Split up into modules

socialmediascraper/cli.py

@@ -0,0 +1,63 @@
+import argparse
+import logging
+import socialmediascraper.base
+import socialmediascraper.modules
+
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+	parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+	parser.add_argument('-v', '--verbose', '--verbosity', dest = 'verbosity', action = 'count', default = 0, help = 'Increase output verbosity')
+	parser.add_argument('--retry', '--retries', dest = 'retries', type = int, default = 3, metavar = 'N',
+		help = 'When the connection fails or the server returns an unexpected response, retry up to N times with an exponential backoff')
+	parser.add_argument('-n', '--max-results', dest = 'maxResults', type = int, metavar = 'N', help = 'Only return the first N results')
+
+	subparsers = parser.add_subparsers(dest = 'scraper', help = 'The scraper you want to use')
+	for cls in socialmediascraper.base.Scraper.__subclasses__():
+		subparser = subparsers.add_parser(cls.name, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+		cls.setup_parser(subparser)
+		subparser.set_defaults(cls = cls)
+
+	args = parser.parse_args()
+
+	# http://bugs.python.org/issue16308 / https://bugs.python.org/issue26510 (fixed in Python 3.7)
+	if not args.scraper:
+		raise RuntimeError('Error: no scraper specified')
+
+	return args
+
+
+def setup_logging(verbosity):
+	rootLogger = logging.getLogger()
+
+	# Set level
+	if verbosity > 0:
+		level = logging.INFO if verbosity == 1 else logging.DEBUG
+		rootLogger.setLevel(level)
+		for handler in rootLogger.handlers:
+			handler.setLevel(level)
+
+	# Create formatter
+	formatter = logging.Formatter('{asctime}  {levelname}  {name}  {message}', datefmt = '%Y-%m-%d %H:%M:%S', style = '{')
+
+	# Add stream handler
+	handler = logging.StreamHandler()
+	handler.setFormatter(formatter)
+	rootLogger.addHandler(handler)
+
+
+def main():
+	args = parse_args()
+	setup_logging(args.verbosity)
+	scraper = args.cls.from_args(args)
+
+	i = 0
+	for i, item in enumerate(scraper.get_items(), start = 1):
+		print(item)
+		if args.maxResults and i >= args.maxResults:
+			logger.info(f'Exiting after {i} results')
+			break
+	else:
+		logger.info(f'Done, found {i} results')