bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-13 04:48:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix behaviour on SIGPIPE/BrokenPipeError

Browse Source
This commit is contained in:
JustAnotherArchivist
2023-02-28 20:20:28 +00:00
parent 42cb6d8170
commit 03ef3debaf
1 changed files with 32 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
snscrape/_cli.py
View File

@@ -6,6 +6,7 @@ import datetime
import importlib.metadata import importlib.metadata
import inspect import inspect
import logging import logging
import os
import requests import requests
# Imported in parse_args() after setting up the logger: # Imported in parse_args() after setting up the logger:
#import snscrape.base #import snscrape.base
@@ -307,32 +308,36 @@ def main():
i = 0 i = 0
with _dump_locals_on_exception(): with _dump_locals_on_exception():
if args.withEntity and (entity := scraper.entity): try:
if args.jsonl: if args.withEntity and (entity := scraper.entity):
print(entity.json()) if args.jsonl:
print(entity.json())
else:
print(entity)
if args.maxResults == 0:
logger.info('Exiting after 0 results')
return
for i, item in enumerate(scraper.get_items(), start = 1):
if args.since is not None and item.date < args.since:
logger.info(f'Exiting due to reaching older results than {args.since}')
break
if args.jsonl:
print(item.json())
elif args.format is not None:
print(args.format.format(item))
else:
print(item)
if args.progress and i % 100 == 0:
print(f'Scraping, {i} results so far', file = sys.stderr)
if args.maxResults and i >= args.maxResults:
logger.info(f'Exiting after {i} results')
if args.progress:
print(f'Stopped scraping after {i} results due to --max-results', file = sys.stderr)
break
else: else:
print(entity) logger.info(f'Done, found {i} results')
if args.maxResults == 0:
logger.info('Exiting after 0 results')
return
for i, item in enumerate(scraper.get_items(), start = 1):
if args.since is not None and item.date < args.since:
logger.info(f'Exiting due to reaching older results than {args.since}')
break
if args.jsonl:
print(item.json())
elif args.format is not None:
print(args.format.format(item))
else:
print(item)
if args.progress and i % 100 == 0:
print(f'Scraping, {i} results so far', file = sys.stderr)
if args.maxResults and i >= args.maxResults:
logger.info(f'Exiting after {i} results')
if args.progress: if args.progress:
print(f'Stopped scraping after {i} results due to --max-results', file = sys.stderr) print(f'Finished, {i} results', file = sys.stderr)
break except BrokenPipeError:
else: os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
logger.info(f'Done, found {i} results') sys.exit(1)
if args.progress:
print(f'Finished, {i} results', file = sys.stderr)