bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-12 13:28:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add an option to scape posts older than the database record as well as newer (Telegram only)

Browse Source
This commit is contained in:
Logan Williams
2022-09-05 13:48:01 +00:00
parent 86656f8ba3
commit c15022402d
5 changed files with 408 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app.py
View File

@@ -74,6 +74,11 @@ def scrape_channels(args):
controller = get_scraper_controller()
controller.scrape_all_channels(archive_media=args.media)
def scrape_channels_old(args):
logger.info(f"Scraping old posts from channels, media: {args.media}")
controller = get_scraper_controller()
controller.scrape_all_channels(archive_media=args.media, fetch_old=True)
def scrape_channel_info(args):
logger.info(f"Scraping channel info")
@@ -147,6 +152,9 @@ if __name__ == "__main__":
elif args.command == "scrape-channels":
logger.add("logs/scrape-channels.log", level="TRACE", rotation="100 MB")
scrape_channels(args)
elif args.command == "scrape-channels-old":
logger.add("logs/scrape-channels-old.log", level="TRACE", rotation="100 MB")
scrape_channels_old(args)
elif args.command == "archive-media":
logger.add("logs/archive-media.log", level="TRACE", rotation="100 MB")
archive_media(args)