bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-13 05:48:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

added capability of running scraper without archiving media, and implemented prototype Telethon scraper for Telegram

Browse Source
This commit is contained in:
Tristan Lee
2022-03-09 12:12:01 -06:00
parent 506fb54a53
commit 739e1d8484
25 changed files with 340 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
test.py
View File

@@ -9,6 +9,7 @@ from cisticola.scraper import (
OdyseeScraper,
RumbleScraper,
TelegramSnscrapeScraper,
TelegramTelethonScraper,
TwitterScraper)
test_channels = [
@@ -117,11 +118,12 @@ scrapers = [
OdyseeScraper(),
RumbleScraper(),
TelegramSnscrapeScraper(),
TwitterScraper()]
TwitterScraper()
TelegramTelethonScraper()]
controller.register_scrapers(scrapers)
engine = create_engine('sqlite:///test3.db')
controller.connect_to_db(engine)
controller.scrape_channels(test_channels)
controller.scrape_channels(test_channels, media = True)