bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-11 21:08:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add method for archiving media, reoranize scraper base classes

Browse Source
This commit is contained in:
Logan Williams
2022-02-24 16:36:55 +01:00
parent e09e0f5202
commit 6092e4caa5
8 changed files with 58 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
test.py
View File

@@ -3,36 +3,39 @@
# still need to do some planning for handling media
import cisticola
import cisticola.scraper.twitter
import cisticola.scraper.telegram_snscrape
from sqlalchemy import create_engine
test_channels = [cisticola.base.Channel(id=0, name="Logan Williams (test)", platform_id=891729132,
category="test", followers=None, platform="Twitter",
url="https://twitter.com/obtusatum", country="US",
url="https://twitter.com/obtusatum", screenname="obtusatum", country="US",
influencer=None, public=True, chat=False,
notes=""),
cisticola.base.Channel(id=1, name="JQHN SPARTAN", platform_id=-1001181961026,
category="qanon", followers=None, platform="Telegram",
url="https://t.me/jqhnspartan", country="FR",
url="https://t.me/jqhnspartan", screenname="jqhnspartan", country="FR",
influencer="JQNH SPARTAN", public=True, chat=False, notes=""),
cisticola.base.Channel(id=2, name="LizardRepublic", platform_id='lizardrepublic',
category="qanon", followers=None, platform="Gettr",
url="https://www.gettr.com/user/lizardrepublic", country="US",
url="https://www.gettr.com/user/lizardrepublic", screenname="lizardrepublic", country="US",
influencer=None, public=True, chat=False, notes=""),
cisticola.base.Channel(id=3, name="Patriot Front", platform_id='OVv9QZL4sEsC',
category="nazi", followers=None, platform="Bitchute",
url="https://www.bitchute.com/channel/OVv9QZL4sEsC/", country="US",
url="https://www.bitchute.com/channel/OVv9QZL4sEsC/", screenname=None, country="US",
influencer=None, public=True, chat=False, notes=""),]
controller = cisticola.ScraperController()
scraper = cisticola.scraper.twitter.TwitterScraper()
# scraper = cisticola.scraper.twitter.TwitterScraper()
# controller.register_scraper(scraper)
scraper = cisticola.scraper.telegram_snscrape.TelegramSnscrapeScraper()
controller.register_scraper(scraper)
engine = create_engine('sqlite:///test.db')
engine = create_engine('sqlite:///test3.db')
controller.connect_to_db(engine)
controller.scrape_channels(test_channels)