bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-12 05:18:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

merged scraper into main and fixed minor merge conflict

Browse Source
This commit is contained in:
Tristan Lee
2022-03-15 09:12:12 -05:00
parent db03cbf141 750f0cc887
commit e287fd03d9
13 changed files with 612 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/scraper/instagram.py Normal file
View File

@@ -0,0 +1,16 @@
from cisticola.base import Channel
from cisticola.scraper import InstagramScraper
def test_scrape_instagram_channel_no_media(controller, channel_kwargs):
channels = [Channel(**channel_kwargs['instagram'])]
controller.register_scraper(scraper = InstagramScraper())
controller.scrape_channels(channels = channels, archive_media = False)
def test_scrape_instagram_channel(controller, channel_kwargs):
controller.reset_db()
channels = [Channel(**channel_kwargs['instagram'])]
controller.register_scraper(scraper = InstagramScraper())
controller.scrape_channels(channels = channels, archive_media = True)

16
tests/scraper/vkontakte.py Normal file
View File

@@ -0,0 +1,16 @@
from cisticola.base import Channel
from cisticola.scraper import VkontakteScraper
def test_scrape_vkontakte_channel_no_media(controller, channel_kwargs):
channels = [Channel(**channel_kwargs['vkontakte'])]
controller.register_scraper(scraper = VkontakteScraper())
controller.scrape_channels(channels = channels, archive_media = False)
def test_scrape_vkontakte_channel(controller, channel_kwargs):
controller.reset_db()
channels = [Channel(**channel_kwargs['vkontakte'])]
controller.register_scraper(scraper = VkontakteScraper())
controller.scrape_channels(channels = channels, archive_media = True)

16
tests/scraper/youtube.py Normal file
View File

@@ -0,0 +1,16 @@
from cisticola.base import Channel
from cisticola.scraper import YoutubeScraper
def test_scrape_youtube_channel_no_media(controller, channel_kwargs):
channels = [Channel(**channel_kwargs['youtube'])]
controller.register_scraper(scraper = YoutubeScraper())
controller.scrape_channels(channels = channels, archive_media = False)
def test_scrape_youtube_channel(controller, channel_kwargs):
controller.reset_db()
channels = [Channel(**channel_kwargs['youtube'])]
controller.register_scraper(scraper = YoutubeScraper())
controller.scrape_channels(channels = channels, archive_media = True)