bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-12 21:38:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

added youtube scraper, moved from official youtube-dl repo to using yt-dlp because download speed for youtube videos is much better

Browse Source
This commit is contained in:
Tristan Lee
2022-03-11 17:19:52 -06:00
parent 821c39004b
commit 965bf1e2dc
8 changed files with 121 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/scraper/youtube.py Normal file
View File

@@ -0,0 +1,16 @@
from cisticola.base import Channel
from cisticola.scraper import YoutubeScraper
def test_scrape_youtube_channel_no_media(controller, channel_kwargs):
channels = [Channel(**channel_kwargs['youtube'])]
controller.register_scraper(scraper = YoutubeScraper())
controller.scrape_channels(channels = channels, archive_media = False)
def test_scrape_youtube_channel(controller, channel_kwargs):
controller.reset_db()
channels = [Channel(**channel_kwargs['youtube'])]
controller.register_scraper(scraper = YoutubeScraper())
controller.scrape_channels(channels = channels, archive_media = True)