From 86656f8ba3c12fd8f92abaeb2ee374b659991951 Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Fri, 26 Aug 2022 15:56:46 +0200 Subject: [PATCH] Scrape snowball_it channels too --- cisticola/scraper/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cisticola/scraper/base.py b/cisticola/scraper/base.py index e8d2d7d..96090e0 100644 --- a/cisticola/scraper/base.py +++ b/cisticola/scraper/base.py @@ -342,7 +342,7 @@ class ScraperController: session = self.session() - channels = session.query(Channel).where(Channel.source=='researcher').all() + channels = session.query(Channel).filter((Channel.source=='researcher')|(Channel.source=='snowball_it')).all() session.close() @@ -359,7 +359,7 @@ class ScraperController: # This will sort the channels by the least recently scraped. most_recently_archived = session.query(func.max(RawChannelInfo.date_archived).label("date"), RawChannelInfo.channel.label("channel")).group_by(RawChannelInfo.channel).subquery() channels = session.query(Channel).\ - where(Channel.source=='researcher').\ + filter((Channel.source=='researcher')|(Channel.source=='snowball_it')).\ outerjoin(most_recently_archived, Channel.id == most_recently_archived.c.channel).\ order_by(nullsfirst(most_recently_archived.c.date.asc())).all()