bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-13 05:48:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add method for archiving media, reoranize scraper base classes

Browse Source
This commit is contained in:
Logan Williams
2022-02-24 16:36:55 +01:00
parent e09e0f5202
commit 6092e4caa5
8 changed files with 58 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cisticola/__init__.py
View File

@@ -1,6 +1,6 @@
from typing import List
import cisticola.scraper
import cisticola.base
import cisticola.scraper.base
from sqlalchemy.orm import sessionmaker
from loguru import logger
@@ -14,7 +14,7 @@ class ScraperController:
self.session = None
self.mapper_registry = None
def register_scraper(self, scraper: cisticola.scraper.Scraper):
def register_scraper(self, scraper: cisticola.scraper.base.Scraper):
self.scrapers.append(scraper)
def scrape_channels(self, channels: List[cisticola.base.Channel]):
@@ -30,7 +30,7 @@ class ScraperController:
# get most recent post
session = self.session()
rows = session.query(cisticola.base.ScraperResult).order_by(
cisticola.base.ScraperResult.date_archived).limit(1).all()
cisticola.base.ScraperResult.date.desc()).limit(1).all()
if len(rows) == 1:
since = rows[0]