bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-13 05:48:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

renamed 'archive_media' and 'media' to avoid name collision, changed scope of test fixture controller to 'function' so that db is fresh for each executed test

Browse Source
This commit is contained in:
Tristan Lee
2022-03-09 13:19:35 -06:00
parent 739e1d8484
commit 6cf3b8842d
20 changed files with 130 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cisticola/scraper/base.py
View File

@@ -69,7 +69,7 @@ class Scraper:
return blob, content_type, key
def archive_media(self, blob: bytes, content_type: str, key: str) -> str:
def archive_blob(self, blob: bytes, content_type: str, key: str) -> str:
filename = self.__version__.replace(' ', '_') + '/' + key
@@ -83,7 +83,7 @@ class Scraper:
def can_handle(self, channel: Channel) -> bool:
raise NotImplementedError
def get_posts(self, channel: Channel, since: ScraperResult = None, media: bool = True) -> Generator[ScraperResult, None, None]:
def get_posts(self, channel: Channel, since: ScraperResult = None, archive_media: bool = True) -> Generator[ScraperResult, None, None]:
raise NotImplementedError
@@ -103,7 +103,7 @@ class ScraperController:
self.scrapers.extend(scraper)
@logger.catch
def scrape_channels(self, channels: List[Channel], media: bool = True):
def scrape_channels(self, channels: List[Channel], archive_media: bool = True):
if self.session is None:
logger.error("No DB session")
return
@@ -128,7 +128,7 @@ class ScraperController:
else:
since = None
posts = scraper.get_posts(channel, since=since, media=media)
posts = scraper.get_posts(channel, since=since, archive_media=archive_media)
for post in posts:
session.add(post)