bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-12 21:38:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove 200 post limit; add log rotation

Browse Source
This commit is contained in:
Logan Williams
2022-04-03 16:32:00 +00:00
parent 4c580519dd
commit fccbad7a93
4 changed files with 145 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cisticola/scraper/base.py
View File

@@ -403,9 +403,6 @@ class ScraperController:
session.commit()
added += 1
if added >= 200:
break
session.commit()
logger.info(
f"{scraper} found {added} new posts from {channel}")
@@ -507,4 +504,4 @@ class ScraperController:
self.connect_to_db(self.engine)
class ChannelDoesNotExistError(Exception):
"""The specified channel does not exist or has been deleted."""
"""The specified channel does not exist or has been deleted."""