bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-08 03:18:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix logging logic in scraper controller

Browse Source
This commit is contained in:
Logan Williams
2022-02-24 18:49:06 +01:00
parent e64d845002
commit d163e6b3d9
2 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cisticola/__init__.py
View File

@@ -46,12 +46,12 @@ class ScraperController:
if not handled:
logger.warning(f"No handler found for Channel {channel}")
else:
session = self.session()
session.bulk_save_objects(posts)
session.commit()
session = self.session()
session.bulk_save_objects(posts)
session.commit()
logger.info(f"Added {len(posts)} entries to database")
logger.info(f"Added {len(posts)} entries to database")
def connect_to_db(self, engine):
# create tables

3
cisticola/scraper/telegram_snscrape.py
View File

@@ -20,6 +20,9 @@ class TelegramSnscrapeScraper(cisticola.scraper.base.Scraper):
g = scr.get_items()
for post in g:
if (len(posts)) >= 10:
break
if since is not None and post.date.replace(tzinfo=timezone.utc) <= since.date.replace(tzinfo=timezone.utc):
break