diff --git a/cisticola/__init__.py b/cisticola/__init__.py index ee24a4e..e87aa19 100644 --- a/cisticola/__init__.py +++ b/cisticola/__init__.py @@ -46,12 +46,12 @@ class ScraperController: if not handled: logger.warning(f"No handler found for Channel {channel}") + else: + session = self.session() + session.bulk_save_objects(posts) + session.commit() - session = self.session() - session.bulk_save_objects(posts) - session.commit() - - logger.info(f"Added {len(posts)} entries to database") + logger.info(f"Added {len(posts)} entries to database") def connect_to_db(self, engine): # create tables diff --git a/cisticola/scraper/telegram_snscrape.py b/cisticola/scraper/telegram_snscrape.py index 6ea7fd7..8bd8f15 100644 --- a/cisticola/scraper/telegram_snscrape.py +++ b/cisticola/scraper/telegram_snscrape.py @@ -20,6 +20,9 @@ class TelegramSnscrapeScraper(cisticola.scraper.base.Scraper): g = scr.get_items() for post in g: + if (len(posts)) >= 10: + break + if since is not None and post.date.replace(tzinfo=timezone.utc) <= since.date.replace(tzinfo=timezone.utc): break