Remove MAX_POSTS, auto detect MIME type

Co-authored-by: Tristan Lee <tristan@bellingcat.com>
This commit is contained in:
Logan Williams
2022-02-25 08:52:42 +01:00
committed by GitHub
parent e6085689b5
commit 8ab56ff5ba
2 changed files with 3 additions and 5 deletions

View File

@@ -4,8 +4,6 @@ import cisticola.scraper.base
from sqlalchemy.orm import sessionmaker
from loguru import logger
MAX_POSTS = 10
class ScraperController:
"""Registers scrapers, uses them to generate ScraperResults. Synchronizes
@@ -49,8 +47,6 @@ class ScraperController:
for post in posts:
session.add(post)
added += 1
if added >= MAX_POSTS:
break
session.commit()
logger.info(