bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-11 04:48:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix issues with new datetime baed 'media_archived' column

Browse Source
This commit is contained in:
Logan Williams
2022-04-02 18:45:08 +00:00
parent 63633617d2
commit 01bbabe0cb
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cisticola/scraper/base.py
View File

@@ -3,6 +3,7 @@ import os
from io import BytesIO
from urllib.parse import urlparse
import tempfile
from datetime import datetime, timezone
import boto3
from loguru import logger
@@ -417,7 +418,7 @@ class ScraperController:
session = self.session()
posts = session.query(ScraperResult).where(ScraperResult.media_archived == False).order_by(func.random()).all()
posts = session.query(ScraperResult).where(ScraperResult.media_archived == None).order_by(func.random()).all()
logger.info(f"Found {len(posts)} posts without media. Archiving now")
@@ -431,7 +432,7 @@ class ScraperController:
post = scraper.archive_files(post)
if post:
session.query(ScraperResult).where(ScraperResult.id == post.id).update({'archived_urls': post.archived_urls, 'media_archived': True})
session.query(ScraperResult).where(ScraperResult.id == post.id).update({'archived_urls': post.archived_urls, 'media_archived': datetime.now(timezone.utc)})
session.commit()
break