mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-11 12:48:28 +03:00
d02e7e0f02
Implements issue #335: improve detection of deleted/missing posts ## Changes ### New Deletion Detection System - Created `deletion_detection.py` utility module with platform-specific indicators for Twitter, Facebook, Instagram, TikTok, YouTube, Reddit, VK, and Telegram - Detects deletion via HTML content, page titles, error messages, and video metadata - Stores detailed deletion context (indicator, source, platform) in metadata for investigators ### Integration Points - **Antibot Extractor**: Checks HTML and page titles after page load; resolves TODO about detecting deleted videos - **Generic Extractor**: Checks yt-dlp video data and error messages for deletion indicators - **Twitter Dropin**: Enhanced detection when user/created_at fields are missing ### Test Coverage - Comprehensive test suite covering all platforms - Tests for HTML, title, error message, and metadata detection - Validates that normal content is not falsely flagged ## Impact for Conflict Documentation This fix is critical for evidence preservation in war-torn regions: - Investigators can now document that evidence existed but was deleted - Prevents wasted archival attempts on deleted content - Tracks patterns of content removal - Preserves metadata about what was deleted and when Twitter example: Detects "Hmm...this page doesn't exist. Try searching for something else" and flags content as deleted_or_unavailable.
84 lines
3.7 KiB
Python
84 lines
3.7 KiB
Python
import re
|
|
import mimetypes
|
|
|
|
from auto_archiver.utils.custom_logger import logger
|
|
from slugify import slugify
|
|
|
|
from auto_archiver.core.metadata import Metadata, Media
|
|
from auto_archiver.utils import url as UrlUtil, get_datetime_from_str
|
|
from auto_archiver.core.extractor import Extractor
|
|
from auto_archiver.utils.deletion_detection import detect_deletion, flag_as_deleted
|
|
from auto_archiver.modules.generic_extractor.dropin import GenericDropin, InfoExtractor
|
|
|
|
|
|
class Twitter(GenericDropin):
|
|
def choose_variant(self, variants):
|
|
# choosing the highest quality possible
|
|
variant, width, height = None, 0, 0
|
|
for var in variants:
|
|
if var.get("content_type", "") == "video/mp4":
|
|
width_height = re.search(r"\/(\d+)x(\d+)\/", var["url"])
|
|
if width_height:
|
|
w, h = int(width_height[1]), int(width_height[2])
|
|
if w > width or h > height:
|
|
width, height = w, h
|
|
variant = var
|
|
else:
|
|
variant = var if not variant else variant
|
|
return variant
|
|
|
|
def extract_post(self, url: str, ie_instance: InfoExtractor):
|
|
twid = ie_instance._match_valid_url(url).group("id")
|
|
return ie_instance._extract_status(twid=twid)
|
|
|
|
def keys_to_clean(self, video_data, info_extractor):
|
|
return ["user", "created_at", "entities", "favorited", "translator_type"]
|
|
|
|
def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
|
|
result = Metadata()
|
|
try:
|
|
if not tweet.get("user") or not tweet.get("created_at"):
|
|
# Check for deletion indicators
|
|
deletion_info = detect_deletion(
|
|
video_data=tweet,
|
|
url=url,
|
|
error_message="Missing user or created_at fields"
|
|
)
|
|
if deletion_info:
|
|
flag_as_deleted(result, deletion_info)
|
|
return result
|
|
|
|
raise ValueError("Error retrieving post. Are you sure it exists?")
|
|
timestamp = get_datetime_from_str(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
|
|
except (ValueError, KeyError) as ex:
|
|
logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")
|
|
return False
|
|
|
|
full_text = tweet.pop("full_text", "")
|
|
author = tweet["user"].get("name", "")
|
|
result.set("author", author).set_url(url)
|
|
|
|
result.set_title(f"{author} - {full_text}").set_content(full_text).set_timestamp(timestamp)
|
|
if not tweet.get("entities", {}).get("media"):
|
|
logger.debug("No media found, archiving tweet text only")
|
|
result.status = "twitter-ytdl"
|
|
return result
|
|
for i, tw_media in enumerate(tweet["entities"]["media"]):
|
|
media = Media(filename="")
|
|
mimetype = ""
|
|
if tw_media["type"] == "photo":
|
|
media.set("src", UrlUtil.twitter_best_quality_url(tw_media["media_url_https"]))
|
|
mimetype = "image/jpeg"
|
|
elif tw_media["type"] == "video":
|
|
variant = self.choose_variant(tw_media["video_info"]["variants"])
|
|
media.set("src", variant["url"])
|
|
mimetype = variant["content_type"]
|
|
elif tw_media["type"] == "animated_gif":
|
|
variant = tw_media["video_info"]["variants"][0]
|
|
media.set("src", variant["url"])
|
|
mimetype = variant["content_type"]
|
|
ext = mimetypes.guess_extension(mimetype)
|
|
media.filename = archiver.download_from_url(media.get("src"), f"{slugify(url)}_{i}{ext}")
|
|
result.add_media(media)
|
|
return result
|