bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-12 21:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactoring filenumber into subfolder

Browse Source
This commit is contained in:
msramalho
2022-05-26 19:18:29 +02:00
parent 03aa02e88b
commit 159adf9afe
13 changed files with 144 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
archivers/base_archiver.py
View File

@@ -18,6 +18,7 @@ from selenium.webdriver.common.by import By
from loguru import logger
from selenium.common.exceptions import TimeoutException
@dataclass
class ArchiveResult:
status: str
@@ -42,7 +43,7 @@ class Archiver(ABC):
return self.__class__.__name__
@abstractmethod
def download(self, url, check_if_exists=False, filenumber=None): pass
def download(self, url, check_if_exists=False): pass
def get_netloc(self, url):
return urlparse(url).netloc
@@ -51,7 +52,7 @@ class Archiver(ABC):
return self.get_key(urlparse(url).path.replace("/", "_") + ".html")
# generate the html page eg SM3013/twitter__minmyatnaing13_status_1499415562937503751.html
def generate_media_page_html(self, url, urls_info: dict, object, thumbnail=None, filenumber=None):
def generate_media_page_html(self, url, urls_info: dict, object, thumbnail=None):
page = f'''<html><head><title>{url}</title><meta charset="UTF-8"></head>
<body>
<h2>Archived media from {self.name}</h2>
@@ -71,10 +72,6 @@ class Archiver(ABC):
page_hash = self.get_hash(page_filename)
if filenumber != None:
logger.trace(f'filenumber for directory is {filenumber}')
page_key = filenumber + "/" + page_key
self.storage.upload(page_filename, page_key, extra_args={
'ACL': 'public-read', 'ContentType': 'text/html'})
@@ -82,7 +79,7 @@ class Archiver(ABC):
return (page_cdn, page_hash, thumbnail)
# eg images in a tweet save to cloud storage
def generate_media_page(self, urls, url, object, filenumber=None):
def generate_media_page(self, urls, url, object):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
}
@@ -102,10 +99,6 @@ class Archiver(ABC):
with open(filename, 'wb') as f:
f.write(d.content)
if filenumber is not None:
logger.debug(f'filenumber for directory is {filenumber}')
key = filenumber + "/" + key
# eg filename: 'tmp/twitter__media_FM7-ggCUYAQHKWW.jpg'
# eg key: 'twitter__media_FM7-ggCUYAQHKWW.jpg'
# or if using filename key: 'SM3013/twitter__media_FM7-ggCUYAQHKWW.jpg'
@@ -120,7 +113,7 @@ class Archiver(ABC):
thumbnail = cdn_url
uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
return self.generate_media_page_html(url, uploaded_media, object, thumbnail=thumbnail, filenumber=filenumber)
return self.generate_media_page_html(url, uploaded_media, object, thumbnail=thumbnail)
def get_key(self, filename):
"""
@@ -140,16 +133,15 @@ class Archiver(ABC):
def get_hash(self, filename):
f = open(filename, "rb")
bytes = f.read() # read entire file as bytes
# TODO: customizable hash
hash = hashlib.sha256(bytes)
# option to use SHA3_512 instead
# hash = hashlib.sha3_512(bytes)
f.close()
return hash.hexdigest()
# eg SA3013/twitter__minmyatnaing13_status_14994155629375037512022-04-27T13:51:43.701962.png
# def get_screenshot(self, url, filenumber, storage="GD"):
def get_screenshot(self, url, filenumber):
def get_screenshot(self, url):
key = self.get_key(urlparse(url).path.replace(
"/", "_") + datetime.datetime.utcnow().isoformat().replace(" ", "_") + ".png")
filename = 'tmp/' + key
@@ -158,8 +150,8 @@ class Archiver(ABC):
if 'facebook.com' in url:
try:
logger.debug(f'Trying fb click accept cookie popup for {url}')
self.driver.get("http://www.facebook.com")
foo = self.driver.find_element(By.XPATH,"//button[@data-cookiebanner='accept_only_essential_button']")
self.driver.get("http://www.facebook.com")
foo = self.driver.find_element(By.XPATH, "//button[@data-cookiebanner='accept_only_essential_button']")
foo.click()
logger.debug(f'fb click worked')
# linux server needs a sleep otherwise facebook cookie wont have worked and we'll get a popup on next page
@@ -174,11 +166,6 @@ class Archiver(ABC):
logger.info("TimeoutException loading page for screenshot")
self.driver.save_screenshot(filename)
if filenumber is not None:
logger.debug(f'filenumber for directory is {filenumber}')
key = filenumber + "/" + key
self.storage.upload(filename, key, extra_args={
'ACL': 'public-read', 'ContentType': 'image/png'})

9
archivers/telegram_archiver.py
View File

@@ -11,7 +11,7 @@ from .base_archiver import Archiver, ArchiveResult
class TelegramArchiver(Archiver):
name = "telegram"
def download(self, url, check_if_exists=False, filenumber=None):
def download(self, url, check_if_exists=False):
# detect URLs that we definitely cannot handle
if 't.me' != self.get_netloc(url):
return False
@@ -27,7 +27,7 @@ class TelegramArchiver(Archiver):
if url[-8:] != "?embed=1":
url += "?embed=1"
screenshot = self.get_screenshot(url, filenumber=filenumber)
screenshot = self.get_screenshot(url)
t = requests.get(url, headers=headers)
s = BeautifulSoup(t.content, 'html.parser')
@@ -42,7 +42,7 @@ class TelegramArchiver(Archiver):
urls = [u.replace("'", "") for u in re.findall(r'url\((.*?)\)', im['style'])]
images += urls
page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content)),filenumber=filenumber)
page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content)))
time_elements = s.find_all('time')
timestamp = time_elements[0].get('datetime') if len(time_elements) else None
@@ -52,9 +52,6 @@ class TelegramArchiver(Archiver):
video_id = video_url.split('/')[-1].split('?')[0]
key = self.get_key(video_id)
if filenumber is not None:
key = filenumber + "/" + key
filename = 'tmp/' + key
cdn_url = self.storage.get_cdn_url(key)

27
archivers/telethon_archiver.py
View File

@@ -7,6 +7,7 @@ from loguru import logger
from storages import Storage
from .base_archiver import Archiver, ArchiveResult
from telethon.sync import TelegramClient
from telethon.errors import ChannelInvalidError
@dataclass
@@ -41,14 +42,14 @@ class TelethonArchiver(Archiver):
media.append(post)
return media
def download(self, url, check_if_exists=False, filenumber=None):
def download(self, url, check_if_exists=False):
# detect URLs that we definitely cannot handle
matches = self.link_pattern.findall(url)
if not len(matches):
return False
status = "success"
screenshot = self.get_screenshot(url, filenumber)
screenshot = self.get_screenshot(url)
# app will ask (stall for user input!) for phone number and auth code if anon.session not found
with self.client.start():
@@ -60,7 +61,11 @@ class TelethonArchiver(Archiver):
try:
post = self.client.get_messages(chat, ids=post_id)
except ValueError as e:
logger.warning(f'Could not fetch telegram {url} possibly it\'s private: {e}')
logger.error(f'Could not fetch telegram {url} possibly it\'s private: {e}')
return False
except ChannelInvalidError as e:
# TODO: check followup here: https://github.com/LonamiWebs/Telethon/issues/3819
logger.error(f'Could not fetch telegram {url} possibly it\'s private or not displayable in : {e}')
return False
media_posts = self._get_media_posts_in_group(chat, post)
@@ -68,11 +73,8 @@ class TelethonArchiver(Archiver):
if len(media_posts) > 1:
key = self.get_html_key(url)
if filenumber is not None:
key = filenumber + "/" + key
if check_if_exists and self.storage.exists(key):
# only s3 storage supports storage.exists as not implemented on gd
# only s3 storage supports storage.exists as not implemented on gd
cdn_url = self.storage.get_cdn_url(key)
status = 'already archived'
return ArchiveResult(status='already archived', cdn_url=cdn_url, title=post.message, timestamp=post.date, screenshot=screenshot)
@@ -84,26 +86,19 @@ class TelethonArchiver(Archiver):
if len(mp.message) > len(message): message = mp.message
filename = self.client.download_media(mp.media, f'tmp/{chat}_{group_id}/{mp.id}')
key = filename.split('tmp/')[1]
if filenumber is not None:
key = filenumber + "/" + key
self.storage.upload(filename, key)
hash = self.get_hash(filename)
cdn_url = self.storage.get_cdn_url(key)
uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
os.remove(filename)
page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)), filenumber=filenumber)
page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)))
return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)
elif len(media_posts) == 1:
key = self.get_key(f'{chat}_{post_id}')
filename = self.client.download_media(post.media, f'tmp/{key}')
key = filename.split('tmp/')[1].replace(" ", "")
if filenumber is not None:
key = filenumber + "/" + key
self.storage.upload(filename, key)
hash = self.get_hash(filename)
cdn_url = self.storage.get_cdn_url(key)
@@ -112,5 +107,5 @@ class TelethonArchiver(Archiver):
return ArchiveResult(status=status, cdn_url=cdn_url, title=post.message, thumbnail=key_thumb, thumbnail_index=thumb_index, timestamp=post.date, hash=hash, screenshot=screenshot)
page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)), filenumber=filenumber)
page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)))
return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)

2
archivers/tiktok_archiver.py
View File

@@ -8,7 +8,7 @@ from .base_archiver import Archiver, ArchiveResult
class TiktokArchiver(Archiver):
name = "tiktok"
def download(self, url, check_if_exists=False, filenumber=None):
def download(self, url, check_if_exists=False):
if 'tiktok.com' not in url:
return False

14
archivers/twitter_archiver.py
View File

@@ -8,15 +8,15 @@ from .base_archiver import Archiver, ArchiveResult
class TwitterArchiver(Archiver):
name = "twitter"
def download(self, url, check_if_exists=False, filenumber=None):
def download(self, url, check_if_exists=False):
if 'twitter.com' != self.get_netloc(url):
return False
tweet_id = urlparse(url).path.split('/')
if 'status' in tweet_id:
i = tweet_id.index('status')
tweet_id = tweet_id[i+1]
tweet_id = tweet_id[i + 1]
else:
return False
@@ -25,9 +25,7 @@ class TwitterArchiver(Archiver):
try:
tweet = next(scr.get_items())
except Exception as ex:
template = "TwitterArchiver cant get tweet and threw, which can happen if a media sensitive tweet. \n type: {0} occurred. \n arguments:{1!r}"
message = template.format(type(ex).__name__, ex.args)
logger.warning(message)
logger.warning(f"can't get tweet: {type(ex).__name__} occurred. args: {ex.args}")
return False
if tweet.media is None:
@@ -48,8 +46,8 @@ class TwitterArchiver(Archiver):
else:
logger.warning(f"Could not get media URL of {media}")
page_cdn, page_hash, thumbnail = self.generate_media_page(urls, url, tweet.json(), filenumber)
page_cdn, page_hash, thumbnail = self.generate_media_page(urls, url, tweet.json())
screenshot = self.get_screenshot(url, filenumber)
screenshot = self.get_screenshot(url)
return ArchiveResult(status="success", cdn_url=page_cdn, screenshot=screenshot, hash=page_hash, thumbnail=thumbnail, timestamp=tweet.date)

4
archivers/wayback_archiver.py
View File

@@ -14,7 +14,7 @@ class WaybackArchiver(Archiver):
super(WaybackArchiver, self).__init__(storage, driver)
self.seen_urls = {}
def download(self, url, check_if_exists=False, filenumber=None):
def download(self, url, check_if_exists=False):
if check_if_exists and url in self.seen_urls:
return self.seen_urls[url]
@@ -75,7 +75,7 @@ class WaybackArchiver(Archiver):
except:
title = "Could not get title"
screenshot = self.get_screenshot(url, filenumber)
screenshot = self.get_screenshot(url)
result = ArchiveResult(status='Internet Archive fallback', cdn_url=archive_url, title=title, screenshot=screenshot)
self.seen_urls[url] = result
return result

17
archivers/youtubedl_archiver.py
View File

@@ -7,6 +7,7 @@ from loguru import logger
from .base_archiver import Archiver, ArchiveResult
from storages import Storage
class YoutubeDLArchiver(Archiver):
name = "youtube_dl"
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
@@ -15,7 +16,7 @@ class YoutubeDLArchiver(Archiver):
super().__init__(storage, driver)
self.fb_cookie = fb_cookie
def download(self, url, check_if_exists=False, filenumber=None):
def download(self, url, check_if_exists=False):
netloc = self.get_netloc(url)
if netloc in ['facebook.com', 'www.facebook.com']:
logger.debug('Using Facebook cookie')
@@ -61,9 +62,6 @@ class YoutubeDLArchiver(Archiver):
key = self.get_key(filename)
if filenumber is not None:
key = filenumber + "/" + key
if self.storage.exists(key):
status = 'already archived'
cdn_url = self.storage.get_cdn_url(key)
@@ -87,10 +85,6 @@ class YoutubeDLArchiver(Archiver):
if status != 'already archived':
key = self.get_key(filename)
if filenumber is not None:
key = filenumber + "/" + key
self.storage.upload(filename, key)
# filename ='tmp/sDE-qZdi8p8.webm'
@@ -98,8 +92,7 @@ class YoutubeDLArchiver(Archiver):
cdn_url = self.storage.get_cdn_url(key)
hash = self.get_hash(filename)
screenshot = self.get_screenshot(url, filenumber)
screenshot = self.get_screenshot(url)
# get duration
duration = info.get('duration')
@@ -115,9 +108,9 @@ class YoutubeDLArchiver(Archiver):
timestamp = datetime.datetime.utcfromtimestamp(info['timestamp']).replace(tzinfo=datetime.timezone.utc).isoformat() \
if 'timestamp' in info else \
datetime.datetime.strptime(info['upload_date'], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) \
datetime.datetime.strptime(info['upload_date'], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) \
if 'upload_date' in info and info['upload_date'] is not None else \
None
None
return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb, thumbnail_index=thumb_index, duration=duration,
title=info['title'] if 'title' in info else None, timestamp=timestamp, hash=hash, screenshot=screenshot)