mirror of
https://github.com/bellingcat/auto-archiver-api.git
synced 2026-06-10 20:48:34 +03:00
7e48f706df
* Update pyproject.toml * add pre-commit * Create .pre-commit-config.yaml * Comment out ruff * Update .pre-commit-config.yaml * General formatting * Create format-and-fail.yml * Update ci.yml * Add pre-commit to dev dependencies * Update pyproject.toml
34 lines
1.3 KiB
Python
34 lines
1.3 KiB
Python
# TODO: code in this file should eventually be moved to the auto-archiver code base
|
|
|
|
from typing import List
|
|
|
|
from auto_archiver.core import Media, Metadata
|
|
from loguru import logger
|
|
|
|
from app.shared.db import models
|
|
|
|
|
|
def get_all_urls(result: Metadata) -> List[models.ArchiveUrl]:
|
|
db_urls = []
|
|
for m in result.media:
|
|
for i, url in enumerate(m.urls): db_urls.append(models.ArchiveUrl(url=url, key=m.get("id", f"media_{i}")))
|
|
for k, prop in m.properties.items():
|
|
if prop_converted := convert_if_media(prop):
|
|
for i, url in enumerate(prop_converted.urls): db_urls.append(models.ArchiveUrl(url=url, key=prop_converted.get("id", f"{k}_{i}")))
|
|
if isinstance(prop, list):
|
|
for i, prop_media in enumerate(prop):
|
|
if prop_media := convert_if_media(prop_media):
|
|
for j, url in enumerate(prop_media.urls):
|
|
db_urls.append(models.ArchiveUrl(url=url, key=prop_media.get("id", f"{k}{prop_media.key}_{i}.{j}")))
|
|
return db_urls
|
|
|
|
|
|
|
|
def convert_if_media(media):
|
|
if isinstance(media, Media): return media
|
|
elif isinstance(media, dict):
|
|
try: return Media.from_dict(media)
|
|
except Exception as e:
|
|
logger.debug(f"error parsing {media} : {e}")
|
|
return False
|