bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-12 21:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add temp options for testing

Browse Source
This commit is contained in:
erinhmclark
2025-03-04 14:03:39 +00:00
parent 77b517cfc1
commit 7e4b44883b
3 changed files with 31 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/auto_archiver/modules/generic_extractor/generic_extractor.py
View File

@@ -285,6 +285,8 @@ class GenericExtractor(Extractor):
"playlistend": self.max_downloads,
# TODO
# "verbose": True,
# 'debug_extractor': True,
# 'cachedir': False,
# "print_traffic": True,
}
@@ -307,22 +309,6 @@ class GenericExtractor(Extractor):
ydl_options['cookiefile'] = auth['cookies_file']
# Applying user-defined extractor_args
if self.extractor_args:
logger.info(f"Applying user-defined extractor_args")
ydl_options.setdefault('extractor_args', {})
for key, args in self.extractor_args.items():
logger.debug(f"Setting extractor_args: {key}")
if isinstance(args, dict):
# Site specific arguments (e.g., youtube: somekey=value)
ydl_options['extractor_args'].setdefault(key, {}).update(args)
else:
# General extractor_args (e.g., somekey=value)
ydl_options['extractor_args'][key] = args
# Applying user-defined extractor_args
if self.extractor_args:
logger.info(f"Applying user-defined extractor_args")
@@ -341,27 +327,7 @@ class GenericExtractor(Extractor):
ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
for info_extractor in self.suitable_extractors(url):
try:
result = self.download_for_extractor(info_extractor, url, ydl)
if result:
return result
except yt_dlp.utils.ExtractorError as e:
# TODO Does this catch empty/ incomplete failures?
if self.extractor_args:
logger.warning(
f"Extraction with custom extractor_args failed for {url}. Retrying without extractor_args...")
# Remove extractor_args and try without
del ydl_options['extractor_args']
ydl = yt_dlp.YoutubeDL(ydl_options)
try:
result = self.download_for_extractor(info_extractor, url, ydl)
if result:
return result
except Exception as retry_error:
logger.error(f"Extraction failed for {url} after retrying: {retry_error}")
return False
else:
logger.error(f"Extraction failed for {url}: {e}")
return False
result = self.download_for_extractor(info_extractor, url, ydl)
if result:
return result
return False