mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-11 20:58:29 +03:00
Separate setup() and module_setup().
This commit is contained in:
@@ -19,7 +19,7 @@ from auto_archiver.core import Storage
|
||||
|
||||
class GDriveStorage(Storage):
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
self.scopes = ['https://www.googleapis.com/auth/drive']
|
||||
# Initialize Google Drive service
|
||||
self._setup_google_drive_service()
|
||||
|
||||
@@ -21,7 +21,7 @@ from . import GWorksheet
|
||||
|
||||
class GsheetsFeeder(Feeder):
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
self.gsheets_client = gspread.service_account(filename=self.service_account)
|
||||
# TODO mv to validators
|
||||
assert self.sheet or self.sheet_id, (
|
||||
|
||||
@@ -17,7 +17,7 @@ class HtmlFormatter(Formatter):
|
||||
environment: Environment = None
|
||||
template: any = None
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
"""Sets up the Jinja2 environment and loads the template."""
|
||||
template_dir = os.path.join(pathlib.Path(__file__).parent.resolve(), "templates/")
|
||||
self.environment = Environment(loader=FileSystemLoader(template_dir), autoescape=True)
|
||||
|
||||
@@ -32,7 +32,7 @@ class InstagramAPIExtractor(Extractor):
|
||||
r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com)\/(stories(?:\/highlights)?|p|reel)?\/?([^\/\?]*)\/?(\d+)?"
|
||||
)
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
if self.api_endpoint[-1] == "/":
|
||||
self.api_endpoint = self.api_endpoint[:-1]
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ class InstagramExtractor(Extractor):
|
||||
profile_pattern = re.compile(r"{valid_url}(\w+)".format(valid_url=valid_url))
|
||||
# TODO: links to stories
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
|
||||
self.insta = instaloader.Instaloader(
|
||||
download_geotags=True, download_comments=True, compress_json=False, dirname_pattern=self.download_folder, filename_pattern="{date_utc}_UTC_{target}__{typename}"
|
||||
|
||||
@@ -27,7 +27,7 @@ class InstagramTbotExtractor(Extractor):
|
||||
https://t.me/instagram_load_bot
|
||||
"""
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
"""
|
||||
1. makes a copy of session_file that is removed in cleanup
|
||||
2. checks if the session file is valid
|
||||
|
||||
@@ -13,7 +13,7 @@ NO_DUPLICATES_FOLDER = "no-dups/"
|
||||
|
||||
class S3Storage(Storage):
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
self.s3 = boto3.client(
|
||||
's3',
|
||||
region_name=self.region,
|
||||
|
||||
@@ -18,7 +18,7 @@ class TelethonExtractor(Extractor):
|
||||
invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)")
|
||||
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
|
||||
"""
|
||||
1. makes a copy of session_file that is removed in cleanup
|
||||
|
||||
@@ -15,7 +15,7 @@ class TwitterApiExtractor(Extractor):
|
||||
|
||||
valid_url: re.Pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
self.api_index = 0
|
||||
self.apis = []
|
||||
if len(self.bearer_tokens):
|
||||
|
||||
@@ -12,7 +12,7 @@ class VkExtractor(Extractor):
|
||||
Currently only works for /wall posts
|
||||
"""
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
self.vks = VkScraper(self.username, self.password, session_file=self.session_file)
|
||||
|
||||
def download(self, item: Metadata) -> Metadata:
|
||||
|
||||
@@ -18,7 +18,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
|
||||
When used as an archiver it will extract the media from the .WACZ archive so it can be enriched.
|
||||
"""
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
|
||||
self.use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER')
|
||||
self.docker_in_docker = os.environ.get('WACZ_ENABLE_DOCKER') and os.environ.get('RUNNING_IN_DOCKER')
|
||||
|
||||
@@ -13,7 +13,7 @@ class WhisperEnricher(Enricher):
|
||||
Only works if an S3 compatible storage is used
|
||||
"""
|
||||
|
||||
def module_setup(self) -> None:
|
||||
def setup(self) -> None:
|
||||
self.stores = self.config['steps']['storages']
|
||||
self.s3 = get_module("s3_storage", self.config)
|
||||
if not "s3_storage" in self.stores:
|
||||
|
||||
Reference in New Issue
Block a user