mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 21:28:29 +03:00
41 lines
2.2 KiB
Python
41 lines
2.2 KiB
Python
m = {
|
|
"name": "S3 Storage",
|
|
"type": ["storage"],
|
|
"requires_setup": True,
|
|
"external_dependencies": {
|
|
"python": ["boto3", "loguru"],
|
|
},
|
|
"configs": {
|
|
# TODO: get base storage configs
|
|
"bucket": {"default": None, "help": "S3 bucket name"},
|
|
"region": {"default": None, "help": "S3 region name"},
|
|
"key": {"default": None, "help": "S3 API key"},
|
|
"secret": {"default": None, "help": "S3 API secret"},
|
|
"random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"},
|
|
"endpoint_url": {
|
|
"default": 'https://{region}.digitaloceanspaces.com',
|
|
"help": "S3 bucket endpoint, {region} are inserted at runtime"
|
|
},
|
|
"cdn_url": {
|
|
"default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}',
|
|
"help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime"
|
|
},
|
|
"private": {"default": False, "help": "if true S3 files will not be readable online"},
|
|
},
|
|
"description": """
|
|
S3Storage: A storage module for saving media files to an S3-compatible object storage.
|
|
|
|
### Features
|
|
- Uploads media files to an S3 bucket with customizable configurations.
|
|
- Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes.
|
|
- Automatically generates unique paths for files when duplicates are found.
|
|
- Configurable endpoint and CDN URL for different S3-compatible providers.
|
|
- Supports both private and public file storage, with public files being readable online.
|
|
|
|
### Notes
|
|
- Requires S3 credentials (API key and secret) and a bucket name to function.
|
|
- The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.
|
|
- Uses `boto3` for interaction with the S3 API.
|
|
"""
|
|
}
|