mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-07-03 23:28:36 +03:00
Compare commits
2 Commits
dev
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b721c47de | ||
|
|
afbe4fac50 |
12
.github/workflows/docker-publish.yaml
vendored
12
.github/workflows/docker-publish.yaml
vendored
@@ -22,30 +22,30 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Check out the repo
|
- name: Check out the repo
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@v4
|
||||||
# https://github.com/docker/setup-buildx-action
|
# https://github.com/docker/setup-buildx-action
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
id: buildx
|
id: buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Log in to Docker Hub
|
- name: Log in to Docker Hub
|
||||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
|
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|
||||||
- name: Extract metadata (tags, labels) for Docker
|
- name: Extract metadata (tags, labels) for Docker
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051
|
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9
|
||||||
with:
|
with:
|
||||||
images: bellingcat/auto-archiver
|
images: bellingcat/auto-archiver
|
||||||
|
|
||||||
- name: Build and push Docker image
|
- name: Build and push Docker image
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@v7
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
|
|||||||
2
.github/workflows/python-publish.yaml
vendored
2
.github/workflows/python-publish.yaml
vendored
@@ -22,7 +22,7 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout Repository
|
- name: Checkout Repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v6
|
||||||
|
|||||||
2
.github/workflows/ruff.yaml
vendored
2
.github/workflows/ruff.yaml
vendored
@@ -20,7 +20,7 @@ jobs:
|
|||||||
build:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
- name: Install Python
|
- name: Install Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
|
|||||||
4
.github/workflows/tests-core.yaml
vendored
4
.github/workflows/tests-core.yaml
vendored
@@ -26,7 +26,7 @@ jobs:
|
|||||||
working-directory: ./
|
working-directory: ./
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Install ffmpeg
|
- name: Install ffmpeg
|
||||||
run: sudo apt-get update && sudo apt-get install -y ffmpeg
|
run: sudo apt-get update && sudo apt-get install -y ffmpeg
|
||||||
@@ -40,7 +40,7 @@ jobs:
|
|||||||
run: pipx install poetry
|
run: pipx install poetry
|
||||||
|
|
||||||
- name: Cache Poetry and pip artifacts
|
- name: Cache Poetry and pip artifacts
|
||||||
uses: actions/cache@v5
|
uses: actions/cache@v6
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cache/pypoetry
|
~/.cache/pypoetry
|
||||||
|
|||||||
4
.github/workflows/tests-download.yaml
vendored
4
.github/workflows/tests-download.yaml
vendored
@@ -20,7 +20,7 @@ jobs:
|
|||||||
working-directory: ./
|
working-directory: ./
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Install ffmpeg
|
- name: Install ffmpeg
|
||||||
run: sudo apt-get update && sudo apt-get install -y ffmpeg
|
run: sudo apt-get update && sudo apt-get install -y ffmpeg
|
||||||
@@ -34,7 +34,7 @@ jobs:
|
|||||||
run: pipx install poetry
|
run: pipx install poetry
|
||||||
|
|
||||||
- name: Cache Poetry and pip artifacts
|
- name: Cache Poetry and pip artifacts
|
||||||
uses: actions/cache@v5
|
uses: actions/cache@v6
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cache/pypoetry
|
~/.cache/pypoetry
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM webrecorder/browsertrix-crawler:1.13.2 AS base
|
FROM webrecorder/browsertrix-crawler:1.12.4 AS base
|
||||||
|
|
||||||
ENV RUNNING_IN_DOCKER=1 \
|
ENV RUNNING_IN_DOCKER=1 \
|
||||||
LANG=C.UTF-8 \
|
LANG=C.UTF-8 \
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ generic_extractor:
|
|||||||
|
|
||||||
**PyPi/ Local**:
|
**PyPi/ Local**:
|
||||||
|
|
||||||
When using the Auto Archiver PyPI package, or running locally, you will need additional system requirements to run the token generation script, namely either Docker, or Node.js and npm (>=9).
|
When using the Auto Archiver PyPI package, or running locally, you will need additional system requirements to run the token generation script, namely either Docker, or Node.js and Yarn.
|
||||||
|
|
||||||
See the [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#a-http-server-option) documentation for more details.
|
See the [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#a-http-server-option) documentation for more details.
|
||||||
|
|
||||||
|
|||||||
1072
poetry.lock
generated
1072
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "auto-archiver"
|
name = "auto-archiver"
|
||||||
version = "1.2.8"
|
version = "1.2.7"
|
||||||
description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
|
description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
|
||||||
|
|
||||||
requires-python = ">=3.10,<3.13"
|
requires-python = ">=3.10,<3.13"
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ class GenericExtractor(Extractor):
|
|||||||
def setup_token_generation_script(self) -> None:
|
def setup_token_generation_script(self) -> None:
|
||||||
"""This function sets up the Proof of Origin Token generation script method for
|
"""This function sets up the Proof of Origin Token generation script method for
|
||||||
bgutil-ytdlp-pot-provider if enabled or in Docker."""
|
bgutil-ytdlp-pot-provider if enabled or in Docker."""
|
||||||
missing_tools = [tool for tool in ("node", "npm", "npx") if shutil.which(tool) is None]
|
missing_tools = [tool for tool in ("node", "yarn", "npx") if shutil.which(tool) is None]
|
||||||
if missing_tools:
|
if missing_tools:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Cannot set up PO Token script; missing required tools: {', '.join(missing_tools)}. "
|
f"Cannot set up PO Token script; missing required tools: {', '.join(missing_tools)}. "
|
||||||
@@ -151,7 +151,7 @@ class GenericExtractor(Extractor):
|
|||||||
shutil.move(os.path.join(extracted_root, "server"), server_dir)
|
shutil.move(os.path.join(extracted_root, "server"), server_dir)
|
||||||
shutil.rmtree(extracted_root)
|
shutil.rmtree(extracted_root)
|
||||||
logger.info("Installing dependencies and transpiling PoT Generator script...")
|
logger.info("Installing dependencies and transpiling PoT Generator script...")
|
||||||
subprocess.run(["npm", "ci"], cwd=server_dir, check=True)
|
subprocess.run(["yarn", "install", "--frozen-lockfile"], cwd=server_dir, check=True)
|
||||||
subprocess.run(["npx", "tsc"], cwd=server_dir, check=True)
|
subprocess.run(["npx", "tsc"], cwd=server_dir, check=True)
|
||||||
|
|
||||||
with open(version_file, "w") as vf:
|
with open(version_file, "w") as vf:
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ class DeletionIndicators:
|
|||||||
# YouTube deletion indicators
|
# YouTube deletion indicators
|
||||||
YOUTUBE = [
|
YOUTUBE = [
|
||||||
"This video isn't available anymore",
|
"This video isn't available anymore",
|
||||||
|
"Video unavailable",
|
||||||
"This video has been removed",
|
"This video has been removed",
|
||||||
"This video is no longer available",
|
"This video is no longer available",
|
||||||
"This video is private",
|
"This video is private",
|
||||||
|
|||||||
Reference in New Issue
Block a user