implements single URL archiving

This commit is contained in:
msramalho
2024-10-30 15:58:57 +00:00
parent bf151ff92e
commit 42fa376f7d
12 changed files with 594 additions and 59 deletions

View File

@@ -1,16 +1,14 @@
<template>
<div class="pane">
<v-app class="bg">
<NavBar />
<v-main>
<router-view />
</v-main>
<v-footer class="legal">
<router-link to="/privacy">Privacy Policy</router-link>
<router-link to="/tos">Terms of Service</router-link>
</v-footer>
</v-app>
</div>
<v-app class="bg">
<NavBar />
<v-main>
<router-view />
</v-main>
<v-footer class="legal">
<router-link to="/privacy">Privacy Policy</router-link>
<router-link to="/tos">Terms of Service</router-link>
</v-footer>
</v-app>
</template>
<script>
@@ -33,8 +31,13 @@ export default {
max-width: 800px;
margin-left: auto;
margin-right: auto;
height: 100vh;
background-color: #d6e8de !important;
/* height: 100vh; */
}
.pane-l {
max-width: 1200px;
/* margin-left: auto;
margin-right: auto; */
}
.bg {

View File

@@ -0,0 +1,212 @@
<template>
<v-container class="pane">
<v-card :loading="loadingGroups || loadingArchive">
<v-card-title class="text-center">
Archive a single URL
</v-card-title>
<v-card-text>
<v-row>
<v-col cols="12">
<v-text-field v-model="url" label="URL" required :rules="[urlValidator]"></v-text-field>
</v-col>
<v-col cols="12" md="4">
<v-radio-group v-model="public" inline>
<v-radio label="Public" :value="true"></v-radio>
<v-radio label="Private" :value="false"></v-radio>
</v-radio-group>
</v-col>
<v-col cols="12" md="4">
<v-select v-show="!public && groupsLoaded" v-model="group" label="Group" :items="availableGroups"
density="compact" :disabled="!groupsLoaded"></v-select>
</v-col>
<v-col cols="12" md="4" class="text-right">
<v-btn @click="archiveUrl" color="primary" :disabled="!validUrl || loadingGroups || loadingArchive || (!public && group == -1)">
Archive
</v-btn>
</v-col>
</v-row>
<v-row>
<v-col cols="12">
<p v-if="loadingArchive">
<v-progress-circular color="primary" indeterminate></v-progress-circular>
Archive in progress task id = <code>{{ taskId }}</code>
</p>
<v-alert color="success" icon="mdi-information" v-if="archiveResult">
Archived successfully with id {{ taskId }} available <a :href="getUrlFromResult(archiveResult)" target="_blank">here</a>.
</v-alert>
<v-alert color="warning" icon="mdi-alert" v-if="archiveFailure">
Failure: {{archiveFailure}}
</v-alert>
<p v-if="validUrl">
You can <strong v-if="archiveFailure">still</strong> <router-link :to="`/urls?url=${encodeURIComponent(url)}`"
target="_blank"><v-icon>mdi-open-in-new</v-icon> search for archives</router-link> of
this URL.
</p>
</v-col>
</v-row>
</v-card-text>
</v-card>
<SnackBar :message="snackbarMessage" :show="snackbar" :color="snackbarColor" @update:show="snackbar = $event" />
</v-container>
</template>
<script>
import { urlValidator, getUrlFromResult } from "@/utils/misc";
import SnackBar from "@/components/SnackBar.vue";
export default {
name: "ArchiveUrl",
components: {
SnackBar,
},
data() {
return {
loadingGroups: false,
groupsLoaded: false,
availableGroups: [],
url: "",
public: true,
group: -1,
loadingArchive: false,
taskId: null,
archiveResult: null,
archiveFailure: null,
snackbar: false,
snackbarMessage: "",
snackbarColor: "red",
};
},
computed: {
urlValidator() {
return urlValidator;
},
getUrlFromResult() {
return getUrlFromResult;
},
validUrl() {
return this.url && this.urlValidator(this.url) === true;
},
},
watch: {
public(val) {
if (!val) this.loadGroups();
},
url(val) {
if (this.validUrl) {
this.archiveResult = null;
this.archiveFailure = null;
this.taskId = null;
}
},
},
methods: {
showSnackbar(message, color = "red") {
this.snackbarMessage = message;
this.snackbarColor = color;
this.snackbar = true;
},
archiveUrl() {
if (this.loadingGroups || this.loadingArchive) return;
this.loadingArchive = true;
this.archiveResult = null;
this.archiveFailure = null;
fetch(`${this.$store.state.API_ENDPOINT}/url/archive`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${this.$store.state.access_token}`,
},
body: JSON.stringify({
url: this.url,
group: this.public ? "" : this.group,
public: this.public,
tags: [],
})
})
.then(response => {
if (!response.ok) {
throw new Error(`API error: ${response.statusText}`);
}
return response.json();
})
.then(res => {
console.log("archiveUrl response", res);
this.taskId = res.id;
this.showSnackbar(`Your URL is being archived with id ${this.taskId}!`, "green");
this.pollForArchiveResults();
})
.catch(error => {
console.error("/archive ", error);
this.showSnackbar(`Unable to archive URL: ${error.message}`);
this.loadingArchive = false;
})
},
pollForArchiveResults() {
this.loadingArchive = true;
const poll = () => {
fetch(`${this.$store.state.API_ENDPOINT}/task/${this.taskId}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${this.$store.state.access_token}`,
}
})
.then(response => response.json())
.then(task => {
if (task.status === "SUCCESS") {
this.showSnackbar(`URL archived successfully with id ${task.result.id}!`, "green");
this.loadingArchive = false;
this.taskId = null;
this.archiveResult = task;
} else if (task.status === "FAILURE") {
this.showSnackbar(`Failed to archive URL: ${task.result.error}`);
this.loadingArchive = false;
this.taskId = null;
this.archiveFailure = task.result.error;
} else {
setTimeout(poll, 5000); // Poll every 5 seconds
}
})
.catch(error => {
console.error("/task ", error);
this.showSnackbar(`Error checking archive status: ${error.message}`);
this.loadingArchive = false;
});
};
poll();
},
loadGroups() {
if (this.groupsLoaded) return;
this.loadingGroups = true;
fetch(`${this.$store.state.API_ENDPOINT}/groups`, {
method: "GET",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${this.$store.state.access_token}`,
}
})
.then(response => response.json())
.then(items => {
if (!Array.isArray(items)) {
throw (`Unexpected response format from API`);
}
this.availableGroups = [{ title: "only me", value: "" }].concat(items.map(g => ({ title: g, value: g })));
this.group = this.availableGroups[0].value;
this.groupsLoaded = true;
})
.catch(error => {
console.error("/groups ", error);
this.showSnackbar(`Unable to fetch groups: ${error}`);
})
.finally(() => {
this.loadingGroups = false;
});
},
},
};
</script>

View File

@@ -1,29 +1,60 @@
<template>
<v-app-bar style="flex-grow: 0" class="text-no-wrap">
<v-toolbar-title>
<router-link to="/" class="nodecoration">
Bellingcat Auto Archiver demo
</router-link>
</v-toolbar-title>
<v-btn v-if="!user" @click="$store.dispatch('signin')">Sign In</v-btn>
<v-chip v-if="$store.state.errorMessage" :title="$store.state.errorMessage" color="red" variant="tonal"
closable class="mx-4">
ERROR: {{ $store.state.errorMessage }}
</v-chip>
<span class="user" v-if="user">
<v-chip v-if="user.active" color="green" prepend-icon="mdi-checkbox-marked-circle" variant="outlined">
<v-chip v-if="user.active" color="green" prepend-icon="mdi-checkbox-marked-circle" variant="outlined">
active
</v-chip>
<v-chip v-if="!user.active" color="red" prepend-icon="mdi-account-cancel" variant="outlined">
<v-chip v-if="!user.active" color="red" prepend-icon="mdi-account-cancel" variant="outlined">
inactive
</v-chip>
<span class="ms-4">{{ user.email }}</span>
<v-tooltip activator="parent" location="bottom">{{ activeUserMessage }}</v-tooltip>
<span class="ms-4">{{ user.email }}</span>
<v-tooltip activator="parent" location="bottom">{{ activeUserMessage }}</v-tooltip>
</span>
<v-btn v-if="user" href="#" @click="$store.dispatch('signout')">Sign Out</v-btn>
<v-btn v-if="!user" @click="$store.dispatch('signin')">Sign In</v-btn>
<v-menu v-if="user">
<template v-slot:activator="{ props }">
<v-app-bar-nav-icon v-bind="props"></v-app-bar-nav-icon>
</template>
<v-list>
<v-list-item to="urls">
<v-btn prepend-icon="mdi-link" variant="plain">URL</v-btn>
</v-list-item>
<v-list-item to="sheets">
<v-btn prepend-icon="mdi-table-large" variant="plain">Sheet</v-btn>
</v-list-item>
<v-list-item @click="$store.dispatch('signout')">
<v-btn prepend-icon="mdi-logout" variant="plain">Sign Out</v-btn>
</v-list-item>
</v-list>
</v-menu>
</v-app-bar>
</template>
<script>
export default {
name: "NavBar",
data() {
return {
drawer: false,
};
},
computed: {
user() {
return this.$store.state.user;

View File

@@ -0,0 +1,57 @@
<template>
<v-snackbar v-model="visible" :timeout="timeout" :top="top" :bottom="bottom" close-on-content-click>
{{ message }}
<template v-slot:actions>
<v-btn :color="color" variant="text" @click="visible = false">
Close
</v-btn>
</template>
</v-snackbar>
</template>
<script>
export default {
name: 'MySnackBar',
props: {
message: {
type: String,
required: true
},
timeout: {
type: Number,
default: 3000
},
color: {
type: String,
default: 'orange'
},
top: {
type: Boolean,
default: false
},
bottom: {
type: Boolean,
default: true
},
show: {
type: Boolean,
required: true
}
},
data() {
return {
visible: this.show
};
},
watch: {
show(val) {
this.visible = val;
},
visible(val) {
if (!val) {
this.$emit('update:show', false);
}
}
}
};
</script>

View File

@@ -2,6 +2,7 @@ import { createApp } from "vue";
import { createVuetify } from "vuetify";
import * as components from 'vuetify/components';
import * as directives from 'vuetify/directives';
import { VDateInput } from 'vuetify/labs/VDateInput';
import App from "./App.vue";
import router from "./router";
import store from "./store";
@@ -10,7 +11,7 @@ import "@mdi/font/css/materialdesignicons.css";
import "./styles/global.css";
const vuetify = createVuetify({
components,
components: { ...components, VDateInput, },
directives,
});

View File

@@ -1,6 +1,7 @@
import { createRouter, createWebHistory } from 'vue-router';
import HomeView from '../views/HomeView.vue';
import SheetView from '../views/SheetView.vue';
import UrlView from '../views/UrlView.vue';
const routes = [
{
@@ -10,9 +11,14 @@ const routes = [
},
{
path: '/sheets',
name: 'sheets',
name: 'Google Sheets Archiving',
component: SheetView,
},
{
path: '/urls',
name: 'URL Archiving',
component: UrlView,
},
{
path: '/privacy',
name: 'Privacy Policy',

View File

@@ -430,6 +430,7 @@ export default createStore({
if (!state.access_token) return true;
try {
const response = await fetch(`https://oauth2.googleapis.com/tokeninfo?access_token=${state.access_token}`);
if (response.status !== 200) return true;
const data = await response.json();
if (data.expires_in > 0) return false;
} catch (error) {

View File

@@ -1,3 +1,13 @@
html {
overflow-y: auto;
}
a {
color: #1a73e8;
text-decoration: none;
}
a:hover {
color: #0c47a1;
text-decoration: underline;
}

19
src/utils/misc.js Normal file
View File

@@ -0,0 +1,19 @@
export function urlValidator(url) {
if (!url) return true;
if (url.length < 10) return "URL is too short";
try {
new URL(url);
return true;
} catch (_) {
return "Not a valid URL";
}
}
export function getUrlFromResult(item) {
const final_media = item.result?.media?.filter(m => m?.properties?.id == '_final_media');
if (final_media && final_media.length > 0) {
return final_media[0].urls;
}
};

View File

@@ -1,14 +1,14 @@
<template>
<v-container>
<v-container class="pane" fluid>
<v-row>
<v-col>
<v-alert color="#f2d97c" icon="mdi-alert" >
<v-alert color="#f2d97c" icon="mdi-alert">
This is a prototype demo service provided on a
best-effort basis. <br />Do not use for mission critical or sensitive
data.
</v-alert>
<v-alert color="orange" icon="mdi-information" v-if="user && !user.active" >
To use this tool you need a Google account and <strong>permission from the Bellingcat team</strong>.<br/>
<v-alert color="orange" icon="mdi-information" v-if="user && !user.active">
To use this tool you need a Google account and <strong>permission from the Bellingcat team</strong>.<br />
You can do so HERE: TODO.
</v-alert>
<v-card style="margin-bottom: 1em">
@@ -20,9 +20,9 @@
This tool can be used to archive digital content via single <router-link to="/urls">URLs</router-link> or
<router-link to="/sheets">Google sheets</router-link>.
</p>
<blockquote>
<p v-if="!user || !user.active">
To use this tool you need a Google account and <strong>permission from the Bellingcat team</strong>.
</blockquote>
</p>
<p>
This tool uses <a href="https://github.com/bellingcat/auto-archiver">Bellingcat's Auto Archiver</a> to
archive online content. For more information about the Auto Archiver see
@@ -33,40 +33,24 @@
</p>
</v-card-text>
</v-card>
</v-col>
</v-row>
</v-container>
<ArchiveUrl v-if="user?.active" />
</template>
<script>
import DocList from "@/components/DocList.vue";
import ArchiveUrl from "@/components/ArchiveUrl.vue";
export default {
name: "HomeView",
components: {
DocList,
},
data() {
return {
docName: "Auto archiver sheet",
spreadsheetUrl: "",
};
ArchiveUrl
},
computed: {
user() {
return this.$store.state.user;
},
spreadsheetId() {
if (
this.spreadsheetUrl.startsWith("http") &&
this.spreadsheetUrl.split("/").length >= 6
) {
return this.spreadsheetUrl.split("/")[5];
}
return this.spreadsheetUrl;
},
},
methods: {},
}
};
</script>

View File

@@ -1,5 +1,5 @@
<template>
<v-container>
<v-container class="pane">
<v-row>
<v-col>
<v-card style="margin-bottom: 1em">
@@ -132,15 +132,7 @@
if it is not
</li>
</ol>
<v-alert
v-if="$store.state.errorMessage"
title="Error"
text
type="error"
variant="outlined"
closable
>{{ $store.state.errorMessage }}</v-alert
>
<v-text-field
label="Google Sheet URL"
v-model="spreadsheetUrl"
@@ -171,7 +163,7 @@
import DocList from "@/components/DocList.vue";
export default {
name: "HomeView",
name: "SheetView",
components: {
DocList,
},

219
src/views/UrlView.vue Normal file
View File

@@ -0,0 +1,219 @@
<template>
<v-container class="pane-l">
<v-row>
<v-col>
<v-card elevation="12">
<v-card-title>
Search archives by URL
TODO: toggle between all/and my latest
</v-card-title>
<v-card-text>
<v-form>
<v-row>
<v-col cols="12" md="6">
<v-date-input v-model="queryAfter" label="Archived After" variant="outlined" min="2022-01-01"
:max="queryBefore || today"></v-date-input>
</v-col>
<v-col cols="12" md="6">
<v-date-input v-model="queryBefore" label="Archived Before" variant="outlined"
:min="queryAfter || '2022-01-01'" :max="today"></v-date-input>
</v-col>
</v-row>
<v-text-field ref="searchInput" v-model="queryUrl" label="Search for this URL" prepend-icon="mdi-web"
variant="outlined" :rules="[urlValidator]" required @keyup.enter="searchForArchives"></v-text-field>
<v-row>
<v-col cols="12" class="text-right">
<v-btn @click="searchForArchives" color="primary" class="mt-4" size="large" :disabled="!validUrl">
Search
</v-btn>
</v-col>
</v-row>
</v-form>
<v-row>
<v-col>
<v-snackbar v-model="snackbar" :timeout="4000" top right close-on-content-click>
{{ snackbarMessage }}
<template v-slot:actions>
<v-btn color="orange" variant="text" @click="snackbar = false">
Close
</v-btn>
</template>
</v-snackbar>
<v-data-table-server density="compact" loading-text="Loading... Please wait"
no-data-text="Nothing found" v-model:items-per-page="itemsPerPage" :headers="headers"
:items="serverItems" :items-length="totalItems" :loading="loading" :search="tableSearch"
@update:options="loadItems" :items-per-page-options="pageOptions" show-expand item-value="id"
fixed-header>
<template v-slot:item.result="{ item }">
<a :href="getUrlFromResult(item)" target="_blank" rel="noopener noreferrer">{{ item.result?.status
}}</a>
</template>
<template v-slot:item.url="{ item }">
<a :href="item.url" target="_blank" rel="noopener noreferrer">{{ item.url }}</a>
</template>
<template v-slot:item.created_at="{ item }">
<time :datetime="item?.created_at">{{ item?.created_at }}</time>
</template>
<template v-slot:item.size="{ item }">
{{ ((item?.result?.metadata?.total_bytes || 0) / (1024 * 1024)).toFixed(2) }}
</template>
<template v-slot:item.files="{ item }">
{{ item?.result?.media?.length }}
</template>
<!-- EXPANDED ROW WITH SUBTABLE -->
<template v-slot:expanded-row="{ columns, item }">
<tr>
<td :colspan="columns.length" class="pa-0">
<v-data-table density="compact" class="sub-table elevation-0 bg-blue-grey-lighten-5"
:items="item?.result?.media" item-key="key" hide-default-footer :headers="fileHeaders">
<template v-slot:item.preview="{ item: media }">
<a :href="media.urls[0]" target="_blank">
<template v-if="media._mimetype?.startsWith('image/')">
<v-img :src="media.urls[0]" max-width="150" max-height="250" class="mx-auto"></v-img>
</template>
<template v-else-if="media._mimetype?.startsWith('video/')">
<video :src="media.urls[0]" controls style="max-width: 150px; max-height: 200px;"
class="mx-auto"></video>
</template>
<template v-else>
<span>{{ media?.properties?.id }}</span>
</template>
</a>
</template>
<template v-slot:item.hash="{ item: media }">
<span style="font-size: small;">{{ media?.properties?.hash }}</span>
</template>
</v-data-table>
</td>
</tr>
</template>
</v-data-table-server>
</v-col>
</v-row>
</v-card-text>
</v-card>
</v-col>
</v-row>
</v-container>
</template>
<script>
import { urlValidator, getUrlFromResult } from "@/utils/misc.js";
export default {
name: "UrlView",
data() {
return {
today: new Date().toISOString().substring(0, 10),
queryAfter: null,
queryBefore: null,
queryUrl: this.$route.query.url || "https://",
tableSearch: "", // used to retrigger the search
loading: false,
itemsPerPage: 5,
totalItems: 0,
pageOptions: [{ value: 5, title: '5' }, { value: 10, title: '10' }, { value: 25, title: '25' }, { value: 50, title: '50' }],
headers: [
{ title: "URL", value: "url" },
{ title: "Result", value: "result" },
{ title: "Archived At", value: "created_at" },
{ title: "Size (MB)", value: "size" },
{ title: "Files", value: "files" },
{ title: '', key: 'data-table-expand' },
],
fileHeaders: [
{ title: 'Preview', value: 'preview', align: 'center' },
{ title: 'Hash', value: 'hash', align: 'end', width: '150px' },
{ title: 'Size', value: 'properties.size', align: 'end', width: '150px' }
],
serverItems: [],
snackbar: false,
snackbarMessage: "",
};
},
computed: {
user() {
return this.$store.state.user;
},
validUrl() {
return this.queryUrl && this.urlValidator(this.queryUrl) === true;
},
urlValidator() {
return urlValidator;
},
getUrlFromResult() {
return getUrlFromResult;
},
},
methods: {
searchForArchives() {
if (!this.validUrl) return;
this.tableSearch = `${this.queryUrl}${this.queryAfter}${this.queryBefore}`;
},
loadItems({ page, itemsPerPage, sortBy }) {
if (!this.validUrl || this.loading === true) return;
this.loading = true;
const params = new URLSearchParams({
url: this.queryUrl,
limit: itemsPerPage,
skip: (page - 1) * itemsPerPage,
});
if (this.queryAfter) {
params.append("archived_after", this.queryAfter.toISOString());
}
if (this.queryBefore) {
params.append("archived_before", this.queryBefore.toISOString());
}
fetch(`${this.$store.state.API_ENDPOINT}/url/search?${params}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${this.$store.state.access_token}`,
}
}).then(response => response.json())
.then(items => {
if (!Array.isArray(items)) {
throw (`Unexpected response format from API`);
}
// Estimate totalItems if not provided by the API
this.serverItems = items;
if (items.length < itemsPerPage) {
this.totalItems = (page - 1) * itemsPerPage + items.length;
} else {
this.totalItems = (page + 1) * itemsPerPage; // Assume there are more items
}
})
.catch(error => {
console.error("/url/search ", error);
this.snackbarMessage = `Error searching for archives: ${error}`;
this.snackbar = true;
})
.finally(() => {
this.loading = false;
});
}
},
};
</script>
<style>
img,
video {
filter: grayscale(100%);
}
img:hover,
video:hover {
filter: none;
}
td {
word-break: break-all;
}
</style>