From ebe7286ce22b0f2d55d51dc0c15c3c2fd73caf88 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:21:44 +0000 Subject: [PATCH] final updates before release --- functions/index.js | 166 +- package.json | 4 +- src/App.vue | 32 +- src/components/AddSheet.vue | 502 +- src/components/ArchiveSheet.vue | 57 +- src/components/ManageSheets.vue | 312 +- src/components/NavBar.vue | 112 +- src/components/PermissionNeeded.vue | 57 +- src/components/SnackBar.vue | 108 +- src/components/WelcomeCard.vue | 110 +- src/main.js | 8 +- src/router/index.js | 36 +- src/store/index.js | 131 +- src/utils/misc.js | 30 +- src/views/ArchiveSearchView.vue | 214 +- src/views/ArchiveUrlView.vue | 187 +- src/views/HomeView.vue | 16 +- yarn.lock | 11087 +++++++++++++------------- 18 files changed, 6884 insertions(+), 6285 deletions(-) diff --git a/functions/index.js b/functions/index.js index 38b56d3..a0aa6bd 100644 --- a/functions/index.js +++ b/functions/index.js @@ -1,96 +1,98 @@ -/** - * Import function triggers from their respective submodules: - * - * const {onCall} = require("firebase-functions/v2/https"); - * const {onDocumentWritten} = require("firebase-functions/v2/firestore"); - * - * See a full list of supported triggers at https://firebase.google.com/docs/functions - */ +//NB: this code has been disabled since the cronjob is now handled by the API -const { onSchedule } = require("firebase-functions/v2/scheduler"); -const logger = require("firebase-functions/logger"); +// /** +// * Import function triggers from their respective submodules: +// * +// * const {onCall} = require("firebase-functions/v2/https"); +// * const {onDocumentWritten} = require("firebase-functions/v2/firestore"); +// * +// * See a full list of supported triggers at https://firebase.google.com/docs/functions +// */ -// The Firebase Admin SDK to access Firestore. -const { initializeApp } = require("firebase-admin/app"); -const { getFirestore } = require("firebase-admin/firestore"); +// const { onSchedule } = require("firebase-functions/v2/scheduler"); +// const logger = require("firebase-functions/logger"); -const { defineSecret } = require('firebase-functions/params'); -const API_TOKEN = defineSecret('API_SERVICE_PASSWORD'); -const CLIENT_EMAIL = defineSecret('GOOGLE_API_CLIENT_EMAIL'); -const PRIVATE_KEY = defineSecret('GOOGLE_API_PRIVATE_KEY'); +// // The Firebase Admin SDK to access Firestore. +// const { initializeApp } = require("firebase-admin/app"); +// const { getFirestore } = require("firebase-admin/firestore"); -const { google } = require('googleapis'); +// const { defineSecret } = require('firebase-functions/params'); +// const API_TOKEN = defineSecret('API_SERVICE_PASSWORD'); +// const CLIENT_EMAIL = defineSecret('GOOGLE_API_CLIENT_EMAIL'); +// const PRIVATE_KEY = defineSecret('GOOGLE_API_PRIVATE_KEY'); -initializeApp(); -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// const { google } = require('googleapis'); -String.prototype.hashCode = function () { - // https://stackoverflow.com/a/7616484/6196010 - // Generating 1M random strings and applying this function shows it's very balanced for modulo 60 - // 0 has double frequency of other numbers, but that's not a problem - var hash = 0, - i, chr; - if (this.length === 0) return hash; - for (i = 0; i < this.length; i++) { - chr = this.charCodeAt(i); - hash = ((hash << 5) - hash) + chr; - hash |= 0; // Convert to 32bit integer - } - return hash; -} +// initializeApp(); +// const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); -//TODO: disable the scheduler -exports.processSheetScheduler = onSchedule( - { secrets: [API_TOKEN, CLIENT_EMAIL, PRIVATE_KEY], schedule: "* * * * *" }, - async (event) => { - // authenticate the service account - const googleAuth = new google.auth.JWT(CLIENT_EMAIL.value(), null, PRIVATE_KEY.value().replace(/\\n/g, '\n'), 'https://www.googleapis.com/auth/spreadsheets'); - const sheets = await google.sheets({ version: 'v4', auth: googleAuth }); +// String.prototype.hashCode = function () { +// // https://stackoverflow.com/a/7616484/6196010 +// // Generating 1M random strings and applying this function shows it's very balanced for modulo 60 +// // 0 has double frequency of other numbers, but that's not a problem +// var hash = 0, +// i, chr; +// if (this.length === 0) return hash; +// for (i = 0; i < this.length; i++) { +// chr = this.charCodeAt(i); +// hash = ((hash << 5) - hash) + chr; +// hash |= 0; // Convert to 32bit integer +// } +// return hash; +// } - // get all documents from firestore sheets collection - const db = getFirestore(); +// //TODO: disable the scheduler +// exports.processSheetScheduler = onSchedule( +// { secrets: [API_TOKEN, CLIENT_EMAIL, PRIVATE_KEY], schedule: "* * * * *" }, +// async (event) => { +// // authenticate the service account +// const googleAuth = new google.auth.JWT(CLIENT_EMAIL.value(), null, PRIVATE_KEY.value().replace(/\\n/g, '\n'), 'https://www.googleapis.com/auth/spreadsheets'); +// const sheets = await google.sheets({ version: 'v4', auth: googleAuth }); - // each sheet runs once per hour, so we hash the sheet id and only process it if the hash % 60 matches the cron minute - const querySnapshot = await db.collection("sheets").get(); - const eventDate = new Date(Date.parse(event.scheduleTime)); - querySnapshot.forEach(async (doc) => { - const hashToSixty = Math.abs(doc.id.hashCode() % 60); - if (hashToSixty != eventDate.getMinutes()) { - return; - } - logger.log(`processing document ${doc.id}, its hash % 60 (${hashToSixty}) matches the cron minute (${eventDate.getMinutes()})`); +// // get all documents from firestore sheets collection +// const db = getFirestore(); - try { - await sheets.spreadsheets.get({ spreadsheetId: doc.data().sheetId }); - } catch (e) { - if (e.status == 404) { - await doc.ref.delete(); - logger.log(`document ${doc.data().sheetId} not found, deleted`); - return; - } - } +// // each sheet runs once per hour, so we hash the sheet id and only process it if the hash % 60 matches the cron minute +// const querySnapshot = await db.collection("sheets").get(); +// const eventDate = new Date(Date.parse(event.scheduleTime)); +// querySnapshot.forEach(async (doc) => { +// const hashToSixty = Math.abs(doc.id.hashCode() % 60); +// if (hashToSixty != eventDate.getMinutes()) { +// return; +// } +// logger.log(`processing document ${doc.id}, its hash % 60 (${hashToSixty}) matches the cron minute (${eventDate.getMinutes()})`); - // send POST request with sheetID to trigger sheet processing - const url = "https://auto-archiver-api.bellingcat.com/sheet_service"; - const data = { - sheet_id: doc.data().sheetId, - author_id: doc.data().email ?? doc.data().uid, - tags: ["setup-tool"] - }; - const options = { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${API_TOKEN.value()}`, - }, - body: JSON.stringify(data), - }; +// try { +// await sheets.spreadsheets.get({ spreadsheetId: doc.data().sheetId }); +// } catch (e) { +// if (e.status == 404) { +// await doc.ref.delete(); +// logger.log(`document ${doc.data().sheetId} not found, deleted`); +// return; +// } +// } - const response = await fetch(url, options); +// // send POST request with sheetID to trigger sheet processing +// const url = "https://auto-archiver-api.bellingcat.com/sheet_service"; +// const data = { +// sheet_id: doc.data().sheetId, +// author_id: doc.data().email ?? doc.data().uid, +// tags: ["setup-tool"] +// }; +// const options = { +// method: "POST", +// headers: { +// "Content-Type": "application/json", +// Authorization: `Bearer ${API_TOKEN.value()}`, +// }, +// body: JSON.stringify(data), +// }; - await doc.ref.update({ lastArchived: Date.now() }); +// const response = await fetch(url, options); - await sleep(100); - }); - } -); +// await doc.ref.update({ lastArchived: Date.now() }); + +// await sleep(100); +// }); +// } +// ); diff --git a/package.json b/package.json index 04bc004..459df47 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "private": true, "scripts": { "serve": "vue-cli-service serve --port 8081 --skip-plugins @vue/cli-plugin-eslint", - "build": "vue-cli-service build", + "build": "VUE_APP_API_ENDPOINT=https://auto-archiver-api.bellingcat.com vue-cli-service build --skip-plugins @vue/cli-plugin-eslint", + "preview": "serve -s dist -l 8081", "lint": "vue-cli-service lint" }, "dependencies": { @@ -35,6 +36,7 @@ "firebase-admin": "^11.11.1", "firebase-functions": "^4.5.0", "prettier": "^2.4.1", + "serve": "^14.2.4", "vue-template-compiler": "^2.6.14" } } diff --git a/src/App.vue b/src/App.vue index c729bb6..cc64020 100644 --- a/src/App.vue +++ b/src/App.vue @@ -13,17 +13,25 @@ - You can deploy your own version of this tool by hosting the API and the UI. -
- This tool uses Bellingcat's Auto Archiver under the hood to archive online content. -
- For more information about it see associated - article. - + You can deploy your own version of this tool by hosting the + API and + the + UI. +
+ This tool uses + Bellingcat's Auto Archiver + under the hood to archive online content. +
+ For more information about it see + associated article. - @@ -96,14 +104,14 @@ p { } code { - background-color: rgba(0, 0, 0, .1); - padding: .2em .4em; + background-color: rgba(0, 0, 0, 0.1); + padding: 0.2em 0.4em; } .v-card .v-card-text { font-size: 1.1rem; font-weight: 400; line-height: 1.7rem; - letter-spacing: .0092em; + letter-spacing: 0.0092em; } diff --git a/src/components/AddSheet.vue b/src/components/AddSheet.vue index 10a575e..e43e95d 100644 --- a/src/components/AddSheet.vue +++ b/src/components/AddSheet.vue @@ -1,210 +1,320 @@ \ No newline at end of file + diff --git a/src/components/ArchiveSheet.vue b/src/components/ArchiveSheet.vue index 38888ff..ffa8662 100644 --- a/src/components/ArchiveSheet.vue +++ b/src/components/ArchiveSheet.vue @@ -1,9 +1,13 @@ \ No newline at end of file + diff --git a/src/components/ManageSheets.vue b/src/components/ManageSheets.vue index 25b07bd..d79c06a 100644 --- a/src/components/ManageSheets.vue +++ b/src/components/ManageSheets.vue @@ -1,139 +1,199 @@ \ No newline at end of file + diff --git a/src/components/NavBar.vue b/src/components/NavBar.vue index 67e45f1..f084ac6 100644 --- a/src/components/NavBar.vue +++ b/src/components/NavBar.vue @@ -6,38 +6,75 @@ - - + ERROR: {{ $store.state.errorMessage }}
- + - + active - + inactive - {{ activeUserMessage }} + {{ + activeUserMessage + }} {{ user.email }} - Sign Out + Sign Out Sign In @@ -48,21 +85,33 @@ - + {{ btn.text }} - {{ btn.tooltip }} + {{ + btn.tooltip + }} - Sign Out + Sign Out - @@ -73,10 +122,25 @@ export default { return { drawer: false, btns: [ - { to: "/", icon: "mdi-table-large", text: "Sheets", tooltip: "Create, manage, and archive Google Sheets." }, - { to: "/url", icon: "mdi-cloud-download-outline", text: "URL", tooltip: "Archive a single URL." }, - { to: "/archives", icon: "mdi-magnify", text: "Archives", tooltip: "Search for archived URLs." }, - ] + { + to: "/", + icon: "mdi-table-large", + text: "Sheets", + tooltip: "Create, manage, and archive Google Sheets.", + }, + { + to: "/url", + icon: "mdi-cloud-download-outline", + text: "URL", + tooltip: "Archive a single URL.", + }, + { + to: "/archives", + icon: "mdi-magnify", + text: "Archives", + tooltip: "Search for archived URLs.", + }, + ], }; }, computed: { @@ -89,9 +153,9 @@ export default { } return "This account is inactive, please reach out to the Bellingcat team for access."; }, - loadingUserState() { - return this.$store.state?.loadingUserState; - } + loadingUserState() { + return this.$store.state?.loadingUserState; + }, }, }; diff --git a/src/components/PermissionNeeded.vue b/src/components/PermissionNeeded.vue index b7c31a0..b48cbf5 100644 --- a/src/components/PermissionNeeded.vue +++ b/src/components/PermissionNeeded.vue @@ -1,32 +1,39 @@ - \ No newline at end of file + diff --git a/src/components/SnackBar.vue b/src/components/SnackBar.vue index 29c80f1..ab908a2 100644 --- a/src/components/SnackBar.vue +++ b/src/components/SnackBar.vue @@ -1,57 +1,63 @@ \ No newline at end of file + diff --git a/src/components/WelcomeCard.vue b/src/components/WelcomeCard.vue index c7b6122..dca0645 100644 --- a/src/components/WelcomeCard.vue +++ b/src/components/WelcomeCard.vue @@ -1,57 +1,67 @@ - \ No newline at end of file + diff --git a/src/main.js b/src/main.js index b39f956..cf6590d 100644 --- a/src/main.js +++ b/src/main.js @@ -1,8 +1,8 @@ import { createApp } from "vue"; import { createVuetify } from "vuetify"; -import * as components from 'vuetify/components'; -import * as directives from 'vuetify/directives'; -import { VDateInput } from 'vuetify/labs/VDateInput'; +import * as components from "vuetify/components"; +import * as directives from "vuetify/directives"; +import { VDateInput } from "vuetify/labs/VDateInput"; import App from "./App.vue"; import router from "./router"; import store from "./store"; @@ -12,7 +12,7 @@ import "@mdi/font/css/materialdesignicons.css"; import "./styles/global.css"; const vuetify = createVuetify({ - components: { ...components, VDateInput, }, + components: { ...components, VDateInput }, directives, }); diff --git a/src/router/index.js b/src/router/index.js index a976ee4..359b569 100644 --- a/src/router/index.js +++ b/src/router/index.js @@ -1,41 +1,41 @@ -import { createRouter, createWebHistory } from 'vue-router'; -import HomeView from '../views/HomeView.vue'; -import ArchiveSearchView from '../views/ArchiveSearchView.vue'; +import { createRouter, createWebHistory } from "vue-router"; +import HomeView from "../views/HomeView.vue"; +import ArchiveSearchView from "../views/ArchiveSearchView.vue"; import ArchiveUrlView from "../views/ArchiveUrlView.vue"; const routes = [ { - path: '/', - name: 'home', + path: "/", + name: "home", component: HomeView, }, { - path: '/url', - name: 'URL Archiving', + path: "/url", + name: "URL Archiving", component: ArchiveUrlView, }, { - path: '/archives', - name: 'Archives search', + path: "/archives", + name: "Archives search", component: ArchiveSearchView, }, { - path: '/privacy', - name: 'Privacy Policy', + path: "/privacy", + name: "Privacy Policy", component: () => - import(/* webpackChunkName: "privacy" */ '../views/PrivacyView.vue'), + import(/* webpackChunkName: "privacy" */ "../views/PrivacyView.vue"), }, { - path: '/tos', - name: 'Terms of Use', + path: "/tos", + name: "Terms of Use", component: () => - import(/* webpackChunkName: "tos" */ '../views/TOSView.vue'), + import(/* webpackChunkName: "tos" */ "../views/TOSView.vue"), }, { - path: '/:pathMatch(.*)*', - redirect: '/', - } + path: "/:pathMatch(.*)*", + redirect: "/", + }, ]; const router = createRouter({ diff --git a/src/store/index.js b/src/store/index.js index a0847f6..e91e96c 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -1,12 +1,13 @@ import { createStore } from "vuex"; -import { gapi, client } from "@/gapi"; +import { gapi } from "@/gapi"; import { signOut, GoogleAuthProvider, signInWithCredential, + browserLocalPersistence, + setPersistence, } from "firebase/auth"; -import { collection, } from "firebase/firestore"; -import { firebaseAuth, firebaseFirestore } from "@/firebase.js"; +import { firebaseAuth } from "@/firebase.js"; function saveToLocalStorage(state) { localStorage.setItem("user", JSON.stringify(state.user)); @@ -25,7 +26,7 @@ function clearLocalStorage() { } async function waitForGapiAuth2() { - return new Promise((resolve, reject) => { + return new Promise((resolve, _reject) => { const checkGapiAuth2 = () => { if (gapi.auth2 && gapi.auth2.getAuthInstance()) { resolve(gapi.auth2.getAuthInstance()); @@ -45,9 +46,8 @@ export default createStore({ sheets: [], loadingUserState: false, errorMessage: "", - // # TODO: reenable production API endpoint // API_ENDPOINT: "https://auto-archiver-api.bellingcat.com" - API_ENDPOINT: "http://localhost:8004" + API_ENDPOINT: process.env.VUE_APP_API_ENDPOINT || "http://localhost:8004", }, mutations: { setUser(state, user) { @@ -60,7 +60,9 @@ export default createStore({ }, setUserPermissions(state, permissions) { state.user.permissions = permissions; - state.user.groups = Object.keys(permissions).filter(key => key !== "all"); + state.user.groups = Object.keys(permissions).filter( + (key) => key !== "all" + ); state.loadingUserState = false; saveToLocalStorage(state); }, @@ -90,6 +92,10 @@ export default createStore({ commit("setAccessToken", access_token); const credential = GoogleAuthProvider.credential(null, access_token); + // Set persistence before signing in + await setPersistence(firebaseAuth, browserLocalPersistence); + + // Sign in with the provided credential const response = await signInWithCredential(firebaseAuth, credential); commit("setUser", response.user); @@ -108,7 +114,7 @@ export default createStore({ callback, }); - client.requestAccessToken(); + await client.requestAccessToken(); }, async signout({ commit }) { @@ -138,16 +144,14 @@ export default createStore({ async checkActiveUser({ state, dispatch, commit }) { try { commit("setErrorMessage", ""); - const r = await fetch( - `${state.API_ENDPOINT}/user/active`, - { - method: "GET", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${state.access_token}`, - }, - } - ) + console.log(`${state.API_ENDPOINT}/user/active`); + const r = await fetch(`${state.API_ENDPOINT}/user/active`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${state.access_token}`, + }, + }); const response = await r.json(); commit("setUserActiveState", response.active); if (response.active === true) { @@ -155,48 +159,51 @@ export default createStore({ } } catch (error) { console.error("checkActiveUser (firebase.js): ", error); - commit("setErrorMessage", "Unable to check user status against the API"); + commit( + "setErrorMessage", + "Unable to check user status against the API" + ); } }, async checkUserPermissions({ state, commit }) { try { commit("setErrorMessage", ""); - const r = await fetch( - `${state.API_ENDPOINT}/user/permissions`, - { - method: "GET", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${state.access_token}`, - }, - } - ); + const r = await fetch(`${state.API_ENDPOINT}/user/permissions`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${state.access_token}`, + }, + }); const response = await r.json(); commit("setUserPermissions", response); } catch (error) { console.error("checkUserPermissions (firebase.js): ", error); - commit("setErrorMessage", "Unable to fetch user permissions from the API"); + commit( + "setErrorMessage", + "Unable to fetch user permissions from the API" + ); } }, async checkUserUsage({ state, commit }) { try { commit("setErrorMessage", ""); - const r = await fetch( - `${state.API_ENDPOINT}/user/usage`, - { - method: "GET", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${state.access_token}`, - }, - } - ); + const r = await fetch(`${state.API_ENDPOINT}/user/usage`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${state.access_token}`, + }, + }); const response = await r.json(); commit("setUserUsage", response); } catch (error) { console.error("checkUserUsage (firebase.js): ", error); - commit("setErrorMessage", "Unable to fetch user usage quota from the API"); + commit( + "setErrorMessage", + "Unable to fetch user usage quota from the API" + ); } }, @@ -210,8 +217,8 @@ export default createStore({ headers: { "Content-Type": "application/json", Authorization: `Bearer ${state.access_token}`, - } - }).then(async response => { + }, + }).then(async (response) => { const res = await response.json(); if (response.status === 200) { commit("setSheets", res); @@ -219,13 +226,14 @@ export default createStore({ throw new Error(JSON.stringify(res)); } }); - } catch (error) { console.error("getSheets (firebase.js): ", error); } - }, - async createSheet({ state, dispatch, commit }, {name, service_account_email}) { + async createSheet( + { _state, dispatch, _commit }, + { name, service_account_email } + ) { return new Promise(async (resolve, reject) => { try { // create new sheet @@ -354,8 +362,7 @@ export default createStore({ resource: { role: "writer", type: "user", - emailAddress: - service_account_email, + emailAddress: service_account_email, }, }); @@ -374,7 +381,9 @@ export default createStore({ isTokenExpired: async (state) => { if (!state.access_token) return true; try { - const response = await fetch(`https://oauth2.googleapis.com/tokeninfo?access_token=${state.access_token}`); + const response = await fetch( + `https://oauth2.googleapis.com/tokeninfo?access_token=${state.access_token}` + ); if (response.status !== 200) return true; const data = await response.json(); if (data.expires_in > 0) return false; @@ -398,18 +407,20 @@ export default createStore({ store.commit("setLoadingUserState", true); store.commit("setUser", user); store.commit("setAccessToken", access_token); - store.getters.isTokenExpired.then((expired) => { - if (expired) { + store.getters.isTokenExpired + .then((expired) => { + if (expired) { + store.dispatch("signout"); + } else { + store.dispatch("checkActiveUser"); + store.dispatch("checkUserPermissions"); + store.dispatch("checkUserUsage"); + } + }) + .catch((error) => { + console.error("Error checking token expiration:", error); store.dispatch("signout"); - } else { - store.dispatch("checkActiveUser"); - store.dispatch("checkUserPermissions"); - store.dispatch("checkUserUsage"); - } - }).catch((error) => { - console.error("Error checking token expiration:", error); - store.dispatch("signout"); - }); + }); } }, ], diff --git a/src/utils/misc.js b/src/utils/misc.js index c10c11d..bb93a59 100644 --- a/src/utils/misc.js +++ b/src/utils/misc.js @@ -1,19 +1,19 @@ - - export function urlValidator(url) { - if (!url) return true; - if (url.length < 10) return "URL is too short"; - try { - new URL(url); - return true; - } catch (_) { - return "Not a valid URL"; - } + if (!url) return true; + if (url.length < 10) return "URL is too short"; + try { + new URL(url); + return true; + } catch (_) { + return "Not a valid URL"; + } } export function getUrlFromResult(item) { - const final_media = item.result?.media?.filter(m => m?.properties?.id == '_final_media'); - if (final_media && final_media.length > 0) { - return final_media[0].urls; - } -}; \ No newline at end of file + const final_media = item.result?.media?.filter( + (m) => m?.properties?.id == "_final_media" + ); + if (final_media && final_media.length > 0) { + return final_media[0].urls; + } +} diff --git a/src/views/ArchiveSearchView.vue b/src/views/ArchiveSearchView.vue index e3d8547..2f66599 100644 --- a/src/views/ArchiveSearchView.vue +++ b/src/views/ArchiveSearchView.vue @@ -1,6 +1,6 @@