first MVP

.github/funding.yml

@@ -1,2 +1,2 @@
-github: fregante
+github: bellingcat
-custom: https://paypal.me/bytemode
+custom: https://www.patreon.com/bellingcat

.github/workflows/template-cleanup.yml

@@ -1,26 +0,0 @@
-# This is a GitHub Actions workflow for cleaning up resources in the original template. When users create
-# a new repository from the template, the workflow deletes and edits files and push a commit.
-#
-# There is no straightforward way to exclude files when a template is used, so this is a workaround for it.
-# https://github.community/t/can-you-ignore-files-folders-when-making-a-repo-from-a-template/3279
-
-name: Template cleanup
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    if: github.event.repository.name != 'browser-extension-template'
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cleanup
-        run: |
-          rm -f \
-            .github/funding.yml \
-            .github/workflows/template-cleanup.yml
-      - uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          commit_message: Template cleanup

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Stichting Bellingcat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package.json

@@ -1,7 +1,7 @@
 {
 	"private": true,
 	"scripts": {
-		"build": "parcel build source/manifest.json --no-content-hash --no-source-maps --dist-dir distribution --no-cache --detailed-report 0",
+		"build": "rm -rf distribution && parcel build source/manifest.json --no-content-hash --no-source-maps --dist-dir distribution --no-cache --detailed-report 0",
 		"lint": "run-p lint:*",
 		"lint-fix": "run-p 'lint:* -- --fix'",
 		"lint:css": "stylelint source/**/*.css",
@@ -23,11 +23,15 @@
 		"extends": "stylelint-config-xo"
 	},
 	"dependencies": {
+		"material-design-icons": "^3.0.1",
+		"materialize-css": "^1.0.0-rc.2",
+		"vue": "^3.2.45",
 		"webext-base-css": "^1.4.1",
 		"webext-options-sync": "^3.1.0"
 	},
 	"devDependencies": {
 		"@parcel/config-webextension": "^2.6.2",
+		"@parcel/transformer-vue": "^2.6.2",
 		"npm-run-all": "^4.1.5",
 		"parcel": "^2.6.2",
 		"stylelint": "^14.9.1",
@@ -42,8 +46,8 @@
 			]
 		}
 	},
- 	"@parcel/bundler-default-bug": "https://github.com/parcel-bundler/parcel/issues/8071",
+	"@parcel/bundler-default-bug": "https://github.com/parcel-bundler/parcel/issues/8071",
- 	"@parcel/bundler-default": {
+	"@parcel/bundler-default": {
- 		"minBundles": 10000000
+		"minBundles": 10000000
- 	}
+	}
 }

privacy-policy.md

@@ -4,4 +4,4 @@ No data or personal information is collected by browser-extension-template.
 
 ##### Contact
 
-If you have any questions or suggestions regarding this privacy policy, do not hesitate to [contact us](https://github.com/fregante/browser-extension-template/issues/new).
+If you have any questions or suggestions regarding this privacy policy, do not hesitate to [contact us](https://github.com/bellingcat/auto-archiver-extension/issues/new).

source/background.js

@@ -1,2 +0,0 @@
-// eslint-disable-next-line import/no-unassigned-import
-import './options-storage.js';

source/content.css

@@ -1,8 +0,0 @@
-#text-notice {
-	position: fixed;
-	top: 10px;
-	left: 10px;
-	padding: 10px;
-	background: #fff;
-	z-index: 999999;
-}

source/content.js

@@ -1,16 +0,0 @@
-import optionsStorage from './options-storage.js';
-
-console.log('💈 Content script loaded for', chrome.runtime.getManifest().name);
-async function init() {
-	const options = await optionsStorage.getAll();
-	const color = 'rgb(' + options.colorRed + ', ' + options.colorGreen + ',' + options.colorBlue + ')';
-	const text = options.text;
-	const notice = document.createElement('div');
-	notice.innerHTML = text;
-	document.body.prepend(notice);
-	notice.id = 'text-notice';
-	notice.style.border = '2px solid ' + color;
-	notice.style.color = color;
-}
-
-init();

source/css/popup.css

@@ -0,0 +1,33 @@
+body {
+    font-size: 100%;
+}
+
+#app {
+    min-width: 40em;
+    margin: 15px;
+}
+
+#icon {
+    max-height: 26px;
+    vertical-align: middle;
+}
+
+#archiveResults .row{
+    /* table-layout: fixed; */
+	width:90%;
+	max-width:100px;
+}
+
+/* #archiveResults td {
+    width: auto;
+}
+
+#archiveResults td:nth-child(2) {
+    width: 150px;
+} */
+
+table td {
+    word-wrap: break-word;
+    overflow-wrap: break-word;
+    padding: 5px;
+}

source/html/options.html

@@ -33,4 +33,4 @@
 	</div>
 </form>
 
-<script src="options.js" type="module"></script>
+<script src="../js/options.js" type="module"></script>

source/html/popup.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+    <link rel="stylesheet" href="../css/popup.css">
+</head>
+
+<body>
+		<div id="app"></div>
+    </div>
+</body>
+
+<script src="../js/popup.js" type="module"></script>
+
+</html>

source/js/background.js

@@ -0,0 +1,98 @@
+// eslint-disable-next-line import/no-unassigned-import
+// import './options-storage.js';
+import optionsStorage from './options-storage.js';
+
+const API_ENDPOINT = 'http://localhost:8000/tasks'
+
+chrome.runtime.onMessage.addListener(((r, s, sR) => {
+	processMessages(r, s, sR)
+	return true; // needed for sendResponse to be async
+}));
+
+async function processMessages(request, sender, sendResponse) {
+	console.info(`action {${request.action}} from ${sender.tab ? 'content-script (' + sender.tab.url + ')' : 'the extension'}`)
+	if (request.action === "archive") {
+		archiveUrl(sendResponse);
+	} else if (request.action === "status") {
+		const task_db = await getTaskById(request.task.task_id);
+		if (task_db?.status == "SUCCESS" || task_db?.status == 'FAILURE') {
+			console.log("ALREADY FINSIHED, NO REQS")
+			sendResponse(task_db)
+		}
+		const task_fresh = await checkTaskStatus(request.task)
+		sendResponse(task_fresh)
+	} else if (request.action === "getTasks") {
+		sendResponse(await getAllTasks());
+	}
+}
+
+function archiveUrl(sendResponse) {
+	chrome.tabs.query({
+		active: true,
+		lastFocusedWindow: true
+	}, async (tabs) => {
+		let url = tabs[0].url;
+		console.log(`url=${url}`);
+		const response = await submitUrlTask(url)
+		const new_archive = { url, task_id: response.task_id, status: 'PENDING', result: {} };
+		await upsertTask(new_archive);
+		sendResponse(new_archive);
+	});
+}
+
+function submitUrlTask(url) {
+	console.log(`API: SUBMIT`)
+	return new Promise((resolve, reject) => {
+		fetch(API_ENDPOINT, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },//, 'X-API-KEY': 'TODO' },
+			body: JSON.stringify({ url }),
+		}).then(
+			response => response.json(),
+		).then(response => resolve(response)
+		).catch(err => {
+			console.log(`There was an error: ${err}`)
+			reject(err)
+		});
+	})
+}
+
+function checkTaskStatus(task) {
+	console.log(`API: STATUS`)
+	return new Promise((resolve, reject) => {
+		fetch(`${API_ENDPOINT}/${task.task_id}`, {
+			method: 'GET',
+			headers: { 'Content-Type': 'application/json' },
+		}).then(
+			response => response.json(),
+		).then(response => {
+			const new_task = {
+				url: task.url,
+				task_id: response.task_id,
+				status: response.task_status,
+				result: JSON.parse(response.task_result),
+			}
+			console.log(new_task);
+			upsertTask(new_task);
+			resolve(new_task)
+		}
+		).catch(err => reject(err));
+	})
+}
+
+async function getAllTasks() {
+	const storage = await optionsStorage.getAll();
+	return storage.archived_urls;
+}
+
+//TODO: improve with less reads from storage
+async function upsertTask(task) {
+	const storage = await optionsStorage.getAll();
+	storage.archived_urls[task.task_id] = task;
+	await optionsStorage.set(storage);
+}
+
+async function getTaskById(task) {
+	const storage = await optionsStorage.getAll();
+	return storage.archived_urls[task.task_id];
+}

source/js/options-storage.js

@@ -2,10 +2,7 @@ import OptionsSync from 'webext-options-sync';
 
 export default new OptionsSync({
 	defaults: {
-		colorRed: 244,
+		archived_urls: {},
-		colorGreen: 67,
-		colorBlue: 54,
-		text: 'Set a text!',
 	},
 	migrations: [
 		OptionsSync.migrations.removeUnused,

source/js/options.js

@@ -1,6 +1,6 @@
 // eslint-disable-next-line import/no-unassigned-import
 import 'webext-base-css';
-import './options.css';
+import '../css/options.css';
 
 import optionsStorage from './options-storage.js';
 

source/js/popup.js

@@ -0,0 +1,22 @@
+import { createApp } from "vue";
+import Popup from "../vue/Popup.vue";
+import 'materialize-css/dist/css/materialize.min.css'
+import 'material-design-icons/iconfont/material-icons.css'
+
+
+const app = createApp(Popup);
+app.mount("#app");
+
+// Import browser from 'webextension-polyfill';
+// import optionsStorage from './options-storage.js';
+
+document.addEventListener('DOMContentLoaded', async () => {
+	// TODO: uncomment if using options
+	// listenForOptionsClick();
+});
+
+// Function listenForOptionsClick() {
+// 	document.querySelector('#optionsBtn').addEventListener('click', () => {
+// 		browser.runtime.openOptionsPage();
+// 	});
+// }

source/manifest.json

@@ -1,39 +1,35 @@
 {
-	"name": "Awesome Extension",
+    "name": "Auto-archiver extension",
-	"version": "0.0.0",
+    "version": "0.0.1",
-	"description": "An awesome new browser extension",
+    "description": "A gateway to effective archiving of online content, including behind private platforms. ",
-	"homepage_url": "https://github.com/fregante/browser-extension-template",
+    "homepage_url": "https://github.com/bellingcat/auto-archiver-extension",
-	"manifest_version": 3,
+    "manifest_version": 3,
-	"minimum_chrome_version": "100",
+    "minimum_chrome_version": "100",
-	"browser_specific_settings": {
+    "browser_specific_settings": {
-		"gecko": {
+        "gecko": {
-			"id": "awesome-extension@notlmn.github.io",
+            "id": "todo@github.io",
-			"strict_min_version": "100.0"
+            "strict_min_version": "100.0"
-		}
+        }
-	},
+    },
-	"icons": {
+    "icons": {
-		"128": "icon.png"
+        "128": "img/icon.png"
-	},
+    },
-	"permissions": [
+    "permissions": [
-		"storage"
+        "storage", "tabs"
-	],
+    ],
-	"host_permissions": [
+    "host_permissions": [
-		"https://github.com/*"
+        "*://*/*"
-	],
+    ],
-	"content_scripts": [
+    "background": {
-		{
+        "service_worker": "js/background.js",
-			"matches": [ "https://github.com/fregante/browser-extension-template/*" ],
+        "type": "module"
-			"js": [ "content.js" ],
+    },
-			"css": [ "content.css" ],
+    "action": {
-			"run_at": "document_end"
+        "default_popup": "html/popup.html"
-		}
+    },
-	],
+    "content_scripts": [],
-	"options_ui": {
+    "options_ui": {
-		"browser_style": true,
+        "browser_style": true,
-		"page": "options.html"
+        "page": "html/options.html"
-	},
+    }
-	"background": {
-		"service_worker": "background.js",
-		"type": "module"
-	}
 }

source/vue/Popup.vue

@@ -0,0 +1,86 @@
+<template>
+    <h5>
+        <img src="../img/icon.png" alt="icon" id="icon">
+        Auto Archiver extension
+        <button v-on:click="archive" class="waves-effect waves-light btn-small right">Archive!</button>
+    </h5>
+    <div class="input-field col s6">
+        <i class="material-icons prefix">search</i>
+        <input id="icon_prefix" type="text" v-model="search">
+        <label for="icon_prefix">Search for URLs</label>
+    </div>
+    <table id="archiveResults">
+        <thead>
+            <tr class="row">
+                <th class="col s1"></th>
+                <th class="col s5">URL</th>
+                <th class="col s2">Result</th>
+                <th class="col s3">Date</th>
+            </tr>
+        </thead>
+        <tbody>
+            <TaskItem v-for="t in displayTasks" :key="t.task_id" :initial-task="t" />
+        </tbody>
+    </table>
+</template>
+
+
+<script>
+import M from 'materialize-css';
+import TaskItem from './TaskItem.vue';
+
+export default {
+    data() {
+        return {
+            tasks: [],
+            isLoading: false,
+            search: ''
+        };
+    },
+    methods: {
+        archive: function () {
+            // M.toast({html: 'DONE'})
+
+            // chrome.tabs.sendMessage
+            this.isLoading = !this.isLoading;
+            (async () => {
+                const response = await chrome.runtime.sendMessage({
+                    action: "archive"
+                });
+                // do something with response here, not outside the function
+                this.url = response.url;
+                this.task_id = response.task_id;
+                this.addTask(response)
+            })();
+        },
+        displayAllTasks: function () {
+            (async () => {
+                const tasks = await chrome.runtime.sendMessage({
+                    action: "getTasks"
+                });
+                console.log(tasks)
+                this.tasks = tasks;
+            })();
+        },
+        addTask: function (task) {
+            this.tasks[task.task_id] = task;
+        }
+    },
+    computed: {
+        displayTasks() {
+            let st = Object.values(this.tasks)
+                .filter(t => t?.url.toLowerCase().includes(this.search.toLowerCase()))
+                .sort((t1, t2) => (t1?.result?._processed_at || 0) - (t2?.result?._processed_at || 0)).slice(0, 25)
+            return st
+        }
+    },
+    mounted() {
+        M.AutoInit()
+        this.displayAllTasks()
+    },
+    created() { },
+    components: {
+        TaskItem
+    }
+};
+</script>

source/vue/TaskItem.vue

@@ -0,0 +1,78 @@
+<template>
+	<tr class="row">
+		<td class="col s1">
+			<div v-if="task.status == 'PENDING'" class="preloader-wrapper small active">
+				<div class="spinner-layer ">
+					<div class="circle-clipper left">
+						<div class="circle"></div>
+					</div>
+					<div class="gap-patch">
+						<div class="circle"></div>
+					</div>
+					<div class="circle-clipper right">
+						<div class="circle"></div>
+					</div>
+				</div>
+			</div>
+			<div v-if="task.status == 'SUCCESS'">
+				<i class="material-icons small green-text darken-4">done</i>
+			</div>
+			<div v-if="task.status == 'FAILURE'">
+				<i class="material-icons small red-text darken-4">clear</i>
+			</div>
+		</td>
+		<td class="col s5"><a :href="task?.url">{{ task.url }}</a></td>
+		<td class="col s2"><a v-if="archiveUrl.length" :href="archiveUrl" target="_blank">{{ task?.result?.status || "open" }}</a> </td>
+		<td class="col s3">{{ readbleDate }}</td>
+	</tr>
+</template>
+
+<script>
+export default {
+	name: 'TaskItem',
+	props: ['initialTask'],
+	data() {
+		return {
+			task: this.initialTask
+		}
+	},
+	methods: {
+		checkStatus: function () {
+			console.log(this.task)
+			if (this.taskFinished(this.task)) return
+			this.intervalId = setInterval(function () {
+				chrome.runtime.sendMessage({
+					action: "status",
+					task: this.task
+				}).then(updated_task => {
+					console.log(updated_task)
+					if (this.taskFinished(updated_task)) {
+						clearInterval(this.intervalId);
+						this.task = updated_task
+					}
+				})
+			}.bind(this), 2500);
+		},
+		taskFinished: function (task) {
+			return task.status == 'SUCCESS' || task.status == 'FAILURE';
+		}
+	},
+	computed: {
+		archiveUrl() {
+			// return this.task?.result?.media?.urls.at(0) || '';
+			console.log(this.task?.result?.media);
+			console.log(this.task?.result?.media?.filter(m=>m?.properties?.id=="_final_media"));
+			console.log(this.task?.result?.media?.filter(m=>m?.properties?.id=="_final_media")?.urls?.at(0));
+			return this.task?.result?.media?.filter(m=>m?.properties?.id=="_final_media")?.at(0)?.urls?.at(0) || '';
+		},
+		readbleDate() {
+			if (this.task?.result?._processed_at) {
+				return new Date(this.task.result._processed_at * 1e3).toISOString().slice(0, 19);
+			}
+		}
+	},
+	mounted() {
+		this.checkStatus();
+	}
+}
+</script>