From 07ee773a54ab346b174ee55d7be02c2f4682c1aa Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 4 Mar 2025 10:54:16 +0000 Subject: [PATCH] Better drag & drop + keep comments in file --- scripts/generate_settings_schema.py | 11 +- scripts/settings/src/App.tsx | 166 +++++++++++++++++++++++----- scripts/settings/src/schema.json | 3 +- 3 files changed, 150 insertions(+), 30 deletions(-) diff --git a/scripts/generate_settings_schema.py b/scripts/generate_settings_schema.py index 92853cd..16cb22f 100644 --- a/scripts/generate_settings_schema.py +++ b/scripts/generate_settings_schema.py @@ -1,9 +1,12 @@ import json import os +import io + +from ruamel.yaml import YAML from auto_archiver.core.module import ModuleFactory from auto_archiver.core.consts import MODULE_TYPES - +from auto_archiver.core.config import EMPTY_CONFIG class SchemaEncoder(json.JSONEncoder): def default(self, obj): @@ -23,6 +26,11 @@ for module in available_modules: all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)) +yaml: YAML = YAML() + +config_string = io.BytesIO() +yaml.dump(EMPTY_CONFIG, config_string) +config_string = config_string.getvalue().decode('utf-8') output_schema = { 'modules': dict((module.name, { @@ -35,6 +43,7 @@ output_schema = { 'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES), 'configs': [m.name for m in all_modules_ordered_by_type if m.configs], 'module_types': MODULE_TYPES, + 'empty_config': config_string } current_file_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/scripts/settings/src/App.tsx b/scripts/settings/src/App.tsx index 923d891..55ff5dd 100644 --- a/scripts/settings/src/App.tsx +++ b/scripts/settings/src/App.tsx @@ -1,9 +1,9 @@ import * as React from 'react'; -import { useEffect, useState } from 'react'; +import { useEffect, useState, useRef } from 'react'; import Container from '@mui/material/Container'; import Typography from '@mui/material/Typography'; import Box from '@mui/material/Box'; - +import FileUploadIcon from '@mui/icons-material/FileUpload'; // import { DndContext, @@ -26,27 +26,28 @@ import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/co import { Module } from './types'; -import { modules, steps, module_types } from './schema.json'; +import { modules, steps, module_types, empty_config } from './schema.json'; import { Stack, Button, } from '@mui/material'; import Grid from '@mui/material/Grid2'; -import { parseDocument, Document } from 'yaml' +import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml' import StepCard from './StepCard'; function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch> }) { const [showError, setShowError] = useState(false); - const [label, setLabel] = useState("Drag and drop your orchestration.yaml file here, or click to select a file."); + const [label, setLabel] = useState(<>Drag and drop your orchestration.yaml file here, or click to select a file.); + const wrapperRef = useRef(null); function openYAMLFile(event: any) { let file = event.target.files[0]; - if (file.type !== 'application/x-yaml') { + if (file.type.indexOf('yaml') === -1) { setShowError(true); - setLabel("Invalid type, only YAML files are accepted.") + setLabel(<>Invalid type, only YAML files are accepted.) return; } let reader = new FileReader(); @@ -57,12 +58,34 @@ function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch 0) { // not a valid yaml file setShowError(true); - setLabel("Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.") + setLabel(<>Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.) return; } else { setShowError(false); - setLabel("File loaded successfully.") + setLabel(<>File loaded successfully.) } + // do some basic validation of 'steps' + let steps = document.get('steps'); + if (!steps) { + setShowError(true); + setLabel(<>Invalid file. Your orchestration file must have a 'steps' section in it.) + return; + } + const replacements = { + feeder: 'feeders', + formatter: 'formatters', + archivers: 'extractors', + }; + + let error = false; + for (let stepType of Object.keys(replacements)) { + if (steps.get(stepType) !== undefined) { + setShowError(true); + setLabel(<>Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '{stepType}' section.
You should manually update your orchestration file first (hint: {stepType} → {replacements[stepType]})); + error = true; + return; + } + }; setYamlFile(document); } catch (e) { console.error(e); @@ -72,10 +95,39 @@ function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch -
- - - +
{ + e.currentTarget.style.backgroundColor = 'var(--mui-palette-LinearProgress-infoBg)'; + }} + onDragLeave={(e) => { + e.currentTarget.style.backgroundColor = ''; + }} + onDrop={(e) => { + e.currentTarget.style.backgroundColor = ''; + }} + > + + + {label}
@@ -216,28 +268,74 @@ export default function App() { // generate the steps config let stepsConfig = enabledModules; - // create a yaml file from - const finalYaml = { - 'steps': Object.keys(steps).reduce((acc, stepType: string) => { - acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name); - return acc; - }, {}) - }; + let finalYamlFile: Document = null; + if (!yamlFile || yamlFile.contents == null) { + // create the yaml file from + finalYamlFile = parseDocument(empty_config as string); + } else { + finalYamlFile = yamlFile; + } - Object.keys(configValues).map((module: string) => { - let module_values = configValues[module]; - if (module_values) { - finalYaml[module] = module_values; - } + // set the steps + module_types.forEach((type: string) => { + let stepType = type + 's'; + let existingSteps = finalYamlFile.getIn(['steps', stepType]) as YAMLSeq; + stepsConfig[stepType].forEach(([name, enabled]: [string, boolean]) => { + let index = existingSteps.items.findIndex((item) => { + return item.value === name + }); + let commentIndex = existingSteps.items.findIndex((item) => { + return item.comment?.indexOf(name) || item.commentBefore?.indexOf() + }); + let stepItem = finalYamlFile.getIn(['steps', stepType], true) as YAMLSeq; + + if (enabled && index === -1) { + finalYamlFile.addIn(['steps', stepType], name); + stepItem.commentBefore = stepItem.commentBefore?.replace("\n - " + name, ''); + stepItem.comment = stepItem.comment?.replace("\n - " + name, ''); + } else if (!enabled && index !== -1) { + // set the value to empty and add a comment before with the commented value + finalYamlFile.deleteIn(['steps', stepType, index]); + stepItem.commentBefore += "\n - " + name; + finalYamlFile.setIn(['steps', stepType], stepItem); + } + }); + existingSteps.flow = existingSteps.items.length ? false : true; }); - let newFile = new Document(finalYaml); + + // set all other settings + // loop through each item that isn't 'steps' in the finalYamlFile and check if it exists in configValues + + Object.keys(configValues).forEach((module_name: string) => { + // get an existing key + let existingConfig = finalYamlFile.get(module_name, true) as YAMLMap; + if (existingConfig) { + Object.keys(configValues[module_name]).forEach((config_name: string) => { + let existingConfigYAML = existingConfig.get(config_name, true) as Scalar; + if (existingConfigYAML) { + console.log(existingConfigYAML.comment); + console.log(existingConfigYAML.commentBefore); + existingConfigYAML.value = configValues[module_name][config_name]; + existingConfig.set(config_name, existingConfigYAML); + } else { + existingConfig.set(config_name, configValues[module_name][config_name]); + } + }); + finalYamlFile.set(module_name, existingConfig); + } else { + if (configValues[module_name] && Object.keys(configValues[module_name]).length > 0) { + finalYamlFile.set(module_name, configValues[module_name]); + } + } + }); + if (copy) { - navigator.clipboard.writeText(String(newFile)).then(() => { + navigator.clipboard.writeText(String(finalYamlFile)).then(() => { alert("Settings copied to clipboard."); }); } else { // offer the file for download - const blob = new Blob([String(newFile)], { type: 'application/x-yaml' }); + const blob = new Blob([String(finalYamlFile)], { type: 'application/x-yaml' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; @@ -274,6 +372,7 @@ export default function App() { let settings = yamlFile.toJS(); // make a deep copy of settings let stepSettings = settings['steps']; + let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => { return [type, steps[type].map((name: string) => { return [name, stepSettings[type].indexOf(name) !== -1]; @@ -295,6 +394,16 @@ export default function App() { return module_types.indexOf(a[0]) - module_types.indexOf(b[0]); })); setEnabledModules(newEnabledModules); + + // set the config values + let newConfigValues = settings; + delete newConfigValues['steps']; + + + setConfigValues(Object.keys(modules).reduce((acc, module) => { + acc[module] = newConfigValues[module] || {}; + return acc; + }, {})); }, [yamlFile]); @@ -306,6 +415,7 @@ export default function App() { 1. Select your orchestration.yaml settings file. + Or skip this step to start from scratch diff --git a/scripts/settings/src/schema.json b/scripts/settings/src/schema.json index 8376c9f..64a903a 100644 --- a/scripts/settings/src/schema.json +++ b/scripts/settings/src/schema.json @@ -2113,5 +2113,6 @@ "database", "storage", "formatter" - ] + ], + "empty_config": "# Auto Archiver Configuration\n\n# Steps are the modules that will be run in the order they are defined\nsteps:\n feeders: []\n extractors: []\n enrichers: []\n databases: []\n storages: []\n formatters: []\n\n# Global configuration\n\n# Authentication\n# a dictionary of authentication information that can be used by extractors to login to website. \n# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)\n# Common login 'types' are username/password, cookie, api key/token.\n# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. \n# Some Examples:\n# facebook.com:\n# username: \"my_username\"\n# password: \"my_password\"\n# or for a site that uses an API key:\n# twitter.com,x.com:\n# api_key\n# api_secret\n# youtube.com:\n# cookie: \"login_cookie=value ; other_cookie=123\" # multiple 'key=value' pairs should be separated by ;\n\nauthentication: {}\n\n# These are the global configurations that are used by the modules\n\nlogging:\n level: INFO\n\n" } \ No newline at end of file