mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 05:08:28 +03:00
Remove unused files, set up for RTD
This commit is contained in:
@@ -27,7 +27,7 @@ import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/co
|
||||
import { Module } from './types';
|
||||
|
||||
import { modules, steps, module_types } from './schema.json';
|
||||
import {
|
||||
import {
|
||||
Stack,
|
||||
Button,
|
||||
} from '@mui/material';
|
||||
@@ -50,7 +50,7 @@ function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateA
|
||||
return;
|
||||
}
|
||||
let reader = new FileReader();
|
||||
reader.onload = function(e) {
|
||||
reader.onload = function (e) {
|
||||
let contents = e.target ? e.target.result : '';
|
||||
try {
|
||||
let document = parseDocument(contents as string);
|
||||
@@ -72,13 +72,13 @@ function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateA
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<div style={{width:'100%', border:'dashed', textAlign:'center', borderWidth:'1px', padding:'20px'}}>
|
||||
<div style={{ width: '100%', border: 'dashed', textAlign: 'center', borderWidth: '1px', padding: '20px' }}>
|
||||
|
||||
<input name="file" type="file" accept=".yaml" onChange={openYAMLFile} />
|
||||
<Typography style={{marginTop:'20px' }} variant="body1" color={showError ? 'error' : ''} >
|
||||
{label}
|
||||
</Typography>
|
||||
</div>
|
||||
<input name="file" type="file" accept=".yaml" onChange={openYAMLFile} />
|
||||
<Typography style={{ marginTop: '20px' }} variant="body1" color={showError ? 'error' : ''} >
|
||||
{label}
|
||||
</Typography>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -91,13 +91,13 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
|
||||
useEffect(() => {
|
||||
setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
|
||||
}
|
||||
, [enabledModules]);
|
||||
, [enabledModules]);
|
||||
|
||||
const toggleModule = (event: any) => {
|
||||
// make sure that 'feeder' and 'formatter' types only have one value
|
||||
let name = event.target.id;
|
||||
let checked = event.target.checked;
|
||||
if (stepType === 'feeder' || stepType === 'formatter') {
|
||||
if (stepType === 'feeders' || stepType === 'formatters') {
|
||||
// check how many modules of this type are enabled
|
||||
const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => {
|
||||
return (m !== name && enabled) || (checked && m === name)
|
||||
@@ -110,11 +110,11 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
|
||||
} else {
|
||||
setShowError(false);
|
||||
}
|
||||
let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type : string) => {
|
||||
let newEnabledModules = Object.fromEntries(Object.keys(enabledModules).map((type: string) => {
|
||||
return [type, enabledModules[type].map(([m, enabled]: [string, boolean]) => {
|
||||
return (m === name) ? [m, checked] : [m, enabled];
|
||||
})];
|
||||
}
|
||||
}
|
||||
));
|
||||
setEnabledModules(newEnabledModules);
|
||||
}
|
||||
@@ -135,61 +135,62 @@ function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues
|
||||
const { active, over } = event;
|
||||
|
||||
if (active.id !== over?.id) {
|
||||
const oldIndex = items.indexOf(active.id as string);
|
||||
const newIndex = items.indexOf(over?.id as string);
|
||||
const oldIndex = items.indexOf(active.id as string);
|
||||
const newIndex = items.indexOf(over?.id as string);
|
||||
|
||||
let newArray = arrayMove(items, oldIndex, newIndex);
|
||||
// set it also on steps
|
||||
let newEnabledModules = { ...enabledModules };
|
||||
newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => {
|
||||
return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
|
||||
})
|
||||
setEnabledModules(newEnabledModules);
|
||||
let newArray = arrayMove(items, oldIndex, newIndex);
|
||||
// set it also on steps
|
||||
let newEnabledModules = { ...enabledModules };
|
||||
newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => {
|
||||
return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
|
||||
})
|
||||
setEnabledModules(newEnabledModules);
|
||||
}
|
||||
};
|
||||
return (
|
||||
<>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
|
||||
{stepType}
|
||||
</Typography>
|
||||
<Typography variant="body1" >
|
||||
Select the {stepType} you wish to enable. You can drag and drop them to reorder them.
|
||||
</Typography>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
|
||||
{stepType}
|
||||
</Typography>
|
||||
<Typography variant="body1" >
|
||||
Select the {stepType} you wish to enable. You can drag and move to reorder.
|
||||
Learn more about {stepType} <a href={`https://auto-archiver.readthedocs.io/en/latest/modules/${stepType.slice(0,-1)}.html`} target="_blank">here</a>.
|
||||
</Typography>
|
||||
</Box>
|
||||
{showError ? <Typography variant="body1" color="error" >Only one {stepType} can be enabled at a time.</Typography> : null}
|
||||
{showError ? <Typography variant="body1" color="error" >Only one {stepType.slice(0,-1)} can be enabled at a time.</Typography> : null}
|
||||
|
||||
<DndContext
|
||||
sensors={sensors}
|
||||
collisionDetection={closestCenter}
|
||||
onDragEnd={handleDragEnd}
|
||||
onDragStart={handleDragStart}
|
||||
>
|
||||
<Grid container spacing={1} key={stepType}>
|
||||
<SortableContext items={items} strategy={rectSortingStrategy}>
|
||||
{items.map((name: string) => {
|
||||
let m: Module = modules[name];
|
||||
return (
|
||||
<StepCard key={name} type={stepType} module={m} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
|
||||
);
|
||||
})}
|
||||
<DragOverlay>
|
||||
{activeId ? (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
backgroundColor: "grey",
|
||||
opacity:0.1,
|
||||
}}
|
||||
></div>
|
||||
sensors={sensors}
|
||||
collisionDetection={closestCenter}
|
||||
onDragEnd={handleDragEnd}
|
||||
onDragStart={handleDragStart}
|
||||
>
|
||||
<Grid container spacing={1} key={stepType}>
|
||||
<SortableContext items={items} strategy={rectSortingStrategy}>
|
||||
{items.map((name: string) => {
|
||||
let m: Module = modules[name];
|
||||
return (
|
||||
<StepCard key={name} type={stepType} module={m} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
|
||||
);
|
||||
})}
|
||||
<DragOverlay>
|
||||
{activeId ? (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
backgroundColor: "grey",
|
||||
opacity: 0.1,
|
||||
}}
|
||||
></div>
|
||||
|
||||
) : null}
|
||||
</DragOverlay>
|
||||
</SortableContext>
|
||||
</Grid>
|
||||
) : null}
|
||||
</DragOverlay>
|
||||
</SortableContext>
|
||||
</Grid>
|
||||
</DndContext>
|
||||
</>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -209,41 +210,41 @@ export default function App() {
|
||||
}, {})
|
||||
);
|
||||
|
||||
const saveSettings = function(copy: boolean = false) {
|
||||
const saveSettings = function (copy: boolean = false) {
|
||||
// edit the yamlFile
|
||||
|
||||
// generate the steps config
|
||||
let stepsConfig = enabledModules;
|
||||
|
||||
// create a yaml file from
|
||||
const finalYaml = {
|
||||
'steps': Object.keys(steps).reduce((acc, stepType: string) => {
|
||||
acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name);
|
||||
return acc;
|
||||
}, {})
|
||||
};
|
||||
// create a yaml file from
|
||||
const finalYaml = {
|
||||
'steps': Object.keys(steps).reduce((acc, stepType: string) => {
|
||||
acc[stepType] = stepsConfig[stepType].filter(([name, enabled]: [string, boolean]) => enabled).map(([name, enabled]: [string, boolean]) => name);
|
||||
return acc;
|
||||
}, {})
|
||||
};
|
||||
|
||||
Object.keys(configValues).map((module: string) => {
|
||||
let module_values = configValues[module];
|
||||
if (module_values) {
|
||||
finalYaml[module] = module_values;
|
||||
}
|
||||
});
|
||||
let newFile = new Document(finalYaml);
|
||||
if (copy) {
|
||||
navigator.clipboard.writeText(String(newFile)).then(() => {
|
||||
alert("Settings copied to clipboard.");
|
||||
});
|
||||
} else {
|
||||
// offer the file for download
|
||||
const blob = new Blob([String(newFile)], { type: 'application/x-yaml' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'orchestration.yaml';
|
||||
a.click();
|
||||
Object.keys(configValues).map((module: string) => {
|
||||
let module_values = configValues[module];
|
||||
if (module_values) {
|
||||
finalYaml[module] = module_values;
|
||||
}
|
||||
});
|
||||
let newFile = new Document(finalYaml);
|
||||
if (copy) {
|
||||
navigator.clipboard.writeText(String(newFile)).then(() => {
|
||||
alert("Settings copied to clipboard.");
|
||||
});
|
||||
} else {
|
||||
// offer the file for download
|
||||
const blob = new Blob([String(newFile)], { type: 'application/x-yaml' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'orchestration.yaml';
|
||||
a.click();
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
// load the configs, and set the default values if they exist
|
||||
@@ -262,7 +263,7 @@ export default function App() {
|
||||
}
|
||||
});
|
||||
})
|
||||
setConfigValues(newConfigValues);
|
||||
setConfigValues(newConfigValues);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -274,25 +275,25 @@ export default function App() {
|
||||
// make a deep copy of settings
|
||||
let stepSettings = settings['steps'];
|
||||
let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => {
|
||||
return [type, steps[type].map((name: string) => {
|
||||
return [name, stepSettings[type].indexOf(name) !== -1];
|
||||
}).sort((a, b) => {
|
||||
let aIndex = stepSettings[type].indexOf(a[0]);
|
||||
let bIndex = stepSettings[type].indexOf(b[0]);
|
||||
if (aIndex === -1 && bIndex === -1) {
|
||||
return a - b;
|
||||
}
|
||||
if (bIndex === -1) {
|
||||
return -1;
|
||||
}
|
||||
if (aIndex === -1) {
|
||||
return 1;
|
||||
}
|
||||
return aIndex - bIndex;
|
||||
})];
|
||||
return [type, steps[type].map((name: string) => {
|
||||
return [name, stepSettings[type].indexOf(name) !== -1];
|
||||
}).sort((a, b) => {
|
||||
return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
|
||||
}));
|
||||
let aIndex = stepSettings[type].indexOf(a[0]);
|
||||
let bIndex = stepSettings[type].indexOf(b[0]);
|
||||
if (aIndex === -1 && bIndex === -1) {
|
||||
return a - b;
|
||||
}
|
||||
if (bIndex === -1) {
|
||||
return -1;
|
||||
}
|
||||
if (aIndex === -1) {
|
||||
return 1;
|
||||
}
|
||||
return aIndex - bIndex;
|
||||
})];
|
||||
}).sort((a, b) => {
|
||||
return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
|
||||
}));
|
||||
setEnabledModules(newEnabledModules);
|
||||
}, [yamlFile]);
|
||||
|
||||
@@ -301,44 +302,41 @@ export default function App() {
|
||||
return (
|
||||
<Container maxWidth="lg">
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h2" >
|
||||
Auto Archiver Settings
|
||||
</Typography>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
1. Select your <pre style={{display:'inline'}}>orchestration.yaml</pre> settings file.
|
||||
</Typography>
|
||||
<FileDrop setYamlFile={setYamlFile}/>
|
||||
<Typography variant="h5" >
|
||||
1. Select your orchestration.yaml settings file.
|
||||
</Typography>
|
||||
<FileDrop setYamlFile={setYamlFile} />
|
||||
</Box>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
2. Choose the Modules you wish to enable/disable
|
||||
</Typography>
|
||||
<Typography variant="h5" >
|
||||
2. Choose the Modules you wish to enable/disable
|
||||
</Typography>
|
||||
{Object.keys(steps).map((stepType: string) => {
|
||||
return (
|
||||
<Box key={stepType} sx={{ my: 4 }}>
|
||||
<ModuleTypes stepType={stepType} setEnabledModules={setEnabledModules} enabledModules={enabledModules} configValues={configValues} />
|
||||
<ModuleTypes stepType={stepType} setEnabledModules={setEnabledModules} enabledModules={enabledModules} configValues={configValues} />
|
||||
</Box>
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
3. Configure your Enabled Modules
|
||||
</Typography>
|
||||
<Typography variant="body1" >
|
||||
Next to each module you've enabled, you can click 'Configure' to set the module's settings.
|
||||
</Typography>
|
||||
<Typography variant="h5" >
|
||||
3. Configure your Enabled Modules
|
||||
</Typography>
|
||||
<Typography variant="body1" >
|
||||
Next to each module you've enabled, you can click 'Configure' to set the module's settings.
|
||||
</Typography>
|
||||
</Box>
|
||||
<Box sx={{ my: 4 }}>
|
||||
<Typography variant="h5" >
|
||||
4. Save your settings
|
||||
</Typography>
|
||||
<Stack direction="row" spacing={2} sx={{ my: 2 }}>
|
||||
<Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
|
||||
<Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
|
||||
<Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
|
||||
<Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
|
||||
</Stack>
|
||||
</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
</Container>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user