mirror of
https://github.com/bellingcat/datasheet-server.git
synced 2026-06-08 03:18:33 +03:00
abstract generic logic from blueprinters to blueprinters.js
The logic in the files in the 'blueprinters' folder is now _only_ the data transformation logic. Instead of taking in arguments like the sheetId, the tabName, and the sheetName, the function now takes a single argument: the list of lists that represents the raw data from the sheet. This setup gives datasheet-server greater value, as it allows developers to only specify the transformation logic, and not worry about the other stuff that datasheet server is doing.
This commit is contained in:
Lachlan Kermode
@@ -1,37 +0,0 @@
|
||||
import R from 'ramda'
|
||||
import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* byColumn - generate a Blueprint from a data sheet by column. Each column
|
||||
* name is a resheet, and all values in that column are the resheet items.
|
||||
*
|
||||
* @param {type} data - list of lists representing sheet data.
|
||||
* @return {type} Blueprint
|
||||
* generated.
|
||||
*/
|
||||
function columns (tabName, sheetName, sheetId, data) {
|
||||
// Define Blueprint props
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
// column names define resources
|
||||
const labels = data[0]
|
||||
labels.forEach(label => {
|
||||
bp.resources[label] = R.clone(defaultResource)
|
||||
})
|
||||
|
||||
// remaining rows as data
|
||||
data.forEach((row, idx) => {
|
||||
if (idx === 0) return
|
||||
labels.forEach((label, idx) => {
|
||||
bp.resources[label].data.push(row[idx])
|
||||
})
|
||||
})
|
||||
return bp
|
||||
}
|
||||
|
||||
export default columns
|
||||
26
src/blueprinters/deeprows.js
Normal file
26
src/blueprinters/deeprows.js
Normal file
@@ -0,0 +1,26 @@
|
||||
import R from 'ramda'
|
||||
import { fmtObj } from '../lib/util'
|
||||
|
||||
/**
|
||||
* Each resource item is an object with values labelled according
|
||||
* to column names specified in the sheet's first row. If two or more
|
||||
* column names are the same except for a different integer at the end
|
||||
* (e.g. 'tag1', and 'tag2'), then the values of those two columns are
|
||||
* aggregated into a list, which is the value of the prefix's key ('tag').
|
||||
*
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @return {type} Array the structured data.
|
||||
*/
|
||||
export default (data) => {
|
||||
// TODO: make these deep rows.
|
||||
const itemLabels = data[0]
|
||||
const fmt = fmtObj(itemLabels)
|
||||
const output = []
|
||||
|
||||
data.forEach((row, idx) => {
|
||||
if (idx === 0) return
|
||||
output.push(fmt(row))
|
||||
})
|
||||
|
||||
return output
|
||||
}
|
||||
@@ -1,39 +1,17 @@
|
||||
import R from 'ramda'
|
||||
import { fmtObj } from '../lib/util'
|
||||
import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* groups - generate a Blueprint from a data sheet grouped by a column called 'group'
|
||||
* The resource name defaults to 'groups', or a custom resource name can be passed.
|
||||
* Each resource item is an object with values labelled according to column
|
||||
* names. Items are inserted in the data list at idx = id.
|
||||
* names. Items are inserted into the data list at idx = id.
|
||||
*
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @param {type} label="groups" name of resource in blueprint.
|
||||
* @param {type} name="" name of blueprint.
|
||||
* @return {type} Blueprint
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @return {type} Array the structured data.
|
||||
*/
|
||||
export default function groups (
|
||||
tabName,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'groups'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
// Column names define resources
|
||||
export default (data) => {
|
||||
const itemLabels = data[0]
|
||||
const fmt = fmtObj(itemLabels)
|
||||
bp.resources[label] = R.clone(defaultResource)
|
||||
bp.resources[label].data = []
|
||||
|
||||
const output = []
|
||||
const dataGroups = {}
|
||||
|
||||
data.forEach((row, idx) => {
|
||||
@@ -45,12 +23,14 @@ export default function groups (
|
||||
dataGroups[group].push(fmt(row))
|
||||
}
|
||||
})
|
||||
Object.keys(dataGroups).forEach(groupKey => {
|
||||
bp.resources[label].data.push({
|
||||
group: groupKey,
|
||||
group_label: dataGroups[groupKey][0].group_label,
|
||||
data: dataGroups[groupKey]
|
||||
Object.keys(dataGroups)
|
||||
.forEach(groupKey => {
|
||||
output.push({
|
||||
group: groupKey,
|
||||
group_label: dataGroups[groupKey][0].group_label,
|
||||
data: dataGroups[groupKey]
|
||||
})
|
||||
})
|
||||
})
|
||||
return bp
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -1,43 +1,23 @@
|
||||
import R from 'ramda'
|
||||
import { fmtObj } from '../lib/util'
|
||||
import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* ids - generate a Blueprint from a data sheet by id, which is an integer.
|
||||
* The resource name defaults to 'ids', or a custom resource name can be passed.
|
||||
* Each resource item is an object with values labelled according to column
|
||||
* names. Items are inserted in the data list at idx = id.
|
||||
* Very similar to the rows blueprinter, but inserts each row as a value in
|
||||
* an object, where the value in the 'id' column of the row will be used as
|
||||
* the search key
|
||||
*
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @param {type} label="ids" name of resource in blueprint.
|
||||
* @param {type} name="" name of blueprint.
|
||||
* @return {type} Blueprint
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @return {type} Object the structured data.
|
||||
*/
|
||||
export default function ids (
|
||||
tabName,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'ids'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
// Column names define resources
|
||||
export default (data) => {
|
||||
const itemLabels = data[0]
|
||||
const fmt = fmtObj(itemLabels)
|
||||
bp.resources[label] = R.clone(defaultResource)
|
||||
bp.resources[label].data = {}
|
||||
const output = {}
|
||||
|
||||
data.forEach((row, idx) => {
|
||||
if (idx === 0) return
|
||||
bp.resources[label].data[fmt(row).id] = fmt(row)
|
||||
output[fmt(row).id] = fmt(row)
|
||||
})
|
||||
|
||||
return bp
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -1,41 +1,22 @@
|
||||
import R from 'ramda'
|
||||
import { fmtObj } from '../lib/util'
|
||||
import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* rows - generate a Blueprint from a data sheet by row. The resource name
|
||||
* defaults to 'rows', or a custom resource name can be passed. Each resource
|
||||
* item is an object with values labelled according to column names.
|
||||
* Each resource item is an object with values labelled according
|
||||
* to column names specified in the sheet's first row.
|
||||
*
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @param {type} label="rows" name of resource in blueprint.
|
||||
* @param {type} name="" name of blueprint.
|
||||
* @return {type} Blueprint
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @return {type} Array the structured data.
|
||||
*/
|
||||
export default function rows (
|
||||
tabName,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'rows'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
// Column names define resources
|
||||
export default (data) => {
|
||||
const itemLabels = data[0]
|
||||
const fmt = fmtObj(itemLabels)
|
||||
bp.resources[label] = R.clone(defaultResource)
|
||||
bp.resources[label].data = []
|
||||
const output = []
|
||||
|
||||
data.forEach((row, idx) => {
|
||||
if (idx === 0) return
|
||||
bp.resources[label].data.push(fmt(row))
|
||||
output.push(fmt(row))
|
||||
})
|
||||
return bp
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -1,36 +1,13 @@
|
||||
import R from 'ramda'
|
||||
import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
|
||||
import { fmtObj } from '../lib/util'
|
||||
|
||||
/**
|
||||
* tree - generate a Blueprint from a data sheet grouped by a column called 'group'
|
||||
* The resource name defaults to 'groups', or a custom resource name can be passed.
|
||||
* Each resource item is an object with values labelled according to column
|
||||
* names. Items are inserted in the data list at idx = id.
|
||||
* Each resource item is inserted into a tree. TODO: describe layout.
|
||||
*
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @param {type} label="groups" name of resource in blueprint.
|
||||
* @param {type} name="" name of blueprint.
|
||||
* @return {type} Blueprint
|
||||
* @param {type} data list of lists representing sheet data.
|
||||
* @return {type} Array the structured data.
|
||||
*/
|
||||
export default function tree (
|
||||
tabName,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'tree'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
// Column names define resources
|
||||
bp.resources[label] = R.clone(defaultResource)
|
||||
bp.resources[label].data = {}
|
||||
|
||||
export default (data) => {
|
||||
const tree = {
|
||||
key: 'tags',
|
||||
children: {}
|
||||
@@ -62,6 +39,5 @@ export default function tree (
|
||||
}
|
||||
})
|
||||
|
||||
bp.resources[label].data = tree
|
||||
return bp
|
||||
return tree
|
||||
}
|
||||
|
||||
@@ -30,6 +30,8 @@ class Controller {
|
||||
} else {
|
||||
throw new Error(copy.errors.update)
|
||||
}
|
||||
}).catch(err => {
|
||||
console.log(err)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -91,9 +91,9 @@ class Fetcher {
|
||||
*/
|
||||
_saveViaBlueprinter (tab, data, blueprinter) {
|
||||
const saturatedBp = blueprinter(
|
||||
tab,
|
||||
this.sheetName,
|
||||
this.sheetId,
|
||||
this.sheetName,
|
||||
tab,
|
||||
data
|
||||
)
|
||||
|
||||
|
||||
@@ -24,13 +24,28 @@ export function buildDesaturated (sheetId, sheetName, tab, resource) {
|
||||
return bp
|
||||
}
|
||||
|
||||
const buildBlueprinter = R.curry((datafierName, datafier, sheetId, sheetName, tabName, data) => {
|
||||
console.log(`blueprinter ${datafierName} called: ${tabName}`)
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
bp.resources[datafierName] = R.clone(defaultResource)
|
||||
bp.resources[datafierName].data = datafier(data)
|
||||
|
||||
return bp
|
||||
})
|
||||
|
||||
// import all default exports from 'blueprinters' folder
|
||||
const allBps = {}
|
||||
const REL_PATH_TO_BPS = '../blueprinters'
|
||||
const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS)
|
||||
fs.readdirSync(normalizedPath).forEach(file => {
|
||||
const bpName = file.replace('.js', '')
|
||||
allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default
|
||||
const datafier = require(`${REL_PATH_TO_BPS}/${file}`).default
|
||||
allBps[bpName] = buildBlueprinter(bpName, datafier)
|
||||
})
|
||||
|
||||
// NB: revert to ES5 'module.exports' required to make blueprinters from
|
||||
|
||||
Reference in New Issue
Block a user