mirror of
https://github.com/bellingcat/datasheet-server.git
synced 2026-06-08 03:18:33 +03:00
rename source to sheet
This commit is contained in:
@@ -15,10 +15,10 @@ export default ({ config, controller }) => {
|
||||
res.json(controller.blueprints())
|
||||
})
|
||||
|
||||
api.get('/:source/:tab/:resource/:frag', (req, res) => {
|
||||
const { source, tab, resource, frag } = req.params
|
||||
api.get('/:sheet/:tab/:resource/:frag', (req, res) => {
|
||||
const { sheet, tab, resource, frag } = req.params
|
||||
controller
|
||||
.retrieveFrag(source, tab, resource, frag)
|
||||
.retrieveFrag(sheet, tab, resource, frag)
|
||||
.then(data => res.json(data))
|
||||
.catch(err =>
|
||||
res.status(err.status || 404)
|
||||
@@ -26,9 +26,9 @@ export default ({ config, controller }) => {
|
||||
)
|
||||
})
|
||||
|
||||
api.get('/:source/:tab/:resource', (req, res) => {
|
||||
api.get('/:sheet/:tab/:resource', (req, res) => {
|
||||
controller
|
||||
.retrieve(req.params.source, req.params.tab, req.params.resource)
|
||||
.retrieve(req.params.sheet, req.params.tab, req.params.resource)
|
||||
.then(data => res.json(data))
|
||||
.catch(err =>
|
||||
res.status(err.status || 404)
|
||||
@@ -53,12 +53,12 @@ export default ({ config, controller }) => {
|
||||
// ERROR routes. Note that it is important that these come AFTER routes
|
||||
// like /update, so that the regex does not greedily match these routes.
|
||||
|
||||
api.get('/:source', (req, res) => {
|
||||
api.get('/:sheet', (req, res) => {
|
||||
res.status(404)
|
||||
.send({ error: copy.errors.onlySource })
|
||||
.send({ error: copy.errors.onlysheet })
|
||||
})
|
||||
|
||||
api.get('/:source/:tab', (req, res) => {
|
||||
api.get('/:sheet/:tab', (req, res) => {
|
||||
res.status(404)
|
||||
.send({ error: copy.errors.onlyTab })
|
||||
})
|
||||
|
||||
@@ -2,19 +2,19 @@ import R from 'ramda'
|
||||
import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* byColumn - generate a Blueprint from a data source by column. Each column
|
||||
* name is a resource, and all values in that column are the resource items.
|
||||
* byColumn - generate a Blueprint from a data sheet by column. Each column
|
||||
* name is a resheet, and all values in that column are the resheet items.
|
||||
*
|
||||
* @param {type} data - list of lists representing sheet data.
|
||||
* @return {type} Blueprint
|
||||
* generated.
|
||||
*/
|
||||
export default function byColumn (tabName, sourceName, sourceId, data) {
|
||||
export default function byColumn (tabName, sheetName, sheetId, data) {
|
||||
// Define Blueprint props
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.source = {
|
||||
name: sourceName,
|
||||
id: sourceId
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { fmtObj } from '../lib/util'
|
||||
import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* byGroup - generate a Blueprint from a data source grouped by a column called 'group'
|
||||
* byGroup - generate a Blueprint from a data sheet grouped by a column called 'group'
|
||||
* The resource name defaults to 'groups', or a custom resource name can be passed.
|
||||
* Each resource item is an object with values labelled according to column
|
||||
* names. Items are inserted in the data list at idx = id.
|
||||
@@ -15,16 +15,16 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
*/
|
||||
export default function byGroup (
|
||||
tabName,
|
||||
sourceName,
|
||||
sourceId,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'groups'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.source = {
|
||||
name: sourceName,
|
||||
id: sourceId
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { fmtObj } from '../lib/util'
|
||||
import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* byId - generate a Blueprint from a data source by id, which is an integer.
|
||||
* byId - generate a Blueprint from a data sheet by id, which is an integer.
|
||||
* The resource name defaults to 'ids', or a custom resource name can be passed.
|
||||
* Each resource item is an object with values labelled according to column
|
||||
* names. Items are inserted in the data list at idx = id.
|
||||
@@ -15,16 +15,16 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
*/
|
||||
export default function byId (
|
||||
tabName,
|
||||
sourceName,
|
||||
sourceId,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'ids'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.source = {
|
||||
name: sourceName,
|
||||
id: sourceId
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { fmtObj } from '../lib/util'
|
||||
import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* byRow - generate a Blueprint from a data source by row. The resource name
|
||||
* byRow - generate a Blueprint from a data sheet by row. The resource name
|
||||
* defaults to 'rows', or a custom resource name can be passed. Each resource
|
||||
* item is an object with values labelled according to column names.
|
||||
*
|
||||
@@ -14,16 +14,16 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
*/
|
||||
export default function byRow (
|
||||
tabName,
|
||||
sourceName,
|
||||
sourceId,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'rows'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.source = {
|
||||
name: sourceName,
|
||||
id: sourceId
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import R from 'ramda'
|
||||
import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
|
||||
/**
|
||||
* byTree - generate a Blueprint from a data source grouped by a column called 'group'
|
||||
* byTree - generate a Blueprint from a data sheet grouped by a column called 'group'
|
||||
* The resource name defaults to 'groups', or a custom resource name can be passed.
|
||||
* Each resource item is an object with values labelled according to column
|
||||
* names. Items are inserted in the data list at idx = id.
|
||||
@@ -14,16 +14,16 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
|
||||
*/
|
||||
export default function byTree (
|
||||
tabName,
|
||||
sourceName,
|
||||
sourceId,
|
||||
sheetName,
|
||||
sheetId,
|
||||
data,
|
||||
label = 'tree'
|
||||
) {
|
||||
// Define Blueprint
|
||||
const bp = R.clone(defaultBlueprint)
|
||||
bp.source = {
|
||||
name: sourceName,
|
||||
id: sourceId
|
||||
bp.sheet = {
|
||||
name: sheetName,
|
||||
id: sheetId
|
||||
}
|
||||
bp.name = tabName
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
export default {
|
||||
errors: {
|
||||
update: 'The server could not update. Check your API credentials and internet connection and try again.',
|
||||
onlySource: 'You cannot query a source directly. The URL needs to be in the format /:source/:tab/:resource.',
|
||||
onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:source/:tab/:resource.',
|
||||
noSource: source => `The source ${source} is not available in this server.`,
|
||||
noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' of the source '${prts[0]}'.`,
|
||||
onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
||||
onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
||||
noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
|
||||
noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' of the sheet '${prts[0]}'.`,
|
||||
noFragment: prts => `Fragment index does not exist`
|
||||
},
|
||||
success: {
|
||||
update: 'All sources updated'
|
||||
update: 'All sheets updated'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,20 +9,20 @@ class Controller {
|
||||
this.fetchers = fetchers
|
||||
}
|
||||
|
||||
sourceExists (source) {
|
||||
return (Object.keys(this.fetchers).indexOf(source) >= 0)
|
||||
sheetExists (sheet) {
|
||||
return (Object.keys(this.fetchers).indexOf(sheet) >= 0)
|
||||
}
|
||||
|
||||
blueprints () {
|
||||
return Object.keys(this.fetchers).map(
|
||||
source => this.fetchers[source].blueprints
|
||||
sheet => this.fetchers[sheet].blueprints
|
||||
)
|
||||
}
|
||||
|
||||
update () {
|
||||
return Promise.all(
|
||||
Object.keys(this.fetchers).map(source => {
|
||||
return this.fetchers[source].update()
|
||||
Object.keys(this.fetchers).map(sheet => {
|
||||
return this.fetchers[sheet].update()
|
||||
})
|
||||
).then(results => {
|
||||
if (results.every(r => r)) {
|
||||
@@ -33,21 +33,21 @@ class Controller {
|
||||
})
|
||||
}
|
||||
|
||||
retrieve (source, tab, resource) {
|
||||
if (this.sourceExists(source)) {
|
||||
const fetcher = this.fetchers[source]
|
||||
retrieve (sheet, tab, resource) {
|
||||
if (this.sheetExists(sheet)) {
|
||||
const fetcher = this.fetchers[sheet]
|
||||
return fetcher.retrieve(tab, resource)
|
||||
} else {
|
||||
return Promise.reject(new Error(copy.errors.noResource(source)))
|
||||
return Promise.reject(new Error(copy.errors.noResource(sheet)))
|
||||
}
|
||||
}
|
||||
|
||||
retrieveFrag (source, tab, resource, frag) {
|
||||
if (this.sourceExists(source)) {
|
||||
const fetcher = this.fetchers[source]
|
||||
retrieveFrag (sheet, tab, resource, frag) {
|
||||
if (this.sheetExists(sheet)) {
|
||||
const fetcher = this.fetchers[sheet]
|
||||
return fetcher.retrieveFrag(tab, resource, frag)
|
||||
} else {
|
||||
return Promise.reject(new Error(copy.errors.noResource(source)))
|
||||
return Promise.reject(new Error(copy.errors.noResource(sheet)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
|
||||
import { google } from 'googleapis'
|
||||
import {
|
||||
fmtSourceTitle,
|
||||
fmtSheetTitle,
|
||||
fmtBlueprinterTitles,
|
||||
bp,
|
||||
isFunction
|
||||
@@ -10,7 +10,7 @@ import { byRow } from './blueprinters'
|
||||
import R from 'ramda'
|
||||
|
||||
class Fetcher {
|
||||
constructor (db, sourceName, sourceId, blueprinters) {
|
||||
constructor (db, sheetName, sheetId, blueprinters) {
|
||||
/*
|
||||
* The database that the fetcher should use. This should be an instance of a model-compliant class.
|
||||
* See models/Interface.js for the specifications for a model-compliant class.
|
||||
@@ -18,15 +18,15 @@ class Fetcher {
|
||||
this.db = db
|
||||
|
||||
/*
|
||||
* ID of the Google Sheet where the data is sourced. Note that the privateKey.clientEmail
|
||||
* ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
|
||||
* loaded here must be added to the sheet as an editor.
|
||||
*/
|
||||
this.sourceId = sourceId
|
||||
this.sheetId = sheetId
|
||||
|
||||
/*
|
||||
* The name of the source. This will prefix tabs saved in the database.
|
||||
* The name of the sheet. This will prefix tabs saved in the database.
|
||||
*/
|
||||
this.sourceName = sourceName
|
||||
this.sheetName = sheetName
|
||||
|
||||
/*
|
||||
* These are the available tabs for storing and retrieving data.
|
||||
@@ -53,8 +53,8 @@ class Fetcher {
|
||||
this._saveBp = R.curry((tab, title, data, blueprinter) => {
|
||||
const saturatedBp = blueprinter(
|
||||
tab,
|
||||
this.sourceName,
|
||||
this.sourceId,
|
||||
this.sheetName,
|
||||
this.sheetId,
|
||||
data
|
||||
)
|
||||
const blueprint = bp(saturatedBp) // TODO: come up with better semantics.
|
||||
@@ -69,14 +69,14 @@ class Fetcher {
|
||||
'https://www.googleapis.com/auth/spreadsheets'
|
||||
])
|
||||
this.auth = googleAuth
|
||||
const { sourceId } = this
|
||||
const { sheetId } = this
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
googleAuth.authorize(function (err) {
|
||||
if (err) {
|
||||
reject(err)
|
||||
} else {
|
||||
resolve(`Connected to ${sourceId}.`)
|
||||
resolve(`Connected to ${sheetId}.`)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -88,13 +88,13 @@ class Fetcher {
|
||||
return this.sheets.spreadsheets
|
||||
.get({
|
||||
auth: this.auth,
|
||||
spreadsheetId: this.sourceId
|
||||
spreadsheetId: this.sheetId
|
||||
})
|
||||
.then(response => {
|
||||
tabTitles = response.data.sheets.map(sheet => sheet.properties.title)
|
||||
return this.sheets.spreadsheets.values.batchGet({
|
||||
auth: this.auth,
|
||||
spreadsheetId: this.sourceId,
|
||||
spreadsheetId: this.sheetId,
|
||||
ranges: tabTitles
|
||||
})
|
||||
})
|
||||
@@ -116,7 +116,7 @@ class Fetcher {
|
||||
}
|
||||
|
||||
save (tab, data) {
|
||||
const title = fmtSourceTitle(tab)
|
||||
const title = fmtSheetTitle(tab)
|
||||
if (Object.keys(this.blueprinters).indexOf(title) > -1) {
|
||||
const bpConfig = this.blueprinters[title]
|
||||
|
||||
@@ -127,20 +127,20 @@ class Fetcher {
|
||||
}
|
||||
} else {
|
||||
// If it can't find a blueprinter for the tab title, default to byRow
|
||||
return this.db.save(byRow(tab, this.sourceName, this.sourceId, data))
|
||||
return this.db.save(byRow(tab, this.sheetName, this.sheetId, data))
|
||||
}
|
||||
}
|
||||
|
||||
// NB: could combine these functions by checking kwargs length
|
||||
retrieve (tab, resource) {
|
||||
const title = fmtSourceTitle(tab)
|
||||
const url = `${this.sourceName}/${tab}/${resource}`
|
||||
const title = fmtSheetTitle(tab)
|
||||
const url = `${this.sheetName}/${tab}/${resource}`
|
||||
return this.db.load(url, this.blueprints[title])
|
||||
}
|
||||
|
||||
retrieveFrag (tab, resource, frag) {
|
||||
const title = fmtSourceTitle(tab)
|
||||
const url = `${this.sourceName}/${tab}/${resource}/${frag}`
|
||||
const title = fmtSheetTitle(tab)
|
||||
const url = `${this.sheetName}/${tab}/${resource}/${frag}`
|
||||
return this.db.load(url, this.blueprints[title])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,27 +55,27 @@ export const idxSearcher = R.curry((attrName, searchValue, myArray) => {
|
||||
|
||||
/* more site specific functions. TODO: maybe move to another folder? */
|
||||
|
||||
export function fmtSourceTitle (name) {
|
||||
export function fmtSheetTitle (name) {
|
||||
return name.replaceAll(' ', '-').toLowerCase()
|
||||
}
|
||||
|
||||
export function fmtBlueprinterTitles (tabs) {
|
||||
const obj = {}
|
||||
Object.keys(tabs).forEach(tab => {
|
||||
const name = fmtSourceTitle(tab)
|
||||
const name = fmtSheetTitle(tab)
|
||||
obj[name] = tabs[tab]
|
||||
})
|
||||
return obj
|
||||
}
|
||||
|
||||
export function deriveFilename (source, tab) {
|
||||
return `${fmtSourceTitle(source)}-${fmtSourceTitle(tab)}.json`
|
||||
export function deriveFilename (sheet, tab) {
|
||||
return `${fmtSheetTitle(sheet)}-${fmtSheetTitle(tab)}.json`
|
||||
}
|
||||
|
||||
export function bp (full) {
|
||||
const blueprint = {
|
||||
name: R.clone(full.name),
|
||||
source: R.clone(full.source),
|
||||
sheet: R.clone(full.sheet),
|
||||
dialects: R.clone(full.dialects),
|
||||
routes: {}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import fs from 'mz/fs'
|
||||
import { fmtSourceTitle } from '../lib/util'
|
||||
import { fmtSheetTitle } from '../lib/util'
|
||||
import copy from '../copy/en'
|
||||
|
||||
const STORAGE_DIRNAME = 'temp'
|
||||
@@ -9,9 +9,9 @@ class StoreJson {
|
||||
return Promise.all(
|
||||
Object.keys(bp.routes).map(route =>
|
||||
fs.writeFile(
|
||||
`${STORAGE_DIRNAME}/${fmtSourceTitle(
|
||||
bp.source.name
|
||||
)}__${fmtSourceTitle(bp.name)}__${route}.json`,
|
||||
`${STORAGE_DIRNAME}/${fmtSheetTitle(
|
||||
bp.sheet.name
|
||||
)}__${fmtSheetTitle(bp.name)}__${route}.json`,
|
||||
JSON.stringify(bp.routes[route].data)
|
||||
)
|
||||
)
|
||||
@@ -43,7 +43,6 @@ class StoreJson {
|
||||
// Do a lookup if fragment is included to filter a relevant item
|
||||
const index = parseInt(parts[3])
|
||||
if (!isNaN(index) && index >= 0 && index < data.length) {
|
||||
console.log(data, index)
|
||||
return data.filter((vl, idx) => idx === index)[0]
|
||||
} else {
|
||||
throw new Error(copy.errors.noFragment(parts))
|
||||
@@ -55,7 +54,7 @@ class StoreJson {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: add method to build blueprint from data source
|
||||
// TODO: add method to build blueprint from data sheet
|
||||
}
|
||||
|
||||
export default StoreJson
|
||||
|
||||
Reference in New Issue
Block a user