bellingcat/datasheet-server
1
0
Fork 0
mirror of https://github.com/bellingcat/datasheet-server.git synced 2026-06-08 03:18:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

change linter to standard

Browse Source
This commit is contained in:
Unknown
2018-11-12 10:28:59 +00:00
parent 19f3edf55a
commit 04d80c7c2f
19 changed files with 837 additions and 691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
package.json
View File

@@ -7,25 +7,10 @@
"dev": "nodemon -w src --exec \"babel-node src\"", "dev": "nodemon -w src --exec \"babel-node src\"",
"build": "npx babel src -d dist", "build": "npx babel src -d dist",
"start": "node dist", "start": "node dist",
"lint": "eslint src", "lint": "standard \"src/**/*.js\"",
"test-watch": "ava --watch", "test-watch": "ava --watch",
"test": "ava --verbose" "test": "ava --verbose"
}, },
"eslintConfig": {
"extends": "eslint:recommended",
"parserOptions": {
"ecmaVersion": 7,
"sourceType": "module"
},
"env": {
"node": true,
"es6": true
},
"rules": {
"no-console": 0,
"no-unused-vars": 1
}
},
"repository": { "repository": {
"type": "git", "type": "git",
"url": "git+https://github.com/developit/express-es6-rest-api.git" "url": "git+https://github.com/developit/express-es6-rest-api.git"
@@ -54,8 +39,8 @@
"@babel/preset-env": "^7.1.0", "@babel/preset-env": "^7.1.0",
"@babel/register": "^7.0.0", "@babel/register": "^7.0.0",
"ava": "1.0.0-beta.8", "ava": "1.0.0-beta.8",
"eslint": "^3.1.1", "nodemon": "^1.9.2",
"nodemon": "^1.9.2" "standard": "^12.0.1"
}, },
"babel": { "babel": {
"presets": [ "presets": [

45
src/api/index.js
View File

@@ -1,22 +1,21 @@
import {version} from "../../package.json"; import { version } from '../../package.json'
import {Router} from "express"; import { Router } from 'express'
import {idxSearcher} from "../lib/util";
export default ({config, controller}) => { export default ({ config, controller }) => {
let api = Router(); let api = Router()
api.get("/", (req, res) => { api.get('/', (req, res) => {
res.json({ res.json({
version version
}); })
}); })
api.get("/blueprints", (req, res) => { api.get('/blueprints', (req, res) => {
res.json(controller.blueprints()); res.json(controller.blueprints())
}); })
api.get("/:source/:tab/:resource/:frag", (req, res) => { api.get('/:source/:tab/:resource/:frag', (req, res) => {
const {source, tab, resource, frag} = req.params; const { source, tab, resource, frag } = req.params
controller controller
.retrieveFrag(source, tab, resource, frag) .retrieveFrag(source, tab, resource, frag)
.then(data => res.json(data)) .then(data => res.json(data))
@@ -24,10 +23,10 @@ export default ({config, controller}) => {
res.json({ res.json({
error: err.message error: err.message
}) })
); )
}); })
api.get("/:source/:tab/:resource", (req, res) => { api.get('/:source/:tab/:resource', (req, res) => {
controller controller
.retrieve(req.params.source, req.params.tab, req.params.resource) .retrieve(req.params.source, req.params.tab, req.params.resource)
.then(data => res.json(data)) .then(data => res.json(data))
@@ -35,10 +34,10 @@ export default ({config, controller}) => {
res.json({ res.json({
error: err.message error: err.message
}) })
); )
}); })
api.get("/update", (req, res) => { api.get('/update', (req, res) => {
controller controller
.update() .update()
.then(msg => .then(msg =>
@@ -50,8 +49,8 @@ export default ({config, controller}) => {
res.json({ res.json({
error: err.message error: err.message
}) })
); )
}); })
return api; return api
}; }

28
src/blueprinters/byColumn.js
View File

@@ -1,5 +1,5 @@
import R from "ramda"; import R from 'ramda'
import {defaultBlueprint, defaultRoute} from "../lib/blueprinters"; import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
/** /**
* byColumn - generate a Blueprint from a data source by column. Each column * byColumn - generate a Blueprint from a data source by column. Each column
@@ -9,27 +9,27 @@ import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
* @return {type} Blueprint * @return {type} Blueprint
* generated. * generated.
*/ */
export default function byColumn(tabName, sourceName, sourceId, data) { export default function byColumn (tabName, sourceName, sourceId, data) {
// Define Blueprint props // Define Blueprint props
const bp = R.clone(defaultBlueprint); const bp = R.clone(defaultBlueprint)
bp.source = { bp.source = {
name: sourceName, name: sourceName,
id: sourceId id: sourceId
}; }
bp.name = tabName; bp.name = tabName
// column names define routes // column names define routes
const labels = data[0]; const labels = data[0]
labels.forEach(label => { labels.forEach(label => {
bp.routes[label] = R.clone(defaultRoute); bp.routes[label] = R.clone(defaultRoute)
}); })
// remaining rows as data // remaining rows as data
data.forEach((row, idx) => { data.forEach((row, idx) => {
if (idx == 0) return; if (idx === 0) return
labels.forEach((label, idx) => { labels.forEach((label, idx) => {
bp.routes[label].data.push(row[idx]); bp.routes[label].data.push(row[idx])
}); })
}); })
return bp; return bp
} }

42
src/blueprinters/byGroup.js
View File

@@ -1,6 +1,6 @@
import R from "ramda"; import R from 'ramda'
import {fmtObj, idxSearcher} from "../lib/util"; import { fmtObj } from '../lib/util'
import {defaultBlueprint, defaultRoute} from "../lib/blueprinters"; import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
/** /**
* byGroup - generate a Blueprint from a data source grouped by a column called 'group' * byGroup - generate a Blueprint from a data source grouped by a column called 'group'
@@ -13,44 +13,44 @@ import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
* @param {type} name="" name of blueprint. * @param {type} name="" name of blueprint.
* @return {type} Blueprint * @return {type} Blueprint
*/ */
export default function byGroup( export default function byGroup (
tabName, tabName,
sourceName, sourceName,
sourceId, sourceId,
data, data,
label = "groups" label = 'groups'
) { ) {
// Define Blueprint // Define Blueprint
const bp = R.clone(defaultBlueprint); const bp = R.clone(defaultBlueprint)
bp.source = { bp.source = {
name: sourceName, name: sourceName,
id: sourceId id: sourceId
}; }
bp.name = tabName; bp.name = tabName
// Column names define routes // Column names define routes
const itemLabels = data[0]; const itemLabels = data[0]
const fmt = fmtObj(itemLabels); const fmt = fmtObj(itemLabels)
bp.routes[label] = R.clone(defaultRoute); bp.routes[label] = R.clone(defaultRoute)
bp.routes[label].data = []; bp.routes[label].data = []
const dataGroups = {}; const dataGroups = {}
data.forEach((row, idx) => { data.forEach((row, idx) => {
if (idx == 0) return; if (idx === 0) return
const group = fmt(row).group; const group = fmt(row).group
if (!dataGroups[group]) { if (!dataGroups[group]) {
dataGroups[group] = [fmt(row)]; dataGroups[group] = [fmt(row)]
} else { } else {
dataGroups[group].push(fmt(row)); dataGroups[group].push(fmt(row))
} }
}); })
Object.keys(dataGroups).forEach(groupKey => { Object.keys(dataGroups).forEach(groupKey => {
bp.routes[label].data.push({ bp.routes[label].data.push({
group: groupKey, group: groupKey,
group_label: dataGroups[groupKey][0].group_label, group_label: dataGroups[groupKey][0].group_label,
data: dataGroups[groupKey] data: dataGroups[groupKey]
}); })
}); })
return bp; return bp
} }

32
src/blueprinters/byId.js
View File

@@ -1,6 +1,6 @@
import R from "ramda"; import R from 'ramda'
import {fmtObj, idxSearcher} from "../lib/util"; import { fmtObj } from '../lib/util'
import {defaultBlueprint, defaultRoute} from "../lib/blueprinters"; import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
/** /**
* byId - generate a Blueprint from a data source by id, which is an integer. * byId - generate a Blueprint from a data source by id, which is an integer.
@@ -13,30 +13,30 @@ import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
* @param {type} name="" name of blueprint. * @param {type} name="" name of blueprint.
* @return {type} Blueprint * @return {type} Blueprint
*/ */
export default function byId( export default function byId (
tabName, tabName,
sourceName, sourceName,
sourceId, sourceId,
data, data,
label = "ids" label = 'ids'
) { ) {
// Define Blueprint // Define Blueprint
const bp = R.clone(defaultBlueprint); const bp = R.clone(defaultBlueprint)
bp.source = { bp.source = {
name: sourceName, name: sourceName,
id: sourceId id: sourceId
}; }
bp.name = tabName; bp.name = tabName
// Column names define routes // Column names define routes
const itemLabels = data[0]; const itemLabels = data[0]
const fmt = fmtObj(itemLabels); const fmt = fmtObj(itemLabels)
bp.routes[label] = R.clone(defaultRoute); bp.routes[label] = R.clone(defaultRoute)
bp.routes[label].data = []; bp.routes[label].data = []
data.forEach((row, idx) => { data.forEach((row, idx) => {
if (idx == 0) return; if (idx === 0) return
bp.routes[label].data[fmt(row).id] = fmt(row); bp.routes[label].data[fmt(row).id] = fmt(row)
}); })
return bp; return bp
} }

32
src/blueprinters/byRow.js
View File

@@ -1,6 +1,6 @@
import R from "ramda"; import R from 'ramda'
import {fmtObj, idxSearcher} from "../lib/util"; import { fmtObj } from '../lib/util'
import {defaultBlueprint, defaultRoute} from "../lib/blueprinters"; import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
/** /**
* byRow - generate a Blueprint from a data source by row. The resource name * byRow - generate a Blueprint from a data source by row. The resource name
@@ -12,30 +12,30 @@ import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
* @param {type} name="" name of blueprint. * @param {type} name="" name of blueprint.
* @return {type} Blueprint * @return {type} Blueprint
*/ */
export default function byRow( export default function byRow (
tabName, tabName,
sourceName, sourceName,
sourceId, sourceId,
data, data,
label = "rows" label = 'rows'
) { ) {
// Define Blueprint // Define Blueprint
const bp = R.clone(defaultBlueprint); const bp = R.clone(defaultBlueprint)
bp.source = { bp.source = {
name: sourceName, name: sourceName,
id: sourceId id: sourceId
}; }
bp.name = tabName; bp.name = tabName
// Column names define routes // Column names define routes
const itemLabels = data[0]; const itemLabels = data[0]
const fmt = fmtObj(itemLabels); const fmt = fmtObj(itemLabels)
bp.routes[label] = R.clone(defaultRoute); bp.routes[label] = R.clone(defaultRoute)
bp.routes[label].data = []; bp.routes[label].data = []
data.forEach((row, idx) => { data.forEach((row, idx) => {
if (idx == 0) return; if (idx === 0) return
bp.routes[label].data.push(fmt(row)); bp.routes[label].data.push(fmt(row))
}); })
return bp; return bp
} }

48
src/blueprinters/byTree.js
View File

@@ -1,6 +1,5 @@
import R from "ramda"; import R from 'ramda'
import {fmtObj, idxSearcher} from "../lib/util"; import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
/** /**
* byTree - generate a Blueprint from a data source grouped by a column called 'group' * byTree - generate a Blueprint from a data source grouped by a column called 'group'
@@ -13,55 +12,56 @@ import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
* @param {type} name="" name of blueprint. * @param {type} name="" name of blueprint.
* @return {type} Blueprint * @return {type} Blueprint
*/ */
export default function byTree( export default function byTree (
tabName, tabName,
sourceName, sourceName,
sourceId, sourceId,
data, data,
label = "tree" label = 'tree'
) { ) {
// Define Blueprint // Define Blueprint
const bp = R.clone(defaultBlueprint); const bp = R.clone(defaultBlueprint)
bp.source = { bp.source = {
name: sourceName, name: sourceName,
id: sourceId id: sourceId
}; }
bp.name = tabName; bp.name = tabName
// Column names define routes // Column names define routes
bp.routes[label] = R.clone(defaultRoute); bp.routes[label] = R.clone(defaultRoute)
bp.routes[label].data = {}; bp.routes[label].data = {}
const tree = { const tree = {
key: "tags", key: 'tags',
children: {} children: {}
}; }
data.forEach(path => { data.forEach(path => {
const root = path[0]; const root = path[0]
if (!tree.children[root]) if (!tree.children[root]) {
tree.children[root] = { tree.children[root] = {
key: root, key: root,
children: {} children: {}
}; }
}
let depth = 1; let depth = 1
let parentNode = tree.children[root]; let parentNode = tree.children[root]
while (depth < path.length) { while (depth < path.length) {
const node = path[depth]; const node = path[depth]
if (!parentNode.children[node]) { if (!parentNode.children[node]) {
parentNode.children[node] = { parentNode.children[node] = {
key: node, key: node,
children: {} children: {}
}; }
} }
parentNode = parentNode.children[node]; parentNode = parentNode.children[node]
depth++; depth++
} }
}); })
bp.routes[label].data = tree; bp.routes[label].data = tree
return bp; return bp
} }

12
src/example.config.js
View File

@@ -1,18 +1,18 @@
import BP from "./lib/blueprinters"; import BP from './lib/blueprinters'
export default { export default {
port: 4040, port: 4040,
googleSheets: { googleSheets: {
email: "project-name@reliable-baptist-23338.iam.gserviceaccount.com", email: 'project-name@reliable-baptist-23338.iam.gserviceaccount.com',
privateKey: "SOME_PRIVATE_KEY", privateKey: 'SOME_PRIVATE_KEY',
sheets: [ sheets: [
{ {
name: "example", name: 'example',
id: "1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI", id: '1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI',
tabs: { tabs: {
objects: [BP.byRow] objects: [BP.byRow]
} }
} }
] ]
} }
}; }

30
src/index.js
View File

@@ -1,12 +1,12 @@
import http from "http"; import http from 'http'
import express from "express"; import express from 'express'
import initialize from "./initialize"; import initialize from './initialize'
import middleware from "./middleware"; import middleware from './middleware'
import api from "./api"; import api from './api'
import config from "./config"; import config from './config'
let app = express(); let app = express()
app.server = http.createServer(app); app.server = http.createServer(app)
initialize(controller => { initialize(controller => {
app.use( app.use(
@@ -14,18 +14,18 @@ initialize(controller => {
config, config,
controller controller
}) })
); )
app.use( app.use(
"/api", '/api',
api({ api({
config, config,
controller controller
}) })
); )
app.server.listen(process.env.PORT || config.port, () => { app.server.listen(process.env.PORT || config.port, () => {
console.log(`Started on port ${app.server.address().port}`); console.log(`Started on port ${app.server.address().port}`)
}); })
}); })
export default app; export default app

48
src/initialize.js
View File

@@ -1,16 +1,16 @@
import StoreJson from "./models/StoreJson"; import StoreJson from './models/StoreJson'
import Fetcher from "./lib/Fetcher"; import Fetcher from './lib/Fetcher'
import Controller from "./lib/Controller"; import Controller from './lib/Controller'
import config from "./config"; import config from './config'
const {googleSheets} = config; const { googleSheets } = config
const {sheets, privateKey, email} = googleSheets; const { sheets, privateKey, email } = googleSheets
function authenticate(_fetcher) { function authenticate (_fetcher) {
return _fetcher.fetcher.authenticate(email, privateKey).then(msg => { return _fetcher.fetcher.authenticate(email, privateKey).then(msg => {
console.log(msg); console.log(msg)
return true; return true
}); })
} }
export default callback => { export default callback => {
@@ -18,29 +18,29 @@ export default callback => {
return { return {
name: sheet.name, name: sheet.name,
fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs) fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs)
}; }
}); })
Promise.all(fetchers.map(authenticate)) Promise.all(fetchers.map(authenticate))
.then(() => { .then(() => {
console.log(`===================`); console.log(`===================`)
console.log(`grant access to: ${email}`); console.log(`grant access to: ${email}`)
console.log(`===================`); console.log(`===================`)
// NB: reformat fetchers as config for controller // NB: reformat fetchers as config for controller
const config = {}; const config = {}
fetchers.forEach(fetcher => { fetchers.forEach(fetcher => {
config[fetcher.name] = fetcher.fetcher; config[fetcher.name] = fetcher.fetcher
}); })
const controller = new Controller(config); const controller = new Controller(config)
callback(controller); callback(controller)
}) })
.catch(err => { .catch(err => {
console.log(err); console.log(err)
console.log( console.log(
`ERROR: the server couldn't connect to all of the sheets you provided. Ensure you have granted access to ${ `ERROR: the server couldn't connect to all of the sheets you provided. Ensure you have granted access to ${
email email
} on ALL listed sheets.` } on ALL listed sheets.`
); )
}); })
}; }

40
src/lib/Controller.js
View File

@@ -3,51 +3,51 @@
* *
*/ */
class Controller { class Controller {
constructor(fetchers) { constructor (fetchers) {
this.fetchers = fetchers; this.fetchers = fetchers
} }
sourceExists(source) { sourceExists (source) {
return (Object.keys(this.fetchers).indexOf(source) >= 0) return (Object.keys(this.fetchers).indexOf(source) >= 0)
} }
blueprints() { blueprints () {
return Object.keys(this.fetchers).map( return Object.keys(this.fetchers).map(
source => this.fetchers[source].blueprints source => this.fetchers[source].blueprints
); )
} }
update() { update () {
return Promise.all( return Promise.all(
Object.keys(this.fetchers).map(source => { Object.keys(this.fetchers).map(source => {
return this.fetchers[source].update(); return this.fetchers[source].update()
}) })
).then(results => { ).then(results => {
return "All sources updated"; return 'All sources updated'
}); })
} }
retrieve(source, tab, resource) { retrieve (source, tab, resource) {
if (this.sourceExists(source)) { if (this.sourceExists(source)) {
const fetcher = this.fetchers[source]; const fetcher = this.fetchers[source]
return fetcher.retrieve(tab, resource); return fetcher.retrieve(tab, resource)
} else { } else {
return Promise.resolve().then(() => { return Promise.resolve().then(() => {
throw new Error(`Source ${source} not available.`); throw new Error(`Source ${source} not available.`)
}); })
} }
} }
retrieveFrag(source, tab, resource, frag) { retrieveFrag (source, tab, resource, frag) {
if (this.sourceExists(source)) { if (this.sourceExists(source)) {
const fetcher = this.fetchers[source]; const fetcher = this.fetchers[source]
return fetcher.retrieveFrag(tab, resource, frag); return fetcher.retrieveFrag(tab, resource, frag)
} else { } else {
return Promise.resolve().then(() => { return Promise.resolve().then(() => {
throw new Error(`Source ${source} not available.`); throw new Error(`Source ${source} not available.`)
}); })
} }
} }
} }
export default Controller; export default Controller

118
src/lib/Fetcher.js
View File

@@ -1,50 +1,49 @@
// FetcherTwo class interfaces with Google Sheet, and saves to a specified db // FetcherTwo class interfaces with Google Sheet, and saves to a specified db
import {google} from "googleapis"; import { google } from 'googleapis'
import { import {
fmtSourceTitle, fmtSourceTitle,
fmtBlueprinterTitles, fmtBlueprinterTitles,
deriveFilename,
bp, bp,
isFunction isFunction
} from "./util"; } from './util'
import {byRow, byId} from "./blueprinters"; import { byRow } from './blueprinters'
import R from "ramda"; import R from 'ramda'
class Fetcher { class Fetcher {
constructor(db, sourceName, sourceId, blueprinters) { constructor (db, sourceName, sourceId, blueprinters) {
/* /*
* The database that the fetcher should use. This should be an instance of a model-compliant class. * The database that the fetcher should use. This should be an instance of a model-compliant class.
* See models/Interface.js for the specifications for a model-compliant class. * See models/Interface.js for the specifications for a model-compliant class.
*/ */
this.db = db; this.db = db
/* /*
* ID of the Google Sheet where the data is sourced. Note that the privateKey.client_email * ID of the Google Sheet where the data is sourced. Note that the privateKey.clientEmail
* loaded here must be added to the sheet as an editor. * loaded here must be added to the sheet as an editor.
*/ */
this.sourceId = sourceId; this.sourceId = sourceId
/* /*
* The name of the source. This will prefix tabs saved in the database. * The name of the source. This will prefix tabs saved in the database.
*/ */
this.sourceName = sourceName; this.sourceName = sourceName
/* /*
* These are the available tabs for storing and retrieving data. * These are the available tabs for storing and retrieving data.
* Each blueprinter is a function that returns a Blueprint from a * Each blueprinter is a function that returns a Blueprint from a
* list of lists (which will be retrieved from gsheets). * list of lists (which will be retrieved from gsheets).
*/ */
this.blueprinters = fmtBlueprinterTitles(blueprinters); this.blueprinters = fmtBlueprinterTitles(blueprinters)
this.blueprints = {}; this.blueprints = {}
Object.keys(this.blueprinters).forEach(key => { Object.keys(this.blueprinters).forEach(key => {
this.blueprints[key] = null; this.blueprints[key] = null
}); })
/* /*
* Google API setup * Google API setup
*/ */
this.sheets = google.sheets("v4"); this.sheets = google.sheets('v4')
this.auth = null; this.auth = null
/** /**
* saveBp is a curried function that takes in a title and * saveBp is a curried function that takes in a title and
@@ -57,35 +56,34 @@ class Fetcher {
this.sourceName, this.sourceName,
this.sourceId, this.sourceId,
data data
); )
const blueprint = bp(saturatedBp); // TODO: come up with better semantics. const blueprint = bp(saturatedBp) // TODO: come up with better semantics.
this.blueprints[title] = blueprint; this.blueprints[title] = blueprint
return this.db.save(saturatedBp); return this.db.save(saturatedBp)
}); })
} }
/** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */ /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
authenticate(client_email, private_key) { authenticate (clientEmail, privateKey) {
const googleAuth = new google.auth.JWT(client_email, null, private_key, [ const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [
"https://www.googleapis.com/auth/spreadsheets" 'https://www.googleapis.com/auth/spreadsheets'
]); ])
this.auth = googleAuth; this.auth = googleAuth
const {sourceId} = this; const { sourceId } = this
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
googleAuth.authorize(function(err, tokens) { googleAuth.authorize(function (err) {
if (err) { if (err) {
reject(err); reject(err)
return;
} else { } else {
resolve(`Connected to ${sourceId}.`); resolve(`Connected to ${sourceId}.`)
} }
}); })
}); })
} }
update() { update () {
let tabTitles; let tabTitles
/* Retrieve all available routes on a given sheet, and store formatted copies of it where a formatter is available */ /* Retrieve all available routes on a given sheet, and store formatted copies of it where a formatter is available */
return this.sheets.spreadsheets return this.sheets.spreadsheets
.get({ .get({
@@ -93,57 +91,57 @@ class Fetcher {
spreadsheetId: this.sourceId spreadsheetId: this.sourceId
}) })
.then(response => { .then(response => {
tabTitles = response.data.sheets.map(sheet => sheet.properties.title); tabTitles = response.data.sheets.map(sheet => sheet.properties.title)
return this.sheets.spreadsheets.values.batchGet({ return this.sheets.spreadsheets.values.batchGet({
auth: this.auth, auth: this.auth,
spreadsheetId: this.sourceId, spreadsheetId: this.sourceId,
ranges: tabTitles ranges: tabTitles
}); })
}) })
.then(results => { .then(results => {
const tabData = results.data.valueRanges; const tabData = results.data.valueRanges
return Promise.all( return Promise.all(
tabData.map((tab, idx) => { tabData.map((tab, idx) => {
const {values} = tab; const { values } = tab
if (values == undefined) { if (values === undefined) {
return Promise.resolve({}); return Promise.resolve({})
} }
const name = tabTitles[idx]; const name = tabTitles[idx]
return this.save(name, values); return this.save(name, values)
}) })
); )
}) })
.then(() => "All tabs updated"); .then(() => 'All tabs updated')
} }
save(tab, data) { save (tab, data) {
const title = fmtSourceTitle(tab); const title = fmtSourceTitle(tab)
if (Object.keys(this.blueprinters).indexOf(title) > -1) { if (Object.keys(this.blueprinters).indexOf(title) > -1) {
const bpConfig = this.blueprinters[title]; const bpConfig = this.blueprinters[title]
if (isFunction(bpConfig)) { if (isFunction(bpConfig)) {
return this._saveBp(tab, title, data, bpConfig); return this._saveBp(tab, title, data, bpConfig)
} else { } else {
return bpConfig.map(this._saveBp(tab, title, data)); return bpConfig.map(this._saveBp(tab, title, data))
} }
} else { } else {
// If it can't find a blueprinter for the tab title, default to byRow // If it can't find a blueprinter for the tab title, default to byRow
return this.db.save(byRow(tab, this.sourceName, this.sourceId, data)); return this.db.save(byRow(tab, this.sourceName, this.sourceId, data))
} }
} }
// NB: could combine these functions by checking kwargs length // NB: could combine these functions by checking kwargs length
retrieve(tab, resource) { retrieve (tab, resource) {
const title = fmtSourceTitle(tab); const title = fmtSourceTitle(tab)
const url = `${this.sourceName}/${tab}/${resource}`; const url = `${this.sourceName}/${tab}/${resource}`
return this.db.load(url, this.blueprints[title]); return this.db.load(url, this.blueprints[title])
} }
retrieveFrag(tab, resource, frag) { retrieveFrag (tab, resource, frag) {
const title = fmtSourceTitle(tab); const title = fmtSourceTitle(tab)
const url = `${this.sourceName}/${tab}/${resource}/${frag}`; const url = `${this.sourceName}/${tab}/${resource}/${frag}`
return this.db.load(url, this.blueprints[title]); return this.db.load(url, this.blueprints[title])
} }
} }
export default Fetcher; export default Fetcher

29
src/lib/blueprinters.js
View File

@@ -1,33 +1,32 @@
import path from "path"; import path from 'path'
import fs from "fs"; import fs from 'fs'
export const defaultBlueprint = { export const defaultBlueprint = {
name: null, name: null,
id: null, id: null,
dialects: ["rest"], // supported dialects, can (eventually) be multiple dialects: ['rest'], // supported dialects, can (eventually) be multiple
routes: {} routes: {}
}; }
export const defaultRoute = { export const defaultRoute = {
options: { options: {
fragment: true fragment: true
}, },
data: [] data: []
}; }
// import all default exports from 'blueprinters' folder // import all default exports from 'blueprinters' folder
const allBps = {}; const allBps = {}
const REL_PATH_TO_BPS = "../blueprinters"; const REL_PATH_TO_BPS = '../blueprinters'
const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS); const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS)
fs.readdirSync(normalizedPath).forEach(file => { fs.readdirSync(normalizedPath).forEach(file => {
const bpName = file.replace(".js", ""); const bpName = file.replace('.js', '')
allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default; allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default
}); })
// NB: revert to ES5 'module.exports' required to make blueprinters from // NB: revert to ES5 'module.exports' required to make blueprinters from
// each file in blueprinters folder available for granular import from here. // each file in blueprinters folder available for granular import from here.
module.exports = { module.exports = Object.assign({
defaultBlueprint, defaultBlueprint,
defaultRoute, defaultRoute
...allBps }, allBps)
}

87
src/lib/util.js
View File

@@ -1,16 +1,17 @@
import R from "ramda"; import R from 'ramda'
String.prototype.replaceAll = function(search, replacement) { /* eslint-disable */
const target = this; String.prototype.replaceAll = function (search, replacement) {
return target.replace(new RegExp(search, "g"), replacement); const target = this
}; return target.replace(new RegExp(search, 'g'), replacement)
}
/* eslint-enable */
function camelize(str) { function camelize (str) {
return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function(match, index) { return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function (match) {
if (+match === 0) return ""; // or if (/\s+/.test(match)) for white spaces if (+match === 0) return '' // or if (/\s+/.test(match)) for white spaces
// return index == 0 ? match.toLowerCase() : match.toUpperCase(); return match.toUpperCase()
return match.toUpperCase(); })
});
} }
export const fmtObj = R.curry( export const fmtObj = R.curry(
@@ -23,71 +24,71 @@ export const fmtObj = R.curry(
camelCaseKeys: false camelCaseKeys: false
} }
) => { ) => {
const obj = {}; const obj = {}
const fmtColName = colName => { const fmtColName = colName => {
if (options.camelCaseKeys) { if (options.camelCaseKeys) {
return camelize(colName); return camelize(colName)
} else if (options.hyphenatedKeys) { } else if (options.hyphenatedKeys) {
return colName.toLowerCase().replaceAll(" ", "-"); return colName.toLowerCase().replaceAll(' ', '-')
} else if (options.noSpacesInKeys) { } else if (options.noSpacesInKeys) {
return colName.replaceAll(" ", ""); return colName.replaceAll(' ', '')
} else { } else {
return colName; return colName
} }
}; }
columnNames.forEach((columnName, idx) => { columnNames.forEach((columnName, idx) => {
obj[fmtColName(columnName)] = row[idx]; obj[fmtColName(columnName)] = row[idx]
}); })
return obj; return obj
} }
); )
/* search for object with key in array. Return index if exists, or -1 if not */ /* search for object with key in array. Return index if exists, or -1 if not */
export const idxSearcher = R.curry((attrName, searchValue, myArray) => { export const idxSearcher = R.curry((attrName, searchValue, myArray) => {
for (var i = 0; i < myArray.length; i++) { for (var i = 0; i < myArray.length; i++) {
if (myArray[i][attrName] == searchValue) { if (myArray[i][attrName] === searchValue) {
return i; return i
} }
} }
return -1; return -1
}); })
/* more site specific functions. TODO: maybe move to another folder? */ /* more site specific functions. TODO: maybe move to another folder? */
export function fmtSourceTitle(name) { export function fmtSourceTitle (name) {
return name.replaceAll(" ", "-").toLowerCase(); return name.replaceAll(' ', '-').toLowerCase()
} }
export function fmtBlueprinterTitles(tabs) { export function fmtBlueprinterTitles (tabs) {
const obj = {}; const obj = {}
Object.keys(tabs).forEach(tab => { Object.keys(tabs).forEach(tab => {
const name = fmtSourceTitle(tab); const name = fmtSourceTitle(tab)
obj[name] = tabs[tab]; obj[name] = tabs[tab]
}); })
return obj; return obj
} }
export function deriveFilename(source, tab) { export function deriveFilename (source, tab) {
return `${fmtSourceTitle(source)}-${fmtSourceTitle(tab)}.json`; return `${fmtSourceTitle(source)}-${fmtSourceTitle(tab)}.json`
} }
export function bp(full) { export function bp (full) {
const blueprint = { const blueprint = {
name: R.clone(full.name), name: R.clone(full.name),
source: R.clone(full.source), source: R.clone(full.source),
dialects: R.clone(full.dialects), dialects: R.clone(full.dialects),
routes: {} routes: {}
}; }
Object.keys(full.routes).forEach(route => { Object.keys(full.routes).forEach(route => {
blueprint.routes[route] = { blueprint.routes[route] = {
options: R.clone(full.routes[route].options) options: R.clone(full.routes[route].options)
}; }
}); })
return blueprint; return blueprint
} }
export function isFunction(functionToCheck) { export function isFunction (functionToCheck) {
return ( return (
functionToCheck && {}.toString.call(functionToCheck) === "[object Function]" functionToCheck && {}.toString.call(functionToCheck) === '[object Function]'
); )
} }

21
src/middleware/index.js
View File

@@ -1,17 +1,18 @@
import {Router, next} from "express"; import { Router } from 'express'
import {mapboxAccessToken} from "../config"; import { mapboxAccessToken } from '../config'
import morgan from "morgan"; import morgan from 'morgan'
import mapbox from "./mapbox"; import mapbox from './mapbox'
export default ({config, db}) => { // eslint-disable-next-line
let routes = Router(); export default ({ config, db }) => {
let routes = Router()
/* logging middleware */ /* logging middleware */
routes.use(morgan("dev")); routes.use(morgan('dev'))
if (mapboxAccessToken) { if (mapboxAccessToken) {
routes.get("/mapbox/:z/:y/:x", mapbox(mapboxAccessToken)); routes.get('/mapbox/:z/:y/:x', mapbox(mapboxAccessToken))
} }
return routes; return routes
}; }

18
src/middleware/mapbox/index.js
View File

@@ -1,16 +1,14 @@
import fetch from "node-fetch"; import fetch from 'node-fetch'
import fs from "fs";
// TODO: load images from mapbox API and store.
const baseUrl = "http://a.tiles.mapbox.com/v4/mapbox.satellite"; const baseUrl = 'http://a.tiles.mapbox.com/v4/mapbox.satellite'
export default accessToken => (req, res) => { export default accessToken => (req, res) => {
const {x, y, z} = req.params; const { x, y, z } = req.params
// const filename = `${z}-${y}-${x}.png` // const filename = `${z}-${y}-${x}.png`
// const fileStream = fs.createWriteStream(`${z}-${y}-${x}.png`) // const fileStream = fs.createWriteStream(`${z}-${y}-${x}.png`)
fetch( fetch(
`http://a.tiles.mapbox.com/v4/mapbox.satellite/${z}/${y}/${x}@2x.png?access_token=${accessToken}` `${baseUrl}/${z}/${y}/${x}@2x.png?access_token=${accessToken}`
).then(result => { ).then(result => {
res.set("Content-Type", "image/png"); res.set('Content-Type', 'image/png')
result.body.pipe(res); result.body.pipe(res)
}); })
}; }

5
src/models/Interface.js
View File

@@ -1,3 +1,4 @@
/* eslint-disable */
/** /**
* Model is a class whose sole responsibility is to save and load blueprints. * Model is a class whose sole responsibility is to save and load blueprints.
* It allows for different storage mechanisms for different kinds of blueprints. * It allows for different storage mechanisms for different kinds of blueprints.
@@ -9,7 +10,7 @@ class Model {
* @param {type} blueprint the Blueprint to be saved. * @param {type} blueprint the Blueprint to be saved.
* @return {type} Promise which returns True. * @return {type} Promise which returns True.
*/ */
save(blueprint) {} save (blueprint) {}
/** /**
* load - load a resource from a data model, using a Blueprint object as * load - load a resource from a data model, using a Blueprint object as
@@ -19,5 +20,5 @@ class Model {
* @param {type} blueprint Blueprint object (desaturated?). * @param {type} blueprint Blueprint object (desaturated?).
* @return {type} Object containing the resource data. * @return {type} Object containing the resource data.
*/ */
load(url, blueprint) {} load (url, blueprint) {}
} }

44
src/models/StoreJson.js
View File

@@ -1,12 +1,10 @@
import fs from "mz/fs"; import fs from 'mz/fs'
import hash from "object-hash"; import { fmtSourceTitle } from '../lib/util'
import {fmtSourceTitle} from "../lib/util";
import path from "path";
const STORAGE_DIRNAME = "temp"; const STORAGE_DIRNAME = 'temp'
class StoreJson { class StoreJson {
save(bp) { save (bp) {
return Promise.all( return Promise.all(
Object.keys(bp.routes).map(route => Object.keys(bp.routes).map(route =>
fs.writeFile( fs.writeFile(
@@ -16,50 +14,50 @@ class StoreJson {
JSON.stringify(bp.routes[route].data) JSON.stringify(bp.routes[route].data)
) )
) )
); )
} }
load(url, bp) { load (url) {
const parts = url.split("/"); const parts = url.split('/')
const fname = `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${ const fname = `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${
parts[2] parts[2]
}.json`; }.json`
return fs return fs
.exists(fname) .exists(fname)
.then(isAvailable => { .then(isAvailable => {
if (isAvailable) return fs.readFile(fname, "utf8"); if (isAvailable) return fs.readFile(fname, 'utf8')
else { else {
throw new Error("No resource exists"); throw new Error('No resource exists')
} }
}) })
.then(data => JSON.parse(data)) .then(data => JSON.parse(data))
.then(data => { .then(data => {
if (parts.length === 3) { if (parts.length === 3) {
// No lookup if the requested url doesn't have a fragment // No lookup if the requested url doesn't have a fragment
return data; return data
} else if (parts[2] === "ids") { } else if (parts[2] === 'ids') {
// Do a lookup if fragment is included to filter a relevant item // Do a lookup if fragment is included to filter a relevant item
// When the resource requested is 'ids' // When the resource requested is 'ids'
const id = parseInt(parts[3]); const id = parseInt(parts[3])
if (!isNaN(id) && id >= 0 && id < data.length) { if (!isNaN(id) && id >= 0 && id < data.length) {
return data[id]; return data[id]
} else { } else {
throw new Error(`Fragment index does not exist`); throw new Error(`Fragment index does not exist`)
} }
} else { } else {
// Do a lookup if fragment is included to filter a relevant item // Do a lookup if fragment is included to filter a relevant item
const index = parseInt(parts[3]); const index = parseInt(parts[3])
if (!isNaN(index) && index >= 0 && index < data.length) { if (!isNaN(index) && index >= 0 && index < data.length) {
console.log(data, index); console.log(data, index)
return data.filter((vl, idx) => idx === index)[0]; return data.filter((vl, idx) => idx === index)[0]
} else { } else {
throw new Error(`Fragment index does not exist`); throw new Error(`Fragment index does not exist`)
} }
} }
}); })
} }
// TODO: add method to build blueprint from data source // TODO: add method to build blueprint from data source
} }
export default StoreJson; export default StoreJson

828
yarn.lock
View File

File diff suppressed because it is too large Load Diff