mirror of
https://github.com/bellingcat/datasheet-server.git
synced 2026-06-11 04:48:32 +03:00
change linter to standard
This commit is contained in:
@@ -3,51 +3,51 @@
|
||||
*
|
||||
*/
|
||||
class Controller {
|
||||
constructor(fetchers) {
|
||||
this.fetchers = fetchers;
|
||||
constructor (fetchers) {
|
||||
this.fetchers = fetchers
|
||||
}
|
||||
|
||||
sourceExists(source) {
|
||||
sourceExists (source) {
|
||||
return (Object.keys(this.fetchers).indexOf(source) >= 0)
|
||||
}
|
||||
|
||||
blueprints() {
|
||||
blueprints () {
|
||||
return Object.keys(this.fetchers).map(
|
||||
source => this.fetchers[source].blueprints
|
||||
);
|
||||
)
|
||||
}
|
||||
|
||||
update() {
|
||||
update () {
|
||||
return Promise.all(
|
||||
Object.keys(this.fetchers).map(source => {
|
||||
return this.fetchers[source].update();
|
||||
return this.fetchers[source].update()
|
||||
})
|
||||
).then(results => {
|
||||
return "All sources updated";
|
||||
});
|
||||
return 'All sources updated'
|
||||
})
|
||||
}
|
||||
|
||||
retrieve(source, tab, resource) {
|
||||
retrieve (source, tab, resource) {
|
||||
if (this.sourceExists(source)) {
|
||||
const fetcher = this.fetchers[source];
|
||||
return fetcher.retrieve(tab, resource);
|
||||
const fetcher = this.fetchers[source]
|
||||
return fetcher.retrieve(tab, resource)
|
||||
} else {
|
||||
return Promise.resolve().then(() => {
|
||||
throw new Error(`Source ${source} not available.`);
|
||||
});
|
||||
throw new Error(`Source ${source} not available.`)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
retrieveFrag(source, tab, resource, frag) {
|
||||
retrieveFrag (source, tab, resource, frag) {
|
||||
if (this.sourceExists(source)) {
|
||||
const fetcher = this.fetchers[source];
|
||||
return fetcher.retrieveFrag(tab, resource, frag);
|
||||
const fetcher = this.fetchers[source]
|
||||
return fetcher.retrieveFrag(tab, resource, frag)
|
||||
} else {
|
||||
return Promise.resolve().then(() => {
|
||||
throw new Error(`Source ${source} not available.`);
|
||||
});
|
||||
throw new Error(`Source ${source} not available.`)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default Controller;
|
||||
export default Controller
|
||||
|
||||
@@ -1,50 +1,49 @@
|
||||
// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
|
||||
import {google} from "googleapis";
|
||||
import { google } from 'googleapis'
|
||||
import {
|
||||
fmtSourceTitle,
|
||||
fmtBlueprinterTitles,
|
||||
deriveFilename,
|
||||
bp,
|
||||
isFunction
|
||||
} from "./util";
|
||||
import {byRow, byId} from "./blueprinters";
|
||||
import R from "ramda";
|
||||
} from './util'
|
||||
import { byRow } from './blueprinters'
|
||||
import R from 'ramda'
|
||||
|
||||
class Fetcher {
|
||||
constructor(db, sourceName, sourceId, blueprinters) {
|
||||
constructor (db, sourceName, sourceId, blueprinters) {
|
||||
/*
|
||||
* The database that the fetcher should use. This should be an instance of a model-compliant class.
|
||||
* See models/Interface.js for the specifications for a model-compliant class.
|
||||
*/
|
||||
this.db = db;
|
||||
this.db = db
|
||||
|
||||
/*
|
||||
* ID of the Google Sheet where the data is sourced. Note that the privateKey.client_email
|
||||
* ID of the Google Sheet where the data is sourced. Note that the privateKey.clientEmail
|
||||
* loaded here must be added to the sheet as an editor.
|
||||
*/
|
||||
this.sourceId = sourceId;
|
||||
this.sourceId = sourceId
|
||||
|
||||
/*
|
||||
* The name of the source. This will prefix tabs saved in the database.
|
||||
*/
|
||||
this.sourceName = sourceName;
|
||||
this.sourceName = sourceName
|
||||
|
||||
/*
|
||||
* These are the available tabs for storing and retrieving data.
|
||||
* Each blueprinter is a function that returns a Blueprint from a
|
||||
* list of lists (which will be retrieved from gsheets).
|
||||
*/
|
||||
this.blueprinters = fmtBlueprinterTitles(blueprinters);
|
||||
this.blueprints = {};
|
||||
this.blueprinters = fmtBlueprinterTitles(blueprinters)
|
||||
this.blueprints = {}
|
||||
Object.keys(this.blueprinters).forEach(key => {
|
||||
this.blueprints[key] = null;
|
||||
});
|
||||
this.blueprints[key] = null
|
||||
})
|
||||
|
||||
/*
|
||||
* Google API setup
|
||||
*/
|
||||
this.sheets = google.sheets("v4");
|
||||
this.auth = null;
|
||||
this.sheets = google.sheets('v4')
|
||||
this.auth = null
|
||||
|
||||
/**
|
||||
* saveBp is a curried function that takes in a title and
|
||||
@@ -57,35 +56,34 @@ class Fetcher {
|
||||
this.sourceName,
|
||||
this.sourceId,
|
||||
data
|
||||
);
|
||||
const blueprint = bp(saturatedBp); // TODO: come up with better semantics.
|
||||
this.blueprints[title] = blueprint;
|
||||
return this.db.save(saturatedBp);
|
||||
});
|
||||
)
|
||||
const blueprint = bp(saturatedBp) // TODO: come up with better semantics.
|
||||
this.blueprints[title] = blueprint
|
||||
return this.db.save(saturatedBp)
|
||||
})
|
||||
}
|
||||
|
||||
/** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
|
||||
authenticate(client_email, private_key) {
|
||||
const googleAuth = new google.auth.JWT(client_email, null, private_key, [
|
||||
"https://www.googleapis.com/auth/spreadsheets"
|
||||
]);
|
||||
this.auth = googleAuth;
|
||||
const {sourceId} = this;
|
||||
authenticate (clientEmail, privateKey) {
|
||||
const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [
|
||||
'https://www.googleapis.com/auth/spreadsheets'
|
||||
])
|
||||
this.auth = googleAuth
|
||||
const { sourceId } = this
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
googleAuth.authorize(function(err, tokens) {
|
||||
googleAuth.authorize(function (err) {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
reject(err)
|
||||
} else {
|
||||
resolve(`Connected to ${sourceId}.`);
|
||||
resolve(`Connected to ${sourceId}.`)
|
||||
}
|
||||
});
|
||||
});
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
update() {
|
||||
let tabTitles;
|
||||
update () {
|
||||
let tabTitles
|
||||
/* Retrieve all available routes on a given sheet, and store formatted copies of it where a formatter is available */
|
||||
return this.sheets.spreadsheets
|
||||
.get({
|
||||
@@ -93,57 +91,57 @@ class Fetcher {
|
||||
spreadsheetId: this.sourceId
|
||||
})
|
||||
.then(response => {
|
||||
tabTitles = response.data.sheets.map(sheet => sheet.properties.title);
|
||||
tabTitles = response.data.sheets.map(sheet => sheet.properties.title)
|
||||
return this.sheets.spreadsheets.values.batchGet({
|
||||
auth: this.auth,
|
||||
spreadsheetId: this.sourceId,
|
||||
ranges: tabTitles
|
||||
});
|
||||
})
|
||||
})
|
||||
.then(results => {
|
||||
const tabData = results.data.valueRanges;
|
||||
const tabData = results.data.valueRanges
|
||||
return Promise.all(
|
||||
tabData.map((tab, idx) => {
|
||||
const {values} = tab;
|
||||
if (values == undefined) {
|
||||
return Promise.resolve({});
|
||||
const { values } = tab
|
||||
if (values === undefined) {
|
||||
return Promise.resolve({})
|
||||
}
|
||||
const name = tabTitles[idx];
|
||||
return this.save(name, values);
|
||||
const name = tabTitles[idx]
|
||||
return this.save(name, values)
|
||||
})
|
||||
);
|
||||
)
|
||||
})
|
||||
.then(() => "All tabs updated");
|
||||
.then(() => 'All tabs updated')
|
||||
}
|
||||
|
||||
save(tab, data) {
|
||||
const title = fmtSourceTitle(tab);
|
||||
save (tab, data) {
|
||||
const title = fmtSourceTitle(tab)
|
||||
if (Object.keys(this.blueprinters).indexOf(title) > -1) {
|
||||
const bpConfig = this.blueprinters[title];
|
||||
const bpConfig = this.blueprinters[title]
|
||||
|
||||
if (isFunction(bpConfig)) {
|
||||
return this._saveBp(tab, title, data, bpConfig);
|
||||
return this._saveBp(tab, title, data, bpConfig)
|
||||
} else {
|
||||
return bpConfig.map(this._saveBp(tab, title, data));
|
||||
return bpConfig.map(this._saveBp(tab, title, data))
|
||||
}
|
||||
} else {
|
||||
// If it can't find a blueprinter for the tab title, default to byRow
|
||||
return this.db.save(byRow(tab, this.sourceName, this.sourceId, data));
|
||||
return this.db.save(byRow(tab, this.sourceName, this.sourceId, data))
|
||||
}
|
||||
}
|
||||
|
||||
// NB: could combine these functions by checking kwargs length
|
||||
retrieve(tab, resource) {
|
||||
const title = fmtSourceTitle(tab);
|
||||
const url = `${this.sourceName}/${tab}/${resource}`;
|
||||
return this.db.load(url, this.blueprints[title]);
|
||||
retrieve (tab, resource) {
|
||||
const title = fmtSourceTitle(tab)
|
||||
const url = `${this.sourceName}/${tab}/${resource}`
|
||||
return this.db.load(url, this.blueprints[title])
|
||||
}
|
||||
|
||||
retrieveFrag(tab, resource, frag) {
|
||||
const title = fmtSourceTitle(tab);
|
||||
const url = `${this.sourceName}/${tab}/${resource}/${frag}`;
|
||||
return this.db.load(url, this.blueprints[title]);
|
||||
retrieveFrag (tab, resource, frag) {
|
||||
const title = fmtSourceTitle(tab)
|
||||
const url = `${this.sourceName}/${tab}/${resource}/${frag}`
|
||||
return this.db.load(url, this.blueprints[title])
|
||||
}
|
||||
}
|
||||
|
||||
export default Fetcher;
|
||||
export default Fetcher
|
||||
|
||||
@@ -1,33 +1,32 @@
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
import path from 'path'
|
||||
import fs from 'fs'
|
||||
|
||||
export const defaultBlueprint = {
|
||||
name: null,
|
||||
id: null,
|
||||
dialects: ["rest"], // supported dialects, can (eventually) be multiple
|
||||
dialects: ['rest'], // supported dialects, can (eventually) be multiple
|
||||
routes: {}
|
||||
};
|
||||
}
|
||||
|
||||
export const defaultRoute = {
|
||||
options: {
|
||||
fragment: true
|
||||
},
|
||||
data: []
|
||||
};
|
||||
}
|
||||
|
||||
// import all default exports from 'blueprinters' folder
|
||||
const allBps = {};
|
||||
const REL_PATH_TO_BPS = "../blueprinters";
|
||||
const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS);
|
||||
const allBps = {}
|
||||
const REL_PATH_TO_BPS = '../blueprinters'
|
||||
const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS)
|
||||
fs.readdirSync(normalizedPath).forEach(file => {
|
||||
const bpName = file.replace(".js", "");
|
||||
allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default;
|
||||
});
|
||||
const bpName = file.replace('.js', '')
|
||||
allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default
|
||||
})
|
||||
|
||||
// NB: revert to ES5 'module.exports' required to make blueprinters from
|
||||
// each file in blueprinters folder available for granular import from here.
|
||||
module.exports = {
|
||||
module.exports = Object.assign({
|
||||
defaultBlueprint,
|
||||
defaultRoute,
|
||||
...allBps
|
||||
}
|
||||
defaultRoute
|
||||
}, allBps)
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
import R from "ramda";
|
||||
import R from 'ramda'
|
||||
|
||||
String.prototype.replaceAll = function(search, replacement) {
|
||||
const target = this;
|
||||
return target.replace(new RegExp(search, "g"), replacement);
|
||||
};
|
||||
/* eslint-disable */
|
||||
String.prototype.replaceAll = function (search, replacement) {
|
||||
const target = this
|
||||
return target.replace(new RegExp(search, 'g'), replacement)
|
||||
}
|
||||
/* eslint-enable */
|
||||
|
||||
function camelize(str) {
|
||||
return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function(match, index) {
|
||||
if (+match === 0) return ""; // or if (/\s+/.test(match)) for white spaces
|
||||
// return index == 0 ? match.toLowerCase() : match.toUpperCase();
|
||||
return match.toUpperCase();
|
||||
});
|
||||
function camelize (str) {
|
||||
return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function (match) {
|
||||
if (+match === 0) return '' // or if (/\s+/.test(match)) for white spaces
|
||||
return match.toUpperCase()
|
||||
})
|
||||
}
|
||||
|
||||
export const fmtObj = R.curry(
|
||||
@@ -23,71 +24,71 @@ export const fmtObj = R.curry(
|
||||
camelCaseKeys: false
|
||||
}
|
||||
) => {
|
||||
const obj = {};
|
||||
const obj = {}
|
||||
const fmtColName = colName => {
|
||||
if (options.camelCaseKeys) {
|
||||
return camelize(colName);
|
||||
return camelize(colName)
|
||||
} else if (options.hyphenatedKeys) {
|
||||
return colName.toLowerCase().replaceAll(" ", "-");
|
||||
return colName.toLowerCase().replaceAll(' ', '-')
|
||||
} else if (options.noSpacesInKeys) {
|
||||
return colName.replaceAll(" ", "");
|
||||
return colName.replaceAll(' ', '')
|
||||
} else {
|
||||
return colName;
|
||||
return colName
|
||||
}
|
||||
};
|
||||
}
|
||||
columnNames.forEach((columnName, idx) => {
|
||||
obj[fmtColName(columnName)] = row[idx];
|
||||
});
|
||||
return obj;
|
||||
obj[fmtColName(columnName)] = row[idx]
|
||||
})
|
||||
return obj
|
||||
}
|
||||
);
|
||||
)
|
||||
|
||||
/* search for object with key in array. Return index if exists, or -1 if not */
|
||||
export const idxSearcher = R.curry((attrName, searchValue, myArray) => {
|
||||
for (var i = 0; i < myArray.length; i++) {
|
||||
if (myArray[i][attrName] == searchValue) {
|
||||
return i;
|
||||
if (myArray[i][attrName] === searchValue) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
});
|
||||
return -1
|
||||
})
|
||||
|
||||
/* more site specific functions. TODO: maybe move to another folder? */
|
||||
|
||||
export function fmtSourceTitle(name) {
|
||||
return name.replaceAll(" ", "-").toLowerCase();
|
||||
export function fmtSourceTitle (name) {
|
||||
return name.replaceAll(' ', '-').toLowerCase()
|
||||
}
|
||||
|
||||
export function fmtBlueprinterTitles(tabs) {
|
||||
const obj = {};
|
||||
export function fmtBlueprinterTitles (tabs) {
|
||||
const obj = {}
|
||||
Object.keys(tabs).forEach(tab => {
|
||||
const name = fmtSourceTitle(tab);
|
||||
obj[name] = tabs[tab];
|
||||
});
|
||||
return obj;
|
||||
const name = fmtSourceTitle(tab)
|
||||
obj[name] = tabs[tab]
|
||||
})
|
||||
return obj
|
||||
}
|
||||
|
||||
export function deriveFilename(source, tab) {
|
||||
return `${fmtSourceTitle(source)}-${fmtSourceTitle(tab)}.json`;
|
||||
export function deriveFilename (source, tab) {
|
||||
return `${fmtSourceTitle(source)}-${fmtSourceTitle(tab)}.json`
|
||||
}
|
||||
|
||||
export function bp(full) {
|
||||
export function bp (full) {
|
||||
const blueprint = {
|
||||
name: R.clone(full.name),
|
||||
source: R.clone(full.source),
|
||||
dialects: R.clone(full.dialects),
|
||||
routes: {}
|
||||
};
|
||||
}
|
||||
Object.keys(full.routes).forEach(route => {
|
||||
blueprint.routes[route] = {
|
||||
options: R.clone(full.routes[route].options)
|
||||
};
|
||||
});
|
||||
return blueprint;
|
||||
}
|
||||
})
|
||||
return blueprint
|
||||
}
|
||||
|
||||
export function isFunction(functionToCheck) {
|
||||
export function isFunction (functionToCheck) {
|
||||
return (
|
||||
functionToCheck && {}.toString.call(functionToCheck) === "[object Function]"
|
||||
);
|
||||
functionToCheck && {}.toString.call(functionToCheck) === '[object Function]'
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user