From 5d7eb0af05ed070d4e2e38c00d403f64f4020224 Mon Sep 17 00:00:00 2001 From: Lachlan Kermode Date: Fri, 27 Mar 2020 12:30:21 +0100 Subject: [PATCH] add xlsx support by factoring out a Fetcher class --- package.json | 1 + src/api/index.js | 7 +- src/index.js | 3 +- src/initialize.js | 51 ++++++-------- src/lib/Controller.js | 1 + src/lib/Fetcher.js | 158 ++++++++++++++++++++++++++---------------- yarn.lock | 94 ++++++++++++++++++++++++- 7 files changed, 226 insertions(+), 89 deletions(-) diff --git a/package.json b/package.json index 5caea99..cda1ff9 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "morgan": "^1.8.0", "mz": "^2.7.0", "node-fetch": "^2.3.0", + "node-xlsx": "^0.15.0", "object-hash": "^1.3.0", "ramda": "^0.25.0", "resource-router-middleware": "^0.6.0" diff --git a/src/api/index.js b/src/api/index.js index db9d76f..4780e57 100755 --- a/src/api/index.js +++ b/src/api/index.js @@ -12,7 +12,12 @@ export default ({ config, controller }) => { }) api.get('/blueprints', (req, res) => { - res.json(controller.blueprints()) + const bps = controller.blueprints() + res.json(bps.map(bp => ({ + source: bp.sheet.name, + tab: bp.name, + urls: bp.urls + }))) }) api.get('/update', (req, res) => { diff --git a/src/index.js b/src/index.js index 1e4ccd2..e341eb1 100755 --- a/src/index.js +++ b/src/index.js @@ -36,7 +36,8 @@ initialize(controller => { ) app.server.listen(process.env.PORT || 4040, () => { - console.log(`Started on port ${app.server.address().port}`) + console.log('===========================================') + console.log(`Server running on port ${app.server.address().port}`) }) }) diff --git a/src/initialize.js b/src/initialize.js index 75c2a94..55765dd 100755 --- a/src/initialize.js +++ b/src/initialize.js @@ -1,37 +1,32 @@ import StoreJson from './models/StoreJson' -import Fetcher from './lib/Fetcher' +import fetchers from './lib/Fetcher' import Controller from './lib/Controller' -import sheetsConfig from './sheets_config' +import config from './sheets_config' +import R from 'ramda' -const { googleSheets } = sheetsConfig -const { sheets } = googleSheets - -function authenticate (_fetcher) { - return _fetcher.fetcher.authenticate(process.env.SERVICE_ACCOUNT_EMAIL, process.env.SERVICE_ACCOUNT_PRIVATE_KEY).then(msg => { - console.log(msg) - return true - }) -} +const isntNull = n => n !== null +const filterNull = ls => R.filter(isntNull, ls) +const flattenfilterNull = ls => filterNull(R.flatten(ls)) +let themFetchers export default callback => { - const fetchers = sheets.map(sheet => { - return { - name: sheet.name, - fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs) - } + return Promise.resolve().then(() => { + return Object.keys(config).map(fType => { + // skip config attrs that don't have corresponding fetchers + if (!(fType in fetchers)) return null + const FFetcher = fetchers[fType] + return config[fType].map(sheet => ({ + name: sheet.name, + fetcher: new FFetcher(new StoreJson(), ...Object.values(sheet)) + })) + }) }) - - Promise.all(fetchers.map(authenticate)) - .then(() => { - console.log(`===================`) - console.log(`grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`) - console.log(`===================`) - - // NB: reformat fetchers as config for controller - const config = {} - fetchers.forEach(fetcher => { - config[fetcher.name] = fetcher.fetcher - }) + .then(res => { + themFetchers = flattenfilterNull(res) + }) + .then(() => Promise.all(themFetchers.map(f => f.fetcher.authenticate(process.env)))) + .then(fetchers => { + const config = R.zipObj(themFetchers.map(f => f.name), fetchers) const controller = new Controller(config) callback(controller) }) diff --git a/src/lib/Controller.js b/src/lib/Controller.js index 970e6f2..0910369 100644 --- a/src/lib/Controller.js +++ b/src/lib/Controller.js @@ -22,6 +22,7 @@ class Controller { update () { return Promise.all( Object.keys(this.fetchers).map(sheet => { + console.log(sheet) return this.fetchers[sheet].update() }) ).then(results => { diff --git a/src/lib/Fetcher.js b/src/lib/Fetcher.js index 2b05bee..fee031a 100644 --- a/src/lib/Fetcher.js +++ b/src/lib/Fetcher.js @@ -8,38 +8,31 @@ import { } from './util' import { createHash } from 'crypto' import R from 'ramda' +import xlsx from 'node-xlsx' +import fs from 'fs' class Fetcher { - constructor (db, sheetName, sheetId, blueprinters) { + constructor (db, name, bps) { /* * The database that the fetcher should use. This should be an instance of a model-compliant class. * See models/Interface.js for the specifications for a model-compliant class. */ this.db = db - - /* - * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail - * loaded here must be added to the sheet as an editor. - */ - this.sheetId = sheetId - /* * The name of the sheet. This will prefix tabs saved in the database. */ - this.sheetName = sheetName - + this.sheetName = name /* * A unique ID for the Fetcher to identify its elements in the model layer */ - this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex') - + const bpsstring = Object.keys(bps).join(';') + this.id = createHash('md5').update(name).update(bpsstring).digest('hex') /* * These are the available tabs for storing and retrieving data. * Each blueprinter is a function that returns a Blueprint from a * list of lists (which will be retrieved from gsheets). */ - this.blueprinters = fmtBlueprinterTitles(blueprinters) - + this.blueprinters = fmtBlueprinterTitles(bps) /* * This object is the canonical represenation for the data that a Fetcher * proxies. When the fetcher is initialized, its model layer (db) is indexed, @@ -49,13 +42,6 @@ class Fetcher { */ this.blueprints = null this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion - - /* - * Google API setup - */ - this.API = google.sheets('v4') - this.auth = null - /** curry to allow convenient syntax with map */ this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter.bind(this)) } @@ -83,12 +69,16 @@ class Fetcher { }, []) return allParts - .map(parts => buildDesaturated( - this.sheetId, - this.sheetName, - parts[0], - parts[1] - )) + .map(parts => { + const bp = buildDesaturated( + this.sheetId, + this.sheetName, + parts[0], + parts[1] + ) + bp.urls = Object.keys(bp.resources).map(r => `/api/${bp.sheet.name}/${bp.name}/${r}`) + return bp + }) }) .then(res => { this.blueprints = res @@ -113,20 +103,81 @@ class Fetcher { ) } + save (_tab, data) { + const tab = fmtName(_tab) + + if (Object.keys(this.blueprinters).indexOf(tab) > -1) { + const bpConfig = this.blueprinters[tab] + + if (isFunction(bpConfig)) { + // if bpConfig specifies a single blueprinter + return this._saveViaBlueprinter(tab, data, bpConfig) + } else { + // if bpConfig specifies an array of blueprinters + return bpConfig.map(this._saveViaBlueprinter(tab, data)) + } + } else { + // NB: if a blueprinter is not specified for a tab, + // just skip it. + return true + } + } + + // NB: could combine these functions by checking kwargs length + retrieve (tab, resource) { + const title = fmtName(tab) + const url = `${this.id}/${tab}/${resource}` + return this.db.load(url, this.blueprints[title]) + } + + retrieveFrag (tab, resource, frag) { + const title = fmtName(tab) + const url = `${this.id}/${tab}/${resource}/${frag || ''}` + return this.db.load(url, this.blueprints[title]) + } + + /** Run on startup. Should be overridden if explicit auth is required **/ + authenticate (env) { + console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`) + return Promise.resolve(this) + } +} + +class GsheetFetcher extends Fetcher { + constructor (db, name, bps, sheetId) { + super(db, name, bps) + this.type = 'Google Sheet' + if (arguments.length < 4) throw Error('You must provide the sheet ID') + + /* + * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail + * loaded here must be added to the sheet as an editor. + */ + this.sheetId = sheetId + /* + * Google API setup + */ + this.API = google.sheets('v4') + this.auth = null + } + /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */ - authenticate (clientEmail, privateKey) { - const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [ + authenticate (env) { + const googleAuth = new google.auth.JWT(env.SERVICE_ACCOUNT_EMAIL, null, env.SERVICE_ACCOUNT_PRIVATE_KEY, [ 'https://www.googleapis.com/auth/spreadsheets' ]) this.auth = googleAuth const { sheetId } = this + const me = this return new Promise((resolve, reject) => { googleAuth.authorize(function (err) { if (err) { reject(err) } else { - resolve(`Connected to ${sheetId}.`) + console.log(`Connected to ${me.sheetName}. (${me.type} with ID ${sheetId}).`) + console.log(` grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`) + resolve(me) } }) }) @@ -168,39 +219,30 @@ class Fetcher { .then(() => true) .catch(() => false) } +} - save (_tab, data) { - const tab = fmtName(_tab) +class XlsxFetcher extends Fetcher { + constructor (db, name, bps, path) { + super(db, name, bps) + this.type = 'XLSX File' + this.path = path + this.isRemote = false - if (Object.keys(this.blueprinters).indexOf(tab) > -1) { - const bpConfig = this.blueprinters[tab] - - if (isFunction(bpConfig)) { - // if bpConfig specifies a single blueprinter - return this._saveViaBlueprinter(tab, data, bpConfig) - } else { - // if bpConfig specifies an array of blueprinters - return bpConfig.map(this._saveViaBlueprinter(tab, data)) - } - } else { - // NB: if a blueprinter is not specified for a tab, - // just skip it. - return true + if (this.path.startsWith('https')) { + this.isRemote = true } } - // NB: could combine these functions by checking kwargs length - retrieve (tab, resource) { - const title = fmtName(tab) - const url = `${this.id}/${tab}/${resource}` - return this.db.load(url, this.blueprints[title]) - } - - retrieveFrag (tab, resource, frag) { - const title = fmtName(tab) - const url = `${this.id}/${tab}/${resource}/${frag || ''}` - return this.db.load(url, this.blueprints[title]) + update () { + const data = xlsx.parse(fs.readFileSync(this.path)) + data.forEach(tab => { + this.save(tab.name, tab.data) + }) + return Promise.resolve(true) } } -export default Fetcher +export default { + 'gsheets': GsheetFetcher, + 'xlsx': XlsxFetcher +} diff --git a/yarn.lock b/yarn.lock index 7755f53..14f648c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -621,6 +621,14 @@ acorn@^6.0.2: version "6.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474" +adler-32@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25" + integrity sha1-aj5r8KY5ALoVZSgIyxXGgT0aXyU= + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + ajv-keywords@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.2.0.tgz#e86b819c602cf8821ad637413698f1dec021847a" @@ -957,7 +965,7 @@ buffer-equal-constant-time@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" -buffer-from@^1.0.0: +buffer-from@^1.0.0, buffer-from@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -1042,6 +1050,16 @@ capture-stack-trace@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz#a6c0bbe1f38f3aa0b92238ecb6ff42c344d4135d" +cfb@^1.1.2: + version "1.1.4" + resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.1.4.tgz#81fd35ede4c919d8f0962a94582e1dfaf7051e2a" + integrity sha512-rwFkl3aFO3f+ljR27YINwC0x8vPjyiEVbYbrTCKzspEf7Q++3THdfHVgJYNUbxNcupJECrLX+L40Mjm9hm/Bgw== + dependencies: + adler-32 "~1.2.0" + commander "^2.16.0" + crc-32 "~1.2.0" + printj "~1.1.2" + chalk@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -1155,6 +1173,14 @@ code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" +codepage@~1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99" + integrity sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k= + dependencies: + commander "~2.14.1" + exit-on-epipe "~1.0.1" + collection-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -1172,10 +1198,25 @@ color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" +commander@^2.16.0: + version "2.20.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + commander@^2.8.1: version "2.19.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a" +commander@~2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa" + integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw== + +commander@~2.17.1: + version "2.17.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf" + integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg== + common-path-prefix@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/common-path-prefix/-/common-path-prefix-1.0.0.tgz#cd52f6f0712e0baab97d6f9732874f22f47752c0" @@ -1299,6 +1340,14 @@ cosmiconfig@^5.0.6: js-yaml "^3.9.0" parse-json "^4.0.0" +crc-32@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208" + integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA== + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + create-error-class@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/create-error-class/-/create-error-class-3.0.2.tgz#06be7abef947a3f14a30fd610671d401bca8b7b6" @@ -1790,6 +1839,11 @@ execa@^1.0.0: signal-exit "^3.0.0" strip-eof "^1.0.0" +exit-on-epipe@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" + integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw== + expand-brackets@^2.1.4: version "2.1.4" resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622" @@ -1988,6 +2042,11 @@ forwarded@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" +frac@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b" + integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA== + fragment-cache@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" @@ -3120,6 +3179,14 @@ node-releases@^1.0.0-alpha.14: dependencies: semver "^5.3.0" +node-xlsx@^0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/node-xlsx/-/node-xlsx-0.15.0.tgz#1f1b0d7adce5c706e86bfd96a5aa0005bf8a9dc3" + integrity sha512-rQyhWDJ/k60wQemov7a8MlToastWTidrAVFRwTWV+s53LN/SRwU4lnmc5xuFXx/ay+uaLAsAQBp6BkVob5OjOA== + dependencies: + buffer-from "^1.1.0" + xlsx "^0.14.1" + nodemon@1.18.7: version "1.18.7" resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-1.18.7.tgz#716b66bf3e89ac4fcfb38a9e61887a03fc82efbb" @@ -3523,6 +3590,11 @@ pretty-ms@^3.2.0: dependencies: parse-ms "^1.0.0" +printj@~1.1.0, printj@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" + integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== + private@^0.1.6: version "0.1.8" resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff" @@ -4039,6 +4111,13 @@ sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" +ssf@~0.10.2: + version "0.10.3" + resolved "https://registry.yarnpkg.com/ssf/-/ssf-0.10.3.tgz#8eae1fc29c90a552e7921208f81892d6f77acb2b" + integrity sha512-pRuUdW0WwyB2doSqqjWyzwCD6PkfxpHAHdZp39K3dp/Hq7f+xfMwNAWIi16DyrRg4gg9c/RvLYkJTSawTPTm1w== + dependencies: + frac "~1.1.2" + stack-utils@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.1.tgz#d4f33ab54e8e38778b0ca5cfd3b3afb12db68620" @@ -4489,6 +4568,19 @@ xdg-basedir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4" +xlsx@^0.14.1: + version "0.14.5" + resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.14.5.tgz#3637e914d791bdca7382816e173f7d725ed0e0d2" + integrity sha512-s/5f4/mjeWREmIWZ+HtDfh/rnz51ar+dZ4LWKZU3u9VBx2zLdSIWTdXgoa52/pnZ9Oe/Vu1W1qzcKzLVe+lq4w== + dependencies: + adler-32 "~1.2.0" + cfb "^1.1.2" + codepage "~1.14.0" + commander "~2.17.1" + crc-32 "~1.2.0" + exit-on-epipe "~1.0.1" + ssf "~0.10.2" + xtend@^4.0.0, xtend@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"