add xlsx support

by factoring out a Fetcher class
2026-06-08 03:18:33 +03:00 · 2020-03-27 12:30:21 +01:00
parent 2e343a17dd
commit 5d7eb0af05
7 changed files with 226 additions and 89 deletions
--- a/package.json
+++ b/package.json
@@ -35,6 +35,7 @@
    "morgan": "^1.8.0",
    "mz": "^2.7.0",
    "node-fetch": "^2.3.0",
+    "node-xlsx": "^0.15.0",
    "object-hash": "^1.3.0",
    "ramda": "^0.25.0",
    "resource-router-middleware": "^0.6.0"
--- a/src/api/index.js
+++ b/src/api/index.js
@@ -12,7 +12,12 @@ export default ({ config, controller }) => {
  })

  api.get('/blueprints', (req, res) => {
-    res.json(controller.blueprints())
+    const bps = controller.blueprints()
+    res.json(bps.map(bp => ({
+      source: bp.sheet.name,
+      tab: bp.name,
+      urls: bp.urls
+    })))
  })

  api.get('/update', (req, res) => {
--- a/src/index.js
+++ b/src/index.js
@@ -36,7 +36,8 @@ initialize(controller => {
  )

  app.server.listen(process.env.PORT || 4040, () => {
-    console.log(`Started on port ${app.server.address().port}`)
+    console.log('===========================================')
+    console.log(`Server running on port ${app.server.address().port}`)
  })
 })

--- a/src/initialize.js
+++ b/src/initialize.js
@@ -1,37 +1,32 @@
 import StoreJson from './models/StoreJson'
-import Fetcher from './lib/Fetcher'
+import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import sheetsConfig from './sheets_config'
+import config from './sheets_config'
+import R from 'ramda'

-const { googleSheets } = sheetsConfig
-const { sheets } = googleSheets
-
-function authenticate (_fetcher) {
-  return _fetcher.fetcher.authenticate(process.env.SERVICE_ACCOUNT_EMAIL, process.env.SERVICE_ACCOUNT_PRIVATE_KEY).then(msg => {
-    console.log(msg)
-    return true
-  })
-}
+const isntNull = n => n !== null
+const filterNull = ls => R.filter(isntNull, ls)
+const flattenfilterNull = ls => filterNull(R.flatten(ls))
+let themFetchers

 export default callback => {
-  const fetchers = sheets.map(sheet => {
-    return {
-      name: sheet.name,
-      fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs)
-    }
+  return Promise.resolve().then(() => {
+    return Object.keys(config).map(fType => {
+      // skip config attrs that don't have corresponding fetchers
+      if (!(fType in fetchers)) return null
+      const FFetcher = fetchers[fType]
+      return config[fType].map(sheet => ({
+        name: sheet.name,
+        fetcher: new FFetcher(new StoreJson(), ...Object.values(sheet))
+      }))
+    })
  })
-
-  Promise.all(fetchers.map(authenticate))
-    .then(() => {
-      console.log(`===================`)
-      console.log(`grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
-      console.log(`===================`)
-
-      // NB: reformat fetchers as config for controller
-      const config = {}
-      fetchers.forEach(fetcher => {
-        config[fetcher.name] = fetcher.fetcher
-      })
+    .then(res => {
+      themFetchers = flattenfilterNull(res)
+    })
+    .then(() => Promise.all(themFetchers.map(f => f.fetcher.authenticate(process.env))))
+    .then(fetchers => {
+      const config = R.zipObj(themFetchers.map(f => f.name), fetchers)
      const controller = new Controller(config)
      callback(controller)
    })
--- a/src/lib/Controller.js
+++ b/src/lib/Controller.js
@@ -22,6 +22,7 @@ class Controller {
  update () {
    return Promise.all(
      Object.keys(this.fetchers).map(sheet => {
+        console.log(sheet)
        return this.fetchers[sheet].update()
      })
    ).then(results => {
--- a/src/lib/Fetcher.js
+++ b/src/lib/Fetcher.js
@@ -8,38 +8,31 @@ import {
 } from './util'
 import { createHash } from 'crypto'
 import R from 'ramda'
+import xlsx from 'node-xlsx'
+import fs from 'fs'

 class Fetcher {
-  constructor (db, sheetName, sheetId, blueprinters) {
+  constructor (db, name, bps) {
    /*
     * The database that the fetcher should use. This should be an instance of a model-compliant class.
     * See models/Interface.js for the specifications for a model-compliant class.
     */
    this.db = db
-
-    /*
-     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
-     * loaded here must be added to the sheet as an editor.
-     */
-    this.sheetId = sheetId
-
    /*
     * The name of the sheet. This will prefix tabs saved in the database.
     */
-    this.sheetName = sheetName
-
+    this.sheetName = name
    /*
     * A unique ID for the Fetcher to identify its elements in the model layer
     */
-    this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex')
-
+    const bpsstring = Object.keys(bps).join(';')
+    this.id = createHash('md5').update(name).update(bpsstring).digest('hex')
    /*
     * These are the available tabs for storing and retrieving data.
     * Each blueprinter is a function that returns a Blueprint from a
     * list of lists (which will be retrieved from gsheets).
     */
-    this.blueprinters = fmtBlueprinterTitles(blueprinters)
-
+    this.blueprinters = fmtBlueprinterTitles(bps)
    /*
     * This object is the canonical represenation for the data that a Fetcher
     * proxies. When the fetcher is initialized, its model layer (db) is indexed,
@@ -49,13 +42,6 @@ class Fetcher {
     */
    this.blueprints = null
    this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion
-
-    /*
-     * Google API setup
-     */
-    this.API = google.sheets('v4')
-    this.auth = null
-
    /** curry to allow convenient syntax with map */
    this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter.bind(this))
  }
@@ -83,12 +69,16 @@ class Fetcher {
        }, [])

        return allParts
-          .map(parts => buildDesaturated(
-            this.sheetId,
-            this.sheetName,
-            parts[0],
-            parts[1]
-          ))
+          .map(parts => {
+            const bp = buildDesaturated(
+              this.sheetId,
+              this.sheetName,
+              parts[0],
+              parts[1]
+            )
+            bp.urls = Object.keys(bp.resources).map(r => `/api/${bp.sheet.name}/${bp.name}/${r}`)
+            return bp
+          })
      })
      .then(res => {
        this.blueprints = res
@@ -113,20 +103,81 @@ class Fetcher {
    )
  }

+  save (_tab, data) {
+    const tab = fmtName(_tab)
+
+    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
+      const bpConfig = this.blueprinters[tab]
+
+      if (isFunction(bpConfig)) {
+        // if bpConfig specifies a single blueprinter
+        return this._saveViaBlueprinter(tab, data, bpConfig)
+      } else {
+        // if bpConfig specifies an array of blueprinters
+        return bpConfig.map(this._saveViaBlueprinter(tab, data))
+      }
+    } else {
+      // NB: if a blueprinter is not specified for a tab,
+      // just skip it.
+      return true
+    }
+  }
+
+  // NB: could combine these functions by checking kwargs length
+  retrieve (tab, resource) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  retrieveFrag (tab, resource, frag) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  /** Run on startup. Should be overridden if explicit auth is required **/
+  authenticate (env) {
+    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
+    return Promise.resolve(this)
+  }
+}
+
+class GsheetFetcher extends Fetcher {
+  constructor (db, name, bps, sheetId) {
+    super(db, name, bps)
+    this.type = 'Google Sheet'
+    if (arguments.length < 4) throw Error('You must provide the sheet ID')
+
+    /*
+     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
+     * loaded here must be added to the sheet as an editor.
+     */
+    this.sheetId = sheetId
+    /*
+     * Google API setup
+     */
+    this.API = google.sheets('v4')
+    this.auth = null
+  }
+
  /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
-  authenticate (clientEmail, privateKey) {
-    const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [
+  authenticate (env) {
+    const googleAuth = new google.auth.JWT(env.SERVICE_ACCOUNT_EMAIL, null, env.SERVICE_ACCOUNT_PRIVATE_KEY, [
      'https://www.googleapis.com/auth/spreadsheets'
    ])
    this.auth = googleAuth
    const { sheetId } = this
+    const me = this

    return new Promise((resolve, reject) => {
      googleAuth.authorize(function (err) {
        if (err) {
          reject(err)
        } else {
-          resolve(`Connected to ${sheetId}.`)
+          console.log(`Connected to ${me.sheetName}. (${me.type} with ID ${sheetId}).`)
+          console.log(`    grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
+          resolve(me)
        }
      })
    })
@@ -168,39 +219,30 @@ class Fetcher {
      .then(() => true)
      .catch(() => false)
  }
+}

-  save (_tab, data) {
-    const tab = fmtName(_tab)
+class XlsxFetcher extends Fetcher {
+  constructor (db, name, bps, path) {
+    super(db, name, bps)
+    this.type = 'XLSX File'
+    this.path = path
+    this.isRemote = false

-    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
-      const bpConfig = this.blueprinters[tab]
-
-      if (isFunction(bpConfig)) {
-        // if bpConfig specifies a single blueprinter
-        return this._saveViaBlueprinter(tab, data, bpConfig)
-      } else {
-        // if bpConfig specifies an array of blueprinters
-        return bpConfig.map(this._saveViaBlueprinter(tab, data))
-      }
-    } else {
-      // NB: if a blueprinter is not specified for a tab,
-      // just skip it.
-      return true
+    if (this.path.startsWith('https')) {
+      this.isRemote = true
    }
  }

-  // NB: could combine these functions by checking kwargs length
-  retrieve (tab, resource) {
-    const title = fmtName(tab)
-    const url = `${this.id}/${tab}/${resource}`
-    return this.db.load(url, this.blueprints[title])
-  }
-
-  retrieveFrag (tab, resource, frag) {
-    const title = fmtName(tab)
-    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
-    return this.db.load(url, this.blueprints[title])
+  update () {
+    const data = xlsx.parse(fs.readFileSync(this.path))
+    data.forEach(tab => {
+      this.save(tab.name, tab.data)
+    })
+    return Promise.resolve(true)
  }
 }

-export default Fetcher
+export default {
+  'gsheets': GsheetFetcher,
+  'xlsx': XlsxFetcher
+}
--- a/yarn.lock
+++ b/yarn.lock
@@ -621,6 +621,14 @@ acorn@^6.0.2:
  version "6.4.1"
  resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474"

+adler-32@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25"
+  integrity sha1-aj5r8KY5ALoVZSgIyxXGgT0aXyU=
+  dependencies:
+    exit-on-epipe "~1.0.1"
+    printj "~1.1.0"
+
 ajv-keywords@^3.0.0:
  version "3.2.0"
  resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.2.0.tgz#e86b819c602cf8821ad637413698f1dec021847a"
@@ -957,7 +965,7 @@ buffer-equal-constant-time@1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"

-buffer-from@^1.0.0:
+buffer-from@^1.0.0, buffer-from@^1.1.0:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"

@@ -1042,6 +1050,16 @@ capture-stack-trace@^1.0.0:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz#a6c0bbe1f38f3aa0b92238ecb6ff42c344d4135d"

+cfb@^1.1.2:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.1.4.tgz#81fd35ede4c919d8f0962a94582e1dfaf7051e2a"
+  integrity sha512-rwFkl3aFO3f+ljR27YINwC0x8vPjyiEVbYbrTCKzspEf7Q++3THdfHVgJYNUbxNcupJECrLX+L40Mjm9hm/Bgw==
+  dependencies:
+    adler-32 "~1.2.0"
+    commander "^2.16.0"
+    crc-32 "~1.2.0"
+    printj "~1.1.2"
+
 chalk@^1.1.3:
  version "1.1.3"
  resolved "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -1155,6 +1173,14 @@ code-point-at@^1.0.0:
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"

+codepage@~1.14.0:
+  version "1.14.0"
+  resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99"
+  integrity sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k=
+  dependencies:
+    commander "~2.14.1"
+    exit-on-epipe "~1.0.1"
+
 collection-visit@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0"
@@ -1172,10 +1198,25 @@ color-name@1.1.3:
  version "1.1.3"
  resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"

+commander@^2.16.0:
+  version "2.20.3"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
+  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
+
 commander@^2.8.1:
  version "2.19.0"
  resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"

+commander@~2.14.1:
+  version "2.14.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa"
+  integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw==
+
+commander@~2.17.1:
+  version "2.17.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
+  integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==
+
 common-path-prefix@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/common-path-prefix/-/common-path-prefix-1.0.0.tgz#cd52f6f0712e0baab97d6f9732874f22f47752c0"
@@ -1299,6 +1340,14 @@ cosmiconfig@^5.0.6:
    js-yaml "^3.9.0"
    parse-json "^4.0.0"

+crc-32@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
+  integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
+  dependencies:
+    exit-on-epipe "~1.0.1"
+    printj "~1.1.0"
+
 create-error-class@^3.0.0:
  version "3.0.2"
  resolved "https://registry.yarnpkg.com/create-error-class/-/create-error-class-3.0.2.tgz#06be7abef947a3f14a30fd610671d401bca8b7b6"
@@ -1790,6 +1839,11 @@ execa@^1.0.0:
    signal-exit "^3.0.0"
    strip-eof "^1.0.0"

+exit-on-epipe@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692"
+  integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==
+
 expand-brackets@^2.1.4:
  version "2.1.4"
  resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622"
@@ -1988,6 +2042,11 @@ forwarded@~0.1.2:
  version "0.1.2"
  resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"

+frac@~1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b"
+  integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==
+
 fragment-cache@^0.2.1:
  version "0.2.1"
  resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19"
@@ -3120,6 +3179,14 @@ node-releases@^1.0.0-alpha.14:
  dependencies:
    semver "^5.3.0"

+node-xlsx@^0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/node-xlsx/-/node-xlsx-0.15.0.tgz#1f1b0d7adce5c706e86bfd96a5aa0005bf8a9dc3"
+  integrity sha512-rQyhWDJ/k60wQemov7a8MlToastWTidrAVFRwTWV+s53LN/SRwU4lnmc5xuFXx/ay+uaLAsAQBp6BkVob5OjOA==
+  dependencies:
+    buffer-from "^1.1.0"
+    xlsx "^0.14.1"
+
 nodemon@1.18.7:
  version "1.18.7"
  resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-1.18.7.tgz#716b66bf3e89ac4fcfb38a9e61887a03fc82efbb"
@@ -3523,6 +3590,11 @@ pretty-ms@^3.2.0:
  dependencies:
    parse-ms "^1.0.0"

+printj@~1.1.0, printj@~1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222"
+  integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==
+
 private@^0.1.6:
  version "0.1.8"
  resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
@@ -4039,6 +4111,13 @@ sprintf-js@~1.0.2:
  version "1.0.3"
  resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"

+ssf@~0.10.2:
+  version "0.10.3"
+  resolved "https://registry.yarnpkg.com/ssf/-/ssf-0.10.3.tgz#8eae1fc29c90a552e7921208f81892d6f77acb2b"
+  integrity sha512-pRuUdW0WwyB2doSqqjWyzwCD6PkfxpHAHdZp39K3dp/Hq7f+xfMwNAWIi16DyrRg4gg9c/RvLYkJTSawTPTm1w==
+  dependencies:
+    frac "~1.1.2"
+
 stack-utils@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.1.tgz#d4f33ab54e8e38778b0ca5cfd3b3afb12db68620"
@@ -4489,6 +4568,19 @@ xdg-basedir@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"

+xlsx@^0.14.1:
+  version "0.14.5"
+  resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.14.5.tgz#3637e914d791bdca7382816e173f7d725ed0e0d2"
+  integrity sha512-s/5f4/mjeWREmIWZ+HtDfh/rnz51ar+dZ4LWKZU3u9VBx2zLdSIWTdXgoa52/pnZ9Oe/Vu1W1qzcKzLVe+lq4w==
+  dependencies:
+    adler-32 "~1.2.0"
+    cfb "^1.1.2"
+    codepage "~1.14.0"
+    commander "~2.17.1"
+    crc-32 "~1.2.0"
+    exit-on-epipe "~1.0.1"
+    ssf "~0.10.2"
+
 xtend@^4.0.0, xtend@^4.0.1:
  version "4.0.1"
  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"