update default sheet

Topic/xlsx

.gitignore

@@ -7,7 +7,10 @@
 
 .env
 *service-account-key\.json
-src/config.js
 /yarn-error.log
 *.pem
-.travis.yml.old
+.travis.yml.old
+tags
+tags.lock
+tags.temp
+src/config.js

.travis.yml

@@ -12,6 +12,3 @@ script:
 - yarn build
 - yarn lint
 - yarn test
-before_install:
-- openssl aes-256-cbc -K $encrypted_eeb74e8d35d1_key -iv $encrypted_eeb74e8d35d1_iv
-  -in .env.enc -out .env -d

Dockerfile

@@ -10,7 +10,7 @@ RUN cd /www; yarn
 COPY . /www
 WORKDIR /www
 RUN yarn build
-RUN mkdir -p temp
+RUN mkdir -p data
 
 # set your port
 ENV PORT 8080

LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2018 Forensic Architecture
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.md

@@ -0,0 +1,66 @@
+Do No Harm License
+
+**Preamble**
+
+Most software today is developed with little to no thought of how it will be used, or the consequences for our society and planet.
+
+As software developers, we engineer the infrastructure of the 21st century. We recognise that our infrastructure has great power to shape the world and the lives of those we share it with, and we choose to consciously take responsibility for the social and environmental impacts of what we build.
+
+We envisage a world free from injustice, inequality, and the reckless destruction of lives and our planet. We reject slavery in all its forms, whether by force, indebtedness, or by algorithms that hack human vulnerabilities. We seek a world where humankind is at peace with our neighbours, nature, and ourselves. We want our work to enrich the physical, mental and spiritual wellbeing of all society.
+
+We build software to further this vision of a just world, or at the very least, to not put that vision further from reach.
+
+**Terms**
+
+*Copyright* (c) 2019 Forensic Architecture. All rights reserved.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+4. This software must not be used by any organisation, website, product or service that:
+
+   a) lobbies for, promotes, or derives a majority of income from actions that support or contribute to:
+      * sex trafficking
+      * human trafficking
+      * slavery
+      * indentured servitude
+      * gambling
+      * tobacco
+      * adversely addictive behaviours
+      * nuclear energy
+      * warfare
+      * weapons manufacturing
+      * war crimes
+      * violence (except when required to protect public safety)
+      * burning of forests
+      * deforestation
+      * hate speech or discrimination based on age, gender, gender identity, race, sexuality, religion, nationality
+
+   b) lobbies against, or derives a majority of income from actions that discourage or frustrate:
+      * peace
+      * access to the rights set out in the Universal Declaration of Human Rights and the Convention on the Rights of the Child
+      * peaceful assembly and association (including worker associations)
+      * a safe environment or action to curtail the use of fossil fuels or prevent climate change
+      * democratic processes
+
+5. All redistribution of source code or binary form, including any modifications must be under these terms. You must inform recipients that the code is governed by these conditions, and how they can obtain a copy of this license. You may not attempt to alter the conditions of who may/may not use this software.
+
+We define:
+
+**Forests** to be 0.5 or more hectares of trees that were either planted more than 50 years ago or were not planted by humans or human made equipment.
+
+**Deforestation** to be the clearing, burning or destruction of 0.5 or more hectares of forests within a 1 year period.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+**Attribution**
+
+Do No Harm License [Contributor Covenant][homepage], (pre 1.0),
+available at https://github.com/raisely/NoHarm
+
+[homepage]: https://github.com/raisely/NoHarm
+

README.md

@@ -34,7 +34,7 @@ Datasheet server is a Node server developed at [Forensic Architecture](https://f
 
 Querying data directly from spreadsheets is brittle, as it relies on the maintenance of a rigid structure in the sheets at all times. By putting Datasheet Server as a proxy that sits in between source sheets and their consumers, it is possible to dynamically modify sheets without breaking applications. A data admin can then use Datasheet Server to ensure that applications always receive eligible data, without foregoing the spreadsheets as sources of truth.
 
-To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/Setting-up-a-local-instance-of-Timemap) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
+To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/running-timemap-and-datasheet-server-locally) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
 
 ### Design Concepts
 The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
@@ -76,7 +76,7 @@ Other configuration options, such as the port at which the server will expose
 resources, are also modifiable from the .env file.
 
 #### [Sources](#sources)
-Sources are specified in [src/sheets_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/sheets_config.js). Datasheet server currently only supports Google Sheets as a source.
+Sources are specified in [src/config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js). Datasheet server currently only supports Google Sheets as a source.
 
 ###### [Google Sheets](#source-google-sheets)
 | sheets | A list of objects, one for each sheet that is being used as a source. Each sheet object has a `name` (String), an `id` (String), and a `tabs` (object) field, which are explained below. | object |

data/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!timemap_data.xlsx

docs/architecture.md

@@ -8,10 +8,10 @@ management of the sheets it represents.
 It exposes an API with four endpoints:
  * `/:sheet/:tab/:resource` - the primary means of accessing data in the
      server. The name of a sheet is specified in
-     [sheets_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/sheets_config.js#L7).
+     [_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js#L7).
      Each tab on the sheet is in lower case, with spaces replaced by
      underscores. The resource name is the same as the name of the blueprinter
-     specified for the tab is sheets_config.js.
+     specified for the tab is config.js.
  * `/:sheet/:tab/:resource/:id` - when the resource is a list, items in the list
      can be accessed individually as well.
  * `/update` - when this route is queried with a GET request, each of the

package.json

@@ -4,19 +4,14 @@
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
   "scripts": {
-    "dev": "NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
-    "build": "NODE_ENV=production npx babel src -d dist",
+    "dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
+    "build": "env NODE_ENV=production npx babel src -d dist",
     "start": "node dist",
     "lint": "standard \"src/**/*.js\" \"test/**/*.js\"",
     "test-watch": "ava --watch",
     "test": "ava --verbose",
     "travis-encrypt": "./scripts/encrypt.sh"
   },
-  "husky": {
-    "hooks": {
-      "pre-push": "./scripts/check-branch.sh && yarn lint && yarn test"
-    }
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/developit/express-es6-rest-api.git"
@@ -30,11 +25,13 @@
     "dotenv": "^6.1.0",
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
+    "express-handlebars": "^4.0.4",
     "googleapis": "^32.0.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
     "node-fetch": "^2.3.0",
+    "node-xlsx": "^0.15.0",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
     "resource-router-middleware": "^0.6.0"
@@ -47,7 +44,6 @@
     "@babel/preset-env": "^7.1.0",
     "@babel/register": "^7.0.0",
     "ava": "1.0.0-beta.8",
-    "husky": "^1.2.0",
     "nodemon": "1.18.7",
     "standard": "^12.0.1"
   },

src/api/index.js

@@ -12,7 +12,14 @@ export default ({ config, controller }) => {
   })
 
   api.get('/blueprints', (req, res) => {
-    res.json(controller.blueprints())
+    const bps = controller.blueprints()
+    res.render('blueprints', {
+      bps: bps.map(bp => ({
+        source: bp.sheet.name,
+        tab: bp.name,
+        urls: bp.urls
+      }))
+    })
   })
 
   api.get('/update', (req, res) => {

src/blueprinters/deeprows.js

@@ -63,8 +63,9 @@ export default (data) => {
     structure.__flat.forEach(label => {
       deepRow[label] = baseRow[label]
     })
-
-    output.push(deepRow)
+    if (deepRow['id'] && deepRow['id'] !== '') {
+      output.push(deepRow)
+    }
   })
 
   return output

src/config.js

@@ -0,0 +1,12 @@
+import { timemap } from './lib'
+
+export default {
+  gsheets: [],
+  xlsx: [
+    {
+      name: 'timemap_data',
+      path: 'data/timemap_data.xlsx',
+      tabs: timemap.default
+    }
+  ]
+}

src/index.js

@@ -3,13 +3,18 @@ import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
 import api from './api'
-// import config from './sheets_config'
 import dotenv from 'dotenv'
+const hbs = require('express-handlebars')
 
 dotenv.config()
 
 let app = express()
 app.server = http.createServer(app)
+app.engine('.hbs', hbs({
+  extname: '.hbs',
+  defaultLayout: 'default'
+}))
+app.set('view engine', '.hbs')
 
 // enable cross origin requests explicitly in development
 if (process.env.NODE_ENV === 'development') {
@@ -35,8 +40,11 @@ initialize(controller => {
     })
   )
 
+  app.use(express.static(__dirname + '/public'))
+
   app.server.listen(process.env.PORT || 4040, () => {
-    console.log(`Started on port ${app.server.address().port}`)
+    console.log('===========================================')
+    console.log(`Server running on port ${app.server.address().port}`)
   })
 })
 

src/initialize.js

@@ -1,37 +1,37 @@
 import StoreJson from './models/StoreJson'
-import Fetcher from './lib/Fetcher'
+import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import sheetsConfig from './sheets_config'
+import config from './config'
+import R from 'ramda'
 
-const { googleSheets } = sheetsConfig
-const { sheets } = googleSheets
-
-function authenticate (_fetcher) {
-  return _fetcher.fetcher.authenticate(process.env.SERVICE_ACCOUNT_EMAIL, process.env.SERVICE_ACCOUNT_PRIVATE_KEY).then(msg => {
-    console.log(msg)
-    return true
-  })
-}
+const isntNull = n => n !== null
+const filterNull = ls => R.filter(isntNull, ls)
+const flattenfilterNull = ls => filterNull(R.flatten(ls))
+let themFetchers
 
 export default callback => {
-  const fetchers = sheets.map(sheet => {
-    return {
-      name: sheet.name,
-      fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs)
-    }
-  })
-
-  Promise.all(fetchers.map(authenticate))
-    .then(() => {
-      console.log(`===================`)
-      console.log(`grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
-      console.log(`===================`)
-
-      // NB: reformat fetchers as config for controller
-      const config = {}
-      fetchers.forEach(fetcher => {
-        config[fetcher.name] = fetcher.fetcher
+  return Promise.resolve().then(() => {
+    return Object.keys(config).map(fType => {
+      // skip config attrs that don't have corresponding fetchers
+      if (!(fType in fetchers)) return null
+      const FFetcher = fetchers[fType]
+      return config[fType].map(sheet => {
+        const otherArgs = { ...sheet }
+        delete otherArgs.name
+        delete otherArgs.tabs
+        return {
+          name: sheet.name,
+          fetcher: new FFetcher(new StoreJson(), sheet.name, sheet.tabs, ...Object.values(otherArgs))
+        }
       })
+    })
+  })
+    .then(res => {
+      themFetchers = flattenfilterNull(res)
+    })
+    .then(() => Promise.all(themFetchers.map(f => f.fetcher.authenticate(process.env))))
+    .then(fetchers => {
+      const config = R.zipObj(themFetchers.map(f => f.name), fetchers)
       const controller = new Controller(config)
       callback(controller)
     })

src/lib.js

@@ -0,0 +1,19 @@
+import BP from './lib/blueprinters'
+
+function prefixedTabs (prefix, cfg) {
+  if (!cfg) cfg = {}
+  const prf = key => cfg[key] ? `${prefix}_` : ''
+  return {
+    [`${prf('events')}export_events`]: BP.deeprows,
+    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
+    [`${prf('filters')}export_filters`]: BP.tree,
+    [`${prf('narratives')}export_narratives`]: BP.rows,
+    [`${prf('sources')}export_sources`]: BP.deepids,
+    [`${prf('sites')}export_sites`]: BP.rows
+  }
+}
+
+export const timemap = {
+  default: prefixedTabs(),
+  prefixedTabs
+}

src/lib/Controller.js

@@ -16,16 +16,22 @@ class Controller {
   blueprints () {
     return Object.keys(this.fetchers).map(
       sheet => this.fetchers[sheet].blueprints
-    )
+    ).reduce((acc, curr) => acc.concat(curr))
+  }
+
+  rebuildBlueprintsAsync () {
+    Object.values(this.fetchers).forEach(t => t._buildBlueprintsAsync())
   }
 
   update () {
+    const me = this
     return Promise.all(
       Object.keys(this.fetchers).map(sheet => {
         return this.fetchers[sheet].update()
       })
     ).then(results => {
       if (results.every(r => r)) {
+        me.rebuildBlueprintsAsync()
         return copy.success.update
       } else {
         throw new Error(copy.errors.update)

src/lib/Fetcher.js

@@ -8,38 +8,31 @@ import {
 } from './util'
 import { createHash } from 'crypto'
 import R from 'ramda'
+import xlsx from 'node-xlsx'
+import fs from 'fs'
 
 class Fetcher {
-  constructor (db, sheetName, sheetId, blueprinters) {
+  constructor (db, name, bps) {
     /*
      * The database that the fetcher should use. This should be an instance of a model-compliant class.
      * See models/Interface.js for the specifications for a model-compliant class.
      */
     this.db = db
-
-    /*
-     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
-     * loaded here must be added to the sheet as an editor.
-     */
-    this.sheetId = sheetId
-
     /*
      * The name of the sheet. This will prefix tabs saved in the database.
      */
-    this.sheetName = sheetName
-
+    this.sheetName = name
     /*
      * A unique ID for the Fetcher to identify its elements in the model layer
      */
-    this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex')
-
+    const bpsstring = Object.keys(bps).join(';')
+    this.id = createHash('md5').update(name).update(bpsstring).digest('hex')
     /*
      * These are the available tabs for storing and retrieving data.
      * Each blueprinter is a function that returns a Blueprint from a
      * list of lists (which will be retrieved from gsheets).
      */
-    this.blueprinters = fmtBlueprinterTitles(blueprinters)
-
+    this.blueprinters = fmtBlueprinterTitles(bps)
     /*
      * This object is the canonical represenation for the data that a Fetcher
      * proxies. When the fetcher is initialized, its model layer (db) is indexed,
@@ -49,13 +42,6 @@ class Fetcher {
      */
     this.blueprints = null
     this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion
-
-    /*
-     * Google API setup
-     */
-    this.API = google.sheets('v4')
-    this.auth = null
-
     /** curry to allow convenient syntax with map */
     this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter.bind(this))
   }
@@ -66,20 +52,33 @@ class Fetcher {
         const allParts = allUrls.reduce((acc, url) => {
           if (url.startsWith(this.id)) {
             const parts = url.split('/')
-            acc.push([ parts[1], parts[2] ])
-            return acc
-          } else {
-            return acc
+            let duplicateTab = acc.reduce((tabFound, p) => {
+              return tabFound || p[0] === parts[1]
+            }, false)
+            if (duplicateTab) {
+              acc.forEach(p => {
+                if (p[0] === parts[1]) {
+                  p[1].push(parts[2])
+                }
+              })
+            } else {
+              acc.push([ parts[1], [ parts[2] ] ])
+            }
           }
+          return acc
         }, [])
 
         return allParts
-          .map(parts => buildDesaturated(
-            this.sheetId,
-            this.sheetName,
-            parts[0],
-            parts[1]
-          ))
+          .map(parts => {
+            const bp = buildDesaturated(
+              this.sheetId,
+              this.sheetName,
+              parts[0],
+              parts[1]
+            )
+            bp.urls = Object.keys(bp.resources).map(r => `/api/${bp.sheet.name}/${bp.name}/${r}`)
+            return bp
+          })
       })
       .then(res => {
         this.blueprints = res
@@ -104,20 +103,81 @@ class Fetcher {
     )
   }
 
+  save (_tab, data) {
+    const tab = fmtName(_tab)
+
+    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
+      const bpConfig = this.blueprinters[tab]
+
+      if (isFunction(bpConfig)) {
+        // if bpConfig specifies a single blueprinter
+        return this._saveViaBlueprinter(tab, data, bpConfig)
+      } else {
+        // if bpConfig specifies an array of blueprinters
+        return bpConfig.map(this._saveViaBlueprinter(tab, data))
+      }
+    } else {
+      // NB: if a blueprinter is not specified for a tab,
+      // just skip it.
+      return true
+    }
+  }
+
+  // NB: could combine these functions by checking kwargs length
+  retrieve (tab, resource) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  retrieveFrag (tab, resource, frag) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  /** Run on startup. Should be overridden if explicit auth is required **/
+  authenticate (env) {
+    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
+    return Promise.resolve(this)
+  }
+}
+
+class GsheetFetcher extends Fetcher {
+  constructor (db, name, bps, sheetId) {
+    super(db, name, bps)
+    this.type = 'Google Sheet'
+    if (arguments.length < 4) throw Error('You must provide the sheet ID')
+
+    /*
+     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
+     * loaded here must be added to the sheet as an editor.
+     */
+    this.sheetId = sheetId
+    /*
+     * Google API setup
+     */
+    this.API = google.sheets('v4')
+    this.auth = null
+  }
+
   /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
-  authenticate (clientEmail, privateKey) {
-    const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [
+  authenticate (env) {
+    const googleAuth = new google.auth.JWT(env.SERVICE_ACCOUNT_EMAIL, null, env.SERVICE_ACCOUNT_PRIVATE_KEY, [
       'https://www.googleapis.com/auth/spreadsheets'
     ])
     this.auth = googleAuth
     const { sheetId } = this
+    const me = this
 
     return new Promise((resolve, reject) => {
       googleAuth.authorize(function (err) {
         if (err) {
           reject(err)
         } else {
-          resolve(`Connected to ${sheetId}.`)
+          console.log(`Connected to ${me.sheetName}. (${me.type} with ID ${sheetId}).`)
+          console.log(`    grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
+          resolve(me)
         }
       })
     })
@@ -159,39 +219,35 @@ class Fetcher {
       .then(() => true)
       .catch(() => false)
   }
+}
 
-  save (_tab, data) {
-    const tab = fmtName(_tab)
+class XlsxFetcher extends Fetcher {
+  constructor (db, name, bps, path) {
+    super(db, name, bps)
+    this.type = 'XLSX File'
+    this.path = path
+    this.isRemote = false
 
-    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
-      const bpConfig = this.blueprinters[tab]
-
-      if (isFunction(bpConfig)) {
-        // if bpConfig specifies a single blueprinter
-        return this._saveViaBlueprinter(tab, data, bpConfig)
-      } else {
-        // if bpConfig specifies an array of blueprinters
-        return bpConfig.map(this._saveViaBlueprinter(tab, data))
-      }
-    } else {
-      // NB: if a blueprinter is not specified for a tab,
-      // just skip it.
-      return true
+    if (this.path.startsWith('https')) {
+      this.isRemote = true
     }
   }
 
-  // NB: could combine these functions by checking kwargs length
-  retrieve (tab, resource) {
-    const title = fmtName(tab)
-    const url = `${this.id}/${tab}/${resource}`
-    return this.db.load(url, this.blueprints[title])
-  }
-
-  retrieveFrag (tab, resource, frag) {
-    const title = fmtName(tab)
-    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
-    return this.db.load(url, this.blueprints[title])
+  update () {
+    const data = xlsx.parse(fs.readFileSync(this.path))
+    data.forEach(tab => {
+      const stringyData = tab.data.map(row =>
+        row.map(d =>
+          typeof (d) === 'number' ? d.toString() : d
+        )
+      )
+      this.save(tab.name, stringyData)
+    })
+    return Promise.resolve(true)
   }
 }
 
-export default Fetcher
+export default {
+  'gsheets': GsheetFetcher,
+  'xlsx': XlsxFetcher
+}

src/lib/blueprinters.js

@@ -15,12 +15,15 @@ export const defaultResource = {
   data: []
 }
 
-export function buildDesaturated (sheetId, sheetName, tab, resource) {
+export function buildDesaturated (sheetId, sheetName, tab, resources) {
   const bp = R.clone(defaultBlueprint)
   bp.sheet.name = sheetName
   bp.sheet.id = sheetId
   bp.name = tab
-  bp.resources[resource] = null
+  bp.resources = resources.reduce((acc, r) => {
+    acc[r] = null
+    return acc
+  }, {})
   return bp
 }
 

src/models/StoreJson.js

@@ -1,7 +1,7 @@
 import fs from 'mz/fs'
 import errors from '../lib/errors'
 
-const STORAGE_DIRNAME = 'temp'
+const STORAGE_DIRNAME = 'data'
 
 function partsFromFilename (fname) {
   const body = fname.slice(0, -5)

src/sheets_config.js

@@ -1,20 +0,0 @@
-import BP from './lib/blueprinters'
-
-export default {
-  googleSheets: {
-    sheets: [
-      {
-        name: 'example',
-        id: '1UC7DkCFeUXHfpUxUGruExwFbP4pqVBdJLOKfo6wDDGk',
-        tabs: {
-          export_events: [BP.deeprows, BP.rows],
-          export_categories: [BP.groups, BP.rows],
-          export_narratives: BP.rows,
-          export_sources: BP.deepids,
-          export_sites: BP.rows,
-          export_tags: BP.tree
-        }
-      }
-    ]
-  }
-}

temp/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

views/blueprints.hbs

@@ -0,0 +1,91 @@
+<h1>Available Endpoints</h1>
+<hr>
+<div class="main-container">
+  {{#each bps}}
+  <div class="blueprint-container">
+    <div class="bp-header">
+      <div class="bp-tab">{{ tab }}</div>
+      <div class="bp-source">{{ source }}</div>
+    </div>
+    {{#each urls}}
+    <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
+    {{/each}}
+  </div>
+  {{ else }}
+  <div>No endpoints found. Have you updated?</div>
+  {{/each}}
+</div>
+
+<a class="bp-update-container" target="_blank" href="http://localhost:4040/api/update">
+  <div class="bp-button">Update</div>
+</div>
+
+
+<style>
+:root {
+  --grey: #8a8a8a;
+  --btn-width: 200px;
+}
+.main-container {
+  display: flex;
+  flex-wrap: wrap;
+}
+
+.blueprint-container {
+  display: flex;
+  flex: 1 0 31%;
+  flex-direction: column;
+  border: 1px solid var(--grey);
+  padding: 1em;
+  margin: 1em;
+}
+
+.bp-header {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+}
+
+.bp-tab {
+  font-size: 24pt;
+}
+
+.bp-source {
+  color: var(--grey);
+}
+
+.bp-update-container {
+  position: fixed;
+  top: 2px;
+  right: 2em;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+
+.bp-button {
+  transition: all 0.3s ease;
+  text-transform: uppercase;
+  font-size: 24pt;
+  font-weight: bold;
+  width: var(--btn-width);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  border: 3px solid black;
+  padding: .5em;
+  text-decoration: none;
+  text-underline: none;
+  color: black;
+}
+
+.bp-update-container:hover {
+  background-color: black;
+  transition: all 0.3s ease;
+  cursor: pointer;
+}
+
+.bp-update-container:hover div {
+  color: white;
+}
+</style>

views/layouts/default.hbs

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+  <title>Datasheet Server</title>
+  <meta name="HandheldFriendly" content="True" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
+</head>
+<body>
+  <div class="container">
+	  {{{body}}}
+  </div>
+</body>
+</html>