Merge branch 'develop' of https://www.github.com/forensic-architecture/datasheet-server into feature/json-api-export

Adding the correct branch to checkout with for workflow

.github/workflows/cd.yml

@@ -0,0 +1,19 @@
+name: CD
+on:
+  push:
+    branches: [ develop ]
+#  pull_request:
+#    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger CD build
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.CI_DISPATCH_TOKEN }}
+          repository: forensic-architecture/configs
+          event-type: remote-build
+          client-payload: '{"runtime_args": "datasheet", "branch": "${GITHUB_REF##*/}"}'
+

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '12'
+
+      - run: npm install
+      - run: npm test
+        env:
+          CI: true
+      - run: npm run lint
+        env:
+          CI: true

.gitignore

@@ -7,7 +7,11 @@
 
 .env
 *service-account-key\.json
-src/config.js
 /yarn-error.log
 *.pem
-.travis.yml.old
+.travis.yml.old
+tags
+tags.lock
+tags.temp
+src/config.js
+src/local.config.js

.travis.yml

@@ -1,17 +0,0 @@
-language: node_js
-node_js:
-- stable
-cache:
-  directories:
-  - node_modules
-before_script:
-- npm install -g yarn
-install:
-- yarn
-script:
-- yarn build
-- yarn lint
-- yarn test
-before_install:
-- openssl aes-256-cbc -K $encrypted_eeb74e8d35d1_key -iv $encrypted_eeb74e8d35d1_iv
-  -in .env.enc -out .env -d

Dockerfile

@@ -10,11 +10,11 @@ RUN cd /www; yarn
 COPY . /www
 WORKDIR /www
 RUN yarn build
-RUN mkdir -p temp
+RUN mkdir -p data
 
 # set your port
-ENV PORT 8080
+ENV PORT 4040
-EXPOSE  8080
+EXPOSE 4040
 
 # start command as per package.json
 CMD ["yarn", "start"]

LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2018 Forensic Architecture
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.md

@@ -0,0 +1,66 @@
+Do No Harm License
+
+**Preamble**
+
+Most software today is developed with little to no thought of how it will be used, or the consequences for our society and planet.
+
+As software developers, we engineer the infrastructure of the 21st century. We recognise that our infrastructure has great power to shape the world and the lives of those we share it with, and we choose to consciously take responsibility for the social and environmental impacts of what we build.
+
+We envisage a world free from injustice, inequality, and the reckless destruction of lives and our planet. We reject slavery in all its forms, whether by force, indebtedness, or by algorithms that hack human vulnerabilities. We seek a world where humankind is at peace with our neighbours, nature, and ourselves. We want our work to enrich the physical, mental and spiritual wellbeing of all society.
+
+We build software to further this vision of a just world, or at the very least, to not put that vision further from reach.
+
+**Terms**
+
+*Copyright* (c) 2019 Forensic Architecture. All rights reserved.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+4. This software must not be used by any organisation, website, product or service that:
+
+   a) lobbies for, promotes, or derives a majority of income from actions that support or contribute to:
+      * sex trafficking
+      * human trafficking
+      * slavery
+      * indentured servitude
+      * gambling
+      * tobacco
+      * adversely addictive behaviours
+      * nuclear energy
+      * warfare
+      * weapons manufacturing
+      * war crimes
+      * violence (except when required to protect public safety)
+      * burning of forests
+      * deforestation
+      * hate speech or discrimination based on age, gender, gender identity, race, sexuality, religion, nationality
+
+   b) lobbies against, or derives a majority of income from actions that discourage or frustrate:
+      * peace
+      * access to the rights set out in the Universal Declaration of Human Rights and the Convention on the Rights of the Child
+      * peaceful assembly and association (including worker associations)
+      * a safe environment or action to curtail the use of fossil fuels or prevent climate change
+      * democratic processes
+
+5. All redistribution of source code or binary form, including any modifications must be under these terms. You must inform recipients that the code is governed by these conditions, and how they can obtain a copy of this license. You may not attempt to alter the conditions of who may/may not use this software.
+
+We define:
+
+**Forests** to be 0.5 or more hectares of trees that were either planted more than 50 years ago or were not planted by humans or human made equipment.
+
+**Deforestation** to be the clearing, burning or destruction of 0.5 or more hectares of forests within a 1 year period.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+**Attribution**
+
+Do No Harm License [Contributor Covenant][homepage], (pre 1.0),
+available at https://github.com/raisely/NoHarm
+
+[homepage]: https://github.com/raisely/NoHarm
+

README.md

@@ -34,49 +34,10 @@ Datasheet server is a Node server developed at [Forensic Architecture](https://f
 
 Querying data directly from spreadsheets is brittle, as it relies on the maintenance of a rigid structure in the sheets at all times. By putting Datasheet Server as a proxy that sits in between source sheets and their consumers, it is possible to dynamically modify sheets without breaking applications. A data admin can then use Datasheet Server to ensure that applications always receive eligible data, without foregoing the spreadsheets as sources of truth.
 
-To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/Setting-up-a-local-instance-of-Timemap) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
+To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/running-timemap-and-datasheet-server-locally) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
-
-### Design Concepts
-The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
-
-**Sources**     
-A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
-
-**Resources**     
-The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
-
-**Blueprints**       
-Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
-
-Blueprints are JSON objects. There have two forms:
-
-1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
-2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
-
-A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
-
-A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
-
-## [Configuration](#configuration)
-Copy the example environment file:
-```
-cp .env.example .env
-```
-Inside this file, you will need to modify at least the `SERVICE_ACCOUNT_EMAIL` and `SERVICE_ACCOUNT_PRIVATE_KEY` fields. These fields refer to the credentials of a [Google service account](https://cloud.google.com/iam/docs/understanding-service-accounts). Google requires that developers create these when attempting to access their services programmatically, so that they can attribute requests to users. Service accounts also contain identity information, which means that asset owners can allow certain service accounts access to certain sheets, just as one might differentially grant certain users access to certain cloud assets. 
-
-Once you have [created a service account](https://support.google.com/a/answer/7378726?hl=en), create and download an [API key for that account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). The JSON file for the API key that you download when you create it contains both a service account private key, and an email associated with the service account: add these respectively in the strings in .env for `SERVICE_ACCOUNT_PRIVATE_KEY` and `SERVICE_ACCOUNT_EMAIL`.
-
-The last thing to do is to grant the service account access to the sheet that
-datasheet-server is pulling from. You can add a service account to a sheet as
-you would any other Google user: just enter the email address associated. (Note
-that this step is not necessary if you are accessing a publicly available
-sheet.)
-
-Other configuration options, such as the port at which the server will expose
-resources, are also modifiable from the .env file.
 
 #### [Sources](#sources)
-Sources are specified in [src/sheets_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/sheets_config.js). Datasheet server currently only supports Google Sheets as a source.
+Sources are specified in [src/config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js). Datasheet server currently only supports Google Sheets as a source.
 
 ###### [Google Sheets](#source-google-sheets)
 | sheets | A list of objects, one for each sheet that is being used as a source. Each sheet object has a `name` (String), an `id` (String), and a `tabs` (object) field, which are explained below. | object |
@@ -86,34 +47,13 @@ Each Google Sheet being used as a as source requires a corresponding object in `
 | Option | Description | Type |
 | ------ | ----------- | ---- |
 | name | Used to refer to data served from this source | string |
-| id | The ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
+| path/id | The path to the XLSX sheet on your local, or the ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
 | tabs | An object that maps each tab in the source to one or more Blueprinters. All of the Blueprinters in the [blueprinters folder](/lib/blueprinters) are available through a single import as at the top of [example.config.js](/src/example.config.js). <br> To correctly associate a Blueprinter, the object key needs to be _the tab name with all lowercase letters, and spaces replaced by a '-'_. For example, if the tab name in Google Sheets is 'Info About SHEEP', the object key should be 'info-about-sheep'. <br> The value should be the Blueprinter function that you want to use for the data in that tab. If you require more than one endpoint for a single tab, you can support multiple blueprinters by making the item an array. See the example of a configuration object below. <br>TODO: no Blueprinter is used by default. | object |
 
-###### Example Configuration Object
+See src/config.js for an example configuration sheet. 
-```js
-import BP from './lib/blueprinters'
-
-export default {
-  googleSheets: {
-    email: 'project-name@reliable-baptist-23338.iam.gserviceaccount.com',
-    privateKey: 'SOME_PRIVATE_KEY',
-    sheets: [
-      {
-        name: 'example',
-        id: '1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI',
-        tabs: {
-          'objects': BP.rows,
-          'fruit': [BP.columns, BP.ids],
-        }
-      },
-    ]
-  }
-}
-
-```
 
 
-## [Quickstart](#quickstart)
+## [Quickstart](#quickstart)
 Clone the repository to your local:
 ```
 git clone https://www.github.com/forensic-architecture/datasheet-server

data/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!timemap_data.xlsx

docs/architecture.md

@@ -8,10 +8,10 @@ management of the sheets it represents.
 It exposes an API with four endpoints:
  * `/:sheet/:tab/:resource` - the primary means of accessing data in the
      server. The name of a sheet is specified in
-     [sheets_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/sheets_config.js#L7).
+     [_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js#L7).
      Each tab on the sheet is in lower case, with spaces replaced by
      underscores. The resource name is the same as the name of the blueprinter
-     specified for the tab is sheets_config.js.
+     specified for the tab is config.js.
  * `/:sheet/:tab/:resource/:id` - when the resource is a list, items in the list
      can be accessed individually as well.
  * `/update` - when this route is queried with a GET request, each of the

docs/design-concepts.md

@@ -0,0 +1,22 @@
+### Design Concepts
+The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
+
+**Sources**     
+A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
+
+**Resources**     
+The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
+
+**Blueprints**       
+Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
+
+Blueprints are JSON objects. There have two forms:
+
+1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
+2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
+
+A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
+
+A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
+
+

docs/gsheet-config.md

@@ -0,0 +1,19 @@
+## [Configuration](#configuration)
+Copy the example environment file:
+```
+cp .env.example .env
+```
+Inside this file, you will need to modify at least the `SERVICE_ACCOUNT_EMAIL` and `SERVICE_ACCOUNT_PRIVATE_KEY` fields. These fields refer to the credentials of a [Google service account](https://cloud.google.com/iam/docs/understanding-service-accounts). Google requires that developers create these when attempting to access their services programmatically, so that they can attribute requests to users. Service accounts also contain identity information, which means that asset owners can allow certain service accounts access to certain sheets, just as one might differentially grant certain users access to certain cloud assets. 
+
+Once you have [created a service account](https://support.google.com/a/answer/7378726?hl=en), create and download an [API key for that account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). The JSON file for the API key that you download when you create it contains both a service account private key, and an email associated with the service account: add these respectively in the strings in .env for `SERVICE_ACCOUNT_PRIVATE_KEY` and `SERVICE_ACCOUNT_EMAIL`.
+
+The last thing to do is to grant the service account access to the sheet that
+datasheet-server is pulling from. You can add a service account to a sheet as
+you would any other Google user: just enter the email address associated. (Note
+that this step is not necessary if you are accessing a publicly available
+sheet.)
+
+Other configuration options, such as the port at which the server will expose
+resources, are also modifiable from the .env file.
+
+

package.json

@@ -3,20 +3,16 @@
   "version": "0.3.0",
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
+  "type": "module",
   "scripts": {
-    "dev": "NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
+    "dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
-    "build": "NODE_ENV=production npx babel src -d dist",
+    "build": "env NODE_ENV=production npx babel src -d dist",
     "start": "node dist",
     "lint": "standard \"src/**/*.js\" \"test/**/*.js\"",
     "test-watch": "ava --watch",
     "test": "ava --verbose",
     "travis-encrypt": "./scripts/encrypt.sh"
   },
-  "husky": {
-    "hooks": {
-      "pre-push": "./scripts/check-branch.sh && yarn lint && yarn test"
-    }
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/developit/express-es6-rest-api.git"
@@ -30,14 +26,17 @@
     "dotenv": "^6.1.0",
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
-    "googleapis": "^32.0.0",
+    "express-handlebars": "^4.0.4",
+    "file-system": "^2.2.2",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.3.0",
+    "node-fetch": "^2.6.1",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",
@@ -47,7 +46,6 @@
     "@babel/preset-env": "^7.1.0",
     "@babel/register": "^7.0.0",
     "ava": "1.0.0-beta.8",
-    "husky": "^1.2.0",
     "nodemon": "1.18.7",
     "standard": "^12.0.1"
   },

scripts/check-branch.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-ENCRYPT_MESSAGE="\nThe .env.enc file has not changed its contents from the version on origin/develop.\nTo ensure that tests pass in Travis, you need to encrypt the contents of .env  before pushing to the remote, so that the Travis server can use your service account credentials.\nEnsure that the Travis CLI is installed on your local, run\n\n\tnpm run travis-encrypt\n\nand then push to the remote again.\nIf you don't care whether your build passes on Travis, you can run:\n\n\tgit push --no-verify\n\nand bypass this check.\n\n"
-
-# check whether .env.enc has changed
-if [ -z "`git diff origin/develop -- .env.enc`" ]; then
-	echo $ENCRYPT_MESSAGE
-	exit 1
-fi
-
-

scripts/encrypt.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-echo "Encrypting .env file for Travis..."
-
-# confirm that the user has the core repo as origin, and is therefore a maintainer
-# if [ -z `git config --get remote.origin.url | grep "forensic-architecture/datasheet-server"` ]; then
-# 	echo "Travis encryption not required for satellite contributors, continuing.."
-# 	exit 0
-# fi
-
-# confirm travis is installed
-if [ ! hash travis 2>/dev/null ]; then
-	echo "============================================================================================"
-	echo "ERROR: Travis CLI is not installed on your local. Please install from:"
-	echo "\thttps://github.com/travis-ci/travis.rb"
-	echo "After installing, make sure that you login with:"
-	echo "\ttravis login --pro"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# confirm there is a .env file to encrypt
-if [ ! -f .env ]; then
-	echo "============================================================================================"
-	echo "ERROR: You must create a .env file and add your credentials. See .env.example for an example"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# regex to match and delete 'before_install' and everything after it
-# necessary to delete these lines to get Travis to build for multiple accounts
-echo "creating new .travis.yml configuration"
-sed -i.old '/^before_install.*/,$ d' .travis.yml
-echo "old config file saved as .travis.yml.old"
-
-travis encrypt-file .env --add --force --org
-git add .env.enc
-git add .travis.yml
-echo ".env.enc created and added to commit"

src/api/index.js

@@ -4,6 +4,7 @@ import copy from '../copy/en'
 
 export default ({ config, controller }) => {
   let api = Router()
+  const fileDest = config.EXPORT_FILE_DEST || null
 
   api.get('/', (req, res) => {
     res.json({
@@ -12,7 +13,28 @@ export default ({ config, controller }) => {
   })
 
   api.get('/blueprints', (req, res) => {
-    res.json(controller.blueprints())
+    const bps = controller.blueprints()
+    res.render('blueprints', {
+      bps: bps.map(bp => ({
+        source: bp.sheet.name,
+        tab: bp.name,
+        urls: bp.urls
+      }))
+    })
+  })
+
+  api.get('/export', (req, res) => {
+    controller
+      .retrieveAll(fileDest)
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
   })
 
   api.get('/update', (req, res) => {

src/blueprinters/deeprows.js

@@ -63,8 +63,9 @@ export default (data) => {
     structure.__flat.forEach(label => {
       deepRow[label] = baseRow[label]
     })
-
+    if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {
-    output.push(deepRow)
+      output.push(deepRow)
+    }
   })
 
   return output

src/config.js

@@ -0,0 +1,12 @@
+import { timemap } from './lib'
+
+export default {
+  gsheets: [],
+  xlsx: [
+    {
+      name: 'timemap_data',
+      path: 'data/timemap_data.xlsx',
+      tabs: timemap.default
+    }
+  ]
+}

src/copy/en.js

@@ -1,6 +1,10 @@
 export default {
   errors: {
     update: 'The server could not update. Check your API credentials and internet connection and try again.',
+    export: {
+      fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
+      writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
+    },
     onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
@@ -9,6 +13,7 @@ export default {
     modelLayer: prts => `Something went wrong at the model layer`
   },
   success: {
-    update: 'All sheets updated'
+    update: 'All sheets updated',
+    export: dest => `All resources exported to the file: ${dest}`
   }
 }

src/index.js

@@ -1,15 +1,21 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
 import api from './api'
-// import config from './sheets_config'
 import dotenv from 'dotenv'
+const hbs = require('express-handlebars')
 
 dotenv.config()
 
 let app = express()
 app.server = http.createServer(app)
+app.engine('.hbs', hbs({
+  extname: '.hbs',
+  defaultLayout: 'default'
+}))
+app.set('view engine', '.hbs')
 
 // enable cross origin requests explicitly in development
 if (process.env.NODE_ENV === 'development') {
@@ -35,8 +41,11 @@ initialize(controller => {
     })
   )
 
+  app.use(express.static(path.join(__dirname, 'public')))
+
   app.server.listen(process.env.PORT || 4040, () => {
-    console.log(`Started on port ${app.server.address().port}`)
+    console.log('===========================================')
+    console.log(`Server running on port ${app.server.address().port}`)
   })
 })
 

src/initialize.js

@@ -1,37 +1,43 @@
 import StoreJson from './models/StoreJson'
-import Fetcher from './lib/Fetcher'
+import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import sheetsConfig from './sheets_config'
+import R from 'ramda'
 
-const { googleSheets } = sheetsConfig
+const isntNull = n => n !== null
-const { sheets } = googleSheets
+const filterNull = ls => R.filter(isntNull, ls)
+const flattenfilterNull = ls => filterNull(R.flatten(ls))
+let themFetchers
 
-function authenticate (_fetcher) {
+let config
-  return _fetcher.fetcher.authenticate(process.env.SERVICE_ACCOUNT_EMAIL, process.env.SERVICE_ACCOUNT_PRIVATE_KEY).then(msg => {
+try {
-    console.log(msg)
+  config = require('./local.config.js').default
-    return true
+} catch (_) {
-  })
+  config = require('./config.js').default
 }
 
 export default callback => {
-  const fetchers = sheets.map(sheet => {
+  return Promise.resolve().then(() => {
-    return {
+    return Object.keys(config).map(fType => {
-      name: sheet.name,
+      // skip config attrs that don't have corresponding fetchers
-      fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs)
+      if (!(fType in fetchers)) return null
-    }
+      const FFetcher = fetchers[fType]
-  })
+      return config[fType].map(sheet => {
-
+        const otherArgs = { ...sheet }
-  Promise.all(fetchers.map(authenticate))
+        delete otherArgs.name
-    .then(() => {
+        delete otherArgs.tabs
-      console.log(`===================`)
+        return {
-      console.log(`grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
+          name: sheet.name,
-      console.log(`===================`)
+          fetcher: new FFetcher(new StoreJson(), sheet.name, sheet.tabs, ...Object.values(otherArgs))
-
+        }
-      // NB: reformat fetchers as config for controller
-      const config = {}
-      fetchers.forEach(fetcher => {
-        config[fetcher.name] = fetcher.fetcher
       })
+    })
+  })
+    .then(res => {
+      themFetchers = flattenfilterNull(res)
+    })
+    .then(() => Promise.all(themFetchers.map(f => f.fetcher.authenticate(process.env))))
+    .then(fetchers => {
+      const config = R.zipObj(themFetchers.map(f => f.name), fetchers)
       const controller = new Controller(config)
       callback(controller)
     })

src/lib.js

@@ -0,0 +1,17 @@
+import BP from './lib/blueprinters'
+
+function prefixedTabs (prefix, cfg) {
+  if (!cfg) cfg = {}
+  const prf = key => cfg[key] ? `${prefix}_` : ''
+  return {
+    [`${prf('events')}export_events`]: BP.deeprows,
+    [`${prf('associations')}export_associations`]: BP.deeprows,
+    [`${prf('sources')}export_sources`]: BP.deepids,
+    [`${prf('sites')}export_sites`]: BP.rows
+  }
+}
+
+export const timemap = {
+  default: prefixedTabs(),
+  prefixedTabs
+}

src/lib/Controller.js

@@ -1,4 +1,5 @@
 import copy from '../copy/en'
+import { exportToFile } from '../lib/util'
 
 /**
  * Controller
@@ -16,16 +17,22 @@ class Controller {
   blueprints () {
     return Object.keys(this.fetchers).map(
       sheet => this.fetchers[sheet].blueprints
-    )
+    ).reduce((acc, curr) => acc.concat(curr))
+  }
+
+  rebuildBlueprintsAsync () {
+    Object.values(this.fetchers).forEach(t => t._buildBlueprintsAsync())
   }
 
   update () {
+    const me = this
     return Promise.all(
       Object.keys(this.fetchers).map(sheet => {
         return this.fetchers[sheet].update()
       })
     ).then(results => {
       if (results.every(r => r)) {
+        me.rebuildBlueprintsAsync()
         return copy.success.update
       } else {
         throw new Error(copy.errors.update)
@@ -33,6 +40,37 @@ class Controller {
     })
   }
 
+  // Controller function to retrieve all blueprints and export to user defined file location
+  retrieveAll (fileDest) {
+    if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
+
+    const indexedData = {}
+    const urls = []
+
+    const bps = this.blueprints()
+    return Promise.all(
+      bps.map(bp => {
+        const resource = Object.keys(bp.resources)[0]
+        urls.push(bp.urls[0])
+        return this.retrieve(bp.sheet.name, bp.name, resource)
+      })
+    ).then(async results => {
+      if (results.every(res => res)) {
+        urls.forEach((item, idx) => {
+          indexedData[item] = results[idx]
+        })
+        try {
+          const message = await exportToFile(fileDest, indexedData)
+          return message
+        } catch (e) {
+          return Promise.reject(e)
+        }
+      } else {
+        throw new Error(copy.errors.export.writeFailed)
+      }
+    })
+  }
+
   retrieve (sheet, tab, resource) {
     if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]

src/lib/Fetcher.js

@@ -1,45 +1,39 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
+import R from 'ramda'
-import { google } from 'googleapis'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
+
-import R from 'ramda'
+/* GsheetFetcher deps */
+import { google } from 'googleapis'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
-  constructor (db, sheetName, sheetId, blueprinters) {
+  constructor (db, name, bps) {
     /*
      * The database that the fetcher should use. This should be an instance of a model-compliant class.
      * See models/Interface.js for the specifications for a model-compliant class.
      */
     this.db = db
-
-    /*
-     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
-     * loaded here must be added to the sheet as an editor.
-     */
-    this.sheetId = sheetId
-
     /*
      * The name of the sheet. This will prefix tabs saved in the database.
      */
-    this.sheetName = sheetName
+    this.sheetName = name
-
     /*
      * A unique ID for the Fetcher to identify its elements in the model layer
      */
-    this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex')
+    const bpsstring = Object.keys(bps).join(';')
-
+    this.id = createHash('md5').update(name).update(bpsstring).digest('hex')
     /*
      * These are the available tabs for storing and retrieving data.
      * Each blueprinter is a function that returns a Blueprint from a
      * list of lists (which will be retrieved from gsheets).
      */
-    this.blueprinters = fmtBlueprinterTitles(blueprinters)
+    this.blueprinters = fmtBlueprinterTitles(bps)
-
     /*
      * This object is the canonical represenation for the data that a Fetcher
      * proxies. When the fetcher is initialized, its model layer (db) is indexed,
@@ -49,13 +43,6 @@ class Fetcher {
      */
     this.blueprints = null
     this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion
-
-    /*
-     * Google API setup
-     */
-    this.API = google.sheets('v4')
-    this.auth = null
-
     /** curry to allow convenient syntax with map */
     this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter.bind(this))
   }
@@ -66,20 +53,33 @@ class Fetcher {
         const allParts = allUrls.reduce((acc, url) => {
           if (url.startsWith(this.id)) {
             const parts = url.split('/')
-            acc.push([ parts[1], parts[2] ])
+            let duplicateTab = acc.reduce((tabFound, p) => {
-            return acc
+              return tabFound || p[0] === parts[1]
-          } else {
+            }, false)
-            return acc
+            if (duplicateTab) {
+              acc.forEach(p => {
+                if (p[0] === parts[1]) {
+                  p[1].push(parts[2])
+                }
+              })
+            } else {
+              acc.push([ parts[1], [ parts[2] ] ])
+            }
           }
+          return acc
         }, [])
 
         return allParts
-          .map(parts => buildDesaturated(
+          .map(parts => {
-            this.sheetId,
+            const bp = buildDesaturated(
-            this.sheetName,
+              this.sheetId,
-            parts[0],
+              this.sheetName,
-            parts[1]
+              parts[0],
-          ))
+              parts[1]
+            )
+            bp.urls = Object.keys(bp.resources).map(r => `/api/${bp.sheet.name}/${bp.name}/${r}`)
+            return bp
+          })
       })
       .then(res => {
         this.blueprints = res
@@ -104,20 +104,80 @@ class Fetcher {
     )
   }
 
+  save (_tab, data) {
+    const tab = fmtName(_tab)
+
+    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
+      const bpConfig = this.blueprinters[tab]
+
+      if (isFunction(bpConfig)) {
+        // if bpConfig specifies a single blueprinter
+        return this._saveViaBlueprinter(tab, data, bpConfig)
+      } else {
+        // if bpConfig specifies an array of blueprinters
+        return bpConfig.map(this._saveViaBlueprinter(tab, data))
+      }
+    } else {
+      // NB: if a blueprinter is not specified for a tab,
+      // just skip it.
+      return true
+    }
+  }
+
+  // NB: could combine these functions by checking kwargs length
+  retrieve (tab, resource) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  retrieveFrag (tab, resource, frag) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  /** Run on startup. Should be overridden if explicit auth is required **/
+  authenticate (env) {
+    return Promise.resolve(this)
+  }
+}
+
+class GsheetFetcher extends Fetcher {
+  constructor (db, name, bps, sheetId) {
+    super(db, name, bps)
+    this.type = 'Google Sheet'
+    if (arguments.length < 4) throw Error('You must provide the sheet ID')
+
+    /*
+     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
+     * loaded here must be added to the sheet as an editor.
+     */
+    this.sheetId = sheetId
+    /*
+     * Google API setup
+     */
+    this.API = google.sheets('v4')
+    this.auth = null
+  }
+
   /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
-  authenticate (clientEmail, privateKey) {
+  authenticate (env) {
-    const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [
+    const googleAuth = new google.auth.JWT(env.SERVICE_ACCOUNT_EMAIL, null, env.SERVICE_ACCOUNT_PRIVATE_KEY, [
       'https://www.googleapis.com/auth/spreadsheets'
     ])
     this.auth = googleAuth
     const { sheetId } = this
+    const me = this
 
     return new Promise((resolve, reject) => {
       googleAuth.authorize(function (err) {
         if (err) {
           reject(err)
         } else {
-          resolve(`Connected to ${sheetId}.`)
+          console.log(`Connected to ${me.sheetName}. (${me.type} with ID ${sheetId}).`)
+          console.log(`    grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
+          resolve(me)
         }
       })
     })
@@ -159,39 +219,32 @@ class Fetcher {
       .then(() => true)
       .catch(() => false)
   }
+}
 
-  save (_tab, data) {
+class LocalFetcher extends Fetcher {
-    const tab = fmtName(_tab)
+  constructor (db, name, bps, path) {
-
+    super(db, name, bps)
-    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
+    this.path = path
-      const bpConfig = this.blueprinters[tab]
+    this.update().then(res =>
-
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
-      if (isFunction(bpConfig)) {
+    )
-        // if bpConfig specifies a single blueprinter
-        return this._saveViaBlueprinter(tab, data, bpConfig)
-      } else {
-        // if bpConfig specifies an array of blueprinters
-        return bpConfig.map(this._saveViaBlueprinter(tab, data))
-      }
-    } else {
-      // NB: if a blueprinter is not specified for a tab,
-      // just skip it.
-      return true
-    }
   }
 
-  // NB: could combine these functions by checking kwargs length
+  update () {
-  retrieve (tab, resource) {
+    const wb = X.readFile(this.path)
-    const title = fmtName(tab)
+    wb.SheetNames.forEach(name => {
-    const url = `${this.id}/${tab}/${resource}`
+      const sh = wb.Sheets[name]
-    return this.db.load(url, this.blueprints[title])
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
-  }
+      const ll = csv.split('\n').map(line => line.split('\t'))
-
+      this.save(name, ll)
-  retrieveFrag (tab, resource, frag) {
+    })
-    const title = fmtName(tab)
+    return Promise.resolve(true)
-    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
-    return this.db.load(url, this.blueprints[title])
   }
 }
 
-export default Fetcher
+export default {
+  'gsheets': GsheetFetcher,
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
+}

src/lib/blueprinters.js

@@ -15,12 +15,15 @@ export const defaultResource = {
   data: []
 }
 
-export function buildDesaturated (sheetId, sheetName, tab, resource) {
+export function buildDesaturated (sheetId, sheetName, tab, resources) {
   const bp = R.clone(defaultBlueprint)
   bp.sheet.name = sheetName
   bp.sheet.id = sheetId
   bp.name = tab
-  bp.resources[resource] = null
+  bp.resources = resources.reduce((acc, r) => {
+    acc[r] = null
+    return acc
+  }, {})
   return bp
 }
 

src/lib/util.js

@@ -1,4 +1,6 @@
 import R from 'ramda'
+import { promises as fs } from 'file-system'
+import copy from '../copy/en'
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {
@@ -14,6 +16,18 @@ function camelize (str) {
   })
 }
 
+export async function exportToFile (fileDest, data) {
+  const stringifiedData = JSON.stringify(data, null, 2)
+  const filePath = `${fileDest}/export.json`
+
+  try {
+    await fs.writeFile(filePath, stringifiedData)
+    return copy.success.export(filePath)
+  } catch (err) {
+    throw new Error(copy.errors.export.writeFailed)
+  }
+}
+
 export const fmtObj = R.curry(
   (
     columnNames,

src/models/StoreJson.js

@@ -1,7 +1,7 @@
 import fs from 'mz/fs'
 import errors from '../lib/errors'
 
-const STORAGE_DIRNAME = 'temp'
+const STORAGE_DIRNAME = 'data'
 
 function partsFromFilename (fname) {
   const body = fname.slice(0, -5)

src/sheets_config.js

@@ -1,20 +0,0 @@
-import BP from './lib/blueprinters'
-
-export default {
-  googleSheets: {
-    sheets: [
-      {
-        name: 'example',
-        id: '1UC7DkCFeUXHfpUxUGruExwFbP4pqVBdJLOKfo6wDDGk',
-        tabs: {
-          export_events: [BP.deeprows, BP.rows],
-          export_categories: [BP.groups, BP.rows],
-          export_narratives: BP.rows,
-          export_sources: BP.deepids,
-          export_sites: BP.rows,
-          export_tags: BP.tree
-        }
-      }
-    ]
-  }
-}

temp/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

test/serverProcess.js

@@ -54,14 +54,7 @@ test('should launch', t => {
 })
 
 const passUrls = [
-  // /
+  '/api/'
-  '/api/',
-  // /blueprints
-  '/api/blueprints',
-  // /:sheet/:tab/:resource
-  '/api/example/export_events/rows',
-  // /:sheet/:tab/:resource/:frag
-  '/api/example/export_events/rows/1'
 ]
 
 const failUrls = [
@@ -77,6 +70,7 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
+        console.info('JSON: ', json)
         t.pass()
       })
   })

views/blueprints.hbs

@@ -0,0 +1,91 @@
+<h1>Available Endpoints</h1>
+<hr>
+<div class="main-container">
+  {{#each bps}}
+  <div class="blueprint-container">
+    <div class="bp-header">
+      <div class="bp-tab">{{ tab }}</div>
+      <div class="bp-source">{{ source }}</div>
+    </div>
+    {{#each urls}}
+    <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
+    {{/each}}
+  </div>
+  {{ else }}
+  <div>No endpoints found. Have you updated?</div>
+  {{/each}}
+</div>
+
+<a class="bp-update-container" target="_blank" href="http://localhost:4040/api/update">
+  <div class="bp-button">Update</div>
+</div>
+
+
+<style>
+:root {
+  --grey: #8a8a8a;
+  --btn-width: 200px;
+}
+.main-container {
+  display: flex;
+  flex-wrap: wrap;
+}
+
+.blueprint-container {
+  display: flex;
+  flex: 1 0 31%;
+  flex-direction: column;
+  border: 1px solid var(--grey);
+  padding: 1em;
+  margin: 1em;
+}
+
+.bp-header {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+}
+
+.bp-tab {
+  font-size: 24pt;
+}
+
+.bp-source {
+  color: var(--grey);
+}
+
+.bp-update-container {
+  position: fixed;
+  top: 2px;
+  right: 2em;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+
+.bp-button {
+  transition: all 0.3s ease;
+  text-transform: uppercase;
+  font-size: 24pt;
+  font-weight: bold;
+  width: var(--btn-width);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  border: 3px solid black;
+  padding: .5em;
+  text-decoration: none;
+  text-underline: none;
+  color: black;
+}
+
+.bp-update-container:hover {
+  background-color: black;
+  transition: all 0.3s ease;
+  cursor: pointer;
+}
+
+.bp-update-container:hover div {
+  color: white;
+}
+</style>

views/layouts/default.hbs

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+  <title>Datasheet Server</title>
+  <meta name="HandheldFriendly" content="True" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
+</head>
+<body>
+  <div class="container">
+	  {{{body}}}
+  </div>
+</body>
+</html>