Merge branch 'develop' of https://www.github.com/forensic-architecture/datasheet-server into feature/json-api-export

Adding the correct branch to checkout with for workflow

.env.example

@@ -0,0 +1,4 @@
+PORT=4040
+MAPBOX_TOKEN=pk.ANOTHERLONGSTRING.pMXNkIn0.xjjmguLIeX-r8FWomVG8Tg
+SERVICE_ACCOUNT_EMAIL="DUMMY-SERVICE-WORKER-NAME@DUMMY-PROJECT-NAME.iam.gserviceaccount.com"
+SERVICE_ACCOUNT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\A-vErY-l0nG-sTr1Ng\n-----END PRIVATE KEY-----\n"

.github/workflows/cd.yml

@@ -0,0 +1,19 @@
+name: CD
+on:
+  push:
+    branches: [ develop ]
+#  pull_request:
+#    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger CD build
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.CI_DISPATCH_TOKEN }}
+          repository: forensic-architecture/configs
+          event-type: remote-build
+          client-payload: '{"runtime_args": "datasheet", "branch": "${GITHUB_REF##*/}"}'
+

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '12'
+
+      - run: npm install
+      - run: npm test
+        env:
+          CI: true
+      - run: npm run lint
+        env:
+          CI: true

.gitignore

@@ -5,6 +5,13 @@
 .DS_Store
 *.swp
 
+.env
 *service-account-key\.json
-src/config.js
 /yarn-error.log
+*.pem
+.travis.yml.old
+tags
+tags.lock
+tags.temp
+src/config.js
+src/local.config.js

.travis.yml

@@ -1,15 +0,0 @@
-language: node_js
-node_js:
-  - "stable"
-cache:
-  directories:
-    - node_modules
-before_script:
-  - npm install -g yarn
-  - cp ./src/example.config.js ./src/config.js
-install:
-  - yarn
-script:
-  - yarn build
-  - yarn lint
-  - yarn test

CONTRIBUTING.md

@@ -45,11 +45,32 @@ WIP
 
 ### Before making changes
 
-WIP. Here we'll describe what the expected process and workflow is when making code changes, with regards to branching, forking and so on.
+1. If you are a contributor, you will need to create a fork of this repository
+   on your own GitHub handle, as you will not have commit access to the
+   forensic architecture repo.
+2. Create a new branch from _develop_ (not master or staging). The branch
+   should be prefixed with 'topic/' if you are intending to submit a feature
+   ('enhancement' tag in the issue), or with 'fix/' if you are fixing a bug
+   ('bug' tag in the issue).
+
+All of your commits go in this branch. When the feature/fix is complete, follow
+the instructions below to submit a PR for the branch.
 
 ### Submitting changes as Pull Requests
 
-WIP
+In order to submit a branch as a PR, you'll need to install the [Travis CLI](https://github.com/travis-ci/travis.rb). The documentation for this is a little shifty: if you're developing on a Mac, you can easily install it with `brew install travis`. The Travis CLI is necessary so that you can encrypt your service account credentials and use them while testing in Travis CI.
+
+To do this, you need to run one extra command before you push commits
+to a remote branch:
+```
+npm run travis-encrypt
+```
+This command encrypts your private key and service account email in .env in
+such a way that they can still be used while running tests on Travis' server.
+This command will add a commit to your branch that modifies the binary file
+.env.enc, and updates your Travis config accordingly. After running this
+command, you should be able to pass the pre-push check and run tests in the
+Travis server.
 
 ## Additional resources
 

Dockerfile

@@ -10,11 +10,11 @@ RUN cd /www; yarn
 COPY . /www
 WORKDIR /www
 RUN yarn build
-RUN mkdir -p temp
+RUN mkdir -p data
 
 # set your port
-ENV PORT 8080
-EXPOSE  8080
+ENV PORT 4040
+EXPOSE 4040
 
 # start command as per package.json
 CMD ["yarn", "start"]

LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2018 Forensic Architecture
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.md

@@ -0,0 +1,66 @@
+Do No Harm License
+
+**Preamble**
+
+Most software today is developed with little to no thought of how it will be used, or the consequences for our society and planet.
+
+As software developers, we engineer the infrastructure of the 21st century. We recognise that our infrastructure has great power to shape the world and the lives of those we share it with, and we choose to consciously take responsibility for the social and environmental impacts of what we build.
+
+We envisage a world free from injustice, inequality, and the reckless destruction of lives and our planet. We reject slavery in all its forms, whether by force, indebtedness, or by algorithms that hack human vulnerabilities. We seek a world where humankind is at peace with our neighbours, nature, and ourselves. We want our work to enrich the physical, mental and spiritual wellbeing of all society.
+
+We build software to further this vision of a just world, or at the very least, to not put that vision further from reach.
+
+**Terms**
+
+*Copyright* (c) 2019 Forensic Architecture. All rights reserved.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+4. This software must not be used by any organisation, website, product or service that:
+
+   a) lobbies for, promotes, or derives a majority of income from actions that support or contribute to:
+      * sex trafficking
+      * human trafficking
+      * slavery
+      * indentured servitude
+      * gambling
+      * tobacco
+      * adversely addictive behaviours
+      * nuclear energy
+      * warfare
+      * weapons manufacturing
+      * war crimes
+      * violence (except when required to protect public safety)
+      * burning of forests
+      * deforestation
+      * hate speech or discrimination based on age, gender, gender identity, race, sexuality, religion, nationality
+
+   b) lobbies against, or derives a majority of income from actions that discourage or frustrate:
+      * peace
+      * access to the rights set out in the Universal Declaration of Human Rights and the Convention on the Rights of the Child
+      * peaceful assembly and association (including worker associations)
+      * a safe environment or action to curtail the use of fossil fuels or prevent climate change
+      * democratic processes
+
+5. All redistribution of source code or binary form, including any modifications must be under these terms. You must inform recipients that the code is governed by these conditions, and how they can obtain a copy of this license. You may not attempt to alter the conditions of who may/may not use this software.
+
+We define:
+
+**Forests** to be 0.5 or more hectares of trees that were either planted more than 50 years ago or were not planted by humans or human made equipment.
+
+**Deforestation** to be the clearing, burning or destruction of 0.5 or more hectares of forests within a 1 year period.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+**Attribution**
+
+Do No Harm License [Contributor Covenant][homepage], (pre 1.0),
+available at https://github.com/raisely/NoHarm
+
+[homepage]: https://github.com/raisely/NoHarm
+

README.md

@@ -34,45 +34,12 @@ Datasheet server is a Node server developed at [Forensic Architecture](https://f
 
 Querying data directly from spreadsheets is brittle, as it relies on the maintenance of a rigid structure in the sheets at all times. By putting Datasheet Server as a proxy that sits in between source sheets and their consumers, it is possible to dynamically modify sheets without breaking applications. A data admin can then use Datasheet Server to ensure that applications always receive eligible data, without foregoing the spreadsheets as sources of truth.
 
-To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/Setting-up-a-local-instance-of-Timemap) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
-
-### Design Concepts
-The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
-
-**Sources**     
-A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
-
-**Resources**     
-The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
-
-**Blueprints**       
-Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
-
-Blueprints are JSON objects. There have two forms:
-
-1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
-2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
-
-A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
-
-A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
-
-## [Configuration](#configuration)
-Copy the [example.config.js](/src/example.config.js) in the [src](/src) directory into a file named 'config.js'. Modify the options in this file accordingly:
-
-| Option  | Description | Type |
-| ------- | ----------- | ---- |
-| port | The port at which the server will make data available.  | integer |
-| googleSheets  | The configuration object for [Google Sheet](https://www.google.co.uk/sheets/about/) data sources. See the [Sources](#source-google-sheets) section below. | object |
+To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/running-timemap-and-datasheet-server-locally) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
 
 #### [Sources](#sources)
-###### [Google Sheets](#source-google-sheets)
-In order to make the data from a Sheet accessible to the server, you need to [create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts). Once created, give the service account email access to each Sheet from which you want to serve data. ('View Only' access is sufficient, as the server never modifies data.)
+Sources are specified in [src/config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js). Datasheet server currently only supports Google Sheets as a source.
 
-| Option | Description | Type |
-| ------ | ----------- | ---- |
-| email | The email address of the service account. This is available in the downloadable service account JSON in the `client_email` field. | string |
-| privateKey | The private key associated with the service account. This is available in the downloadable service account JSON in the `private_key` field. | string |
+###### [Google Sheets](#source-google-sheets)
 | sheets | A list of objects, one for each sheet that is being used as a source. Each sheet object has a `name` (String), an `id` (String), and a `tabs` (object) field, which are explained below. | object |
 
 Each Google Sheet being used as a as source requires a corresponding object in `sheets`. The object should be structured as follows:
@@ -80,41 +47,23 @@ Each Google Sheet being used as a as source requires a corresponding object in `
 | Option | Description | Type |
 | ------ | ----------- | ---- |
 | name | Used to refer to data served from this source | string |
-| id | The ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
+| path/id | The path to the XLSX sheet on your local, or the ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
 | tabs | An object that maps each tab in the source to one or more Blueprinters. All of the Blueprinters in the [blueprinters folder](/lib/blueprinters) are available through a single import as at the top of [example.config.js](/src/example.config.js). <br> To correctly associate a Blueprinter, the object key needs to be _the tab name with all lowercase letters, and spaces replaced by a '-'_. For example, if the tab name in Google Sheets is 'Info About SHEEP', the object key should be 'info-about-sheep'. <br> The value should be the Blueprinter function that you want to use for the data in that tab. If you require more than one endpoint for a single tab, you can support multiple blueprinters by making the item an array. See the example of a configuration object below. <br>TODO: no Blueprinter is used by default. | object |
 
-###### Example Configuration Object
-```js
-import BP from './lib/blueprinters'
-
-export default {
-  port: 4040,
-  googleSheets: {
-    email: 'project-name@reliable-baptist-23338.iam.gserviceaccount.com',
-    privateKey: 'SOME_PRIVATE_KEY',
-    sheets: [
-      {
-        name: 'example',
-        id: '1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI',
-        tabs: {
-          'objects': BP.byRow,
-          'fruit': [BP.byRow, BP.byID],
-        }
-      },
-    ]
-  }
-}
-
-```
+See src/config.js for an example configuration sheet. 
 
 
-## [Quickstart](#quickstart)
+## [Quickstart](#quickstart)
 Clone the repository to your local:
 ```
 git clone https://www.github.com/forensic-architecture/datasheet-server
 ```
+
+Follow the steps in the [configuration](#configuration) section of this
+document.
+
 ### Run with Docker
-To create a new instance of the server with [Docker](https://www.docker.com/) installed, clone the repository, create a `config.js`, and build the image:
+To create a new instance of the server with [Docker](https://www.docker.com/) installed, ensure that you have followed the steps in the quickstart guide above, then and build an image locally. (Note that Docker must be installed):
 ```sh
 docker build -t datasheet-server .
 ```
@@ -152,4 +101,4 @@ If you have any questions or just want to chat, please join our team [fa_open_so
 
 ## License
 
-TimeMap is distributed under the MIT License.
+Datasheet Server is distributed under the [MIT License](https://github.com/forensic-architecture/datasheet-server/blob/develop/LICENSE).

data/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!timemap_data.xlsx

docs/architecture.md

@@ -0,0 +1,41 @@
+# Architecture
+
+![](datasheet-server-graphic.jpg)
+
+Datasheet Server uses the architecture diagrammed above to allow effective
+management of the sheets it represents.
+
+It exposes an API with four endpoints:
+ * `/:sheet/:tab/:resource` - the primary means of accessing data in the
+     server. The name of a sheet is specified in
+     [_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js#L7).
+     Each tab on the sheet is in lower case, with spaces replaced by
+     underscores. The resource name is the same as the name of the blueprinter
+     specified for the tab is config.js.
+ * `/:sheet/:tab/:resource/:id` - when the resource is a list, items in the list
+     can be accessed individually as well.
+ * `/update` - when this route is queried with a GET request, each of the
+     fetchers in the server will update their models from the relevant sheet.
+ * `/blueprints` - a JSON object that represents all available sheets, tabs, and
+     resources in the server.
+
+# Controller
+The [controller](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/lib/Controller.js)
+manages all of the fetchers in the server. Its `update` triggers the update
+mechanisms of all the `update`s in the fetchers it manages. Its `retrieve` and
+`retrieveFrag` methods find the appropriate fetcher, and trigger its 
+respectively named method. Its `bluerprints` method collects the blueprints
+from all fetchers, and presents them as a single list.
+
+# Fetcher
+The [fetcher](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/lib/Fetcher.js)
+is the most complex component in datasheet server. Its responsiblity is to
+interface with a sheet, and store the data from that sheet in a model. It also
+makes information from that model available to the controller.
+
+# Model
+The model layer is an [interface](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/models/Interface.js)
+that specifies how stored data must be made available to a fetcher. The
+implementation of the model layer ([StoreJson](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/models/StoreJson.js)
+is one example) determines the time and space complexity of retrieval.
+

docs/design-concepts.md

@@ -0,0 +1,22 @@
+### Design Concepts
+The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
+
+**Sources**     
+A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
+
+**Resources**     
+The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
+
+**Blueprints**       
+Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
+
+Blueprints are JSON objects. There have two forms:
+
+1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
+2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
+
+A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
+
+A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
+
+

docs/gsheet-config.md

@@ -0,0 +1,19 @@
+## [Configuration](#configuration)
+Copy the example environment file:
+```
+cp .env.example .env
+```
+Inside this file, you will need to modify at least the `SERVICE_ACCOUNT_EMAIL` and `SERVICE_ACCOUNT_PRIVATE_KEY` fields. These fields refer to the credentials of a [Google service account](https://cloud.google.com/iam/docs/understanding-service-accounts). Google requires that developers create these when attempting to access their services programmatically, so that they can attribute requests to users. Service accounts also contain identity information, which means that asset owners can allow certain service accounts access to certain sheets, just as one might differentially grant certain users access to certain cloud assets. 
+
+Once you have [created a service account](https://support.google.com/a/answer/7378726?hl=en), create and download an [API key for that account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). The JSON file for the API key that you download when you create it contains both a service account private key, and an email associated with the service account: add these respectively in the strings in .env for `SERVICE_ACCOUNT_PRIVATE_KEY` and `SERVICE_ACCOUNT_EMAIL`.
+
+The last thing to do is to grant the service account access to the sheet that
+datasheet-server is pulling from. You can add a service account to a sheet as
+you would any other Google user: just enter the email address associated. (Note
+that this step is not necessary if you are accessing a publicly available
+sheet.)
+
+Other configuration options, such as the port at which the server will expose
+resources, are also modifiable from the .env file.
+
+

package.json

@@ -1,15 +1,17 @@
 {
-  "name": "grenfell-server",
+  "name": "datasheet-server",
   "version": "0.3.0",
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
+  "type": "module",
   "scripts": {
-    "dev": "NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
-    "build": "NODE_ENV=production npx babel src -d dist",
+    "dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
+    "build": "env NODE_ENV=production npx babel src -d dist",
     "start": "node dist",
-    "lint": "standard \"src/**/*.js\" \"test/**/*/js\"",
+    "lint": "standard \"src/**/*.js\" \"test/**/*.js\"",
     "test-watch": "ava --watch",
-    "test": "ava --verbose"
+    "test": "ava --verbose",
+    "travis-encrypt": "./scripts/encrypt.sh"
   },
   "repository": {
     "type": "git",
@@ -21,21 +23,26 @@
     "body-parser": "^1.13.3",
     "compression": "^1.5.2",
     "cors": "^2.8.5",
+    "dotenv": "^6.1.0",
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
-    "googleapis": "^32.0.0",
+    "express-handlebars": "^4.0.4",
+    "file-system": "^2.2.2",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.2.0",
+    "node-fetch": "^2.6.1",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",
     "@babel/core": "^7.1.2",
     "@babel/node": "^7.0.0",
+    "@babel/polyfill": "^7.0.0",
     "@babel/preset-env": "^7.1.0",
     "@babel/register": "^7.0.0",
     "ava": "1.0.0-beta.8",
@@ -52,7 +59,8 @@
       "test/**/*.js"
     ],
     "require": [
-      "@babel/register"
+      "@babel/register",
+      "@babel/polyfill"
     ]
   },
   "bugs": {

src/api/index.js

@@ -4,6 +4,7 @@ import copy from '../copy/en'
 
 export default ({ config, controller }) => {
   let api = Router()
+  const fileDest = config.EXPORT_FILE_DEST || null
 
   api.get('/', (req, res) => {
     res.json({
@@ -12,7 +13,42 @@ export default ({ config, controller }) => {
   })
 
   api.get('/blueprints', (req, res) => {
-    res.json(controller.blueprints())
+    const bps = controller.blueprints()
+    res.render('blueprints', {
+      bps: bps.map(bp => ({
+        source: bp.sheet.name,
+        tab: bp.name,
+        urls: bp.urls
+      }))
+    })
+  })
+
+  api.get('/export', (req, res) => {
+    controller
+      .retrieveAll(fileDest)
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
+  })
+
+  api.get('/update', (req, res) => {
+    controller
+      .update()
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
   })
 
   api.get('/:sheet/:tab/:resource/:frag', (req, res) => {
@@ -27,8 +63,9 @@ export default ({ config, controller }) => {
   })
 
   api.get('/:sheet/:tab/:resource', (req, res) => {
+    const { sheet, tab, resource } = req.params
     controller
-      .retrieve(req.params.sheet, req.params.tab, req.params.resource)
+      .retrieve(sheet, tab, resource)
       .then(data => res.json(data))
       .catch(err =>
         res.status(err.status || 404)
@@ -36,30 +73,16 @@ export default ({ config, controller }) => {
       )
   })
 
-  api.get('/update', (req, res) => {
-    controller
-      .update()
-      .then(msg =>
-        res.json({
-          success: msg
-        })
-      )
-      .catch(err =>
-        res.status(404)
-          .send({ error: err.message })
-      )
-  })
-
   // ERROR routes. Note that it is important that these come AFTER routes
   // like /update, so that the regex does not greedily match these routes.
 
   api.get('/:sheet', (req, res) => {
-    res.status(404)
+    res.status(400)
       .send({ error: copy.errors.onlysheet })
   })
 
   api.get('/:sheet/:tab', (req, res) => {
-    res.status(404)
+    res.status(400)
       .send({ error: copy.errors.onlyTab })
   })
 

src/blueprinters/columns.js

@@ -1,37 +1,27 @@
-import R from 'ramda'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
-
 /**
- * byColumn - generate a Blueprint from a data sheet by column. Each column
- * name is a resheet, and all values in that column are the resheet items.
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
  *
- * @param  {type} data - list of lists representing sheet data.
- * @return {type} Blueprint
- * generated.
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-function columns (tabName, sheetName, sheetId, data) {
-  // Define Blueprint props
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
+export default (data) => {
+  const columnNames = data[0]
+  const columns = columnNames.map(name => ([]))
 
-  // column names define resources
-  const labels = data[0]
-  labels.forEach(label => {
-    bp.resources[label] = R.clone(defaultResource)
-  })
-
-  // remaining rows as data
   data.forEach((row, idx) => {
     if (idx === 0) return
-    labels.forEach((label, idx) => {
-      bp.resources[label].data.push(row[idx])
+    row.forEach((item, idx) => {
+      columns[idx].push(item)
     })
   })
-  return bp
-}
 
-export default columns
+  return columns.map((column, idx) => ({
+    name: columnNames[idx],
+    items: column
+  }))
+}

src/blueprinters/deepids.js

@@ -0,0 +1,22 @@
+import deeprows from './deeprows'
+
+/**
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
+ *
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Object      the structured data.
+ */
+export default (data) => {
+  const output = {}
+
+  deeprows(data).forEach(row => {
+    output[row.id] = row
+  })
+
+  return output
+}

src/blueprinters/deeprows.js

@@ -0,0 +1,72 @@
+import { fmtObj } from '../lib/util'
+
+/**
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
+ *
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
+ */
+export default (data) => {
+  const itemLabels = data[0]
+  const baseFmt = fmtObj(itemLabels)
+  const output = []
+
+  // create a structure to indicate which columns needs to be aggregated
+  const endsWithNumber = new RegExp('(.*?)[0-9]+$')
+  const structure = {
+    __flat: []
+  }
+
+  itemLabels.forEach(label => {
+    const matches = label.match(endsWithNumber)
+    if (!matches) {
+      structure.__flat.push(label)
+    } else {
+      const labelPrefix = `${matches[1]}s`
+      if (labelPrefix in structure) {
+        structure[labelPrefix].push(label)
+      } else {
+        structure[labelPrefix] = [ label ]
+      }
+    }
+  })
+
+  // generate the value for deep labels using the structure created
+  data.forEach((row, idx) => {
+    if (idx === 0) return
+    const baseRow = baseFmt(row)
+    const deepRow = {}
+
+    // generate deep row labels using structure
+    Object.keys(structure)
+      .forEach(newLabel => {
+        if (newLabel !== '__flat') {
+          const oldLabels = structure[newLabel]
+          // only add new value if not ''
+          const labelValues = []
+          oldLabels.forEach(l => {
+            const vl = baseRow[l]
+            if (vl !== '') {
+              labelValues.push(vl)
+            }
+          })
+          deepRow[newLabel] = labelValues
+        }
+      })
+
+    // move values for flat labels over from base
+    structure.__flat.forEach(label => {
+      deepRow[label] = baseRow[label]
+    })
+    if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {
+      output.push(deepRow)
+    }
+  })
+
+  return output
+}

src/blueprinters/groups.js

@@ -1,39 +1,16 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * groups - generate a Blueprint from a data sheet grouped by a column called 'group'
- * The resource name defaults to 'groups', or a custom resource name can be passed.
  * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * names. Items are inserted into the data list at idx = id.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="groups"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function groups (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'groups'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
-
+  const output = []
   const dataGroups = {}
 
   data.forEach((row, idx) => {
@@ -45,12 +22,14 @@ export default function groups (
       dataGroups[group].push(fmt(row))
     }
   })
-  Object.keys(dataGroups).forEach(groupKey => {
-    bp.resources[label].data.push({
-      group: groupKey,
-      group_label: dataGroups[groupKey][0].group_label,
-      data: dataGroups[groupKey]
+  Object.keys(dataGroups)
+    .forEach(groupKey => {
+      output.push({
+        group: groupKey,
+        group_label: dataGroups[groupKey][0].group_label,
+        data: dataGroups[groupKey]
+      })
     })
-  })
-  return bp
+
+  return output
 }

src/blueprinters/ids.js

@@ -1,42 +1,22 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * ids - generate a Blueprint from a data sheet by id, which is an integer.
- * The resource name defaults to 'ids', or a custom resource name can be passed.
- * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * Very similar to the rows blueprinter, but inserts each row as a value in
+ * an object, where the value in the 'id' column of the row will be used as
+ * the search key
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="ids"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Object      the structured data.
  */
-export default function ids (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'ids'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
+  const output = {}
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.resources[label].data[fmt(row).id] = fmt(row)
+    output[fmt(row).id] = fmt(row)
   })
-  return bp
+
+  return output
 }

src/blueprinters/rows.js

@@ -1,41 +1,21 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * rows - generate a Blueprint from a data sheet by row. The resource name
- * defaults to 'rows', or a custom resource name can be passed. Each resource
- * item is an object with values labelled according to column names.
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="rows" name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function rows (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'rows'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
+  const output = []
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.resources[label].data.push(fmt(row))
+    output.push(fmt(row))
   })
-  return bp
+
+  return output
 }

src/blueprinters/tree.js

@@ -1,38 +1,12 @@
-import R from 'ramda'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
-
 /**
- * tree - generate a Blueprint from a data sheet grouped by a column called 'group'
- * The resource name defaults to 'groups', or a custom resource name can be passed.
- * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * Each resource item is inserted into a tree. TODO: describe layout.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="groups"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function tree (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'tree'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = {}
-
+export default (data) => {
   const tree = {
-    key: 'tags',
+    key: '_root',
     children: {}
   }
 
@@ -62,6 +36,5 @@ export default function tree (
     }
   })
 
-  bp.resources[label].data = tree
-  return bp
+  return tree
 }

src/config.js

@@ -0,0 +1,12 @@
+import { timemap } from './lib'
+
+export default {
+  gsheets: [],
+  xlsx: [
+    {
+      name: 'timemap_data',
+      path: 'data/timemap_data.xlsx',
+      tabs: timemap.default
+    }
+  ]
+}

src/copy/en.js

@@ -1,13 +1,19 @@
 export default {
   errors: {
     update: 'The server could not update. Check your API credentials and internet connection and try again.',
+    export: {
+      fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
+      writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
+    },
     onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
     noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' in this sheet.`,
-    noFragment: prts => `Fragment index does not exist`
+    noFragment: prts => `Fragment index does not exist`,
+    modelLayer: prts => `Something went wrong at the model layer`
   },
   success: {
-    update: 'All sheets updated'
+    update: 'All sheets updated',
+    export: dest => `All resources exported to the file: ${dest}`
   }
 }

src/example.config.js

@@ -1,21 +0,0 @@
-import BP from './lib/blueprinters'
-
-export default {
-  port: 4040,
-  googleSheets: {
-    email: 'SOME_SERVICE_ACCOUNT_EMAIL',
-    privateKey: 'SOME_SERVICE_ACCOUNT_PRIVATE_KEY',
-    sheets: [
-      {
-        name: 'example',
-        id: '1UC7DkCFeUXHfpUxUGruExwFbP4pqVBdJLOKfo6wDDGk',
-        tabs: {
-          export_events: [BP.byId, BP.byRow],
-          export_categories: [BP.byGroup, BP.byRow],
-          export_sites: BP.byRow,
-          export_tags: BP.byTree
-        }
-      }
-    ]
-  }
-}

src/index.js

@@ -1,12 +1,21 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
 import api from './api'
-import config from './config'
+import dotenv from 'dotenv'
+const hbs = require('express-handlebars')
+
+dotenv.config()
 
 let app = express()
 app.server = http.createServer(app)
+app.engine('.hbs', hbs({
+  extname: '.hbs',
+  defaultLayout: 'default'
+}))
+app.set('view engine', '.hbs')
 
 // enable cross origin requests explicitly in development
 if (process.env.NODE_ENV === 'development') {
@@ -15,6 +24,8 @@ if (process.env.NODE_ENV === 'development') {
   app.use(cors())
 }
 
+const config = process.env
+
 initialize(controller => {
   app.use(
     middleware({
@@ -30,8 +41,11 @@ initialize(controller => {
     })
   )
 
-  app.server.listen(process.env.PORT || config.port, () => {
-    console.log(`Started on port ${app.server.address().port}`)
+  app.use(express.static(path.join(__dirname, 'public')))
+
+  app.server.listen(process.env.PORT || 4040, () => {
+    console.log('===========================================')
+    console.log(`Server running on port ${app.server.address().port}`)
   })
 })
 

src/initialize.js

@@ -1,37 +1,43 @@
 import StoreJson from './models/StoreJson'
-import Fetcher from './lib/Fetcher'
+import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
+import R from 'ramda'
 
-const { googleSheets } = config
-const { sheets, privateKey, email } = googleSheets
+const isntNull = n => n !== null
+const filterNull = ls => R.filter(isntNull, ls)
+const flattenfilterNull = ls => filterNull(R.flatten(ls))
+let themFetchers
 
-function authenticate (_fetcher) {
-  return _fetcher.fetcher.authenticate(email, privateKey).then(msg => {
-    console.log(msg)
-    return true
-  })
+let config
+try {
+  config = require('./local.config.js').default
+} catch (_) {
+  config = require('./config.js').default
 }
 
 export default callback => {
-  const fetchers = sheets.map(sheet => {
-    return {
-      name: sheet.name,
-      fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs)
-    }
-  })
-
-  Promise.all(fetchers.map(authenticate))
-    .then(() => {
-      console.log(`===================`)
-      console.log(`grant access to: ${email}`)
-      console.log(`===================`)
-
-      // NB: reformat fetchers as config for controller
-      const config = {}
-      fetchers.forEach(fetcher => {
-        config[fetcher.name] = fetcher.fetcher
+  return Promise.resolve().then(() => {
+    return Object.keys(config).map(fType => {
+      // skip config attrs that don't have corresponding fetchers
+      if (!(fType in fetchers)) return null
+      const FFetcher = fetchers[fType]
+      return config[fType].map(sheet => {
+        const otherArgs = { ...sheet }
+        delete otherArgs.name
+        delete otherArgs.tabs
+        return {
+          name: sheet.name,
+          fetcher: new FFetcher(new StoreJson(), sheet.name, sheet.tabs, ...Object.values(otherArgs))
+        }
       })
+    })
+  })
+    .then(res => {
+      themFetchers = flattenfilterNull(res)
+    })
+    .then(() => Promise.all(themFetchers.map(f => f.fetcher.authenticate(process.env))))
+    .then(fetchers => {
+      const config = R.zipObj(themFetchers.map(f => f.name), fetchers)
       const controller = new Controller(config)
       callback(controller)
     })
@@ -39,7 +45,7 @@ export default callback => {
       console.log(err)
       console.log(
         `ERROR: the server couldn't connect to all of the sheets you provided. Ensure you have granted access to ${
-          email
+          process.env.SERVICE_ACCOUNT_EMAIL
         } on ALL listed sheets.`
       )
     })

src/lib.js

@@ -0,0 +1,17 @@
+import BP from './lib/blueprinters'
+
+function prefixedTabs (prefix, cfg) {
+  if (!cfg) cfg = {}
+  const prf = key => cfg[key] ? `${prefix}_` : ''
+  return {
+    [`${prf('events')}export_events`]: BP.deeprows,
+    [`${prf('associations')}export_associations`]: BP.deeprows,
+    [`${prf('sources')}export_sources`]: BP.deepids,
+    [`${prf('sites')}export_sites`]: BP.rows
+  }
+}
+
+export const timemap = {
+  default: prefixedTabs(),
+  prefixedTabs
+}

src/lib/Controller.js

@@ -1,4 +1,5 @@
 import copy from '../copy/en'
+import { exportToFile } from '../lib/util'
 
 /**
  * Controller
@@ -16,16 +17,22 @@ class Controller {
   blueprints () {
     return Object.keys(this.fetchers).map(
       sheet => this.fetchers[sheet].blueprints
-    )
+    ).reduce((acc, curr) => acc.concat(curr))
+  }
+
+  rebuildBlueprintsAsync () {
+    Object.values(this.fetchers).forEach(t => t._buildBlueprintsAsync())
   }
 
   update () {
+    const me = this
     return Promise.all(
       Object.keys(this.fetchers).map(sheet => {
         return this.fetchers[sheet].update()
       })
     ).then(results => {
       if (results.every(r => r)) {
+        me.rebuildBlueprintsAsync()
         return copy.success.update
       } else {
         throw new Error(copy.errors.update)
@@ -33,6 +40,37 @@ class Controller {
     })
   }
 
+  // Controller function to retrieve all blueprints and export to user defined file location
+  retrieveAll (fileDest) {
+    if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
+
+    const indexedData = {}
+    const urls = []
+
+    const bps = this.blueprints()
+    return Promise.all(
+      bps.map(bp => {
+        const resource = Object.keys(bp.resources)[0]
+        urls.push(bp.urls[0])
+        return this.retrieve(bp.sheet.name, bp.name, resource)
+      })
+    ).then(async results => {
+      if (results.every(res => res)) {
+        urls.forEach((item, idx) => {
+          indexedData[item] = results[idx]
+        })
+        try {
+          const message = await exportToFile(fileDest, indexedData)
+          return message
+        } catch (e) {
+          return Promise.reject(e)
+        }
+      } else {
+        throw new Error(copy.errors.export.writeFailed)
+      }
+    })
+  }
+
   retrieve (sheet, tab, resource) {
     if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]

src/lib/Fetcher.js

@@ -1,45 +1,39 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
-import { google } from 'googleapis'
+import R from 'ramda'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
-import R from 'ramda'
+
+/* GsheetFetcher deps */
+import { google } from 'googleapis'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
-  constructor (db, sheetName, sheetId, blueprinters) {
+  constructor (db, name, bps) {
     /*
      * The database that the fetcher should use. This should be an instance of a model-compliant class.
      * See models/Interface.js for the specifications for a model-compliant class.
      */
     this.db = db
-
-    /*
-     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
-     * loaded here must be added to the sheet as an editor.
-     */
-    this.sheetId = sheetId
-
     /*
      * The name of the sheet. This will prefix tabs saved in the database.
      */
-    this.sheetName = sheetName
-
+    this.sheetName = name
     /*
      * A unique ID for the Fetcher to identify its elements in the model layer
      */
-    this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex')
-
+    const bpsstring = Object.keys(bps).join(';')
+    this.id = createHash('md5').update(name).update(bpsstring).digest('hex')
     /*
      * These are the available tabs for storing and retrieving data.
      * Each blueprinter is a function that returns a Blueprint from a
      * list of lists (which will be retrieved from gsheets).
      */
-    this.blueprinters = fmtBlueprinterTitles(blueprinters)
-
+    this.blueprinters = fmtBlueprinterTitles(bps)
     /*
      * This object is the canonical represenation for the data that a Fetcher
      * proxies. When the fetcher is initialized, its model layer (db) is indexed,
@@ -49,13 +43,6 @@ class Fetcher {
      */
     this.blueprints = null
     this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion
-
-    /*
-     * Google API setup
-     */
-    this.API = google.sheets('v4')
-    this.auth = null
-
     /** curry to allow convenient syntax with map */
     this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter.bind(this))
   }
@@ -66,20 +53,33 @@ class Fetcher {
         const allParts = allUrls.reduce((acc, url) => {
           if (url.startsWith(this.id)) {
             const parts = url.split('/')
-            acc.push([ parts[1], parts[2] ])
-            return acc
-          } else {
-            return acc
+            let duplicateTab = acc.reduce((tabFound, p) => {
+              return tabFound || p[0] === parts[1]
+            }, false)
+            if (duplicateTab) {
+              acc.forEach(p => {
+                if (p[0] === parts[1]) {
+                  p[1].push(parts[2])
+                }
+              })
+            } else {
+              acc.push([ parts[1], [ parts[2] ] ])
+            }
           }
+          return acc
         }, [])
 
         return allParts
-          .map(parts => buildDesaturated(
-            this.sheetId,
-            this.sheetName,
-            parts[0],
-            parts[1]
-          ))
+          .map(parts => {
+            const bp = buildDesaturated(
+              this.sheetId,
+              this.sheetName,
+              parts[0],
+              parts[1]
+            )
+            bp.urls = Object.keys(bp.resources).map(r => `/api/${bp.sheet.name}/${bp.name}/${r}`)
+            return bp
+          })
       })
       .then(res => {
         this.blueprints = res
@@ -91,9 +91,9 @@ class Fetcher {
    */
   _saveViaBlueprinter (tab, data, blueprinter) {
     const saturatedBp = blueprinter(
-      tab,
-      this.sheetName,
       this.sheetId,
+      this.sheetName,
+      tab,
       data
     )
 
@@ -104,20 +104,80 @@ class Fetcher {
     )
   }
 
+  save (_tab, data) {
+    const tab = fmtName(_tab)
+
+    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
+      const bpConfig = this.blueprinters[tab]
+
+      if (isFunction(bpConfig)) {
+        // if bpConfig specifies a single blueprinter
+        return this._saveViaBlueprinter(tab, data, bpConfig)
+      } else {
+        // if bpConfig specifies an array of blueprinters
+        return bpConfig.map(this._saveViaBlueprinter(tab, data))
+      }
+    } else {
+      // NB: if a blueprinter is not specified for a tab,
+      // just skip it.
+      return true
+    }
+  }
+
+  // NB: could combine these functions by checking kwargs length
+  retrieve (tab, resource) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  retrieveFrag (tab, resource, frag) {
+    const title = fmtName(tab)
+    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
+    return this.db.load(url, this.blueprints[title])
+  }
+
+  /** Run on startup. Should be overridden if explicit auth is required **/
+  authenticate (env) {
+    return Promise.resolve(this)
+  }
+}
+
+class GsheetFetcher extends Fetcher {
+  constructor (db, name, bps, sheetId) {
+    super(db, name, bps)
+    this.type = 'Google Sheet'
+    if (arguments.length < 4) throw Error('You must provide the sheet ID')
+
+    /*
+     * ID of the Google Sheet where the data is sheetd. Note that the privateKey.clientEmail
+     * loaded here must be added to the sheet as an editor.
+     */
+    this.sheetId = sheetId
+    /*
+     * Google API setup
+     */
+    this.API = google.sheets('v4')
+    this.auth = null
+  }
+
   /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
-  authenticate (clientEmail, privateKey) {
-    const googleAuth = new google.auth.JWT(clientEmail, null, privateKey, [
+  authenticate (env) {
+    const googleAuth = new google.auth.JWT(env.SERVICE_ACCOUNT_EMAIL, null, env.SERVICE_ACCOUNT_PRIVATE_KEY, [
       'https://www.googleapis.com/auth/spreadsheets'
     ])
     this.auth = googleAuth
     const { sheetId } = this
+    const me = this
 
     return new Promise((resolve, reject) => {
       googleAuth.authorize(function (err) {
         if (err) {
           reject(err)
         } else {
-          resolve(`Connected to ${sheetId}.`)
+          console.log(`Connected to ${me.sheetName}. (${me.type} with ID ${sheetId}).`)
+          console.log(`    grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
+          resolve(me)
         }
       })
     })
@@ -159,39 +219,32 @@ class Fetcher {
       .then(() => true)
       .catch(() => false)
   }
+}
 
-  save (_tab, data) {
-    const tab = fmtName(_tab)
-
-    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
-      const bpConfig = this.blueprinters[tab]
-
-      if (isFunction(bpConfig)) {
-        // if bpConfig specifies a single blueprinter
-        return this._saveViaBlueprinter(tab, data, bpConfig)
-      } else {
-        // if bpConfig specifies an array of blueprinters
-        return bpConfig.map(this._saveViaBlueprinter(tab, data))
-      }
-    } else {
-      // NB: if a blueprinter is not specified for a tab,
-      // just skip it.
-      return true
-    }
+class LocalFetcher extends Fetcher {
+  constructor (db, name, bps, path) {
+    super(db, name, bps)
+    this.path = path
+    this.update().then(res =>
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
+    )
   }
 
-  // NB: could combine these functions by checking kwargs length
-  retrieve (tab, resource) {
-    const title = fmtName(tab)
-    const url = `${this.id}/${tab}/${resource}`
-    return this.db.load(url, this.blueprints[title])
-  }
-
-  retrieveFrag (tab, resource, frag) {
-    const title = fmtName(tab)
-    const url = `${this.sheetName}/${tab}/${resource}/${frag}`
-    return this.db.load(url, this.blueprints[title])
+  update () {
+    const wb = X.readFile(this.path)
+    wb.SheetNames.forEach(name => {
+      const sh = wb.Sheets[name]
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
+      const ll = csv.split('\n').map(line => line.split('\t'))
+      this.save(name, ll)
+    })
+    return Promise.resolve(true)
   }
 }
 
-export default Fetcher
+export default {
+  'gsheets': GsheetFetcher,
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
+}

src/lib/blueprinters.js

@@ -15,22 +15,39 @@ export const defaultResource = {
   data: []
 }
 
-export function buildDesaturated (sheetId, sheetName, tab, resource) {
+export function buildDesaturated (sheetId, sheetName, tab, resources) {
   const bp = R.clone(defaultBlueprint)
   bp.sheet.name = sheetName
   bp.sheet.id = sheetId
   bp.name = tab
-  bp.resources[resource] = null
+  bp.resources = resources.reduce((acc, r) => {
+    acc[r] = null
+    return acc
+  }, {})
   return bp
 }
 
+const buildBlueprinter = R.curry((datafierName, datafier, sheetId, sheetName, tabName, data) => {
+  const bp = R.clone(defaultBlueprint)
+  bp.sheet = {
+    name: sheetName,
+    id: sheetId
+  }
+  bp.name = tabName
+  bp.resources[datafierName] = R.clone(defaultResource)
+  bp.resources[datafierName].data = datafier(data)
+
+  return bp
+})
+
 // import all default exports from 'blueprinters' folder
 const allBps = {}
 const REL_PATH_TO_BPS = '../blueprinters'
 const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS)
 fs.readdirSync(normalizedPath).forEach(file => {
   const bpName = file.replace('.js', '')
-  allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default
+  const datafier = require(`${REL_PATH_TO_BPS}/${file}`).default
+  allBps[bpName] = buildBlueprinter(bpName, datafier)
 })
 
 // NB: revert to ES5 'module.exports' required to make blueprinters from

src/lib/errors.js

@@ -0,0 +1,19 @@
+import copy from '../copy/en'
+
+export function modelLayerGeneric (parts) {
+  return new Error(copy.errors.modelLayer(parts))
+}
+
+export function noFragment (parts) {
+  return new Error(copy.errors.noFragment(parts))
+}
+
+export function noResource (parts) {
+  return new Error(copy.errors.noResource(parts))
+}
+
+export default {
+  modelLayerGeneric,
+  noFragment,
+  noResource
+}

src/lib/util.js

@@ -1,4 +1,6 @@
 import R from 'ramda'
+import { promises as fs } from 'file-system'
+import copy from '../copy/en'
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {
@@ -14,6 +16,18 @@ function camelize (str) {
   })
 }
 
+export async function exportToFile (fileDest, data) {
+  const stringifiedData = JSON.stringify(data, null, 2)
+  const filePath = `${fileDest}/export.json`
+
+  try {
+    await fs.writeFile(filePath, stringifiedData)
+    return copy.success.export(filePath)
+  } catch (err) {
+    throw new Error(copy.errors.export.writeFailed)
+  }
+}
+
 export const fmtObj = R.curry(
   (
     columnNames,
@@ -37,7 +51,8 @@ export const fmtObj = R.curry(
       }
     }
     columnNames.forEach((columnName, idx) => {
-      obj[fmtColName(columnName)] = row[idx]
+      const value = row[idx] ? row[idx] : ''
+      obj[fmtColName(columnName)] = value
     })
     return obj
   }

src/middleware/index.js

@@ -1,5 +1,4 @@
 import { Router } from 'express'
-import { mapboxAccessToken } from '../config'
 import morgan from 'morgan'
 import mapbox from './mapbox'
 
@@ -10,8 +9,8 @@ export default ({ config, db }) => {
   /* logging middleware */
   routes.use(morgan('dev'))
 
-  if (mapboxAccessToken) {
-    routes.get('/mapbox/:z/:y/:x', mapbox(mapboxAccessToken))
+  if (process.env.MAPBOX_TOKEN) {
+    routes.get('/mapbox/:z/:y/:x', mapbox(process.env.MAPBOX_TOKEN))
   }
 
   return routes

src/models/Interface.js

@@ -1,30 +1,41 @@
 /* eslint-disable */
 /**
- * Model is a class whose sole responsibility is to save and load blueprints.
- * It allows for different storage mechanisms for different kinds of blueprints.
+ * Model is a class whose sole responsibility is to save and load data through a custom URL format.
+ * As an interfacce, it allows for different storage mechanisms, and different scale/performance for different kinds of data.
  * 
  * ERRORS:
  * When a load function fails, it should throw either:
- * 1. A __ error if the resource doesn't exist on that sheet/tab.
- * 2. A __ error if a fragment lookup fails because it doesn't exist.
- * 3. A __ error if something else goes wrong.
+ * 1. noResource(parts) if the resource doesn't exist on that sheet/tab.
+ * 2. noFragment(parts) if a fragment lookup fails because it doesn't exist.
+ * 3. modelLayerGeneric(parts) if something else goes wrong.
+ *
+ * This is a WIP layer. See StoreJson.js for an example in action.
  */
 class Model {
   /**
-   * save - save a Blueprint, using the information it contains.
-   *
-   * @param  {type} blueprint the Blueprint to be saved.
-   * @return {type} Promise   which returns True.
+   * Index the data stored by this model, returning a list of the available URLs. 
+   * @return {Promise(boolean)} Unpacks to a list of available URLs if successful, throws an error otherwise.
    */
-  save (blueprint) {}
+  index () {}
+  
+  /**
+   * Save data at a URL. The URL is in the format
+   *    /:fetcherID/:tab/:resource
+   * Fetcher IDs must be unique, tabs and resources can be duplicated across
+   * different fetchers.
+   * 
+   * @param  {string} url - the URL at which to save the data.
+   * @param  {object} data - the data to be saved.
+   * @return {Promise(boolean)} Unpacks to true if the update was successful, false if otherwise.
+   */
+  save (url, data) {}
 
   /**
-   * load - load a resource from a data model, using a Blueprint object as
-   * well as a REST-like  URL of the format /:source/:tab/:resource.
-   *
-   * @param  {type} url       String that represents the path to resource.
-   * @param  {type} blueprint Blueprint object (desaturated?).
-   * @return {type}           Object containing the resource data.
+   * Load data from a URL, in the format
+   *    /:fetcherID/:tab/:resource
+   * 
+   * @param  {string} url - the URL at which to load the data.
+   * @return {Promise(object)} a Promise that unpacks to the data retrieved. An error will be thrown if the URL is invalid.
    */
-  load (url, blueprint) {}
+  load (url) {}
 }

src/models/StoreJson.js

@@ -1,7 +1,7 @@
 import fs from 'mz/fs'
-import copy from '../copy/en'
+import errors from '../lib/errors'
 
-const STORAGE_DIRNAME = 'temp'
+const STORAGE_DIRNAME = 'data'
 
 function partsFromFilename (fname) {
   const body = fname.slice(0, -5)
@@ -45,7 +45,7 @@ class StoreJson {
             if (!isNaN(id) && id >= 0 && id < data.length) {
               return data[id]
             } else {
-              throw new Error(copy.errors.noFragment(parts))
+              throw errors.noFragment(parts)
             }
           } else {
             // Do a lookup if fragment is included to filter a relevant item
@@ -53,12 +53,12 @@ class StoreJson {
             if (!isNaN(index) && index >= 0 && index < data.length) {
               return data.filter((vl, idx) => idx === index)[0]
             } else {
-              throw new Error(copy.errors.noFragment(parts))
+              throw errors.noFragment(parts)
             }
           }
         })
     } else {
-      return Promise.reject(new Error(copy.errors.noResource(parts)))
+      return Promise.reject(errors.noResource(parts))
     }
   }
 

temp/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

test/internals.js

@@ -1,12 +1,11 @@
 import test from 'ava'
-import R from 'ramda'
 import {
-  defaultBlueprint,
-  defaultResource,
-  columns,
-  rows
+  defaultBlueprint
 } from '../src/lib/blueprinters'
 
+import rows from '../src/blueprinters/rows'
+import deeprows from '../src/blueprinters/deeprows'
+
 const egInput1 = [
   ['h1', 'h2', 'h3'],
   [1, 2, 3],
@@ -25,41 +24,20 @@ test('defaultBlueprint exports', t => {
   t.deepEqual(expected, defaultBlueprint)
 })
 
-test('columns blueprinter generates expected output', t => {
-  const actual = columns('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1)
-  const expected = R.clone(defaultBlueprint)
-  expected.name = 'eg ColumnBlueprint'
-  expected.sheet = {
-    id: 'egSheetId',
-    name: 'egSheetName'
-  }
-  expected.resources['h1'] = R.clone(defaultResource)
-  expected.resources['h1'].data = [1, 4]
-  expected.resources['h2'] = R.clone(defaultResource)
-  expected.resources['h2'].data = [2, 5]
-  expected.resources['h3'] = R.clone(defaultResource)
-  expected.resources['h3'].data = [3, 6]
+test('rows blueprinter', t => {
+  const expected = [
+    { h1: 1, h2: 2, h3: 3 },
+    { h1: 4, h2: 5, h3: 6 }
+  ]
+  const actual = rows(egInput1)
   t.deepEqual(expected, actual)
 })
 
-test('rows blueprinter generates expected output', t => {
-  const actual = rows('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items')
-  const expected = R.clone(defaultBlueprint)
-  expected.name = 'egRowBlueprint'
-  expected.sheet = {
-    id: 'egSheetId',
-    name: 'egSheetName'
-  }
-  expected.resources['items'] = R.clone(defaultResource)
-  expected.resources['items'].data = [{
-    h1: 1,
-    h2: 2,
-    h3: 3
-  },
-  {
-    h1: 4,
-    h2: 5,
-    h3: 6
-  }]
+test('deeprows blueprinter', t => {
+  const expected = [
+    { 'hs': [1, 2, 3] },
+    { 'hs': [4, 5, 6] }
+  ]
+  const actual = deeprows(egInput1)
   t.deepEqual(expected, actual)
 })

test/serverProcess.js

@@ -0,0 +1,90 @@
+import test from 'ava'
+import fetch from 'node-fetch'
+import childProcess from 'child_process'
+
+const SERVER_LAUNCH_WAIT_TIME = 10 * 1000
+const SERVER_ROOT = 'http://localhost:4040'
+let serverProc = null
+let serverExited = false
+function checkStatus (res) {
+  if (res.ok) {
+    return res
+  } else {
+    throw new Error('Route is not present')
+  }
+}
+
+/* SETUP: launch a development server with a wait time */
+test.before.cb(t => {
+  console.log('SETUP: launching server and updating...')
+  serverProc = childProcess.spawn('yarn', ['dev'], {
+    cwd: '.',
+    stdio: 'ignore'
+  })
+
+  serverProc.on('exit', function (code, signal) {
+    serverExited = true
+  })
+
+  function pingUpdate () {
+    const expected = {
+      success: 'All sheets updated'
+    }
+
+    return fetch(`${SERVER_ROOT}/api/update`)
+      .then(checkStatus)
+      .then(res => res.json())
+      .then(json => {
+        t.deepEqual(json, expected)
+        t.end()
+      })
+  }
+
+  setTimeout(pingUpdate, SERVER_LAUNCH_WAIT_TIME)
+})
+
+/* CLEANUP: kill the server */
+test.after(function () {
+  console.log('killing server...')
+  serverProc.kill('SIGKILL')
+})
+
+test('should launch', t => {
+  t.false(serverExited)
+})
+
+const passUrls = [
+  '/api/'
+]
+
+const failUrls = [
+  // /:sheet
+  '/api/example',
+  // /:sheet/:tab
+  '/api/example/events'
+]
+
+passUrls.forEach(function (url) {
+  test(`should respond successfully to request for ${url}`, t => {
+    return fetch(`${SERVER_ROOT}${url}`)
+      .then(checkStatus)
+      .then(res => res.json())
+      .then(json => {
+        console.info('JSON: ', json)
+        t.pass()
+      })
+  })
+})
+
+failUrls.forEach(function (url) {
+  test(`should respond with 404 for ${url}`, t => {
+    return fetch(`${SERVER_ROOT}${url}`)
+      .then(res => {
+        if (!res.ok) {
+          t.pass()
+        } else {
+          t.fail()
+        }
+      })
+  })
+})

views/blueprints.hbs

@@ -0,0 +1,91 @@
+<h1>Available Endpoints</h1>
+<hr>
+<div class="main-container">
+  {{#each bps}}
+  <div class="blueprint-container">
+    <div class="bp-header">
+      <div class="bp-tab">{{ tab }}</div>
+      <div class="bp-source">{{ source }}</div>
+    </div>
+    {{#each urls}}
+    <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
+    {{/each}}
+  </div>
+  {{ else }}
+  <div>No endpoints found. Have you updated?</div>
+  {{/each}}
+</div>
+
+<a class="bp-update-container" target="_blank" href="http://localhost:4040/api/update">
+  <div class="bp-button">Update</div>
+</div>
+
+
+<style>
+:root {
+  --grey: #8a8a8a;
+  --btn-width: 200px;
+}
+.main-container {
+  display: flex;
+  flex-wrap: wrap;
+}
+
+.blueprint-container {
+  display: flex;
+  flex: 1 0 31%;
+  flex-direction: column;
+  border: 1px solid var(--grey);
+  padding: 1em;
+  margin: 1em;
+}
+
+.bp-header {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+}
+
+.bp-tab {
+  font-size: 24pt;
+}
+
+.bp-source {
+  color: var(--grey);
+}
+
+.bp-update-container {
+  position: fixed;
+  top: 2px;
+  right: 2em;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+
+.bp-button {
+  transition: all 0.3s ease;
+  text-transform: uppercase;
+  font-size: 24pt;
+  font-weight: bold;
+  width: var(--btn-width);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  border: 3px solid black;
+  padding: .5em;
+  text-decoration: none;
+  text-underline: none;
+  color: black;
+}
+
+.bp-update-container:hover {
+  background-color: black;
+  transition: all 0.3s ease;
+  cursor: pointer;
+}
+
+.bp-update-container:hover div {
+  color: white;
+}
+</style>

views/layouts/default.hbs

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+  <title>Datasheet Server</title>
+  <meta name="HandheldFriendly" content="True" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
+</head>
+<body>
+  <div class="container">
+	  {{{body}}}
+  </div>
+</body>
+</html>