Merge pull request #36 from forensic-architecture/topic/add-columns

Topic/add columns

.env.example

@@ -0,0 +1,4 @@
+PORT=4040
+MAPBOX_TOKEN=pk.ANOTHERLONGSTRING.pMXNkIn0.xjjmguLIeX-r8FWomVG8Tg
+SERVICE_ACCOUNT_EMAIL="DUMMY-SERVICE-WORKER-NAME@DUMMY-PROJECT-NAME.iam.gserviceaccount.com"
+SERVICE_ACCOUNT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\A-vErY-l0nG-sTr1Ng\n-----END PRIVATE KEY-----\n"

.gitignore

@@ -5,6 +5,9 @@
 .DS_Store
 *.swp
 
+.env
 *service-account-key\.json
 src/config.js
 /yarn-error.log
+*.pem
+.travis.yml.old

.travis.yml

@@ -1,15 +1,17 @@
 language: node_js
 node_js:
-  - "stable"
+- stable
 cache:
   directories:
-    - node_modules
+  - node_modules
 before_script:
-  - npm install -g yarn
-  - cp ./src/example.config.js ./src/config.js
+- npm install -g yarn
 install:
-  - yarn
+- yarn
 script:
-  - yarn build
-  - yarn lint
-  - yarn test
+- yarn build
+- yarn lint
+- yarn test
+before_install:
+- openssl aes-256-cbc -K $encrypted_eeb74e8d35d1_key -iv $encrypted_eeb74e8d35d1_iv
+  -in .env.enc -out .env -d

CONTRIBUTING.md

@@ -45,11 +45,32 @@ WIP
 
 ### Before making changes
 
-WIP. Here we'll describe what the expected process and workflow is when making code changes, with regards to branching, forking and so on.
+1. If you are a contributor, you will need to create a fork of this repository
+   on your own GitHub handle, as you will not have commit access to the
+   forensic architecture repo.
+2. Create a new branch from _develop_ (not master or staging). The branch
+   should be prefixed with 'topic/' if you are intending to submit a feature
+   ('enhancement' tag in the issue), or with 'fix/' if you are fixing a bug
+   ('bug' tag in the issue).
+
+All of your commits go in this branch. When the feature/fix is complete, follow
+the instructions below to submit a PR for the branch.
 
 ### Submitting changes as Pull Requests
 
-WIP
+In order to submit a branch as a PR, you'll need to install the [Travis CLI](https://github.com/travis-ci/travis.rb). The documentation for this is a little shifty: if you're developing on a Mac, you can easily install it with `brew install travis`. The Travis CLI is necessary so that you can encrypt your service account credentials and use them while testing in Travis CI.
+
+To do this, you need to run one extra command before you push commits
+to a remote branch:
+```
+npm run travis-encrypt
+```
+This command encrypts your private key and service account email in .env in
+such a way that they can still be used while running tests on Travis' server.
+This command will add a commit to your branch that modifies the binary file
+.env.enc, and updates your Travis config accordingly. After running this
+command, you should be able to pass the pre-push check and run tests in the
+Travis server.
 
 ## Additional resources
 

README.md

@@ -58,21 +58,27 @@ A desaturated Blueprint can be saturated by retrieving its data from the server'
 A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
 
 ## [Configuration](#configuration)
-Copy the [example.config.js](/src/example.config.js) in the [src](/src) directory into a file named 'config.js'. Modify the options in this file accordingly:
+Copy the example environment file:
+```
+cp .env.example .env
+```
+Inside this file, you will need to modify at least the `SERVICE_ACCOUNT_EMAIL` and `SERVICE_ACCOUNT_PRIVATE_KEY` fields. These fields refer to the credentials of a [Google service account](https://cloud.google.com/iam/docs/understanding-service-accounts). Google requires that developers create these when attempting to access their services programmatically, so that they can attribute requests to users. Service accounts also contain identity information, which means that asset owners can allow certain service accounts access to certain sheets, just as one might differentially grant certain users access to certain cloud assets. 
 
-| Option  | Description | Type |
-| ------- | ----------- | ---- |
-| port | The port at which the server will make data available.  | integer |
-| googleSheets  | The configuration object for [Google Sheet](https://www.google.co.uk/sheets/about/) data sources. See the [Sources](#source-google-sheets) section below. | object |
+Once you have [created a service account](https://support.google.com/a/answer/7378726?hl=en), create and download an [API key for that account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). The JSON file for the API key that you download when you create it contains both a service account private key, and an email associated with the service account: add these respectively in the strings in .env for `SERVICE_ACCOUNT_PRIVATE_KEY` and `SERVICE_ACCOUNT_EMAIL`.
+
+The last thing to do is to grant the service account access to the sheet that
+datasheet-server is pulling from. You can add a service account to a sheet as
+you would any other Google user: just enter the email address associated. (Note
+that this step is not necessary if you are accessing a publicly available
+sheet.)
+
+Other configuration options, such as the port at which the server will expose
+resources, are also modifiable from the .env file.
 
 #### [Sources](#sources)
-###### [Google Sheets](#source-google-sheets)
-In order to make the data from a Sheet accessible to the server, you need to [create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts). Once created, give the service account email access to each Sheet from which you want to serve data. ('View Only' access is sufficient, as the server never modifies data.)
+Sources are specified in [src/sheets_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/sheets_config.js). Datasheet server currently only supports Google Sheets as a source.
 
-| Option | Description | Type |
-| ------ | ----------- | ---- |
-| email | The email address of the service account. This is available in the downloadable service account JSON in the `client_email` field. | string |
-| privateKey | The private key associated with the service account. This is available in the downloadable service account JSON in the `private_key` field. | string |
+###### [Google Sheets](#source-google-sheets)
 | sheets | A list of objects, one for each sheet that is being used as a source. Each sheet object has a `name` (String), an `id` (String), and a `tabs` (object) field, which are explained below. | object |
 
 Each Google Sheet being used as a as source requires a corresponding object in `sheets`. The object should be structured as follows:
@@ -88,7 +94,6 @@ Each Google Sheet being used as a as source requires a corresponding object in `
 import BP from './lib/blueprinters'
 
 export default {
-  port: 4040,
   googleSheets: {
     email: 'project-name@reliable-baptist-23338.iam.gserviceaccount.com',
     privateKey: 'SOME_PRIVATE_KEY',
@@ -97,8 +102,8 @@ export default {
         name: 'example',
         id: '1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI',
         tabs: {
-          'objects': BP.byRow,
-          'fruit': [BP.byRow, BP.byID],
+          'objects': BP.rows,
+          'fruit': [BP.columns, BP.ids],
         }
       },
     ]
@@ -113,8 +118,12 @@ Clone the repository to your local:
 ```
 git clone https://www.github.com/forensic-architecture/datasheet-server
 ```
+
+Follow the steps in the [configuration](#configuration) section of this
+document.
+
 ### Run with Docker
-To create a new instance of the server with [Docker](https://www.docker.com/) installed, clone the repository, create a `config.js`, and build the image:
+To create a new instance of the server with [Docker](https://www.docker.com/) installed, ensure that you have followed the steps in the quickstart guide above, then and build an image locally. (Note that Docker must be installed):
 ```sh
 docker build -t datasheet-server .
 ```
@@ -152,4 +161,4 @@ If you have any questions or just want to chat, please join our team [fa_open_so
 
 ## License
 
-TimeMap is distributed under the MIT License.
+Datasheet Server is distributed under the [MIT License](https://github.com/forensic-architecture/datasheet-server/blob/develop/LICENSE).

docs/architecture.md

@@ -0,0 +1,41 @@
+# Architecture
+
+![](datasheet-server-graphic.jpg)
+
+Datasheet Server uses the architecture diagrammed above to allow effective
+management of the sheets it represents.
+
+It exposes an API with four endpoints:
+ * `/:sheet/:tab/:resource` - the primary means of accessing data in the
+     server. The name of a sheet is specified in
+     [sheets_config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/sheets_config.js#L7).
+     Each tab on the sheet is in lower case, with spaces replaced by
+     underscores. The resource name is the same as the name of the blueprinter
+     specified for the tab is sheets_config.js.
+ * `/:sheet/:tab/:resource/:id` - when the resource is a list, items in the list
+     can be accessed individually as well.
+ * `/update` - when this route is queried with a GET request, each of the
+     fetchers in the server will update their models from the relevant sheet.
+ * `/blueprints` - a JSON object that represents all available sheets, tabs, and
+     resources in the server.
+
+# Controller
+The [controller](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/lib/Controller.js)
+manages all of the fetchers in the server. Its `update` triggers the update
+mechanisms of all the `update`s in the fetchers it manages. Its `retrieve` and
+`retrieveFrag` methods find the appropriate fetcher, and trigger its 
+respectively named method. Its `bluerprints` method collects the blueprints
+from all fetchers, and presents them as a single list.
+
+# Fetcher
+The [fetcher](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/lib/Fetcher.js)
+is the most complex component in datasheet server. Its responsiblity is to
+interface with a sheet, and store the data from that sheet in a model. It also
+makes information from that model available to the controller.
+
+# Model
+The model layer is an [interface](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/models/Interface.js)
+that specifies how stored data must be made available to a fetcher. The
+implementation of the model layer ([StoreJson](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/models/StoreJson.js)
+is one example) determines the time and space complexity of retrieval.
+

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "grenfell-server",
+  "name": "datasheet-server",
   "version": "0.3.0",
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
@@ -7,9 +7,15 @@
     "dev": "NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
     "build": "NODE_ENV=production npx babel src -d dist",
     "start": "node dist",
-    "lint": "standard \"src/**/*.js\" \"test/**/*/js\"",
+    "lint": "standard \"src/**/*.js\" \"test/**/*.js\"",
     "test-watch": "ava --watch",
-    "test": "ava --verbose"
+    "test": "ava --verbose",
+    "travis-encrypt": "./scripts/encrypt.sh"
+  },
+  "husky": {
+    "hooks": {
+      "pre-push": "./scripts/check-branch.sh && yarn lint && yarn test"
+    }
   },
   "repository": {
     "type": "git",
@@ -21,13 +27,14 @@
     "body-parser": "^1.13.3",
     "compression": "^1.5.2",
     "cors": "^2.8.5",
+    "dotenv": "^6.1.0",
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "googleapis": "^32.0.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.2.0",
+    "node-fetch": "^2.3.0",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
     "resource-router-middleware": "^0.6.0"
@@ -36,9 +43,11 @@
     "@babel/cli": "^7.1.2",
     "@babel/core": "^7.1.2",
     "@babel/node": "^7.0.0",
+    "@babel/polyfill": "^7.0.0",
     "@babel/preset-env": "^7.1.0",
     "@babel/register": "^7.0.0",
     "ava": "1.0.0-beta.8",
+    "husky": "^1.2.0",
     "nodemon": "1.18.7",
     "standard": "^12.0.1"
   },
@@ -52,7 +61,8 @@
       "test/**/*.js"
     ],
     "require": [
-      "@babel/register"
+      "@babel/register",
+      "@babel/polyfill"
     ]
   },
   "bugs": {

scripts/check-branch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+ENCRYPT_MESSAGE="\nThe .env.enc file has not changed its contents from the version on origin/develop.\nTo ensure that tests pass in Travis, you need to encrypt the contents of .env  before pushing to the remote, so that the Travis server can use your service account credentials.\nEnsure that the Travis CLI is installed on your local, run\n\n\tnpm run travis-encrypt\n\nand then push to the remote again.\nIf you don't care whether your build passes on Travis, you can run:\n\n\tgit push --no-verify\n\nand bypass this check.\n\n"
+
+# check whether .env.enc has changed
+if [ -z "`git diff origin/develop -- .env.enc`" ]; then
+	echo $ENCRYPT_MESSAGE
+	exit 1
+fi
+
+

scripts/encrypt.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+echo "Encrypting .env file for Travis..."
+
+# confirm that the user has the core repo as origin, and is therefore a maintainer
+# if [ -z `git config --get remote.origin.url | grep "forensic-architecture/datasheet-server"` ]; then
+# 	echo "Travis encryption not required for satellite contributors, continuing.."
+# 	exit 0
+# fi
+
+# confirm travis is installed
+if [ ! hash travis 2>/dev/null ]; then
+	echo "============================================================================================"
+	echo "ERROR: Travis CLI is not installed on your local. Please install from:"
+	echo "\thttps://github.com/travis-ci/travis.rb"
+	echo "After installing, make sure that you login with:"
+	echo "\ttravis login --pro"
+	echo "============================================================================================"
+	exit 3
+fi
+
+# confirm there is a .env file to encrypt
+if [ ! -f .env ]; then
+	echo "============================================================================================"
+	echo "ERROR: You must create a .env file and add your credentials. See .env.example for an example"
+	echo "============================================================================================"
+	exit 3
+fi
+
+# regex to match and delete 'before_install' and everything after it
+# necessary to delete these lines to get Travis to build for multiple accounts
+echo "creating new .travis.yml configuration"
+sed -i.old '/^before_install.*/,$ d' .travis.yml
+echo "old config file saved as .travis.yml.old"
+
+travis encrypt-file .env --add --force --org
+git add .env.enc
+git add .travis.yml
+echo ".env.enc created and added to commit"

src/api/index.js

@@ -15,6 +15,20 @@ export default ({ config, controller }) => {
     res.json(controller.blueprints())
   })
 
+  api.get('/update', (req, res) => {
+    controller
+      .update()
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
+  })
+
   api.get('/:sheet/:tab/:resource/:frag', (req, res) => {
     const { sheet, tab, resource, frag } = req.params
     controller
@@ -27,8 +41,9 @@ export default ({ config, controller }) => {
   })
 
   api.get('/:sheet/:tab/:resource', (req, res) => {
+    const { sheet, tab, resource } = req.params
     controller
-      .retrieve(req.params.sheet, req.params.tab, req.params.resource)
+      .retrieve(sheet, tab, resource)
       .then(data => res.json(data))
       .catch(err =>
         res.status(err.status || 404)
@@ -36,30 +51,16 @@ export default ({ config, controller }) => {
       )
   })
 
-  api.get('/update', (req, res) => {
-    controller
-      .update()
-      .then(msg =>
-        res.json({
-          success: msg
-        })
-      )
-      .catch(err =>
-        res.status(404)
-          .send({ error: err.message })
-      )
-  })
-
   // ERROR routes. Note that it is important that these come AFTER routes
   // like /update, so that the regex does not greedily match these routes.
 
   api.get('/:sheet', (req, res) => {
-    res.status(404)
+    res.status(400)
       .send({ error: copy.errors.onlysheet })
   })
 
   api.get('/:sheet/:tab', (req, res) => {
-    res.status(404)
+    res.status(400)
       .send({ error: copy.errors.onlyTab })
   })
 

src/blueprinters/columns.js

@@ -1,37 +1,27 @@
-import R from 'ramda'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
-
 /**
- * byColumn - generate a Blueprint from a data sheet by column. Each column
- * name is a resheet, and all values in that column are the resheet items.
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
  *
- * @param  {type} data - list of lists representing sheet data.
- * @return {type} Blueprint
- * generated.
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-function columns (tabName, sheetName, sheetId, data) {
-  // Define Blueprint props
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
+export default (data) => {
+  const columnNames = data[0]
+  const columns = columnNames.map(name => ([]))
 
-  // column names define resources
-  const labels = data[0]
-  labels.forEach(label => {
-    bp.resources[label] = R.clone(defaultResource)
-  })
-
-  // remaining rows as data
   data.forEach((row, idx) => {
     if (idx === 0) return
-    labels.forEach((label, idx) => {
-      bp.resources[label].data.push(row[idx])
+    row.forEach((item, idx) => {
+      columns[idx].push(item)
     })
   })
-  return bp
-}
 
-export default columns
+  return columns.map((column, idx) => ({
+    name: columnNames[idx],
+    items: column
+  }))
+}

src/blueprinters/deepids.js

@@ -0,0 +1,22 @@
+import deeprows from './deeprows'
+
+/**
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
+ *
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Object      the structured data.
+ */
+export default (data) => {
+  const output = {}
+
+  deeprows(data).forEach(row => {
+    output[row.id] = row
+  })
+
+  return output
+}

src/blueprinters/deeprows.js

@@ -0,0 +1,71 @@
+import { fmtObj } from '../lib/util'
+
+/**
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
+ *
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
+ */
+export default (data) => {
+  const itemLabels = data[0]
+  const baseFmt = fmtObj(itemLabels)
+  const output = []
+
+  // create a structure to indicate which columns needs to be aggregated
+  const endsWithNumber = new RegExp('(.*?)[0-9]+$')
+  const structure = {
+    __flat: []
+  }
+
+  itemLabels.forEach(label => {
+    const matches = label.match(endsWithNumber)
+    if (!matches) {
+      structure.__flat.push(label)
+    } else {
+      const labelPrefix = `${matches[1]}s`
+      if (labelPrefix in structure) {
+        structure[labelPrefix].push(label)
+      } else {
+        structure[labelPrefix] = [ label ]
+      }
+    }
+  })
+
+  // generate the value for deep labels using the structure created
+  data.forEach((row, idx) => {
+    if (idx === 0) return
+    const baseRow = baseFmt(row)
+    const deepRow = {}
+
+    // generate deep row labels using structure
+    Object.keys(structure)
+      .forEach(newLabel => {
+        if (newLabel !== '__flat') {
+          const oldLabels = structure[newLabel]
+          // only add new value if not ''
+          const labelValues = []
+          oldLabels.forEach(l => {
+            const vl = baseRow[l]
+            if (vl !== '') {
+              labelValues.push(vl)
+            }
+          })
+          deepRow[newLabel] = labelValues
+        }
+      })
+
+    // move values for flat labels over from base
+    structure.__flat.forEach(label => {
+      deepRow[label] = baseRow[label]
+    })
+
+    output.push(deepRow)
+  })
+
+  return output
+}

src/blueprinters/groups.js

@@ -1,39 +1,16 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * groups - generate a Blueprint from a data sheet grouped by a column called 'group'
- * The resource name defaults to 'groups', or a custom resource name can be passed.
  * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * names. Items are inserted into the data list at idx = id.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="groups"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function groups (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'groups'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
-
+  const output = []
   const dataGroups = {}
 
   data.forEach((row, idx) => {
@@ -45,12 +22,14 @@ export default function groups (
       dataGroups[group].push(fmt(row))
     }
   })
-  Object.keys(dataGroups).forEach(groupKey => {
-    bp.resources[label].data.push({
-      group: groupKey,
-      group_label: dataGroups[groupKey][0].group_label,
-      data: dataGroups[groupKey]
+  Object.keys(dataGroups)
+    .forEach(groupKey => {
+      output.push({
+        group: groupKey,
+        group_label: dataGroups[groupKey][0].group_label,
+        data: dataGroups[groupKey]
+      })
     })
-  })
-  return bp
+
+  return output
 }

src/blueprinters/ids.js

@@ -1,42 +1,22 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * ids - generate a Blueprint from a data sheet by id, which is an integer.
- * The resource name defaults to 'ids', or a custom resource name can be passed.
- * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * Very similar to the rows blueprinter, but inserts each row as a value in
+ * an object, where the value in the 'id' column of the row will be used as
+ * the search key
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="ids"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Object      the structured data.
  */
-export default function ids (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'ids'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
+  const output = {}
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.resources[label].data[fmt(row).id] = fmt(row)
+    output[fmt(row).id] = fmt(row)
   })
-  return bp
+
+  return output
 }

src/blueprinters/rows.js

@@ -1,41 +1,21 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * rows - generate a Blueprint from a data sheet by row. The resource name
- * defaults to 'rows', or a custom resource name can be passed. Each resource
- * item is an object with values labelled according to column names.
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="rows" name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function rows (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'rows'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
+  const output = []
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.resources[label].data.push(fmt(row))
+    output.push(fmt(row))
   })
-  return bp
+
+  return output
 }

src/blueprinters/tree.js

@@ -1,38 +1,12 @@
-import R from 'ramda'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
-
 /**
- * tree - generate a Blueprint from a data sheet grouped by a column called 'group'
- * The resource name defaults to 'groups', or a custom resource name can be passed.
- * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * Each resource item is inserted into a tree. TODO: describe layout.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="groups"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function tree (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'tree'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = {}
-
+export default (data) => {
   const tree = {
-    key: 'tags',
+    key: '_root',
     children: {}
   }
 
@@ -62,6 +36,5 @@ export default function tree (
     }
   })
 
-  bp.resources[label].data = tree
-  return bp
+  return tree
 }

src/copy/en.js

@@ -5,7 +5,8 @@ export default {
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
     noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' in this sheet.`,
-    noFragment: prts => `Fragment index does not exist`
+    noFragment: prts => `Fragment index does not exist`,
+    modelLayer: prts => `Something went wrong at the model layer`
   },
   success: {
     update: 'All sheets updated'

src/example.config.js

@@ -1,21 +0,0 @@
-import BP from './lib/blueprinters'
-
-export default {
-  port: 4040,
-  googleSheets: {
-    email: 'SOME_SERVICE_ACCOUNT_EMAIL',
-    privateKey: 'SOME_SERVICE_ACCOUNT_PRIVATE_KEY',
-    sheets: [
-      {
-        name: 'example',
-        id: '1UC7DkCFeUXHfpUxUGruExwFbP4pqVBdJLOKfo6wDDGk',
-        tabs: {
-          export_events: [BP.byId, BP.byRow],
-          export_categories: [BP.byGroup, BP.byRow],
-          export_sites: BP.byRow,
-          export_tags: BP.byTree
-        }
-      }
-    ]
-  }
-}

src/index.js

@@ -3,7 +3,10 @@ import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
 import api from './api'
-import config from './config'
+// import config from './sheets_config'
+import dotenv from 'dotenv'
+
+dotenv.config()
 
 let app = express()
 app.server = http.createServer(app)
@@ -15,6 +18,8 @@ if (process.env.NODE_ENV === 'development') {
   app.use(cors())
 }
 
+const config = process.env
+
 initialize(controller => {
   app.use(
     middleware({
@@ -30,7 +35,7 @@ initialize(controller => {
     })
   )
 
-  app.server.listen(process.env.PORT || config.port, () => {
+  app.server.listen(process.env.PORT || 4040, () => {
     console.log(`Started on port ${app.server.address().port}`)
   })
 })

src/initialize.js

@@ -1,13 +1,13 @@
 import StoreJson from './models/StoreJson'
 import Fetcher from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
+import sheetsConfig from './sheets_config'
 
-const { googleSheets } = config
-const { sheets, privateKey, email } = googleSheets
+const { googleSheets } = sheetsConfig
+const { sheets } = googleSheets
 
 function authenticate (_fetcher) {
-  return _fetcher.fetcher.authenticate(email, privateKey).then(msg => {
+  return _fetcher.fetcher.authenticate(process.env.SERVICE_ACCOUNT_EMAIL, process.env.SERVICE_ACCOUNT_PRIVATE_KEY).then(msg => {
     console.log(msg)
     return true
   })
@@ -24,7 +24,7 @@ export default callback => {
   Promise.all(fetchers.map(authenticate))
     .then(() => {
       console.log(`===================`)
-      console.log(`grant access to: ${email}`)
+      console.log(`grant access to: ${process.env.SERVICE_ACCOUNT_EMAIL}`)
       console.log(`===================`)
 
       // NB: reformat fetchers as config for controller
@@ -39,7 +39,7 @@ export default callback => {
       console.log(err)
       console.log(
         `ERROR: the server couldn't connect to all of the sheets you provided. Ensure you have granted access to ${
-          email
+          process.env.SERVICE_ACCOUNT_EMAIL
         } on ALL listed sheets.`
       )
     })

src/lib/Fetcher.js

@@ -91,9 +91,9 @@ class Fetcher {
    */
   _saveViaBlueprinter (tab, data, blueprinter) {
     const saturatedBp = blueprinter(
-      tab,
-      this.sheetName,
       this.sheetId,
+      this.sheetName,
+      tab,
       data
     )
 
@@ -189,7 +189,7 @@ class Fetcher {
 
   retrieveFrag (tab, resource, frag) {
     const title = fmtName(tab)
-    const url = `${this.sheetName}/${tab}/${resource}/${frag}`
+    const url = `${this.id}/${tab}/${resource}/${frag || ''}`
     return this.db.load(url, this.blueprints[title])
   }
 }

src/lib/blueprinters.js

@@ -24,13 +24,27 @@ export function buildDesaturated (sheetId, sheetName, tab, resource) {
   return bp
 }
 
+const buildBlueprinter = R.curry((datafierName, datafier, sheetId, sheetName, tabName, data) => {
+  const bp = R.clone(defaultBlueprint)
+  bp.sheet = {
+    name: sheetName,
+    id: sheetId
+  }
+  bp.name = tabName
+  bp.resources[datafierName] = R.clone(defaultResource)
+  bp.resources[datafierName].data = datafier(data)
+
+  return bp
+})
+
 // import all default exports from 'blueprinters' folder
 const allBps = {}
 const REL_PATH_TO_BPS = '../blueprinters'
 const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS)
 fs.readdirSync(normalizedPath).forEach(file => {
   const bpName = file.replace('.js', '')
-  allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default
+  const datafier = require(`${REL_PATH_TO_BPS}/${file}`).default
+  allBps[bpName] = buildBlueprinter(bpName, datafier)
 })
 
 // NB: revert to ES5 'module.exports' required to make blueprinters from

src/lib/errors.js

@@ -0,0 +1,19 @@
+import copy from '../copy/en'
+
+export function modelLayerGeneric (parts) {
+  return new Error(copy.errors.modelLayer(parts))
+}
+
+export function noFragment (parts) {
+  return new Error(copy.errors.noFragment(parts))
+}
+
+export function noResource (parts) {
+  return new Error(copy.errors.noResource(parts))
+}
+
+export default {
+  modelLayerGeneric,
+  noFragment,
+  noResource
+}

src/lib/util.js

@@ -37,7 +37,8 @@ export const fmtObj = R.curry(
       }
     }
     columnNames.forEach((columnName, idx) => {
-      obj[fmtColName(columnName)] = row[idx]
+      const value = row[idx] ? row[idx] : ''
+      obj[fmtColName(columnName)] = value
     })
     return obj
   }

src/middleware/index.js

@@ -1,5 +1,4 @@
 import { Router } from 'express'
-import { mapboxAccessToken } from '../config'
 import morgan from 'morgan'
 import mapbox from './mapbox'
 
@@ -10,8 +9,8 @@ export default ({ config, db }) => {
   /* logging middleware */
   routes.use(morgan('dev'))
 
-  if (mapboxAccessToken) {
-    routes.get('/mapbox/:z/:y/:x', mapbox(mapboxAccessToken))
+  if (process.env.MAPBOX_TOKEN) {
+    routes.get('/mapbox/:z/:y/:x', mapbox(process.env.MAPBOX_TOKEN))
   }
 
   return routes

src/models/Interface.js

@@ -1,30 +1,41 @@
 /* eslint-disable */
 /**
- * Model is a class whose sole responsibility is to save and load blueprints.
- * It allows for different storage mechanisms for different kinds of blueprints.
+ * Model is a class whose sole responsibility is to save and load data through a custom URL format.
+ * As an interfacce, it allows for different storage mechanisms, and different scale/performance for different kinds of data.
  * 
  * ERRORS:
  * When a load function fails, it should throw either:
- * 1. A __ error if the resource doesn't exist on that sheet/tab.
- * 2. A __ error if a fragment lookup fails because it doesn't exist.
- * 3. A __ error if something else goes wrong.
+ * 1. noResource(parts) if the resource doesn't exist on that sheet/tab.
+ * 2. noFragment(parts) if a fragment lookup fails because it doesn't exist.
+ * 3. modelLayerGeneric(parts) if something else goes wrong.
+ *
+ * This is a WIP layer. See StoreJson.js for an example in action.
  */
 class Model {
   /**
-   * save - save a Blueprint, using the information it contains.
-   *
-   * @param  {type} blueprint the Blueprint to be saved.
-   * @return {type} Promise   which returns True.
+   * Index the data stored by this model, returning a list of the available URLs. 
+   * @return {Promise(boolean)} Unpacks to a list of available URLs if successful, throws an error otherwise.
    */
-  save (blueprint) {}
+  index () {}
+  
+  /**
+   * Save data at a URL. The URL is in the format
+   *    /:fetcherID/:tab/:resource
+   * Fetcher IDs must be unique, tabs and resources can be duplicated across
+   * different fetchers.
+   * 
+   * @param  {string} url - the URL at which to save the data.
+   * @param  {object} data - the data to be saved.
+   * @return {Promise(boolean)} Unpacks to true if the update was successful, false if otherwise.
+   */
+  save (url, data) {}
 
   /**
-   * load - load a resource from a data model, using a Blueprint object as
-   * well as a REST-like  URL of the format /:source/:tab/:resource.
-   *
-   * @param  {type} url       String that represents the path to resource.
-   * @param  {type} blueprint Blueprint object (desaturated?).
-   * @return {type}           Object containing the resource data.
+   * Load data from a URL, in the format
+   *    /:fetcherID/:tab/:resource
+   * 
+   * @param  {string} url - the URL at which to load the data.
+   * @return {Promise(object)} a Promise that unpacks to the data retrieved. An error will be thrown if the URL is invalid.
    */
-  load (url, blueprint) {}
+  load (url) {}
 }

src/models/StoreJson.js

@@ -1,5 +1,5 @@
 import fs from 'mz/fs'
-import copy from '../copy/en'
+import errors from '../lib/errors'
 
 const STORAGE_DIRNAME = 'temp'
 
@@ -45,7 +45,7 @@ class StoreJson {
             if (!isNaN(id) && id >= 0 && id < data.length) {
               return data[id]
             } else {
-              throw new Error(copy.errors.noFragment(parts))
+              throw errors.noFragment(parts)
             }
           } else {
             // Do a lookup if fragment is included to filter a relevant item
@@ -53,12 +53,12 @@ class StoreJson {
             if (!isNaN(index) && index >= 0 && index < data.length) {
               return data.filter((vl, idx) => idx === index)[0]
             } else {
-              throw new Error(copy.errors.noFragment(parts))
+              throw errors.noFragment(parts)
             }
           }
         })
     } else {
-      return Promise.reject(new Error(copy.errors.noResource(parts)))
+      return Promise.reject(errors.noResource(parts))
     }
   }
 

src/sheets_config.js

@@ -0,0 +1,20 @@
+import BP from './lib/blueprinters'
+
+export default {
+  googleSheets: {
+    sheets: [
+      {
+        name: 'example',
+        id: '1UC7DkCFeUXHfpUxUGruExwFbP4pqVBdJLOKfo6wDDGk',
+        tabs: {
+          export_events: [BP.deeprows, BP.rows],
+          export_categories: [BP.groups, BP.rows],
+          export_narratives: BP.rows,
+          export_sources: BP.deepids,
+          export_sites: BP.rows,
+          export_tags: BP.tree
+        }
+      }
+    ]
+  }
+}

test/internals.js

@@ -1,12 +1,11 @@
 import test from 'ava'
-import R from 'ramda'
 import {
-  defaultBlueprint,
-  defaultResource,
-  columns,
-  rows
+  defaultBlueprint
 } from '../src/lib/blueprinters'
 
+import rows from '../src/blueprinters/rows'
+import deeprows from '../src/blueprinters/deeprows'
+
 const egInput1 = [
   ['h1', 'h2', 'h3'],
   [1, 2, 3],
@@ -25,41 +24,20 @@ test('defaultBlueprint exports', t => {
   t.deepEqual(expected, defaultBlueprint)
 })
 
-test('columns blueprinter generates expected output', t => {
-  const actual = columns('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1)
-  const expected = R.clone(defaultBlueprint)
-  expected.name = 'eg ColumnBlueprint'
-  expected.sheet = {
-    id: 'egSheetId',
-    name: 'egSheetName'
-  }
-  expected.resources['h1'] = R.clone(defaultResource)
-  expected.resources['h1'].data = [1, 4]
-  expected.resources['h2'] = R.clone(defaultResource)
-  expected.resources['h2'].data = [2, 5]
-  expected.resources['h3'] = R.clone(defaultResource)
-  expected.resources['h3'].data = [3, 6]
+test('rows blueprinter', t => {
+  const expected = [
+    { h1: 1, h2: 2, h3: 3 },
+    { h1: 4, h2: 5, h3: 6 }
+  ]
+  const actual = rows(egInput1)
   t.deepEqual(expected, actual)
 })
 
-test('rows blueprinter generates expected output', t => {
-  const actual = rows('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items')
-  const expected = R.clone(defaultBlueprint)
-  expected.name = 'egRowBlueprint'
-  expected.sheet = {
-    id: 'egSheetId',
-    name: 'egSheetName'
-  }
-  expected.resources['items'] = R.clone(defaultResource)
-  expected.resources['items'].data = [{
-    h1: 1,
-    h2: 2,
-    h3: 3
-  },
-  {
-    h1: 4,
-    h2: 5,
-    h3: 6
-  }]
+test('deeprows blueprinter', t => {
+  const expected = [
+    { 'hs': [1, 2, 3] },
+    { 'hs': [4, 5, 6] }
+  ]
+  const actual = deeprows(egInput1)
   t.deepEqual(expected, actual)
 })

test/serverProcess.js

@@ -0,0 +1,96 @@
+import test from 'ava'
+import fetch from 'node-fetch'
+import childProcess from 'child_process'
+
+const SERVER_LAUNCH_WAIT_TIME = 10 * 1000
+const SERVER_ROOT = 'http://localhost:4040'
+let serverProc = null
+let serverExited = false
+function checkStatus (res) {
+  if (res.ok) {
+    return res
+  } else {
+    throw new Error('Route is not present')
+  }
+}
+
+/* SETUP: launch a development server with a wait time */
+test.before.cb(t => {
+  console.log('SETUP: launching server and updating...')
+  serverProc = childProcess.spawn('yarn', ['dev'], {
+    cwd: '.',
+    stdio: 'ignore'
+  })
+
+  serverProc.on('exit', function (code, signal) {
+    serverExited = true
+  })
+
+  function pingUpdate () {
+    const expected = {
+      success: 'All sheets updated'
+    }
+
+    return fetch(`${SERVER_ROOT}/api/update`)
+      .then(checkStatus)
+      .then(res => res.json())
+      .then(json => {
+        t.deepEqual(json, expected)
+        t.end()
+      })
+  }
+
+  setTimeout(pingUpdate, SERVER_LAUNCH_WAIT_TIME)
+})
+
+/* CLEANUP: kill the server */
+test.after(function () {
+  console.log('killing server...')
+  serverProc.kill('SIGKILL')
+})
+
+test('should launch', t => {
+  t.false(serverExited)
+})
+
+const passUrls = [
+  // /
+  '/api/',
+  // /blueprints
+  '/api/blueprints',
+  // /:sheet/:tab/:resource
+  '/api/example/export_events/rows',
+  // /:sheet/:tab/:resource/:frag
+  '/api/example/export_events/rows/1'
+]
+
+const failUrls = [
+  // /:sheet
+  '/api/example',
+  // /:sheet/:tab
+  '/api/example/events'
+]
+
+passUrls.forEach(function (url) {
+  test(`should respond successfully to request for ${url}`, t => {
+    return fetch(`${SERVER_ROOT}${url}`)
+      .then(checkStatus)
+      .then(res => res.json())
+      .then(json => {
+        t.pass()
+      })
+  })
+})
+
+failUrls.forEach(function (url) {
+  test(`should respond with 404 for ${url}`, t => {
+    return fetch(`${SERVER_ROOT}${url}`)
+      .then(res => {
+        if (!res.ok) {
+          t.pass()
+        } else {
+          t.fail()
+        }
+      })
+  })
+})

yarn.lock

@@ -996,16 +996,32 @@ call-signature@0.0.2:
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/call-signature/-/call-signature-0.0.2.tgz#a84abc825a55ef4cb2b028bd74e205a65b9a4996"
 
+caller-callsite@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/caller-callsite/-/caller-callsite-2.0.0.tgz#847e0fce0a223750a9a027c54b33731ad3154134"
+  dependencies:
+    callsites "^2.0.0"
+
 caller-path@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/caller-path/-/caller-path-0.1.0.tgz#94085ef63581ecd3daa92444a8fe94e82577751f"
   dependencies:
     callsites "^0.2.0"
 
+caller-path@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/caller-path/-/caller-path-2.0.0.tgz#468f83044e369ab2010fac5f06ceee15bb2cb1f4"
+  dependencies:
+    caller-callsite "^2.0.0"
+
 callsites@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-0.2.0.tgz#afab96262910a7f33c19a5775825c69f34e350ca"
 
+callsites@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/callsites/-/callsites-2.0.0.tgz#06eb84f00eea413da86affefacbffb36093b3c50"
+
 camelcase-keys@^4.0.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-4.2.0.tgz#a2aa5fb1af688758259c32c141426d78923b9b77"
@@ -1274,6 +1290,15 @@ cors@^2.8.5:
     object-assign "^4"
     vary "^1"
 
+cosmiconfig@^5.0.6:
+  version "5.0.7"
+  resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.0.7.tgz#39826b292ee0d78eda137dfa3173bd1c21a43b04"
+  dependencies:
+    import-fresh "^2.0.0"
+    is-directory "^0.3.1"
+    js-yaml "^3.9.0"
+    parse-json "^4.0.0"
+
 create-error-class@^3.0.0:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/create-error-class/-/create-error-class-3.0.2.tgz#06be7abef947a3f14a30fd610671d401bca8b7b6"
@@ -1288,7 +1313,7 @@ cross-spawn@^5.0.1:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
-cross-spawn@^6.0.5:
+cross-spawn@^6.0.0, cross-spawn@^6.0.5:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
   dependencies:
@@ -1470,6 +1495,10 @@ dot-prop@^4.1.0, dot-prop@^4.2.0:
   dependencies:
     is-obj "^1.0.0"
 
+dotenv@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-6.1.0.tgz#9853b6ca98292acb7dec67a95018fa40bccff42c"
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -1503,6 +1532,12 @@ encodeurl@~1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
 
+end-of-stream@^1.1.0:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43"
+  dependencies:
+    once "^1.4.0"
+
 equal-length@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/equal-length/-/equal-length-1.0.1.tgz#21ca112d48ab24b4e1e7ffc0e5339d31fdfc274c"
@@ -1741,6 +1776,18 @@ execa@^0.7.0:
     signal-exit "^3.0.0"
     strip-eof "^1.0.0"
 
+execa@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
+  dependencies:
+    cross-spawn "^6.0.0"
+    get-stream "^4.0.0"
+    is-stream "^1.1.0"
+    npm-run-path "^2.0.0"
+    p-finally "^1.0.0"
+    signal-exit "^3.0.0"
+    strip-eof "^1.0.0"
+
 expand-brackets@^2.1.4:
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622"
@@ -1910,6 +1957,12 @@ find-up@^2.0.0, find-up@^2.1.0:
   dependencies:
     locate-path "^2.0.0"
 
+find-up@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
+  dependencies:
+    locate-path "^3.0.0"
+
 flat-cache@^1.2.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.3.0.tgz#d3030b32b38154f4e3b7e9c709f490f7ef97c481"
@@ -2009,6 +2062,12 @@ get-stream@^3.0.0:
   version "3.0.0"
   resolved "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14"
 
+get-stream@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
+  dependencies:
+    pump "^3.0.0"
+
 get-value@^2.0.3, get-value@^2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
@@ -2226,6 +2285,21 @@ http-errors@^1.3.0:
     statuses ">= 1.5.0 < 2"
     toidentifier "1.0.0"
 
+husky@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/husky/-/husky-1.2.0.tgz#d631dda1e4a9ee8ba69a10b0c51a0e2c66e711e5"
+  dependencies:
+    cosmiconfig "^5.0.6"
+    execa "^1.0.0"
+    find-up "^3.0.0"
+    get-stdin "^6.0.0"
+    is-ci "^1.2.1"
+    pkg-dir "^3.0.0"
+    please-upgrade-node "^3.1.1"
+    read-pkg "^4.0.1"
+    run-node "^1.0.0"
+    slash "^2.0.0"
+
 iconv-lite@0.4.23:
   version "0.4.23"
   resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
@@ -2256,6 +2330,13 @@ ignore@^4.0.2:
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
 
+import-fresh@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-2.0.0.tgz#d81355c15612d386c61f9ddd3922d4304822a546"
+  dependencies:
+    caller-path "^2.0.0"
+    resolve-from "^3.0.0"
+
 import-lazy@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43"
@@ -2358,7 +2439,7 @@ is-callable@^1.1.3, is-callable@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.4.tgz#1e1adf219e1eeb684d691f9d6a05ff0d30a24d75"
 
-is-ci@^1.0.10, is-ci@^1.2.0:
+is-ci@^1.0.10, is-ci@^1.2.0, is-ci@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-1.2.1.tgz#e3779c8ee17fccf428488f6e281187f2e632841c"
   dependencies:
@@ -2396,6 +2477,10 @@ is-descriptor@^1.0.0, is-descriptor@^1.0.2:
     is-data-descriptor "^1.0.0"
     kind-of "^6.0.2"
 
+is-directory@^0.3.1:
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/is-directory/-/is-directory-0.3.1.tgz#61339b6f2475fc772fd9c9d83f5c8575dc154ae1"
+
 is-error@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/is-error/-/is-error-2.2.1.tgz#684a96d84076577c98f4cdb40c6d26a5123bf19c"
@@ -2577,7 +2662,7 @@ js-tokens@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
 
-js-yaml@^3.10.0, js-yaml@^3.11.0:
+js-yaml@^3.10.0, js-yaml@^3.11.0, js-yaml@^3.9.0:
   version "3.12.0"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1"
   dependencies:
@@ -2687,6 +2772,13 @@ locate-path@^2.0.0:
     p-locate "^2.0.0"
     path-exists "^3.0.0"
 
+locate-path@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e"
+  dependencies:
+    p-locate "^3.0.0"
+    path-exists "^3.0.0"
+
 lodash.clone@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.clone/-/lodash.clone-4.5.0.tgz#195870450f5a13192478df4bc3d23d2dea1907b6"
@@ -2989,9 +3081,9 @@ nice-try@^1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
 
-node-fetch@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5"
+node-fetch@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5"
 
 node-forge@^0.7.4:
   version "0.7.6"
@@ -3025,7 +3117,6 @@ node-releases@^1.0.0-alpha.14:
 nodemon@1.18.7:
   version "1.18.7"
   resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-1.18.7.tgz#716b66bf3e89ac4fcfb38a9e61887a03fc82efbb"
-  integrity sha512-xuC1V0F5EcEyKQ1VhHYD13owznQbUw29JKvZ8bVH7TmuvVNHvvbp9pLgE4PjTMRJVe0pJ8fGRvwR2nMiosIsPQ==
   dependencies:
     chokidar "^2.0.4"
     debug "^3.1.0"
@@ -3145,7 +3236,7 @@ on-headers@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.1.tgz#928f5d0f470d49342651ea6794b0857c100693f7"
 
-once@^1.3.0:
+once@^1.3.0, once@^1.3.1, once@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
   dependencies:
@@ -3212,12 +3303,24 @@ p-limit@^1.1.0:
   dependencies:
     p-try "^1.0.0"
 
+p-limit@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.0.0.tgz#e624ed54ee8c460a778b3c9f3670496ff8a57aec"
+  dependencies:
+    p-try "^2.0.0"
+
 p-locate@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43"
   dependencies:
     p-limit "^1.1.0"
 
+p-locate@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4"
+  dependencies:
+    p-limit "^2.0.0"
+
 p-map@^1.1.1:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-1.2.0.tgz#e4e94f311eabbc8633a1e79908165fca26241b6b"
@@ -3226,6 +3329,10 @@ p-try@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"
 
+p-try@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.0.0.tgz#85080bb87c64688fa47996fe8f7dfbe8211760b1"
+
 package-hash@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/package-hash/-/package-hash-2.0.0.tgz#78ae326c89e05a4d813b68601977af05c00d2a0d"
@@ -3370,6 +3477,18 @@ pkg-dir@^2.0.0:
   dependencies:
     find-up "^2.1.0"
 
+pkg-dir@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3"
+  dependencies:
+    find-up "^3.0.0"
+
+please-upgrade-node@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.1.1.tgz#ed320051dfcc5024fae696712c8288993595e8ac"
+  dependencies:
+    semver-compare "^1.0.0"
+
 plur@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/plur/-/plur-3.0.1.tgz#268652d605f816699b42b86248de73c9acd06a7c"
@@ -3431,7 +3550,13 @@ pseudomap@^1.0.2:
 pstree.remy@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.2.tgz#4448bbeb4b2af1fed242afc8dc7416a6f504951a"
-  integrity sha512-vL6NLxNHzkNTjGJUpMm5PLC+94/0tTlC1vkP9bdU0pOHih+EujMjgMTwfZopZvHWRFbqJ5Y73OMoau50PewDDA==
+
+pump@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
+  dependencies:
+    end-of-stream "^1.1.0"
+    once "^1.3.1"
 
 punycode@^2.1.0:
   version "2.1.1"
@@ -3501,6 +3626,14 @@ read-pkg@^3.0.0:
     normalize-package-data "^2.3.2"
     path-type "^3.0.0"
 
+read-pkg@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-4.0.1.tgz#963625378f3e1c4d48c85872b5a6ec7d5d093237"
+  dependencies:
+    normalize-package-data "^2.3.2"
+    parse-json "^4.0.0"
+    pify "^3.0.0"
+
 readable-stream@^2.0.2, readable-stream@^2.0.6:
   version "2.3.6"
   resolved "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
@@ -3677,6 +3810,10 @@ run-async@^2.2.0:
   dependencies:
     is-promise "^2.1.0"
 
+run-node@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/run-node/-/run-node-1.0.0.tgz#46b50b946a2aa2d4947ae1d886e9856fd9cabe5e"
+
 run-parallel@^1.1.2:
   version "1.1.9"
   resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679"
@@ -3705,6 +3842,10 @@ sax@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
 
+semver-compare@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc"
+
 semver-diff@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36"