export as deeprowsWithSchema

Adding the correct branch to checkout with for workflow

.github/workflows/cd.yml

@@ -0,0 +1,19 @@
+name: CD
+on:
+  push:
+    branches: [ develop ]
+#  pull_request:
+#    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger CD build
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.CI_DISPATCH_TOKEN }}
+          repository: forensic-architecture/configs
+          event-type: remote-build
+          client-payload: '{"runtime_args": "datasheet", "branch": "${GITHUB_REF##*/}"}'
+

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '12'
+
+      - run: npm install
+      - run: npm test
+        env:
+          CI: true
+      - run: npm run lint
+        env:
+          CI: true

.gitignore

@@ -14,3 +14,4 @@ tags
 tags.lock
 tags.temp
 src/config.js
+src/local.config.js

.travis.yml

@@ -1,14 +0,0 @@
-language: node_js
-node_js:
-- stable
-cache:
-  directories:
-  - node_modules
-before_script:
-- npm install -g yarn
-install:
-- yarn
-script:
-- yarn build
-- yarn lint
-- yarn test

Dockerfile

@@ -13,8 +13,8 @@ RUN yarn build
 RUN mkdir -p data
 
 # set your port
-ENV PORT 8080
-EXPOSE  8080
+ENV PORT 4040
+EXPOSE 4040
 
 # start command as per package.json
 CMD ["yarn", "start"]

README.md

@@ -36,45 +36,6 @@ Querying data directly from spreadsheets is brittle, as it relies on the mainten
 
 To see how to get a local instance of datasheet server running in practice, see [this wiki](https://github.com/forensic-architecture/timemap/wiki/running-timemap-and-datasheet-server-locally) explaining how to use it to feed data from a Google Sheet to a local instance of [Timemap](https://github.com/forensic-architecture/timemap).
 
-### Design Concepts
-The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
-
-**Sources**     
-A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
-
-**Resources**     
-The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
-
-**Blueprints**       
-Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
-
-Blueprints are JSON objects. There have two forms:
-
-1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
-2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
-
-A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
-
-A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
-
-## [Configuration](#configuration)
-Copy the example environment file:
-```
-cp .env.example .env
-```
-Inside this file, you will need to modify at least the `SERVICE_ACCOUNT_EMAIL` and `SERVICE_ACCOUNT_PRIVATE_KEY` fields. These fields refer to the credentials of a [Google service account](https://cloud.google.com/iam/docs/understanding-service-accounts). Google requires that developers create these when attempting to access their services programmatically, so that they can attribute requests to users. Service accounts also contain identity information, which means that asset owners can allow certain service accounts access to certain sheets, just as one might differentially grant certain users access to certain cloud assets. 
-
-Once you have [created a service account](https://support.google.com/a/answer/7378726?hl=en), create and download an [API key for that account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). The JSON file for the API key that you download when you create it contains both a service account private key, and an email associated with the service account: add these respectively in the strings in .env for `SERVICE_ACCOUNT_PRIVATE_KEY` and `SERVICE_ACCOUNT_EMAIL`.
-
-The last thing to do is to grant the service account access to the sheet that
-datasheet-server is pulling from. You can add a service account to a sheet as
-you would any other Google user: just enter the email address associated. (Note
-that this step is not necessary if you are accessing a publicly available
-sheet.)
-
-Other configuration options, such as the port at which the server will expose
-resources, are also modifiable from the .env file.
-
 #### [Sources](#sources)
 Sources are specified in [src/config.js](https://github.com/forensic-architecture/datasheet-server/blob/develop/src/config.js). Datasheet server currently only supports Google Sheets as a source.
 
@@ -86,34 +47,13 @@ Each Google Sheet being used as a as source requires a corresponding object in `
 | Option | Description | Type |
 | ------ | ----------- | ---- |
 | name | Used to refer to data served from this source | string |
-| id | The ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
+| path/id | The path to the XLSX sheet on your local, or the ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
 | tabs | An object that maps each tab in the source to one or more Blueprinters. All of the Blueprinters in the [blueprinters folder](/lib/blueprinters) are available through a single import as at the top of [example.config.js](/src/example.config.js). <br> To correctly associate a Blueprinter, the object key needs to be _the tab name with all lowercase letters, and spaces replaced by a '-'_. For example, if the tab name in Google Sheets is 'Info About SHEEP', the object key should be 'info-about-sheep'. <br> The value should be the Blueprinter function that you want to use for the data in that tab. If you require more than one endpoint for a single tab, you can support multiple blueprinters by making the item an array. See the example of a configuration object below. <br>TODO: no Blueprinter is used by default. | object |
 
-###### Example Configuration Object
-```js
-import BP from './lib/blueprinters'
-
-export default {
-  googleSheets: {
-    email: 'project-name@reliable-baptist-23338.iam.gserviceaccount.com',
-    privateKey: 'SOME_PRIVATE_KEY',
-    sheets: [
-      {
-        name: 'example',
-        id: '1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI',
-        tabs: {
-          'objects': BP.rows,
-          'fruit': [BP.columns, BP.ids],
-        }
-      },
-    ]
-  }
-}
-
-```
+See src/config.js for an example configuration sheet. 
 
 
-## [Quickstart](#quickstart)
+## [Quickstart](#quickstart)
 Clone the repository to your local:
 ```
 git clone https://www.github.com/forensic-architecture/datasheet-server

docs/design-concepts.md

@@ -0,0 +1,22 @@
+### Design Concepts
+The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
+
+**Sources**     
+A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
+
+**Resources**     
+The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
+
+**Blueprints**       
+Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
+
+Blueprints are JSON objects. There have two forms:
+
+1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
+2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
+
+A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
+
+A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
+
+

docs/gsheet-config.md

@@ -0,0 +1,19 @@
+## [Configuration](#configuration)
+Copy the example environment file:
+```
+cp .env.example .env
+```
+Inside this file, you will need to modify at least the `SERVICE_ACCOUNT_EMAIL` and `SERVICE_ACCOUNT_PRIVATE_KEY` fields. These fields refer to the credentials of a [Google service account](https://cloud.google.com/iam/docs/understanding-service-accounts). Google requires that developers create these when attempting to access their services programmatically, so that they can attribute requests to users. Service accounts also contain identity information, which means that asset owners can allow certain service accounts access to certain sheets, just as one might differentially grant certain users access to certain cloud assets. 
+
+Once you have [created a service account](https://support.google.com/a/answer/7378726?hl=en), create and download an [API key for that account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). The JSON file for the API key that you download when you create it contains both a service account private key, and an email associated with the service account: add these respectively in the strings in .env for `SERVICE_ACCOUNT_PRIVATE_KEY` and `SERVICE_ACCOUNT_EMAIL`.
+
+The last thing to do is to grant the service account access to the sheet that
+datasheet-server is pulling from. You can add a service account to a sheet as
+you would any other Google user: just enter the email address associated. (Note
+that this step is not necessary if you are accessing a publicly available
+sheet.)
+
+Other configuration options, such as the port at which the server will expose
+resources, are also modifiable from the .env file.
+
+

package.json

@@ -26,15 +26,15 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
-    "googleapis": "^32.0.0",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.3.0",
-    "node-xlsx": "^0.15.0",
+    "node-fetch": "^2.6.1",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",

scripts/check-branch.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-ENCRYPT_MESSAGE="\nThe .env.enc file has not changed its contents from the version on origin/develop.\nTo ensure that tests pass in Travis, you need to encrypt the contents of .env  before pushing to the remote, so that the Travis server can use your service account credentials.\nEnsure that the Travis CLI is installed on your local, run\n\n\tnpm run travis-encrypt\n\nand then push to the remote again.\nIf you don't care whether your build passes on Travis, you can run:\n\n\tgit push --no-verify\n\nand bypass this check.\n\n"
-
-# check whether .env.enc has changed
-if [ -z "`git diff origin/develop -- .env.enc`" ]; then
-	echo $ENCRYPT_MESSAGE
-	exit 1
-fi
-
-

scripts/encrypt.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-echo "Encrypting .env file for Travis..."
-
-# confirm that the user has the core repo as origin, and is therefore a maintainer
-# if [ -z `git config --get remote.origin.url | grep "forensic-architecture/datasheet-server"` ]; then
-# 	echo "Travis encryption not required for satellite contributors, continuing.."
-# 	exit 0
-# fi
-
-# confirm travis is installed
-if [ ! hash travis 2>/dev/null ]; then
-	echo "============================================================================================"
-	echo "ERROR: Travis CLI is not installed on your local. Please install from:"
-	echo "\thttps://github.com/travis-ci/travis.rb"
-	echo "After installing, make sure that you login with:"
-	echo "\ttravis login --pro"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# confirm there is a .env file to encrypt
-if [ ! -f .env ]; then
-	echo "============================================================================================"
-	echo "ERROR: You must create a .env file and add your credentials. See .env.example for an example"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# regex to match and delete 'before_install' and everything after it
-# necessary to delete these lines to get Travis to build for multiple accounts
-echo "creating new .travis.yml configuration"
-sed -i.old '/^before_install.*/,$ d' .travis.yml
-echo "old config file saved as .travis.yml.old"
-
-travis encrypt-file .env --add --force --org
-git add .env.enc
-git add .travis.yml
-echo ".env.enc created and added to commit"

src/api/index.js

@@ -30,10 +30,10 @@ export default ({ config, controller }) => {
           success: msg
         })
       )
-      .catch(err =>
+      .catch(err => {
         res.status(404)
           .send({ error: err.message, err })
-      )
+      })
   })
 
   api.get('/:sheet/:tab/:resource/:frag', (req, res) => {

src/blueprinters/deeprows.js

@@ -63,7 +63,7 @@ export default (data) => {
     structure.__flat.forEach(label => {
       deepRow[label] = baseRow[label]
     })
-    if (deepRow['id'] && deepRow['id'] !== '') {
+    if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {
       output.push(deepRow)
     }
   })

src/index.js

@@ -1,4 +1,5 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
@@ -40,7 +41,7 @@ initialize(controller => {
     })
   )
 
-  app.use(express.static(__dirname + '/public'))
+  app.use(express.static(path.join(__dirname, 'public')))
 
   app.server.listen(process.env.PORT || 4040, () => {
     console.log('===========================================')

src/initialize.js

@@ -1,7 +1,6 @@
 import StoreJson from './models/StoreJson'
 import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
 import R from 'ramda'
 
 const isntNull = n => n !== null
@@ -9,6 +8,13 @@ const filterNull = ls => R.filter(isntNull, ls)
 const flattenfilterNull = ls => filterNull(R.flatten(ls))
 let themFetchers
 
+let config
+try {
+  config = require('./local.config.js').default
+} catch (_) {
+  config = require('./config.js').default
+}
+
 export default callback => {
   return Promise.resolve().then(() => {
     return Object.keys(config).map(fType => {

src/lib.js

@@ -5,9 +5,7 @@ function prefixedTabs (prefix, cfg) {
   const prf = key => cfg[key] ? `${prefix}_` : ''
   return {
     [`${prf('events')}export_events`]: BP.deeprows,
-    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
-    [`${prf('filters')}export_filters`]: BP.tree,
-    [`${prf('narratives')}export_narratives`]: BP.rows,
+    [`${prf('associations')}export_associations`]: BP.deeprows,
     [`${prf('sources')}export_sources`]: BP.deepids,
     [`${prf('sites')}export_sites`]: BP.rows
   }

src/lib/Controller.js

@@ -1,7 +1,7 @@
 import copy from '../copy/en'
 
 /**
- * Controller
+ * Controller class
  *
  */
 class Controller {

src/lib/Fetcher.js

@@ -1,15 +1,16 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
-import { google } from 'googleapis'
+import R from 'ramda'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
-import R from 'ramda'
-import xlsx from 'node-xlsx'
-import fs from 'fs'
+
+/* GsheetFetcher deps */
+import { google } from 'googleapis'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
   constructor (db, name, bps) {
@@ -138,7 +139,6 @@ class Fetcher {
 
   /** Run on startup. Should be overridden if explicit auth is required **/
   authenticate (env) {
-    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
     return Promise.resolve(this)
   }
 }
@@ -217,31 +217,29 @@ class GsheetFetcher extends Fetcher {
       })
       .then(this._buildBlueprintsAsync())
       .then(() => true)
-      .catch(() => false)
+      .catch((err) => {
+        console.log(`Error fetching gsheets: ${err.message} `)
+        return false
+      })
   }
 }
 
-class XlsxFetcher extends Fetcher {
+class LocalFetcher extends Fetcher {
   constructor (db, name, bps, path) {
     super(db, name, bps)
-    this.type = 'XLSX File'
     this.path = path
-    this.isRemote = false
-
-    if (this.path.startsWith('https')) {
-      this.isRemote = true
-    }
+    this.update().then(res =>
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
+    )
   }
 
   update () {
-    const data = xlsx.parse(fs.readFileSync(this.path))
-    data.forEach(tab => {
-      const stringyData = tab.data.map(row =>
-        row.map(d =>
-          typeof (d) === 'number' ? d.toString() : d
-        )
-      )
-      this.save(tab.name, stringyData)
+    const wb = X.readFile(this.path)
+    wb.SheetNames.forEach(name => {
+      const sh = wb.Sheets[name]
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
+      const ll = csv.split('\n').map(line => line.split('\t'))
+      this.save(name, ll)
     })
     return Promise.resolve(true)
   }
@@ -249,5 +247,7 @@ class XlsxFetcher extends Fetcher {
 
 export default {
   'gsheets': GsheetFetcher,
-  'xlsx': XlsxFetcher
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
 }

src/lib/blueprinters.js

@@ -2,7 +2,7 @@ import path from 'path'
 import fs from 'fs'
 import R from 'ramda'
 
-export const defaultBlueprint = {
+const defaultBlueprint = {
   name: null,
   sheet: {
     name: null,
@@ -11,11 +11,11 @@ export const defaultBlueprint = {
   resources: {}
 }
 
-export const defaultResource = {
+const defaultResource = {
   data: []
 }
 
-export function buildDesaturated (sheetId, sheetName, tab, resources) {
+function buildDesaturated (sheetId, sheetName, tab, resources) {
   const bp = R.clone(defaultBlueprint)
   bp.sheet.name = sheetName
   bp.sheet.id = sheetId
@@ -50,10 +50,25 @@ fs.readdirSync(normalizedPath).forEach(file => {
   allBps[bpName] = buildBlueprinter(bpName, datafier)
 })
 
+function deeprowsWithSchema (datafierName, schema) {
+  const datafier = data => {
+    const transformedData = allBps.deeprows('', '', '', data).resources.deeprows.data
+    return transformedData.map(row => {
+      Object.keys(schema).forEach(key => {
+        row[key] = schema[key](row[key])
+      })
+      return row
+    })
+  }
+
+  return buildBlueprinter(`deeprows_${datafierName}`, datafier)
+}
+
 // NB: revert to ES5 'module.exports' required to make blueprinters from
 // each file in blueprinters folder available for granular import from here.
 module.exports = Object.assign({
   defaultBlueprint,
   defaultResource,
-  buildDesaturated
+  buildDesaturated,
+  deeprowsWithSchema
 }, allBps)

src/models/StoreJson.js

@@ -19,7 +19,6 @@ class StoreJson {
 
   save (url, data) {
     const parts = url.split('/')
-
     return fs.writeFile(
       `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`,
       JSON.stringify(data)

test/internals.js

@@ -12,6 +12,8 @@ const egInput1 = [
   [4, 5, 6]
 ]
 
+// Test default blueprint exports
+// Smoke tests
 test('defaultBlueprint exports', t => {
   const expected = {
     sheet: {

test/serverProcess.js

@@ -54,14 +54,7 @@ test('should launch', t => {
 })
 
 const passUrls = [
-  // /
-  '/api/',
-  // /blueprints
-  '/api/blueprints',
-  // /:sheet/:tab/:resource
-  '/api/example/export_events/rows',
-  // /:sheet/:tab/:resource/:frag
-  '/api/example/export_events/rows/1'
+  '/api/'
 ]
 
 const failUrls = [
@@ -77,6 +70,7 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
+        console.info('JSON: ', json)
         t.pass()
       })
   })