Use npm in Dockerfile rather than yarn

* Update package-lock.json

* Update caniuse-lite

* Update XLSX data to be more descriptive of timemap's features

.github/workflows/cd.yml

@@ -0,0 +1,19 @@
+name: CD
+on:
+  push:
+    branches: [ develop ]
+#  pull_request:
+#    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger CD build
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.CI_DISPATCH_TOKEN }}
+          repository: forensic-architecture/configs
+          event-type: remote-build
+          client-payload: '{"runtime_args": "datasheet", "branch": "${GITHUB_REF##*/}"}'
+

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '12'
+
+      - run: npm install
+      - run: npm test
+        env:
+          CI: true
+      - run: npm run lint
+        env:
+          CI: true

.gitignore

@@ -14,3 +14,4 @@ tags
 tags.lock
 tags.temp
 src/config.js
+src/local.config.js

.travis.yml

@@ -1,14 +0,0 @@
-language: node_js
-node_js:
-- stable
-cache:
-  directories:
-  - node_modules
-before_script:
-- npm install -g yarn
-install:
-- yarn
-script:
-- yarn build
-- yarn lint
-- yarn test

Dockerfile

@@ -4,17 +4,17 @@ LABEL authors="Lachlan Kermode <lk@forensic-architecture.org>"
 
 # Install app dependencies
 COPY package.json /www/package.json
-RUN cd /www; yarn
+RUN cd /www; npm install
 
 # Copy app source
 COPY . /www
 WORKDIR /www
-RUN yarn build
+RUN npm run build
 RUN mkdir -p data
 
 # set your port
-ENV PORT 8080
+ENV PORT 4040
-EXPOSE  8080
+EXPOSE 4040
 
 # start command as per package.json
-CMD ["yarn", "start"]
+CMD ["npm", "start"]

README.md

@@ -53,7 +53,7 @@ Each Google Sheet being used as a as source requires a corresponding object in `
 See src/config.js for an example configuration sheet. 
 
 
-## [Quickstart](#quickstart)
+## [Quickstart](#quickstart)
 Clone the repository to your local:
 ```
 git clone https://www.github.com/forensic-architecture/datasheet-server

package.json

@@ -26,15 +26,15 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
-    "googleapis": "^32.0.0",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.3.0",
+    "node-fetch": "^2.6.1",
-    "node-xlsx": "^0.15.0",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",

scripts/check-branch.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-ENCRYPT_MESSAGE="\nThe .env.enc file has not changed its contents from the version on origin/develop.\nTo ensure that tests pass in Travis, you need to encrypt the contents of .env  before pushing to the remote, so that the Travis server can use your service account credentials.\nEnsure that the Travis CLI is installed on your local, run\n\n\tnpm run travis-encrypt\n\nand then push to the remote again.\nIf you don't care whether your build passes on Travis, you can run:\n\n\tgit push --no-verify\n\nand bypass this check.\n\n"
-
-# check whether .env.enc has changed
-if [ -z "`git diff origin/develop -- .env.enc`" ]; then
-	echo $ENCRYPT_MESSAGE
-	exit 1
-fi
-
-

scripts/encrypt.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-echo "Encrypting .env file for Travis..."
-
-# confirm that the user has the core repo as origin, and is therefore a maintainer
-# if [ -z `git config --get remote.origin.url | grep "forensic-architecture/datasheet-server"` ]; then
-# 	echo "Travis encryption not required for satellite contributors, continuing.."
-# 	exit 0
-# fi
-
-# confirm travis is installed
-if [ ! hash travis 2>/dev/null ]; then
-	echo "============================================================================================"
-	echo "ERROR: Travis CLI is not installed on your local. Please install from:"
-	echo "\thttps://github.com/travis-ci/travis.rb"
-	echo "After installing, make sure that you login with:"
-	echo "\ttravis login --pro"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# confirm there is a .env file to encrypt
-if [ ! -f .env ]; then
-	echo "============================================================================================"
-	echo "ERROR: You must create a .env file and add your credentials. See .env.example for an example"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# regex to match and delete 'before_install' and everything after it
-# necessary to delete these lines to get Travis to build for multiple accounts
-echo "creating new .travis.yml configuration"
-sed -i.old '/^before_install.*/,$ d' .travis.yml
-echo "old config file saved as .travis.yml.old"
-
-travis encrypt-file .env --add --force --org
-git add .env.enc
-git add .travis.yml
-echo ".env.enc created and added to commit"

src/api/index.js

@@ -30,10 +30,10 @@ export default ({ config, controller }) => {
           success: msg
         })
       )
-      .catch(err =>
+      .catch(err => {
         res.status(404)
           .send({ error: err.message, err })
-      )
+      })
   })
 
   api.get('/:sheet/:tab/:resource/:frag', (req, res) => {

src/blueprinters/deeprows.js

@@ -63,7 +63,7 @@ export default (data) => {
     structure.__flat.forEach(label => {
       deepRow[label] = baseRow[label]
     })
-    if (deepRow['id'] && deepRow['id'] !== '') {
+    if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {
       output.push(deepRow)
     }
   })

src/index.js

@@ -1,4 +1,5 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
@@ -40,7 +41,7 @@ initialize(controller => {
     })
   )
 
-  app.use(express.static(__dirname + '/public'))
+  app.use(express.static(path.join(__dirname, 'public')))
 
   app.server.listen(process.env.PORT || 4040, () => {
     console.log('===========================================')

src/initialize.js

@@ -1,7 +1,6 @@
 import StoreJson from './models/StoreJson'
 import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
 import R from 'ramda'
 
 const isntNull = n => n !== null
@@ -9,6 +8,13 @@ const filterNull = ls => R.filter(isntNull, ls)
 const flattenfilterNull = ls => filterNull(R.flatten(ls))
 let themFetchers
 
+let config
+try {
+  config = require('./local.config.js').default
+} catch (_) {
+  config = require('./config.js').default
+}
+
 export default callback => {
   return Promise.resolve().then(() => {
     return Object.keys(config).map(fType => {

src/lib.js

@@ -5,10 +5,9 @@ function prefixedTabs (prefix, cfg) {
   const prf = key => cfg[key] ? `${prefix}_` : ''
   return {
     [`${prf('events')}export_events`]: BP.deeprows,
-    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
+    [`${prf('associations')}export_associations`]: BP.deeprows,
-    [`${prf('filters')}export_filters`]: BP.tree,
-    [`${prf('narratives')}export_narratives`]: BP.rows,
     [`${prf('sources')}export_sources`]: BP.deepids,
+    [`${prf('shapes')}export_shapes`]: BP.deeprows,
     [`${prf('sites')}export_sites`]: BP.rows
   }
 }

src/lib/Controller.js

@@ -1,7 +1,7 @@
 import copy from '../copy/en'
 
 /**
- * Controller
+ * Controller class
  *
  */
 class Controller {

src/lib/Fetcher.js

@@ -1,15 +1,16 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
+import R from 'ramda'
-import { google } from 'googleapis'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
+
-import R from 'ramda'
+/* GsheetFetcher deps */
-import xlsx from 'node-xlsx'
+import { google } from 'googleapis'
-import fs from 'fs'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
   constructor (db, name, bps) {
@@ -138,7 +139,6 @@ class Fetcher {
 
   /** Run on startup. Should be overridden if explicit auth is required **/
   authenticate (env) {
-    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
     return Promise.resolve(this)
   }
 }
@@ -217,31 +217,29 @@ class GsheetFetcher extends Fetcher {
       })
       .then(this._buildBlueprintsAsync())
       .then(() => true)
-      .catch(() => false)
+      .catch((err) => {
+        console.log(`Error fetching gsheets: ${err.message} `)
+        return false
+      })
   }
 }
 
-class XlsxFetcher extends Fetcher {
+class LocalFetcher extends Fetcher {
   constructor (db, name, bps, path) {
     super(db, name, bps)
-    this.type = 'XLSX File'
     this.path = path
-    this.isRemote = false
+    this.update().then(res =>
-
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
-    if (this.path.startsWith('https')) {
+    )
-      this.isRemote = true
-    }
   }
 
   update () {
-    const data = xlsx.parse(fs.readFileSync(this.path))
+    const wb = X.readFile(this.path)
-    data.forEach(tab => {
+    wb.SheetNames.forEach(name => {
-      const stringyData = tab.data.map(row =>
+      const sh = wb.Sheets[name]
-        row.map(d =>
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
-          typeof (d) === 'number' ? d.toString() : d
+      const ll = csv.split('\n').map(line => line.split('\t'))
-        )
+      this.save(name, ll)
-      )
-      this.save(tab.name, stringyData)
     })
     return Promise.resolve(true)
   }
@@ -249,5 +247,7 @@ class XlsxFetcher extends Fetcher {
 
 export default {
   'gsheets': GsheetFetcher,
-  'xlsx': XlsxFetcher
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
 }

src/models/StoreJson.js

@@ -19,7 +19,6 @@ class StoreJson {
 
   save (url, data) {
     const parts = url.split('/')
-
     return fs.writeFile(
       `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`,
       JSON.stringify(data)

test/internals.js

@@ -12,6 +12,8 @@ const egInput1 = [
   [4, 5, 6]
 ]
 
+// Test default blueprint exports
+// Smoke tests
 test('defaultBlueprint exports', t => {
   const expected = {
     sheet: {

test/serverProcess.js

@@ -54,14 +54,7 @@ test('should launch', t => {
 })
 
 const passUrls = [
-  // /
+  '/api/'
-  '/api/',
-  // /blueprints
-  '/api/blueprints',
-  // /:sheet/:tab/:resource
-  '/api/example/export_events/rows',
-  // /:sheet/:tab/:resource/:frag
-  '/api/example/export_events/rows/1'
 ]
 
 const failUrls = [
@@ -77,6 +70,7 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
+        console.info('JSON: ', json)
         t.pass()
       })
   })