Merge branch 'develop' of https://www.github.com/forensic-architecture/datasheet-server into feature/json-api-export

Adding the correct branch to checkout with for workflow

.github/workflows/cd.yml

@@ -0,0 +1,19 @@
+name: CD
+on:
+  push:
+    branches: [ develop ]
+#  pull_request:
+#    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger CD build
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.CI_DISPATCH_TOKEN }}
+          repository: forensic-architecture/configs
+          event-type: remote-build
+          client-payload: '{"runtime_args": "datasheet", "branch": "${GITHUB_REF##*/}"}'
+

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '12'
+
+      - run: npm install
+      - run: npm test
+        env:
+          CI: true
+      - run: npm run lint
+        env:
+          CI: true

.gitignore

@@ -14,3 +14,4 @@ tags
 tags.lock
 tags.temp
 src/config.js
+src/local.config.js

.travis.yml

@@ -1,14 +0,0 @@
-language: node_js
-node_js:
-- stable
-cache:
-  directories:
-  - node_modules
-before_script:
-- npm install -g yarn
-install:
-- yarn
-script:
-- yarn build
-- yarn lint
-- yarn test

Dockerfile

@@ -13,8 +13,8 @@ RUN yarn build
 RUN mkdir -p data
 
 # set your port
-ENV PORT 8080
-EXPOSE  8080
+ENV PORT 4040
+EXPOSE 4040
 
 # start command as per package.json
 CMD ["yarn", "start"]

package.json

@@ -3,6 +3,7 @@
   "version": "0.3.0",
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
+  "type": "module",
   "scripts": {
     "dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
     "build": "env NODE_ENV=production npx babel src -d dist",
@@ -26,16 +27,16 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
-    "googleapis": "^32.0.0",
+    "file-system": "^2.2.2",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
-    "moment": "^2.27.0",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.3.0",
-    "node-xlsx": "^0.15.0",
+    "node-fetch": "^2.6.1",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",

scripts/check-branch.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-ENCRYPT_MESSAGE="\nThe .env.enc file has not changed its contents from the version on origin/develop.\nTo ensure that tests pass in Travis, you need to encrypt the contents of .env  before pushing to the remote, so that the Travis server can use your service account credentials.\nEnsure that the Travis CLI is installed on your local, run\n\n\tnpm run travis-encrypt\n\nand then push to the remote again.\nIf you don't care whether your build passes on Travis, you can run:\n\n\tgit push --no-verify\n\nand bypass this check.\n\n"
-
-# check whether .env.enc has changed
-if [ -z "`git diff origin/develop -- .env.enc`" ]; then
-	echo $ENCRYPT_MESSAGE
-	exit 1
-fi
-
-

scripts/encrypt.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-echo "Encrypting .env file for Travis..."
-
-# confirm that the user has the core repo as origin, and is therefore a maintainer
-# if [ -z `git config --get remote.origin.url | grep "forensic-architecture/datasheet-server"` ]; then
-# 	echo "Travis encryption not required for satellite contributors, continuing.."
-# 	exit 0
-# fi
-
-# confirm travis is installed
-if [ ! hash travis 2>/dev/null ]; then
-	echo "============================================================================================"
-	echo "ERROR: Travis CLI is not installed on your local. Please install from:"
-	echo "\thttps://github.com/travis-ci/travis.rb"
-	echo "After installing, make sure that you login with:"
-	echo "\ttravis login --pro"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# confirm there is a .env file to encrypt
-if [ ! -f .env ]; then
-	echo "============================================================================================"
-	echo "ERROR: You must create a .env file and add your credentials. See .env.example for an example"
-	echo "============================================================================================"
-	exit 3
-fi
-
-# regex to match and delete 'before_install' and everything after it
-# necessary to delete these lines to get Travis to build for multiple accounts
-echo "creating new .travis.yml configuration"
-sed -i.old '/^before_install.*/,$ d' .travis.yml
-echo "old config file saved as .travis.yml.old"
-
-travis encrypt-file .env --add --force --org
-git add .env.enc
-git add .travis.yml
-echo ".env.enc created and added to commit"

src/api/index.js

@@ -4,6 +4,7 @@ import copy from '../copy/en'
 
 export default ({ config, controller }) => {
   let api = Router()
+  const fileDest = config.EXPORT_FILE_DEST || null
 
   api.get('/', (req, res) => {
     res.json({
@@ -22,6 +23,20 @@ export default ({ config, controller }) => {
     })
   })
 
+  api.get('/export', (req, res) => {
+    controller
+      .retrieveAll(fileDest)
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
+  })
+
   api.get('/update', (req, res) => {
     controller
       .update()
@@ -58,17 +73,6 @@ export default ({ config, controller }) => {
       )
   })
 
-  // api.get('/:sheet/:tab/:resource/validate', (req, res) => {
-  //   const { sheet, tab, resource } = req.params
-  //   controller
-  //     .retrieve(sheet, tab, resource)
-  //     .then(data => res.json(data))
-  //     .catch(err =>
-  //       res.status(err.status || 404)
-  //         .send({ error: err.message })
-  //     )
-  // })
-
   // ERROR routes. Note that it is important that these come AFTER routes
   // like /update, so that the regex does not greedily match these routes.
 

src/blueprinters/deeprows.js

@@ -1,5 +1,4 @@
 import { fmtObj } from '../lib/util'
-import { getColumnValidation } from '../lib/validation'
 
 /**
  * Each resource item is an object with values labelled according
@@ -36,6 +35,7 @@ export default (data) => {
       }
     }
   })
+
   // generate the value for deep labels using the structure created
   data.forEach((row, idx) => {
     if (idx === 0) return
@@ -61,8 +61,6 @@ export default (data) => {
 
     // move values for flat labels over from base
     structure.__flat.forEach(label => {
-      const validatedLabel = getColumnValidation(label, baseRow[label])
-      console.log(validatedLabel)
       deepRow[label] = baseRow[label]
     })
     if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {

src/config.js

@@ -1,13 +1,7 @@
 import { timemap } from './lib'
 
 export default {
-  gsheets: [
-    {
-      name: 'us2020',
-      id: '1I_pgyTQJlIorTIEHBxw1mM1STn-SrIi66FKYxut61iM',
-      tabs: timemap.default
-    }
-  ],
+  gsheets: [],
   xlsx: [
     {
       name: 'timemap_data',

src/copy/en.js

@@ -1,6 +1,10 @@
 export default {
   errors: {
     update: 'The server could not update. Check your API credentials and internet connection and try again.',
+    export: {
+      fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
+      writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
+    },
     onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
@@ -9,6 +13,7 @@ export default {
     modelLayer: prts => `Something went wrong at the model layer`
   },
   success: {
-    update: 'All sheets updated'
+    update: 'All sheets updated',
+    export: dest => `All resources exported to the file: ${dest}`
   }
 }

src/index.js

@@ -1,4 +1,5 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
@@ -40,7 +41,7 @@ initialize(controller => {
     })
   )
 
-  app.use(express.static(__dirname + '/public'))
+  app.use(express.static(path.join(__dirname, 'public')))
 
   app.server.listen(process.env.PORT || 4040, () => {
     console.log('===========================================')

src/initialize.js

@@ -1,7 +1,6 @@
 import StoreJson from './models/StoreJson'
 import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
 import R from 'ramda'
 
 const isntNull = n => n !== null
@@ -9,6 +8,13 @@ const filterNull = ls => R.filter(isntNull, ls)
 const flattenfilterNull = ls => filterNull(R.flatten(ls))
 let themFetchers
 
+let config
+try {
+  config = require('./local.config.js').default
+} catch (_) {
+  config = require('./config.js').default
+}
+
 export default callback => {
   return Promise.resolve().then(() => {
     return Object.keys(config).map(fType => {

src/lib.js

@@ -5,9 +5,7 @@ function prefixedTabs (prefix, cfg) {
   const prf = key => cfg[key] ? `${prefix}_` : ''
   return {
     [`${prf('events')}export_events`]: BP.deeprows,
-    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
-    [`${prf('filters')}export_filters`]: BP.tree,
-    [`${prf('narratives')}export_narratives`]: BP.rows,
+    [`${prf('associations')}export_associations`]: BP.deeprows,
     [`${prf('sources')}export_sources`]: BP.deepids,
     [`${prf('sites')}export_sites`]: BP.rows
   }

src/lib/Controller.js

@@ -1,4 +1,5 @@
 import copy from '../copy/en'
+import { exportToFile } from '../lib/util'
 
 /**
  * Controller
@@ -39,6 +40,37 @@ class Controller {
     })
   }
 
+  // Controller function to retrieve all blueprints and export to user defined file location
+  retrieveAll (fileDest) {
+    if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
+
+    const indexedData = {}
+    const urls = []
+
+    const bps = this.blueprints()
+    return Promise.all(
+      bps.map(bp => {
+        const resource = Object.keys(bp.resources)[0]
+        urls.push(bp.urls[0])
+        return this.retrieve(bp.sheet.name, bp.name, resource)
+      })
+    ).then(async results => {
+      if (results.every(res => res)) {
+        urls.forEach((item, idx) => {
+          indexedData[item] = results[idx]
+        })
+        try {
+          const message = await exportToFile(fileDest, indexedData)
+          return message
+        } catch (e) {
+          return Promise.reject(e)
+        }
+      } else {
+        throw new Error(copy.errors.export.writeFailed)
+      }
+    })
+  }
+
   retrieve (sheet, tab, resource) {
     if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]

src/lib/Fetcher.js

@@ -1,15 +1,16 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
-import { google } from 'googleapis'
+import R from 'ramda'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
-import R from 'ramda'
-import xlsx from 'node-xlsx'
-import fs from 'fs'
+
+/* GsheetFetcher deps */
+import { google } from 'googleapis'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
   constructor (db, name, bps) {
@@ -138,7 +139,6 @@ class Fetcher {
 
   /** Run on startup. Should be overridden if explicit auth is required **/
   authenticate (env) {
-    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
     return Promise.resolve(this)
   }
 }
@@ -221,27 +221,22 @@ class GsheetFetcher extends Fetcher {
   }
 }
 
-class XlsxFetcher extends Fetcher {
+class LocalFetcher extends Fetcher {
   constructor (db, name, bps, path) {
     super(db, name, bps)
-    this.type = 'XLSX File'
     this.path = path
-    this.isRemote = false
-
-    if (this.path.startsWith('https')) {
-      this.isRemote = true
-    }
+    this.update().then(res =>
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
+    )
   }
 
   update () {
-    const data = xlsx.parse(fs.readFileSync(this.path))
-    data.forEach(tab => {
-      const stringyData = tab.data.map(row =>
-        row.map(d =>
-          typeof (d) === 'number' ? d.toString() : d
-        )
-      )
-      this.save(tab.name, stringyData)
+    const wb = X.readFile(this.path)
+    wb.SheetNames.forEach(name => {
+      const sh = wb.Sheets[name]
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
+      const ll = csv.split('\n').map(line => line.split('\t'))
+      this.save(name, ll)
     })
     return Promise.resolve(true)
   }
@@ -249,5 +244,7 @@ class XlsxFetcher extends Fetcher {
 
 export default {
   'gsheets': GsheetFetcher,
-  'xlsx': XlsxFetcher
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
 }

src/lib/util.js

@@ -1,4 +1,6 @@
-import R from 'ramda';
+import R from 'ramda'
+import { promises as fs } from 'file-system'
+import copy from '../copy/en'
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {
@@ -14,6 +16,18 @@ function camelize (str) {
   })
 }
 
+export async function exportToFile (fileDest, data) {
+  const stringifiedData = JSON.stringify(data, null, 2)
+  const filePath = `${fileDest}/export.json`
+
+  try {
+    await fs.writeFile(filePath, stringifiedData)
+    return copy.success.export(filePath)
+  } catch (err) {
+    throw new Error(copy.errors.export.writeFailed)
+  }
+}
+
 export const fmtObj = R.curry(
   (
     columnNames,

src/lib/validation.js

@@ -1,20 +0,0 @@
-import moment from 'moment';
-
-const DATE_FORMAT = "MM/DD/YYYY";
-const TIME_REGEX = new RegExp("^([01]\d|2[0-3]):?([0-5]\d)$")
-
-
-export const getColumnValidation = (colName, value) => {
-  switch(colName) {
-    case 'longitude':
-      return isFinite(value) && Math.abs(value) <= 180;
-    case 'latitude':
-      return isFinite(value) && Math.abs(value) <= 90;
-    case 'date':
-      return moment(value, DATE_FORMAT, true).isValid();
-    case 'time':
-      return TIME_REGEX.test(value);
-    default:
-      return true
-  }
-}

test/serverProcess.js

@@ -54,14 +54,7 @@ test('should launch', t => {
 })
 
 const passUrls = [
-  // /
-  '/api/',
-  // /blueprints
-  '/api/blueprints',
-  // /:sheet/:tab/:resource
-  '/api/example/export_events/rows',
-  // /:sheet/:tab/:resource/:frag
-  '/api/example/export_events/rows/1'
+  '/api/'
 ]
 
 const failUrls = [
@@ -77,6 +70,7 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
+        console.info('JSON: ', json)
         t.pass()
       })
   })

views/blueprints.hbs

@@ -9,7 +9,6 @@
     </div>
     {{#each urls}}
     <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
-    <div class="bp-validate-button" target="_blank" href="http://localhost:4040{{ this }}">Validate</div>
     {{/each}}
   </div>
   {{ else }}
@@ -26,7 +25,6 @@
 :root {
   --grey: #8a8a8a;
   --btn-width: 200px;
-  --validate-btn-width: 150px;
 }
 .main-container {
   display: flex;
@@ -81,21 +79,6 @@
   color: black;
 }
 
-.bp-validate-button {
-  transition: all 0.3s ease;
-  text-transform: uppercase;
-  font-size: 12pt;
-  font-weight: bold;
-  width: var(--validate-btn-width);
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  border: 3px solid red;
-  padding: .5em;
-  text-decoration: none;
-  color: red;
-}
-
 .bp-update-container:hover {
   background-color: black;
   transition: all 0.3s ease;