Release v0.5.0 (#61)

* remove explicit ID from events in timemap setup

* Fixed internal anchor link headline

* update fmt

* Bump lodash from 4.17.15 to 4.17.19

Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.19.
- [Release notes](https://github.com/lodash/lodash/releases)
- [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.19)

Signed-off-by: dependabot[bot] <support@github.com>

* Wrote validation functions and getter to grab appropriate validation function

* Collapsed filters and narratives into associations; modified default tabs to reflect

* Refactoring events to have only associations; created associations tab and related export tab

* Clean up with event related associations

* Delete validation for now; not relevant to this PR

* Bump node-fetch from 2.6.0 to 2.6.1

Bumps [node-fetch](https://github.com/bitinn/node-fetch) from 2.6.0 to 2.6.1.
- [Release notes](https://github.com/bitinn/node-fetch/releases)
- [Changelog](https://github.com/node-fetch/node-fetch/blob/master/docs/CHANGELOG.md)
- [Commits](https://github.com/bitinn/node-fetch/compare/v2.6.0...v2.6.1)

Signed-off-by: dependabot[bot] <support@github.com>

* Removed gsheets tab in config, unused import in package json

* try/catch to prioritise local.config.js

* Removing export categories endpoint

* Removed categories from list and using test_export_events

* Test associations

* Removed comment regarding categories endpoint

* Removing unnecessary test_export_events endpoint

* Linting threw error stating that we use path module to join instead of +

* generalise LocalFetcher to support many sheet types (xlsx, ods, etc)

* rm getFileExt

* use tab as delimiter in intermediate representation

* Bump googleapis from 32.0.0 to 39.1.0

Bumps [googleapis](https://github.com/googleapis/google-api-nodejs-client) from 32.0.0 to 39.1.0.
- [Release notes](https://github.com/googleapis/google-api-nodejs-client/releases)
- [Changelog](https://github.com/googleapis/google-api-nodejs-client/blob/master/CHANGELOG.md)
- [Commits](https://github.com/googleapis/google-api-nodejs-client/compare/v32.0.0...v39.1.0)

Signed-off-by: dependabot[bot] <support@github.com>

* Fixed tests

* fix example sheet

Co-authored-by: Christoph Knoth <github@christoph-knoth.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: efarooqui <efarooqui@pandora.com>
Co-authored-by: Ebrahem Farooqui <ebefarooqui@gmail.com>
Co-authored-by: Lachlan <Kermode>

.gitignore

@@ -14,3 +14,4 @@ tags
 tags.lock
 tags.temp
 src/config.js
+src/local.config.js

README.md

@@ -53,7 +53,7 @@ Each Google Sheet being used as a as source requires a corresponding object in `
 See src/config.js for an example configuration sheet. 
 
 
-## [Quickstart](#quickstart)
+## [Quickstart](#quickstart)
 Clone the repository to your local:
 ```
 git clone https://www.github.com/forensic-architecture/datasheet-server

package.json

@@ -26,15 +26,15 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
-    "googleapis": "^32.0.0",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.3.0",
-    "node-xlsx": "^0.15.0",
+    "node-fetch": "^2.6.1",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",

src/blueprinters/deeprows.js

@@ -63,7 +63,10 @@ export default (data) => {
     structure.__flat.forEach(label => {
       deepRow[label] = baseRow[label]
     })
-    if (deepRow['id'] && deepRow['id'] !== '') {
+    if (!Object.keys(deepRow).every(k => (
+      (deepRow[k] === '') ||
+      (Array.isArray(deepRow[k]) && deepRow[k].length === 0)
+    ))) {
       output.push(deepRow)
     }
   })

src/index.js

@@ -1,4 +1,5 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
@@ -40,7 +41,7 @@ initialize(controller => {
     })
   )
 
-  app.use(express.static(__dirname + '/public'))
+  app.use(express.static(path.join(__dirname, 'public')))
 
   app.server.listen(process.env.PORT || 4040, () => {
     console.log('===========================================')

src/initialize.js

@@ -1,7 +1,6 @@
 import StoreJson from './models/StoreJson'
 import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
 import R from 'ramda'
 
 const isntNull = n => n !== null
@@ -9,6 +8,13 @@ const filterNull = ls => R.filter(isntNull, ls)
 const flattenfilterNull = ls => filterNull(R.flatten(ls))
 let themFetchers
 
+let config
+try {
+  config = require('./local.config.js').default
+} catch (_) {
+  config = require('./config.js').default
+}
+
 export default callback => {
   return Promise.resolve().then(() => {
     return Object.keys(config).map(fType => {

src/lib.js

@@ -5,9 +5,7 @@ function prefixedTabs (prefix, cfg) {
   const prf = key => cfg[key] ? `${prefix}_` : ''
   return {
     [`${prf('events')}export_events`]: BP.deeprows,
-    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
-    [`${prf('filters')}export_filters`]: BP.tree,
-    [`${prf('narratives')}export_narratives`]: BP.rows,
+    [`${prf('associations')}export_associations`]: BP.deeprows,
     [`${prf('sources')}export_sources`]: BP.deepids,
     [`${prf('sites')}export_sites`]: BP.rows
   }

src/lib/Fetcher.js

@@ -1,15 +1,16 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
-import { google } from 'googleapis'
+import R from 'ramda'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
-import R from 'ramda'
-import xlsx from 'node-xlsx'
-import fs from 'fs'
+
+/* GsheetFetcher deps */
+import { google } from 'googleapis'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
   constructor (db, name, bps) {
@@ -138,7 +139,6 @@ class Fetcher {
 
   /** Run on startup. Should be overridden if explicit auth is required **/
   authenticate (env) {
-    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
     return Promise.resolve(this)
   }
 }
@@ -221,27 +221,22 @@ class GsheetFetcher extends Fetcher {
   }
 }
 
-class XlsxFetcher extends Fetcher {
+class LocalFetcher extends Fetcher {
   constructor (db, name, bps, path) {
     super(db, name, bps)
-    this.type = 'XLSX File'
     this.path = path
-    this.isRemote = false
-
-    if (this.path.startsWith('https')) {
-      this.isRemote = true
-    }
+    this.update().then(res =>
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
+    )
   }
 
   update () {
-    const data = xlsx.parse(fs.readFileSync(this.path))
-    data.forEach(tab => {
-      const stringyData = tab.data.map(row =>
-        row.map(d =>
-          typeof (d) === 'number' ? d.toString() : d
-        )
-      )
-      this.save(tab.name, stringyData)
+    const wb = X.readFile(this.path)
+    wb.SheetNames.forEach(name => {
+      const sh = wb.Sheets[name]
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
+      const ll = csv.split('\n').map(line => line.split('\t'))
+      this.save(name, ll)
     })
     return Promise.resolve(true)
   }
@@ -249,5 +244,7 @@ class XlsxFetcher extends Fetcher {
 
 export default {
   'gsheets': GsheetFetcher,
-  'xlsx': XlsxFetcher
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
 }

test/serverProcess.js

@@ -54,14 +54,7 @@ test('should launch', t => {
 })
 
 const passUrls = [
-  // /
   '/api/',
-  // /blueprints
-  '/api/blueprints',
-  // /:sheet/:tab/:resource
-  '/api/example/export_events/rows',
-  // /:sheet/:tab/:resource/:frag
-  '/api/example/export_events/rows/1'
 ]
 
 const failUrls = [
@@ -77,6 +70,7 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
+        console.info('JSON: ', json)
         t.pass()
       })
   })