Release v0.5.0 (#61)

* remove explicit ID from events in timemap setup

* Fixed internal anchor link headline

* update fmt

* Bump lodash from 4.17.15 to 4.17.19

Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.19.
- [Release notes](https://github.com/lodash/lodash/releases)
- [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.19)

Signed-off-by: dependabot[bot] <support@github.com>

* Wrote validation functions and getter to grab appropriate validation function

* Collapsed filters and narratives into associations; modified default tabs to reflect

* Refactoring events to have only associations; created associations tab and related export tab

* Clean up with event related associations

* Delete validation for now; not relevant to this PR

* Bump node-fetch from 2.6.0 to 2.6.1

Bumps [node-fetch](https://github.com/bitinn/node-fetch) from 2.6.0 to 2.6.1.
- [Release notes](https://github.com/bitinn/node-fetch/releases)
- [Changelog](https://github.com/node-fetch/node-fetch/blob/master/docs/CHANGELOG.md)
- [Commits](https://github.com/bitinn/node-fetch/compare/v2.6.0...v2.6.1)

Signed-off-by: dependabot[bot] <support@github.com>

* Removed gsheets tab in config, unused import in package json

* try/catch to prioritise local.config.js

* Removing export categories endpoint

* Removed categories from list and using test_export_events

* Test associations

* Removed comment regarding categories endpoint

* Removing unnecessary test_export_events endpoint

* Linting threw error stating that we use path module to join instead of +

* generalise LocalFetcher to support many sheet types (xlsx, ods, etc)

* rm getFileExt

* use tab as delimiter in intermediate representation

* Bump googleapis from 32.0.0 to 39.1.0

Bumps [googleapis](https://github.com/googleapis/google-api-nodejs-client) from 32.0.0 to 39.1.0.
- [Release notes](https://github.com/googleapis/google-api-nodejs-client/releases)
- [Changelog](https://github.com/googleapis/google-api-nodejs-client/blob/master/CHANGELOG.md)
- [Commits](https://github.com/googleapis/google-api-nodejs-client/compare/v32.0.0...v39.1.0)

Signed-off-by: dependabot[bot] <support@github.com>

* Fixed tests

* fix example sheet

Co-authored-by: Christoph Knoth <github@christoph-knoth.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: efarooqui <efarooqui@pandora.com>
Co-authored-by: Ebrahem Farooqui <ebefarooqui@gmail.com>
Co-authored-by: Lachlan <Kermode>

.gitignore

@@ -14,3 +14,4 @@ tags
 tags.lock
 tags.temp
 src/config.js
+src/local.config.js

package.json

@@ -26,16 +26,15 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
-    "googleapis": "^32.0.0",
+    "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
-    "moment": "^2.27.0",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.3.0",
-    "node-xlsx": "^0.15.0",
+    "node-fetch": "^2.6.1",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0"
+    "resource-router-middleware": "^0.6.0",
+    "xlsx": "^0.16.8"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",

src/api/index.js

@@ -58,17 +58,6 @@ export default ({ config, controller }) => {
       )
   })
 
-  // api.get('/:sheet/:tab/:resource/validate', (req, res) => {
-  //   const { sheet, tab, resource } = req.params
-  //   controller
-  //     .retrieve(sheet, tab, resource)
-  //     .then(data => res.json(data))
-  //     .catch(err =>
-  //       res.status(err.status || 404)
-  //         .send({ error: err.message })
-  //     )
-  // })
-
   // ERROR routes. Note that it is important that these come AFTER routes
   // like /update, so that the regex does not greedily match these routes.
 

src/blueprinters/deeprows.js

@@ -1,5 +1,4 @@
 import { fmtObj } from '../lib/util'
-import { getColumnValidation } from '../lib/validation'
 
 /**
  * Each resource item is an object with values labelled according
@@ -36,6 +35,7 @@ export default (data) => {
       }
     }
   })
+
   // generate the value for deep labels using the structure created
   data.forEach((row, idx) => {
     if (idx === 0) return
@@ -61,11 +61,12 @@ export default (data) => {
 
     // move values for flat labels over from base
     structure.__flat.forEach(label => {
-      const validatedLabel = getColumnValidation(label, baseRow[label])
-      console.log(validatedLabel)
       deepRow[label] = baseRow[label]
     })
-    if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {
+    if (!Object.keys(deepRow).every(k => (
+      (deepRow[k] === '') ||
+      (Array.isArray(deepRow[k]) && deepRow[k].length === 0)
+    ))) {
       output.push(deepRow)
     }
   })

src/config.js

@@ -1,13 +1,7 @@
 import { timemap } from './lib'
 
 export default {
-  gsheets: [
-    {
-      name: 'us2020',
-      id: '1I_pgyTQJlIorTIEHBxw1mM1STn-SrIi66FKYxut61iM',
-      tabs: timemap.default
-    }
-  ],
+  gsheets: [],
   xlsx: [
     {
       name: 'timemap_data',

src/index.js

@@ -1,4 +1,5 @@
 import http from 'http'
+import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
@@ -40,7 +41,7 @@ initialize(controller => {
     })
   )
 
-  app.use(express.static(__dirname + '/public'))
+  app.use(express.static(path.join(__dirname, 'public')))
 
   app.server.listen(process.env.PORT || 4040, () => {
     console.log('===========================================')

src/initialize.js

@@ -1,7 +1,6 @@
 import StoreJson from './models/StoreJson'
 import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
-import config from './config'
 import R from 'ramda'
 
 const isntNull = n => n !== null
@@ -9,6 +8,13 @@ const filterNull = ls => R.filter(isntNull, ls)
 const flattenfilterNull = ls => filterNull(R.flatten(ls))
 let themFetchers
 
+let config
+try {
+  config = require('./local.config.js').default
+} catch (_) {
+  config = require('./config.js').default
+}
+
 export default callback => {
   return Promise.resolve().then(() => {
     return Object.keys(config).map(fType => {

src/lib.js

@@ -5,9 +5,7 @@ function prefixedTabs (prefix, cfg) {
   const prf = key => cfg[key] ? `${prefix}_` : ''
   return {
     [`${prf('events')}export_events`]: BP.deeprows,
-    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
-    [`${prf('filters')}export_filters`]: BP.tree,
-    [`${prf('narratives')}export_narratives`]: BP.rows,
+    [`${prf('associations')}export_associations`]: BP.deeprows,
     [`${prf('sources')}export_sources`]: BP.deepids,
     [`${prf('sites')}export_sites`]: BP.rows
   }

src/lib/Fetcher.js

@@ -1,15 +1,16 @@
-// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
-import { google } from 'googleapis'
+import R from 'ramda'
+import { createHash } from 'crypto'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-import { createHash } from 'crypto'
-import R from 'ramda'
-import xlsx from 'node-xlsx'
-import fs from 'fs'
+
+/* GsheetFetcher deps */
+import { google } from 'googleapis'
+/* LocalFetcher deps */
+import X from 'xlsx'
 
 class Fetcher {
   constructor (db, name, bps) {
@@ -138,7 +139,6 @@ class Fetcher {
 
   /** Run on startup. Should be overridden if explicit auth is required **/
   authenticate (env) {
-    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
     return Promise.resolve(this)
   }
 }
@@ -221,27 +221,22 @@ class GsheetFetcher extends Fetcher {
   }
 }
 
-class XlsxFetcher extends Fetcher {
+class LocalFetcher extends Fetcher {
   constructor (db, name, bps, path) {
     super(db, name, bps)
-    this.type = 'XLSX File'
     this.path = path
-    this.isRemote = false
-
-    if (this.path.startsWith('https')) {
-      this.isRemote = true
-    }
+    this.update().then(res =>
+      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
+    )
   }
 
   update () {
-    const data = xlsx.parse(fs.readFileSync(this.path))
-    data.forEach(tab => {
-      const stringyData = tab.data.map(row =>
-        row.map(d =>
-          typeof (d) === 'number' ? d.toString() : d
-        )
-      )
-      this.save(tab.name, stringyData)
+    const wb = X.readFile(this.path)
+    wb.SheetNames.forEach(name => {
+      const sh = wb.Sheets[name]
+      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
+      const ll = csv.split('\n').map(line => line.split('\t'))
+      this.save(name, ll)
     })
     return Promise.resolve(true)
   }
@@ -249,5 +244,7 @@ class XlsxFetcher extends Fetcher {
 
 export default {
   'gsheets': GsheetFetcher,
-  'xlsx': XlsxFetcher
+  'xlsx': LocalFetcher,
+  'ods': LocalFetcher,
+  'local': LocalFetcher
 }

src/lib/util.js

@@ -1,4 +1,4 @@
-import R from 'ramda';
+import R from 'ramda'
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {

src/lib/validation.js

@@ -1,20 +0,0 @@
-import moment from 'moment';
-
-const DATE_FORMAT = "MM/DD/YYYY";
-const TIME_REGEX = new RegExp("^([01]\d|2[0-3]):?([0-5]\d)$")
-
-
-export const getColumnValidation = (colName, value) => {
-  switch(colName) {
-    case 'longitude':
-      return isFinite(value) && Math.abs(value) <= 180;
-    case 'latitude':
-      return isFinite(value) && Math.abs(value) <= 90;
-    case 'date':
-      return moment(value, DATE_FORMAT, true).isValid();
-    case 'time':
-      return TIME_REGEX.test(value);
-    default:
-      return true
-  }
-}

test/serverProcess.js

@@ -54,14 +54,7 @@ test('should launch', t => {
 })
 
 const passUrls = [
-  // /
   '/api/',
-  // /blueprints
-  '/api/blueprints',
-  // /:sheet/:tab/:resource
-  '/api/example/export_events/rows',
-  // /:sheet/:tab/:resource/:frag
-  '/api/example/export_events/rows/1'
 ]
 
 const failUrls = [
@@ -77,6 +70,7 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
+        console.info('JSON: ', json)
         t.pass()
       })
   })

views/blueprints.hbs

@@ -9,7 +9,6 @@
     </div>
     {{#each urls}}
     <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
-    <div class="bp-validate-button" target="_blank" href="http://localhost:4040{{ this }}">Validate</div>
     {{/each}}
   </div>
   {{ else }}
@@ -26,7 +25,6 @@
 :root {
   --grey: #8a8a8a;
   --btn-width: 200px;
-  --validate-btn-width: 150px;
 }
 .main-container {
   display: flex;
@@ -81,21 +79,6 @@
   color: black;
 }
 
-.bp-validate-button {
-  transition: all 0.3s ease;
-  text-transform: uppercase;
-  font-size: 12pt;
-  font-weight: bold;
-  width: var(--validate-btn-width);
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  border: 3px solid red;
-  padding: .5em;
-  text-decoration: none;
-  color: red;
-}
-
 .bp-update-container:hover {
   background-color: black;
   transition: all 0.3s ease;