Updated view with validation button

Bump lodash from 4.17.15 to 4.17.19

.gitignore

@@ -14,4 +14,3 @@ tags
 tags.lock
 tags.temp
 src/config.js
-src/local.config.js

package.json

@@ -26,15 +26,16 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
-    "googleapis": "^39.1.0",
+    "googleapis": "^32.0.0",
     "graphql": "^0.13.2",
+    "moment": "^2.27.0",
     "morgan": "^1.8.0",
     "mz": "^2.7.0",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "^2.3.0",
+    "node-xlsx": "^0.15.0",
     "object-hash": "^1.3.0",
     "ramda": "^0.25.0",
-    "resource-router-middleware": "^0.6.0",
-    "xlsx": "^0.16.8"
+    "resource-router-middleware": "^0.6.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.2",

src/api/index.js

@@ -58,6 +58,17 @@ export default ({ config, controller }) => {
       )
   })
 
+  // api.get('/:sheet/:tab/:resource/validate', (req, res) => {
+  //   const { sheet, tab, resource } = req.params
+  //   controller
+  //     .retrieve(sheet, tab, resource)
+  //     .then(data => res.json(data))
+  //     .catch(err =>
+  //       res.status(err.status || 404)
+  //         .send({ error: err.message })
+  //     )
+  // })
+
   // ERROR routes. Note that it is important that these come AFTER routes
   // like /update, so that the regex does not greedily match these routes.
 

src/blueprinters/deeprows.js

@@ -1,4 +1,5 @@
 import { fmtObj } from '../lib/util'
+import { getColumnValidation } from '../lib/validation'
 
 /**
  * Each resource item is an object with values labelled according
@@ -35,7 +36,6 @@ export default (data) => {
       }
     }
   })
-
   // generate the value for deep labels using the structure created
   data.forEach((row, idx) => {
     if (idx === 0) return
@@ -61,12 +61,11 @@ export default (data) => {
 
     // move values for flat labels over from base
     structure.__flat.forEach(label => {
+      const validatedLabel = getColumnValidation(label, baseRow[label])
+      console.log(validatedLabel)
       deepRow[label] = baseRow[label]
     })
-    if (!Object.keys(deepRow).every(k => (
-      (deepRow[k] === '') ||
-      (Array.isArray(deepRow[k]) && deepRow[k].length === 0)
-    ))) {
+    if (!Object.keys(deepRow).every(k => deepRow[k] === '')) {
       output.push(deepRow)
     }
   })

src/config.js

@@ -1,7 +1,13 @@
 import { timemap } from './lib'
 
 export default {
-  gsheets: [],
+  gsheets: [
+    {
+      name: 'us2020',
+      id: '1I_pgyTQJlIorTIEHBxw1mM1STn-SrIi66FKYxut61iM',
+      tabs: timemap.default
+    }
+  ],
   xlsx: [
     {
       name: 'timemap_data',

src/index.js

@@ -1,5 +1,4 @@
 import http from 'http'
-import path from 'path'
 import express from 'express'
 import initialize from './initialize'
 import middleware from './middleware'
@@ -41,7 +40,7 @@ initialize(controller => {
     })
   )
 
-  app.use(express.static(path.join(__dirname, 'public')))
+  app.use(express.static(__dirname + '/public'))
 
   app.server.listen(process.env.PORT || 4040, () => {
     console.log('===========================================')

src/initialize.js

@@ -1,6 +1,7 @@
 import StoreJson from './models/StoreJson'
 import fetchers from './lib/Fetcher'
 import Controller from './lib/Controller'
+import config from './config'
 import R from 'ramda'
 
 const isntNull = n => n !== null
@@ -8,13 +9,6 @@ const filterNull = ls => R.filter(isntNull, ls)
 const flattenfilterNull = ls => filterNull(R.flatten(ls))
 let themFetchers
 
-let config
-try {
-  config = require('./local.config.js').default
-} catch (_) {
-  config = require('./config.js').default
-}
-
 export default callback => {
   return Promise.resolve().then(() => {
     return Object.keys(config).map(fType => {

src/lib.js

@@ -5,7 +5,9 @@ function prefixedTabs (prefix, cfg) {
   const prf = key => cfg[key] ? `${prefix}_` : ''
   return {
     [`${prf('events')}export_events`]: BP.deeprows,
-    [`${prf('associations')}export_associations`]: BP.deeprows,
+    [`${prf('categories')}export_categories`]: [BP.groups, BP.rows],
+    [`${prf('filters')}export_filters`]: BP.tree,
+    [`${prf('narratives')}export_narratives`]: BP.rows,
     [`${prf('sources')}export_sources`]: BP.deepids,
     [`${prf('sites')}export_sites`]: BP.rows
   }

src/lib/Fetcher.js

@@ -1,16 +1,15 @@
-import R from 'ramda'
-import { createHash } from 'crypto'
+// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
+import { google } from 'googleapis'
 import { buildDesaturated } from './blueprinters'
 import {
   fmtName,
   fmtBlueprinterTitles,
   isFunction
 } from './util'
-
-/* GsheetFetcher deps */
-import { google } from 'googleapis'
-/* LocalFetcher deps */
-import X from 'xlsx'
+import { createHash } from 'crypto'
+import R from 'ramda'
+import xlsx from 'node-xlsx'
+import fs from 'fs'
 
 class Fetcher {
   constructor (db, name, bps) {
@@ -139,6 +138,7 @@ class Fetcher {
 
   /** Run on startup. Should be overridden if explicit auth is required **/
   authenticate (env) {
+    console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
     return Promise.resolve(this)
   }
 }
@@ -221,22 +221,27 @@ class GsheetFetcher extends Fetcher {
   }
 }
 
-class LocalFetcher extends Fetcher {
+class XlsxFetcher extends Fetcher {
   constructor (db, name, bps, path) {
     super(db, name, bps)
+    this.type = 'XLSX File'
     this.path = path
-    this.update().then(res =>
-      console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
-    )
+    this.isRemote = false
+
+    if (this.path.startsWith('https')) {
+      this.isRemote = true
+    }
   }
 
   update () {
-    const wb = X.readFile(this.path)
-    wb.SheetNames.forEach(name => {
-      const sh = wb.Sheets[name]
-      const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
-      const ll = csv.split('\n').map(line => line.split('\t'))
-      this.save(name, ll)
+    const data = xlsx.parse(fs.readFileSync(this.path))
+    data.forEach(tab => {
+      const stringyData = tab.data.map(row =>
+        row.map(d =>
+          typeof (d) === 'number' ? d.toString() : d
+        )
+      )
+      this.save(tab.name, stringyData)
     })
     return Promise.resolve(true)
   }
@@ -244,7 +249,5 @@ class LocalFetcher extends Fetcher {
 
 export default {
   'gsheets': GsheetFetcher,
-  'xlsx': LocalFetcher,
-  'ods': LocalFetcher,
-  'local': LocalFetcher
+  'xlsx': XlsxFetcher
 }

src/lib/util.js

@@ -1,4 +1,4 @@
-import R from 'ramda'
+import R from 'ramda';
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {

src/lib/validation.js

@@ -0,0 +1,20 @@
+import moment from 'moment';
+
+const DATE_FORMAT = "MM/DD/YYYY";
+const TIME_REGEX = new RegExp("^([01]\d|2[0-3]):?([0-5]\d)$")
+
+
+export const getColumnValidation = (colName, value) => {
+  switch(colName) {
+    case 'longitude':
+      return isFinite(value) && Math.abs(value) <= 180;
+    case 'latitude':
+      return isFinite(value) && Math.abs(value) <= 90;
+    case 'date':
+      return moment(value, DATE_FORMAT, true).isValid();
+    case 'time':
+      return TIME_REGEX.test(value);
+    default:
+      return true
+  }
+}

test/serverProcess.js

@@ -54,7 +54,14 @@ test('should launch', t => {
 })
 
 const passUrls = [
+  // /
   '/api/',
+  // /blueprints
+  '/api/blueprints',
+  // /:sheet/:tab/:resource
+  '/api/example/export_events/rows',
+  // /:sheet/:tab/:resource/:frag
+  '/api/example/export_events/rows/1'
 ]
 
 const failUrls = [
@@ -70,7 +77,6 @@ passUrls.forEach(function (url) {
       .then(checkStatus)
       .then(res => res.json())
       .then(json => {
-        console.info('JSON: ', json)
         t.pass()
       })
   })

views/blueprints.hbs

@@ -9,6 +9,7 @@
     </div>
     {{#each urls}}
     <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
+    <div class="bp-validate-button" target="_blank" href="http://localhost:4040{{ this }}">Validate</div>
     {{/each}}
   </div>
   {{ else }}
@@ -25,6 +26,7 @@
 :root {
   --grey: #8a8a8a;
   --btn-width: 200px;
+  --validate-btn-width: 150px;
 }
 .main-container {
   display: flex;
@@ -79,6 +81,21 @@
   color: black;
 }
 
+.bp-validate-button {
+  transition: all 0.3s ease;
+  text-transform: uppercase;
+  font-size: 12pt;
+  font-weight: bold;
+  width: var(--validate-btn-width);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  border: 3px solid red;
+  padding: .5em;
+  text-decoration: none;
+  color: red;
+}
+
 .bp-update-container:hover {
   background-color: black;
   transition: all 0.3s ease;