Merge pull request #22 from forensic-architecture/topic/build-bp

Topic/build bp closes #1

src/blueprinters/columns.js

@@ -1,5 +1,5 @@
 import R from 'ramda'
-import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
+import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
  * byColumn - generate a Blueprint from a data sheet by column. Each column
@@ -9,7 +9,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
  * @return {type} Blueprint
  * generated.
  */
-export default function byColumn (tabName, sheetName, sheetId, data) {
+function columns (tabName, sheetName, sheetId, data) {
   // Define Blueprint props
   const bp = R.clone(defaultBlueprint)
   bp.sheet = {
@@ -18,18 +18,20 @@ export default function byColumn (tabName, sheetName, sheetId, data) {
   }
   bp.name = tabName
 
-  // column names define routes
+  // column names define resources
   const labels = data[0]
   labels.forEach(label => {
-    bp.routes[label] = R.clone(defaultRoute)
+    bp.resources[label] = R.clone(defaultResource)
   })
 
   // remaining rows as data
   data.forEach((row, idx) => {
     if (idx === 0) return
     labels.forEach((label, idx) => {
-      bp.routes[label].data.push(row[idx])
+      bp.resources[label].data.push(row[idx])
     })
   })
   return bp
 }
+
+export default columns

src/blueprinters/groups.js

@@ -1,9 +1,9 @@
 import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
+import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * byGroup - generate a Blueprint from a data sheet grouped by a column called 'group'
+ * groups - generate a Blueprint from a data sheet grouped by a column called 'group'
  * The resource name defaults to 'groups', or a custom resource name can be passed.
  * Each resource item is an object with values labelled according to column
  * names. Items are inserted in the data list at idx = id.
@@ -13,7 +13,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
  * @param  {type} name=""      name of blueprint.
  * @return {type} Blueprint
  */
-export default function byGroup (
+export default function groups (
   tabName,
   sheetName,
   sheetId,
@@ -28,11 +28,11 @@ export default function byGroup (
   }
   bp.name = tabName
 
-  // Column names define routes
+  // Column names define resources
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.routes[label] = R.clone(defaultRoute)
+  bp.resources[label] = R.clone(defaultResource)
-  bp.routes[label].data = []
+  bp.resources[label].data = []
 
   const dataGroups = {}
 
@@ -46,7 +46,7 @@ export default function byGroup (
     }
   })
   Object.keys(dataGroups).forEach(groupKey => {
-    bp.routes[label].data.push({
+    bp.resources[label].data.push({
       group: groupKey,
       group_label: dataGroups[groupKey][0].group_label,
       data: dataGroups[groupKey]

src/blueprinters/ids.js

@@ -1,9 +1,9 @@
 import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
+import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * byId - generate a Blueprint from a data sheet by id, which is an integer.
+ * ids - generate a Blueprint from a data sheet by id, which is an integer.
  * The resource name defaults to 'ids', or a custom resource name can be passed.
  * Each resource item is an object with values labelled according to column
  * names. Items are inserted in the data list at idx = id.
@@ -13,7 +13,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
  * @param  {type} name=""      name of blueprint.
  * @return {type} Blueprint
  */
-export default function byId (
+export default function ids (
   tabName,
   sheetName,
   sheetId,
@@ -28,15 +28,15 @@ export default function byId (
   }
   bp.name = tabName
 
-  // Column names define routes
+  // Column names define resources
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.routes[label] = R.clone(defaultRoute)
+  bp.resources[label] = R.clone(defaultResource)
-  bp.routes[label].data = []
+  bp.resources[label].data = []
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.routes[label].data[fmt(row).id] = fmt(row)
+    bp.resources[label].data[fmt(row).id] = fmt(row)
   })
   return bp
 }

src/blueprinters/rows.js

@@ -1,9 +1,9 @@
 import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
+import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * byRow - generate a Blueprint from a data sheet by row. The resource name
+ * rows - generate a Blueprint from a data sheet by row. The resource name
  * defaults to 'rows', or a custom resource name can be passed. Each resource
  * item is an object with values labelled according to column names.
  *
@@ -12,7 +12,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
  * @param  {type} name=""      name of blueprint.
  * @return {type} Blueprint
  */
-export default function byRow (
+export default function rows (
   tabName,
   sheetName,
   sheetId,
@@ -27,15 +27,15 @@ export default function byRow (
   }
   bp.name = tabName
 
-  // Column names define routes
+  // Column names define resources
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.routes[label] = R.clone(defaultRoute)
+  bp.resources[label] = R.clone(defaultResource)
-  bp.routes[label].data = []
+  bp.resources[label].data = []
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.routes[label].data.push(fmt(row))
+    bp.resources[label].data.push(fmt(row))
   })
   return bp
 }

src/blueprinters/tree.js

@@ -1,8 +1,8 @@
 import R from 'ramda'
-import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
+import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * byTree - generate a Blueprint from a data sheet grouped by a column called 'group'
+ * tree - generate a Blueprint from a data sheet grouped by a column called 'group'
  * The resource name defaults to 'groups', or a custom resource name can be passed.
  * Each resource item is an object with values labelled according to column
  * names. Items are inserted in the data list at idx = id.
@@ -12,7 +12,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters'
  * @param  {type} name=""      name of blueprint.
  * @return {type} Blueprint
  */
-export default function byTree (
+export default function tree (
   tabName,
   sheetName,
   sheetId,
@@ -27,9 +27,9 @@ export default function byTree (
   }
   bp.name = tabName
 
-  // Column names define routes
+  // Column names define resources
-  bp.routes[label] = R.clone(defaultRoute)
+  bp.resources[label] = R.clone(defaultResource)
-  bp.routes[label].data = {}
+  bp.resources[label].data = {}
 
   const tree = {
     key: 'tags',
@@ -62,6 +62,6 @@ export default function byTree (
     }
   })
 
-  bp.routes[label].data = tree
+  bp.resources[label].data = tree
   return bp
 }

src/copy/en.js

@@ -4,7 +4,7 @@ export default {
     onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
-    noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' of the sheet '${prts[0]}'.`,
+    noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' in this sheet.`,
     noFragment: prts => `Fragment index does not exist`
   },
   success: {

src/lib/Controller.js

@@ -9,7 +9,7 @@ class Controller {
     this.fetchers = fetchers
   }
 
-  sheetExists (sheet) {
+  _sheetExists (sheet) {
     return (Object.keys(this.fetchers).indexOf(sheet) >= 0)
   }
 
@@ -34,7 +34,7 @@ class Controller {
   }
 
   retrieve (sheet, tab, resource) {
-    if (this.sheetExists(sheet)) {
+    if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]
       return fetcher.retrieve(tab, resource)
     } else {
@@ -43,7 +43,7 @@ class Controller {
   }
 
   retrieveFrag (sheet, tab, resource, frag) {
-    if (this.sheetExists(sheet)) {
+    if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]
       return fetcher.retrieveFrag(tab, resource, frag)
     } else {

src/lib/Fetcher.js

@@ -1,12 +1,12 @@
 // FetcherTwo class interfaces with Google Sheet, and saves to a specified db
 import { google } from 'googleapis'
+import { buildDesaturated } from './blueprinters'
 import {
-  fmtSheetTitle,
+  fmtName,
   fmtBlueprinterTitles,
-  bp,
   isFunction
 } from './util'
-import { byRow } from './blueprinters'
+import { createHash } from 'crypto'
 import R from 'ramda'
 
 class Fetcher {
@@ -28,39 +28,80 @@ class Fetcher {
      */
     this.sheetName = sheetName
 
+    /*
+     * A unique ID for the Fetcher to identify its elements in the model layer
+     */
+    this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex')
+
     /*
      * These are the available tabs for storing and retrieving data.
      * Each blueprinter is a function that returns a Blueprint from a
      * list of lists (which will be retrieved from gsheets).
      */
     this.blueprinters = fmtBlueprinterTitles(blueprinters)
-    this.blueprints = {}
+
-    Object.keys(this.blueprinters).forEach(key => {
+    /*
-      this.blueprints[key] = null
+     * This object is the canonical represenation for the data that a Fetcher
-    })
+     * proxies. When the fetcher is initialized, its model layer (db) is indexed,
+     * and this object populated accordingly. Whenever the fetcher updates, this
+     * data structure updates as well. It is the model layer that determines the
+     * performance of indexing the blueprints.
+     */
+    this.blueprints = null
+    this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion
 
     /*
      * Google API setup
      */
-    this.sheets = google.sheets('v4')
+    this.API = google.sheets('v4')
     this.auth = null
 
-    /**
+    /** curry to allow convenient syntax with map */
-     * saveBp is a curried function that takes in a title and
+    this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter)
-     * a blueprinter. NB: it sits here in the constructor as
+  }
-     * I am not sure how to curry a class method with Ramda.
+
+  _buildBlueprintsAsync () {
+    return this.db.index()
+      .then(allUrls => {
+        const allParts = allUrls.reduce((acc, url) => {
+          if (url.startsWith(this.id)) {
+            const parts = url.split('/')
+            acc.push([ parts[1], parts[2] ])
+            return acc
+          } else {
+            return acc
+          }
+        }, [])
+
+        return allParts
+          .map(parts => buildDesaturated(
+            this.sheetId,
+            this.sheetName,
+            parts[0],
+            parts[1]
+          ))
+      })
+      .then(res => {
+        this.blueprints = res
+      })
+  }
+
+  /** save data under a given tab name via its blueprinter, which generates
+   * its resource name. Note that this is curried in the constructor.
    */
-    this._saveBp = R.curry((tab, title, data, blueprinter) => {
+  _saveViaBlueprinter (tab, data, blueprinter) {
     const saturatedBp = blueprinter(
       tab,
       this.sheetName,
       this.sheetId,
       data
     )
-      const blueprint = bp(saturatedBp) // TODO: come up with better semantics.
+
-      this.blueprints[title] = blueprint
+    return Promise.all(
-      return this.db.save(saturatedBp)
+      Object.keys(saturatedBp.resources).map(route =>
-    })
+        this.db.save(`${this.id}/${tab}/${route}`, saturatedBp.resources[route].data)
+      )
+    )
   }
 
   /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
@@ -84,15 +125,15 @@ class Fetcher {
 
   update () {
     let tabTitles
-    /* Retrieve all available routes on a given sheet, and store formatted copies of it where a formatter is available */
+    /* Retrieve all available resources on a given sheet, and store formatted copies of it where a formatter is available */
-    return this.sheets.spreadsheets
+    return this.API.spreadsheets
       .get({
         auth: this.auth,
         spreadsheetId: this.sheetId
       })
       .then(response => {
         tabTitles = response.data.sheets.map(sheet => sheet.properties.title)
-        return this.sheets.spreadsheets.values.batchGet({
+        return this.API.spreadsheets.values.batchGet({
           auth: this.auth,
           spreadsheetId: this.sheetId,
           ranges: tabTitles
@@ -100,12 +141,15 @@ class Fetcher {
       })
       .then(results => {
         const tabData = results.data.valueRanges
+
         return Promise.all(
           tabData.map((tab, idx) => {
             const { values } = tab
+
             if (values === undefined) {
               return Promise.resolve({})
             }
+
             const name = tabTitles[idx]
             return this.save(name, values)
           })
@@ -115,31 +159,35 @@ class Fetcher {
       .catch(() => false)
   }
 
-  save (tab, data) {
+  save (_tab, data) {
-    const title = fmtSheetTitle(tab)
+    const tab = fmtName(_tab)
-    if (Object.keys(this.blueprinters).indexOf(title) > -1) {
+
-      const bpConfig = this.blueprinters[title]
+    if (Object.keys(this.blueprinters).indexOf(tab) > -1) {
+      const bpConfig = this.blueprinters[tab]
 
       if (isFunction(bpConfig)) {
-        return this._saveBp(tab, title, data, bpConfig)
+        // if bpConfig specifies a single blueprinter
+        return this._saveViaBlueprinter(tab, data, bpConfig)
       } else {
-        return bpConfig.map(this._saveBp(tab, title, data))
+        // if bpConfig specifies an array of blueprinters
+        return bpConfig.map(this._saveViaBlueprinter(tab, data))
       }
     } else {
-      // If it can't find a blueprinter for the tab title, default to byRow
+      // NB: if a blueprinter is not specified for a tab,
-      return this.db.save(byRow(tab, this.sheetName, this.sheetId, data))
+      // just skip it.
+      return true
     }
   }
 
   // NB: could combine these functions by checking kwargs length
   retrieve (tab, resource) {
-    const title = fmtSheetTitle(tab)
+    const title = fmtName(tab)
-    const url = `${this.sheetName}/${tab}/${resource}`
+    const url = `${this.id}/${tab}/${resource}`
     return this.db.load(url, this.blueprints[title])
   }
 
   retrieveFrag (tab, resource, frag) {
-    const title = fmtSheetTitle(tab)
+    const title = fmtName(tab)
     const url = `${this.sheetName}/${tab}/${resource}/${frag}`
     return this.db.load(url, this.blueprints[title])
   }

src/lib/blueprinters.js

@@ -1,20 +1,29 @@
 import path from 'path'
 import fs from 'fs'
+import R from 'ramda'
 
 export const defaultBlueprint = {
   name: null,
-  id: null,
+  sheet: {
-  dialects: ['rest'], // supported dialects, can (eventually) be multiple
+    name: null,
-  routes: {}
+    id: null
+  },
+  resources: {}
 }
 
-export const defaultRoute = {
+export const defaultResource = {
-  options: {
-    fragment: true
-  },
   data: []
 }
 
+export function buildDesaturated (sheetId, sheetName, tab, resource) {
+  const bp = R.clone(defaultBlueprint)
+  bp.sheet.name = sheetName
+  bp.sheet.id = sheetId
+  bp.name = tab
+  bp.resources[resource] = null
+  return bp
+}
+
 // import all default exports from 'blueprinters' folder
 const allBps = {}
 const REL_PATH_TO_BPS = '../blueprinters'
@@ -28,5 +37,6 @@ fs.readdirSync(normalizedPath).forEach(file => {
 // each file in blueprinters folder available for granular import from here.
 module.exports = Object.assign({
   defaultBlueprint,
-  defaultRoute
+  defaultResource,
+  buildDesaturated
 }, allBps)

src/lib/util.js

@@ -55,33 +55,33 @@ export const idxSearcher = R.curry((attrName, searchValue, myArray) => {
 
 /* more site specific functions. TODO: maybe move to another folder? */
 
-export function fmtSheetTitle (name) {
+export function fmtName (name) {
   return name.replaceAll(' ', '-').toLowerCase()
 }
 
 export function fmtBlueprinterTitles (tabs) {
   const obj = {}
   Object.keys(tabs).forEach(tab => {
-    const name = fmtSheetTitle(tab)
+    const name = fmtName(tab)
     obj[name] = tabs[tab]
   })
   return obj
 }
 
 export function deriveFilename (sheet, tab) {
-  return `${fmtSheetTitle(sheet)}-${fmtSheetTitle(tab)}.json`
+  return `${fmtName(sheet)}-${fmtName(tab)}.json`
 }
 
-export function bp (full) {
+export function desaturate (full) {
   const blueprint = {
     name: R.clone(full.name),
     sheet: R.clone(full.sheet),
     dialects: R.clone(full.dialects),
-    routes: {}
+    resources: {}
   }
-  Object.keys(full.routes).forEach(route => {
+  Object.keys(full.resources).forEach(route => {
-    blueprint.routes[route] = {
+    blueprint.resources[route] = {
-      options: R.clone(full.routes[route].options)
+      options: R.clone(full.resources[route].options)
     }
   })
   return blueprint

src/models/StoreJson.js

@@ -1,20 +1,28 @@
 import fs from 'mz/fs'
-import { fmtSheetTitle } from '../lib/util'
 import copy from '../copy/en'
 
 const STORAGE_DIRNAME = 'temp'
 
+function partsFromFilename (fname) {
+  const body = fname.slice(0, -5)
+  return body.split('__')
+}
+
 class StoreJson {
-  save (bp) {
+  index () {
-    return Promise.all(
+    return Promise.resolve()
-      Object.keys(bp.routes).map(route =>
+      .then(() => fs.readdir(STORAGE_DIRNAME))
-        fs.writeFile(
+      .then(files => files.filter(f => f.match(/.*\.json$/)))
-          `${STORAGE_DIRNAME}/${fmtSheetTitle(
+      .then(jsons => jsons.map(partsFromFilename))
-            bp.sheet.name
+      .then(parts => parts.map(p => `${p[0]}/${p[1]}/${p[2]}`))
-          )}__${fmtSheetTitle(bp.name)}__${route}.json`,
+  }
-          JSON.stringify(bp.routes[route].data)
+
-        )
+  save (url, data) {
-      )
+    const parts = url.split('/')
+
+    return fs.writeFile(
+      `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`,
+      JSON.stringify(data)
     )
   }
 

test/internals.js

@@ -2,9 +2,9 @@ import test from 'ava'
 import R from 'ramda'
 import {
   defaultBlueprint,
-  defaultRoute,
+  defaultResource,
-  byColumn,
+  columns,
-  byRow
+  rows
 } from '../src/lib/blueprinters'
 
 const egInput1 = [
@@ -15,41 +15,43 @@ const egInput1 = [
 
 test('defaultBlueprint exports', t => {
   const expected = {
+    sheet: {
       name: null,
-    id: null,
+      id: null
-    dialects: ['rest'],
+    },
-    routes: {}
+    name: null,
+    resources: {}
   }
   t.deepEqual(expected, defaultBlueprint)
 })
 
-test('byColumn blueprinter generates expected output', t => {
+test('columns blueprinter generates expected output', t => {
-  const actual = byColumn('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1)
+  const actual = columns('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1)
   const expected = R.clone(defaultBlueprint)
   expected.name = 'eg ColumnBlueprint'
   expected.sheet = {
     id: 'egSheetId',
     name: 'egSheetName'
   }
-  expected.routes['h1'] = R.clone(defaultRoute)
+  expected.resources['h1'] = R.clone(defaultResource)
-  expected.routes['h1'].data = [1, 4]
+  expected.resources['h1'].data = [1, 4]
-  expected.routes['h2'] = R.clone(defaultRoute)
+  expected.resources['h2'] = R.clone(defaultResource)
-  expected.routes['h2'].data = [2, 5]
+  expected.resources['h2'].data = [2, 5]
-  expected.routes['h3'] = R.clone(defaultRoute)
+  expected.resources['h3'] = R.clone(defaultResource)
-  expected.routes['h3'].data = [3, 6]
+  expected.resources['h3'].data = [3, 6]
   t.deepEqual(expected, actual)
 })
 
-test('byRow blueprinter generates expected output', t => {
+test('rows blueprinter generates expected output', t => {
-  const actual = byRow('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items')
+  const actual = rows('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items')
   const expected = R.clone(defaultBlueprint)
   expected.name = 'egRowBlueprint'
   expected.sheet = {
     id: 'egSheetId',
     name: 'egSheetName'
   }
-  expected.routes['items'] = R.clone(defaultRoute)
+  expected.resources['items'] = R.clone(defaultResource)
-  expected.routes['items'].data = [{
+  expected.resources['items'].data = [{
     h1: 1,
     h2: 2,
     h3: 3