From 5431b2be3f9800fa7390e8d5372aac5d9e98b74c Mon Sep 17 00:00:00 2001
From: Lachlan Kermode <lachiekermode@gmail.com>
Date: Thu, 13 Dec 2018 15:56:54 +0000
Subject: [PATCH 1/6] abstract generic logic from blueprinters to
 blueprinters.js

The logic in the files in the 'blueprinters' folder is now _only_ the data transformation logic. Instead of taking in arguments like the sheetId, the tabName, and the sheetName, the function now takes a single argument: the list of lists that represents the raw data from the sheet.

This setup gives datasheet-server greater value, as it allows developers to only specify the transformation logic, and not worry about the other stuff that datasheet server is doing.
---
 src/blueprinters/columns.js  | 37 ---------------------------
 src/blueprinters/deeprows.js | 26 +++++++++++++++++++
 src/blueprinters/groups.js   | 48 +++++++++++-------------------------
 src/blueprinters/ids.js      | 38 +++++++---------------------
 src/blueprinters/rows.js     | 37 +++++++--------------------
 src/blueprinters/tree.js     | 36 +++++----------------------
 src/lib/Controller.js        |  2 ++
 src/lib/Fetcher.js           |  4 +--
 src/lib/blueprinters.js      | 17 ++++++++++++-
 9 files changed, 84 insertions(+), 161 deletions(-)
 delete mode 100644 src/blueprinters/columns.js
 create mode 100644 src/blueprinters/deeprows.js

diff --git a/src/blueprinters/columns.js b/src/blueprinters/columns.js
deleted file mode 100644
index eee7bc8..0000000
--- a/src/blueprinters/columns.js
+++ /dev/null
@@ -1,37 +0,0 @@
-import R from 'ramda'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
-
-/**
- * byColumn - generate a Blueprint from a data sheet by column. Each column
- * name is a resheet, and all values in that column are the resheet items.
- *
- * @param  {type} data - list of lists representing sheet data.
- * @return {type} Blueprint
- * generated.
- */
-function columns (tabName, sheetName, sheetId, data) {
-  // Define Blueprint props
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // column names define resources
-  const labels = data[0]
-  labels.forEach(label => {
-    bp.resources[label] = R.clone(defaultResource)
-  })
-
-  // remaining rows as data
-  data.forEach((row, idx) => {
-    if (idx === 0) return
-    labels.forEach((label, idx) => {
-      bp.resources[label].data.push(row[idx])
-    })
-  })
-  return bp
-}
-
-export default columns
diff --git a/src/blueprinters/deeprows.js b/src/blueprinters/deeprows.js
new file mode 100644
index 0000000..7e00e69
--- /dev/null
+++ b/src/blueprinters/deeprows.js
@@ -0,0 +1,26 @@
+import R from 'ramda'
+import { fmtObj } from '../lib/util'
+
+/**
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row. If two or more
+ * column names are the same except for a different integer at the end
+ * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
+ * aggregated into a list, which is the value of the prefix's key ('tag').
+ *
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
+ */
+export default (data) => {
+  // TODO: make these deep rows.
+  const itemLabels = data[0]
+  const fmt = fmtObj(itemLabels)
+  const output = []
+
+  data.forEach((row, idx) => {
+    if (idx === 0) return
+    output.push(fmt(row))
+  })
+
+  return output
+}
diff --git a/src/blueprinters/groups.js b/src/blueprinters/groups.js
index 30ef755..9b08ca8 100644
--- a/src/blueprinters/groups.js
+++ b/src/blueprinters/groups.js
@@ -1,39 +1,17 @@
 import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * groups - generate a Blueprint from a data sheet grouped by a column called 'group'
- * The resource name defaults to 'groups', or a custom resource name can be passed.
  * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * names. Items are inserted into the data list at idx = id.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="groups"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function groups (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'groups'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
-
+  const output = []
   const dataGroups = {}
 
   data.forEach((row, idx) => {
@@ -45,12 +23,14 @@ export default function groups (
       dataGroups[group].push(fmt(row))
     }
   })
-  Object.keys(dataGroups).forEach(groupKey => {
-    bp.resources[label].data.push({
-      group: groupKey,
-      group_label: dataGroups[groupKey][0].group_label,
-      data: dataGroups[groupKey]
+  Object.keys(dataGroups)
+    .forEach(groupKey => {
+      output.push({
+        group: groupKey,
+        group_label: dataGroups[groupKey][0].group_label,
+        data: dataGroups[groupKey]
+      })
     })
-  })
-  return bp
+
+  return output
 }
diff --git a/src/blueprinters/ids.js b/src/blueprinters/ids.js
index 26705b9..9423f8b 100644
--- a/src/blueprinters/ids.js
+++ b/src/blueprinters/ids.js
@@ -1,43 +1,23 @@
 import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * ids - generate a Blueprint from a data sheet by id, which is an integer.
- * The resource name defaults to 'ids', or a custom resource name can be passed.
- * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * Very similar to the rows blueprinter, but inserts each row as a value in
+ * an object, where the value in the 'id' column of the row will be used as
+ * the search key
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="ids"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Object      the structured data.
  */
-export default function ids (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'ids'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = {}
+  const output = {}
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.resources[label].data[fmt(row).id] = fmt(row)
+    output[fmt(row).id] = fmt(row)
   })
 
-  return bp
+  return output
 }
diff --git a/src/blueprinters/rows.js b/src/blueprinters/rows.js
index 6966480..f3cf8a2 100644
--- a/src/blueprinters/rows.js
+++ b/src/blueprinters/rows.js
@@ -1,41 +1,22 @@
 import R from 'ramda'
 import { fmtObj } from '../lib/util'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
 
 /**
- * rows - generate a Blueprint from a data sheet by row. The resource name
- * defaults to 'rows', or a custom resource name can be passed. Each resource
- * item is an object with values labelled according to column names.
+ * Each resource item is an object with values labelled according
+ * to column names specified in the sheet's first row.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="rows" name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function rows (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'rows'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
+export default (data) => {
   const itemLabels = data[0]
   const fmt = fmtObj(itemLabels)
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = []
+  const output = []
 
   data.forEach((row, idx) => {
     if (idx === 0) return
-    bp.resources[label].data.push(fmt(row))
+    output.push(fmt(row))
   })
-  return bp
+
+  return output
 }
diff --git a/src/blueprinters/tree.js b/src/blueprinters/tree.js
index 143d419..5a80cd3 100644
--- a/src/blueprinters/tree.js
+++ b/src/blueprinters/tree.js
@@ -1,36 +1,13 @@
 import R from 'ramda'
-import { defaultBlueprint, defaultResource } from '../lib/blueprinters'
+import { fmtObj } from '../lib/util'
 
 /**
- * tree - generate a Blueprint from a data sheet grouped by a column called 'group'
- * The resource name defaults to 'groups', or a custom resource name can be passed.
- * Each resource item is an object with values labelled according to column
- * names. Items are inserted in the data list at idx = id.
+ * Each resource item is inserted into a tree. TODO: describe layout.
  *
- * @param  {type} data         list of lists representing sheet data.
- * @param  {type} label="groups"  name of resource in blueprint.
- * @param  {type} name=""      name of blueprint.
- * @return {type} Blueprint
+ * @param  {type} data        list of lists representing sheet data.
+ * @return {type} Array       the structured data.
  */
-export default function tree (
-  tabName,
-  sheetName,
-  sheetId,
-  data,
-  label = 'tree'
-) {
-  // Define Blueprint
-  const bp = R.clone(defaultBlueprint)
-  bp.sheet = {
-    name: sheetName,
-    id: sheetId
-  }
-  bp.name = tabName
-
-  // Column names define resources
-  bp.resources[label] = R.clone(defaultResource)
-  bp.resources[label].data = {}
-
+export default (data) => {
   const tree = {
     key: 'tags',
     children: {}
@@ -62,6 +39,5 @@ export default function tree (
     }
   })
 
-  bp.resources[label].data = tree
-  return bp
+  return tree
 }
diff --git a/src/lib/Controller.js b/src/lib/Controller.js
index b9e9351..8433fcd 100644
--- a/src/lib/Controller.js
+++ b/src/lib/Controller.js
@@ -30,6 +30,8 @@ class Controller {
       } else {
         throw new Error(copy.errors.update)
       }
+    }).catch(err => {
+      console.log(err)
     })
   }
 
diff --git a/src/lib/Fetcher.js b/src/lib/Fetcher.js
index 20beca9..f62c79f 100644
--- a/src/lib/Fetcher.js
+++ b/src/lib/Fetcher.js
@@ -91,9 +91,9 @@ class Fetcher {
    */
   _saveViaBlueprinter (tab, data, blueprinter) {
     const saturatedBp = blueprinter(
-      tab,
-      this.sheetName,
       this.sheetId,
+      this.sheetName,
+      tab,
       data
     )
 
diff --git a/src/lib/blueprinters.js b/src/lib/blueprinters.js
index 3dba25d..064922d 100644
--- a/src/lib/blueprinters.js
+++ b/src/lib/blueprinters.js
@@ -24,13 +24,28 @@ export function buildDesaturated (sheetId, sheetName, tab, resource) {
   return bp
 }
 
+const buildBlueprinter = R.curry((datafierName, datafier, sheetId, sheetName, tabName, data) => {
+  console.log(`blueprinter ${datafierName} called: ${tabName}`)
+  const bp = R.clone(defaultBlueprint)
+  bp.sheet = {
+    name: sheetName,
+    id: sheetId
+  }
+  bp.name = tabName
+  bp.resources[datafierName] = R.clone(defaultResource)
+  bp.resources[datafierName].data = datafier(data)
+
+  return bp
+})
+
 // import all default exports from 'blueprinters' folder
 const allBps = {}
 const REL_PATH_TO_BPS = '../blueprinters'
 const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS)
 fs.readdirSync(normalizedPath).forEach(file => {
   const bpName = file.replace('.js', '')
-  allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default
+  const datafier = require(`${REL_PATH_TO_BPS}/${file}`).default
+  allBps[bpName] = buildBlueprinter(bpName, datafier)
 })
 
 // NB: revert to ES5 'module.exports' required to make blueprinters from

From df239c8f58b0073abe44afdf6e0fee93279e917b Mon Sep 17 00:00:00 2001
From: Lachlan Kermode <lachiekermode@gmail.com>
Date: Thu, 13 Dec 2018 15:59:03 +0000
Subject: [PATCH 2/6] rm debugging logic

---
 src/lib/Controller.js   | 2 --
 src/lib/blueprinters.js | 1 -
 2 files changed, 3 deletions(-)

diff --git a/src/lib/Controller.js b/src/lib/Controller.js
index 8433fcd..b9e9351 100644
--- a/src/lib/Controller.js
+++ b/src/lib/Controller.js
@@ -30,8 +30,6 @@ class Controller {
       } else {
         throw new Error(copy.errors.update)
       }
-    }).catch(err => {
-      console.log(err)
     })
   }
 
diff --git a/src/lib/blueprinters.js b/src/lib/blueprinters.js
index 064922d..6193972 100644
--- a/src/lib/blueprinters.js
+++ b/src/lib/blueprinters.js
@@ -25,7 +25,6 @@ export function buildDesaturated (sheetId, sheetName, tab, resource) {
 }
 
 const buildBlueprinter = R.curry((datafierName, datafier, sheetId, sheetName, tabName, data) => {
-  console.log(`blueprinter ${datafierName} called: ${tabName}`)
   const bp = R.clone(defaultBlueprint)
   bp.sheet = {
     name: sheetName,

From bbee0c28960fb7e288710149bda588533549da33 Mon Sep 17 00:00:00 2001
From: Lachlan Kermode <lachiekermode@gmail.com>
Date: Thu, 13 Dec 2018 16:44:56 +0000
Subject: [PATCH 3/6] functional deeprows blueprinter

---
 src/blueprinters/deeprows.js | 52 +++++++++++++++++++++++++++++++++---
 1 file changed, 49 insertions(+), 3 deletions(-)

diff --git a/src/blueprinters/deeprows.js b/src/blueprinters/deeprows.js
index 7e00e69..a27c7f2 100644
--- a/src/blueprinters/deeprows.js
+++ b/src/blueprinters/deeprows.js
@@ -7,19 +7,65 @@ import { fmtObj } from '../lib/util'
  * column names are the same except for a different integer at the end
  * (e.g. 'tag1', and 'tag2'), then the values of those two columns are
  * aggregated into a list, which is the value of the prefix's key ('tag').
+ * Any values in those columns that are empty will NOT be added to the list.
  *
  * @param  {type} data        list of lists representing sheet data.
  * @return {type} Array       the structured data.
  */
 export default (data) => {
-  // TODO: make these deep rows.
   const itemLabels = data[0]
-  const fmt = fmtObj(itemLabels)
+  const baseFmt = fmtObj(itemLabels)
   const output = []
 
+  // create a structure to indicate which columns needs to be aggregated
+  const endsWithNumber = new RegExp('(.*)[0-9]+$')
+  const structure = {
+    __flat: []
+  }
+
+  itemLabels.forEach(label => {
+    const matches = label.match(endsWithNumber)
+    if (!matches) {
+      structure.__flat.push(label)
+    } else {
+      const labelPrefix = `${matches[1]}s`
+      if (labelPrefix in structure) {
+        structure[labelPrefix].push(label)
+      } else {
+        structure[labelPrefix] = [ label ]
+      }
+    }
+  })
+
+  // generate the value for deep labels using the structure created
   data.forEach((row, idx) => {
     if (idx === 0) return
-    output.push(fmt(row))
+    const baseRow = baseFmt(row)
+    const deepRow = {}
+
+    // generate deep row labels using structure
+    Object.keys(structure)
+      .forEach(newLabel => {
+        if (newLabel != '__flat') {
+          const oldLabels = structure[newLabel]
+          // only add new value if not ''
+          const labelValues = []
+          oldLabels.forEach(l => {
+            const vl = baseRow[l]
+            if (vl !== '') {
+              labelValues.push(vl)
+            }
+          })
+          deepRow[newLabel] = labelValues
+        }
+      })
+
+    // move values for flat labels over from base
+    structure.__flat.forEach(label => {
+      deepRow[label] = baseRow[label]
+    })
+
+    output.push(deepRow)
   })
 
   return output

From b37e49880aeaf1bff9ed5bc1104d6c279adef392 Mon Sep 17 00:00:00 2001
From: Lachlan Kermode <lachiekermode@gmail.com>
Date: Thu, 13 Dec 2018 16:49:23 +0000
Subject: [PATCH 4/6] fix lint

---
 src/blueprinters/deeprows.js | 3 +--
 src/blueprinters/groups.js   | 1 -
 src/blueprinters/ids.js      | 1 -
 src/blueprinters/rows.js     | 1 -
 src/blueprinters/tree.js     | 3 ---
 5 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/blueprinters/deeprows.js b/src/blueprinters/deeprows.js
index a27c7f2..036d4af 100644
--- a/src/blueprinters/deeprows.js
+++ b/src/blueprinters/deeprows.js
@@ -1,4 +1,3 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
 
 /**
@@ -46,7 +45,7 @@ export default (data) => {
     // generate deep row labels using structure
     Object.keys(structure)
       .forEach(newLabel => {
-        if (newLabel != '__flat') {
+        if (newLabel !== '__flat') {
           const oldLabels = structure[newLabel]
           // only add new value if not ''
           const labelValues = []
diff --git a/src/blueprinters/groups.js b/src/blueprinters/groups.js
index 9b08ca8..49ab684 100644
--- a/src/blueprinters/groups.js
+++ b/src/blueprinters/groups.js
@@ -1,4 +1,3 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
 
 /**
diff --git a/src/blueprinters/ids.js b/src/blueprinters/ids.js
index 9423f8b..6af7840 100644
--- a/src/blueprinters/ids.js
+++ b/src/blueprinters/ids.js
@@ -1,4 +1,3 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
 
 /**
diff --git a/src/blueprinters/rows.js b/src/blueprinters/rows.js
index f3cf8a2..25e9999 100644
--- a/src/blueprinters/rows.js
+++ b/src/blueprinters/rows.js
@@ -1,4 +1,3 @@
-import R from 'ramda'
 import { fmtObj } from '../lib/util'
 
 /**
diff --git a/src/blueprinters/tree.js b/src/blueprinters/tree.js
index 5a80cd3..2e4dd21 100644
--- a/src/blueprinters/tree.js
+++ b/src/blueprinters/tree.js
@@ -1,6 +1,3 @@
-import R from 'ramda'
-import { fmtObj } from '../lib/util'
-
 /**
  * Each resource item is inserted into a tree. TODO: describe layout.
  *

From f47fc311c1c758e0b207cee6f235ea1e50ba500b Mon Sep 17 00:00:00 2001
From: Lachlan Kermode <lachiekermode@gmail.com>
Date: Thu, 13 Dec 2018 17:10:14 +0000
Subject: [PATCH 5/6] update tests

---
 test/internals.js | 50 ++++++++++++++---------------------------------
 1 file changed, 15 insertions(+), 35 deletions(-)

diff --git a/test/internals.js b/test/internals.js
index 49c6ba4..99274a1 100644
--- a/test/internals.js
+++ b/test/internals.js
@@ -3,10 +3,11 @@ import R from 'ramda'
 import {
   defaultBlueprint,
   defaultResource,
-  columns,
-  rows
 } from '../src/lib/blueprinters'
 
+import rows from '../src/blueprinters/rows'
+import deeprows from '../src/blueprinters/deeprows'
+
 const egInput1 = [
   ['h1', 'h2', 'h3'],
   [1, 2, 3],
@@ -25,41 +26,20 @@ test('defaultBlueprint exports', t => {
   t.deepEqual(expected, defaultBlueprint)
 })
 
-test('columns blueprinter generates expected output', t => {
-  const actual = columns('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1)
-  const expected = R.clone(defaultBlueprint)
-  expected.name = 'eg ColumnBlueprint'
-  expected.sheet = {
-    id: 'egSheetId',
-    name: 'egSheetName'
-  }
-  expected.resources['h1'] = R.clone(defaultResource)
-  expected.resources['h1'].data = [1, 4]
-  expected.resources['h2'] = R.clone(defaultResource)
-  expected.resources['h2'].data = [2, 5]
-  expected.resources['h3'] = R.clone(defaultResource)
-  expected.resources['h3'].data = [3, 6]
+test('rows blueprinter', t => {
+  const expected = [
+    { h1: 1, h2: 2, h3: 3 },
+    { h1: 4, h2: 5, h3: 6 },
+  ]
+  const actual = rows(egInput1)
   t.deepEqual(expected, actual)
 })
 
-test('rows blueprinter generates expected output', t => {
-  const actual = rows('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items')
-  const expected = R.clone(defaultBlueprint)
-  expected.name = 'egRowBlueprint'
-  expected.sheet = {
-    id: 'egSheetId',
-    name: 'egSheetName'
-  }
-  expected.resources['items'] = R.clone(defaultResource)
-  expected.resources['items'].data = [{
-    h1: 1,
-    h2: 2,
-    h3: 3
-  },
-  {
-    h1: 4,
-    h2: 5,
-    h3: 6
-  }]
+test('deeprows blueprinter', t => {
+  const expected = [
+    { 'hs': [1,2,3] },
+    { 'hs': [4,5,6] }
+  ]
+  const actual = deeprows(egInput1)
   t.deepEqual(expected, actual)
 })

From 5d8a6b19276ac959e40c6f51b50a9c5e2dab6395 Mon Sep 17 00:00:00 2001
From: Lachlan Kermode <lachiekermode@gmail.com>
Date: Fri, 14 Dec 2018 09:56:18 +0000
Subject: [PATCH 6/6] update example.config.js

---
 src/example.config.js | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/example.config.js b/src/example.config.js
index da4c52c..c4c190c 100644
--- a/src/example.config.js
+++ b/src/example.config.js
@@ -10,10 +10,11 @@ export default {
         name: 'example',
         id: '1UC7DkCFeUXHfpUxUGruExwFbP4pqVBdJLOKfo6wDDGk',
         tabs: {
-          export_events: [BP.byId, BP.byRow],
-          export_categories: [BP.byGroup, BP.byRow],
-          export_sites: BP.byRow,
-          export_tags: BP.byTree
+          export_events: [BP.deeprows, BP.rows],
+          export_categories: [BP.groups, BP.rows],
+          export_sources: BP.ids,
+          export_sites: BP.rows,
+          export_tags: BP.tree
         }
       }
     ]