Merge branch 'develop' of https://www.github.com/forensic-architecture/datasheet-server into feature/json-api-export

package.json

@@ -3,6 +3,7 @@
   "version": "0.3.0",
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
+  "type": "module",
   "scripts": {
     "dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
     "build": "env NODE_ENV=production npx babel src -d dist",
@@ -26,6 +27,7 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
+    "file-system": "^2.2.2",
     "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",

src/api/index.js

@@ -4,6 +4,7 @@ import copy from '../copy/en'
 
 export default ({ config, controller }) => {
   let api = Router()
+  const fileDest = config.EXPORT_FILE_DEST || null
 
   api.get('/', (req, res) => {
     res.json({
@@ -22,6 +23,20 @@ export default ({ config, controller }) => {
     })
   })
 
+  api.get('/export', (req, res) => {
+    controller
+      .retrieveAll(fileDest)
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
+  })
+
   api.get('/update', (req, res) => {
     controller
       .update()
@@ -30,10 +45,10 @@ export default ({ config, controller }) => {
           success: msg
         })
       )
-      .catch(err => {
+      .catch(err =>
         res.status(404)
           .send({ error: err.message, err })
-      })
+      )
   })
 
   api.get('/:sheet/:tab/:resource/:frag', (req, res) => {

src/copy/en.js

@@ -1,6 +1,10 @@
 export default {
   errors: {
     update: 'The server could not update. Check your API credentials and internet connection and try again.',
+    export: {
+      fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
+      writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
+    },
     onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
@@ -9,6 +13,7 @@ export default {
     modelLayer: prts => `Something went wrong at the model layer`
   },
   success: {
-    update: 'All sheets updated'
+    update: 'All sheets updated',
+    export: dest => `All resources exported to the file: ${dest}`
   }
 }

src/lib/Controller.js

@@ -1,7 +1,8 @@
 import copy from '../copy/en'
+import { exportToFile } from '../lib/util'
 
 /**
- * Controller class
+ * Controller
  *
  */
 class Controller {
@@ -39,6 +40,37 @@ class Controller {
     })
   }
 
+  // Controller function to retrieve all blueprints and export to user defined file location
+  retrieveAll (fileDest) {
+    if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
+
+    const indexedData = {}
+    const urls = []
+
+    const bps = this.blueprints()
+    return Promise.all(
+      bps.map(bp => {
+        const resource = Object.keys(bp.resources)[0]
+        urls.push(bp.urls[0])
+        return this.retrieve(bp.sheet.name, bp.name, resource)
+      })
+    ).then(async results => {
+      if (results.every(res => res)) {
+        urls.forEach((item, idx) => {
+          indexedData[item] = results[idx]
+        })
+        try {
+          const message = await exportToFile(fileDest, indexedData)
+          return message
+        } catch (e) {
+          return Promise.reject(e)
+        }
+      } else {
+        throw new Error(copy.errors.export.writeFailed)
+      }
+    })
+  }
+
   retrieve (sheet, tab, resource) {
     if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]

src/lib/Fetcher.js

@@ -217,10 +217,7 @@ class GsheetFetcher extends Fetcher {
       })
       .then(this._buildBlueprintsAsync())
       .then(() => true)
-      .catch((err) => {
-        console.log(`Error fetching gsheets: ${err.message} `)
-        return false
-      })
+      .catch(() => false)
   }
 }
 

src/lib/blueprinters.js

@@ -2,7 +2,7 @@ import path from 'path'
 import fs from 'fs'
 import R from 'ramda'
 
-const defaultBlueprint = {
+export const defaultBlueprint = {
   name: null,
   sheet: {
     name: null,
@@ -11,11 +11,11 @@ const defaultBlueprint = {
   resources: {}
 }
 
-const defaultResource = {
+export const defaultResource = {
   data: []
 }
 
-function buildDesaturated (sheetId, sheetName, tab, resources) {
+export function buildDesaturated (sheetId, sheetName, tab, resources) {
   const bp = R.clone(defaultBlueprint)
   bp.sheet.name = sheetName
   bp.sheet.id = sheetId
@@ -50,25 +50,10 @@ fs.readdirSync(normalizedPath).forEach(file => {
   allBps[bpName] = buildBlueprinter(bpName, datafier)
 })
 
-function deeprowsWithSchema (datafierName, schema) {
-  const datafier = data => {
-    const transformedData = allBps.deeprows('', '', '', data).resources.deeprows.data
-    return transformedData.map(row => {
-      Object.keys(schema).forEach(key => {
-        row[key] = schema[key](row[key])
-      })
-      return row
-    })
-  }
-
-  return buildBlueprinter(`deeprows_${datafierName}`, datafier)
-}
-
 // NB: revert to ES5 'module.exports' required to make blueprinters from
 // each file in blueprinters folder available for granular import from here.
 module.exports = Object.assign({
   defaultBlueprint,
   defaultResource,
-  buildDesaturated,
-  deeprowsWithSchema
+  buildDesaturated
 }, allBps)

src/lib/util.js

@@ -1,4 +1,6 @@
 import R from 'ramda'
+import { promises as fs } from 'file-system'
+import copy from '../copy/en'
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {
@@ -14,6 +16,18 @@ function camelize (str) {
   })
 }
 
+export async function exportToFile (fileDest, data) {
+  const stringifiedData = JSON.stringify(data, null, 2)
+  const filePath = `${fileDest}/export.json`
+
+  try {
+    await fs.writeFile(filePath, stringifiedData)
+    return copy.success.export(filePath)
+  } catch (err) {
+    throw new Error(copy.errors.export.writeFailed)
+  }
+}
+
 export const fmtObj = R.curry(
   (
     columnNames,

src/models/StoreJson.js

@@ -19,6 +19,7 @@ class StoreJson {
 
   save (url, data) {
     const parts = url.split('/')
+
     return fs.writeFile(
       `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`,
       JSON.stringify(data)

test/internals.js

@@ -12,8 +12,6 @@ const egInput1 = [
   [4, 5, 6]
 ]
 
-// Test default blueprint exports
-// Smoke tests
 test('defaultBlueprint exports', t => {
   const expected = {
     sheet: {