Merge branch 'develop' of https://www.github.com/forensic-architecture/datasheet-server into feature/json-api-export

.gitignore

@@ -15,4 +15,3 @@ tags.lock
 tags.temp
 src/config.js
 src/local.config.js
-src/config.js

Dockerfile

@@ -4,12 +4,12 @@ LABEL authors="Lachlan Kermode <lk@forensic-architecture.org>"
 
 # Install app dependencies
 COPY package.json /www/package.json
-RUN cd /www; npm install
+RUN cd /www; yarn
 
 # Copy app source
 COPY . /www
 WORKDIR /www
-RUN npm run build
+RUN yarn build
 RUN mkdir -p data
 
 # set your port
@@ -17,4 +17,4 @@ ENV PORT 4040
 EXPOSE 4040
 
 # start command as per package.json
-CMD ["npm", "start"]
+CMD ["yarn", "start"]

README.md

@@ -59,11 +59,6 @@ Clone the repository to your local:
 git clone https://www.github.com/forensic-architecture/datasheet-server
 ```
 
-Copy [src/example.config.js](src/example.config.js) into `src/config.js` and modify
-```
-cp src/example.config.js src/config.js
-```
-
 Follow the steps in the [configuration](#configuration) section of this
 document.
 

package.json

@@ -3,6 +3,7 @@
   "version": "0.3.0",
   "description": "Starter project for an ES6 RESTful Express API",
   "main": "dist",
+  "type": "module",
   "scripts": {
     "dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
     "build": "env NODE_ENV=production npx babel src -d dist",
@@ -26,6 +27,7 @@
     "express": "^4.13.3",
     "express-graphql": "^0.6.12",
     "express-handlebars": "^4.0.4",
+    "file-system": "^2.2.2",
     "googleapis": "^39.1.0",
     "graphql": "^0.13.2",
     "morgan": "^1.8.0",

src/api/index.js

@@ -4,6 +4,7 @@ import copy from '../copy/en'
 
 export default ({ config, controller }) => {
   let api = Router()
+  const fileDest = config.EXPORT_FILE_DEST || null
 
   api.get('/', (req, res) => {
     res.json({
@@ -22,6 +23,20 @@ export default ({ config, controller }) => {
     })
   })
 
+  api.get('/export', (req, res) => {
+    controller
+      .retrieveAll(fileDest)
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.status(404)
+          .send({ error: err.message, err })
+      )
+  })
+
   api.get('/update', (req, res) => {
     controller
       .update()
@@ -30,10 +45,10 @@ export default ({ config, controller }) => {
           success: msg
         })
       )
-      .catch(err => {
+      .catch(err =>
         res.status(404)
           .send({ error: err.message, err })
-      })
+      )
   })
 
   api.get('/:sheet/:tab/:resource/:frag', (req, res) => {

src/config.js

@@ -0,0 +1,12 @@
+import { timemap } from './lib'
+
+export default {
+  gsheets: [],
+  xlsx: [
+    {
+      name: 'timemap_data',
+      path: 'data/timemap_data.xlsx',
+      tabs: timemap.default
+    }
+  ]
+}

src/copy/en.js

@@ -1,6 +1,10 @@
 export default {
   errors: {
     update: 'The server could not update. Check your API credentials and internet connection and try again.',
+    export: {
+      fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
+      writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
+    },
     onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
     noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
@@ -9,6 +13,7 @@ export default {
     modelLayer: prts => `Something went wrong at the model layer`
   },
   success: {
-    update: 'All sheets updated'
+    update: 'All sheets updated',
+    export: dest => `All resources exported to the file: ${dest}`
   }
 }

src/example.config.js

@@ -1,19 +0,0 @@
-import { timemap } from './lib'
-
-export default {
-  gsheets: [],
-  xlsx: [
-    {
-      name: 'timemap_data',
-      path: 'data/timemap_data.xlsx',
-      tabs: timemap.default
-    }
-  ],
-  cors: {
-    active: false,
-    corsOptions: { // meaningless unless active=true
-      origin: 'http://example.com',
-      optionsSuccessStatus: 200 // some legacy browsers (IE11, various SmartTVs) choke on 204
-    }
-  }
-}

src/index.js

@@ -7,14 +7,6 @@ import api from './api'
 import dotenv from 'dotenv'
 const hbs = require('express-handlebars')
 
-
-let configJS
-try {
-  configJS = require('./local.config.js').default
-} catch (_) {
-  configJS = require('./config.js').default
-}
-
 dotenv.config()
 
 let app = express()
@@ -25,17 +17,13 @@ app.engine('.hbs', hbs({
 }))
 app.set('view engine', '.hbs')
 
-// enable cross origin requests explicitly in development OR if active in prod
-const cors = require('cors')
+// enable cross origin requests explicitly in development
 if (process.env.NODE_ENV === 'development') {
+  const cors = require('cors')
   console.log('Enabling CORS in development...')
   app.use(cors())
-} else if (configJS.cors.active === true) {
-  console.log('Enabling CORS in from config.js in prod...')
-  app.use(cors(configJS.cors.corsOptions))
 }
 
-
 const config = process.env
 
 initialize(controller => {

src/lib.js

@@ -7,7 +7,6 @@ function prefixedTabs (prefix, cfg) {
     [`${prf('events')}export_events`]: BP.deeprows,
     [`${prf('associations')}export_associations`]: BP.deeprows,
     [`${prf('sources')}export_sources`]: BP.deepids,
-    [`${prf('shapes')}export_shapes`]: BP.deeprows,
     [`${prf('sites')}export_sites`]: BP.rows
   }
 }

src/lib/Controller.js

@@ -1,7 +1,8 @@
 import copy from '../copy/en'
+import { exportToFile } from '../lib/util'
 
 /**
- * Controller class
+ * Controller
  *
  */
 class Controller {
@@ -39,6 +40,37 @@ class Controller {
     })
   }
 
+  // Controller function to retrieve all blueprints and export to user defined file location
+  retrieveAll (fileDest) {
+    if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
+
+    const indexedData = {}
+    const urls = []
+
+    const bps = this.blueprints()
+    return Promise.all(
+      bps.map(bp => {
+        const resource = Object.keys(bp.resources)[0]
+        urls.push(bp.urls[0])
+        return this.retrieve(bp.sheet.name, bp.name, resource)
+      })
+    ).then(async results => {
+      if (results.every(res => res)) {
+        urls.forEach((item, idx) => {
+          indexedData[item] = results[idx]
+        })
+        try {
+          const message = await exportToFile(fileDest, indexedData)
+          return message
+        } catch (e) {
+          return Promise.reject(e)
+        }
+      } else {
+        throw new Error(copy.errors.export.writeFailed)
+      }
+    })
+  }
+
   retrieve (sheet, tab, resource) {
     if (this._sheetExists(sheet)) {
       const fetcher = this.fetchers[sheet]

src/lib/Fetcher.js

@@ -217,10 +217,7 @@ class GsheetFetcher extends Fetcher {
       })
       .then(this._buildBlueprintsAsync())
       .then(() => true)
-      .catch((err) => {
-        console.log(`Error fetching gsheets: ${err.message} `)
-        return false
-      })
+      .catch(() => false)
   }
 }
 

src/lib/util.js

@@ -1,4 +1,6 @@
 import R from 'ramda'
+import { promises as fs } from 'file-system'
+import copy from '../copy/en'
 
 /* eslint-disable */
 String.prototype.replaceAll = function (search, replacement) {
@@ -14,6 +16,18 @@ function camelize (str) {
   })
 }
 
+export async function exportToFile (fileDest, data) {
+  const stringifiedData = JSON.stringify(data, null, 2)
+  const filePath = `${fileDest}/export.json`
+
+  try {
+    await fs.writeFile(filePath, stringifiedData)
+    return copy.success.export(filePath)
+  } catch (err) {
+    throw new Error(copy.errors.export.writeFailed)
+  }
+}
+
 export const fmtObj = R.curry(
   (
     columnNames,

src/models/StoreJson.js

@@ -19,6 +19,7 @@ class StoreJson {
 
   save (url, data) {
     const parts = url.split('/')
+
     return fs.writeFile(
       `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`,
       JSON.stringify(data)

test/internals.js

@@ -12,8 +12,6 @@ const egInput1 = [
   [4, 5, 6]
 ]
 
-// Test default blueprint exports
-// Smoke tests
 test('defaultBlueprint exports', t => {
   const expected = {
     sheet: {

test/serverProcess.js

@@ -3,7 +3,7 @@ import fetch from 'node-fetch'
 import childProcess from 'child_process'
 
 const SERVER_LAUNCH_WAIT_TIME = 10 * 1000
-const SERVER_ROOT = ''
+const SERVER_ROOT = 'http://localhost:4040'
 let serverProc = null
 let serverExited = false
 function checkStatus (res) {

views/blueprints.hbs

@@ -8,7 +8,7 @@
       <div class="bp-source">{{ source }}</div>
     </div>
     {{#each urls}}
-    <div><a target="_blank" href="{{ this }}">{{ this }}</a></div>
+    <div><a target="_blank" href="http://localhost:4040{{ this }}">{{ this }}</a></div>
     {{/each}}
   </div>
   {{ else }}
@@ -16,7 +16,7 @@
   {{/each}}
 </div>
 
-<a class="bp-update-container" target="_blank" href="/api/update">
+<a class="bp-update-container" target="_blank" href="http://localhost:4040/api/update">
   <div class="bp-button">Update</div>
 </div>