diff --git a/src/blueprinters/byColumn.js b/src/blueprinters/columns.js similarity index 69% rename from src/blueprinters/byColumn.js rename to src/blueprinters/columns.js index 68abb0e..eee7bc8 100644 --- a/src/blueprinters/byColumn.js +++ b/src/blueprinters/columns.js @@ -1,5 +1,5 @@ import R from 'ramda' -import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' +import { defaultBlueprint, defaultResource } from '../lib/blueprinters' /** * byColumn - generate a Blueprint from a data sheet by column. Each column @@ -9,7 +9,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' * @return {type} Blueprint * generated. */ -export default function byColumn (tabName, sheetName, sheetId, data) { +function columns (tabName, sheetName, sheetId, data) { // Define Blueprint props const bp = R.clone(defaultBlueprint) bp.sheet = { @@ -18,18 +18,20 @@ export default function byColumn (tabName, sheetName, sheetId, data) { } bp.name = tabName - // column names define routes + // column names define resources const labels = data[0] labels.forEach(label => { - bp.routes[label] = R.clone(defaultRoute) + bp.resources[label] = R.clone(defaultResource) }) // remaining rows as data data.forEach((row, idx) => { if (idx === 0) return labels.forEach((label, idx) => { - bp.routes[label].data.push(row[idx]) + bp.resources[label].data.push(row[idx]) }) }) return bp } + +export default columns diff --git a/src/blueprinters/byGroup.js b/src/blueprinters/groups.js similarity index 77% rename from src/blueprinters/byGroup.js rename to src/blueprinters/groups.js index 0dac039..30ef755 100644 --- a/src/blueprinters/byGroup.js +++ b/src/blueprinters/groups.js @@ -1,9 +1,9 @@ import R from 'ramda' import { fmtObj } from '../lib/util' -import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' +import { defaultBlueprint, defaultResource } from '../lib/blueprinters' /** - * byGroup - generate a Blueprint from a data sheet grouped by a column called 'group' + * groups - generate a Blueprint from a data sheet grouped by a column called 'group' * The resource name defaults to 'groups', or a custom resource name can be passed. * Each resource item is an object with values labelled according to column * names. Items are inserted in the data list at idx = id. @@ -13,7 +13,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' * @param {type} name="" name of blueprint. * @return {type} Blueprint */ -export default function byGroup ( +export default function groups ( tabName, sheetName, sheetId, @@ -28,11 +28,11 @@ export default function byGroup ( } bp.name = tabName - // Column names define routes + // Column names define resources const itemLabels = data[0] const fmt = fmtObj(itemLabels) - bp.routes[label] = R.clone(defaultRoute) - bp.routes[label].data = [] + bp.resources[label] = R.clone(defaultResource) + bp.resources[label].data = [] const dataGroups = {} @@ -46,7 +46,7 @@ export default function byGroup ( } }) Object.keys(dataGroups).forEach(groupKey => { - bp.routes[label].data.push({ + bp.resources[label].data.push({ group: groupKey, group_label: dataGroups[groupKey][0].group_label, data: dataGroups[groupKey] diff --git a/src/blueprinters/byId.js b/src/blueprinters/ids.js similarity index 70% rename from src/blueprinters/byId.js rename to src/blueprinters/ids.js index 0f40a31..d027be1 100644 --- a/src/blueprinters/byId.js +++ b/src/blueprinters/ids.js @@ -1,9 +1,9 @@ import R from 'ramda' import { fmtObj } from '../lib/util' -import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' +import { defaultBlueprint, defaultResource } from '../lib/blueprinters' /** - * byId - generate a Blueprint from a data sheet by id, which is an integer. + * ids - generate a Blueprint from a data sheet by id, which is an integer. * The resource name defaults to 'ids', or a custom resource name can be passed. * Each resource item is an object with values labelled according to column * names. Items are inserted in the data list at idx = id. @@ -13,7 +13,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' * @param {type} name="" name of blueprint. * @return {type} Blueprint */ -export default function byId ( +export default function ids ( tabName, sheetName, sheetId, @@ -28,15 +28,15 @@ export default function byId ( } bp.name = tabName - // Column names define routes + // Column names define resources const itemLabels = data[0] const fmt = fmtObj(itemLabels) - bp.routes[label] = R.clone(defaultRoute) - bp.routes[label].data = [] + bp.resources[label] = R.clone(defaultResource) + bp.resources[label].data = [] data.forEach((row, idx) => { if (idx === 0) return - bp.routes[label].data[fmt(row).id] = fmt(row) + bp.resources[label].data[fmt(row).id] = fmt(row) }) return bp } diff --git a/src/blueprinters/byRow.js b/src/blueprinters/rows.js similarity index 69% rename from src/blueprinters/byRow.js rename to src/blueprinters/rows.js index 637e397..6966480 100644 --- a/src/blueprinters/byRow.js +++ b/src/blueprinters/rows.js @@ -1,9 +1,9 @@ import R from 'ramda' import { fmtObj } from '../lib/util' -import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' +import { defaultBlueprint, defaultResource } from '../lib/blueprinters' /** - * byRow - generate a Blueprint from a data sheet by row. The resource name + * rows - generate a Blueprint from a data sheet by row. The resource name * defaults to 'rows', or a custom resource name can be passed. Each resource * item is an object with values labelled according to column names. * @@ -12,7 +12,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' * @param {type} name="" name of blueprint. * @return {type} Blueprint */ -export default function byRow ( +export default function rows ( tabName, sheetName, sheetId, @@ -27,15 +27,15 @@ export default function byRow ( } bp.name = tabName - // Column names define routes + // Column names define resources const itemLabels = data[0] const fmt = fmtObj(itemLabels) - bp.routes[label] = R.clone(defaultRoute) - bp.routes[label].data = [] + bp.resources[label] = R.clone(defaultResource) + bp.resources[label].data = [] data.forEach((row, idx) => { if (idx === 0) return - bp.routes[label].data.push(fmt(row)) + bp.resources[label].data.push(fmt(row)) }) return bp } diff --git a/src/blueprinters/byTree.js b/src/blueprinters/tree.js similarity index 78% rename from src/blueprinters/byTree.js rename to src/blueprinters/tree.js index e483d3d..143d419 100644 --- a/src/blueprinters/byTree.js +++ b/src/blueprinters/tree.js @@ -1,8 +1,8 @@ import R from 'ramda' -import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' +import { defaultBlueprint, defaultResource } from '../lib/blueprinters' /** - * byTree - generate a Blueprint from a data sheet grouped by a column called 'group' + * tree - generate a Blueprint from a data sheet grouped by a column called 'group' * The resource name defaults to 'groups', or a custom resource name can be passed. * Each resource item is an object with values labelled according to column * names. Items are inserted in the data list at idx = id. @@ -12,7 +12,7 @@ import { defaultBlueprint, defaultRoute } from '../lib/blueprinters' * @param {type} name="" name of blueprint. * @return {type} Blueprint */ -export default function byTree ( +export default function tree ( tabName, sheetName, sheetId, @@ -27,9 +27,9 @@ export default function byTree ( } bp.name = tabName - // Column names define routes - bp.routes[label] = R.clone(defaultRoute) - bp.routes[label].data = {} + // Column names define resources + bp.resources[label] = R.clone(defaultResource) + bp.resources[label].data = {} const tree = { key: 'tags', @@ -62,6 +62,6 @@ export default function byTree ( } }) - bp.routes[label].data = tree + bp.resources[label].data = tree return bp } diff --git a/src/copy/en.js b/src/copy/en.js index 24a9fbc..ef50d69 100644 --- a/src/copy/en.js +++ b/src/copy/en.js @@ -4,7 +4,7 @@ export default { onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.', onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.', noSheet: sheet => `The sheet ${sheet} is not available in this server.`, - noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' of the sheet '${prts[0]}'.`, + noResource: prts => `The resource '${prts[2]}' does not exists in the tab '${prts[1]}' in this sheet.`, noFragment: prts => `Fragment index does not exist` }, success: { diff --git a/src/lib/Controller.js b/src/lib/Controller.js index de37f97..b9e9351 100644 --- a/src/lib/Controller.js +++ b/src/lib/Controller.js @@ -9,7 +9,7 @@ class Controller { this.fetchers = fetchers } - sheetExists (sheet) { + _sheetExists (sheet) { return (Object.keys(this.fetchers).indexOf(sheet) >= 0) } @@ -34,7 +34,7 @@ class Controller { } retrieve (sheet, tab, resource) { - if (this.sheetExists(sheet)) { + if (this._sheetExists(sheet)) { const fetcher = this.fetchers[sheet] return fetcher.retrieve(tab, resource) } else { @@ -43,7 +43,7 @@ class Controller { } retrieveFrag (sheet, tab, resource, frag) { - if (this.sheetExists(sheet)) { + if (this._sheetExists(sheet)) { const fetcher = this.fetchers[sheet] return fetcher.retrieveFrag(tab, resource, frag) } else { diff --git a/src/lib/Fetcher.js b/src/lib/Fetcher.js index 89477b4..3103145 100644 --- a/src/lib/Fetcher.js +++ b/src/lib/Fetcher.js @@ -1,12 +1,12 @@ // FetcherTwo class interfaces with Google Sheet, and saves to a specified db import { google } from 'googleapis' +import { buildDesaturated } from './blueprinters' import { - fmtSheetTitle, + fmtName, fmtBlueprinterTitles, - bp, isFunction } from './util' -import { byRow } from './blueprinters' +import { createHash } from 'crypto' import R from 'ramda' class Fetcher { @@ -28,39 +28,80 @@ class Fetcher { */ this.sheetName = sheetName + /* + * A unique ID for the Fetcher to identify its elements in the model layer + */ + this.id = createHash('md5').update(sheetName).update(sheetId).digest('hex') + /* * These are the available tabs for storing and retrieving data. * Each blueprinter is a function that returns a Blueprint from a * list of lists (which will be retrieved from gsheets). */ this.blueprinters = fmtBlueprinterTitles(blueprinters) - this.blueprints = {} - Object.keys(this.blueprinters).forEach(key => { - this.blueprints[key] = null - }) + + /* + * This object is the canonical represenation for the data that a Fetcher + * proxies. When the fetcher is initialized, its model layer (db) is indexed, + * and this object populated accordingly. Whenever the fetcher updates, this + * data structure updates as well. It is the model layer that determines the + * performance of indexing the blueprints. + */ + this.blueprints = null + this._buildBlueprintsAsync() // NB: modifies this.blueprints on completion /* * Google API setup */ - this.sheets = google.sheets('v4') + this.API = google.sheets('v4') this.auth = null - /** - * saveBp is a curried function that takes in a title and - * a blueprinter. NB: it sits here in the constructor as - * I am not sure how to curry a class method with Ramda. - */ - this._saveBp = R.curry((tab, title, data, blueprinter) => { - const saturatedBp = blueprinter( - tab, - this.sheetName, - this.sheetId, - data + /** curry to allow convenient syntax with map */ + this._saveViaBlueprinter = R.curry(this._saveViaBlueprinter) + } + + _buildBlueprintsAsync () { + return this.db.index() + .then(allUrls => { + const allParts = allUrls.reduce((acc, url) => { + if (url.startsWith(this.id)) { + const parts = url.split('/') + acc.push([ parts[1], parts[2] ]) + return acc + } else { + return acc + } + }, []) + + return allParts + .map(parts => buildDesaturated( + this.sheetId, + this.sheetName, + parts[0], + parts[1] + )) + }) + .then(res => { + this.blueprints = res + }) + } + + /** save data under a given tab name via its blueprinter, which generates + * its resource name. Note that this is curried in the constructor. + */ + _saveViaBlueprinter (tab, data, blueprinter) { + const saturatedBp = blueprinter( + tab, + this.sheetName, + this.sheetId, + data + ) + + return Promise.all( + Object.keys(saturatedBp.resources).map(route => + this.db.save(`${this.id}/${tab}/${route}`, saturatedBp.resources[route].data) ) - const blueprint = bp(saturatedBp) // TODO: come up with better semantics. - this.blueprints[title] = blueprint - return this.db.save(saturatedBp) - }) + ) } /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */ @@ -84,15 +125,15 @@ class Fetcher { update () { let tabTitles - /* Retrieve all available routes on a given sheet, and store formatted copies of it where a formatter is available */ - return this.sheets.spreadsheets + /* Retrieve all available resources on a given sheet, and store formatted copies of it where a formatter is available */ + return this.API.spreadsheets .get({ auth: this.auth, spreadsheetId: this.sheetId }) .then(response => { tabTitles = response.data.sheets.map(sheet => sheet.properties.title) - return this.sheets.spreadsheets.values.batchGet({ + return this.API.spreadsheets.values.batchGet({ auth: this.auth, spreadsheetId: this.sheetId, ranges: tabTitles @@ -100,12 +141,15 @@ class Fetcher { }) .then(results => { const tabData = results.data.valueRanges + return Promise.all( tabData.map((tab, idx) => { const { values } = tab + if (values === undefined) { return Promise.resolve({}) } + const name = tabTitles[idx] return this.save(name, values) }) @@ -115,31 +159,35 @@ class Fetcher { .catch(() => false) } - save (tab, data) { - const title = fmtSheetTitle(tab) - if (Object.keys(this.blueprinters).indexOf(title) > -1) { - const bpConfig = this.blueprinters[title] + save (_tab, data) { + const tab = fmtName(_tab) + + if (Object.keys(this.blueprinters).indexOf(tab) > -1) { + const bpConfig = this.blueprinters[tab] if (isFunction(bpConfig)) { - return this._saveBp(tab, title, data, bpConfig) + // if bpConfig specifies a single blueprinter + return this._saveViaBlueprinter(tab, data, bpConfig) } else { - return bpConfig.map(this._saveBp(tab, title, data)) + // if bpConfig specifies an array of blueprinters + return bpConfig.map(this._saveViaBlueprinter(tab, data)) } } else { - // If it can't find a blueprinter for the tab title, default to byRow - return this.db.save(byRow(tab, this.sheetName, this.sheetId, data)) + // NB: if a blueprinter is not specified for a tab, + // just skip it. + return true } } // NB: could combine these functions by checking kwargs length retrieve (tab, resource) { - const title = fmtSheetTitle(tab) - const url = `${this.sheetName}/${tab}/${resource}` + const title = fmtName(tab) + const url = `${this.id}/${tab}/${resource}` return this.db.load(url, this.blueprints[title]) } retrieveFrag (tab, resource, frag) { - const title = fmtSheetTitle(tab) + const title = fmtName(tab) const url = `${this.sheetName}/${tab}/${resource}/${frag}` return this.db.load(url, this.blueprints[title]) } diff --git a/src/lib/blueprinters.js b/src/lib/blueprinters.js index 6bb4c64..3dba25d 100644 --- a/src/lib/blueprinters.js +++ b/src/lib/blueprinters.js @@ -1,20 +1,29 @@ import path from 'path' import fs from 'fs' +import R from 'ramda' export const defaultBlueprint = { name: null, - id: null, - dialects: ['rest'], // supported dialects, can (eventually) be multiple - routes: {} + sheet: { + name: null, + id: null + }, + resources: {} } -export const defaultRoute = { - options: { - fragment: true - }, +export const defaultResource = { data: [] } +export function buildDesaturated (sheetId, sheetName, tab, resource) { + const bp = R.clone(defaultBlueprint) + bp.sheet.name = sheetName + bp.sheet.id = sheetId + bp.name = tab + bp.resources[resource] = null + return bp +} + // import all default exports from 'blueprinters' folder const allBps = {} const REL_PATH_TO_BPS = '../blueprinters' @@ -28,5 +37,6 @@ fs.readdirSync(normalizedPath).forEach(file => { // each file in blueprinters folder available for granular import from here. module.exports = Object.assign({ defaultBlueprint, - defaultRoute + defaultResource, + buildDesaturated }, allBps) diff --git a/src/lib/util.js b/src/lib/util.js index 5e71f3a..c6b29b8 100755 --- a/src/lib/util.js +++ b/src/lib/util.js @@ -55,33 +55,33 @@ export const idxSearcher = R.curry((attrName, searchValue, myArray) => { /* more site specific functions. TODO: maybe move to another folder? */ -export function fmtSheetTitle (name) { +export function fmtName (name) { return name.replaceAll(' ', '-').toLowerCase() } export function fmtBlueprinterTitles (tabs) { const obj = {} Object.keys(tabs).forEach(tab => { - const name = fmtSheetTitle(tab) + const name = fmtName(tab) obj[name] = tabs[tab] }) return obj } export function deriveFilename (sheet, tab) { - return `${fmtSheetTitle(sheet)}-${fmtSheetTitle(tab)}.json` + return `${fmtName(sheet)}-${fmtName(tab)}.json` } -export function bp (full) { +export function desaturate (full) { const blueprint = { name: R.clone(full.name), sheet: R.clone(full.sheet), dialects: R.clone(full.dialects), - routes: {} + resources: {} } - Object.keys(full.routes).forEach(route => { - blueprint.routes[route] = { - options: R.clone(full.routes[route].options) + Object.keys(full.resources).forEach(route => { + blueprint.resources[route] = { + options: R.clone(full.resources[route].options) } }) return blueprint diff --git a/src/models/StoreJson.js b/src/models/StoreJson.js index e94e2f8..1124960 100644 --- a/src/models/StoreJson.js +++ b/src/models/StoreJson.js @@ -1,20 +1,28 @@ import fs from 'mz/fs' -import { fmtSheetTitle } from '../lib/util' import copy from '../copy/en' const STORAGE_DIRNAME = 'temp' +function partsFromFilename (fname) { + const body = fname.slice(0, -5) + return body.split('__') +} + class StoreJson { - save (bp) { - return Promise.all( - Object.keys(bp.routes).map(route => - fs.writeFile( - `${STORAGE_DIRNAME}/${fmtSheetTitle( - bp.sheet.name - )}__${fmtSheetTitle(bp.name)}__${route}.json`, - JSON.stringify(bp.routes[route].data) - ) - ) + index () { + return Promise.resolve() + .then(() => fs.readdir(STORAGE_DIRNAME)) + .then(files => files.filter(f => f.match(/.*\.json$/))) + .then(jsons => jsons.map(partsFromFilename)) + .then(parts => parts.map(p => `${p[0]}/${p[1]}/${p[2]}`)) + } + + save (url, data) { + const parts = url.split('/') + + return fs.writeFile( + `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`, + JSON.stringify(data) ) } diff --git a/test/internals.js b/test/internals.js index c99f3ed..49c6ba4 100644 --- a/test/internals.js +++ b/test/internals.js @@ -2,9 +2,9 @@ import test from 'ava' import R from 'ramda' import { defaultBlueprint, - defaultRoute, - byColumn, - byRow + defaultResource, + columns, + rows } from '../src/lib/blueprinters' const egInput1 = [ @@ -15,41 +15,43 @@ const egInput1 = [ test('defaultBlueprint exports', t => { const expected = { + sheet: { + name: null, + id: null + }, name: null, - id: null, - dialects: ['rest'], - routes: {} + resources: {} } t.deepEqual(expected, defaultBlueprint) }) -test('byColumn blueprinter generates expected output', t => { - const actual = byColumn('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1) +test('columns blueprinter generates expected output', t => { + const actual = columns('eg ColumnBlueprint', 'egSheetName', 'egSheetId', egInput1) const expected = R.clone(defaultBlueprint) expected.name = 'eg ColumnBlueprint' expected.sheet = { id: 'egSheetId', name: 'egSheetName' } - expected.routes['h1'] = R.clone(defaultRoute) - expected.routes['h1'].data = [1, 4] - expected.routes['h2'] = R.clone(defaultRoute) - expected.routes['h2'].data = [2, 5] - expected.routes['h3'] = R.clone(defaultRoute) - expected.routes['h3'].data = [3, 6] + expected.resources['h1'] = R.clone(defaultResource) + expected.resources['h1'].data = [1, 4] + expected.resources['h2'] = R.clone(defaultResource) + expected.resources['h2'].data = [2, 5] + expected.resources['h3'] = R.clone(defaultResource) + expected.resources['h3'].data = [3, 6] t.deepEqual(expected, actual) }) -test('byRow blueprinter generates expected output', t => { - const actual = byRow('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items') +test('rows blueprinter generates expected output', t => { + const actual = rows('egRowBlueprint', 'egSheetName', 'egSheetId', egInput1, 'items') const expected = R.clone(defaultBlueprint) expected.name = 'egRowBlueprint' expected.sheet = { id: 'egSheetId', name: 'egSheetName' } - expected.routes['items'] = R.clone(defaultRoute) - expected.routes['items'].data = [{ + expected.resources['items'] = R.clone(defaultResource) + expected.resources['items'].data = [{ h1: 1, h2: 2, h3: 3