Clean master commit

src/api/index.js

@@ -0,0 +1,57 @@
+import {version} from "../../package.json";
+import {Router} from "express";
+import {idxSearcher} from "../lib/util";
+
+export default ({config, controller}) => {
+  let api = Router();
+
+  api.get("/", (req, res) => {
+    res.json({
+      version
+    });
+  });
+
+  api.get("/blueprints", (req, res) => {
+    res.json(controller.blueprints());
+  });
+
+  api.get("/:source/:tab/:resource/:frag", (req, res) => {
+    const {source, tab, resource, frag} = req.params;
+    controller
+      .retrieveFrag(source, tab, resource, frag)
+      .then(data => res.json(data))
+      .catch(err =>
+        res.json({
+          error: err.message
+        })
+      );
+  });
+
+  api.get("/:source/:tab/:resource", (req, res) => {
+    controller
+      .retrieve(req.params.source, req.params.tab, req.params.resource)
+      .then(data => res.json(data))
+      .catch(err =>
+        res.json({
+          error: err.message
+        })
+      );
+  });
+
+  api.get("/update", (req, res) => {
+    controller
+      .update()
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.json({
+          error: err.message
+        })
+      );
+  });
+
+  return api;
+};

src/blueprinters/byColumn.js

@@ -0,0 +1,34 @@
+import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
+
+/**
+ * byColumn - generate a Blueprint from a data source by column. Each column
+ * name is a resource, and all values in that column are the resource items.
+ *
+ * @param  {type} data - list of lists representing sheet data.
+ * @return {type} Blueprint
+ * generated.
+ */
+export default function byColumn(tabName, sourceName, sourceId, data) {
+  // Define Blueprint props
+  const bp = R.clone(defaultBlueprint);
+  bp.source = {
+    name: sourceName,
+    id: sourceId
+  };
+  bp.name = tabName;
+
+  // column names define routes
+  const labels = data[0];
+  labels.forEach(label => {
+    bp.routes[label] = R.clone(defaultRoute);
+  });
+
+  // remaining rows as data
+  data.forEach((row, idx) => {
+    if (idx == 0) return;
+    labels.forEach((label, idx) => {
+      bp.routes[label].data.push(row[idx]);
+    });
+  });
+  return bp;
+}

src/blueprinters/byGroup.js

@@ -0,0 +1,56 @@
+import R from "ramda";
+import {fmtObj, idxSearcher} from "../lib/util";
+import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
+
+/**
+ * byGroup - generate a Blueprint from a data source grouped by a column called 'group'
+ * The resource name defaults to 'groups', or a custom resource name can be passed.
+ * Each resource item is an object with values labelled according to column
+ * names. Items are inserted in the data list at idx = id.
+ *
+ * @param  {type} data         list of lists representing sheet data.
+ * @param  {type} label="groups"  name of resource in blueprint.
+ * @param  {type} name=""      name of blueprint.
+ * @return {type} Blueprint
+ */
+export default function byGroup(
+  tabName,
+  sourceName,
+  sourceId,
+  data,
+  label = "groups"
+) {
+  // Define Blueprint
+  const bp = R.clone(defaultBlueprint);
+  bp.source = {
+    name: sourceName,
+    id: sourceId
+  };
+  bp.name = tabName;
+
+  // Column names define routes
+  const itemLabels = data[0];
+  const fmt = fmtObj(itemLabels);
+  bp.routes[label] = R.clone(defaultRoute);
+  bp.routes[label].data = [];
+
+  const dataGroups = {};
+
+  data.forEach((row, idx) => {
+    if (idx == 0) return;
+    const group = fmt(row).group;
+    if (!dataGroups[group]) {
+      dataGroups[group] = [fmt(row)];
+    } else {
+      dataGroups[group].push(fmt(row));
+    }
+  });
+  Object.keys(dataGroups).forEach(groupKey => {
+    bp.routes[label].data.push({
+      group: groupKey,
+      group_label: dataGroups[groupKey][0].group_label,
+      data: dataGroups[groupKey]
+    });
+  });
+  return bp;
+}

src/blueprinters/byId.js

@@ -0,0 +1,42 @@
+import R from "ramda";
+import {fmtObj, idxSearcher} from "../lib/util";
+import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
+
+/**
+ * byId - generate a Blueprint from a data source by id, which is an integer.
+ * The resource name defaults to 'ids', or a custom resource name can be passed.
+ * Each resource item is an object with values labelled according to column
+ * names. Items are inserted in the data list at idx = id.
+ *
+ * @param  {type} data         list of lists representing sheet data.
+ * @param  {type} label="ids"  name of resource in blueprint.
+ * @param  {type} name=""      name of blueprint.
+ * @return {type} Blueprint
+ */
+export default function byId(
+  tabName,
+  sourceName,
+  sourceId,
+  data,
+  label = "ids"
+) {
+  // Define Blueprint
+  const bp = R.clone(defaultBlueprint);
+  bp.source = {
+    name: sourceName,
+    id: sourceId
+  };
+  bp.name = tabName;
+
+  // Column names define routes
+  const itemLabels = data[0];
+  const fmt = fmtObj(itemLabels);
+  bp.routes[label] = R.clone(defaultRoute);
+  bp.routes[label].data = [];
+
+  data.forEach((row, idx) => {
+    if (idx == 0) return;
+    bp.routes[label].data[fmt(row).id] = fmt(row);
+  });
+  return bp;
+}

src/blueprinters/byRow.js

@@ -0,0 +1,41 @@
+import R from "ramda";
+import {fmtObj, idxSearcher} from "../lib/util";
+import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
+
+/**
+ * byRow - generate a Blueprint from a data source by row. The resource name
+ * defaults to 'rows', or a custom resource name can be passed. Each resource
+ * item is an object with values labelled according to column names.
+ *
+ * @param  {type} data         list of lists representing sheet data.
+ * @param  {type} label="rows" name of resource in blueprint.
+ * @param  {type} name=""      name of blueprint.
+ * @return {type} Blueprint
+ */
+export default function byRow(
+  tabName,
+  sourceName,
+  sourceId,
+  data,
+  label = "rows"
+) {
+  // Define Blueprint
+  const bp = R.clone(defaultBlueprint);
+  bp.source = {
+    name: sourceName,
+    id: sourceId
+  };
+  bp.name = tabName;
+
+  // Column names define routes
+  const itemLabels = data[0];
+  const fmt = fmtObj(itemLabels);
+  bp.routes[label] = R.clone(defaultRoute);
+  bp.routes[label].data = [];
+
+  data.forEach((row, idx) => {
+    if (idx == 0) return;
+    bp.routes[label].data.push(fmt(row));
+  });
+  return bp;
+}

src/blueprinters/byTree.js

@@ -0,0 +1,67 @@
+import R from "ramda";
+import {fmtObj, idxSearcher} from "../lib/util";
+import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
+
+/**
+ * byTree - generate a Blueprint from a data source grouped by a column called 'group'
+ * The resource name defaults to 'groups', or a custom resource name can be passed.
+ * Each resource item is an object with values labelled according to column
+ * names. Items are inserted in the data list at idx = id.
+ *
+ * @param  {type} data         list of lists representing sheet data.
+ * @param  {type} label="groups"  name of resource in blueprint.
+ * @param  {type} name=""      name of blueprint.
+ * @return {type} Blueprint
+ */
+export default function byTree(
+  tabName,
+  sourceName,
+  sourceId,
+  data,
+  label = "tree"
+) {
+  // Define Blueprint
+  const bp = R.clone(defaultBlueprint);
+  bp.source = {
+    name: sourceName,
+    id: sourceId
+  };
+  bp.name = tabName;
+
+  // Column names define routes
+  bp.routes[label] = R.clone(defaultRoute);
+  bp.routes[label].data = {};
+
+  const tree = {
+    key: "tags",
+    children: {}
+  };
+
+  data.forEach(path => {
+    const root = path[0];
+    if (!tree.children[root])
+      tree.children[root] = {
+        key: root,
+        children: {}
+      };
+
+    let depth = 1;
+    let parentNode = tree.children[root];
+
+    while (depth < path.length) {
+      const node = path[depth];
+      if (!parentNode.children[node]) {
+        parentNode.children[node] = {
+          key: node,
+          children: {}
+        };
+      }
+      parentNode = parentNode.children[node];
+
+      depth++;
+    }
+  });
+
+  bp.routes[label].data = tree;
+  return bp;
+}

src/example.config.js

@@ -0,0 +1,18 @@
+import BP from "./lib/blueprinters";
+
+export default {
+  port: 4040,
+  googleSheets: {
+    email: "project-name@reliable-baptist-23338.iam.gserviceaccount.com",
+    privateKey: "SOME_PRIVATE_KEY",
+    sheets: [
+      {
+        name: "example",
+        id: "1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI",
+        tabs: {
+          objects: BP.byRow
+        }
+      }
+    ]
+  }
+};

src/index.js

@@ -0,0 +1,31 @@
+import http from "http";
+import express from "express";
+import initialize from "./initialize";
+import middleware from "./middleware";
+import api from "./api";
+import config from "./config";
+
+let app = express();
+app.server = http.createServer(app);
+
+initialize(controller => {
+  app.use(
+    middleware({
+      config,
+      controller
+    })
+  );
+  app.use(
+    "/api",
+    api({
+      config,
+      controller
+    })
+  );
+
+  app.server.listen(process.env.PORT || config.port, () => {
+    console.log(`Started on port ${app.server.address().port}`);
+  });
+});
+
+export default app;

src/initialize.js

@@ -0,0 +1,46 @@
+import StoreJson from "./models/StoreJson";
+import Fetcher from "./lib/Fetcher";
+import Controller from "./lib/Controller";
+import config from "./config";
+
+const {googleSheets} = config;
+const {sheets, privateKey, email} = googleSheets;
+
+function authenticate(_fetcher) {
+  return _fetcher.fetcher.authenticate(email, privateKey).then(msg => {
+    console.log(msg);
+    return true;
+  });
+}
+
+export default callback => {
+  const fetchers = sheets.map(sheet => {
+    return {
+      name: sheet.name,
+      fetcher: new Fetcher(new StoreJson(), sheet.name, sheet.id, sheet.tabs)
+    };
+  });
+
+  Promise.all(fetchers.map(authenticate))
+    .then(() => {
+      console.log(`===================`);
+      console.log(`grant access to: ${email}`);
+      console.log(`===================`);
+
+      // NB: reformat fetchers as config for controller
+      const config = {};
+      fetchers.forEach(fetcher => {
+        config[fetcher.name] = fetcher.fetcher;
+      });
+      const controller = new Controller(config);
+      callback(controller);
+    })
+    .catch(err => {
+      console.log(err);
+      console.log(
+        `ERROR: the server couldn't connect to all of the sheets you provided. Ensure you have granted access to ${
+          serviceAccount.email
+        } on ALL listed sheets.`
+      );
+    });
+};

src/lib/Controller.js

@@ -0,0 +1,54 @@
+/**
+ * Controller
+ *
+ */
+class Controller {
+  constructor(fetchers) {
+    this.fetchers = fetchers;
+  }
+
+  sourceExists(source) {
+    return true;
+    if (Object.keys(this.fetchers).indexOf(source) == -1) return false;
+  }
+
+  blueprints() {
+    return Object.keys(this.fetchers).map(
+      source => this.fetchers[source].blueprints
+    );
+  }
+
+  update() {
+    return Promise.all(
+      Object.keys(this.fetchers).map(source => {
+        return this.fetchers[source].update();
+      })
+    ).then(results => {
+      return "All sources updated";
+    });
+  }
+
+  retrieve(source, tab, resource) {
+    if (this.sourceExists(source)) {
+      const fetcher = this.fetchers[source];
+      return fetcher.retrieve(tab, resource);
+    } else {
+      return Promise.resolve().then(() => {
+        throw new Error(`Source ${source} not available.`);
+      });
+    }
+  }
+
+  retrieveFrag(source, tab, resource, frag) {
+    if (this.sourceExists(source)) {
+      const fetcher = this.fetchers[source];
+      return fetcher.retrieveFrag(tab, resource, frag);
+    } else {
+      return Promise.resolve().then(() => {
+        throw new Error(`Source ${source} not available.`);
+      });
+    }
+  }
+}
+
+export default Controller;

src/lib/Fetcher.js

@@ -0,0 +1,132 @@
+// FetcherTwo class interfaces with Google Sheet, and saves to a specified db
+import {google} from "googleapis";
+import {fmtSourceTitle, fmtBlueprinterTitles, deriveFilename, bp} from "./util";
+import {byRow, byId} from "./blueprinters";
+import R from "ramda";
+
+class Fetcher {
+  constructor(db, sourceName, sourceId, blueprinters) {
+    /*
+     * The database that the fetcher should use. This should be an instance of a model-compliant class.
+     * See models/Interface.js for the specifications for a model-compliant class.
+     */
+    this.db = db;
+
+    /*
+     * ID of the Google Sheet where the data is sourced. Note that the privateKey.client_email
+     * loaded here must be added to the sheet as an editor.
+     */
+    this.sourceId = sourceId;
+
+    /*
+     * The name of the source. This will prefix tabs saved in the database.
+     */
+    this.sourceName = sourceName;
+
+    /*
+     * These are the available tabs for storing and retrieving data.
+     * Each blueprinter is a function that returns a Blueprint from a
+     * list of lists (which will be retrieved from gsheets).
+     */
+    this.blueprinters = fmtBlueprinterTitles(blueprinters);
+    this.blueprints = {};
+    Object.keys(this.blueprinters).forEach(key => {
+      this.blueprints[key] = null;
+    });
+
+    /*
+     * Google API setup
+     */
+    this.sheets = google.sheets("v4");
+    this.auth = null;
+  }
+
+  /** returns a Promise that resolves if access is granted to the account, and rejects otherwise. */
+  authenticate(client_email, private_key) {
+    const googleAuth = new google.auth.JWT(client_email, null, private_key, [
+      "https://www.googleapis.com/auth/spreadsheets"
+    ]);
+    this.auth = googleAuth;
+    const {sourceId} = this;
+
+    return new Promise((resolve, reject) => {
+      googleAuth.authorize(function(err, tokens) {
+        if (err) {
+          reject(err);
+          return;
+        } else {
+          resolve(`Connected to ${sourceId}.`);
+        }
+      });
+    });
+  }
+
+  update() {
+    let tabTitles;
+    /* Retrieve all available routes on a given sheet, and store formatted copies of it where a formatter is available */
+    return this.sheets.spreadsheets
+      .get({
+        auth: this.auth,
+        spreadsheetId: this.sourceId
+      })
+      .then(response => {
+        tabTitles = response.data.sheets.map(sheet => sheet.properties.title);
+        return this.sheets.spreadsheets.values.batchGet({
+          auth: this.auth,
+          spreadsheetId: this.sourceId,
+          ranges: tabTitles
+        });
+      })
+      .then(results => {
+        const tabData = results.data.valueRanges;
+        return Promise.all(
+          tabData.map((tab, idx) => {
+            const {values} = tab;
+            if (values == undefined) {
+              return Promise.resolve({});
+            }
+            const name = tabTitles[idx];
+            return this.save(name, values);
+          })
+        );
+      })
+      .then(() => "All tabs updated");
+  }
+
+  save(tab, data) {
+    const title = fmtSourceTitle(tab);
+    if (Object.keys(this.blueprinters).indexOf(title) > -1) {
+      const blueprinters = this.blueprinters[title];
+
+      return blueprinters.map(blueprinter => {
+        const saturatedBp = blueprinter(
+          tab,
+          this.sourceName,
+          this.sourceId,
+          data
+        );
+        const blueprint = bp(saturatedBp); // TODO: come up with better semantics.
+        this.blueprints[title] = blueprint;
+        return this.db.save(saturatedBp);
+      });
+    } else {
+      // If it can't find a blueprinter for the tab title, default to byRow
+      return this.db.save(byRow(tab, this.sourceName, this.sourceId, data));
+    }
+  }
+
+  // NB: could combine these functions by checking kwargs length
+  retrieve(tab, resource) {
+    const title = fmtSourceTitle(tab);
+    const url = `${this.sourceName}/${tab}/${resource}`;
+    return this.db.load(url, this.blueprints[title]);
+  }
+
+  retrieveFrag(tab, resource, frag) {
+    const title = fmtSourceTitle(tab);
+    const url = `${this.sourceName}/${tab}/${resource}/${frag}`;
+    return this.db.load(url, this.blueprints[title]);
+  }
+}
+
+export default Fetcher;

src/lib/blueprinters.js

@@ -0,0 +1,28 @@
+import path from "path";
+import fs from "fs";
+
+const REL_PATH_TO_BPS = "../blueprinters";
+const allBps = {};
+
+export const defaultBlueprint = {
+  name: null,
+  id: null,
+  dialects: ["rest"], // supported dialects, can (eventually) be multiple
+  routes: {}
+};
+
+export const defaultRoute = {
+  options: {
+    fragment: true
+  },
+  data: []
+};
+
+// import all default exports from 'blueprinters' folder
+const normalizedPath = path.join(__dirname, REL_PATH_TO_BPS);
+fs.readdirSync(normalizedPath).forEach(file => {
+  const bpName = file.replace(".js", "");
+  allBps[bpName] = require(`${REL_PATH_TO_BPS}/${file}`).default;
+});
+
+module.exports = allBps;

src/lib/util.js

@@ -0,0 +1,87 @@
+import R from "ramda";
+
+String.prototype.replaceAll = function(search, replacement) {
+  const target = this;
+  return target.replace(new RegExp(search, "g"), replacement);
+};
+
+function camelize(str) {
+  return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function(match, index) {
+    if (+match === 0) return ""; // or if (/\s+/.test(match)) for white spaces
+    // return index == 0 ? match.toLowerCase() : match.toUpperCase();
+    return match.toUpperCase();
+  });
+}
+
+export const fmtObj = R.curry(
+  (
+    columnNames,
+    row,
+    options = {
+      noSpacesInKeys: false,
+      hyphenatedKeys: false,
+      camelCaseKeys: false
+    }
+  ) => {
+    const obj = {};
+    const fmtColName = colName => {
+      if (options.camelCaseKeys) {
+        return camelize(colName);
+      } else if (options.hyphenatedKeys) {
+        return colName.toLowerCase().replaceAll(" ", "-");
+      } else if (options.noSpacesInKeys) {
+        return colName.replaceAll(" ", "");
+      } else {
+        return colName;
+      }
+    };
+    columnNames.forEach((columnName, idx) => {
+      obj[fmtColName(columnName)] = row[idx];
+    });
+    return obj;
+  }
+);
+
+/* search for object with key in array. Return index if exists, or -1 if not */
+export const idxSearcher = R.curry((attrName, searchValue, myArray) => {
+  for (var i = 0; i < myArray.length; i++) {
+    if (myArray[i][attrName] == searchValue) {
+      return i;
+    }
+  }
+  return -1;
+});
+
+/* more site specific functions. TODO: maybe move to another folder? */
+
+export function fmtSourceTitle(name) {
+  return name.replaceAll(" ", "-").toLowerCase();
+}
+
+export function fmtBlueprinterTitles(tabs) {
+  const obj = {};
+  Object.keys(tabs).forEach(tab => {
+    const name = fmtSourceTitle(tab);
+    obj[name] = tabs[tab];
+  });
+  return obj;
+}
+
+export function deriveFilename(source, tab) {
+  return `${fmtSourceTitle(source)}-${fmtSourceTitle(tab)}.json`;
+}
+
+export function bp(full) {
+  const blueprint = {
+    name: R.clone(full.name),
+    source: R.clone(full.source),
+    dialects: R.clone(full.dialects),
+    routes: {}
+  };
+  Object.keys(full.routes).forEach(route => {
+    blueprint.routes[route] = {
+      options: R.clone(full.routes[route].options)
+    };
+  });
+  return blueprint;
+}

src/middleware/index.js

@@ -0,0 +1,17 @@
+import {Router, next} from "express";
+import {mapboxAccessToken} from "../config";
+import morgan from "morgan";
+import mapbox from "./mapbox";
+
+export default ({config, db}) => {
+  let routes = Router();
+
+  /* logging middleware */
+  routes.use(morgan("dev"));
+
+  if (mapboxAccessToken) {
+    routes.get("/mapbox/:z/:y/:x", mapbox(mapboxAccessToken));
+  }
+
+  return routes;
+};

src/middleware/mapbox/index.js

@@ -0,0 +1,16 @@
+import fetch from "node-fetch";
+import fs from "fs";
+// TODO: load images from mapbox API and store.
+
+const baseUrl = "http://a.tiles.mapbox.com/v4/mapbox.satellite";
+export default accessToken => (req, res) => {
+  const {x, y, z} = req.params;
+  // const filename = `${z}-${y}-${x}.png`
+  // const fileStream = fs.createWriteStream(`${z}-${y}-${x}.png`)
+  fetch(
+    `http://a.tiles.mapbox.com/v4/mapbox.satellite/${z}/${y}/${x}@2x.png?access_token=${accessToken}`
+  ).then(result => {
+    res.set("Content-Type", "image/png");
+    result.body.pipe(res);
+  });
+};

src/models/Interface.js

@@ -0,0 +1,23 @@
+/**
+ * Model is a class whose sole responsibility is to save and load blueprints.
+ * It allows for different storage mechanisms for different kinds of blueprints.
+ */
+class Model {
+  /**
+   * save - save a Blueprint, using the information it contains.
+   *
+   * @param  {type} blueprint the Blueprint to be saved.
+   * @return {type} Promise   which returns True.
+   */
+  save(blueprint) {}
+
+  /**
+   * load - load a resource from a data model, using a Blueprint object as
+   * well as a REST-like  URL of the format /:source/:tab/:resource.
+   *
+   * @param  {type} url       String that represents the path to resource.
+   * @param  {type} blueprint Blueprint object (desaturated?).
+   * @return {type}           Object containing the resource data.
+   */
+  load(url, blueprint) {}
+}

src/models/StoreJson.js

@@ -0,0 +1,65 @@
+import fs from "mz/fs";
+import hash from "object-hash";
+import {fmtSourceTitle} from "../lib/util";
+import path from "path";
+
+const STORAGE_DIRNAME = "temp";
+
+class StoreJson {
+  save(bp) {
+    return Promise.all(
+      Object.keys(bp.routes).map(route =>
+        fs.writeFile(
+          `${STORAGE_DIRNAME}/${fmtSourceTitle(
+            bp.source.name
+          )}__${fmtSourceTitle(bp.name)}__${route}.json`,
+          JSON.stringify(bp.routes[route].data)
+        )
+      )
+    );
+  }
+
+  load(url, bp) {
+    const parts = url.split("/");
+    const fname = `${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${
+      parts[2]
+    }.json`;
+    return fs
+      .exists(fname)
+      .then(isAvailable => {
+        if (isAvailable) return fs.readFile(fname, "utf8");
+        else {
+          throw new Error("No resource exists");
+        }
+      })
+      .then(data => JSON.parse(data))
+      .then(data => {
+        if (parts.length === 3) {
+          // No lookup if the requested url doesn't have a fragment
+          return data;
+        } else if (parts[2] === "ids") {
+          // Do a lookup if fragment is included to filter a relevant item
+          // When the resource requested is 'ids'
+          const id = parseInt(parts[3]);
+          if (id !== NaN && id >= 0 && id < data.length) {
+            return data[id];
+          } else {
+            throw new Error(`Fragment index does not exist`);
+          }
+        } else {
+          // Do a lookup if fragment is included to filter a relevant item
+          const index = parseInt(parts[3]);
+          if (index !== NaN && index >= 0 && index < data.length) {
+            console.log(data, index);
+            return data.filter((vl, idx) => idx === index)[0];
+          } else {
+            throw new Error(`Fragment index does not exist`);
+          }
+        }
+      });
+  }
+
+  // TODO: add method to build blueprint from data source
+}
+
+export default StoreJson;