mirror of
https://github.com/bellingcat/datasheet-server.git
synced 2026-06-09 20:08:32 +03:00
Compare commits
1 Commits
feature/js
...
grenfell
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9455348001 |
7
.github/workflows/ci.yml
vendored
7
.github/workflows/ci.yml
vendored
@@ -10,16 +10,11 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
with:
|
|
||||||
ref: ${{ github.head_ref }}
|
|
||||||
- uses: actions/setup-node@v2-beta
|
- uses: actions/setup-node@v2-beta
|
||||||
with:
|
with:
|
||||||
node-version: '12'
|
node-version: '12'
|
||||||
|
|
||||||
- run: npm install
|
- run: npm install
|
||||||
- run: npm test
|
- run: npm test
|
||||||
env:
|
|
||||||
CI: true
|
|
||||||
- run: npm run lint
|
- run: npm run lint
|
||||||
env:
|
|
||||||
CI: true
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
"version": "0.3.0",
|
"version": "0.3.0",
|
||||||
"description": "Starter project for an ES6 RESTful Express API",
|
"description": "Starter project for an ES6 RESTful Express API",
|
||||||
"main": "dist",
|
"main": "dist",
|
||||||
"type": "module",
|
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
|
"dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
|
||||||
"build": "env NODE_ENV=production npx babel src -d dist",
|
"build": "env NODE_ENV=production npx babel src -d dist",
|
||||||
@@ -27,7 +26,6 @@
|
|||||||
"express": "^4.13.3",
|
"express": "^4.13.3",
|
||||||
"express-graphql": "^0.6.12",
|
"express-graphql": "^0.6.12",
|
||||||
"express-handlebars": "^4.0.4",
|
"express-handlebars": "^4.0.4",
|
||||||
"file-system": "^2.2.2",
|
|
||||||
"googleapis": "^39.1.0",
|
"googleapis": "^39.1.0",
|
||||||
"graphql": "^0.13.2",
|
"graphql": "^0.13.2",
|
||||||
"morgan": "^1.8.0",
|
"morgan": "^1.8.0",
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import copy from '../copy/en'
|
|||||||
|
|
||||||
export default ({ config, controller }) => {
|
export default ({ config, controller }) => {
|
||||||
let api = Router()
|
let api = Router()
|
||||||
const fileDest = config.EXPORT_FILE_DEST || null
|
|
||||||
|
|
||||||
api.get('/', (req, res) => {
|
api.get('/', (req, res) => {
|
||||||
res.json({
|
res.json({
|
||||||
@@ -23,20 +22,6 @@ export default ({ config, controller }) => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
api.get('/export', (req, res) => {
|
|
||||||
controller
|
|
||||||
.retrieveAll(fileDest)
|
|
||||||
.then(msg =>
|
|
||||||
res.json({
|
|
||||||
success: msg
|
|
||||||
})
|
|
||||||
)
|
|
||||||
.catch(err =>
|
|
||||||
res.status(404)
|
|
||||||
.send({ error: err.message, err })
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
api.get('/update', (req, res) => {
|
api.get('/update', (req, res) => {
|
||||||
controller
|
controller
|
||||||
.update()
|
.update()
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
export default {
|
export default {
|
||||||
errors: {
|
errors: {
|
||||||
update: 'The server could not update. Check your API credentials and internet connection and try again.',
|
update: 'The server could not update. Check your API credentials and internet connection and try again.',
|
||||||
export: {
|
|
||||||
fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
|
|
||||||
writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
|
|
||||||
},
|
|
||||||
onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
||||||
onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
||||||
noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
|
noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
|
||||||
@@ -13,7 +9,6 @@ export default {
|
|||||||
modelLayer: prts => `Something went wrong at the model layer`
|
modelLayer: prts => `Something went wrong at the model layer`
|
||||||
},
|
},
|
||||||
success: {
|
success: {
|
||||||
update: 'All sheets updated',
|
update: 'All sheets updated'
|
||||||
export: dest => `All resources exported to the file: ${dest}`
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
1
src/datasheet.config.js
Symbolic link
1
src/datasheet.config.js
Symbolic link
@@ -0,0 +1 @@
|
|||||||
|
CONFIG=../configs/timemap/grenfell/datasheet.config.js
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
import copy from '../copy/en'
|
import copy from '../copy/en'
|
||||||
import { exportToFile } from '../lib/util'
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Controller
|
* Controller
|
||||||
@@ -40,37 +39,6 @@ class Controller {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Controller function to retrieve all blueprints and export to user defined file location
|
|
||||||
retrieveAll (fileDest) {
|
|
||||||
if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
|
|
||||||
|
|
||||||
const indexedData = {}
|
|
||||||
const urls = []
|
|
||||||
|
|
||||||
const bps = this.blueprints()
|
|
||||||
return Promise.all(
|
|
||||||
bps.map(bp => {
|
|
||||||
const resource = Object.keys(bp.resources)[0]
|
|
||||||
urls.push(bp.urls[0])
|
|
||||||
return this.retrieve(bp.sheet.name, bp.name, resource)
|
|
||||||
})
|
|
||||||
).then(async results => {
|
|
||||||
if (results.every(res => res)) {
|
|
||||||
urls.forEach((item, idx) => {
|
|
||||||
indexedData[item] = results[idx]
|
|
||||||
})
|
|
||||||
try {
|
|
||||||
const message = await exportToFile(fileDest, indexedData)
|
|
||||||
return message
|
|
||||||
} catch (e) {
|
|
||||||
return Promise.reject(e)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw new Error(copy.errors.export.writeFailed)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
retrieve (sheet, tab, resource) {
|
retrieve (sheet, tab, resource) {
|
||||||
if (this._sheetExists(sheet)) {
|
if (this._sheetExists(sheet)) {
|
||||||
const fetcher = this.fetchers[sheet]
|
const fetcher = this.fetchers[sheet]
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
import R from 'ramda'
|
import R from 'ramda'
|
||||||
import { promises as fs } from 'file-system'
|
|
||||||
import copy from '../copy/en'
|
|
||||||
|
|
||||||
/* eslint-disable */
|
/* eslint-disable */
|
||||||
String.prototype.replaceAll = function (search, replacement) {
|
String.prototype.replaceAll = function (search, replacement) {
|
||||||
@@ -16,18 +14,6 @@ function camelize (str) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function exportToFile (fileDest, data) {
|
|
||||||
const stringifiedData = JSON.stringify(data, null, 2)
|
|
||||||
const filePath = `${fileDest}/export.json`
|
|
||||||
|
|
||||||
try {
|
|
||||||
await fs.writeFile(filePath, stringifiedData)
|
|
||||||
return copy.success.export(filePath)
|
|
||||||
} catch (err) {
|
|
||||||
throw new Error(copy.errors.export.writeFailed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const fmtObj = R.curry(
|
export const fmtObj = R.curry(
|
||||||
(
|
(
|
||||||
columnNames,
|
columnNames,
|
||||||
|
|||||||
Reference in New Issue
Block a user