mirror of
https://github.com/bellingcat/datasheet-server.git
synced 2026-06-09 20:08:32 +03:00
Compare commits
15 Commits
feature/js
...
project/ye
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fdee3fa581 | ||
|
|
f093253d87 | ||
|
|
011c667805 | ||
|
|
66d7d42497 | ||
|
|
71a8687003 | ||
|
|
63d168a964 | ||
|
|
4d310270d5 | ||
|
|
a6cc9491c6 | ||
|
|
0ee207dc75 | ||
|
|
7e97c5c227 | ||
|
|
978d572660 | ||
|
|
8bd5f38ac4 | ||
|
|
6ab37ccb70 | ||
|
|
4976bfd985 | ||
|
|
d713a15879 |
@@ -3,7 +3,6 @@
|
||||
"version": "0.3.0",
|
||||
"description": "Starter project for an ES6 RESTful Express API",
|
||||
"main": "dist",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "env NODE_ENV=development nodemon -w src --exec \"babel-node src\"",
|
||||
"build": "env NODE_ENV=production npx babel src -d dist",
|
||||
@@ -27,7 +26,6 @@
|
||||
"express": "^4.13.3",
|
||||
"express-graphql": "^0.6.12",
|
||||
"express-handlebars": "^4.0.4",
|
||||
"file-system": "^2.2.2",
|
||||
"googleapis": "^39.1.0",
|
||||
"graphql": "^0.13.2",
|
||||
"morgan": "^1.8.0",
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { version } from '../../package.json'
|
||||
import fetch from 'node-fetch'
|
||||
import { Router } from 'express'
|
||||
import { getParameterByName } from '../lib/util'
|
||||
import copy from '../copy/en'
|
||||
|
||||
export default ({ config, controller }) => {
|
||||
let api = Router()
|
||||
const fileDest = config.EXPORT_FILE_DEST || null
|
||||
|
||||
api.get('/', (req, res) => {
|
||||
res.json({
|
||||
@@ -23,18 +24,19 @@ export default ({ config, controller }) => {
|
||||
})
|
||||
})
|
||||
|
||||
api.get('/export', (req, res) => {
|
||||
controller
|
||||
.retrieveAll(fileDest)
|
||||
.then(msg =>
|
||||
res.json({
|
||||
success: msg
|
||||
})
|
||||
)
|
||||
.catch(err =>
|
||||
res.status(404)
|
||||
.send({ error: err.message, err })
|
||||
)
|
||||
api.get('/media/:code', (req, res) => {
|
||||
const { code } = req.params
|
||||
fetch(`${process.env.MEDIA_API_ENDPOINT}/${code}`, {
|
||||
method: 'get',
|
||||
headers: {
|
||||
'Authorization': 'Basic ' + new Buffer(process.env.MEDIA_AUTH_USER + ":" + process.env.MEDIA_AUTH_PWORD).toString("base64")
|
||||
}
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => res.send(data))
|
||||
.catch(err => {
|
||||
res.status(err.status || 404).send({ error: err.message })
|
||||
});
|
||||
})
|
||||
|
||||
api.get('/update', (req, res) => {
|
||||
@@ -45,10 +47,10 @@ export default ({ config, controller }) => {
|
||||
success: msg
|
||||
})
|
||||
)
|
||||
.catch(err =>
|
||||
.catch(err => {
|
||||
res.status(404)
|
||||
.send({ error: err.message, err })
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
api.get('/:sheet/:tab/:resource/:frag', (req, res) => {
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
export default {
|
||||
errors: {
|
||||
update: 'The server could not update. Check your API credentials and internet connection and try again.',
|
||||
export: {
|
||||
fileMissing: 'The server could not export. Check that you provided a file path to export to and try again.',
|
||||
writeFailed: 'The server could not export the data to the file. There is an issue with the data format'
|
||||
},
|
||||
onlySheet: 'You cannot query a sheet directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
||||
onlyTab: 'You cannot query a tab directly. The URL needs to be in the format /:sheet/:tab/:resource.',
|
||||
noSheet: sheet => `The sheet ${sheet} is not available in this server.`,
|
||||
@@ -13,7 +9,6 @@ export default {
|
||||
modelLayer: prts => `Something went wrong at the model layer`
|
||||
},
|
||||
success: {
|
||||
update: 'All sheets updated',
|
||||
export: dest => `All resources exported to the file: ${dest}`
|
||||
update: 'All sheets updated'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ function prefixedTabs (prefix, cfg) {
|
||||
[`${prf('events')}export_events`]: BP.deeprows,
|
||||
[`${prf('associations')}export_associations`]: BP.deeprows,
|
||||
[`${prf('sources')}export_sources`]: BP.deepids,
|
||||
[`${prf('shapes')}export_shapes`]: BP.deeprows,
|
||||
[`${prf('sites')}export_sites`]: BP.rows
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import copy from '../copy/en'
|
||||
import { exportToFile } from '../lib/util'
|
||||
|
||||
/**
|
||||
* Controller
|
||||
* Controller class
|
||||
*
|
||||
*/
|
||||
class Controller {
|
||||
@@ -40,37 +39,6 @@ class Controller {
|
||||
})
|
||||
}
|
||||
|
||||
// Controller function to retrieve all blueprints and export to user defined file location
|
||||
retrieveAll (fileDest) {
|
||||
if (!fileDest) return Promise.reject(new Error(copy.errors.export.fileMissing))
|
||||
|
||||
const indexedData = {}
|
||||
const urls = []
|
||||
|
||||
const bps = this.blueprints()
|
||||
return Promise.all(
|
||||
bps.map(bp => {
|
||||
const resource = Object.keys(bp.resources)[0]
|
||||
urls.push(bp.urls[0])
|
||||
return this.retrieve(bp.sheet.name, bp.name, resource)
|
||||
})
|
||||
).then(async results => {
|
||||
if (results.every(res => res)) {
|
||||
urls.forEach((item, idx) => {
|
||||
indexedData[item] = results[idx]
|
||||
})
|
||||
try {
|
||||
const message = await exportToFile(fileDest, indexedData)
|
||||
return message
|
||||
} catch (e) {
|
||||
return Promise.reject(e)
|
||||
}
|
||||
} else {
|
||||
throw new Error(copy.errors.export.writeFailed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
retrieve (sheet, tab, resource) {
|
||||
if (this._sheetExists(sheet)) {
|
||||
const fetcher = this.fetchers[sheet]
|
||||
|
||||
@@ -217,7 +217,10 @@ class GsheetFetcher extends Fetcher {
|
||||
})
|
||||
.then(this._buildBlueprintsAsync())
|
||||
.then(() => true)
|
||||
.catch(() => false)
|
||||
.catch((err) => {
|
||||
console.log(`Error fetching gsheets: ${err.message} `)
|
||||
return false
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import R from 'ramda'
|
||||
import { promises as fs } from 'file-system'
|
||||
import copy from '../copy/en'
|
||||
|
||||
/* eslint-disable */
|
||||
String.prototype.replaceAll = function (search, replacement) {
|
||||
@@ -16,18 +14,6 @@ function camelize (str) {
|
||||
})
|
||||
}
|
||||
|
||||
export async function exportToFile (fileDest, data) {
|
||||
const stringifiedData = JSON.stringify(data, null, 2)
|
||||
const filePath = `${fileDest}/export.json`
|
||||
|
||||
try {
|
||||
await fs.writeFile(filePath, stringifiedData)
|
||||
return copy.success.export(filePath)
|
||||
} catch (err) {
|
||||
throw new Error(copy.errors.export.writeFailed)
|
||||
}
|
||||
}
|
||||
|
||||
export const fmtObj = R.curry(
|
||||
(
|
||||
columnNames,
|
||||
|
||||
@@ -19,7 +19,6 @@ class StoreJson {
|
||||
|
||||
save (url, data) {
|
||||
const parts = url.split('/')
|
||||
|
||||
return fs.writeFile(
|
||||
`${STORAGE_DIRNAME}/${parts[0]}__${parts[1]}__${parts[2]}.json`,
|
||||
JSON.stringify(data)
|
||||
|
||||
@@ -12,6 +12,8 @@ const egInput1 = [
|
||||
[4, 5, 6]
|
||||
]
|
||||
|
||||
// Test default blueprint exports
|
||||
// Smoke tests
|
||||
test('defaultBlueprint exports', t => {
|
||||
const expected = {
|
||||
sheet: {
|
||||
|
||||
Reference in New Issue
Block a user