bellingcat/datasheet-server
1
0
Fork 0
mirror of https://github.com/bellingcat/datasheet-server.git synced 2026-06-08 03:18:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://www.github.com/forensic-architecture/datasheet-server into develop

Browse Source
This commit is contained in:
efarooqui
2020-11-16 08:55:00 -08:00
parent f3f574380c 7dd9d69d1f
commit 17607cf69f
3 changed files with 1046 additions and 815 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1814
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
package.json
View File

@@ -31,10 +31,10 @@
"morgan": "^1.8.0", "morgan": "^1.8.0",
"mz": "^2.7.0", "mz": "^2.7.0",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"node-xlsx": "^0.15.0",
"object-hash": "^1.3.0", "object-hash": "^1.3.0",
"ramda": "^0.25.0", "ramda": "^0.25.0",
"resource-router-middleware": "^0.6.0" "resource-router-middleware": "^0.6.0",
"xlsx": "^0.16.8"
}, },
"devDependencies": { "devDependencies": {
"@babel/cli": "^7.1.2", "@babel/cli": "^7.1.2",

43
src/lib/Fetcher.js
View File

@@ -1,15 +1,16 @@
// FetcherTwo class interfaces with Google Sheet, and saves to a specified db import R from 'ramda'
import { google } from 'googleapis' import { createHash } from 'crypto'
import { buildDesaturated } from './blueprinters' import { buildDesaturated } from './blueprinters'
import { import {
fmtName, fmtName,
fmtBlueprinterTitles, fmtBlueprinterTitles,
isFunction isFunction
} from './util' } from './util'
import { createHash } from 'crypto'
import R from 'ramda' /* GsheetFetcher deps */
import xlsx from 'node-xlsx' import { google } from 'googleapis'
import fs from 'fs' /* LocalFetcher deps */
import X from 'xlsx'
class Fetcher { class Fetcher {
constructor (db, name, bps) { constructor (db, name, bps) {
@@ -138,7 +139,6 @@ class Fetcher {
/** Run on startup. Should be overridden if explicit auth is required **/ /** Run on startup. Should be overridden if explicit auth is required **/
authenticate (env) { authenticate (env) {
console.log(`Connected to ${this.sheetName}. No explicit authentication required for ${this.type}s.`)
return Promise.resolve(this) return Promise.resolve(this)
} }
} }
@@ -221,27 +221,22 @@ class GsheetFetcher extends Fetcher {
} }
} }
class XlsxFetcher extends Fetcher { class LocalFetcher extends Fetcher {
constructor (db, name, bps, path) { constructor (db, name, bps, path) {
super(db, name, bps) super(db, name, bps)
this.type = 'XLSX File'
this.path = path this.path = path
this.isRemote = false this.update().then(res =>
console.log(`${res ? 'Successful' : 'Couldn\'t'} update ${name}`)
if (this.path.startsWith('https')) { )
this.isRemote = true
}
} }
update () { update () {
const data = xlsx.parse(fs.readFileSync(this.path)) const wb = X.readFile(this.path)
data.forEach(tab => { wb.SheetNames.forEach(name => {
const stringyData = tab.data.map(row => const sh = wb.Sheets[name]
row.map(d => const csv = X.utils.sheet_to_csv(sh, { FS: '\t' })
typeof (d) === 'number' ? d.toString() : d const ll = csv.split('\n').map(line => line.split('\t'))
) this.save(name, ll)
)
this.save(tab.name, stringyData)
}) })
return Promise.resolve(true) return Promise.resolve(true)
} }
@@ -249,5 +244,7 @@ class XlsxFetcher extends Fetcher {
export default { export default {
'gsheets': GsheetFetcher, 'gsheets': GsheetFetcher,
'xlsx': XlsxFetcher 'xlsx': LocalFetcher,
'ods': LocalFetcher,
'local': LocalFetcher
} }