Skip to content

Commit 35c9890

Browse files
EvgeniiaVakEvgeniiaVak
authored
Merge pull request #20 from datopian/feature/downloads
Feature/downloads
2 parents c8c4e5e + 7f73216 commit 35c9890

5 files changed

Lines changed: 463 additions & 61 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"scripts": {
66
"start": "node -r dotenv/config ./bin/www",
77
"test": "mocha -r dotenv/config --recursive",
8+
"dev": "watch 'node -r dotenv/config ./bin/www'",
89
"test:integration": "mocha -r dotenv/config --recursive \"./test/integration/*.js\"",
910
"test:unit": "mocha -r dotenv/config --recursive \"./test/unit/*.js\""
1011
},
@@ -19,7 +20,7 @@
1920
"http-errors": "~1.6.3",
2021
"morgan": "~1.9.1",
2122
"node-fetch": "^2.6.1",
22-
"pg": "^8.3.3"
23+
"xlsx": "^0.16.8"
2324
},
2425
"devDependencies": {
2526
"chai": "^4.2.0",

‎routes/index.js‎

Lines changed: 106 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
var express = require('express')
22
var router = express.Router()
3+
const { Readable, finished } = require('stream')
4+
const { once } = require('events')
5+
36
const { request, gql } = require('graphql-request')
47

58
const { queryForData } = require('./queryGraphQL')
9+
const XLSX = require('xlsx')
10+
const path = require('path')
611

712
const APP_VERSION = 'v1'
813

@@ -54,7 +59,7 @@ function beautifyGQLSchema(gqlSchema) {
5459

5560
/* GET home page. */
5661
router.get('/', function (req, res, next) {
57-
res.send('Home Page!!')
62+
res.send('DATA-API Home Page!!')
5863
})
5964

6065
router.get(`/${APP_VERSION}/datastore_search/help`, function (req, res, next) {
@@ -83,19 +88,12 @@ router.get(`/${APP_VERSION}/datastore_search`, async function (req, res, next) {
8388
if (!('resource_id' in req.query)) {
8489
return res.redirect(303, `/${APP_VERSION}/datastore_search/help`)
8590
}
86-
// console.log("Request: " + JSON.stringify(req))
87-
// console.log('Query: ' + JSON.stringify(req.query))
88-
// console.log('Params: ' + JSON.stringify(req.params))
89-
// console.log("Headers: " + JSON.stringify(req.headers))
9091
const table = req.query.resource_id
9192
// query for schema -> this should be already in Frictionless format
9293
// const schema = await queryForSchema()
9394
const schema = await getGraphQLTableSchema(table)
94-
// console.log('SCHEMA: ' + JSON.stringify(schema))
9595
// query for data -> basically the call to queryGraphQL
9696
const data = await queryForData(schema, req.query)
97-
98-
// console.log('RESPONSE: ' + JSON.stringify(data))
9997
/*TODO*/
10098
/* Auth handling ... maybe JWT? */
10199
// Mandatory GET parameters check
@@ -110,4 +108,104 @@ router.get(`/${APP_VERSION}/datastore_search`, async function (req, res, next) {
110108
}
111109
})
112110

111+
//TODO finish this test function to manually check downloads
112+
// router.get(`/test/download`, async function (req, res, next) {
113+
// // res.sendFile('./test-download.html')
114+
// const ppath = __dirname.split(path.sep).slice(0, -1).join(path.sep)
115+
// res.sendFile(path.join(ppath + '/test/test-download.html'))
116+
// })
117+
/**
118+
*
119+
*/
120+
121+
DOWNLOAD_FORMATS_SUPPORTED = ['json', 'csv', 'xlsx', 'ods']
122+
123+
router.post(`/${APP_VERSION}/download`, async function (req, res, next) {
124+
console.log(' Download CALLED')
125+
// get the graphql query from body
126+
const query = req.body.query ? req.body.query : req.body
127+
// call GraphQL
128+
try {
129+
// TODO check graphql syntax BEFORE sending it
130+
const gqlRes = await request(`${process.env.HASURA_URL}/v1/graphql`, query)
131+
132+
// // capture graphql response
133+
const ext = (req.params.format || req.query.format || 'json')
134+
.toLowerCase()
135+
.trim()
136+
if (!DOWNLOAD_FORMATS_SUPPORTED.includes(ext)) {
137+
res
138+
.status(400)
139+
.send(
140+
'Bad format. Supported Formats: ' +
141+
JSON.stringify(DOWNLOAD_FORMATS_SUPPORTED)
142+
)
143+
}
144+
const colSep = (req.query.field_separator || ',').trim()
145+
res.set(
146+
'Content-Disposition',
147+
'attachment; filename="download.' + ext + '";'
148+
)
149+
if (ext != 'json') {
150+
// any spreadsheet supported by [js-xlsx](https://github.com/SheetJS/sheetjs)
151+
let wb = XLSX.utils.book_new()
152+
// TODO control the column/field order:
153+
// https://stackoverflow.com/questions/56854160/sort-and-filter-columns-with-xlsx-js-after-json-to-sheet
154+
// https://github.com/SheetJS/sheetjs/issues/738
155+
// it needs to receive the header parameter with the desired column order
156+
157+
//iterate over the result sets and create a work sheet to append to the book
158+
Object.keys(gqlRes).map((k) => {
159+
const ws = XLSX.utils.json_to_sheet(gqlRes[k])
160+
XLSX.utils.book_append_sheet(wb, ws, k)
161+
})
162+
if (ext === 'csv' && colSep != ',') {
163+
res.set('Content-Type', 'text/csv')
164+
// only send the first sheet
165+
const sname = wb.SheetNames[0]
166+
const ws = wb.Sheets[sname]
167+
// TODO deal with record separator
168+
// const recSep = (req.query.record_separator || undefined).trim() // req.params.field_separator ||
169+
// const csv = XLSX.utils.sheet_to_csv(ws, { FS: colSep, RS: recSep })
170+
const csv = XLSX.utils.sheet_to_csv(ws, { FS: colSep })
171+
const readable = Readable.from(csv, { encoding: 'utf8' })
172+
for await (const chunk of readable) {
173+
if (!res.write(chunk)) {
174+
// Handle backpressure
175+
await once(res, 'drain')
176+
}
177+
}
178+
res.end()
179+
} else {
180+
if (ext === 'csv') {
181+
res.set('Content-Type', 'text/csv')
182+
} else {
183+
res.set(
184+
'Content-Type',
185+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
186+
)
187+
}
188+
res.end(XLSX.write(wb, { type: 'buffer', bookType: ext }))
189+
}
190+
} else {
191+
// pure JSON, GraphQL already returns us that
192+
// Examples and docs here: https://nodesource.com/blog/understanding-streams-in-nodejs/
193+
// is json format, need to convert it to stream type it and stream it back to the client
194+
res.set('Content-Type', 'application/json')
195+
const readable = Readable.from(JSON.stringify(gqlRes), {
196+
encoding: 'utf8',
197+
})
198+
for await (const chunk of readable) {
199+
if (!res.write(chunk)) {
200+
await once(res, 'drain')
201+
}
202+
}
203+
res.end()
204+
}
205+
} catch (e) {
206+
console.error('Error during graphql call', e)
207+
res.status(500).end()
208+
}
209+
})
210+
113211
module.exports = router

‎test/integration/download.test.js‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
const request = require('supertest')
2+
const app = require('../../app')
3+
const assert = require('assert')
4+
5+
const QUERY = `
6+
query MyQuery {
7+
test_table(limit: 100) {
8+
float_column
9+
int_column
10+
text_column
11+
time_column
12+
}
13+
}
14+
`
15+
16+
describe('data-api-download', function () {
17+
it('should return JSON when no format parameter is passed', function (done) {
18+
request(app)
19+
.post('/v1/download')
20+
.send({
21+
query: `{
22+
__type(name: "test_table") {
23+
name
24+
fields {
25+
name
26+
type {
27+
name
28+
}
29+
}
30+
}
31+
}
32+
`,
33+
})
34+
.expect(200)
35+
.end(function (err, res) {
36+
// console.log('ERROR = ', err)
37+
// console.log('response Text: ', res.text)
38+
// console.log('Response Body: ', res.body)
39+
// Check headers
40+
// console.log(res.header)
41+
assert(
42+
res.header['content-disposition'] ==
43+
'attachment; filename="download.json";'
44+
)
45+
assert(res.header['content-type'] == 'application/json; charset=utf-8')
46+
// check that is a JSON
47+
JSON.parse(res.text)
48+
done()
49+
})
50+
})
51+
52+
it('should return JSON when format=JSON', function (done) {
53+
request(app)
54+
.post('/v1/download?format=json')
55+
.send({
56+
query: QUERY,
57+
})
58+
.expect(200)
59+
.end(function (err, res) {
60+
// console.log('ERROR = ', err)
61+
// console.log('response: ', res.text)
62+
assert(
63+
res.header['content-disposition'] ==
64+
'attachment; filename="download.json";'
65+
)
66+
assert(res.header['content-type'] == 'application/json; charset=utf-8')
67+
// check that is a JSON
68+
JSON.parse(res.text)
69+
done()
70+
})
71+
})
72+
73+
it('should return CSV when format=CSV', function (done) {
74+
request(app)
75+
.post('/v1/download?format=csv')
76+
.send({
77+
query: QUERY,
78+
})
79+
.expect(200)
80+
.end(function (err, res) {
81+
console.log('ERROR = ', err)
82+
// console.log('response TEXT: ', res.text)
83+
// console.log('response BODY: ', res.body)
84+
console.log(res.header)
85+
assert(
86+
res.header['content-disposition'] ==
87+
'attachment; filename="download.csv";'
88+
)
89+
assert(res.header['content-type'] == 'text/csv; charset=utf-8')
90+
// check that is a CSV
91+
done()
92+
})
93+
})
94+
95+
it('should return pipe-separated-CSV when format=CSV and field_separator=|', function (done) {
96+
request(app)
97+
.post('/v1/download?format=csv&field_separator=|')
98+
.send({
99+
query: QUERY,
100+
})
101+
.expect(200)
102+
.end(function (err, res) {
103+
console.log('ERROR = ', err)
104+
// console.log('response TEXT: ', res.text)
105+
// console.log('response BODY: ', res.body)
106+
console.log(res.header)
107+
assert(
108+
res.header['content-disposition'] ==
109+
'attachment; filename="download.csv";'
110+
)
111+
assert(res.header['content-type'] == 'text/csv; charset=utf-8')
112+
// check that is a pipe-separated-CSV
113+
114+
done()
115+
})
116+
})
117+
118+
it('should return Excel when format=xlsx', function (done) {
119+
request(app)
120+
.post('/v1/download?format=xlsx')
121+
.send({
122+
query: QUERY,
123+
})
124+
.expect(200)
125+
.end(function (err, res) {
126+
console.log('ERROR = ', err)
127+
// console.log('response TEXT: ', res.text)
128+
// console.log('response BODY: ', res.body)
129+
console.log(res.header)
130+
assert(
131+
res.header['content-disposition'] ==
132+
'attachment; filename="download.xlsx";'
133+
)
134+
assert(
135+
res.header['content-type'] ==
136+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
137+
)
138+
// check that is an Excel file
139+
done()
140+
})
141+
})
142+
143+
it('should return 400 when asked format is not supported', function (done) {
144+
request(app)
145+
.post('/v1/download?format=not_supported')
146+
.send({
147+
query: QUERY,
148+
})
149+
.expect(400)
150+
.end(function (err, res) {
151+
console.log('ERROR = ', err)
152+
assert(res.statusCode === 400)
153+
console.log(res.statusCode)
154+
console.log(
155+
res.text ===
156+
'Bad format. Supported Formats: ["json","csv","xlsx","ods"]'
157+
)
158+
done()
159+
})
160+
})
161+
})

0 commit comments

Comments
 (0)