Skip to content

Commit 8bf0ccb

Browse files
pyramationpyramation
authored
Merge pull request #265 from launchql/feat/csv-fix-headers
Feat/csv fix headers
2 parents 870ab4d + 5ecf2f7 commit 8bf0ccb

13 files changed

Lines changed: 309 additions & 11 deletions

‎packages/pgsql-test/__tests__/postgres-test.csv-edge-cases.test.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
process.env.LOG_SCOPE = 'pgsql-test';
2+
3+
import path from 'path';
4+
5+
import { seed } from '../src';
6+
import { getConnections } from '../src/connect';
7+
import { PgTestClient } from '../src/test-client';
8+
9+
const csv = (file: string) => path.resolve(__dirname, '../csv', file);
10+
11+
describe('CSV edge cases', () => {
12+
describe('quoted commas', () => {
13+
let pg: PgTestClient;
14+
let teardown: () => Promise<void>;
15+
16+
beforeAll(async () => {
17+
({ pg, teardown } = await getConnections({}, [
18+
seed.fn(async ({ pg }) => {
19+
await pg.query(`
20+
CREATE SCHEMA custom;
21+
CREATE TABLE custom.posts (
22+
id SERIAL PRIMARY KEY,
23+
user_id INT NOT NULL,
24+
content TEXT NOT NULL,
25+
title TEXT
26+
);
27+
`);
28+
}),
29+
30+
seed.csv({
31+
'custom.posts': csv('posts-quoted-commas.csv')
32+
})
33+
]));
34+
});
35+
36+
afterAll(async () => {
37+
await teardown();
38+
});
39+
40+
it('handles quoted commas in CSV fields', async () => {
41+
const res = await pg.query(`
42+
SELECT id, user_id, content, title
43+
FROM custom.posts
44+
ORDER BY id
45+
`);
46+
47+
expect(res.rows).toEqual([
48+
{ id: 1, user_id: 1, content: 'Hello, world!', title: 'First Post, Ever' },
49+
{ id: 2, user_id: 2, content: 'Graphile is cool!', title: 'GraphQL, PostGraphile' }
50+
]);
51+
});
52+
});
53+
54+
describe('escaped quotes', () => {
55+
let pg: PgTestClient;
56+
let teardown: () => Promise<void>;
57+
58+
beforeAll(async () => {
59+
({ pg, teardown } = await getConnections({}, [
60+
seed.fn(async ({ pg }) => {
61+
await pg.query(`
62+
CREATE SCHEMA custom;
63+
CREATE TABLE custom.posts (
64+
id SERIAL PRIMARY KEY,
65+
user_id INT NOT NULL,
66+
content TEXT NOT NULL
67+
);
68+
`);
69+
}),
70+
71+
seed.csv({
72+
'custom.posts': csv('posts-escaped-quotes.csv')
73+
})
74+
]));
75+
});
76+
77+
afterAll(async () => {
78+
await teardown();
79+
});
80+
81+
it('handles escaped quotes in CSV fields', async () => {
82+
const res = await pg.query(`
83+
SELECT id, user_id, content
84+
FROM custom.posts
85+
ORDER BY id
86+
`);
87+
88+
expect(res.rows).toEqual([
89+
{ id: 1, user_id: 1, content: 'He said "hello"' },
90+
{ id: 2, user_id: 2, content: 'She replied "hi"' }
91+
]);
92+
});
93+
});
94+
});

‎packages/pgsql-test/__tests__/postgres-test.csv-optional-fields.test.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
process.env.LOG_SCOPE = 'pgsql-test';
2+
3+
import path from 'path';
4+
5+
import { seed } from '../src';
6+
import { getConnections } from '../src/connect';
7+
import { PgTestClient } from '../src/test-client';
8+
9+
const csv = (file: string) => path.resolve(__dirname, '../csv', file);
10+
11+
let pg: PgTestClient;
12+
let teardown: () => Promise<void>;
13+
14+
beforeAll(async () => {
15+
({ pg, teardown } = await getConnections({}, [
16+
seed.fn(async ({ pg }) => {
17+
await pg.query(`
18+
CREATE SCHEMA custom;
19+
CREATE TABLE custom.posts (
20+
id SERIAL PRIMARY KEY,
21+
user_id INT NOT NULL,
22+
content TEXT NOT NULL,
23+
title TEXT,
24+
published BOOLEAN
25+
);
26+
`);
27+
}),
28+
29+
seed.csv({
30+
'custom.posts': csv('posts-with-optional.csv')
31+
}),
32+
33+
seed.fn(async ({ pg }) => {
34+
await pg.query(`SELECT setval(pg_get_serial_sequence('custom.posts', 'id'), (SELECT MAX(id) FROM custom.posts));`);
35+
})
36+
]));
37+
});
38+
39+
afterAll(async () => {
40+
await teardown();
41+
});
42+
43+
it('csv with optional fields', async () => {
44+
const res = await pg.query(`
45+
SELECT id, user_id, content, title, published
46+
FROM custom.posts
47+
ORDER BY id
48+
`);
49+
50+
expect(res.rows).toEqual([
51+
{ id: 1, user_id: 1, content: 'Hello world!', title: 'My First Post', published: true },
52+
{ id: 2, user_id: 2, content: 'Graphile is cool!', title: 'GraphQL Rocks', published: false }
53+
]);
54+
});

‎packages/pgsql-test/__tests__/postgres-test.csv-subset-header.test.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
process.env.LOG_SCOPE = 'pgsql-test';
2+
3+
import fs from 'fs';
4+
import path from 'path';
5+
6+
import { seed } from '../src';
7+
import { getConnections } from '../src/connect';
8+
import { exportTableToCsv } from '../src/seed/csv';
9+
import { PgTestClient } from '../src/test-client';
10+
11+
const csv = (file: string) => path.resolve(__dirname, '../csv', file);
12+
13+
let pg: PgTestClient;
14+
let teardown: () => Promise<void>;
15+
16+
beforeAll(async () => {
17+
({ pg, teardown } = await getConnections({}, [
18+
// 1. Create schema with SERIAL primary keys
19+
seed.fn(async ({ pg }) => {
20+
await pg.query(`
21+
CREATE SCHEMA custom;
22+
CREATE TABLE custom.users (
23+
id SERIAL PRIMARY KEY,
24+
name TEXT NOT NULL
25+
);
26+
27+
CREATE TABLE custom.posts (
28+
id SERIAL PRIMARY KEY,
29+
user_id INT REFERENCES custom.users(id),
30+
content TEXT NOT NULL
31+
);
32+
`);
33+
}),
34+
35+
// 2. Seed from CSV using COPY FROM STDIN
36+
seed.csv({
37+
'custom.users': csv('users.csv'),
38+
'custom.posts': csv('posts-subset-header.csv')
39+
}),
40+
41+
// 3. Fix SERIAL sequences to match the highest used ID
42+
seed.fn(async ({ pg }) => {
43+
await pg.query(`SELECT setval(pg_get_serial_sequence('custom.users', 'id'), (SELECT MAX(id) FROM custom.users));`);
44+
await pg.query(`SELECT setval(pg_get_serial_sequence('custom.posts', 'id'), (SELECT MAX(id) FROM custom.posts));`);
45+
})
46+
]));
47+
});
48+
49+
afterAll(async () => {
50+
await teardown();
51+
});
52+
53+
it('csv in/out', async () => {
54+
// 4. Insert new data without specifying IDs (uses SERIAL)
55+
await pg.query(`
56+
INSERT INTO custom.users (name) VALUES ('Carol');
57+
INSERT INTO custom.posts (user_id, content) VALUES (3, 'Carol''s first post');
58+
`);
59+
60+
// 5. Validate full contents
61+
const res = await pg.query(`
62+
SELECT custom.users.name, custom.posts.content
63+
FROM custom.posts
64+
JOIN custom.users ON custom.users.id = custom.posts.user_id
65+
ORDER BY custom.users.id
66+
`);
67+
68+
expect(res.rows).toEqual([
69+
{ name: 'Alice', content: 'Hello world!' },
70+
{ name: 'Bob', content: 'Graphile is cool!' },
71+
{ name: 'Carol', content: "Carol's first post" }
72+
]);
73+
74+
// 6. Ensure output directory exists
75+
const outDir = path.resolve(__dirname, '../output');
76+
fs.mkdirSync(outDir, { recursive: true });
77+
78+
// 7. Export updated tables to CSV
79+
await exportTableToCsv(pg, 'custom.users', path.join(outDir, 'users.csv'));
80+
await exportTableToCsv(pg, 'custom.posts', path.join(outDir, 'posts.csv'));
81+
82+
console.log(`📤 Exported users and posts to ${outDir}`);
83+
});

‎packages/pgsql-test/__tests__/postgres-test.csv.test.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,17 @@ beforeAll(async () => {
2121
CREATE SCHEMA custom;
2222
CREATE TABLE custom.users (
2323
id SERIAL PRIMARY KEY,
24-
name TEXT NOT NULL
24+
name TEXT NOT NULL,
25+
email TEXT,
26+
bio TEXT
2527
);
2628
2729
CREATE TABLE custom.posts (
2830
id SERIAL PRIMARY KEY,
2931
user_id INT REFERENCES custom.users(id),
30-
content TEXT NOT NULL
32+
content TEXT NOT NULL,
33+
title TEXT,
34+
published BOOLEAN
3135
);
3236
`);
3337
}),

‎packages/pgsql-test/csv/posts-escaped-quotes.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,user_id,content
2+
1,1,"He said ""hello"""
3+
2,2,"She replied ""hi"""

‎packages/pgsql-test/csv/posts-quoted-commas.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,user_id,content,title
2+
1,1,"Hello, world!","First Post, Ever"
3+
2,2,"Graphile is cool!","GraphQL, PostGraphile"

‎packages/pgsql-test/csv/posts-subset-header.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,content,user_id
2+
1,Hello world!,1
3+
2,Graphile is cool!,2

‎packages/pgsql-test/csv/posts-with-optional.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,user_id,content,title,published
2+
1,1,Hello world!,My First Post,true
3+
2,2,Graphile is cool!,GraphQL Rocks,false

‎packages/pgsql-test/output/posts.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
id,user_id,content
2-
1,1,Hello world!
3-
2,2,Graphile is cool!
4-
3,3,Carol's first post
1+
id,user_id,content,title,published
2+
1,1,Hello world!,,
3+
2,2,Graphile is cool!,,
4+
3,3,Carol's first post,,

‎packages/pgsql-test/output/users.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
id,name
2-
1,Alice
3-
2,Bob
4-
3,Carol
1+
id,name,email,bio
2+
1,Alice,,
3+
2,Bob,,
4+
3,Carol,,

0 commit comments

Comments
 (0)