Skip to content

Commit 046833c

Browse files
dem1fodem1fo
committed
fix texts with new config API
1 parent 3166917 commit 046833c

5 files changed

Lines changed: 275 additions & 123 deletions

File tree

‎tests/config/loadConfig.test.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,15 @@ import { statSync, readFileSync, writeFileSync } from 'node:fs';
1818
import { fileURLToPath } from 'node:url';
1919
import { join, dirname } from 'node:path';
2020
import { tmpdir } from 'node:os';
21-
import { loadConfig } from '../../src/config/loadConfig';
22-
import { generateConfigHash } from '../../src/config/generateConfigHash';
21+
import { loadConfig, generateConfigHash } from '../../src/config';
2322

2423
const __dirname = dirname(fileURLToPath(import.meta.url));
2524
const ALGORITHM_ID = 'ANY';
2625
const FILES_PATH = join(__dirname, 'files');
2726

2827
test('loadConfig ok', () => {
2928
const TEST_FILE_PATH = join(FILES_PATH, 'test-config.json');
30-
const loadResult = loadConfig(TEST_FILE_PATH, ALGORITHM_ID);
29+
const loadResult = loadConfig({ configPath: TEST_FILE_PATH, algorithmId: ALGORITHM_ID });
3130

3231
expect(loadResult.success).toEqual(true);
3332
if (!loadResult.success) throw new TypeError();
@@ -42,7 +41,7 @@ test('loadConfig ok', () => {
4241

4342
test('loadConfig invalid', () => {
4443
const TEST_FILE_PATH = join(FILES_PATH, 'test-appconfig.json');
45-
expect(() => loadConfig(TEST_FILE_PATH, ALGORITHM_ID)).toThrow();
44+
expect(() => loadConfig({ configPath: TEST_FILE_PATH, algorithmId: ALGORITHM_ID })).toThrow();
4645
});
4746

4847
test('loadConfig salt', () => {
@@ -57,7 +56,7 @@ test('loadConfig salt', () => {
5756

5857
writeFileSync(TEST_FILE_PATH, JSON.stringify(cfg), 'utf-8');
5958

60-
const loadResult = loadConfig(TEST_FILE_PATH, ALGORITHM_ID);
59+
const loadResult = loadConfig({ configPath: TEST_FILE_PATH, algorithmId: ALGORITHM_ID });
6160

6261
expect(loadResult.success).toEqual(true);
6362
if (!loadResult.success) throw new TypeError();
@@ -78,7 +77,7 @@ test('loadConfig salt error', () => {
7877

7978
writeFileSync(TEST_FILE_PATH, JSON.stringify(cfg), 'utf-8');
8079

81-
const loadResult = loadConfig(TEST_FILE_PATH, ALGORITHM_ID);
80+
const loadResult = loadConfig({ configPath: TEST_FILE_PATH, algorithmId: ALGORITHM_ID });
8281

8382
expect(loadResult.success).toEqual(false);
8483
});

‎tests/config/loadSaltFile.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ const VALIDATOR_REGEXP = /BEGIN TEST[a-z\s]*END TEST/;
2424
const SALT_FILE_PATH = join(__dirname, 'files', 'test.salt');
2525

2626
test('loadSaltFile', () => {
27-
expect(loadSaltFile(SALT_FILE_PATH, VALIDATOR_REGEXP)).toEqual(readFileSync(SALT_FILE_PATH, 'utf-8'));
27+
expect(loadSaltFile({ saltFilePath: SALT_FILE_PATH, validatorRegexp: VALIDATOR_REGEXP })).toEqual(readFileSync(SALT_FILE_PATH, 'utf-8'));
2828

29-
expect(loadSaltFile('NON-EXISTANT-FILE', VALIDATOR_REGEXP)).toEqual(null);
29+
expect(loadSaltFile({ saltFilePath: 'NON_EXISTENT_FILE', validatorRegexp: VALIDATOR_REGEXP })).toEqual(null);
3030

31-
expect(loadSaltFile(join(__dirname, 'files', 'test-config.json'), VALIDATOR_REGEXP)).toEqual(null);
31+
expect(loadSaltFile({ saltFilePath: join(__dirname, 'files', 'test-config.json'), validatorRegexp: VALIDATOR_REGEXP })).toEqual(null);
3232
});

‎tests/processing/postprocess.test.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Common Identifier Application
2+
// Copyright (C) 2024 World Food Programme
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
import { tmpdir } from 'node:os';
18+
import { parse } from 'csv-parse/sync';
19+
import { readFileSync } from 'node:fs';
20+
import { fileURLToPath } from 'node:url';
21+
import { join, dirname } from 'node:path';
22+
23+
import { processFile } from '../../src/processing';
24+
import { BaseHasher } from '../../src/hashing/base';
25+
import { SUPPORTED_FILE_TYPES } from '../../src/document';
26+
import { extractAlgoColumnsFromObject } from '../../src/hashing/utils';
27+
import { SUPPORTED_VALIDATORS, type Validator } from '../../src/validation/Validation';
28+
29+
import type { Config } from '../../src/config/Config';
30+
import type { makeHasherFunction } from '../../src/hashing/base';
31+
32+
const __dirname = dirname(fileURLToPath(import.meta.url));
33+
34+
const CONFIG: Config.FileConfiguration = {
35+
meta: { id: '', version: '', signature: '' },
36+
source: {
37+
columns: [
38+
{ name: 'A', alias: 'col_a' },
39+
{ name: 'B', alias: 'col_b' },
40+
],
41+
},
42+
algorithm: {
43+
hash: { strategy: 'SHA256' },
44+
salt: { source: 'STRING', value: 'TEST' },
45+
columns: {
46+
static: ['col_a'],
47+
process: [],
48+
reference: [],
49+
},
50+
},
51+
validations: {
52+
col_a: [{ op: SUPPORTED_VALIDATORS.MAX_FIELD_LENGTH, value: 2 }],
53+
},
54+
destination: {
55+
columns: [
56+
{ name: 'A', alias: 'col_a' },
57+
{ name: 'Test', alias: 'test' },
58+
],
59+
postfix: '_OUTPUT',
60+
},
61+
destination_errors: {
62+
columns: [
63+
{ name: 'Errors', alias: 'errors' },
64+
{ name: 'A', alias: 'col_a' },
65+
],
66+
postfix: '_ERRORS',
67+
},
68+
destination_map: {
69+
columns: [
70+
{ name: 'A', alias: 'col_a' },
71+
{ name: 'Test', alias: 'test' },
72+
],
73+
postfix: '_MAPPING',
74+
},
75+
post_processing: {
76+
encryption: {
77+
key_path: ""
78+
}
79+
}
80+
};
81+
82+
83+
test('postprocessFile', async () => {
84+
// TODO
85+
})

‎tests/processing/preprocess.test.ts‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Common Identifier Application
2+
// Copyright (C) 2024 World Food Programme
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
import { existsSync } from 'node:fs';
18+
import { join, dirname } from 'node:path';
19+
import { fileURLToPath } from 'node:url';
20+
21+
import { preprocessFile, processFile } from '../../src/processing';
22+
import { SUPPORTED_VALIDATORS } from '../../src/validation/Validation';
23+
24+
import type { Config } from '../../src/config/Config';
25+
26+
const __dirname = dirname(fileURLToPath(import.meta.url));
27+
28+
const CONFIG: Config.FileConfiguration = {
29+
meta: { id: '', version: '', signature: '' },
30+
source: {
31+
columns: [
32+
{ name: 'A', alias: 'col_a' },
33+
{ name: 'B', alias: 'col_b' },
34+
],
35+
},
36+
algorithm: {
37+
hash: { strategy: 'SHA256' },
38+
salt: { source: 'STRING', value: 'TEST' },
39+
columns: {
40+
static: ['col_a'],
41+
process: [],
42+
reference: [],
43+
},
44+
},
45+
validations: {
46+
col_a: [{ op: SUPPORTED_VALIDATORS.MAX_FIELD_LENGTH, value: 2 }],
47+
},
48+
destination: {
49+
columns: [
50+
{ name: 'A', alias: 'col_a' },
51+
{ name: 'Test', alias: 'test' },
52+
],
53+
postfix: '_OUTPUT',
54+
},
55+
destination_errors: {
56+
columns: [
57+
{ name: 'Errors', alias: 'errors' },
58+
{ name: 'A', alias: 'col_a' },
59+
],
60+
postfix: '_ERRORS',
61+
},
62+
destination_map: {
63+
columns: [
64+
{ name: 'A', alias: 'col_a' },
65+
{ name: 'Test', alias: 'test' },
66+
],
67+
postfix: '_MAPPING',
68+
},
69+
};
70+
71+
test('preprocessFile invalid', async () => {
72+
const fn = async () =>
73+
await processFile({
74+
config: CONFIG,
75+
inputFilePath: '',
76+
outputPath: '',
77+
// @ts-ignore
78+
format: null,
79+
limit: 10,
80+
});
81+
await expect(fn).rejects.toThrow();
82+
});
83+
84+
test('preprocessFile', async () => {
85+
const filePath = join(__dirname, 'files', 'input_ok.csv');
86+
const results = await preprocessFile({
87+
config: CONFIG,
88+
inputFilePath: filePath,
89+
limit: 10,
90+
});
91+
92+
expect(results.document.data[0]).toEqual({ col_a: 'A0', col_b: 'B0' });
93+
expect(results.isValid).toEqual(true);
94+
expect(results.isMappingDocument).toEqual(false);
95+
expect(results.inputFilePath).toEqual(filePath);
96+
expect(results.errorFilePath).toEqual(undefined);
97+
});
98+
99+
test('preprocessFile::args', async () => {
100+
const filePath = join(__dirname, 'files', 'input_ok.csv');
101+
let results = await preprocessFile({
102+
config: CONFIG,
103+
inputFilePath: filePath,
104+
limit: 1,
105+
});
106+
107+
expect(results.document.data.length).toEqual(1);
108+
109+
results = await preprocessFile({ config: CONFIG, inputFilePath: filePath });
110+
expect(results.document.data.length).toEqual(2);
111+
});
112+
113+
test('preprocessFile mapping', async () => {
114+
const filePath = join(__dirname, 'files', 'input_mapping_ok.csv');
115+
const results = await preprocessFile({
116+
config: CONFIG,
117+
inputFilePath: filePath,
118+
limit: 10,
119+
});
120+
121+
expect(results.document.data[0]).toEqual({ col_a: 'A0' });
122+
123+
expect(results.isValid).toEqual(true);
124+
expect(results.isMappingDocument).toEqual(true);
125+
expect(results.inputFilePath).toEqual(filePath);
126+
expect(results.errorFilePath).toEqual(undefined);
127+
});
128+
129+
test('preprocessFile mapping invalid', async () => {
130+
const filePath = join(__dirname, 'files', 'input_mapping_ok.csv');
131+
132+
const newConfig = JSON.parse(JSON.stringify(CONFIG));
133+
newConfig.validations.col_a.push({ op: 'options', value: ['NO', 'WAY'] });
134+
135+
const results = await preprocessFile({
136+
config: newConfig,
137+
inputFilePath: filePath,
138+
limit: 10,
139+
});
140+
141+
expect(results.isMappingDocument).toEqual(true);
142+
expect(results.isValid).toEqual(false);
143+
});
144+
145+
test('preprocessFile invalid', async () => {
146+
const filePath = join(__dirname, 'files', 'input_invalid.csv');
147+
148+
const results = await preprocessFile({
149+
config: CONFIG,
150+
inputFilePath: filePath,
151+
limit: 10,
152+
});
153+
154+
expect(results.isMappingDocument).toEqual(false);
155+
expect(results.document.data[0]).toEqual({
156+
col_a: 'A0',
157+
col_b: 'B0',
158+
errors: '',
159+
row_number: 2,
160+
});
161+
expect(results.document.data[1]).toEqual({
162+
col_a: 'A1 TOO LONG',
163+
col_b: 'B1',
164+
errors: 'A must be shorter than 2 characters;',
165+
row_number: 3,
166+
});
167+
168+
expect(results.isValid).toEqual(false);
169+
170+
const errorFile = results.errorFilePath;
171+
expect(errorFile).not.toEqual(undefined);
172+
expect(existsSync(errorFile as string)).toEqual(true);
173+
});

0 commit comments

Comments
 (0)