Skip to content

Commit 94cc7f9

Browse files
Arpith Siromoneyclaude
Arpith Siromoney
and
claude
committed
Add JavaScript integration tests for feed fetching
- Create integration tests that mirror Go integration tests - Test feed fetching with real Redis and S3 (MinIO) - Verify article storage in both Redis sorted sets and S3 - Use shared testdata/feed-get-tests.yaml for consistency - Tests run with INTEGRATION=true environment variable 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent f37f5e8 commit 94cc7f9

1 file changed

Lines changed: 344 additions & 0 deletions

File tree

‎src/lib/feeds.integration.test.js‎

Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
// Integration tests for feed fetching
2+
// Run with: INTEGRATION=true node src/lib/feeds.integration.test.js
3+
// Requires Redis and MinIO to be running (use docker-compose up)
4+
5+
const fs = require('fs');
6+
const http = require('http');
7+
const yaml = require('js-yaml');
8+
const redis = require('redis');
9+
const AWS = require('aws-sdk');
10+
11+
// Exit if not running in integration mode
12+
if (process.env.INTEGRATION !== 'true') {
13+
console.log('Skipping integration tests (set INTEGRATION=true to run)');
14+
process.exit(0);
15+
}
16+
17+
// Load test cases from YAML
18+
const testCasesYaml = fs.readFileSync('./testdata/feed-get-tests.yaml', 'utf8');
19+
const testCases = yaml.load(testCasesYaml);
20+
21+
// Setup Redis client
22+
const redisHost = process.env.REDIS_HOST || 'localhost';
23+
const redisPort = process.env.REDIS_PORT || '6379';
24+
const redisClient = redis.createClient({
25+
host: redisHost,
26+
port: redisPort,
27+
});
28+
29+
// Setup S3 client (MinIO)
30+
const s3Endpoint = process.env.S3_ENDPOINT || 'http://localhost:9000';
31+
const s3AccessKey = process.env.S3_ACCESS_KEY || 'minioadmin';
32+
const s3SecretKey = process.env.S3_SECRET_KEY || 'minioadmin';
33+
const s3Bucket = process.env.S3_BUCKET || 'feedreader2018-articles';
34+
35+
AWS.config.update({
36+
accessKeyId: s3AccessKey,
37+
secretAccessKey: s3SecretKey,
38+
s3ForcePathStyle: true,
39+
signatureVersion: 'v4',
40+
});
41+
42+
const s3 = new AWS.S3({
43+
endpoint: s3Endpoint,
44+
params: { Bucket: s3Bucket },
45+
});
46+
47+
// Import feed utilities
48+
const { buildRedisKeys, buildArticleKey, extractArticleIds } = require('./feedUtils.js');
49+
50+
// Simple test runner
51+
let passed = 0;
52+
let failed = 0;
53+
let testServer = null;
54+
55+
function test(name, fn) {
56+
return new Promise((resolve) => {
57+
fn()
58+
.then(() => {
59+
passed++;
60+
console.log(`✓ ${name}`);
61+
resolve();
62+
})
63+
.catch((error) => {
64+
failed++;
65+
console.error(`✗ ${name}`);
66+
console.error(` ${error.message}`);
67+
if (error.stack) {
68+
console.error(` ${error.stack}`);
69+
}
70+
resolve();
71+
});
72+
});
73+
}
74+
75+
// Helper to create a mock feed.get wrapper that doesn't require Express
76+
function createFeedFetcher(feedURI, redisClient, s3Client) {
77+
const request = require('request');
78+
const FeedParser = require('feedparser');
79+
const { buildRequestHeaders, isValidArticle, processArticle, shouldStoreArticle, generateArticleBody } = require('./feedUtils.js');
80+
const { hash, score } = require('./articleUtils.js');
81+
82+
return new Promise((resolve, reject) => {
83+
const { feedKey, articlesKey } = buildRedisKeys(feedURI);
84+
const result = {
85+
success: false,
86+
articles: [],
87+
};
88+
89+
redisClient.hgetall(feedKey, (e, storedFeed) => {
90+
let fetchedFeed = storedFeed || {};
91+
const headers = buildRequestHeaders(fetchedFeed);
92+
93+
const requ = request({
94+
uri: feedURI,
95+
headers,
96+
}, (requestErr, response) => {
97+
if (requestErr) {
98+
result.success = false;
99+
result.error = requestErr.message;
100+
return reject(new Error(requestErr.message));
101+
}
102+
103+
const lastModified = response.headers['last-modified'] || '';
104+
const etag = response.headers.etag || '';
105+
result.statusCode = response.statusCode;
106+
107+
if (response.statusCode === 304) {
108+
// Not modified, return cached articles
109+
redisClient.zrevrange(articlesKey, 0, -1, (rangeErr, allArticles) => {
110+
if (rangeErr) return reject(rangeErr);
111+
result.success = true;
112+
result.articles = extractArticleIds(allArticles);
113+
result.title = fetchedFeed.title;
114+
result.link = fetchedFeed.link;
115+
result.lastModified = fetchedFeed.lastModified;
116+
result.etag = fetchedFeed.etag;
117+
resolve(result);
118+
});
119+
return;
120+
}
121+
122+
redisClient.hmset(feedKey, 'lastModified', lastModified, 'etag', etag, (hmsetErr) => {
123+
if (hmsetErr) return reject(hmsetErr);
124+
});
125+
});
126+
127+
const options = {};
128+
if (fetchedFeed.link) options.feedurl = fetchedFeed.link;
129+
130+
const feedparser = new FeedParser(options);
131+
requ.pipe(feedparser);
132+
133+
feedparser.on('error', (parseErr) => {
134+
result.success = false;
135+
result.error = parseErr.message;
136+
reject(parseErr);
137+
});
138+
139+
feedparser.on('meta', (meta) => {
140+
result.title = meta.title;
141+
result.link = meta.link;
142+
redisClient.hmset(feedKey, 'title', meta.title, 'link', meta.link, (hmsetErr) => {
143+
if (hmsetErr) reject(hmsetErr);
144+
});
145+
});
146+
147+
feedparser.on('readable', function() {
148+
const stream = this;
149+
for (;;) {
150+
const article = stream.read();
151+
if (!isValidArticle(article)) {
152+
return;
153+
}
154+
155+
const processedArticle = processArticle(article, feedURI, hash, score);
156+
const key = processedArticle.hash;
157+
const rank = processedArticle.score;
158+
const articleKey = buildArticleKey(key);
159+
160+
redisClient.zscore(articlesKey, articleKey, (zscoreErr, oldscore) => {
161+
if (zscoreErr) return;
162+
163+
redisClient.zadd(articlesKey, rank, articleKey, (zaddErr) => {
164+
if (zaddErr) return;
165+
166+
if (shouldStoreArticle(oldscore, rank)) {
167+
const body = generateArticleBody(processedArticle);
168+
s3Client.putObject({
169+
Key: key + '.json',
170+
Body: body,
171+
ContentType: 'application/json',
172+
}, (s3PutErr) => {
173+
if (s3PutErr) console.error('S3 error:', s3PutErr);
174+
});
175+
}
176+
});
177+
});
178+
}
179+
});
180+
181+
feedparser.on('end', () => {
182+
redisClient.zrevrange(articlesKey, 0, -1, (rangeErr, allArticles) => {
183+
if (rangeErr) return reject(rangeErr);
184+
result.success = true;
185+
result.articles = extractArticleIds(allArticles);
186+
resolve(result);
187+
});
188+
});
189+
});
190+
});
191+
}
192+
193+
// Helper to clear Redis keys
194+
function clearRedisKeys(feedURI, callback) {
195+
const { feedKey, articlesKey } = buildRedisKeys(feedURI);
196+
redisClient.del([feedKey, articlesKey], callback);
197+
}
198+
199+
// Run all integration tests
200+
async function runTests() {
201+
console.log('\n=== Testing Feed Fetching (Integration) ===\n');
202+
203+
// Test: Fetch and process Atom feed
204+
const atomTest = testCases.feed_get_tests.find(tc => tc.description.includes('Atom'));
205+
if (atomTest) {
206+
await test(atomTest.description, () => {
207+
return new Promise((resolve, reject) => {
208+
// Clear Redis
209+
clearRedisKeys('http://localhost:8888/xkcd', (clearErr) => {
210+
if (clearErr) return reject(clearErr);
211+
212+
// Read feed fixture
213+
const feedData = fs.readFileSync(atomTest.feed_fixture, 'utf8');
214+
215+
// Create test HTTP server
216+
testServer = http.createServer((req, res) => {
217+
res.writeHead(200, { 'Content-Type': 'application/xml' });
218+
res.end(feedData);
219+
});
220+
221+
testServer.listen(8888, async () => {
222+
try {
223+
const result = await createFeedFetcher('http://localhost:8888/xkcd', redisClient, s3);
224+
225+
// Verify success
226+
if (!result.success) {
227+
throw new Error(`Expected success=true, got ${result.success}`);
228+
}
229+
230+
// Verify feed metadata
231+
if (result.title !== atomTest.expected_feed_metadata.title) {
232+
throw new Error(`Title mismatch: got ${result.title}, want ${atomTest.expected_feed_metadata.title}`);
233+
}
234+
235+
if (result.link !== atomTest.expected_feed_metadata.link) {
236+
throw new Error(`Link mismatch: got ${result.link}, want ${atomTest.expected_feed_metadata.link}`);
237+
}
238+
239+
// Verify article count
240+
if (result.articles.length !== atomTest.expected_articles_count) {
241+
throw new Error(`Article count mismatch: got ${result.articles.length}, want ${atomTest.expected_articles_count}`);
242+
}
243+
244+
// Verify specific articles
245+
for (const expectedArticle of atomTest.expected_articles) {
246+
if (!result.articles.includes(expectedArticle.hash)) {
247+
throw new Error(`Expected article ${expectedArticle.hash} not found`);
248+
}
249+
250+
// Verify article is in Redis
251+
const { articlesKey } = buildRedisKeys('http://localhost:8888/xkcd');
252+
const articleKey = buildArticleKey(expectedArticle.hash);
253+
254+
await new Promise((res, rej) => {
255+
redisClient.zscore(articlesKey, articleKey, (err, score) => {
256+
if (err) return rej(err);
257+
if (parseInt(score) !== expectedArticle.score) {
258+
return rej(new Error(`Score mismatch: got ${score}, want ${expectedArticle.score}`));
259+
}
260+
res();
261+
});
262+
});
263+
264+
// Verify article is in S3
265+
await new Promise((res, rej) => {
266+
s3.headObject({ Key: expectedArticle.hash + '.json' }, (err) => {
267+
if (err) return rej(new Error(`Article ${expectedArticle.hash} not found in S3`));
268+
res();
269+
});
270+
});
271+
}
272+
273+
testServer.close();
274+
resolve();
275+
} catch (error) {
276+
testServer.close();
277+
reject(error);
278+
}
279+
});
280+
});
281+
});
282+
});
283+
}
284+
285+
// Test: Fetch and process RSS feed
286+
const rssTest = testCases.feed_get_tests.find(tc => tc.description.includes('RSS'));
287+
if (rssTest) {
288+
await test(rssTest.description, () => {
289+
return new Promise((resolve, reject) => {
290+
clearRedisKeys('http://localhost:8889/hn', (clearErr) => {
291+
if (clearErr) return reject(clearErr);
292+
293+
const feedData = fs.readFileSync(rssTest.feed_fixture, 'utf8');
294+
295+
testServer = http.createServer((req, res) => {
296+
res.writeHead(200, { 'Content-Type': 'application/xml' });
297+
res.end(feedData);
298+
});
299+
300+
testServer.listen(8889, async () => {
301+
try {
302+
const result = await createFeedFetcher('http://localhost:8889/hn', redisClient, s3);
303+
304+
if (!result.success) {
305+
throw new Error(`Expected success=true, got ${result.success}`);
306+
}
307+
308+
if (result.title !== rssTest.expected_feed_metadata.title) {
309+
throw new Error(`Title mismatch: got ${result.title}, want ${rssTest.expected_feed_metadata.title}`);
310+
}
311+
312+
if (result.articles.length !== rssTest.expected_articles_count) {
313+
throw new Error(`Article count mismatch: got ${result.articles.length}, want ${rssTest.expected_articles_count}`);
314+
}
315+
316+
testServer.close();
317+
resolve();
318+
} catch (error) {
319+
testServer.close();
320+
reject(error);
321+
}
322+
});
323+
});
324+
});
325+
});
326+
}
327+
328+
// Print summary
329+
console.log(`\n=== Test Summary ===`);
330+
console.log(`Passed: ${passed}`);
331+
console.log(`Failed: ${failed}`);
332+
console.log(`Total: ${passed + failed}\n`);
333+
334+
redisClient.quit();
335+
process.exit(failed > 0 ? 1 : 0);
336+
}
337+
338+
// Run tests
339+
runTests().catch((error) => {
340+
console.error('Test runner error:', error);
341+
if (testServer) testServer.close();
342+
redisClient.quit();
343+
process.exit(1);
344+
});

0 commit comments

Comments
 (0)