Handle extremely large feeds more efficiently when viewing articles (#433)

Author: synzen

services/user-feeds-next/src/articles/parser/article-parser.ts

@@ -60,6 +60,22 @@ export function flattenArticle(
     formatOptions?: UserFeedFormatOptions;
     useParserRules?: PostProcessParserRule[];
   }
+): FlattenedArticleWithoutId {
+  const base = flattenArticleLightweight(input, options);
+
+  return enrichFlattenedArticle(base, options);
+}
+
+/**
+ * Lightweight flatten: same as flattenArticle but skips extractExtraInfo and runPostProcessRules.
+ * Used for pagination/search where extracted::/processed:: fields aren't needed.
+ */
+export function flattenArticleLightweight(
+  input: Record<string, unknown>,
+  options: {
+    formatOptions?: UserFeedFormatOptions;
+    useParserRules?: PostProcessParserRule[];
+  }
 ): FlattenedArticleWithoutId {
   const flattened = flatten(input, {
     delimiter: ARTICLE_FIELD_DELIMITER,
@@ -89,7 +105,6 @@ export function flattenArticle(
       try {
         // eslint-disable-next-line no-control-regex
         if (/[^\x00-\x7F]/.test(requestedTimezone)) {
-          // Non-ASCII characters (e.g., Unicode minus) aren't valid in timezones
           throw new Error("Invalid timezone");
         }
         dateVal = dayjs(value).tz(requestedTimezone);
@@ -139,8 +154,20 @@ export function flattenArticle(
     newRecord[key] = String(value);
   });
 
-  // Extract images and anchors from HTML content
-  const entries = Object.entries(newRecord);
+  return newRecord;
+}
+
+/**
+ * Enrich a lightweight-flattened article with extracted:: and processed:: fields.
+ * Runs the expensive operations that flattenArticleLightweight skips.
+ */
+export function enrichFlattenedArticle(
+  flattened: FlattenedArticleWithoutId,
+  options: { useParserRules?: PostProcessParserRule[] }
+): FlattenedArticleWithoutId {
+  const enriched = { ...flattened };
+
+  const entries = Object.entries(flattened);
 
   for (let i = 0; i < entries.length; i++) {
     const [key, value] = entries[i]!;
@@ -149,20 +176,18 @@ export function flattenArticle(
 
     if (imageList.length) {
       for (let j = 0; j < imageList.length; j++) {
-        const image = imageList[j];
-        newRecord[`extracted::${key}::image${j + 1}`] = image!;
+        enriched[`extracted::${key}::image${j + 1}`] = imageList[j]!;
       }
     }
 
     if (anchorList.length) {
       for (let j = 0; j < anchorList.length; j++) {
-        const anchor = anchorList[j];
-        newRecord[`extracted::${key}::anchor${j + 1}`] = anchor!;
+        enriched[`extracted::${key}::anchor${j + 1}`] = anchorList[j]!;
       }
     }
   }
 
-  return runPostProcessRules(newRecord, options.useParserRules);
+  return runPostProcessRules(enriched, options.useParserRules);
 }
 
 /**
@@ -176,6 +201,7 @@ export async function parseArticlesFromXml(
     useParserRules?: PostProcessParserRule[];
     externalFeedProperties?: ExternalFeedProperty[];
     externalFetchFn?: ExternalFetchFn;
+    lightweight?: boolean;
   } = {}
 ): Promise<ParseArticlesResult> {
   const feedparser = new FeedParser({});
@@ -240,7 +266,10 @@ export async function parseArticlesFromXml(
             idType
           );
 
-          const flattened = flattenArticle(rawArticle as never, {
+          const flattenFn = options.lightweight
+            ? flattenArticleLightweight
+            : flattenArticle;
+          const flattened = flattenFn(rawArticle as never, {
             formatOptions: options.formatOptions,
             useParserRules: options.useParserRules,
           });
@@ -287,8 +316,9 @@ export async function parseArticlesFromXml(
           idHashes.add(idHash);
         }
 
-        // Inject external content if configured
+        // Inject external content if configured (skip in lightweight mode)
         if (
+          !options.lightweight &&
           options.externalFeedProperties?.length &&
           options.externalFetchFn &&
           mappedArticles.length <= MAX_ARTICLE_INJECTION_ARTICLE_COUNT

services/user-feeds-next/src/articles/parser/index.ts

@@ -1,6 +1,8 @@
 export {
   parseArticlesFromXml,
   flattenArticle,
+  flattenArticleLightweight,
+  enrichFlattenedArticle,
   FeedParseTimeoutException,
   InvalidFeedException,
 } from "./article-parser";

services/user-feeds-next/src/articles/parser/worker/feed-parser-pool.ts

@@ -40,6 +40,7 @@ export interface ParseArticlesOptions {
   useParserRules?: PostProcessParserRule[];
   externalFeedProperties?: ExternalFeedProperty[];
   externalFetchFn?: ExternalFetchFn;
+  lightweight?: boolean;
 }
 
 /**

services/user-feeds-next/src/feed-fetcher/feed-fetcher.ts

@@ -18,6 +18,29 @@ import {
 
 const API_KEY = process.env.USER_FEEDS_FEED_REQUESTS_API_KEY || "";
 
+function getErrorDetails(err: unknown): string {
+  const parts: string[] = [];
+  let current = err;
+
+  while (current instanceof Error) {
+    const e = current as Error & { code?: string };
+    let part = e.message;
+
+    if (e.code) {
+      part += ` [${e.code}]`;
+    }
+
+    parts.push(part);
+    current = e.cause;
+  }
+
+  if (current !== undefined && !(current instanceof Error)) {
+    parts.push(String(current));
+  }
+
+  return parts.join(" -> ");
+}
+
 export async function fetchFeed(
   url: string,
   options: {
@@ -54,15 +77,15 @@ export async function fetchFeed(
             accept: "application/json",
             "api-key": API_KEY,
           },
-        }),
+        } as RequestInit),
       {
         retries: options?.retries ?? 2,
         randomize: true,
       }
     );
   } catch (err) {
     throw new FeedRequestNetworkException(
-      `Failed to execute request to feed requests API: ${(err as Error).message}`
+      `Failed to execute request to feed requests API: ${getErrorDetails(err)}`
     );
   }
 
@@ -194,15 +217,15 @@ export async function fetchFeedForDeliveryPreview(
             accept: "application/json",
             "api-key": API_KEY,
           },
-        }),
+        } as RequestInit),
       {
         retries: 2,
         randomize: true,
       }
     );
   } catch (err) {
     throw new FeedRequestNetworkException(
-      `Failed to execute request to feed requests API: ${(err as Error).message}`
+      `Failed to execute request to feed requests API: ${getErrorDetails(err)}`
     );
   }
 

services/user-feeds-next/src/feeds/services/articles.service.ts

@@ -44,6 +44,7 @@ export interface FindOrFetchFeedArticlesOptions extends FetchFeedArticleOptions
   executeFetch?: boolean;
   executeFetchIfStale?: boolean;
   parsedArticlesCacheStore?: ParsedArticlesCacheStore;
+  lightweight?: boolean;
 }
 
 export interface FetchFeedArticlesResult {
@@ -123,8 +124,11 @@ export async function findOrFetchFeedArticles(
       dateTimezone: options.formatOptions?.dateTimezone,
       dateLocale: options.formatOptions?.dateLocale,
     },
-    externalFeedProperties: options.externalFeedProperties,
+    externalFeedProperties: options.lightweight
+      ? undefined
+      : options.externalFeedProperties,
     requestLookupDetails: options.requestLookupDetails ?? undefined,
+    lightweight: options.lightweight,
   };
 
   // Check cache first
@@ -177,6 +181,7 @@ export async function findOrFetchFeedArticles(
   try {
     const parsed = await parseArticlesFromXml(result.body, {
       formatOptions: options.formatOptions,
+      lightweight: options.lightweight,
     });
     articles = parsed.articles;
     feed = parsed.feed;
@@ -212,9 +217,9 @@ export async function findOrFetchFeedArticles(
     throw err;
   }
 
-  // Inject external content if external properties are specified
+  // Inject external content if external properties are specified (skip in lightweight mode)
   let externalContentErrors: ExternalContentError[] = [];
-  if (options.externalFeedProperties?.length) {
+  if (!options.lightweight && options.externalFeedProperties?.length) {
     externalContentErrors = await injectExternalContent(
       articles,
       options.externalFeedProperties,

services/user-feeds-next/src/feeds/services/feeds.service.ts

@@ -4,56 +4,48 @@
  */
 
 import type { Article } from "../../articles/parser";
-import {
-  evaluateExpression,
-  buildFilterReferences,
-  type LogicalExpression,
-} from "../../articles/filters";
 import { INJECTED_ARTICLE_PLACEHOLDER_PREFIX } from "../../shared/constants";
 import {
-  GetUserFeedArticlesFilterReturnType,
   SelectPropertyType,
   type CustomPlaceholder,
 } from "../../http/schemas";
 import { getNumbersInRange } from "./utils";
 
-export interface QueryForArticlesInput {
+export interface PaginateArticlesInput {
   articles: Article[];
   limit: number;
   skip: number;
   random?: boolean;
   selectProperties?: string[];
   selectPropertyTypes?: SelectPropertyType[];
+  customPlaceholders?: CustomPlaceholder[] | null;
   filters?: {
-    returnType?: GetUserFeedArticlesFilterReturnType;
-    expression?: Record<string, unknown>;
     articleId?: string;
     articleIdHashes?: string[];
     search?: string;
   };
-  customPlaceholders?: CustomPlaceholder[] | null;
 }
 
-export interface QueryForArticlesOutput {
+export interface PaginateArticlesOutput {
   articles: Article[];
-  properties: string[];
   totalArticles: number;
-  filterEvalResults?: Array<{ passed: boolean }>;
+  properties: string[];
 }
 
 /**
- * Query for articles with filtering, sorting, and pagination.
+ * Paginate articles: sort, filter by ID/search, and apply skip/limit.
+ * Does NOT trim properties or evaluate filter expressions — those happen after formatting.
  */
-export async function queryForArticles({
+export function paginateArticles({
   articles,
   limit,
   skip,
   random,
   selectProperties,
   selectPropertyTypes,
-  filters,
   customPlaceholders,
-}: QueryForArticlesInput): Promise<QueryForArticlesOutput> {
+  filters,
+}: PaginateArticlesInput): PaginateArticlesOutput {
   const placeholdersFromCustomPlaceholders =
     customPlaceholders?.map((c) => c.sourcePlaceholder) || [];
   const properties = queryForArticleProperties(
@@ -63,12 +55,7 @@ export async function queryForArticles({
   );
 
   if (articles.length === 0) {
-    return {
-      articles: [],
-      properties,
-      totalArticles: 0,
-      filterEvalResults: [],
-    };
+    return { articles: [], properties, totalArticles: 0 };
   }
 
   // Sort by date, latest first
@@ -106,7 +93,7 @@ export async function queryForArticles({
     const filtersSearch = filters?.search;
 
     if (filtersSearch && typeof filtersSearch === "string") {
-      matchedArticles = articles.filter((article) => {
+      matchedArticles = matchedArticles.filter((article) => {
         return properties.some((property) =>
           article.flattened[property]
             ?.toLowerCase()
@@ -134,54 +121,10 @@ export async function queryForArticles({
     }
   }
 
-  // Trim articles to only include selected properties
-  const matchedArticlesWithProperties = matchedArticles.map((article) => {
-    const trimmed: Article = {
-      ...article,
-      flattened: {
-        id: article.flattened.id,
-        idHash: article.flattened.idHash,
-      },
-      raw: article.raw,
-    };
-
-    properties.forEach((property) => {
-      trimmed.flattened[property] = article.flattened[property] || "";
-    });
-
-    return trimmed;
-  });
-
-  // Evaluate filter expressions if requested
-  let filterEvalResults: Array<{ passed: boolean }> | undefined;
-
-  if (
-    filters?.returnType ===
-    GetUserFeedArticlesFilterReturnType.IncludeEvaluationResults
-  ) {
-    if (filters.expression) {
-      filterEvalResults = await Promise.all(
-        matchedArticles.map(async (article) => {
-          const { result: passed } = evaluateExpression(
-            filters.expression as unknown as LogicalExpression,
-            buildFilterReferences(article)
-          );
-
-          return {
-            passed,
-          };
-        })
-      );
-    } else {
-      filterEvalResults = matchedArticles.map(() => ({ passed: true }));
-    }
-  }
-
   return {
-    articles: matchedArticlesWithProperties,
+    articles: matchedArticles,
     totalArticles: totalMatchedArticles,
     properties,
-    filterEvalResults,
   };
 }