fix: pagination returns less than the page size when decryption fails (#1547)

Author: rodrigopavezi

packages/data-access/src/combined-data-access.ts

@@ -31,18 +31,14 @@ export abstract class CombinedDataAccess implements DataAccessTypes.IDataAccess
   async getChannelsByTopic(
     topic: string,
     updatedBetween?: DataAccessTypes.ITimestampBoundaries | undefined,
-    page?: number,
-    pageSize?: number,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    return await this.reader.getChannelsByTopic(topic, updatedBetween, page, pageSize);
+    return await this.reader.getChannelsByTopic(topic, updatedBetween);
   }
   async getChannelsByMultipleTopics(
     topics: string[],
     updatedBetween?: DataAccessTypes.ITimestampBoundaries,
-    page?: number | undefined,
-    pageSize?: number | undefined,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween, page, pageSize);
+    return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween);
   }
   async persistTransaction(
     transactionData: DataAccessTypes.ITransaction,

packages/data-access/src/data-read.ts

@@ -49,26 +49,15 @@ export class DataAccessRead implements DataAccessTypes.IDataRead {
   async getChannelsByTopic(
     topic: string,
     updatedBetween?: DataAccessTypes.ITimestampBoundaries | undefined,
-    page?: number | undefined,
-    pageSize?: number | undefined,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    return this.getChannelsByMultipleTopics([topic], updatedBetween, page, pageSize);
+    return this.getChannelsByMultipleTopics([topic], updatedBetween);
   }
 
   async getChannelsByMultipleTopics(
     topics: string[],
     updatedBetween?: DataAccessTypes.ITimestampBoundaries,
-    page?: number,
-    pageSize?: number,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    // Validate pagination parameters
-    if (page !== undefined && page < 1) {
-      throw new Error(`Page number must be greater than or equal to 1, but it is ${page}`);
-    }
-    if (pageSize !== undefined && pageSize < 1) {
-      throw new Error(`Page size must be greater than 0, but it is ${pageSize}`);
-    }
-
+    const result = await this.storage.getTransactionsByTopics(topics);
     const pending = this.pendingStore?.findByTopics(topics) || [];
 
     const pendingItems = pending.map((item) => ({
@@ -84,33 +73,6 @@ export class DataAccessRead implements DataAccessTypes.IDataRead {
       topics: item.topics || [],
     }));
 
-    // Calculate adjusted pagination
-    let adjustedPage = page;
-    let adjustedPageSize = pageSize;
-    let pendingItemsOnCurrentPage = 0;
-    if (page !== undefined && pageSize !== undefined) {
-      const totalPending = pendingItems.length;
-      const itemsPerPage = (page - 1) * pageSize;
-
-      if (totalPending > itemsPerPage) {
-        pendingItemsOnCurrentPage = Math.min(totalPending - itemsPerPage, pageSize);
-        adjustedPageSize = pageSize - pendingItemsOnCurrentPage;
-        adjustedPage = 1;
-        if (adjustedPageSize === 0) {
-          adjustedPageSize = 1;
-          pendingItemsOnCurrentPage--;
-        }
-      } else {
-        adjustedPage = page - Math.floor(totalPending / pageSize);
-      }
-    }
-
-    const result = await this.storage.getTransactionsByTopics(
-      topics,
-      adjustedPage,
-      adjustedPageSize,
-    );
-
     const transactions = result.transactions.concat(...pendingItems);
 
     // list of channels having at least one tx updated during the updatedBetween boundaries
@@ -138,17 +100,6 @@ export class DataAccessRead implements DataAccessTypes.IDataRead {
           prev[curr.channelId].push(curr.hash);
           return prev;
         }, {} as Record<string, string[]>),
-        pagination:
-          page && pageSize
-            ? {
-                total: result.transactions.length + pendingItems.length,
-                page,
-                pageSize,
-                hasMore:
-                  (page - 1) * pageSize + filteredTxs.length - pendingItemsOnCurrentPage <
-                  result.transactions.length,
-              }
-            : undefined,
       },
       result: {
         transactions: filteredTxs.reduce((prev, curr) => {

packages/data-access/src/in-memory-indexer.ts

@@ -55,38 +55,11 @@ export class InMemoryIndexer implements StorageTypes.IIndexer {
     };
   }
 
-  async getTransactionsByTopics(
-    topics: string[],
-    page?: number,
-    pageSize?: number,
-  ): Promise<StorageTypes.IGetTransactionsResponse> {
-    if (page !== undefined && page < 1) {
-      throw new Error('Page must be greater than or equal to 1');
-    }
-    if (pageSize !== undefined && pageSize <= 0) {
-      throw new Error('Page size must be greater than 0');
-    }
-
+  async getTransactionsByTopics(topics: string[]): Promise<StorageTypes.IGetTransactionsResponse> {
     // Efficiently get total count without creating intermediate array
     const channelIdsSet = new Set(topics.flatMap((topic) => this.#topicToChannelsIndex.get(topic)));
-    const total = channelIdsSet.size;
-    let channelIds = Array.from(channelIdsSet);
+    const channelIds = Array.from(channelIdsSet);
 
-    if (page && pageSize) {
-      const start = (page - 1) * pageSize;
-      // Return empty result if page exceeds available data
-      if (start >= total) {
-        return {
-          blockNumber: 0,
-          transactions: [],
-          pagination:
-            page && pageSize
-              ? { total, page, pageSize, hasMore: page * pageSize < total }
-              : undefined,
-        };
-      }
-      channelIds = channelIds.slice(start, start + pageSize);
-    }
     const locations = channelIds
       .map((channel) => this.#channelToLocationsIndex.get(channel))
       .flat();

packages/integration-test/test/node-client.test.ts

@@ -273,7 +273,8 @@ describe('Request client using a request node', () => {
     await waitForConfirmation(requestDataCancel);
 
     // get requests without boundaries
-    let requests = await requestNetwork.fromTopic(topicsRequest1and2[0]);
+    const response = await requestNetwork.fromTopic(topicsRequest1and2[0]);
+    let requests = Array.isArray(response) ? response : response.requests;
     expect(requests.length).toBe(2);
     expect(requests[0].requestId).toBe(request1.requestId);
     expect(requests[1].requestId).toBe(request2.requestId);
@@ -286,9 +287,10 @@ describe('Request client using a request node', () => {
     expect(requestData2.state).toBe(Types.RequestLogic.STATE.CREATED);
 
     // get requests with boundaries
-    requests = await requestNetwork.fromTopic(topicsRequest1and2[0], {
+    const result = await requestNetwork.fromTopic(topicsRequest1and2[0], {
       to: timestampBeforeReduce,
     });
+    requests = Array.isArray(result) ? result : result.requests;
     expect(requests.length).toBe(1);
     expect(requests[0].requestId).toBe(request1.requestId);
 
@@ -341,9 +343,10 @@ describe('Request client using a request node', () => {
     await new Promise((r) => setTimeout(r, 1500));
 
     // get requests with boundaries
-    const requests = await requestNetwork.fromIdentity(payerSmartContract, {
+    const result = await requestNetwork.fromIdentity(payerSmartContract, {
       from: timestampCreation,
     });
+    const requests = Array.isArray(result) ? result : result.requests;
     expect(requests.length).toBe(1);
     expect(requests[0].requestId).toBe(request1.requestId);
   });

packages/request-client.js/src/api/request-network.ts

@@ -17,7 +17,7 @@ import {
   SignatureProviderTypes,
   TransactionTypes,
 } from '@requestnetwork/types';
-import { deepCopy, supportedIdentities, validatePaginationParams } from '@requestnetwork/utils';
+import { deepCopy, supportedIdentities } from '@requestnetwork/utils';
 import { CurrencyManager, UnsupportedCurrencyError } from '@requestnetwork/currency';
 import * as Types from '../types';
 import ContentDataExtension from './content-data-extension';
@@ -294,7 +294,9 @@ export default class RequestNetwork {
       page?: number;
       pageSize?: number;
     },
-  ): Promise<Request[]> {
+  ): Promise<
+    Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
+  > {
     if (!this.supportedIdentities.includes(identity.type)) {
       throw new Error(`${identity.type} is not supported`);
     }
@@ -317,7 +319,9 @@ export default class RequestNetwork {
       page?: number;
       pageSize?: number;
     },
-  ): Promise<Request[]> {
+  ): Promise<
+    Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
+  > {
     const identityNotSupported = identities.find(
       (identity) => !this.supportedIdentities.includes(identity.type),
     );
@@ -345,9 +349,9 @@ export default class RequestNetwork {
       page?: number;
       pageSize?: number;
     },
-  ): Promise<Request[]> {
-    validatePaginationParams(options?.page, options?.pageSize);
-
+  ): Promise<
+    Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
+  > {
     // Gets all the requests indexed by the value of the identity
     const requestsAndMeta: RequestLogicTypes.IReturnGetRequestsByTopic =
       await this.requestLogic.getRequestsByTopic(
@@ -389,8 +393,16 @@ export default class RequestNetwork {
         return request;
       },
     );
-
-    return Promise.all(requestPromises);
+    const requests = await Promise.all(requestPromises);
+
+    if (options?.page && options?.pageSize) {
+      return {
+        requests,
+        meta: requestsAndMeta.meta,
+      };
+    } else {
+      return requests;
+    }
   }
 
   /**
@@ -409,9 +421,9 @@ export default class RequestNetwork {
       page?: number;
       pageSize?: number;
     },
-  ): Promise<Request[]> {
-    validatePaginationParams(options?.page, options?.pageSize);
-
+  ): Promise<
+    Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
+  > {
     // Gets all the requests indexed by the value of the identity
     const requestsAndMeta: RequestLogicTypes.IReturnGetRequestsByTopic =
       await this.requestLogic.getRequestsByMultipleTopics(
@@ -454,8 +466,15 @@ export default class RequestNetwork {
         return request;
       },
     );
-
-    return Promise.all(requestPromises);
+    const requests = await Promise.all(requestPromises);
+    if (options?.page && options?.pageSize) {
+      return {
+        requests,
+        meta: requestsAndMeta.meta,
+      };
+    } else {
+      return requests;
+    }
   }
 
   /*

packages/request-client.js/src/http-data-access.ts

@@ -110,10 +110,8 @@ export default class HttpDataAccess extends CombinedDataAccess {
   public async getChannelsByTopic(
     topic: string,
     updatedBetween?: DataAccessTypes.ITimestampBoundaries,
-    page?: number,
-    pageSize?: number,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    return await this.reader.getChannelsByTopic(topic, updatedBetween, page, pageSize);
+    return await this.reader.getChannelsByTopic(topic, updatedBetween);
   }
 
   /**
@@ -125,10 +123,8 @@ export default class HttpDataAccess extends CombinedDataAccess {
   public async getChannelsByMultipleTopics(
     topics: string[],
     updatedBetween?: DataAccessTypes.ITimestampBoundaries,
-    page?: number,
-    pageSize?: number,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween, page, pageSize);
+    return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween);
   }
 
   /**

packages/request-client.js/src/http-data-read.ts

@@ -1,5 +1,4 @@
 import { DataAccessTypes } from '@requestnetwork/types';
-import { validatePaginationParams } from '@requestnetwork/utils';
 import { HttpDataAccessConfig } from './http-data-access-config';
 
 export class HttpDataRead implements DataAccessTypes.IDataRead {
@@ -53,16 +52,10 @@ export class HttpDataRead implements DataAccessTypes.IDataRead {
   public async getChannelsByTopic(
     topic: string,
     updatedBetween?: DataAccessTypes.ITimestampBoundaries,
-    page?: number,
-    pageSize?: number,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    validatePaginationParams(page, pageSize);
-
     const params = {
       topic,
       updatedBetween,
-      ...(page !== undefined && { page }),
-      ...(pageSize !== undefined && { pageSize }),
     };
 
     return await this.dataAccessConfig.fetchAndRetry('/getChannelsByTopic', params);
@@ -77,16 +70,10 @@ export class HttpDataRead implements DataAccessTypes.IDataRead {
   public async getChannelsByMultipleTopics(
     topics: string[],
     updatedBetween?: DataAccessTypes.ITimestampBoundaries,
-    page?: number,
-    pageSize?: number,
   ): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
-    validatePaginationParams(page, pageSize);
-
     return await this.dataAccessConfig.fetchAndRetry('/getChannelsByMultipleTopics', {
       topics,
       updatedBetween,
-      page,
-      pageSize,
     });
   }
 }