move paging functions ro rest_list

Signed-off-by: Laurent Martin <laurent.martin.l@gmail.com>

Author: laurent-martin

lib/aspera/api/aoc.rb

@@ -157,8 +157,8 @@ def call_paging(query: {})
           Aspera.assert(block_given?)
           # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
           query['per_page'] = 1000 unless query.key?('per_page')
-          max_items = query.delete(Rest::MAX_ITEMS)
-          max_pages = query.delete(Rest::MAX_PAGES)
+          max_items = query.delete(RestList::MAX_ITEMS)
+          max_pages = query.delete(RestList::MAX_PAGES)
           item_list = []
           total_count = nil
           current_page = query['page']

lib/aspera/api/node.rb

@@ -521,6 +521,46 @@ def transfer_spec_gen4(file_id, direction, ts_merge = nil)
         return transfer_spec
       end
 
+      # Executes `GET` call in loop using `iteration_token` (`/ops/transfers`)
+      # @param iteration [Array]   a single element array with the iteration token or nil
+      # @param call_args [Hash]    additional arguments to pass to `Rest.call`
+      # @return [Array] list of items returned by the API call
+      def read_with_paging(subpath, query = nil, iteration: nil, **call_args)
+        Aspera.assert_type(iteration, Array, NilClass){'iteration'}
+        Aspera.assert_type(query, Hash, NilClass){'query'}
+        Aspera.assert(!call_args.key?(:query))
+        query = {} if query.nil?
+        query[:iteration_token] = iteration[0] unless iteration.nil?
+        max = query.delete(RestList::MAX_ITEMS)
+        item_list = []
+        loop do
+          data, http = read(subpath, query, **call_args, ret: :both)
+          Aspera.assert_type(data, Array){"Expected data to be an Array, got: #{data.class}"}
+          # no data
+          break if data.empty?
+          # get next iteration token from link
+          next_iteration_token = nil
+          link_info = http['Link']
+          unless link_info.nil?
+            m = link_info.match(/<([^>]+)>/)
+            Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
+            next_iteration_token = Rest.query_to_h(URI.parse(m[1]).query)['iteration_token']
+          end
+          # same as last iteration: stop
+          break if next_iteration_token&.eql?(query[:iteration_token])
+          query[:iteration_token] = next_iteration_token
+          item_list.concat(data)
+          if max&.<=(item_list.length)
+            item_list = item_list.slice(0, max)
+            break
+          end
+          break if next_iteration_token.nil?
+        end
+        # save iteration token if needed
+        iteration[0] = query[:iteration_token] unless iteration.nil?
+        item_list
+      end
+
       private
 
       # Method called in loop for each entry for `resolve_api_fid`

lib/aspera/cli/plugins/base.rb

@@ -14,10 +14,6 @@ class Base
         INSTANCE_OPS = %i[modify delete show].freeze
         # All standard operations (create list modify delete show)
         ALL_OPS = (GLOBAL_OPS + INSTANCE_OPS).freeze
-        # Special query parameter: `max`: max number of items for list command
-        MAX_ITEMS = 'max'
-        # Special query parameter: `pmax`: max number of pages for list command
-        MAX_PAGES = 'pmax'
 
         class << self
           def declare_options(options)
@@ -190,7 +186,7 @@ def entity_execute(
             return Main.result_single_object(api.read(one_res_path), fields: display_fields)
           when :list
             if tclo
-              data, total = api.list_entities_limit_offset_total_count(entity:, items_key: items_key, query: query_read_delete(default: list_query))
+              data, total = api.list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query_read_delete(default: list_query))
               return Main.result_object_list(data, total: total, fields: display_fields)
             end
             data, http = api.read(entity, query_read_delete, ret: :both)

lib/aspera/cli/plugins/faspex.rb

@@ -9,6 +9,7 @@
 require 'aspera/transfer/uri'
 require 'aspera/transfer/spec'
 require 'aspera/persistency_action_once'
+require 'aspera/rest_list'
 require 'aspera/environment'
 require 'aspera/nagios'
 require 'aspera/id_generator'
@@ -31,7 +32,7 @@ class Faspex < BasicAuth
         # allowed parameters for inbox.atom
         ATOM_PARAMS = %w[page count startIndex].freeze
         # with special parameters (from Plugin class) : max and pmax (from Plugin)
-        ATOM_EXT_PARAMS = [MAX_ITEMS, MAX_PAGES].concat(ATOM_PARAMS).freeze
+        ATOM_EXT_PARAMS = [RestList::MAX_ITEMS, RestList::MAX_PAGES].concat(ATOM_PARAMS).freeze
         # sub path in url for public link delivery
         PUB_LINK_EXTERNAL_MATCH = 'external_deliveries/'
         STANDARD_PATH = '/aspera/faspex'
@@ -176,10 +177,9 @@ def mailbox_filtered_entries(stop_at_id: nil)
             Aspera.assert_type(mailbox_query, Hash){'query'}
             Aspera.assert((mailbox_query.keys - ATOM_EXT_PARAMS).empty?){"query: supported params: #{ATOM_EXT_PARAMS}"}
             Aspera.assert(!(mailbox_query.key?('startIndex') && mailbox_query.key?('page'))){'query: startIndex and page are exclusive'}
-            max_items = mailbox_query[MAX_ITEMS]
-            mailbox_query.delete(MAX_ITEMS)
-            max_pages = mailbox_query[MAX_PAGES]
-            mailbox_query.delete(MAX_PAGES)
+            # Extract pagination control parameters (not part of API query)
+            max_items = mailbox_query.delete(RestList::MAX_ITEMS)
+            max_pages = mailbox_query.delete(RestList::MAX_PAGES)
           end
           loop do
             # get a batch of package information

lib/aspera/cli/plugins/faspex5.rb

@@ -332,7 +332,7 @@ def browse_folder(browse_endpoint, base_query = {})
           Aspera.assert_type(filters, Hash)
           filters['basenames'] ||= []
           Aspera.assert_type(filters, Hash){'filters'}
-          max_items = query.delete(MAX_ITEMS)
+          max_items = query.delete(RestList::MAX_ITEMS)
           recursive = query.delete('recursive')
           use_paging = query.delete('paging'){true}
           if use_paging

lib/aspera/cli/plugins/node.rb

@@ -11,6 +11,7 @@
 require 'aspera/api/node'
 require 'aspera/oauth'
 require 'aspera/node_simulator'
+require 'aspera/rest_list'
 require 'aspera/assert'
 require 'base64'
 require 'zlib'
@@ -166,7 +167,7 @@ def wizard(wizard, app_url)
         # @param api [Rest] an existing API object for the Node API
         # @param prefix_path [String,nil] for Faspex 4, allows browsing a package without full path in node (removes storage prefix)
         def initialize(context:, api: nil, prefix_path: nil)
-          @prefixer = prefix_path ? NodePathPrefix.new(prefix_path) : nil
+          @node_path_prefix = prefix_path ? NodePathPrefix.new(prefix_path) : nil
           super(context: context, basic_options: api.nil?)
           Node.declare_options(options)
           return if context.only_manual?
@@ -196,11 +197,11 @@ def initialize(context:, api: nil, prefix_path: nil)
         # Gen3 API
         def browse_gen3
           folders_to_process = options.get_next_argument('path', validation: String)
-          folders_to_process = @prefixer.add_to_path(folders_to_process) unless @prefixer.nil?
+          folders_to_process = @node_path_prefix.add_to_path(folders_to_process) unless @node_path_prefix.nil?
           folders_to_process = [folders_to_process]
           query = options.get_option(:query) || {}
           # special parameter: max number of entries in result
-          max_items = query.delete(MAX_ITEMS)
+          max_items = query.delete(RestList::MAX_ITEMS)
           # special parameter: recursive browsing
           recursive = query.delete('recursive')
           # special parameter: only return one entry for the path, even if folder
@@ -221,7 +222,7 @@ def browse_gen3
               response = @api_node.create('files/browse', query)
               # 'file','symbolic_link'
               if !Node.gen3_entry_folder?(response['self']) || only_path
-                @prefixer&.remove_in_object_list!([response['self']])
+                @node_path_prefix&.remove_in_object_list!([response['self']])
                 return Main.result_single_object(response['self'])
               end
               items = response['items']
@@ -243,7 +244,7 @@ def browse_gen3
             end
             query.delete('skip')
           end
-          @prefixer&.remove_in_object_list!(all_items)
+          @node_path_prefix&.remove_in_object_list!(all_items)
           return Main.result_object_list(all_items)
         ensure
           RestParameters.instance.spinner_cb.call(action: :success)
@@ -286,12 +287,12 @@ def execute_command_gen3(command)
           when :delete
             # TODO: add query for recursive
             paths_to_delete = options.get_next_argument('file list', multiple: true)
-            @prefixer&.add_to_paths!(paths_to_delete)
+            @node_path_prefix&.add_to_paths!(paths_to_delete)
             resp = @api_node.create('files/delete', {paths: paths_to_delete.map{ |i| {'path' => i.start_with?('/') ? i : "/#{i}"}}})
             return cli_result_from_paths_response(resp, 'file deleted')
           when :search
             search_root = options.get_next_argument('search root', validation: String)
-            search_root = @prefixer.add_to_path(search_root) unless @prefixer.nil?
+            search_root = @node_path_prefix.add_to_path(search_root) unless @node_path_prefix.nil?
             parameters = {'path' => search_root}
             other_options = options.get_option(:query)
             parameters.merge!(other_options) unless other_options.nil?
@@ -300,40 +301,40 @@ def execute_command_gen3(command)
             fields = resp['items'].first.keys.reject{ |i| SEARCH_REMOVE_FIELDS.include?(i)}
             formatter.display_item_count(resp['item_count'], resp['total_count'])
             formatter.display_status("params: #{resp['parameters'].keys.map{ |k| "#{k}:#{resp['parameters'][k]}"}.join(',')}")
-            @prefixer&.remove_in_object_list!(resp['items'])
+            @node_path_prefix&.remove_in_object_list!(resp['items'])
             return Main.result_object_list(resp['items'], fields: fields)
           when :space
             path_list = options.get_next_argument('folder path or ext.val. list', multiple: true)
-            @prefixer&.add_to_paths!(path_list)
+            @node_path_prefix&.add_to_paths!(path_list)
             resp = @api_node.create('space', {'paths' => path_list.map{ |i| {path: i}}})
-            @prefixer&.remove_in_object_list!(resp['paths'])
+            @node_path_prefix&.remove_in_object_list!(resp['paths'])
             return Main.result_object_list(resp['paths'])
           when :mkdir
             path_list = options.get_next_argument('folder path or ext.val. list', multiple: true)
-            @prefixer&.add_to_paths!(path_list)
+            @node_path_prefix&.add_to_paths!(path_list)
             resp = @api_node.create('files/create', {'paths' => path_list.map{ |i| {type: :directory, path: i}}})
             return cli_result_from_paths_response(resp, 'folder created')
           when :mklink
             target = options.get_next_argument('target', validation: String)
-            target = @prefixer.add_to_path(target) unless @prefixer.nil?
+            target = @node_path_prefix.add_to_path(target) unless @node_path_prefix.nil?
             one_path = options.get_next_argument('link path', validation: String)
-            one_path = @prefixer.add_to_path(one_path) unless @prefixer.nil?
+            one_path = @node_path_prefix.add_to_path(one_path) unless @node_path_prefix.nil?
             resp = @api_node.create('files/create', {'paths' => [{type: :symbolic_link, path: one_path, target: {path: target}}]})
             return cli_result_from_paths_response(resp, 'link created')
           when :mkfile
             one_path = options.get_next_argument('file path', validation: String)
-            one_path = @prefixer.add_to_path(one_path) unless @prefixer.nil?
+            one_path = @node_path_prefix.add_to_path(one_path) unless @node_path_prefix.nil?
             contents64 = Base64.strict_encode64(options.get_next_argument('contents'))
             resp = @api_node.create('files/create', {'paths' => [{type: :file, path: one_path, contents: contents64}]})
             return cli_result_from_paths_response(resp, 'file created')
           when :rename
             # TODO: multiple ?
             path_base = options.get_next_argument('path_base', validation: String)
-            path_base = @prefixer.add_to_path(path_base) unless @prefixer.nil?
+            path_base = @node_path_prefix.add_to_path(path_base) unless @node_path_prefix.nil?
             path_src = options.get_next_argument('path_src', validation: String)
-            path_src = @prefixer.add_to_path(path_src) unless @prefixer.nil?
+            path_src = @node_path_prefix.add_to_path(path_src) unless @node_path_prefix.nil?
             path_dst = options.get_next_argument('path_dst', validation: String)
-            path_dst = @prefixer.add_to_path(path_dst) unless @prefixer.nil?
+            path_dst = @node_path_prefix.add_to_path(path_dst) unless @node_path_prefix.nil?
             resp = @api_node.create('files/rename', {'paths' => [{'path' => path_base, 'source' => path_src, 'destination' => path_dst}]})
             return cli_result_from_paths_response(resp, 'entry moved')
           when :browse
@@ -378,7 +379,7 @@ def execute_command_gen3(command)
             return Main.result_transfer(transfer.start(transfer_spec))
           when :cat
             remote_path = options.get_next_argument('remote path', validation: String)
-            remote_path = @prefixer.add_to_path(remote_path) unless @prefixer.nil?
+            remote_path = @node_path_prefix.add_to_path(remote_path) unless @node_path_prefix.nil?
             File.basename(remote_path)
             http = @api_node.read("files/#{URI.encode_www_form_component(remote_path)}/contents", ret: :resp)
             return Main.result_text(http.body)
@@ -882,10 +883,10 @@ def execute_action(command = nil)
                   iteration_persistency.save
                   return Main.result_status('Persistency reset')
                 end
+              else
+                Aspera.assert(!transfer_filter.key?('reset'), type: Cli::BadArgument){'reset only with once_only'}
               end
-              raise Cli::BadArgument, 'reset only with once_only' if transfer_filter.key?('reset') && iteration_persistency.nil?
-              max_items = transfer_filter.delete(MAX_ITEMS)
-              transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', max: max_items, query: transfer_filter, iteration: iteration_persistency&.data)
+              transfers_data = @api_node.read_with_paging('ops/transfers', transfer_filter, iteration: iteration_persistency&.data)
               iteration_persistency&.save
               return Main.result_object_list(transfers_data, fields: %w[id status start_spec.direction start_spec.remote_user start_spec.remote_host start_spec.destination_path])
             when :sessions
@@ -1092,7 +1093,7 @@ def execute_action(command = nil)
             }
             loop do
               timestamp = Time.now
-              transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', query: {active_only: true})
+              transfers_data = @api_node.read_with_paging('ops/transfers', {active_only: true})
               datapoint[:asInt] = transfers_data.length
               datapoint[:timeUnixNano] = timestamp.to_i * 1_000_000_000 + timestamp.nsec
               Log.log.info("#{datapoint[:asInt]} active transfers")
@@ -1130,50 +1131,9 @@ def response_to_result(response, success_msg)
         # Translates paths results into CLI result, and removes prefix
         def cli_result_from_paths_response(response, success_msg)
           obj_list = response_to_result(response, success_msg)
-          @prefixer&.remove_in_object_list!(obj_list)
+          @node_path_prefix&.remove_in_object_list!(obj_list)
           return Main.result_object_list(obj_list, fields: %w[path result])
         end
-
-        # Executes the provided API call in loop
-        # @param api       [Rest]    the API to call
-        # @param iteration [Array]   a single element array with the iteration token or nil
-        # @param max       [Integer] maximum number of items to return, or nil for no limit
-        # @param query     [Hash]    query parameters to use for the API call
-        # @param call_args [Hash]    additional arguments to pass to the API call
-        # @return [Array] list of items returned by the API call
-        def call_with_iteration(api:, iteration: nil, max: nil, query: nil, **call_args)
-          Aspera.assert_type(iteration, Array, NilClass){'iteration'}
-          Aspera.assert_type(query, Hash, NilClass){'query'}
-          query_token = query&.dup || {}
-          item_list = []
-          query_token[:iteration_token] = iteration[0] unless iteration.nil?
-          loop do
-            data, http = api.call(**call_args, query: query_token, ret: :both)
-            Aspera.assert_type(data, Array){"Expected data to be an Array, got: #{data.class}"}
-            # no data
-            break if data.empty?
-            # get next iteration token from link
-            next_iteration_token = nil
-            link_info = http['Link']
-            unless link_info.nil?
-              m = link_info.match(/<([^>]+)>/)
-              Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
-              next_iteration_token = Rest.query_to_h(URI.parse(m[1]).query)['iteration_token']
-            end
-            # same as last iteration: stop
-            break if next_iteration_token&.eql?(query_token[:iteration_token])
-            query_token[:iteration_token] = next_iteration_token
-            item_list.concat(data)
-            if max&.<=(item_list.length)
-              item_list = item_list.slice(0, max)
-              break
-            end
-            break if next_iteration_token.nil?
-          end
-          # save iteration token if needed
-          iteration[0] = query_token[:iteration_token] unless iteration.nil?
-          item_list
-        end
       end
     end
   end

lib/aspera/rest.rb

@@ -72,11 +72,6 @@ def json?(mime)
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # Special query parameter: max number of items for list command
-    MAX_ITEMS = 'max'
-    # Special query parameter: max number of pages for list command
-    MAX_PAGES = 'pmax'
-
     class << self
       # @return [String] Basic auth token
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end

lib/aspera/rest_list.rb

@@ -2,8 +2,13 @@
 
 module Aspera
   # List and lookup methods for Rest
-  # include in classes inheriting Rest
+  # To be included in classes inheriting Rest that require those methods.
   module RestList
+    # `max`: special query parameter: max number of items for list command
+    MAX_ITEMS = 'max'
+    # `pmax`: special query parameter: max number of pages for list command
+    MAX_PAGES = 'pmax'
+
     # Query entity by general search (read with parameter `q`)
     #
     # @param subpath     [String] Path of entity in API