From 5ed3718a1af20f38c9d1fc7aa434b922540e838d Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 7 May 2026 19:28:27 +0000 Subject: [PATCH 01/10] feat(dev): api tokens; - Dump and load user api tokens. --- ckanapi/cli/dump.py | 20 ++++++++++++ ckanapi/cli/load.py | 74 +++++++++++++++++++++++++++++++++------------ ckanapi/cli/main.py | 7 +++-- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py index 9c6870a..ce1968c 100644 --- a/ckanapi/cli/dump.py +++ b/ckanapi/cli/dump.py @@ -204,8 +204,11 @@ def reply(error, record=None): if thing == 'datasets' and arguments['--resource-views']: for res in obj.get('resources', []): populate_res_views(ckan, res) + if thing == 'users' and arguments['--api-tokens']: + populate_api_tokens(ckan, obj) reply(None, obj) + def _worker_command_line(thing, arguments): """ Create a worker command line suitable for Popen with only the @@ -227,6 +230,7 @@ def b(name): + b('--datastore-fields') + b('--resource-views') + b('--include-users') + + b('--api-tokens') + ['value-here-to-make-docopt-happy'] ) @@ -248,3 +252,19 @@ def populate_res_views(ckan, res): return # return if the resource views list is empty res['resource_views'] = views + +def populate_api_tokens(ckan, user): + """ + Update user dict in-place with api_tokens_list + """ + try: + tokens = ckan.call_action('api_token_list', { + 'user_id': user['name'], + 'limit': 0}) + except CKANAPIError: + return + except NotFound: + return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass + if not tokens: + return # return if the user api token list is empty + user['api_tokens_list'] = tokens diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 6c1a9b9..7e1db55 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -110,6 +110,18 @@ def line_reader(): return 3 +def reply(action, error, response, out): + """ + format messages to be sent back to parent process + """ + out.write(compact_json([ + datetime.now().isoformat(), + action, + error, + response]) + b'\n') + out.flush() + + def load_things_worker(ckan, thing, arguments, stdin=None, stdout=None): """ @@ -144,23 +156,12 @@ def load_things_worker(ckan, thing, arguments, 'related_show','related_create','related_update'), }[thing] - def reply(action, error, response): - """ - format messages to be sent back to parent process - """ - stdout.write(compact_json([ - datetime.now().isoformat(), - action, - error, - response]) + b'\n') - stdout.flush() - for line in iter(stdin.readline, b''): try: obj = json.loads(line.decode('utf-8')) except UnicodeDecodeError as e: obj = None - reply('read', 'UnicodeDecodeError', str(e)) + reply('read', 'UnicodeDecodeError', str(e), stdout) continue requests_kwargs = None @@ -184,7 +185,7 @@ def reply(action, error, response): except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e)) + reply('show', 'NotAuthorized', str(e), stdout) continue name = obj.get('name') if not existing and name: @@ -194,7 +195,7 @@ def reply(action, error, response): except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e)) + reply('show', 'NotAuthorized', str(e), stdout) continue if existing: @@ -203,7 +204,7 @@ def reply(action, error, response): # FIXME: compare and reply when 'unchanged'? if not existing and arguments['--update-only']: - reply('show', 'NotFound', [obj.get('id'), obj.get('name')]) + reply('show', 'NotFound', [obj.get('id'), obj.get('name')], stdout) continue act = 'update' if existing else 'create' @@ -223,16 +224,19 @@ def reply(action, error, response): obj['users'] = users ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) + elif thing == 'users' and arguments['--api-tokens'] and obj.get('api_tokens_list'): # check if it is needed to create user api tokens when creating/updating users + _load_user_api_tokens(ckan, obj, arguments, stdout) except ValidationError as e: - reply(act, 'ValidationError', e.error_dict) + reply(act, 'ValidationError', e.error_dict, stdout) except SearchIndexError as e: - reply(act, 'SearchIndexError', str(e)) + reply(act, 'SearchIndexError', str(e), stdout) except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e)) + reply(act, 'NotAuthorized', str(e), stdout) except NotFound: - reply(act, 'NotFound', obj) + reply(act, 'NotFound', obj, stdout) else: - reply(act, None, r.get('name',r.get('id'))) + reply(act, None, r.get('name',r.get('id')), stdout) + def _worker_command_line(thing, arguments): """ @@ -255,6 +259,7 @@ def b(name): + b('--update-only') + b('--upload-resources') + b('--upload-logo') + + b('--api-tokens') ) @@ -309,3 +314,32 @@ def _upload_logo(ckan,obj_orig): obj['image_upload'] = (new_url, f.raw) ckan.action.group_update(**obj) return obj + + +def _load_user_api_tokens(ckan, obj, arguments, stdout): + """ + Loads user API Tokens from api_tokens_list + """ + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + act = 'create-api-token' + for token in obj['api_tokens_list']: + args = { + 'id': token['id'], + 'created_at': token['created_at'], + 'last_access': token['last_access'], + 'name': token['name'], + 'user': token['user_id'] + } + try: + r = ckan.call_action('api_token_create', + args, requests_kwargs=requests_kwargs) + except ValidationError as e: + reply(act, 'ValidationError', e.error_dict, stdout) + except NotAuthorized as e: + reply(act, 'NotAuthorized', str(e), stdout) + except NotFound: + reply(act, 'NotFound', obj, stdout) + else: + reply(act, None, r.get('name', r.get('id')), stdout) diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 140708b..928b310 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -14,7 +14,7 @@ [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi dump (datasets | groups | organizations | users | related) (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) - [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted] + [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted --api-tokens] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi load datasets [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX] @@ -26,7 +26,7 @@ [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (users | related) [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] - [-n | -o] [-qwz] + [-n | -o] [-qwz --api-tokens] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi search datasets [(KEY=STRING | KEY:JSON ) ... | -i | -I JSON_INPUT] @@ -76,6 +76,9 @@ -u --ckan-user=USER perform actions as user with this name, uses the site sysadmin user when not specified -U --include-users include users of a group/organization + --api-tokens export API Token information along with + user metadata as api_tokens_list lists (dump). + create API Tokens for users (load). --upload-logo upload logo image of a group/organization if the image is stored in the original server, otherwise its image url will be used From 2ce3ba63f0ecf3be90d7a72e1becf0f5df4af14f Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 14:25:22 +0000 Subject: [PATCH 02/10] feat(dev): api tokens; - Dump and load user api tokens. --- ckanapi/cli/dump.py | 4 +-- ckanapi/cli/load.py | 69 ++++++++++++++++++++------------------------- ckanapi/cli/main.py | 16 ++++++++--- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py index ce1968c..f24119c 100644 --- a/ckanapi/cli/dump.py +++ b/ckanapi/cli/dump.py @@ -255,7 +255,7 @@ def populate_res_views(ckan, res): def populate_api_tokens(ckan, user): """ - Update user dict in-place with api_tokens_list + Update user dict in-place with api_token_list """ try: tokens = ckan.call_action('api_token_list', { @@ -267,4 +267,4 @@ def populate_api_tokens(ckan, user): return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass if not tokens: return # return if the user api token list is empty - user['api_tokens_list'] = tokens + user['api_token_list'] = tokens diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 7e1db55..15a88c8 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -110,18 +110,6 @@ def line_reader(): return 3 -def reply(action, error, response, out): - """ - format messages to be sent back to parent process - """ - out.write(compact_json([ - datetime.now().isoformat(), - action, - error, - response]) + b'\n') - out.flush() - - def load_things_worker(ckan, thing, arguments, stdin=None, stdout=None): """ @@ -156,12 +144,23 @@ def load_things_worker(ckan, thing, arguments, 'related_show','related_create','related_update'), }[thing] + def reply(action, error, response): + """ + format messages to be sent back to parent process + """ + stdout.write(compact_json([ + datetime.now().isoformat(), + action, + error, + response]) + b'\n') + stdout.flush() + for line in iter(stdin.readline, b''): try: obj = json.loads(line.decode('utf-8')) except UnicodeDecodeError as e: obj = None - reply('read', 'UnicodeDecodeError', str(e), stdout) + reply('read', 'UnicodeDecodeError', str(e)) continue requests_kwargs = None @@ -185,7 +184,7 @@ def load_things_worker(ckan, thing, arguments, except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e), stdout) + reply('show', 'NotAuthorized', str(e)) continue name = obj.get('name') if not existing and name: @@ -195,7 +194,7 @@ def load_things_worker(ckan, thing, arguments, except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e), stdout) + reply('show', 'NotAuthorized', str(e)) continue if existing: @@ -204,11 +203,12 @@ def load_things_worker(ckan, thing, arguments, # FIXME: compare and reply when 'unchanged'? if not existing and arguments['--update-only']: - reply('show', 'NotFound', [obj.get('id'), obj.get('name')], stdout) + reply('show', 'NotFound', [obj.get('id'), obj.get('name')]) continue act = 'update' if existing else 'create' try: + api_token_list = obj.pop('api_token_list', None) # do not send api_token_list to user actions if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) @@ -216,7 +216,7 @@ def load_things_worker(ckan, thing, arguments, r = ckan.call_action(thing_create, obj) if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages _upload_resources(ckan,obj,arguments) - elif thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations + if thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] obj = _upload_logo(ckan,obj) @@ -224,18 +224,18 @@ def load_things_worker(ckan, thing, arguments, obj['users'] = users ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) - elif thing == 'users' and arguments['--api-tokens'] and obj.get('api_tokens_list'): # check if it is needed to create user api tokens when creating/updating users - _load_user_api_tokens(ckan, obj, arguments, stdout) + if thing == 'users' and arguments['--api-tokens'] and api_token_list: # check if it is needed to create user api tokens when creating/updating users + _load_user_api_tokens(ckan, api_token_list, arguments) except ValidationError as e: - reply(act, 'ValidationError', e.error_dict, stdout) + reply(act, 'ValidationError', e.error_dict) except SearchIndexError as e: - reply(act, 'SearchIndexError', str(e), stdout) + reply(act, 'SearchIndexError', str(e)) except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e), stdout) + reply(act, 'NotAuthorized', str(e)) except NotFound: - reply(act, 'NotFound', obj, stdout) + reply(act, 'NotFound', obj) else: - reply(act, None, r.get('name',r.get('id')), stdout) + reply(act, None, r.get('name',r.get('id'))) def _worker_command_line(thing, arguments): @@ -316,15 +316,14 @@ def _upload_logo(ckan,obj_orig): return obj -def _load_user_api_tokens(ckan, obj, arguments, stdout): +def _load_user_api_tokens(ckan, api_token_list, arguments): """ - Loads user API Tokens from api_tokens_list + Loads user API Tokens from api_token_list """ requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} - act = 'create-api-token' - for token in obj['api_tokens_list']: + for token in api_token_list: args = { 'id': token['id'], 'created_at': token['created_at'], @@ -332,14 +331,6 @@ def _load_user_api_tokens(ckan, obj, arguments, stdout): 'name': token['name'], 'user': token['user_id'] } - try: - r = ckan.call_action('api_token_create', - args, requests_kwargs=requests_kwargs) - except ValidationError as e: - reply(act, 'ValidationError', e.error_dict, stdout) - except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e), stdout) - except NotFound: - reply(act, 'NotFound', obj, stdout) - else: - reply(act, None, r.get('name', r.get('id')), stdout) + # exceptions handled in load_things_worker + ckan.call_action('api_token_create', + args, requests_kwargs=requests_kwargs) diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 928b310..9b3e308 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -12,9 +12,13 @@ (ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX]) [-p PROCESSES] [-l LOG_FILE] [-qwz] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] - ckanapi dump (datasets | groups | organizations | users | related) + ckanapi dump (datasets | groups | organizations | related) (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) - [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted --api-tokens] + [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted] + [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] + ckanapi dump users + (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) + [-p PROCESSES] [-qwz --api-tokens] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi load datasets [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX] @@ -24,10 +28,14 @@ [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] - ckanapi load (users | related) + ckanapi load users [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz --api-tokens] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] + ckanapi load related + [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] + [-n | -o] [-qwz] + [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi search datasets [(KEY=STRING | KEY:JSON ) ... | -i | -I JSON_INPUT] [-O JSONL_OUTPUT] [-z] @@ -77,7 +85,7 @@ site sysadmin user when not specified -U --include-users include users of a group/organization --api-tokens export API Token information along with - user metadata as api_tokens_list lists (dump). + user metadata as api_token_list lists (dump). create API Tokens for users (load). --upload-logo upload logo image of a group/organization if the image is stored in the original server, otherwise From baa0c7cfdf0a27a60dca70f1becb2f998699a062 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 14:32:35 +0000 Subject: [PATCH 03/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/tests/test_cli_dump.py | 9 ++++++++- ckanapi/tests/test_cli_load.py | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py index 95534fa..b842077 100644 --- a/ckanapi/tests/test_cli_dump.py +++ b/ckanapi/tests/test_cli_dump.py @@ -145,6 +145,7 @@ def test_parent_dump_all(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, @@ -177,6 +178,7 @@ def test_parent_parallel_limit(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, @@ -206,6 +208,7 @@ def test_parent_id_argument(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, @@ -237,6 +240,7 @@ def test_parent_maintain_order(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool_reversed, stdout=self.stdout, @@ -272,6 +276,7 @@ def test_parent_datapackages(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._worker_pool_with_data, stdout=self.stdout, @@ -326,6 +331,7 @@ def test_resource_views(self): '--resource-views': True, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._worker_pool_with_resource_views, stdout=self.stdout, @@ -379,6 +385,7 @@ def test_include_params_default(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }) action = ckan.method_calls[0].args[0] @@ -412,7 +419,7 @@ def test_include_params_true(self): '--resource-views': False, '--insecure': False, '--include-users': False, - + '--api-tokens': False, '--include-private': True, '--include-drafts': True, '--include-deleted': True, diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 7926940..68bab20 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -345,6 +345,7 @@ def test_parent_load_two(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -380,6 +381,7 @@ def test_parent_load_start_max(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -418,6 +420,7 @@ def test_parent_parallel_limit(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( From c0f4cb625016bc0c34a23cdeeb53fc5549855136 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 15:20:10 +0000 Subject: [PATCH 04/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/cli/load.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 15a88c8..88611f9 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -324,13 +324,14 @@ def _load_user_api_tokens(ckan, api_token_list, arguments): if arguments['--insecure']: requests_kwargs = {'verify': False} for token in api_token_list: - args = { - 'id': token['id'], - 'created_at': token['created_at'], - 'last_access': token['last_access'], - 'name': token['name'], - 'user': token['user_id'] - } # exceptions handled in load_things_worker - ckan.call_action('api_token_create', - args, requests_kwargs=requests_kwargs) + ckan.call_action( + 'api_token_create', + { + 'id': token['id'], + 'created_at': token['created_at'], + 'last_access': token['last_access'], + 'name': token['name'], + 'user': token['user_id'] + }, + requests_kwargs=requests_kwargs) From a3e80b24fb80bddfe7ff8501def42e73fff05dc1 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 13:44:52 -0400 Subject: [PATCH 05/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/cli/dump.py | 5 +- ckanapi/cli/load.py | 3 +- ckanapi/tests/test_cli_dump.py | 117 ++++++++++++++++++++++++++++++++- ckanapi/tests/test_cli_load.py | 44 +++++++++++++ 4 files changed, 162 insertions(+), 7 deletions(-) diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py index f24119c..da826b8 100644 --- a/ckanapi/cli/dump.py +++ b/ckanapi/cli/dump.py @@ -66,7 +66,6 @@ def dump_things(ckan, thing, arguments, include_drafts=arguments['--include-drafts'] if '--include-drafts' in arguments else False, include_deleted=arguments['--include-deleted'] if '--include-deleted' in arguments else False, ) - names = ckan.call_action(get_thing_list, params) else: @@ -258,9 +257,7 @@ def populate_api_tokens(ckan, user): Update user dict in-place with api_token_list """ try: - tokens = ckan.call_action('api_token_list', { - 'user_id': user['name'], - 'limit': 0}) + tokens = ckan.call_action('api_token_list', {'user_id': user['name']}) except CKANAPIError: return except NotFound: diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 88611f9..7cf2382 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -235,7 +235,8 @@ def reply(action, error, response): except NotFound: reply(act, 'NotFound', obj) else: - reply(act, None, r.get('name',r.get('id'))) + log_obj = {'id': r.get('id'), 'name': r.get('name')} + reply(act, None, log_obj) def _worker_command_line(thing, arguments): diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py index b842077..c9c7c89 100644 --- a/ckanapi/tests/test_cli_dump.py +++ b/ckanapi/tests/test_cli_dump.py @@ -56,7 +56,23 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'resource_id': 'd902fafc-5717-4dd0-87f2-7a6fc96989b7', 'responsive': True, 'show_fields': ['_id']}]}, - }[name][data_dict.get('id') or data_dict.get('resource_id')] + 'user_list': { + None: [ + 'test_user', + ], + }, + 'user_show': { + 'test_user': { + 'id': '123', + 'name': 'test_user', + }, + }, + 'api_token_list': { + 'test_user': { + 'token': 'this-is-a-token', + }, + }, + }[name][data_dict.get('id') or data_dict.get('resource_id') or data_dict.get('user_id')] except KeyError: raise NotFound() @@ -434,6 +450,82 @@ def test_include_params_true(self): self.assertEqual(data_dict["include_drafts"], True) self.assertEqual(data_dict["include_deleted"], True) + def test_dump_users(self): + """ + Dumping all users should use user_list with all_fields=False + """ + dump_things(self.ckan, 'users', { + '--all': True, + '--quiet': False, + '--ckan-user': None, + '--config': None, + '--remote': None, + '--apikey': None, + '--worker': False, + '--log': None, + '--output': None, + '--datapackages': None, + '--gzip': False, + '--processes': '1', + '--get-request': False, + '--datastore-fields': False, + '--resource-views': False, + '--insecure': False, + '--include-users': False, + '--api-tokens': False, + '--include-private': False, + '--include-drafts': False, + '--include-deleted': False, + }, + worker_pool=self._worker_pool_for_users, + stdout=self.stdout, + stderr=self.stderr) + + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + data = json.loads(response.decode('UTF-8')) + + self.assertEqual(data, {'id': '123', 'name': 'test_user'}) + + + def test_dump_users_with_tokens(self): + """ + Dumping all users w/ --api-tokens should use user_list + with all_fields=False and use + """ + dump_things(self.ckan, 'users', { + '--all': True, + '--quiet': False, + '--ckan-user': None, + '--config': None, + '--remote': None, + '--apikey': None, + '--worker': False, + '--log': None, + '--output': None, + '--datapackages': None, + '--gzip': False, + '--processes': '1', + '--get-request': False, + '--datastore-fields': False, + '--resource-views': False, + '--insecure': False, + '--include-users': False, + '--api-tokens': True, + '--include-private': False, + '--include-drafts': False, + '--include-deleted': False, + }, + worker_pool=self._worker_pool_for_users_with_tokens, + stdout=self.stdout, + stderr=self.stderr) + + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + data = json.loads(response.decode('UTF-8')) + + self.assertEqual(data, {'api_token_list': {'token': 'this-is-a-token'}, 'id': '123', 'name': 'test_user'}) + def _mock_worker_pool(self, cmd, processes, job_iter): self.worker_cmd = cmd self.worker_processes = processes @@ -460,7 +552,6 @@ def _worker_pool_with_data(self, cmd, processes, job_iter): for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): yield [[], i, v] - def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter): worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) worker_stdout = BytesIO() @@ -473,3 +564,25 @@ def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter): stdout=worker_stdout) for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): yield [[], i, v] + + def _worker_pool_for_users(self, cmd, proccesses, job_iter): + worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) + worker_stdout = BytesIO() + dump_things_worker(self.ckan, 'users', { + '--api-tokens': False, + '--insecure': False,}, + stdin=worker_stdin, + stdout=worker_stdout) + for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): + yield [[], i, v] + + def _worker_pool_for_users_with_tokens(self, cmd, proccesses, job_iter): + worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) + worker_stdout = BytesIO() + dump_things_worker(self.ckan, 'users', { + '--api-tokens': True, + '--insecure': False,}, + stdin=worker_stdin, + stdout=worker_stdout) + for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): + yield [[], i, v] diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 68bab20..cd7b903 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -4,6 +4,7 @@ import unittest from io import BytesIO +import mock class MockCKAN(object): def call_action(self, name, data_dict, requests_kwargs=None): @@ -49,6 +50,12 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'organization_create': { None: {'name': 'org-created'}, }, + 'user_show': { + 'test_user': {'id': 'some-generated-uuid', 'name': 'test_user'}, + }, + 'user_create': { + None: {'id': 'some-generated-uuid', 'name': 'test_user'} + } }[name][data_dict.get('id')] except KeyError: raise NotFound() @@ -433,6 +440,43 @@ def test_parent_parallel_limit(self): 'ckanapi', 'load', 'datasets', '--worker']) self.assertEqual(self.worker_processes, 2) + def test_create_user(self): + load_things_worker(self.ckan, 'users', { + '--create-only': True, + '--update-only': False, + '--insecure': False, + '--api-tokens': False, + }, + stdin=BytesIO(b'{"name":"test_user"}\n'), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) + + # @mock.patch("ckan.model.User.get") + # def test_create_user_with_api_token(self, mock_get): + # mock_user = mock.MagicMock() + # mock_user.name = 'test_user' + # mock_user.id = 'test_user' + # mock_get.return_value = mock_user + # load_things_worker(self.ckan, 'users', { + # '--create-only': True, + # '--update-only': False, + # '--insecure': False, + # '--api-tokens': True, + # }, + # stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), + # stdout=self.stdout) + # response = self.stdout.getvalue() + # self.assertEqual(response[-1:], b'\n') + # timstamp, action, error, data = json.loads(response.decode('UTF-8')) + # self.assertEqual(action, 'create') + # self.assertEqual(error, None) + # self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) + def _mock_worker_pool(self, cmd, processes, job_iter): self.worker_cmd = cmd self.worker_processes = processes From d5d102a83d3d53f7b0c7dec2fd8a6feb362e47e3 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 13:47:12 -0400 Subject: [PATCH 06/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/tests/test_cli_load.py | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index cd7b903..5e14bb7 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -4,7 +4,6 @@ import unittest from io import BytesIO -import mock class MockCKAN(object): def call_action(self, name, data_dict, requests_kwargs=None): @@ -456,26 +455,27 @@ def test_create_user(self): self.assertEqual(error, None) self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) - # @mock.patch("ckan.model.User.get") - # def test_create_user_with_api_token(self, mock_get): - # mock_user = mock.MagicMock() - # mock_user.name = 'test_user' - # mock_user.id = 'test_user' - # mock_get.return_value = mock_user - # load_things_worker(self.ckan, 'users', { - # '--create-only': True, - # '--update-only': False, - # '--insecure': False, - # '--api-tokens': True, - # }, - # stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), - # stdout=self.stdout) - # response = self.stdout.getvalue() - # self.assertEqual(response[-1:], b'\n') - # timstamp, action, error, data = json.loads(response.decode('UTF-8')) - # self.assertEqual(action, 'create') - # self.assertEqual(error, None) - # self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) + @unittest.mock.patch("ckan.model.User.get") + def test_create_user_with_api_token(self, mock_get): + mock_user = unittest.mock.MagicMock() + mock_user.name = 'test_user' + mock_user.id = 'test_user' + mock_get.return_value = mock_user + load_things_worker(self.ckan, 'users', { + '--create-only': True, + '--update-only': False, + '--insecure': False, + '--api-tokens': True, + }, + stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + # FIXME: how to mock this... + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) def _mock_worker_pool(self, cmd, processes, job_iter): self.worker_cmd = cmd From dc06c4101f8392c25495d7eeb4da1ffce5344fc5 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 13:57:42 -0400 Subject: [PATCH 07/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/tests/test_cli_load.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 5e14bb7..5c109fe 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -455,20 +455,20 @@ def test_create_user(self): self.assertEqual(error, None) self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) - @unittest.mock.patch("ckan.model.User.get") def test_create_user_with_api_token(self, mock_get): mock_user = unittest.mock.MagicMock() mock_user.name = 'test_user' mock_user.id = 'test_user' - mock_get.return_value = mock_user - load_things_worker(self.ckan, 'users', { - '--create-only': True, - '--update-only': False, - '--insecure': False, - '--api-tokens': True, - }, - stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), - stdout=self.stdout) + with unittest.mock.patch('ckan.model.User.get') as mock_get: + mock_get.return_value = mock_user + load_things_worker(self.ckan, 'users', { + '--create-only': True, + '--update-only': False, + '--insecure': False, + '--api-tokens': True, + }, + stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), + stdout=self.stdout) response = self.stdout.getvalue() self.assertEqual(response[-1:], b'\n') timstamp, action, error, data = json.loads(response.decode('UTF-8')) From 2da12ecffc23307b0c31f435f35d53db052d91e5 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 14:03:23 -0400 Subject: [PATCH 08/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/tests/test_cli_load.py | 46 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 5c109fe..5bccd24 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -33,24 +33,24 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'unused': {'users': ['people']}, }, 'package_create': { - None: {'name': 'something-new'}, + None: {'id': 'some-generated-uuid', 'name': 'something-new'}, }, 'package_update': { - '34': {'name': 'something-updated'}, + '34': {'id': '34', 'name': 'something-updated'}, }, 'group_update': { - 'ab': {'name': 'group-updated'}, + 'ab': {'id': 'ab', 'name': 'group-updated'}, }, 'organization_update': { - 'cd': {'name': 'org-updated'}, - 'used': {'name': 'users-unchanged'}, - 'unused': {'name': 'users-cleared'}, + 'cd': {'id': 'cd', 'name': 'org-updated'}, + 'used': {'id': 'used', 'name': 'users-unchanged'}, + 'unused': {'id': 'unused', 'name': 'users-cleared'}, }, 'organization_create': { - None: {'name': 'org-created'}, + None: {'id': 'some-generated-uuid', 'name': 'org-created'}, }, 'user_show': { - 'test_user': {'id': 'some-generated-uuid', 'name': 'test_user'}, + 'test_user': {'id': 'test_user', 'name': 'test_user'}, }, 'user_create': { None: {'id': 'some-generated-uuid', 'name': 'test_user'} @@ -80,7 +80,7 @@ def test_create_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -96,7 +96,7 @@ def test_create_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -114,7 +114,7 @@ def test_create_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_only(self): load_things_worker(self.ckan, 'datasets', { @@ -130,7 +130,7 @@ def test_create_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_empty_dict(self): load_things_worker(self.ckan, 'datasets', { @@ -146,7 +146,7 @@ def test_create_empty_dict(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -176,7 +176,7 @@ def test_update_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -192,7 +192,7 @@ def test_update_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -210,7 +210,7 @@ def test_update_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_only(self): load_things_worker(self.ckan, 'datasets', { @@ -226,7 +226,7 @@ def test_update_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -274,7 +274,7 @@ def test_update_group(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'group-updated') + self.assertEqual(data, {'id': 'ab', 'name': 'group-updated'}) def test_update_organization_two(self): load_things_worker(self.ckan, 'organizations', { @@ -294,11 +294,11 @@ def test_update_organization_two(self): timstamp, action, error, data = json.loads(r1.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'org-updated') + self.assertEqual(data, {'id': 'cd', 'name': 'org-updated'}) timstamp, action, error, data = json.loads(r2.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'org-created') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'org-created'}) def test_update_organization_with_users_unchanged(self): load_things_worker(self.ckan, 'organizations', { @@ -314,7 +314,7 @@ def test_update_organization_with_users_unchanged(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'users-unchanged') + self.assertEqual(data, {'id': 'used', 'name': 'users-unchanged'}) def test_update_organization_with_users_cleared(self): load_things_worker(self.ckan, 'organizations', { @@ -330,7 +330,7 @@ def test_update_organization_with_users_cleared(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'users-cleared') + self.assertEqual(data, {'id': 'unused', 'name': 'users-cleared'}) def test_parent_load_two(self): load_things(self.ckan, 'datasets', { @@ -455,7 +455,7 @@ def test_create_user(self): self.assertEqual(error, None) self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) - def test_create_user_with_api_token(self, mock_get): + def test_create_user_with_api_token(self): mock_user = unittest.mock.MagicMock() mock_user.name = 'test_user' mock_user.id = 'test_user' From 61b382102ca3367c0d46c614d92ad7ff72946b63 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 14:18:01 -0400 Subject: [PATCH 09/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/cli/load.py | 7 ++++++- ckanapi/tests/test_cli_load.py | 31 +++++++++++++++---------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 7cf2382..3e21f73 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -225,7 +225,7 @@ def reply(action, error, response): ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) if thing == 'users' and arguments['--api-tokens'] and api_token_list: # check if it is needed to create user api tokens when creating/updating users - _load_user_api_tokens(ckan, api_token_list, arguments) + created_tokens = _load_user_api_tokens(ckan, api_token_list, arguments) except ValidationError as e: reply(act, 'ValidationError', e.error_dict) except SearchIndexError as e: @@ -236,6 +236,8 @@ def reply(action, error, response): reply(act, 'NotFound', obj) else: log_obj = {'id': r.get('id'), 'name': r.get('name')} + if arguments['--api-tokens'] and api_token_list and created_tokens: + log_obj['created_tokens'] = created_tokens reply(act, None, log_obj) @@ -324,6 +326,7 @@ def _load_user_api_tokens(ckan, api_token_list, arguments): requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} + created_tokens = [] for token in api_token_list: # exceptions handled in load_things_worker ckan.call_action( @@ -336,3 +339,5 @@ def _load_user_api_tokens(ckan, api_token_list, arguments): 'user': token['user_id'] }, requests_kwargs=requests_kwargs) + created_tokens.append(token['name']) + return created_tokens diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 5bccd24..9c1a42d 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -54,7 +54,12 @@ def call_action(self, name, data_dict, requests_kwargs=None): }, 'user_create': { None: {'id': 'some-generated-uuid', 'name': 'test_user'} - } + }, + 'api_token_create': { + 'this-is-a-token': { + 'id': 'this-is-a-token' + } + }, }[name][data_dict.get('id')] except KeyError: raise NotFound() @@ -456,26 +461,20 @@ def test_create_user(self): self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) def test_create_user_with_api_token(self): - mock_user = unittest.mock.MagicMock() - mock_user.name = 'test_user' - mock_user.id = 'test_user' - with unittest.mock.patch('ckan.model.User.get') as mock_get: - mock_get.return_value = mock_user - load_things_worker(self.ckan, 'users', { - '--create-only': True, - '--update-only': False, - '--insecure': False, - '--api-tokens': True, - }, - stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), - stdout=self.stdout) + load_things_worker(self.ckan, 'users', { + '--create-only': True, + '--update-only': False, + '--insecure': False, + '--api-tokens': True, + }, + stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), + stdout=self.stdout) response = self.stdout.getvalue() self.assertEqual(response[-1:], b'\n') timstamp, action, error, data = json.loads(response.decode('UTF-8')) - # FIXME: how to mock this... self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) + self.assertEqual(data, {'created_tokens': ['this-is-a-token'], 'id': 'some-generated-uuid', 'name': 'test_user'}) def _mock_worker_pool(self, cmd, processes, job_iter): self.worker_cmd = cmd From a15b0fc1bb1857e3c90c5cd8d952c5a47630a272 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 14:19:53 -0400 Subject: [PATCH 10/10] feat(tests): coverage; - Added new options to tests. --- ckanapi/cli/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 3e21f73..fcc06da 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -236,7 +236,7 @@ def reply(action, error, response): reply(act, 'NotFound', obj) else: log_obj = {'id': r.get('id'), 'name': r.get('name')} - if arguments['--api-tokens'] and api_token_list and created_tokens: + if thing == 'users' and arguments['--api-tokens'] and api_token_list and created_tokens: log_obj['created_tokens'] = created_tokens reply(act, None, log_obj)