Skip to content

Dump & Load User API Tokens #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wardi merged 10 commits into from
May 23, 2026
+255 −29
Merged

Dump & Load User API Tokens #230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5ed3718
feat(dev): api tokens;
JVickery-TBS May 7, 2026
2ce3ba6
feat(dev): api tokens;
JVickery-TBS May 11, 2026
baa0c7c
feat(tests): coverage;
JVickery-TBS May 11, 2026
c0f4cb6
feat(tests): coverage;
JVickery-TBS May 11, 2026
a3e80b2
feat(tests): coverage;
JVickery-TBS May 22, 2026
d5d102a
feat(tests): coverage;
JVickery-TBS May 22, 2026
dc06c41
feat(tests): coverage;
JVickery-TBS May 22, 2026
2da12ec
feat(tests): coverage;
JVickery-TBS May 22, 2026
61b3821
feat(tests): coverage;
JVickery-TBS May 22, 2026
a15b0fc
feat(tests): coverage;
JVickery-TBS May 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion ckanapi/cli/dump.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ def dump_things(ckan, thing, arguments,
include_drafts=arguments['--include-drafts'] if '--include-drafts' in arguments else False,
include_deleted=arguments['--include-deleted'] if '--include-deleted' in arguments else False,
)

names = ckan.call_action(get_thing_list, params)

else:
Expand Down Expand Up @@ -204,8 +203,11 @@ def reply(error, record=None):
if thing == 'datasets' and arguments['--resource-views']:
for res in obj.get('resources', []):
populate_res_views(ckan, res)
if thing == 'users' and arguments['--api-tokens']:
populate_api_tokens(ckan, obj)
reply(None, obj)


def _worker_command_line(thing, arguments):
"""
Create a worker command line suitable for Popen with only the
Expand All @@ -227,6 +229,7 @@ def b(name):
+ b('--datastore-fields')
+ b('--resource-views')
+ b('--include-users')
+ b('--api-tokens')
+ ['value-here-to-make-docopt-happy']
)

Expand All @@ -248,3 +251,17 @@ def populate_res_views(ckan, res):
return # return if the resource views list is empty
res['resource_views'] = views


def populate_api_tokens(ckan, user):
"""
Update user dict in-place with api_token_list
"""
try:
tokens = ckan.call_action('api_token_list', {'user_id': user['name']})
except CKANAPIError:
return
except NotFound:
return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass
Comment on lines +261 to +264

@wardi wardi May 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The way this is written multiple exceptions would be caught in the RemoteCKAN case but only NotFound will be caught in the LocalCKAN case, that's strange.

Suggested change
except CKANAPIError:
return
except NotFound:
return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass
except NotFound:
return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass

But, would we ever expect a NotFound? Won't that only happen if the user doesn't exist? If so it's better to not catch and ignore any exceptions.

@wardi wardi May 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But, I'll merge as-is and we can clean it up later.

if not tokens:
return # return if the user api token list is empty
user['api_token_list'] = tokens
36 changes: 34 additions & 2 deletions ckanapi/cli/load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,21 +208,24 @@ def reply(action, error, response):

act = 'update' if existing else 'create'
try:
api_token_list = obj.pop('api_token_list', None) # do not send api_token_list to user actions
if existing:
r = ckan.call_action(thing_update, obj,
requests_kwargs=requests_kwargs)
else:
r = ckan.call_action(thing_create, obj)
if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages
_upload_resources(ckan,obj,arguments)
elif thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations
if thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations
if arguments['--upload-logo']:
users = obj['users']
obj = _upload_logo(ckan,obj)
obj.pop('image_upload')
obj['users'] = users
ckan.call_action(thing_update, obj,
requests_kwargs=requests_kwargs)
if thing == 'users' and arguments['--api-tokens'] and api_token_list: # check if it is needed to create user api tokens when creating/updating users
created_tokens = _load_user_api_tokens(ckan, api_token_list, arguments)
except ValidationError as e:
reply(act, 'ValidationError', e.error_dict)
except SearchIndexError as e:
Expand All @@ -232,7 +235,11 @@ def reply(action, error, response):
except NotFound:
reply(act, 'NotFound', obj)
else:
reply(act, None, r.get('name',r.get('id')))
log_obj = {'id': r.get('id'), 'name': r.get('name')}
if thing == 'users' and arguments['--api-tokens'] and api_token_list and created_tokens:
log_obj['created_tokens'] = created_tokens
reply(act, None, log_obj)


def _worker_command_line(thing, arguments):
"""
Expand All @@ -255,6 +262,7 @@ def b(name):
+ b('--update-only')
+ b('--upload-resources')
+ b('--upload-logo')
+ b('--api-tokens')
)


Expand Down Expand Up @@ -309,3 +317,27 @@ def _upload_logo(ckan,obj_orig):
obj['image_upload'] = (new_url, f.raw)
ckan.action.group_update(**obj)
return obj


def _load_user_api_tokens(ckan, api_token_list, arguments):
"""
Loads user API Tokens from api_token_list
"""
requests_kwargs = None
if arguments['--insecure']:
requests_kwargs = {'verify': False}
created_tokens = []
for token in api_token_list:
# exceptions handled in load_things_worker
ckan.call_action(
'api_token_create',
{
'id': token['id'],
'created_at': token['created_at'],
'last_access': token['last_access'],
'name': token['name'],
'user': token['user_id']
},
Comment thread
wardi marked this conversation as resolved.
requests_kwargs=requests_kwargs)
created_tokens.append(token['name'])
return created_tokens
15 changes: 13 additions & 2 deletions ckanapi/cli/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,14 @@
(ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX])
[-p PROCESSES] [-l LOG_FILE] [-qwz]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
ckanapi dump (datasets | groups | organizations | users | related)
ckanapi dump (datasets | groups | organizations | related)
(ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
[-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
ckanapi dump users
(ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
[-p PROCESSES] [-qwz --api-tokens]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
ckanapi load datasets
[--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX]
[-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz]
Expand All @@ -24,7 +28,11 @@
[--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX]
[-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
ckanapi load (users | related)
ckanapi load users
[-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE]
[-n | -o] [-qwz --api-tokens]
Comment thread
JVickery-TBS marked this conversation as resolved.
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
ckanapi load related
[-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE]
[-n | -o] [-qwz]
[[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
Expand Down Expand Up @@ -76,6 +84,9 @@
-u --ckan-user=USER perform actions as user with this name, uses the
site sysadmin user when not specified
-U --include-users include users of a group/organization
--api-tokens export API Token information along with
user metadata as api_token_list lists (dump).
create API Tokens for users (load).
--upload-logo upload logo image of a group/organization if the
image is stored in the original server, otherwise
its image url will be used
Expand Down
126 changes: 123 additions & 3 deletions ckanapi/tests/test_cli_dump.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,23 @@ def call_action(self, name, data_dict, requests_kwargs=None):
'resource_id': 'd902fafc-5717-4dd0-87f2-7a6fc96989b7',
'responsive': True,
'show_fields': ['_id']}]},
}[name][data_dict.get('id') or data_dict.get('resource_id')]
'user_list': {
None: [
'test_user',
],
},
'user_show': {
'test_user': {
'id': '123',
'name': 'test_user',
},
},
'api_token_list': {
'test_user': {
'token': 'this-is-a-token',
},
},
}[name][data_dict.get('id') or data_dict.get('resource_id') or data_dict.get('user_id')]
except KeyError:
raise NotFound()

Expand Down Expand Up @@ -145,6 +161,7 @@ def test_parent_dump_all(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdout=self.stdout,
Expand Down Expand Up @@ -177,6 +194,7 @@ def test_parent_parallel_limit(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool,
stdout=self.stdout,
Expand Down Expand Up @@ -206,6 +224,7 @@ def test_parent_id_argument(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},

worker_pool=self._mock_worker_pool,
Expand Down Expand Up @@ -237,6 +256,7 @@ def test_parent_maintain_order(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._mock_worker_pool_reversed,
stdout=self.stdout,
Expand Down Expand Up @@ -272,6 +292,7 @@ def test_parent_datapackages(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._worker_pool_with_data,
stdout=self.stdout,
Expand Down Expand Up @@ -326,6 +347,7 @@ def test_resource_views(self):
'--resource-views': True,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
},
worker_pool=self._worker_pool_with_resource_views,
stdout=self.stdout,
Expand Down Expand Up @@ -379,6 +401,7 @@ def test_include_params_default(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
})

action = ckan.method_calls[0].args[0]
Expand Down Expand Up @@ -412,7 +435,7 @@ def test_include_params_true(self):
'--resource-views': False,
'--insecure': False,
'--include-users': False,

'--api-tokens': False,
'--include-private': True,
'--include-drafts': True,
'--include-deleted': True,
Expand All @@ -427,6 +450,82 @@ def test_include_params_true(self):
self.assertEqual(data_dict["include_drafts"], True)
self.assertEqual(data_dict["include_deleted"], True)

def test_dump_users(self):
"""
Dumping all users should use user_list with all_fields=False
"""
dump_things(self.ckan, 'users', {
'--all': True,
'--quiet': False,
'--ckan-user': None,
'--config': None,
'--remote': None,
'--apikey': None,
'--worker': False,
'--log': None,
'--output': None,
'--datapackages': None,
'--gzip': False,
'--processes': '1',
'--get-request': False,
'--datastore-fields': False,
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': False,
'--include-private': False,
'--include-drafts': False,
'--include-deleted': False,
},
worker_pool=self._worker_pool_for_users,
stdout=self.stdout,
stderr=self.stderr)

response = self.stdout.getvalue()
self.assertEqual(response[-1:], b'\n')
data = json.loads(response.decode('UTF-8'))

self.assertEqual(data, {'id': '123', 'name': 'test_user'})


def test_dump_users_with_tokens(self):
"""
Dumping all users w/ --api-tokens should use user_list
with all_fields=False and use
"""
dump_things(self.ckan, 'users', {
'--all': True,
'--quiet': False,
'--ckan-user': None,
'--config': None,
'--remote': None,
'--apikey': None,
'--worker': False,
'--log': None,
'--output': None,
'--datapackages': None,
'--gzip': False,
'--processes': '1',
'--get-request': False,
'--datastore-fields': False,
'--resource-views': False,
'--insecure': False,
'--include-users': False,
'--api-tokens': True,
'--include-private': False,
'--include-drafts': False,
'--include-deleted': False,
},
worker_pool=self._worker_pool_for_users_with_tokens,
stdout=self.stdout,
stderr=self.stderr)

response = self.stdout.getvalue()
self.assertEqual(response[-1:], b'\n')
data = json.loads(response.decode('UTF-8'))

self.assertEqual(data, {'api_token_list': {'token': 'this-is-a-token'}, 'id': '123', 'name': 'test_user'})

def _mock_worker_pool(self, cmd, processes, job_iter):
self.worker_cmd = cmd
self.worker_processes = processes
Expand All @@ -453,7 +552,6 @@ def _worker_pool_with_data(self, cmd, processes, job_iter):
for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')):
yield [[], i, v]


def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter):
worker_stdin = BytesIO(b''.join(v for i, v in job_iter))
worker_stdout = BytesIO()
Expand All @@ -466,3 +564,25 @@ def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter):
stdout=worker_stdout)
for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')):
yield [[], i, v]

def _worker_pool_for_users(self, cmd, proccesses, job_iter):
worker_stdin = BytesIO(b''.join(v for i, v in job_iter))
worker_stdout = BytesIO()
dump_things_worker(self.ckan, 'users', {
'--api-tokens': False,
'--insecure': False,},
stdin=worker_stdin,
stdout=worker_stdout)
for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')):
yield [[], i, v]

def _worker_pool_for_users_with_tokens(self, cmd, proccesses, job_iter):
worker_stdin = BytesIO(b''.join(v for i, v in job_iter))
worker_stdout = BytesIO()
dump_things_worker(self.ckan, 'users', {
'--api-tokens': True,
'--insecure': False,},
stdin=worker_stdin,
stdout=worker_stdout)
for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')):
yield [[], i, v]
Loading
Loading