diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py index 9c6870a..da826b8 100644 --- a/ckanapi/cli/dump.py +++ b/ckanapi/cli/dump.py @@ -66,7 +66,6 @@ def dump_things(ckan, thing, arguments, include_drafts=arguments['--include-drafts'] if '--include-drafts' in arguments else False, include_deleted=arguments['--include-deleted'] if '--include-deleted' in arguments else False, ) - names = ckan.call_action(get_thing_list, params) else: @@ -204,8 +203,11 @@ def reply(error, record=None): if thing == 'datasets' and arguments['--resource-views']: for res in obj.get('resources', []): populate_res_views(ckan, res) + if thing == 'users' and arguments['--api-tokens']: + populate_api_tokens(ckan, obj) reply(None, obj) + def _worker_command_line(thing, arguments): """ Create a worker command line suitable for Popen with only the @@ -227,6 +229,7 @@ def b(name): + b('--datastore-fields') + b('--resource-views') + b('--include-users') + + b('--api-tokens') + ['value-here-to-make-docopt-happy'] ) @@ -248,3 +251,17 @@ def populate_res_views(ckan, res): return # return if the resource views list is empty res['resource_views'] = views + +def populate_api_tokens(ckan, user): + """ + Update user dict in-place with api_token_list + """ + try: + tokens = ckan.call_action('api_token_list', {'user_id': user['name']}) + except CKANAPIError: + return + except NotFound: + return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass + if not tokens: + return # return if the user api token list is empty + user['api_token_list'] = tokens diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 6c1a9b9..fcc06da 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -208,6 +208,7 @@ def reply(action, error, response): act = 'update' if existing else 'create' try: + api_token_list = obj.pop('api_token_list', None) # do not send api_token_list to user actions if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) @@ -215,7 +216,7 @@ def reply(action, error, response): r = ckan.call_action(thing_create, obj) if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages _upload_resources(ckan,obj,arguments) - elif thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations + if thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] obj = _upload_logo(ckan,obj) @@ -223,6 +224,8 @@ def reply(action, error, response): obj['users'] = users ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) + if thing == 'users' and arguments['--api-tokens'] and api_token_list: # check if it is needed to create user api tokens when creating/updating users + created_tokens = _load_user_api_tokens(ckan, api_token_list, arguments) except ValidationError as e: reply(act, 'ValidationError', e.error_dict) except SearchIndexError as e: @@ -232,7 +235,11 @@ def reply(action, error, response): except NotFound: reply(act, 'NotFound', obj) else: - reply(act, None, r.get('name',r.get('id'))) + log_obj = {'id': r.get('id'), 'name': r.get('name')} + if thing == 'users' and arguments['--api-tokens'] and api_token_list and created_tokens: + log_obj['created_tokens'] = created_tokens + reply(act, None, log_obj) + def _worker_command_line(thing, arguments): """ @@ -255,6 +262,7 @@ def b(name): + b('--update-only') + b('--upload-resources') + b('--upload-logo') + + b('--api-tokens') ) @@ -309,3 +317,27 @@ def _upload_logo(ckan,obj_orig): obj['image_upload'] = (new_url, f.raw) ckan.action.group_update(**obj) return obj + + +def _load_user_api_tokens(ckan, api_token_list, arguments): + """ + Loads user API Tokens from api_token_list + """ + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + created_tokens = [] + for token in api_token_list: + # exceptions handled in load_things_worker + ckan.call_action( + 'api_token_create', + { + 'id': token['id'], + 'created_at': token['created_at'], + 'last_access': token['last_access'], + 'name': token['name'], + 'user': token['user_id'] + }, + requests_kwargs=requests_kwargs) + created_tokens.append(token['name']) + return created_tokens diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 140708b..9b3e308 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -12,10 +12,14 @@ (ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX]) [-p PROCESSES] [-l LOG_FILE] [-qwz] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] - ckanapi dump (datasets | groups | organizations | users | related) + ckanapi dump (datasets | groups | organizations | related) (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] + ckanapi dump users + (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) + [-p PROCESSES] [-qwz --api-tokens] + [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi load datasets [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz] @@ -24,7 +28,11 @@ [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] - ckanapi load (users | related) + ckanapi load users + [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] + [-n | -o] [-qwz --api-tokens] + [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] + ckanapi load related [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] @@ -76,6 +84,9 @@ -u --ckan-user=USER perform actions as user with this name, uses the site sysadmin user when not specified -U --include-users include users of a group/organization + --api-tokens export API Token information along with + user metadata as api_token_list lists (dump). + create API Tokens for users (load). --upload-logo upload logo image of a group/organization if the image is stored in the original server, otherwise its image url will be used diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py index 95534fa..c9c7c89 100644 --- a/ckanapi/tests/test_cli_dump.py +++ b/ckanapi/tests/test_cli_dump.py @@ -56,7 +56,23 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'resource_id': 'd902fafc-5717-4dd0-87f2-7a6fc96989b7', 'responsive': True, 'show_fields': ['_id']}]}, - }[name][data_dict.get('id') or data_dict.get('resource_id')] + 'user_list': { + None: [ + 'test_user', + ], + }, + 'user_show': { + 'test_user': { + 'id': '123', + 'name': 'test_user', + }, + }, + 'api_token_list': { + 'test_user': { + 'token': 'this-is-a-token', + }, + }, + }[name][data_dict.get('id') or data_dict.get('resource_id') or data_dict.get('user_id')] except KeyError: raise NotFound() @@ -145,6 +161,7 @@ def test_parent_dump_all(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, @@ -177,6 +194,7 @@ def test_parent_parallel_limit(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, @@ -206,6 +224,7 @@ def test_parent_id_argument(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, @@ -237,6 +256,7 @@ def test_parent_maintain_order(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool_reversed, stdout=self.stdout, @@ -272,6 +292,7 @@ def test_parent_datapackages(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._worker_pool_with_data, stdout=self.stdout, @@ -326,6 +347,7 @@ def test_resource_views(self): '--resource-views': True, '--insecure': False, '--include-users': False, + '--api-tokens': False, }, worker_pool=self._worker_pool_with_resource_views, stdout=self.stdout, @@ -379,6 +401,7 @@ def test_include_params_default(self): '--resource-views': False, '--insecure': False, '--include-users': False, + '--api-tokens': False, }) action = ckan.method_calls[0].args[0] @@ -412,7 +435,7 @@ def test_include_params_true(self): '--resource-views': False, '--insecure': False, '--include-users': False, - + '--api-tokens': False, '--include-private': True, '--include-drafts': True, '--include-deleted': True, @@ -427,6 +450,82 @@ def test_include_params_true(self): self.assertEqual(data_dict["include_drafts"], True) self.assertEqual(data_dict["include_deleted"], True) + def test_dump_users(self): + """ + Dumping all users should use user_list with all_fields=False + """ + dump_things(self.ckan, 'users', { + '--all': True, + '--quiet': False, + '--ckan-user': None, + '--config': None, + '--remote': None, + '--apikey': None, + '--worker': False, + '--log': None, + '--output': None, + '--datapackages': None, + '--gzip': False, + '--processes': '1', + '--get-request': False, + '--datastore-fields': False, + '--resource-views': False, + '--insecure': False, + '--include-users': False, + '--api-tokens': False, + '--include-private': False, + '--include-drafts': False, + '--include-deleted': False, + }, + worker_pool=self._worker_pool_for_users, + stdout=self.stdout, + stderr=self.stderr) + + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + data = json.loads(response.decode('UTF-8')) + + self.assertEqual(data, {'id': '123', 'name': 'test_user'}) + + + def test_dump_users_with_tokens(self): + """ + Dumping all users w/ --api-tokens should use user_list + with all_fields=False and use + """ + dump_things(self.ckan, 'users', { + '--all': True, + '--quiet': False, + '--ckan-user': None, + '--config': None, + '--remote': None, + '--apikey': None, + '--worker': False, + '--log': None, + '--output': None, + '--datapackages': None, + '--gzip': False, + '--processes': '1', + '--get-request': False, + '--datastore-fields': False, + '--resource-views': False, + '--insecure': False, + '--include-users': False, + '--api-tokens': True, + '--include-private': False, + '--include-drafts': False, + '--include-deleted': False, + }, + worker_pool=self._worker_pool_for_users_with_tokens, + stdout=self.stdout, + stderr=self.stderr) + + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + data = json.loads(response.decode('UTF-8')) + + self.assertEqual(data, {'api_token_list': {'token': 'this-is-a-token'}, 'id': '123', 'name': 'test_user'}) + def _mock_worker_pool(self, cmd, processes, job_iter): self.worker_cmd = cmd self.worker_processes = processes @@ -453,7 +552,6 @@ def _worker_pool_with_data(self, cmd, processes, job_iter): for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): yield [[], i, v] - def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter): worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) worker_stdout = BytesIO() @@ -466,3 +564,25 @@ def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter): stdout=worker_stdout) for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): yield [[], i, v] + + def _worker_pool_for_users(self, cmd, proccesses, job_iter): + worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) + worker_stdout = BytesIO() + dump_things_worker(self.ckan, 'users', { + '--api-tokens': False, + '--insecure': False,}, + stdin=worker_stdin, + stdout=worker_stdout) + for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): + yield [[], i, v] + + def _worker_pool_for_users_with_tokens(self, cmd, proccesses, job_iter): + worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) + worker_stdout = BytesIO() + dump_things_worker(self.ckan, 'users', { + '--api-tokens': True, + '--insecure': False,}, + stdin=worker_stdin, + stdout=worker_stdout) + for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): + yield [[], i, v] diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 7926940..9c1a42d 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -33,22 +33,33 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'unused': {'users': ['people']}, }, 'package_create': { - None: {'name': 'something-new'}, + None: {'id': 'some-generated-uuid', 'name': 'something-new'}, }, 'package_update': { - '34': {'name': 'something-updated'}, + '34': {'id': '34', 'name': 'something-updated'}, }, 'group_update': { - 'ab': {'name': 'group-updated'}, + 'ab': {'id': 'ab', 'name': 'group-updated'}, }, 'organization_update': { - 'cd': {'name': 'org-updated'}, - 'used': {'name': 'users-unchanged'}, - 'unused': {'name': 'users-cleared'}, + 'cd': {'id': 'cd', 'name': 'org-updated'}, + 'used': {'id': 'used', 'name': 'users-unchanged'}, + 'unused': {'id': 'unused', 'name': 'users-cleared'}, }, 'organization_create': { - None: {'name': 'org-created'}, + None: {'id': 'some-generated-uuid', 'name': 'org-created'}, }, + 'user_show': { + 'test_user': {'id': 'test_user', 'name': 'test_user'}, + }, + 'user_create': { + None: {'id': 'some-generated-uuid', 'name': 'test_user'} + }, + 'api_token_create': { + 'this-is-a-token': { + 'id': 'this-is-a-token' + } + }, }[name][data_dict.get('id')] except KeyError: raise NotFound() @@ -74,7 +85,7 @@ def test_create_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -90,7 +101,7 @@ def test_create_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -108,7 +119,7 @@ def test_create_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_only(self): load_things_worker(self.ckan, 'datasets', { @@ -124,7 +135,7 @@ def test_create_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_empty_dict(self): load_things_worker(self.ckan, 'datasets', { @@ -140,7 +151,7 @@ def test_create_empty_dict(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -170,7 +181,7 @@ def test_update_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -186,7 +197,7 @@ def test_update_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -204,7 +215,7 @@ def test_update_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_only(self): load_things_worker(self.ckan, 'datasets', { @@ -220,7 +231,7 @@ def test_update_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -268,7 +279,7 @@ def test_update_group(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'group-updated') + self.assertEqual(data, {'id': 'ab', 'name': 'group-updated'}) def test_update_organization_two(self): load_things_worker(self.ckan, 'organizations', { @@ -288,11 +299,11 @@ def test_update_organization_two(self): timstamp, action, error, data = json.loads(r1.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'org-updated') + self.assertEqual(data, {'id': 'cd', 'name': 'org-updated'}) timstamp, action, error, data = json.loads(r2.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'org-created') + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'org-created'}) def test_update_organization_with_users_unchanged(self): load_things_worker(self.ckan, 'organizations', { @@ -308,7 +319,7 @@ def test_update_organization_with_users_unchanged(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'users-unchanged') + self.assertEqual(data, {'id': 'used', 'name': 'users-unchanged'}) def test_update_organization_with_users_cleared(self): load_things_worker(self.ckan, 'organizations', { @@ -324,7 +335,7 @@ def test_update_organization_with_users_cleared(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'users-cleared') + self.assertEqual(data, {'id': 'unused', 'name': 'users-cleared'}) def test_parent_load_two(self): load_things(self.ckan, 'datasets', { @@ -345,6 +356,7 @@ def test_parent_load_two(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -380,6 +392,7 @@ def test_parent_load_start_max(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -418,6 +431,7 @@ def test_parent_parallel_limit(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--api-tokens': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -430,6 +444,38 @@ def test_parent_parallel_limit(self): 'ckanapi', 'load', 'datasets', '--worker']) self.assertEqual(self.worker_processes, 2) + def test_create_user(self): + load_things_worker(self.ckan, 'users', { + '--create-only': True, + '--update-only': False, + '--insecure': False, + '--api-tokens': False, + }, + stdin=BytesIO(b'{"name":"test_user"}\n'), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'test_user'}) + + def test_create_user_with_api_token(self): + load_things_worker(self.ckan, 'users', { + '--create-only': True, + '--update-only': False, + '--insecure': False, + '--api-tokens': True, + }, + stdin=BytesIO(b'{"name":"test_user","api_token_list":[{"user_id":"test_user","id":"this-is-a-token","name":"this-is-a-token","created_at":null,"last_access":null}]}\n'), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'created_tokens': ['this-is-a-token'], 'id': 'some-generated-uuid', 'name': 'test_user'}) + def _mock_worker_pool(self, cmd, processes, job_iter): self.worker_cmd = cmd self.worker_processes = processes