From c9153b8f680f34e2806ae1ea95e7fb588dd850fc Mon Sep 17 00:00:00 2001 From: Vlad0n20 <137097005+Vlad0n20@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:29:52 +0300 Subject: [PATCH 01/43] add required_metadata_template (#11680) --- ...ractprovider_required_metadata_template.py | 25 +++++++++++++++++++ osf/models/provider.py | 7 ++++++ 2 files changed, 32 insertions(+) create mode 100644 osf/migrations/0038_abstractprovider_required_metadata_template.py diff --git a/osf/migrations/0038_abstractprovider_required_metadata_template.py b/osf/migrations/0038_abstractprovider_required_metadata_template.py new file mode 100644 index 00000000000..f641e8741f3 --- /dev/null +++ b/osf/migrations/0038_abstractprovider_required_metadata_template.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2 on 2026-04-03 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0037_notification_refactor_post_release'), + ] + + operations = [ + migrations.AddField( + model_name='abstractprovider', + name='required_metadata_template', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='required_by_providers', + to='osf.cedarmetadatatemplate', + ), + ), + ] diff --git a/osf/models/provider.py b/osf/models/provider.py index 92681173240..f1dbdec65b3 100644 --- a/osf/models/provider.py +++ b/osf/models/provider.py @@ -157,6 +157,13 @@ def update_or_create_from_json(cls, provider_data, user): share_source = models.CharField(blank=True, default='', max_length=200) share_title = models.TextField(default='', blank=True) doi_prefix = models.CharField(blank=True, null=True, max_length=32) + required_metadata_template = models.ForeignKey( + 'osf.CedarMetadataTemplate', + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name='required_by_providers', + ) def __repr__(self): return ('(name={self.name!r}, default_license={self.default_license!r}, ' From b11d69cce470c8062081ce2cdbef7bc36b9eb5ea Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Fri, 3 Apr 2026 17:22:16 +0300 Subject: [PATCH 02/43] add required_metadata_template to API --- api/providers/serializers.py | 6 +++ .../views/test_collection_provider_detail.py | 38 +++++++++++++++++++ .../views/test_preprint_provider_detail.py | 38 +++++++++++++++++++ .../test_registration_provider_detail.py | 38 +++++++++++++++++++ 4 files changed, 120 insertions(+) diff --git a/api/providers/serializers.py b/api/providers/serializers.py index 7431d221fef..8f7b1ef3af2 100644 --- a/api/providers/serializers.py +++ b/api/providers/serializers.py @@ -77,6 +77,12 @@ class Meta: related_view_kwargs={'provider_id': '<_id>'}, ) + required_metadata_template = RelationshipField( + related_view='cedar-metadata-templates:cedar-metadata-template-detail', + related_view_kwargs={'template_id': ''}, + read_only=True, + ) + schemas = TypedRelationshipField( related_view='providers:registration-providers:registration-schema-list', related_view_kwargs={'provider_id': '<_id>'}, diff --git a/api_tests/providers/collections/views/test_collection_provider_detail.py b/api_tests/providers/collections/views/test_collection_provider_detail.py index 25f1f4b80ca..f4bf1f40991 100644 --- a/api_tests/providers/collections/views/test_collection_provider_detail.py +++ b/api_tests/providers/collections/views/test_collection_provider_detail.py @@ -2,6 +2,7 @@ from api.base.settings.defaults import API_BASE from api_tests.providers.mixins import ProviderExistsMixin +from osf.models import CedarMetadataTemplate from osf_tests.factories import ( CollectionProviderFactory, BrandFactory, @@ -55,3 +56,40 @@ def test_registration_provider_with_special_fields(self, app, provider_with_bran data = res.json['data'] assert data['relationships']['brand']['data']['id'] == str(brand.id) + + +@pytest.mark.django_db +class TestCollectionProviderRequiredMetadataTemplate: + + @pytest.fixture() + def provider(self): + return CollectionProviderFactory() + + @pytest.fixture() + def provider_url(self, provider): + return f'/{API_BASE}providers/collections/{provider._id}/' + + @pytest.fixture() + def cedar_template(self): + return CedarMetadataTemplate.objects.create( + schema_name='Test Schema', + cedar_id='https://repo.metadatacenter.org/templates/test', + template_version=1, + template={}, + active=True, + ) + + def test_required_metadata_template_is_null_by_default(self, app, provider, provider_url): + res = app.get(provider_url) + assert res.status_code == 200 + assert res.json['data']['relationships']['required_metadata_template']['data'] is None + + def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): + provider.required_metadata_template = cedar_template + provider.save() + + res = app.get(provider_url) + assert res.status_code == 200 + rel = res.json['data']['relationships']['required_metadata_template'] + assert rel['data']['id'] == cedar_template._id + assert rel['data']['type'] == 'cedar-metadata-templates' diff --git a/api_tests/providers/preprints/views/test_preprint_provider_detail.py b/api_tests/providers/preprints/views/test_preprint_provider_detail.py index e4d6536dcf7..241d6d35728 100644 --- a/api_tests/providers/preprints/views/test_preprint_provider_detail.py +++ b/api_tests/providers/preprints/views/test_preprint_provider_detail.py @@ -3,6 +3,7 @@ from api.base.settings.defaults import API_BASE from api.base.settings import REST_FRAMEWORK from api_tests.providers.mixins import ProviderExistsMixin +from osf.models import CedarMetadataTemplate from osf_tests.factories import ( PreprintProviderFactory, ProviderAssetFileFactory, @@ -216,3 +217,40 @@ def test_asset_attribute_correct(self, app, provider_one, provider_two, provider res = app.get(provider_two_url) assert res.json['data']['attributes']['assets'][provider_asset_two.name] == provider_asset_two.file.url + + +@pytest.mark.django_db +class TestPreprintProviderRequiredMetadataTemplate: + + @pytest.fixture() + def provider(self): + return PreprintProviderFactory() + + @pytest.fixture() + def provider_url(self, provider): + return f'/{API_BASE}providers/preprints/{provider._id}/' + + @pytest.fixture() + def cedar_template(self): + return CedarMetadataTemplate.objects.create( + schema_name='Test Schema', + cedar_id='https://repo.metadatacenter.org/templates/test', + template_version=1, + template={}, + active=True, + ) + + def test_required_metadata_template_is_null_by_default(self, app, provider, provider_url): + res = app.get(provider_url) + assert res.status_code == 200 + assert res.json['data']['relationships']['required_metadata_template']['data'] is None + + def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): + provider.required_metadata_template = cedar_template + provider.save() + + res = app.get(provider_url) + assert res.status_code == 200 + rel = res.json['data']['relationships']['required_metadata_template'] + assert rel['data']['id'] == cedar_template._id + assert rel['data']['type'] == 'cedar-metadata-templates' diff --git a/api_tests/providers/registrations/views/test_registration_provider_detail.py b/api_tests/providers/registrations/views/test_registration_provider_detail.py index 4538c3e2545..ac09d5e262b 100644 --- a/api_tests/providers/registrations/views/test_registration_provider_detail.py +++ b/api_tests/providers/registrations/views/test_registration_provider_detail.py @@ -2,6 +2,7 @@ from api.base.settings.defaults import API_BASE from api_tests.providers.mixins import ProviderDetailViewTestBaseMixin +from osf.models import CedarMetadataTemplate from osf_tests.factories import ( BrandFactory, RegistrationProviderFactory, @@ -58,3 +59,40 @@ def test_registration_provider_with_special_fields(self, app, provider_with_bran assert data['relationships']['brand']['data']['id'] == str(brand.id) assert data['attributes']['branded_discovery_page'] == provider_with_brand.branded_discovery_page + + +@pytest.mark.django_db +class TestRegistrationProviderRequiredMetadataTemplate: + + @pytest.fixture() + def provider(self): + return RegistrationProviderFactory() + + @pytest.fixture() + def provider_url(self, provider): + return f'/{API_BASE}providers/registrations/{provider._id}/' + + @pytest.fixture() + def cedar_template(self): + return CedarMetadataTemplate.objects.create( + schema_name='Test Schema', + cedar_id='https://repo.metadatacenter.org/templates/test', + template_version=1, + template={}, + active=True, + ) + + def test_required_metadata_template_is_null_by_default(self, app, provider, provider_url): + res = app.get(provider_url) + assert res.status_code == 200 + assert res.json['data']['relationships']['required_metadata_template']['data'] is None + + def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): + provider.required_metadata_template = cedar_template + provider.save() + + res = app.get(provider_url) + assert res.status_code == 200 + rel = res.json['data']['relationships']['required_metadata_template'] + assert rel['data']['id'] == cedar_template._id + assert rel['data']['type'] == 'cedar-metadata-templates' From 1034d792a7235cba57eeced492553b6714e365f7 Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Tue, 7 Apr 2026 15:34:38 +0300 Subject: [PATCH 03/43] Fix tests --- .../collections/views/test_collection_provider_detail.py | 6 +++--- .../preprints/views/test_preprint_provider_detail.py | 6 +++--- .../views/test_registration_provider_detail.py | 6 +++--- osf_tests/metadata/test_serialized_metadata.py | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api_tests/providers/collections/views/test_collection_provider_detail.py b/api_tests/providers/collections/views/test_collection_provider_detail.py index f4bf1f40991..a44c35f7c09 100644 --- a/api_tests/providers/collections/views/test_collection_provider_detail.py +++ b/api_tests/providers/collections/views/test_collection_provider_detail.py @@ -67,7 +67,7 @@ def provider(self): @pytest.fixture() def provider_url(self, provider): - return f'/{API_BASE}providers/collections/{provider._id}/' + return f'/{API_BASE}providers/collections/{provider._id}/?version=2.20' @pytest.fixture() def cedar_template(self): @@ -85,8 +85,8 @@ def test_required_metadata_template_is_null_by_default(self, app, provider, prov assert res.json['data']['relationships']['required_metadata_template']['data'] is None def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): - provider.required_metadata_template = cedar_template - provider.save() + from osf.models import AbstractProvider + AbstractProvider.objects.filter(pk=provider.pk).update(required_metadata_template=cedar_template) res = app.get(provider_url) assert res.status_code == 200 diff --git a/api_tests/providers/preprints/views/test_preprint_provider_detail.py b/api_tests/providers/preprints/views/test_preprint_provider_detail.py index 241d6d35728..575ffc52f88 100644 --- a/api_tests/providers/preprints/views/test_preprint_provider_detail.py +++ b/api_tests/providers/preprints/views/test_preprint_provider_detail.py @@ -228,7 +228,7 @@ def provider(self): @pytest.fixture() def provider_url(self, provider): - return f'/{API_BASE}providers/preprints/{provider._id}/' + return f'/{API_BASE}providers/preprints/{provider._id}/?version=2.20' @pytest.fixture() def cedar_template(self): @@ -246,8 +246,8 @@ def test_required_metadata_template_is_null_by_default(self, app, provider, prov assert res.json['data']['relationships']['required_metadata_template']['data'] is None def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): - provider.required_metadata_template = cedar_template - provider.save() + from osf.models import AbstractProvider + AbstractProvider.objects.filter(pk=provider.pk).update(required_metadata_template=cedar_template) res = app.get(provider_url) assert res.status_code == 200 diff --git a/api_tests/providers/registrations/views/test_registration_provider_detail.py b/api_tests/providers/registrations/views/test_registration_provider_detail.py index ac09d5e262b..16f453decb8 100644 --- a/api_tests/providers/registrations/views/test_registration_provider_detail.py +++ b/api_tests/providers/registrations/views/test_registration_provider_detail.py @@ -70,7 +70,7 @@ def provider(self): @pytest.fixture() def provider_url(self, provider): - return f'/{API_BASE}providers/registrations/{provider._id}/' + return f'/{API_BASE}providers/registrations/{provider._id}/?version=2.20' @pytest.fixture() def cedar_template(self): @@ -88,8 +88,8 @@ def test_required_metadata_template_is_null_by_default(self, app, provider, prov assert res.json['data']['relationships']['required_metadata_template']['data'] is None def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): - provider.required_metadata_template = cedar_template - provider.save() + from osf.models import AbstractProvider + AbstractProvider.objects.filter(pk=provider.pk).update(required_metadata_template=cedar_template) res = app.get(provider_url) assert res.status_code == 200 diff --git a/osf_tests/metadata/test_serialized_metadata.py b/osf_tests/metadata/test_serialized_metadata.py index 5dc4029aaf4..ce59b830ba1 100644 --- a/osf_tests/metadata/test_serialized_metadata.py +++ b/osf_tests/metadata/test_serialized_metadata.py @@ -210,6 +210,7 @@ def setUp(self): mock.patch('osf.models.base.Guid.objects.get_or_create', new=osfguid_sequence.get_or_create), mock.patch('django.utils.timezone.now', new=forever_now), mock.patch('osf.models.mixins.timezone.now', new=forever_now), + mock.patch('osf.models.nodelog.timezone.now', new=forever_now), mock.patch('osf.models.metaschema.RegistrationSchema.absolute_api_v2_url', new='http://fake.example/schema/for/test'), mock.patch('osf.models.node.Node.get_verified_links', return_value=[ {'target_url': 'https://foo.bar', 'resource_type': 'Other'} From 7924acc18aa33c1bedf8a7dfc09dee3e7cdcf0b4 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 7 Apr 2026 16:36:57 +0300 Subject: [PATCH 04/43] removed discovery legacy search --- website/discovery/__init__.py | 0 website/discovery/views.py | 5 ----- website/routes.py | 12 ------------ 3 files changed, 17 deletions(-) delete mode 100644 website/discovery/__init__.py delete mode 100644 website/discovery/views.py diff --git a/website/discovery/__init__.py b/website/discovery/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/website/discovery/views.py b/website/discovery/views.py deleted file mode 100644 index c17c5ca0e7a..00000000000 --- a/website/discovery/views.py +++ /dev/null @@ -1,5 +0,0 @@ -from framework.flask import redirect - - -def redirect_activity_to_search(**kwargs): - return redirect('/search/') diff --git a/website/routes.py b/website/routes.py index 226f03fb1f1..a4bbbe4f71f 100644 --- a/website/routes.py +++ b/website/routes.py @@ -45,7 +45,6 @@ from website.profile import views as profile_views from website.project import views as project_views from addons.base import views as addon_views -from website.discovery import views as discovery_views from website.conferences import views as conference_views from website.policies import views as policy_views from website.preprints import views as preprint_views @@ -524,17 +523,6 @@ def make_url_map(app): Rule('/forms/forgot_password/', 'get', website_views.forgot_password_form, json_renderer), ], prefix='/api/v1') - ### Discovery ### - - process_rules(app, [ - Rule( - ['/activity/', '/explore/activity/', '/explore/'], - 'get', - discovery_views.redirect_activity_to_search, - notemplate - ), - ]) - ### Auth ### process_rules(app, [ From 35dca89827824352df8f10eac7b34b7c5f89a793 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 7 Apr 2026 16:53:59 +0300 Subject: [PATCH 05/43] removed /search/ legacy route --- osf_tests/test_search_views.py | 5 - webpack.common.config.js | 1 - website/routes.py | 6 - website/search/views.py | 3 - website/static/css/pages/search-page.css | 112 ------- website/static/css/search-bar.css | 73 ----- website/static/js/pages/search-page.js | 6 - website/templates/search.mako | 393 ----------------------- 8 files changed, 599 deletions(-) delete mode 100644 website/static/css/pages/search-page.css delete mode 100644 website/static/css/search-bar.css delete mode 100644 website/static/js/pages/search-page.js delete mode 100644 website/templates/search.mako diff --git a/osf_tests/test_search_views.py b/osf_tests/test_search_views.py index 858decb9d1a..edc2bc8026b 100644 --- a/osf_tests/test_search_views.py +++ b/osf_tests/test_search_views.py @@ -102,11 +102,6 @@ def test_search_views(self): assert page == 2 assert pages == 3 - # Test search projects - url = '/search/' - res = self.app.get(url, query_string={'q': self.project.title}) - assert res.status_code == 200 - # Test search node res = self.app.post( api_url_for('search_node'), diff --git a/webpack.common.config.js b/webpack.common.config.js index e76b755dbdf..811645937c8 100644 --- a/webpack.common.config.js +++ b/webpack.common.config.js @@ -46,7 +46,6 @@ var entry = { 'conference-page': staticPath('js/pages/conference-page.js'), 'meetings-page': staticPath('js/pages/meetings-page.js'), 'view-file-tree-page': staticPath('js/pages/view-file-tree-page.js'), - 'search-page': staticPath('js/pages/search-page.js'), 'profile-settings-addons-page': staticPath('js/pages/profile-settings-addons-page.js'), 'forgotpassword-page': staticPath('js/pages/forgotpassword-page.js'), 'resetpassword-page': staticPath('js/pages/resetpassword-page.js'), diff --git a/website/routes.py b/website/routes.py index a4bbbe4f71f..ab291fab096 100644 --- a/website/routes.py +++ b/website/routes.py @@ -937,12 +937,6 @@ def make_url_map(app): process_rules(app, [ - Rule( - '/search/', - 'get', - search_views.search_view, - OsfWebRenderer('search.mako', trust=False) - ), Rule( '/share/registration/', 'get', diff --git a/website/search/views.py b/website/search/views.py index 828cd0e5661..00fec37891d 100644 --- a/website/search/views.py +++ b/website/search/views.py @@ -13,7 +13,6 @@ from framework.utils import sanitize_html from website import language from osf.models import OSFUser, AbstractNode -from website import settings from website.project.views.contributor import get_node_contributors_abbrev from website.search import exceptions import website.search.search as search @@ -71,8 +70,6 @@ def search_search(**kwargs): results['time'] = round(time.time() - tick, 2) return results -def search_view(): - return {'shareUrl': settings.SHARE_URL}, def conditionally_add_query_item(query, item, condition, value): """ Helper for the search_projects_by_title function which will add a condition to a query diff --git a/website/static/css/pages/search-page.css b/website/static/css/pages/search-page.css deleted file mode 100644 index 4369eebabf5..00000000000 --- a/website/static/css/pages/search-page.css +++ /dev/null @@ -1,112 +0,0 @@ -.add-query -{ - width: 190px; - margin-left: 10px; - display: inline-block; - font-weight: normal; -} -.navigate -{ - padding:25px; - font-size: 18px; -} -.remove-button -{ - color: #ffffff; -} -.search-types a -{ - font-size: 16px; -} - -.registration a { - color: #666; -} - -.registration small{ - color: #000000; -} - -.registration { - background-color: #AAA; - color: #ffffff; -} - -.query-label -{ - margin-right: .5em; -} -.result -{ - margin-bottom: 10px; - overflow: hidden; -} -.title h4 -{ - padding: 0; - padding-bottom: 5px; - margin: 0; -} -.private-title -{ - font-weight: normal; - font-style: italic; -} -.search-field -{ - padding: 0; - padding-top: 2px; - padding-bottom: 2px; - margin: 0; - line-height: 16px; -} -.search-field p -{ - margin: 0; -} -.description h5 -{ - padding: 0; - padding-bottom: 4px; - margin: 0; - font-weight: normal; - line-height: 16px; -} -.search-tags -{ - padding-top: 2px; - line-height: 24px; -} -.search-tags a -{ - margin-right: .5em; -} - -div.search-result { - margin-bottom: 10pt; - padding: 10px; - border-bottom: 1px solid #d3d3d3; - overflow-x: hidden; -} - -.search-results.search-result h5 small{ - color: white; - font-size: 10pt; -} - - -.search-results.search-result { - overflow-x: hidden; - -} -/* Search Contributors */ -.search-contributor-result td { - padding-right: 10px; -} -.search-contributor-links { - padding-top: 10px; -} - -#searchControls .osf-search { - z-index: 1029; /* No larger than 1030 */ -} diff --git a/website/static/css/search-bar.css b/website/static/css/search-bar.css deleted file mode 100644 index ef52190258f..00000000000 --- a/website/static/css/search-bar.css +++ /dev/null @@ -1,73 +0,0 @@ -/* Search bar */ -.search-label-placeholder { - position:absolute; - left: 0; - right: 0; - bottom: 0; - font-size: 20px; - color: #738EA2; - font-weight: 300; - visibility: hidden; -} - -.osf-search { - padding: 10px 0; - background: #B8ECC0; - position: fixed; - width: 100%; - box-shadow: 0 0 9px -2px #464545; - left: 0; - top: 50px; - z-index: 1030; /* Should not less than 1030 */ -} - -.osf-search-input { - background: none; - border: none; - box-shadow: none; - border-bottom: 1px dotted #FFF; - border-radius: 0; - padding: 0 0; - font-size: 20px; - color: #214762; - font-weight: 300; -} -.osf-search-input:focus { - outline : 0 !important; - box-shadow: none; - border-bottom: 1px dotted #FFF; -} -.osf-search-btn { - color: #214762; - background: rgba(0, 0, 0, 0); -} - -.osf-search-btn:hover { - color: #738EA2; -} - - -#searchPageFullBar::-webkit-input-placeholder { - color: #738EA2; -} - -#searchPageFullBar:-moz-placeholder { /* Firefox 18- */ - color: #738EA2; -} - -#searchPageFullBar::-moz-placeholder { /* Firefox 19+ */ - color: #738EA2; -} - -#searchPageFullBar:-ms-input-placeholder { - color: #738EA2; -} -#searchControls>.row { - padding-top: 60px; -} - -/*Moves maintenance alert under search bar*/ -#maintenance.alert { - position: relative; - top: 55px; -} diff --git a/website/static/js/pages/search-page.js b/website/static/js/pages/search-page.js deleted file mode 100644 index ca5144148c0..00000000000 --- a/website/static/js/pages/search-page.js +++ /dev/null @@ -1,6 +0,0 @@ -var $ = require('jquery'); -$('input[name=q]').remove(); - -var Search = require('../search.js'); -require('../../css/search-bar.css'); -new Search('#searchControls', '/api/v1/search/', ''); diff --git a/website/templates/search.mako b/website/templates/search.mako deleted file mode 100644 index 7961e5a80ee..00000000000 --- a/website/templates/search.mako +++ /dev/null @@ -1,393 +0,0 @@ -<%inherit file="base.mako"/> -<%def name="title()">Search -<%def name="stylesheets()"> - ${parent.stylesheets()} - - - -<%def name="content()"> -
- <%include file='./search_bar.mako' /> -
-
-
- -
-
-
-
    - - -
  • - -
    • - - -
  • - -
    • - -
-
-
- -
-
-

Improve your search:

- - - - - - - - - - - - - - - - - - - - -
-
-
- - -
-
- -
- -
-
-
- - - - - - - - -
-
-
- - -
-
-
-
-
-
- - - - - - - - - - - - - -<%def name="javascript_bottom()"> - - - - - - From 7a459826b3cb01ca09c5a86ed39aa9beceaae71a Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 7 Apr 2026 17:02:56 +0300 Subject: [PATCH 06/43] remove api/v1/user/search/ legacy route --- osf_tests/test_elastic_search.py | 95 ------------------------------- osf_tests/test_search_views.py | 58 ------------------- website/routes.py | 5 -- website/search/elastic_search.py | 86 +--------------------------- website/search/search.py | 8 --- website/search/views.py | 15 ----- website/static/js/contribAdder.js | 28 ++------- 7 files changed, 5 insertions(+), 290 deletions(-) diff --git a/osf_tests/test_elastic_search.py b/osf_tests/test_elastic_search.py index b6c20e5af4a..e21ed1f44b2 100644 --- a/osf_tests/test_elastic_search.py +++ b/osf_tests/test_elastic_search.py @@ -1015,101 +1015,6 @@ def test_tag_aggregation(self): assert doc['key'] in tags -@pytest.mark.enable_search -@pytest.mark.enable_enqueue_task -class TestAddContributor(OsfTestCase): - # Tests of the search.search_contributor method - - def setUp(self): - self.name1 = 'Roger1 Taylor1' - self.name2 = 'John2 Deacon2' - self.name3 = 'j\xc3\xb3ebert3 Smith3' - self.name4 = 'B\xc3\xb3bbert4 Jones4' - - with run_celery_tasks(): - super().setUp() - self.user = factories.UserFactory(fullname=self.name1) - self.user3 = factories.UserFactory(fullname=self.name3) - - def test_unreg_users_dont_show_in_search(self): - unreg = factories.UnregUserFactory() - contribs = search.search_contributor(unreg.fullname) - assert len(contribs['users']) == 0 - - def test_unreg_users_do_show_on_projects(self): - with run_celery_tasks(): - unreg = factories.UnregUserFactory(fullname='Robert Paulson') - self.project = factories.ProjectFactory( - title='Glamour Rock', - creator=unreg, - is_public=True, - ) - results = query(unreg.fullname)['results'] - assert len(results) == 1 - - def test_search_fullname(self): - # Searching for full name yields exactly one result. - contribs = search.search_contributor(self.name1) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name2) - assert len(contribs['users']) == 0 - - def test_search_firstname(self): - # Searching for first name yields exactly one result. - contribs = search.search_contributor(self.name1.split(' ')[0]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name2.split(' ')[0]) - assert len(contribs['users']) == 0 - - def test_search_partial(self): - # Searching for part of first name yields exactly one - # result. - contribs = search.search_contributor(self.name1.split(' ')[0][:-1]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name2.split(' ')[0][:-1]) - assert len(contribs['users']) == 0 - - def test_search_fullname_special_character(self): - # Searching for a fullname with a special character yields - # exactly one result. - contribs = search.search_contributor(self.name3) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name4) - assert len(contribs['users']) == 0 - - def test_search_firstname_special_character(self): - # Searching for a first name with a special character yields - # exactly one result. - contribs = search.search_contributor(self.name3.split(' ')[0]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name4.split(' ')[0]) - assert len(contribs['users']) == 0 - - def test_search_partial_special_character(self): - # Searching for a partial name with a special character yields - # exactly one result. - contribs = search.search_contributor(self.name3.split(' ')[0][:-1]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name4.split(' ')[0][:-1]) - assert len(contribs['users']) == 0 - - def test_search_profile(self): - orcid = '123456' - user = factories.UserFactory() - user.social['orcid'] = orcid - user.save() - contribs = search.search_contributor(orcid) - assert len(contribs['users']) == 1 - assert len(contribs['users'][0]['social']) == 1 - assert contribs['users'][0]['social']['orcid'] == user.social_links['orcid'] - - @pytest.mark.enable_search @pytest.mark.enable_enqueue_task class TestProjectSearchResults(OsfTestCase): diff --git a/osf_tests/test_search_views.py b/osf_tests/test_search_views.py index edc2bc8026b..f3110699c96 100644 --- a/osf_tests/test_search_views.py +++ b/osf_tests/test_search_views.py @@ -44,64 +44,6 @@ def tearDown(self): reason="for some reason elasticsearch environment isn't properly set up locally, so this test passes only in CI" ) def test_search_views(self): - # Test search contributor - url = api_url_for('search_contributor') - res = self.app.get(url, query_string={'query': self.contrib.fullname}) - assert res.status_code == 200 - result = res.json['users'] - assert len(result) == 1 - brian = result[0] - assert brian['fullname'] == self.contrib.fullname - assert 'profile_image_url' in brian - assert brian['registered'] == self.contrib.is_registered - assert brian['active'] == self.contrib.is_active - - # Test search pagination - res = self.app.get(url, query_string={'query': 'fr'}) - assert res.status_code == 200 - result = res.json['users'] - pages = res.json['pages'] - page = res.json['page'] - assert len(result) == 5 - assert pages == 3 - assert page == 0 - - # Test default page 1 - res = self.app.get(url, query_string={'query': 'fr', 'page': 1}) - assert res.status_code == 200 - result = res.json['users'] - page = res.json['page'] - assert len(result) == 5 - assert page == 1 - - # Test default page 2 - res = self.app.get(url, query_string={'query': 'fr', 'page': 2}) - assert res.status_code == 200 - result = res.json['users'] - page = res.json['page'] - assert len(result) == 4 - assert page == 2 - - # Test smaller pages - res = self.app.get(url, query_string={'query': 'fr', 'size': 5}) - assert res.status_code == 200 - result = res.json['users'] - pages = res.json['pages'] - page = res.json['page'] - assert len(result) == 5 - assert page == 0 - assert pages == 3 - - # Test smaller pages page 2 - res = self.app.get(url, query_string={'query': 'fr', 'size': 5, 'page': 2}) - assert res.status_code == 200 - result = res.json['users'] - pages = res.json['pages'] - page = res.json['page'] - assert len(result) == 4 - assert page == 2 - assert pages == 3 - # Test search node res = self.app.post( api_url_for('search_node'), diff --git a/website/routes.py b/website/routes.py index ab291fab096..a9f3890d58c 100644 --- a/website/routes.py +++ b/website/routes.py @@ -943,11 +943,6 @@ def make_url_map(app): {'register': settings.SHARE_REGISTRATION_URL}, json_renderer ), - Rule( - '/api/v1/user/search/', - 'get', search_views.search_contributor, - json_renderer - ), Rule( '/api/v1/search/node/', 'post', diff --git a/website/search/elastic_search.py b/website/search/elastic_search.py index 107feda2010..ab9f339e0da 100644 --- a/website/search/elastic_search.py +++ b/website/search/elastic_search.py @@ -1,8 +1,6 @@ import copy import functools import logging -import math -import re import unicodedata from framework import sentry @@ -26,11 +24,9 @@ from osf.utils.sanitize import unescape_entities from osf.utils.workflows import CollectionSubmissionStates from website import settings -from website.filters import profile_image_url from osf.models.licenses import serialize_node_license_record from website.search import exceptions -from website.search.util import build_query, clean_splitters -from website.views import validate_page_num +from website.search.util import clean_splitters logger = logging.getLogger(__name__) @@ -886,86 +882,6 @@ def delete_group_doc(deleted_id, index=None): index = index or INDEX client().delete(index=index, doc_type='group', id=deleted_id, refresh=True, ignore=[404]) -@requires_search -def search_contributor(query, page=0, size=10, exclude=None, current_user=None): - """Search for contributors to add to a project using elastic search. Request must - include JSON data with a "query" field. - - :param query: The substring of the username to search for - :param page: For pagination, the page number to use for results - :param size: For pagination, the number of results per page - :param exclude: A list of User objects to exclude from the search - :param current_user: A User object of the current user - - :return: List of dictionaries, each containing the ID, full name, - most recent employment and education, profile_image URL of an OSF user - - """ - start = (page * size) - items = re.split(r'[\s-]+', query) - exclude = exclude or [] - normalized_items = [] - for item in items: - normalized_item = unicodedata.normalize('NFKD', item) - normalized_items.append(normalized_item) - items = normalized_items - - query = ' AND '.join(f'{re.escape(item)}*~' for item in items) + \ - ''.join(f' NOT id:"{excluded._id}"' for excluded in exclude) - - results = search(build_query(query, start=start, size=size), index=INDEX, doc_type='user') - docs = results['results'] - pages = math.ceil(results['counts'].get('user', 0) / size) - validate_page_num(page, pages) - - users = [] - for doc in docs: - # TODO: use utils.serialize_user - user = OSFUser.load(doc['id']) - - if current_user and current_user._id == user._id: - n_projects_in_common = -1 - elif current_user: - n_projects_in_common = current_user.n_projects_in_common(user) - else: - n_projects_in_common = 0 - - if user is None: - logger.error(f"Could not load user {doc['id']}") - continue - if user.is_active: # exclude merged, unregistered, etc. - current_employment = None - education = None - - if user.jobs: - current_employment = user.jobs[0]['institution'] - - if user.schools: - education = user.schools[0]['institution'] - - users.append({ - 'fullname': doc['user'], - 'id': doc['id'], - 'employment': current_employment, - 'education': education, - 'social': user.social_links, - 'n_projects_in_common': n_projects_in_common, - 'profile_image_url': profile_image_url(settings.PROFILE_IMAGE_PROVIDER, - user, - use_ssl=True, - size=settings.PROFILE_IMAGE_MEDIUM), - 'profile_url': user.profile_url, - 'registered': user.is_registered, - 'active': user.is_active - }) - - return { - 'users': users, - 'total': results['counts']['total'], - 'pages': pages, - 'page': page, - } - def serialize_guid_metadata(guid): serialized_guid_metadata = {} diff --git a/website/search/search.py b/website/search/search.py index 4632ad0b3dd..d2a3af1b58b 100644 --- a/website/search/search.py +++ b/website/search/search.py @@ -159,11 +159,3 @@ def delete_index(index): def create_index(index=None): index = index or settings.ELASTIC_INDEX search_engine.create_index(index=index) - - -@requires_search -def search_contributor(query, page=0, size=10, exclude=None, current_user=None): - exclude = exclude or [] - result = search_engine.search_contributor(query=query, page=page, size=size, - exclude=exclude, current_user=current_user) - return result diff --git a/website/search/views.py b/website/search/views.py index 00fec37891d..4455bff7614 100644 --- a/website/search/views.py +++ b/website/search/views.py @@ -6,11 +6,9 @@ from django.db.models import Q from flask import request -from framework.auth.decorators import collect_auth from framework.auth.decorators import must_be_logged_in from framework.exceptions import HTTPError from framework import sentry -from framework.utils import sanitize_html from website import language from osf.models import OSFUser, AbstractNode from website.project.views.contributor import get_node_contributors_abbrev @@ -184,16 +182,3 @@ def process_project_search_results(results, **kwargs): }) return ret - - -@collect_auth -def search_contributor(auth): - user = auth.user if auth else None - nid = request.args.get('excludeNode') - exclude = AbstractNode.load(nid).contributors if nid else [] - # TODO: Determine whether bleach is appropriate for ES payload. Also, inconsistent with website.sanitize.util.strip_html - query = sanitize_html(request.args.get('query', ''), tags=set(), strip=True) - page = int(sanitize_html(request.args.get('page', '0'), tags=set(), strip=True)) - size = int(sanitize_html(request.args.get('size', '5'), tags=set(), strip=True)) - return search.search_contributor(query=query, page=page, size=size, - exclude=exclude, current_user=user) diff --git a/website/static/js/contribAdder.js b/website/static/js/contribAdder.js index b7436c4d9d2..216a3a8797d 100644 --- a/website/static/js/contribAdder.js +++ b/website/static/js/contribAdder.js @@ -210,30 +210,10 @@ AddContributorViewModel = oop.extend(Paginator, { var self = this; self.doneSearching(false); self.notification(false); - if (self.query()) { - return $.getJSON( - '/api/v1/user/search/', { - query: self.query(), - page: self.pageToGet - }, - function (result) { - var contributors = result.users.map(function (userData) { - userData.added = (self.contributors().indexOf(userData.id) !== -1); - return new Contributor(userData); - }); - self.doneSearching(true); - self.results(contributors); - self.currentPage(result.page); - self.numberOfPages(result.pages); - self.addNewPaginators(false); - } - ); - } else { - self.results([]); - self.currentPage(0); - self.totalPages(0); - self.doneSearching(true); - } + self.results([]); + self.currentPage(0); + self.totalPages(0); + self.doneSearching(true); } }, getContributors: function () { From f3e63d48bf8c839e4b056b2e693de8ef893740f3 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 7 Apr 2026 17:07:57 +0300 Subject: [PATCH 07/43] removed api/v1/search/node/ legacy route --- osf_tests/test_search_views.py | 32 -------------- tests/test_serializers.py | 15 +------ website/project/views/node.py | 76 +--------------------------------- website/routes.py | 6 --- 4 files changed, 4 insertions(+), 125 deletions(-) diff --git a/osf_tests/test_search_views.py b/osf_tests/test_search_views.py index f3110699c96..87175cb05bb 100644 --- a/osf_tests/test_search_views.py +++ b/osf_tests/test_search_views.py @@ -44,38 +44,6 @@ def tearDown(self): reason="for some reason elasticsearch environment isn't properly set up locally, so this test passes only in CI" ) def test_search_views(self): - # Test search node - res = self.app.post( - api_url_for('search_node'), - json={'query': self.project.title}, - auth=factories.AuthUserFactory().auth - ) - assert res.status_code == 200 - - # Test search node includePublic true - res = self.app.post( - api_url_for('search_node'), - json={'query': 'a', 'includePublic': True}, - auth=self.user_one.auth - ) - node_ids = [node['id'] for node in res.json['nodes']] - assert self.project_private_user_one._id in node_ids - assert self.project_public_user_one._id in node_ids - assert self.project_public_user_two._id in node_ids - assert self.project_private_user_two._id not in node_ids - - # Test search node includePublic false - res = self.app.post( - api_url_for('search_node'), - json={'query': 'a', 'includePublic': False}, - auth=self.user_one.auth - ) - node_ids = [node['id'] for node in res.json['nodes']] - assert self.project_private_user_one._id in node_ids - assert self.project_public_user_one._id in node_ids - assert self.project_public_user_two._id not in node_ids - assert self.project_private_user_two._id not in node_ids - # Test search user url = '/api/v1/search/user/' res = self.app.get(url, query_string={'q': 'Umwali'}) diff --git a/tests/test_serializers.py b/tests/test_serializers.py index 686c7a4c8ea..f7d9b47037e 100644 --- a/tests/test_serializers.py +++ b/tests/test_serializers.py @@ -1,4 +1,3 @@ -from unittest import mock import datetime as dt import pytest @@ -11,11 +10,11 @@ ) from osf.models import NodeRelation from osf.utils import permissions -from tests.base import OsfTestCase, get_default_metaschema +from tests.base import OsfTestCase from framework.auth import Auth from tests.utils import capture_notifications -from website.project.views.node import _view_project, _serialize_node_search, _get_children, _get_readable_descendants +from website.project.views.node import _view_project, _get_children, _get_readable_descendants from website.views import serialize_node_summary from website.profile import utils from website import filters, settings @@ -189,16 +188,6 @@ def test_serialize_node_summary_child_exists(self): result = _view_project(parent_node, Auth(user)) assert result['node']['child_exists'] == True - def test_serialize_node_search_returns_only_visible_contributors(self): - node = NodeFactory() - non_visible_contributor = UserFactory() - node.add_contributor(non_visible_contributor, visible=False) - serialized_node = _serialize_node_search(node) - - assert serialized_node['firstAuthor'] == node.visible_contributors[0].family_name - assert len(node.visible_contributors) == 1 - assert not serialized_node['etal'] - @pytest.mark.enable_bookmark_creation class TestViewProject(OsfTestCase): diff --git a/website/project/views/node.py b/website/project/views/node.py index 94e4a63fe47..9315eb59065 100644 --- a/website/project/views/node.py +++ b/website/project/views/node.py @@ -1,7 +1,6 @@ import os import logging from rest_framework import status as http_status -import math from collections import defaultdict import waffle @@ -9,7 +8,7 @@ from django.apps import apps from django.utils import timezone from django.core.exceptions import ValidationError -from django.db.models import Q, OuterRef, Subquery +from django.db.models import OuterRef, Subquery from framework import status from framework.forms import push_errors_to_status @@ -53,7 +52,7 @@ from osf.utils.permissions import ADMIN, READ, WRITE, CREATOR_PERMISSIONS, ADMIN_NODE from osf.utils.workflows import CollectionSubmissionStates from website import settings -from website.views import find_bookmark_collection, validate_page_num +from website.views import find_bookmark_collection from website.views import serialize_node_summary, get_storage_region_list from website.profile import utils from addons.mendeley.provider import MendeleyCitationsProvider @@ -1229,77 +1228,6 @@ def project_private_link_edit(auth, **kwargs): ) -def _serialize_node_search(node): - """Serialize a node for use in pointer search. - - :param Node node: Node to serialize - :return: Dictionary of node data - - """ - data = { - 'id': node._id, - 'title': node.title, - 'etal': len(node.visible_contributors) > 1, - 'isRegistration': node.is_registration - } - if node.is_registration: - data['title'] += ' (registration)' - data['dateRegistered'] = node.registered_date.isoformat() - else: - data['dateCreated'] = node.created.isoformat() - data['dateModified'] = node.modified.isoformat() - - first_author = node.visible_contributors[0] - data['firstAuthor'] = first_author.family_name or first_author.given_name or first_author.fullname - - return data - - -@must_be_logged_in -def search_node(auth, **kwargs): - """ - - """ - # Get arguments - node = AbstractNode.load(request.json.get('nodeId')) - include_public = request.json.get('includePublic') - size = float(request.json.get('size', '5').strip()) - page = request.json.get('page', 0) - query = request.json.get('query', '').strip() - - start = (page * size) - if not query: - return {'nodes': []} - - # Exclude current node from query if provided - nin = [node.id] + list(node._nodes.values_list('pk', flat=True)) if node else [] - - can_view_query = Q(_contributors=auth.user) - if include_public: - can_view_query = can_view_query | Q(is_public=True) - - nodes = (AbstractNode.objects.filter( - can_view_query, - title__icontains=query, - is_deleted=False - ).exclude(id__in=nin).exclude(type='osf.collection')) - - count = nodes.count() - pages = math.ceil(count / size) - validate_page_num(page, pages) - - return { - 'nodes': [ - _serialize_node_search(each) - for each in nodes[start: start + size] - if each.contributors - ], - 'total': count, - 'pages': pages, - 'page': page - } - - def _add_pointers(node, pointers, auth): """ diff --git a/website/routes.py b/website/routes.py index a9f3890d58c..4f4bb53ba29 100644 --- a/website/routes.py +++ b/website/routes.py @@ -943,12 +943,6 @@ def make_url_map(app): {'register': settings.SHARE_REGISTRATION_URL}, json_renderer ), - Rule( - '/api/v1/search/node/', - 'post', - project_views.node.search_node, - json_renderer, - ), ]) From f6abdb46584ae0a062051a641a846b2d97b11ec7 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 7 Apr 2026 17:13:09 +0300 Subject: [PATCH 08/43] removed api/v1/share/search/ legacy route --- website/routes.py | 1 - website/views.py | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/website/routes.py b/website/routes.py index 4f4bb53ba29..336532a4fc7 100644 --- a/website/routes.py +++ b/website/routes.py @@ -952,7 +952,6 @@ def make_url_map(app): Rule(['/search/', '/search//'], ['get', 'post'], search_views.search_search, json_renderer), Rule('/search/projects/', 'get', search_views.search_projects_by_title, json_renderer), - Rule('/share/search/', 'get', website_views.legacy_share_v1_search, json_renderer), ], prefix='/api/v1') diff --git a/website/views.py b/website/views.py index 1a4bf4942da..74a81f0501a 100644 --- a/website/views.py +++ b/website/views.py @@ -333,16 +333,6 @@ def redirect_to_registration_workflow(**kwargs): return redirect(furl(DOMAIN).add(path='registries/osf/new').url) -# Return error for legacy SHARE v1 search route -def legacy_share_v1_search(**kwargs): - return HTTPError( - http_status.HTTP_400_BAD_REQUEST, - data=dict( - message_long=f'Please use v2 of the SHARE search API available at {settings.SHARE_URL}api/v2/share/search/creativeworks/_search.' - ) - ) - - def get_storage_region_list(user, node=False): if not user: # Preserves legacy frontend test behavior return [] From be6e09018b9483a4d1c8203361d189831c4808cb Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 14:08:00 +0300 Subject: [PATCH 09/43] removed api/v1/search/projects/ legacy route --- osf_tests/test_search_views.py | 133 --------------------------------- website/routes.py | 1 - website/search/views.py | 87 +-------------------- 3 files changed, 1 insertion(+), 220 deletions(-) diff --git a/osf_tests/test_search_views.py b/osf_tests/test_search_views.py index 87175cb05bb..d34f85d84ca 100644 --- a/osf_tests/test_search_views.py +++ b/osf_tests/test_search_views.py @@ -5,8 +5,6 @@ from osf_tests import factories from tests.base import OsfTestCase -from website.util import api_url_for -from website.views import find_bookmark_collection from osf.external.spam import tasks as spam_tasks @@ -116,134 +114,3 @@ def test_search_views(self): assert res.json['results'][0]['social']['baiduScholar'] == f'http://xueshu.baidu.com/scholarID/{user_two.given_name}' assert res.json['results'][0]['social']['linkedIn'] == f'https://www.linkedin.com/{user_two.given_name}' assert res.json['results'][0]['social']['scholar'] == f'http://scholar.google.com/citations?user={user_two.given_name}' - - -@pytest.mark.enable_bookmark_creation -class TestODMTitleSearch(OsfTestCase): - """ Docs from original method: - :arg term: The substring of the title. - :arg category: Category of the node. - :arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search. - :arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search. - :arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search. - :arg includePublic: yes or no. Whether the projects listed should include public projects. - :arg includeContributed: yes or no. Whether the search should include projects the current user has - contributed to. - :arg ignoreNode: a list of nodes that should not be included in the search. - :return: a list of dictionaries of projects - """ - - def setUp(self): - super().setUp() - - self.user = factories.AuthUserFactory() - self.user_two = factories.AuthUserFactory() - self.project = factories.ProjectFactory(creator=self.user, title='foo') - self.project_two = factories.ProjectFactory(creator=self.user_two, title='bar') - self.public_project = factories.ProjectFactory(creator=self.user_two, is_public=True, title='baz') - self.registration_project = factories.RegistrationFactory(creator=self.user, title='qux') - self.folder = factories.CollectionFactory(creator=self.user, title='quux') - self.dashboard = find_bookmark_collection(self.user) - self.url = api_url_for('search_projects_by_title') - - def test_search_projects_by_title(self): - res = self.app.get(self.url, query_string={'term': self.project.title}, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.public_project.title, - 'includePublic': 'yes', - 'includeContributed': 'no' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.project.title, - 'includePublic': 'no', - 'includeContributed': 'yes' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.project.title, - 'includePublic': 'no', - 'includeContributed': 'yes', - 'isRegistration': 'no' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.project.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isRegistration': 'either' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.public_project.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isRegistration': 'either' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.registration_project.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isRegistration': 'either' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 2 - res = self.app.get(self.url, - query_string={ - 'term': self.registration_project.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isRegistration': 'no' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 1 - res = self.app.get(self.url, - query_string={ - 'term': self.folder.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isFolder': 'yes' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 0 - res = self.app.get(self.url, - query_string={ - 'term': self.folder.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isFolder': 'no' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 0 - res = self.app.get(self.url, - query_string={ - 'term': self.dashboard.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isFolder': 'no' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 0 - res = self.app.get(self.url, - query_string={ - 'term': self.dashboard.title, - 'includePublic': 'yes', - 'includeContributed': 'yes', - 'isFolder': 'yes' - }, auth=self.user.auth) - assert res.status_code == 200 - assert len(res.json) == 0 diff --git a/website/routes.py b/website/routes.py index 336532a4fc7..ce326426c27 100644 --- a/website/routes.py +++ b/website/routes.py @@ -951,7 +951,6 @@ def make_url_map(app): process_rules(app, [ Rule(['/search/', '/search//'], ['get', 'post'], search_views.search_search, json_renderer), - Rule('/search/projects/', 'get', search_views.search_projects_by_title, json_renderer), ], prefix='/api/v1') diff --git a/website/search/views.py b/website/search/views.py index 4455bff7614..7553bfed1ce 100644 --- a/website/search/views.py +++ b/website/search/views.py @@ -3,14 +3,13 @@ import logging import time -from django.db.models import Q from flask import request from framework.auth.decorators import must_be_logged_in from framework.exceptions import HTTPError from framework import sentry from website import language -from osf.models import OSFUser, AbstractNode +from osf.models import OSFUser from website.project.views.contributor import get_node_contributors_abbrev from website.search import exceptions import website.search.search as search @@ -69,90 +68,6 @@ def search_search(**kwargs): return results -def conditionally_add_query_item(query, item, condition, value): - """ Helper for the search_projects_by_title function which will add a condition to a query - It will give an error if the proper search term is not used. - :param query: The modular ODM query that you want to modify - :param item: the field to query on - :param condition: yes, no, or either - :return: the modified query - """ - - condition = condition.lower() - - if condition == 'yes': - return query & Q(**{item: value}) - elif condition == 'no': - return query & ~Q(**{item: value}) - elif condition == 'either': - return query - - raise HTTPError(http_status.HTTP_400_BAD_REQUEST) - - -@must_be_logged_in -def search_projects_by_title(**kwargs): - """ Search for nodes by title. Can pass in arguments from the URL to modify the search - :arg term: The substring of the title. - :arg category: Category of the node. - :arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search. - :arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search. - :arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search. - :arg includePublic: yes or no. Whether the projects listed should include public projects. - :arg includeContributed: yes or no. Whether the search should include projects the current user has - contributed to. - :arg ignoreNode: a list of nodes that should not be included in the search. - :return: a list of dictionaries of projects - - """ - # TODO(fabianvf): At some point, it would be nice to do this with elastic search - user = kwargs['auth'].user - - term = request.args.get('term', '') - max_results = int(request.args.get('maxResults', '10')) - category = request.args.get('category', 'project').lower() - is_deleted = request.args.get('isDeleted', 'no').lower() - is_collection = request.args.get('isFolder', 'no').lower() - is_registration = request.args.get('isRegistration', 'no').lower() - include_public = request.args.get('includePublic', 'yes').lower() - include_contributed = request.args.get('includeContributed', 'yes').lower() - ignore_nodes = request.args.getlist('ignoreNode', []) - - matching_title = Q( - title__icontains=term, # search term (case-insensitive) - category=category # is a project - ) - - matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted, True) - matching_title = conditionally_add_query_item(matching_title, 'type', is_registration, 'osf.registration') - matching_title = conditionally_add_query_item(matching_title, 'type', is_collection, 'osf.collection') - - if len(ignore_nodes) > 0: - for node_id in ignore_nodes: - matching_title = matching_title & ~Q(_id=node_id) - - my_projects = [] - my_project_count = 0 - public_projects = [] - - if include_contributed == 'yes': - my_projects = AbstractNode.objects.filter( - matching_title & - Q(_contributors=user) # user is a contributor - )[:max_results] - my_project_count = my_project_count - - if my_project_count < max_results and include_public == 'yes': - public_projects = AbstractNode.objects.filter( - matching_title & - Q(is_public=True) # is public - )[:max_results - my_project_count] - - results = list(my_projects) + list(public_projects) - ret = process_project_search_results(results, **kwargs) - return ret - - @must_be_logged_in def process_project_search_results(results, **kwargs): """ From a8c11bd184b221bab81f2bf4793bd8ace3733cf0 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 14:25:35 +0300 Subject: [PATCH 10/43] removed api/v1/search/ and api/v1/search// legacy routes --- osf_tests/test_search_views.py | 116 --------------------------------- website/language.py | 2 - website/routes.py | 9 --- website/search/views.py | 59 ----------------- 4 files changed, 186 deletions(-) delete mode 100644 osf_tests/test_search_views.py diff --git a/osf_tests/test_search_views.py b/osf_tests/test_search_views.py deleted file mode 100644 index d34f85d84ca..00000000000 --- a/osf_tests/test_search_views.py +++ /dev/null @@ -1,116 +0,0 @@ -from os import environ -from unittest import mock - -import pytest - -from osf_tests import factories -from tests.base import OsfTestCase -from osf.external.spam import tasks as spam_tasks - - -@pytest.mark.enable_search -@pytest.mark.enable_enqueue_task -class TestSearchViews(OsfTestCase): - - def setUp(self): - super().setUp() - import website.search.search as search - search.delete_all() - - robbie = factories.UserFactory(fullname='Robbie Williams') - self.project = factories.ProjectFactory(creator=robbie) - self.contrib = factories.UserFactory(fullname='Brian May') - for i in range(0, 12): - factories.UserFactory(fullname=f'Freddie Mercury{i}') - - self.user_one = factories.AuthUserFactory() - self.user_two = factories.AuthUserFactory() - self.project_private_user_one = factories.ProjectFactory(title='aaa', creator=self.user_one, is_public=False) - self.project_private_user_two = factories.ProjectFactory(title='aaa', creator=self.user_two, is_public=False) - self.project_public_user_one = factories.ProjectFactory(title='aaa', creator=self.user_one, is_public=True) - self.project_public_user_two = factories.ProjectFactory(title='aaa', creator=self.user_two, is_public=True) - - def tearDown(self): - super().tearDown() - import website.search.search as search - search.delete_all() - - # TODO: this might be failing b/c celery fails to propagate the changes to share - # see https://github.com/CenterForOpenScience/osf.io/pull/10599 - @pytest.mark.skipif( - not environ.get('CI'), - reason="for some reason elasticsearch environment isn't properly set up locally, so this test passes only in CI" - ) - def test_search_views(self): - # Test search user - url = '/api/v1/search/user/' - res = self.app.get(url, query_string={'q': 'Umwali'}) - assert res.status_code == 200 - assert not res.json['results'] - - user_one = factories.AuthUserFactory(fullname='Joe Umwali') - user_two = factories.AuthUserFactory(fullname='Joan Uwase') - - res = self.app.get(url, query_string={'q': 'Umwali'}) - - assert res.status_code == 200 - assert len(res.json['results']) == 1 - assert not res.json['results'][0]['social'] - - user_one.social = { - 'github': user_one.given_name, - 'twitter': user_one.given_name, - 'ssrn': user_one.given_name - } - user_one.save() - - res = self.app.get(url, query_string={'q': 'Umwali'}) - - assert res.status_code == 200 - assert len(res.json['results']) == 1 - assert 'Joan' not in res.text - assert res.json['results'][0]['social'] - assert res.json['results'][0]['names']['fullname'] == user_one.fullname - assert res.json['results'][0]['social']['github'] == f'http://github.com/{user_one.given_name}' - assert res.json['results'][0]['social']['twitter'] == f'http://twitter.com/{user_one.given_name}' - assert res.json['results'][0]['social']['ssrn'] == f'http://papers.ssrn.com/sol3/cf_dev/AbsByAuth.cfm?per_id={user_one.given_name}' - - user_two.social = { - 'profileWebsites': [f'http://me.com/{user_two.given_name}'], - 'orcid': user_two.given_name, - 'linkedIn': user_two.given_name, - 'scholar': user_two.given_name, - 'impactStory': user_two.given_name, - 'baiduScholar': user_two.given_name - } - with mock.patch.object(spam_tasks.requests, 'head'): - user_two.save() - - user_three = factories.AuthUserFactory(fullname='Janet Umwali') - user_three.social = { - 'github': user_three.given_name, - 'ssrn': user_three.given_name - } - user_three.save() - - res = self.app.get(url, query_string={'q': 'Umwali'}) - - assert res.status_code == 200 - assert len(res.json['results']) == 2 - assert res.json['results'][0]['social'] - assert res.json['results'][1]['social'] - assert res.json['results'][0]['social']['ssrn'] != res.json['results'][1]['social']['ssrn'] - assert res.json['results'][0]['social']['github'] != res.json['results'][1]['social']['github'] - - res = self.app.get(url, query_string={'q': 'Uwase'}) - - assert res.status_code == 200 - assert len(res.json['results']) == 1 - assert res.json['results'][0]['social'] - assert 'ssrn' not in res.json['results'][0]['social'] - assert res.json['results'][0]['social']['profileWebsites'][0] == f'http://me.com/{user_two.given_name}' - assert res.json['results'][0]['social']['impactStory'] == f'https://impactstory.org/u/{user_two.given_name}' - assert res.json['results'][0]['social']['orcid'] == f'http://orcid.org/{user_two.given_name}' - assert res.json['results'][0]['social']['baiduScholar'] == f'http://xueshu.baidu.com/scholarID/{user_two.given_name}' - assert res.json['results'][0]['social']['linkedIn'] == f'https://www.linkedin.com/{user_two.given_name}' - assert res.json['results'][0]['social']['scholar'] == f'http://scholar.google.com/citations?user={user_two.given_name}' diff --git a/website/language.py b/website/language.py index 8f292955680..024783d13ec 100644 --- a/website/language.py +++ b/website/language.py @@ -100,8 +100,6 @@ # ########### # Search-related errors -SEARCH_QUERY_HELP = ('Please check our help (the question mark beside the search box) for more information ' - 'on advanced search queries.') # Shown at error page if an expired/revokes email confirmation link is clicked EXPIRED_EMAIL_CONFIRM_TOKEN = 'This confirmation link has expired. Please log in to continue.' diff --git a/website/routes.py b/website/routes.py index ce326426c27..8b53a2c216c 100644 --- a/website/routes.py +++ b/website/routes.py @@ -38,7 +38,6 @@ from website import landing_pages as landing_page_views from website import views as website_views from website.citations import views as citation_views -from website.search import views as search_views from website.oauth import views as oauth_views from addons.osfstorage import views as osfstorage_views from website.profile.utils import get_profile_image_url @@ -946,14 +945,6 @@ def make_url_map(app): ]) - # API - - process_rules(app, [ - - Rule(['/search/', '/search//'], ['get', 'post'], search_views.search_search, json_renderer), - - ], prefix='/api/v1') - # Web process_rules(app, [ diff --git a/website/search/views.py b/website/search/views.py index 7553bfed1ce..3f092ce2060 100644 --- a/website/search/views.py +++ b/website/search/views.py @@ -1,73 +1,14 @@ -import functools -from rest_framework import status as http_status import logging -import time - -from flask import request from framework.auth.decorators import must_be_logged_in -from framework.exceptions import HTTPError -from framework import sentry -from website import language from osf.models import OSFUser from website.project.views.contributor import get_node_contributors_abbrev -from website.search import exceptions -import website.search.search as search -from website.search.util import build_query logger = logging.getLogger(__name__) RESULTS_PER_PAGE = 250 -def handle_search_errors(func): - @functools.wraps(func) - def wrapped(*args, **kwargs): - try: - return func(*args, **kwargs) - except exceptions.MalformedQueryError: - raise HTTPError(http_status.HTTP_400_BAD_REQUEST, data={ - 'message_short': 'Bad search query', - 'message_long': language.SEARCH_QUERY_HELP, - }) - except exceptions.SearchUnavailableError: - raise HTTPError(http_status.HTTP_503_SERVICE_UNAVAILABLE, data={ - 'message_short': 'Search unavailable', - 'message_long': ('Our search service is currently unavailable, if the issue persists, ' - + language.SUPPORT_LINK), - }) - except exceptions.SearchException as e: - # Interim fix for issue where ES fails with 500 in some settings- ensure exception is still logged until it can be better debugged. See OSF-4538 - sentry.log_exception(e) - sentry.log_message('Elasticsearch returned an unexpected error response') - # TODO: Add a test; may need to mock out the error response due to inability to reproduce error code locally - raise HTTPError(http_status.HTTP_400_BAD_REQUEST, data={ - 'message_short': 'Could not perform search query', - 'message_long': language.SEARCH_QUERY_HELP, - }) - return wrapped - - -@handle_search_errors -def search_search(**kwargs): - _type = kwargs.get('type', None) - - tick = time.time() - results = {} - - if request.method == 'POST': - results = search.search(request.get_json(), doc_type=_type) - elif request.method == 'GET': - q = request.args.get('q', '*') - # TODO Match javascript params? - start = request.args.get('from', '0') - size = request.args.get('size', '10') - results = search.search(build_query(q, start, size), doc_type=_type) - - results['time'] = round(time.time() - tick, 2) - return results - - @must_be_logged_in def process_project_search_results(results, **kwargs): """ From d6d33574bbf53f593f70358ebf0ebbb1c3229fa1 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 14:56:53 +0300 Subject: [PATCH 11/43] removed unused website/search process_project_search_results function --- website/search/views.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 website/search/views.py diff --git a/website/search/views.py b/website/search/views.py deleted file mode 100644 index 3f092ce2060..00000000000 --- a/website/search/views.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging - -from framework.auth.decorators import must_be_logged_in -from osf.models import OSFUser -from website.project.views.contributor import get_node_contributors_abbrev - -logger = logging.getLogger(__name__) - -RESULTS_PER_PAGE = 250 - - -@must_be_logged_in -def process_project_search_results(results, **kwargs): - """ - :param results: list of projects from the modular ODM search - :return: we return the entire search result, which is a list of - dictionaries. This includes the list of contributors. - """ - user = kwargs['auth'].user - - ret = [] - - for project in results: - authors = get_node_contributors_abbrev(project=project, auth=kwargs['auth']) - authors_html = '' - for author in authors['contributors']: - a = OSFUser.load(author['user_id']) - authors_html += f'{a.fullname}' - authors_html += author['separator'] + ' ' - authors_html += ' ' + authors['others_count'] - - ret.append({ - 'id': project._id, - 'label': project.title, - 'value': project.title, - 'category': 'My Projects' if user in project.contributors else 'Public Projects', - 'authors': authors_html, - }) - - return ret From c8a27241ef379bcf37a12c1963c4e058f7e8f774 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 18:55:35 +0300 Subject: [PATCH 12/43] removed api/v2/search/ endpoint --- api/search/serializers.py | 58 ------------- api/search/urls.py | 1 - api/search/views.py | 37 -------- api_tests/search/serializers/__init__.py | 0 .../search/serializers/test_serializers.py | 50 ----------- api_tests/search/views/test_views.py | 87 ------------------- website/routes.py | 2 - 7 files changed, 235 deletions(-) delete mode 100644 api/search/serializers.py delete mode 100644 api_tests/search/serializers/__init__.py delete mode 100644 api_tests/search/serializers/test_serializers.py diff --git a/api/search/serializers.py b/api/search/serializers.py deleted file mode 100644 index 6d07a131856..00000000000 --- a/api/search/serializers.py +++ /dev/null @@ -1,58 +0,0 @@ -from api.base.serializers import ( - JSONAPISerializer, -) -from api.base.utils import absolute_reverse -from api.files.serializers import FileSerializer -from api.nodes.serializers import NodeSerializer -from api.registrations.serializers import RegistrationSerializer -from api.users.serializers import UserSerializer -from api.institutions.serializers import InstitutionSerializer -from api.collections.serializers import CollectionSubmissionSerializer - -from osf.models import ( - AbstractNode, - OSFUser, - BaseFileNode, - Institution, - CollectionSubmission, -) - -class SearchSerializer(JSONAPISerializer): - - def to_representation(self, data, envelope='data'): - - if isinstance(data, AbstractNode): - if data.is_registration: - serializer = RegistrationSerializer(data, context=self.context) - return RegistrationSerializer.to_representation(serializer, data) - serializer = NodeSerializer(data, context=self.context) - return NodeSerializer.to_representation(serializer, data) - - if isinstance(data, OSFUser): - serializer = UserSerializer(data, context=self.context) - return UserSerializer.to_representation(serializer, data) - - if isinstance(data, BaseFileNode): - serializer = FileSerializer(data, context=self.context) - return FileSerializer.to_representation(serializer, data) - - if isinstance(data, Institution): - serializer = InstitutionSerializer(data, context=self.context) - return InstitutionSerializer.to_representation(serializer, data) - - if isinstance(data, CollectionSubmission): - serializer = CollectionSubmissionSerializer(data, context=self.context) - return CollectionSubmissionSerializer.to_representation(serializer, data) - - return None - - def get_absolute_url(self, obj): - return absolute_reverse( - view_name='search:search-search', - kwargs={ - 'version': self.context['request'].parser_context['kwargs']['version'], - }, - ) - - class Meta: - type_ = 'search' diff --git a/api/search/urls.py b/api/search/urls.py index 3375e0eb8a6..fcd7cf7e58e 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^$', views.Search.as_view(), name=views.Search.view_name), re_path(r'^components/$', views.SearchComponents.as_view(), name=views.SearchComponents.view_name), re_path(r'^files/$', views.SearchFiles.as_view(), name=views.SearchFiles.view_name), re_path(r'^projects/$', views.SearchProjects.as_view(), name=views.SearchProjects.view_name), diff --git a/api/search/views.py b/api/search/views.py index b8bf9fba3af..f88687bf69f 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -11,7 +11,6 @@ from api.nodes.serializers import NodeSerializer from api.registrations.serializers import RegistrationSerializer from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch -from api.search.serializers import SearchSerializer from api.users.serializers import UserSerializer from api.institutions.serializers import InstitutionSerializer from api.collections.serializers import CollectionSubmissionSerializer @@ -70,42 +69,6 @@ def get_queryset(self, query=None): return results -class Search(BaseSearchView): - """ - *Read-Only* - - Objects (including projects, components, registrations, users, files, and institutions) that have been found by the given - Elasticsearch query. Each object is serialized with the appropriate serializer for its type (files are serialized as - files, users are serialized as users, etc.) and returned collectively. - - ## Search Fields - - # either projects, components, registrations, users, files, or institutions - related - href # the canonical api endpoint to search within a certain object type, e.g `/v2/search/users/` - meta - total # the number of results found that are of the enclosing object type - - ## Links - - See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination). - - ## Query Params - - + `q=` -- Query to search projects, components, registrations, users, and files for. - - + `page=` -- page number of results to view, default 1 - - # This Request/Response - - """ - - serializer_class = SearchSerializer - - view_category = 'search' - view_name = 'search-search' - - class SearchComponents(BaseSearchView): """ *Read-Only* diff --git a/api_tests/search/serializers/__init__.py b/api_tests/search/serializers/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/api_tests/search/serializers/test_serializers.py b/api_tests/search/serializers/test_serializers.py deleted file mode 100644 index 92f267f77f9..00000000000 --- a/api_tests/search/serializers/test_serializers.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest - -from api.search.serializers import SearchSerializer -from api_tests import utils -from osf.models import RegistrationSchema -from osf_tests.factories import ( - AuthUserFactory, - NodeFactory, - ProjectFactory, -) -from tests.utils import make_drf_request_with_version, mock_archive - -SCHEMA_VERSION = 2 - - -@pytest.mark.django_db -class TestSearchSerializer: - - @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') - def test_search_serializer_mixed_model(self): - - user = AuthUserFactory() - project = ProjectFactory(creator=user, is_public=True) - component = NodeFactory(parent=project, creator=user, is_public=True) - file_component = utils.create_test_file(component, user) - context = {'request': make_drf_request_with_version(version='2.0')} - schema = RegistrationSchema.objects.filter( - name='Replication Recipe (Brandt et al., 2013): Post-Completion', - schema_version=SCHEMA_VERSION).first() - - # test_search_serializer_mixed_model_project - result = SearchSerializer(project, context=context).data - assert result['data']['type'] == 'nodes' - - # test_search_serializer_mixed_model_component - result = SearchSerializer(component, context=context).data - assert result['data']['type'] == 'nodes' - - # test_search_serializer_mixed_model_registration - with mock_archive(project, autocomplete=True, autoapprove=True, schema=schema) as registration: - result = SearchSerializer(registration, context=context).data - assert result['data']['type'] == 'registrations' - - # test_search_serializer_mixed_model_file - result = SearchSerializer(file_component, context=context).data - assert result['data']['type'] == 'files' - - # test_search_serializer_mixed_model_user - result = SearchSerializer(user, context=context).data - assert result['data']['type'] == 'users' diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index c535828d955..c4f24438466 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -157,93 +157,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -class TestSearch(ApiSearchTestCase): - - @pytest.fixture() - def url_search(self): - return f'/{API_BASE}search/' - - def test_search_results( - self, app, url_search, user, user_one, user_two, - institution, component, component_private, - component_public, file_component, file_private, - file_public, project, project_public, project_private): - - # test_search_no_auth - res = app.get(url_search) - assert res.status_code == 200 - - search_fields = res.json['search_fields'] - users_found = search_fields['users']['related']['meta']['total'] - files_found = search_fields['files']['related']['meta']['total'] - projects_found = search_fields['projects']['related']['meta']['total'] - components_found = search_fields['components']['related']['meta']['total'] - registrations_found = search_fields['registrations']['related']['meta']['total'] - - assert users_found == 3 - assert files_found == 2 - assert projects_found == 2 - assert components_found == 2 - assert registrations_found == 0 - - # test_search_auth - res = app.get(url_search, auth=user.auth) - assert res.status_code == 200 - - search_fields = res.json['search_fields'] - users_found = search_fields['users']['related']['meta']['total'] - files_found = search_fields['files']['related']['meta']['total'] - projects_found = search_fields['projects']['related']['meta']['total'] - components_found = search_fields['components']['related']['meta']['total'] - registrations_found = search_fields['registrations']['related']['meta']['total'] - - assert users_found == 3 - assert files_found == 2 - assert projects_found == 2 - assert components_found == 2 - assert registrations_found == 0 - - # test_search_fields_links - res = app.get(url_search) - assert res.status_code == 200 - - search_fields = res.json['search_fields'] - users_link = search_fields['users']['related']['href'] - files_link = search_fields['files']['related']['href'] - projects_link = search_fields['projects']['related']['href'] - components_link = search_fields['components']['related']['href'] - registrations_link = search_fields['registrations']['related']['href'] - - assert f'/{API_BASE}search/users/?q=%2A' in users_link - assert f'/{API_BASE}search/files/?q=%2A' in files_link - assert f'/{API_BASE}search/projects/?q=%2A' in projects_link - assert '/{}search/components/?q=%2A'.format( - API_BASE) in components_link - assert '/{}search/registrations/?q=%2A'.format( - API_BASE) in registrations_link - - # test_search_fields_links_with_query - url = f'{url_search}?q=science' - res = app.get(url) - assert res.status_code == 200 - - search_fields = res.json['search_fields'] - users_link = search_fields['users']['related']['href'] - files_link = search_fields['files']['related']['href'] - projects_link = search_fields['projects']['related']['href'] - components_link = search_fields['components']['related']['href'] - registrations_link = search_fields['registrations']['related']['href'] - - assert f'/{API_BASE}search/users/?q=science' in users_link - assert f'/{API_BASE}search/files/?q=science' in files_link - assert '/{}search/projects/?q=science'.format( - API_BASE) in projects_link - assert '/{}search/components/?q=science'.format( - API_BASE) in components_link - assert '/{}search/registrations/?q=science'.format( - API_BASE) in registrations_link - - class TestSearchComponents(ApiSearchTestCase): @pytest.fixture() diff --git a/website/routes.py b/website/routes.py index 8b53a2c216c..58b5f78ddb6 100644 --- a/website/routes.py +++ b/website/routes.py @@ -930,8 +930,6 @@ def make_url_map(app): ], prefix='/api/v1',) - ### Search ### - # Web process_rules(app, [ From b3788adec55284a78a91615f225754b5499ac061 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:01:28 +0300 Subject: [PATCH 13/43] removed api/v2/search/components/ endpoint --- api/search/urls.py | 1 - api/search/views.py | 115 --------------------------- api_tests/search/views/test_views.py | 112 -------------------------- 3 files changed, 228 deletions(-) diff --git a/api/search/urls.py b/api/search/urls.py index fcd7cf7e58e..bc6a2d52528 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^components/$', views.SearchComponents.as_view(), name=views.SearchComponents.view_name), re_path(r'^files/$', views.SearchFiles.as_view(), name=views.SearchFiles.view_name), re_path(r'^projects/$', views.SearchProjects.as_view(), name=views.SearchProjects.view_name), re_path(r'^registrations/$', views.SearchRegistrations.as_view(), name=views.SearchRegistrations.view_name), diff --git a/api/search/views.py b/api/search/views.py index f88687bf69f..5a3164fcf86 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -69,121 +69,6 @@ def get_queryset(self, query=None): return results -class SearchComponents(BaseSearchView): - """ - *Read-Only* - - Components that have been found by the given Elasticsearch query. - - - - On the front end, nodes are considered 'projects' or 'components'. The difference between a project and a component - is that a project is the top-level node, and components are children of the project. There is also a [category - field](/v2/#osf-node-categories) that includes 'project' as an option. The categorization essentially determines - which icon is displayed by the node in the front-end UI and helps with search organization. Top-level nodes may have - a category other than project, and children nodes may have a category of project. - - ##Node Attributes - - - - OSF Node entities have the "nodes" `type`. - - name type description - ================================================================================= - title string title of project or component - description string description of the node - category string node category, must be one of the allowed values - date_created iso8601 timestamp timestamp that the node was created - date_modified iso8601 timestamp timestamp when the node was last updated - tags array of strings list of tags that describe the node - current_user_can_comment boolean Whether the current user is allowed to post comments - current_user_permissions array of strings list of strings representing the permissions for the current user on this node - registration boolean is this a registration? (always false - may be deprecated in future versions) - fork boolean is this node a fork of another node? - public boolean has this node been made publicly-visible? - preprint boolean is this a preprint? - collection boolean is this a collection? (always false - may be deprecated in future versions) - node_license object details of the license applied to the node - year string date range of the license - copyright_holders array of strings holders of the applied license - - ##Relationships - - ###Children - - List of nodes that are children of this node. New child nodes may be added through this endpoint. - - ###Comments - - List of comments on this node. New comments can be left on the node through this endpoint. - - ###Contributors - - List of users who are contributors to this node. Contributors may have "read", "write", or "admin" permissions. - A node must always have at least one "admin" contributor. Contributors may be added via this endpoint. - - ###Draft Registrations - - List of draft registrations of the current node. - - ###Files - - List of top-level folders (actually cloud-storage providers) associated with this node. This is the starting point - for accessing the actual files stored within this node. - - ###Forked From - - If this node was forked from another node, the canonical endpoint of the node that was forked from will be - available in the `/forked_from/links/related/href` key. Otherwise, it will be null. - - ###Logs - - List of read-only log actions pertaining to the node. - - ###Node Links - - List of links (pointers) to other nodes on the OSF. Node links can be added through this endpoint. - - ###Parent - - If this node is a child node of another node, the parent's canonical endpoint will be available in the - `/parent/links/related/href` key. Otherwise, it will be null. - - ###Registrations - - List of registrations of the current node. - - ###Root - - Returns the top-level node associated with the current node. If the current node is the top-level node, the root is - the current node. - - ##Links - - self: the canonical api endpoint of this node - html: this node's page on the OSF website - - See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination). - - ##Query Params - - + `q=` -- Query to search components for, searches across a component's title, description, tags, and contributor names. - - + `page=` -- page number of results to view, default 1 - - #This Request/Response - - """ - - model_class = AbstractNode - serializer_class = NodeSerializer - - doc_type = 'component' - view_category = 'search' - view_name = 'search-component' - - class SearchFiles(BaseSearchView): """ *Read-Only* diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index c4f24438466..bfd539fbfab 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -157,118 +157,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -class TestSearchComponents(ApiSearchTestCase): - - @pytest.fixture() - def url_component_search(self): - return f'/{API_BASE}search/components/' - - def test_search_components( - self, app, url_component_search, user, user_one, user_two, - component, component_public, component_private): - - # test_search_public_component_no_auth - res = app.get(url_component_search) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert component_public.title in res - assert component.title in res - - # test_search_public_component_auth - res = app.get(url_component_search, auth=user) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert component_public.title in res - assert component.title in res - - # test_search_public_component_contributor - res = app.get(url_component_search, auth=user_two) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert component_public.title in res - assert component.title in res - - # test_search_private_component_no_auth - res = app.get(url_component_search) - assert res.status_code == 200 - assert component_private.title not in res - - # test_search_private_component_auth - res = app.get(url_component_search, auth=user) - assert res.status_code == 200 - assert component_private.title not in res - - # test_search_private_component_contributor - res = app.get(url_component_search, auth=user_two) - assert res.status_code == 200 - assert component_private.title not in res - - # test_search_component_by_title - url = '{}?q={}'.format(url_component_search, 'beam') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert component_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_component_by_description - url = '{}?q={}'.format(url_component_search, 'speak') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert component_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_component_by_tags - url = '{}?q={}'.format(url_component_search, 'trumpets') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert component_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_component_by_contributor - url = '{}?q={}'.format(url_component_search, 'Chance') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert component_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_component_no_results - url = '{}?q={}'.format(url_component_search, 'Ocean') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 0 - assert total == 0 - - # test_search_component_bad_query - url = '{}?q={}'.format( - url_component_search, - 'www.spam.com/help/twitter/') - res = app.get(url, expect_errors=True) - assert res.status_code == 400 - - class TestSearchFiles(ApiSearchTestCase): @pytest.fixture() From 408b47ef113e9d089e9a55a9229debbe31a1ecde Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:05:42 +0300 Subject: [PATCH 14/43] removed api/v2/search/files/ endpoint --- api/search/urls.py | 1 - api/search/views.py | 116 +-------------------------- api_tests/search/views/test_views.py | 66 --------------- 3 files changed, 1 insertion(+), 182 deletions(-) diff --git a/api/search/urls.py b/api/search/urls.py index bc6a2d52528..96ff301d800 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^files/$', views.SearchFiles.as_view(), name=views.SearchFiles.view_name), re_path(r'^projects/$', views.SearchProjects.as_view(), name=views.SearchProjects.view_name), re_path(r'^registrations/$', views.SearchRegistrations.as_view(), name=views.SearchRegistrations.view_name), re_path(r'^users/$', views.SearchUsers.as_view(), name=views.SearchUsers.view_name), diff --git a/api/search/views.py b/api/search/views.py index 5a3164fcf86..bb8d41776c3 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -7,7 +7,6 @@ from api.base.pagination import SearchPagination from api.base.parsers import SearchParser from api.base.settings import REST_FRAMEWORK, MAX_PAGE_SIZE -from api.files.serializers import FileSerializer from api.nodes.serializers import NodeSerializer from api.registrations.serializers import RegistrationSerializer from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch @@ -16,7 +15,7 @@ from api.collections.serializers import CollectionSubmissionSerializer from framework.auth.oauth_scopes import CoreScopes -from osf.models import Institution, BaseFileNode, AbstractNode, OSFUser, CollectionSubmission +from osf.models import Institution, AbstractNode, OSFUser, CollectionSubmission from website.search import search from website.search.exceptions import MalformedQueryError @@ -69,119 +68,6 @@ def get_queryset(self, query=None): return results -class SearchFiles(BaseSearchView): - """ - *Read-Only* - - Files that have been found by the given Elasticsearch query. - - - - ####File Entity - - name type description - ========================================================================= - guid string OSF GUID for this file (if one has been assigned) - name string name of the file - path string unique identifier for this file entity for this - project and storage provider. may not end with '/' - materialized string the full path of the file relative to the storage - root. may not end with '/' - kind string "file" - etag string etag - http caching identifier w/o wrapping quotes - modified timestamp last modified timestamp - format depends on provider - contentType string MIME-type when available - provider string id of provider e.g. "osfstorage", "s3", "googledrive". - equivalent to addon_short_name on the OSF - size integer size of file in bytes - tags array of strings list of tags that describes the file (osfstorage only) - extra object may contain additional data beyond what's described here, - depending on the provider - version integer version number of file. will be 1 on initial upload - downloads integer count of the number times the file has been downloaded - hashes object - md5 string md5 hash of file - sha256 string SHA-256 hash of file - - ##Attributes - - For an OSF File entity, the `type` is "files" regardless of whether the entity is actually a file or folder, because - it belongs to the `files` collection of the API. They can be distinguished by the `kind` attribute. Files and - folders use the same representation, but some attributes may be null for one kind but not the other. `size` will be - null for folders. See the [list of storage provider keys](/v2/#storage-providers). - - name type description - ================================================================================================================ - name string name of the file or folder; used for display - kind string "file" or "folder" - path string same as for corresponding WaterButler entity - materialized_path string the unix-style path to the file relative to the provider root - size integer size of file in bytes, null for folders - provider string storage provider for this file. "osfstorage" if stored on the - OSF. other examples include "s3" for Amazon S3, "googledrive" - for Google Drive, "box" for Box.com. - current_user_can_comment boolean Whether the current user is allowed to post comments - - last_touched iso8601 timestamp last time the metadata for the file was retrieved. only - applies to non-OSF storage providers. - date_modified iso8601 timestamp timestamp of when this file was last updated* - date_created iso8601 timestamp timestamp of when this file was created* - extra object may contain additional data beyond what's described here, - depending on the provider - hashes object - md5 string md5 hash of file, null for folders - sha256 string SHA-256 hash of file, null for folders - - * A note on timestamps: for files stored in osfstorage, `date_created` refers to the time that the file was - first uploaded to osfstorage, and `date_modified` is the time that the file was last updated while in osfstorage. - Other providers may or may not provide this information, but if they do it will correspond to the provider's - semantics for created/modified times. These timestamps may also be stale; metadata retrieved via the File Detail - endpoint is cached. The `last_touched` field describes the last time the metadata was retrieved from the external - provider. To force a metadata update, access the parent folder via its Node Files List endpoint. - - - - ##Relationships - - ###Node - - The `node` endpoint describes the project or registration that this file belongs to. - - ###Files (*folders*) - - The `files` endpoint lists all of the subfiles and folders of the current folder. Will be null for files. - - ###Versions (*files*) - - The `versions` endpoint provides version history for files. Will be null for folders. - - ##Links - - info: the canonical api endpoint for the folder's contents or file's most recent version - new_folder: url to target when creating new subfolders (null for files) - move: url to target for move, copy, and rename actions - upload: url to target for uploading new files and updating existing files - download: url to request a download of the latest version of the file (null for folders) - delete: url to target for deleting files and folders - - ## Query Params - - + `q=` -- Query to search files for, searches across a file's name. - - + `page=` -- page number of results to view, default 1 - - #This Request/Response - - """ - - model_class = BaseFileNode - serializer_class = FileSerializer - - doc_type = 'file' - view_category = 'search' - view_name = 'search-file' - - class SearchProjects(BaseSearchView): """ *Read-Only* diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index bfd539fbfab..00529de4a88 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -157,72 +157,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -class TestSearchFiles(ApiSearchTestCase): - - @pytest.fixture() - def url_file_search(self): - return f'/{API_BASE}search/files/' - - def test_search_files( - self, app, url_file_search, user, user_one, - file_public, file_component, file_private): - - # test_search_public_file_no_auth - res = app.get(url_file_search) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert file_public.name in res - assert file_component.name in res - - # test_search_public_file_auth - res = app.get(url_file_search, auth=user) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert file_public.name in res - assert file_component.name in res - - # test_search_public_file_contributor - res = app.get(url_file_search, auth=user_one) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert file_public.name in res - assert file_component.name in res - - # test_search_private_file_no_auth - res = app.get(url_file_search) - assert res.status_code == 200 - assert file_private.name not in res - - # test_search_private_file_auth - res = app.get(url_file_search, auth=user) - assert res.status_code == 200 - assert file_private.name not in res - - # test_search_private_file_contributor - res = app.get(url_file_search, auth=user_one) - assert res.status_code == 200 - assert file_private.name not in res - - # test_search_file_by_name - url = '{}?q={}'.format(url_file_search, 'highlights') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert file_component.name == res.json['data'][0]['attributes']['name'] - - class TestSearchProjects(ApiSearchTestCase): @pytest.fixture() From d3fbfbd129bc4896654babb3600fccfae9347181 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:09:41 +0300 Subject: [PATCH 15/43] removed api/v2/search/projects/ endpoint --- api/search/urls.py | 1 - api/search/views.py | 117 --------------------------- api_tests/search/views/test_views.py | 113 -------------------------- 3 files changed, 231 deletions(-) diff --git a/api/search/urls.py b/api/search/urls.py index 96ff301d800..16fdf865051 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^projects/$', views.SearchProjects.as_view(), name=views.SearchProjects.view_name), re_path(r'^registrations/$', views.SearchRegistrations.as_view(), name=views.SearchRegistrations.view_name), re_path(r'^users/$', views.SearchUsers.as_view(), name=views.SearchUsers.view_name), re_path(r'^institutions/$', views.SearchInstitutions.as_view(), name=views.SearchInstitutions.view_name), diff --git a/api/search/views.py b/api/search/views.py index bb8d41776c3..e17985fd7c3 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -7,7 +7,6 @@ from api.base.pagination import SearchPagination from api.base.parsers import SearchParser from api.base.settings import REST_FRAMEWORK, MAX_PAGE_SIZE -from api.nodes.serializers import NodeSerializer from api.registrations.serializers import RegistrationSerializer from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch from api.users.serializers import UserSerializer @@ -68,122 +67,6 @@ def get_queryset(self, query=None): return results -class SearchProjects(BaseSearchView): - """ - *Read-Only* - - Projects that have been found by the given Elasticsearch query. - - - - On the front end, nodes are considered 'projects' or 'components'. The difference between a project and a component - is that a project is the top-level node, and components are children of the project. There is also a [category - field](/v2/#osf-node-categories) that includes 'project' as an option. The categorization essentially determines - which icon is displayed by the node in the front-end UI and helps with search organization. Top-level nodes may have - a category other than project, and children nodes may have a category of project. - - ##Node Attributes - - - - OSF Node entities have the "nodes" `type`. - - name type description - ================================================================================= - title string title of project or component - description string description of the node - category string node category, must be one of the allowed values - date_created iso8601 timestamp timestamp that the node was created - date_modified iso8601 timestamp timestamp when the node was last updated - tags array of strings list of tags that describe the node - current_user_can_comment boolean Whether the current user is allowed to post comments - current_user_permissions array of strings list of strings representing the permissions for the current user on this node - registration boolean is this a registration? (always false - may be deprecated in future versions) - fork boolean is this node a fork of another node? - public boolean has this node been made publicly-visible? - preprint boolean is this a preprint? - collection boolean is this a collection? (always false - may be deprecated in future versions) - node_license object details of the license applied to the node - year string date range of the license - copyright_holders array of strings holders of the applied license - - ##Relationships - - ###Children - - List of nodes that are children of this node. New child nodes may be added through this endpoint. - - ###Comments - - List of comments on this node. New comments can be left on the node through this endpoint. - - ###Contributors - - List of users who are contributors to this node. Contributors may have "read", "write", or "admin" permissions. - A node must always have at least one "admin" contributor. Contributors may be added via this endpoint. - - ###Draft Registrations - - List of draft registrations of the current node. - - ###Files - - List of top-level folders (actually cloud-storage providers) associated with this node. This is the starting point - for accessing the actual files stored within this node. - - ###Forked From - - If this node was forked from another node, the canonical endpoint of the node that was forked from will be - available in the `/forked_from/links/related/href` key. Otherwise, it will be null. - - ###Logs - - List of read-only log actions pertaining to the node. - - ###Node Links - - List of links (pointers) to other nodes on the OSF. Node links can be added through this endpoint. - - ###Parent - - If this node is a child node of another node, the parent's canonical endpoint will be available in the - `/parent/links/related/href` key. Otherwise, it will be null. - - ###Registrations - - List of registrations of the current node. - - ###Root - - Returns the top-level node associated with the current node. If the current node is the top-level node, the root is - the current node. - - ##Links - - self: the canonical api endpoint of this node - html: this node's page on the OSF website - - See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination). - - ##Query Params - - + `q=` -- Query to search projects for, searches across a project's title, description, tags, and contributor names. - - + `page=` -- page number of results to view, default 1 - - - #This Request/Response - - """ - - model_class = AbstractNode - serializer_class = NodeSerializer - - doc_type = 'project' - view_category = 'search' - view_name = 'search-project' - - class SearchRegistrations(BaseSearchView): """ *Read-Only* diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index 00529de4a88..833a4befa91 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -157,119 +157,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -class TestSearchProjects(ApiSearchTestCase): - - @pytest.fixture() - def url_project_search(self): - return f'/{API_BASE}search/projects/' - - def test_search_projects( - self, app, url_project_search, user, user_one, - user_two, project, project_public, project_private): - - # test_search_public_project_no_auth - res = app.get(url_project_search) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert project_public.title in res - assert project.title in res - - # test_search_public_project_auth - res = app.get(url_project_search, auth=user) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert project_public.title in res - assert project.title in res - - # test_search_public_project_contributor - res = app.get(url_project_search, auth=user_one) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert project_public.title in res - assert project.title in res - - # test_search_private_project_no_auth - res = app.get(url_project_search) - assert res.status_code == 200 - assert project_private.title not in res - - # test_search_private_project_auth - res = app.get(url_project_search, auth=user) - assert res.status_code == 200 - assert project_private.title not in res - - # test_search_private_project_contributor - res = app.get(url_project_search, auth=user_two) - assert res.status_code == 200 - assert project_private.title not in res - - # test_search_project_by_title - url = '{}?q={}'.format(url_project_search, 'pablo') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert project_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_project_by_description - url = '{}?q={}'.format(url_project_search, 'genius') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert project_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_project_by_tags - url = '{}?q={}'.format(url_project_search, 'Yeezus') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert project_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_project_by_contributor - url = '{}?q={}'.format(url_project_search, 'kanye') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert project_public.title in res - assert project.title in res - - # test_search_project_no_results - url = '{}?q={}'.format(url_project_search, 'chicago') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 0 - assert total == 0 - - # test_search_project_bad_query - url = '{}?q={}'.format( - url_project_search, - 'www.spam.com/help/facebook/') - res = app.get(url, expect_errors=True) - assert res.status_code == 400 - - @pytest.mark.django_db class TestSearchRegistrations(ApiSearchTestCase): From 44a3c8bef803477d8a45140d3ea048013f091d6d Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:11:30 +0300 Subject: [PATCH 16/43] removed api/v2/search/registrations/ endpoint --- api/search/urls.py | 1 - api/search/views.py | 92 +---------------- api_tests/search/views/test_views.py | 144 --------------------------- 3 files changed, 1 insertion(+), 236 deletions(-) diff --git a/api/search/urls.py b/api/search/urls.py index 16fdf865051..8487bcf6f44 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^registrations/$', views.SearchRegistrations.as_view(), name=views.SearchRegistrations.view_name), re_path(r'^users/$', views.SearchUsers.as_view(), name=views.SearchUsers.view_name), re_path(r'^institutions/$', views.SearchInstitutions.as_view(), name=views.SearchInstitutions.view_name), re_path(r'^collections/$', views.SearchCollections.as_view(), name=views.SearchCollections.view_name), diff --git a/api/search/views.py b/api/search/views.py index e17985fd7c3..4b2c841b969 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -7,14 +7,13 @@ from api.base.pagination import SearchPagination from api.base.parsers import SearchParser from api.base.settings import REST_FRAMEWORK, MAX_PAGE_SIZE -from api.registrations.serializers import RegistrationSerializer from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch from api.users.serializers import UserSerializer from api.institutions.serializers import InstitutionSerializer from api.collections.serializers import CollectionSubmissionSerializer from framework.auth.oauth_scopes import CoreScopes -from osf.models import Institution, AbstractNode, OSFUser, CollectionSubmission +from osf.models import Institution, OSFUser, CollectionSubmission from website.search import search from website.search.exceptions import MalformedQueryError @@ -67,95 +66,6 @@ def get_queryset(self, query=None): return results -class SearchRegistrations(BaseSearchView): - """ - *Read-Only* - - Registrations that have been found by the given Elasticsearch query. - - - - Node Registrations. - - Registrations are read-only snapshots of a project. This view is a list of all current registrations for which a user - has access. A withdrawn registration will display a limited subset of information, namely, title, description, - created, registration, withdrawn, date_registered, withdrawal_justification, and registration supplement. All - other fields will be displayed as null. Additionally, the only relationships permitted to be accessed for a withdrawn - registration are the contributors - other relationships will return a 403. - - Each resource contains the full representation of the registration, meaning additional requests to an individual - registrations's detail view are not necessary. Unregistered nodes cannot be accessed through this endpoint. - - - ##Registration Attributes - - Registrations have the "registrations" `type`. - - name type description - ======================================================================================================= - title string title of the registered project or component - description string description of the registered node - category string bode category, must be one of the allowed values - date_created iso8601 timestamp timestamp that the node was created - date_modified iso8601 timestamp timestamp when the node was last updated - tags array of strings list of tags that describe the registered node - current_user_can_comment boolean Whether the current user is allowed to post comments - current_user_permissions array of strings list of strings representing the permissions for the current user on this node - fork boolean is this project a fork? - registration boolean has this project been registered? (always true - may be deprecated in future versions) - collection boolean is this registered node a collection? (always false - may be deprecated in future versions) - node_license object details of the license applied to the node - year string date range of the license - copyright_holders array of strings holders of the applied license - public boolean has this registration been made publicly-visible? - withdrawn boolean has this registration been withdrawn? - date_registered iso8601 timestamp timestamp that the registration was created - embargo_end_date iso8601 timestamp when the embargo on this registration will be lifted (if applicable) - withdrawal_justification string reasons for withdrawing the registration - pending_withdrawal boolean is this registration pending withdrawal? - pending_withdrawal_approval boolean is this registration pending approval? - pending_embargo_approval boolean is the associated Embargo awaiting approval by project admins? - registered_meta dictionary registration supplementary information - registration_supplement string registration template - - - - ##Relationships - - ###Registered from - - The registration is branched from this node. - - ###Registered by - - The registration was initiated by this user. - - ###Other Relationships - - See documentation on registered_from detail view. A registration has many of the same properties as a node. - - ##Links - - See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination). - - ## Query Params - - + `q=` -- Query to search registrations for, searches across a registration's title, description, tags, and contributor names. - - + `page=` -- page number of results to view, default 1 - - #This Request/Response - - """ - - model_class = AbstractNode - serializer_class = RegistrationSerializer - - doc_type = 'registration' - view_category = 'search' - view_name = 'search-registration' - - class SearchUsers(BaseSearchView): """ *Read-Only* diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index 833a4befa91..3989712dc82 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -6,7 +6,6 @@ from api_tests import utils from framework.auth.cas import CasResponse from framework.auth.core import Auth -from osf.models import RegistrationSchema from osf_tests.factories import ( AuthUserFactory, NodeFactory, @@ -17,7 +16,6 @@ CollectionProviderFactory, RegistrationProviderFactory, ) -from osf_tests.utils import mock_archive from tests.utils import capture_notifications from website import settings from website.search import elastic_search @@ -157,148 +155,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -@pytest.mark.django_db -class TestSearchRegistrations(ApiSearchTestCase): - - @pytest.fixture() - def url_registration_search(self): - return f'/{API_BASE}search/registrations/' - - @pytest.fixture() - def schema(self): - schema = RegistrationSchema.objects.filter( - name='Replication Recipe (Brandt et al., 2013): Post-Completion', - schema_version=SCHEMA_VERSION).first() - return schema - - @pytest.fixture() - def registration(self, project, schema): - with mock_archive(project, autocomplete=True, autoapprove=True, schema=schema) as registration: - return registration - - @pytest.fixture() - def registration_public(self, project_public, schema): - with mock_archive(project_public, autocomplete=True, autoapprove=True, schema=schema) as registration_public: - return registration_public - - @pytest.fixture() - def registration_private(self, project_private, schema): - with mock_archive(project_private, autocomplete=True, autoapprove=True, schema=schema) as registration_private: - registration_private.is_public = False - registration_private.save() - # TODO: This shouldn't be necessary, but tests fail if we don't do - # this. Investigate further. - registration_private.update_search() - return registration_private - - @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links') - def test_search_registrations( - self, app, url_registration_search, user, user_one, user_two, - registration, registration_public, registration_private): - - # test_search_public_registration_no_auth - res = app.get(url_registration_search) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert registration_public.title in res - assert registration.title in res - - # test_search_public_registration_auth - res = app.get(url_registration_search, auth=user) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert registration_public.title in res - assert registration.title in res - - # test_search_public_registration_contributor - res = app.get(url_registration_search, auth=user_one) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert registration_public.title in res - assert registration.title in res - - # test_search_private_registration_no_auth - res = app.get(url_registration_search) - assert res.status_code == 200 - assert registration_private.title not in res - - # test_search_private_registration_auth - res = app.get(url_registration_search, auth=user) - assert res.status_code == 200 - assert registration_private.title not in res - - # test_search_private_registration_contributor - res = app.get(url_registration_search, auth=user_two) - assert res.status_code == 200 - assert registration_private.title not in res - - # test_search_registration_by_title - url = '{}?q={}'.format(url_registration_search, 'graduation') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert registration.title == res.json['data'][0]['attributes']['title'] - - # test_search_registration_by_description - url = '{}?q={}'.format(url_registration_search, 'crazy') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert registration_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_registration_by_tags - url = '{}?q={}'.format(url_registration_search, 'yeezus') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert registration_public.title == res.json['data'][0]['attributes']['title'] - - # test_search_registration_by_contributor - url = '{}?q={}'.format(url_registration_search, 'west') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 2 - assert total == 2 - assert registration_public.title in res - assert registration.title in res - - # test_search_registration_no_results - url = '{}?q={}'.format(url_registration_search, '79th') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 0 - assert total == 0 - - # test_search_registration_bad_query - url = '{}?q={}'.format( - url_registration_search, - 'www.spam.com/help/snapchat/') - res = app.get(url, expect_errors=True) - assert res.status_code == 400 - - class TestSearchUsers(ApiSearchTestCase): @pytest.fixture() From 2f763bc63b409ec0d796ef38872b77e8dd15f040 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:12:47 +0300 Subject: [PATCH 17/43] removed api/v2/search/users/ endpoint --- api/search/urls.py | 1 - api/search/views.py | 67 +----------------------- api_tests/search/views/test_views.py | 77 ---------------------------- 3 files changed, 1 insertion(+), 144 deletions(-) diff --git a/api/search/urls.py b/api/search/urls.py index 8487bcf6f44..0e70d2d3bab 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^users/$', views.SearchUsers.as_view(), name=views.SearchUsers.view_name), re_path(r'^institutions/$', views.SearchInstitutions.as_view(), name=views.SearchInstitutions.view_name), re_path(r'^collections/$', views.SearchCollections.as_view(), name=views.SearchCollections.view_name), diff --git a/api/search/views.py b/api/search/views.py index 4b2c841b969..68ef9fee726 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -8,12 +8,11 @@ from api.base.parsers import SearchParser from api.base.settings import REST_FRAMEWORK, MAX_PAGE_SIZE from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch -from api.users.serializers import UserSerializer from api.institutions.serializers import InstitutionSerializer from api.collections.serializers import CollectionSubmissionSerializer from framework.auth.oauth_scopes import CoreScopes -from osf.models import Institution, OSFUser, CollectionSubmission +from osf.models import Institution, CollectionSubmission from website.search import search from website.search.exceptions import MalformedQueryError @@ -66,70 +65,6 @@ def get_queryset(self, query=None): return results -class SearchUsers(BaseSearchView): - """ - *Read-Only* - - Users that have been found by the given Elasticsearch query. - - - - The User Detail endpoint retrieves information about the user whose id is the final part of the path. If `me` - is given as the id, the record of the currently logged-in user will be returned. The returned information includes - the user's bibliographic information and the date that the user registered. - - Note that if an anonymous view_only key is being used, user information will not be serialized, and the id will be - an empty string. Relationships to a user object will not show in this case, either. - - - - ##Attributes - - OSF User entities have the "users" `type`. - - name type description - ======================================================================================== - full_name string full name of the user; used for display - given_name string given name of the user; for bibliographic citations - middle_names string middle name of user; for bibliographic citations - family_name string family name of user; for bibliographic citations - suffix string suffix of user's name for bibliographic citations - date_registered iso8601 timestamp timestamp when the user's account was created - - - - ##Relationships - - ###Nodes - - A list of all nodes the user has contributed to. If the user id in the path is the same as the logged-in user, all - nodes will be visible. Otherwise, you will only be able to see the other user's publicly-visible nodes. - - ##Links - - self: the canonical api endpoint of this user - html: this user's page on the OSF website - profile_image_url: a url to the user's profile image - - ## Query Params - - + `q=` -- Query to search users for, searches across a users's given name, middle names, family name, - first listed job, and first listed school. - - + `page=` -- page number of results to view, default 1 - - # This Request/Response - - """ - - model_class = OSFUser - serializer_class = UserSerializer - - doc_type = 'user' - view_category = 'search' - view_name = 'search-user' - - class SearchInstitutions(BaseSearchView): """ *Read-Only* diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index 3989712dc82..1c722ada403 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -155,83 +155,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -class TestSearchUsers(ApiSearchTestCase): - - @pytest.fixture() - def url_user_search(self): - return f'/{API_BASE}search/users/' - - def test_search_user(self, app, url_user_search, user, user_one, user_two): - - # test_search_users_no_auth - res = app.get(url_user_search) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 3 - assert total == 3 - assert user.fullname in res - - # test_search_users_auth - res = app.get(url_user_search, auth=user) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 3 - assert total == 3 - assert user.fullname in res - - # test_search_users_by_given_name - url = '{}?q={}'.format(url_user_search, 'Kanye') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert user_one.given_name == res.json['data'][0]['attributes']['given_name'] - - # test_search_users_by_middle_name - url = '{}?q={}'.format(url_user_search, 'Omari') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert user_one.middle_names[0] == res.json['data'][0]['attributes']['middle_names'][0] - - # test_search_users_by_family_name - url = '{}?q={}'.format(url_user_search, 'West') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert user_one.family_name == res.json['data'][0]['attributes']['family_name'] - - # test_search_users_by_job - url = '{}?q={}'.format(url_user_search, 'producer') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert user_one.fullname == res.json['data'][0]['attributes']['full_name'] - - # test_search_users_by_school - url = '{}?q={}'.format(url_user_search, 'Chicago') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert user_one.fullname == res.json['data'][0]['attributes']['full_name'] - - class TestSearchInstitutions(ApiSearchTestCase): @pytest.fixture() From 0f7f0965c4cc2afb80d38c76dc42f0504571430a Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:14:17 +0300 Subject: [PATCH 18/43] removed api/v2/search/institutions/ endpoint --- api/search/urls.py | 1 - api/search/views.py | 46 +--------------------------- api_tests/search/views/test_views.py | 37 ---------------------- 3 files changed, 1 insertion(+), 83 deletions(-) diff --git a/api/search/urls.py b/api/search/urls.py index 0e70d2d3bab..ab8d62e5121 100644 --- a/api/search/urls.py +++ b/api/search/urls.py @@ -5,7 +5,6 @@ app_name = 'osf' urlpatterns = [ - re_path(r'^institutions/$', views.SearchInstitutions.as_view(), name=views.SearchInstitutions.view_name), re_path(r'^collections/$', views.SearchCollections.as_view(), name=views.SearchCollections.view_name), # not currently supported by v1, but should be supported by v2 diff --git a/api/search/views.py b/api/search/views.py index 68ef9fee726..beb9d922c97 100644 --- a/api/search/views.py +++ b/api/search/views.py @@ -8,11 +8,10 @@ from api.base.parsers import SearchParser from api.base.settings import REST_FRAMEWORK, MAX_PAGE_SIZE from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch -from api.institutions.serializers import InstitutionSerializer from api.collections.serializers import CollectionSubmissionSerializer from framework.auth.oauth_scopes import CoreScopes -from osf.models import Institution, CollectionSubmission +from osf.models import CollectionSubmission from website.search import search from website.search.exceptions import MalformedQueryError @@ -65,49 +64,6 @@ def get_queryset(self, query=None): return results -class SearchInstitutions(BaseSearchView): - """ - *Read-Only* - - Institutions that have been found by the given Elasticsearch query. - - - - ##Attributes - - OSF Institutions have the "institutions" `type`. - - name type description - ========================================================================= - name string title of the institution - id string unique identifier in the OSF - logo_path string a path to the institution's static logo - - ##Relationships - - ###Nodes - List of nodes that have this institution as its primary institution. - - ###Users - List of users that are affiliated with this institution. - - ##Links - - self: the canonical api endpoint of this institution - html: this institution's page on the OSF website - - # This Request/Response - - """ - - model_class = Institution - serializer_class = InstitutionSerializer - - doc_type = 'institution' - view_category = 'search' - view_name = 'search-institution' - - class SearchCollections(BaseSearchView): """ """ diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py index 1c722ada403..72cd936ecda 100644 --- a/api_tests/search/views/test_views.py +++ b/api_tests/search/views/test_views.py @@ -155,43 +155,6 @@ def file_private(self, component_private, user_one): component_private, user_one, filename='Wavves.mp3') -class TestSearchInstitutions(ApiSearchTestCase): - - @pytest.fixture() - def url_institution_search(self): - return f'/{API_BASE}search/institutions/' - - def test_search_institutions( - self, app, url_institution_search, user, institution): - - # test_search_institutions_no_auth - res = app.get(url_institution_search) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert institution.name in res - - # test_search_institutions_auth - res = app.get(url_institution_search, auth=user) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert institution.name in res - - # test_search_institutions_by_name - url = '{}?q={}'.format(url_institution_search, 'Social') - res = app.get(url) - assert res.status_code == 200 - num_results = len(res.json['data']) - total = res.json['links']['meta']['total'] - assert num_results == 1 - assert total == 1 - assert institution.name == res.json['data'][0]['attributes']['name'] - class TestSearchCollections(ApiSearchTestCase): def get_ids(self, data): From 0e8f7ea49161ee09fe2a2aca1eefcbbbe6a9bd46 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 8 Apr 2026 19:37:02 +0300 Subject: [PATCH 19/43] removed api/v2/search/collections/ endpoint, scopes and its dependencies --- api/base/filters.py | 20 -- api/base/pagination.py | 179 ------------ api/base/parsers.py | 50 ---- api/search/__init__.py | 0 api/search/permissions.py | 8 - api/search/urls.py | 12 - api/search/views.py | 100 ------- api_tests/base/test_views.py | 3 +- api_tests/search/__init__.py | 0 api_tests/search/views/__init__.py | 0 api_tests/search/views/test_views.py | 420 --------------------------- framework/auth/oauth_scopes.py | 4 - website/search/elastic_search.py | 14 - 13 files changed, 1 insertion(+), 809 deletions(-) delete mode 100644 api/search/__init__.py delete mode 100644 api/search/permissions.py delete mode 100644 api/search/urls.py delete mode 100644 api/search/views.py delete mode 100644 api_tests/search/__init__.py delete mode 100644 api_tests/search/views/__init__.py delete mode 100644 api_tests/search/views/test_views.py diff --git a/api/base/filters.py b/api/base/filters.py index 7a5e6ed2450..897144a8f8b 100644 --- a/api/base/filters.py +++ b/api/base/filters.py @@ -114,26 +114,6 @@ def remove_invalid_fields(self, queryset, fields, view, request): return valid_fields -class ElasticOSFOrderingFilter(OSFOrderingFilter): - """ This is too enable sorting for ES endpoints that use ES results instead of a typical queryset""" - def filter_queryset(self, request, queryset, view): - sorted_list = queryset.copy() - sort = request.query_params.get('sort') - reverse = False - if sort: - if sort.startswith('-'): - sort = sort.lstrip('-') - reverse = True - - try: - source = view.get_serializer_class()._declared_fields[sort].source - sorted_list['results'] = sorted(queryset['results'], key=lambda item: item['_source'][source], reverse=reverse) - except KeyError: - pass - - return sorted_list - - class FilterMixin: """ View mixin with helper functions for filtering. """ diff --git a/api/base/pagination.py b/api/base/pagination.py index f26e54bfbad..25fba6c4abe 100644 --- a/api/base/pagination.py +++ b/api/base/pagination.py @@ -11,10 +11,8 @@ ) from api.base.serializers import is_anonymized from api.base.settings import MAX_PAGE_SIZE, MAX_SIZE_OF_ES_QUERY -from api.base.utils import absolute_reverse from osf.models import AbstractNode, Comment, Preprint, Guid, DraftRegistration -from website.search.elastic_search import DOC_TYPE_TO_MODEL class JSONAPIPagination(pagination.PageNumberPagination): @@ -263,180 +261,3 @@ class DraftRegistrationContributorPagination(NodeContributorPagination): def get_resource(self, kwargs): resource_id = kwargs.get('draft_id') return DraftRegistration.load(resource_id) - - -class SearchPaginator(DjangoPaginator): - - def __init__(self, object_list, per_page): - super().__init__(object_list, per_page) - - def search_type_to_model(self, obj_id, obj_type): - model = DOC_TYPE_TO_MODEL[obj_type] - return model.load(obj_id) - - def _get_count(self): - self._count = self.object_list['aggs']['total'] - return self._count - count = property(_get_count) - - def page(self, number): - number = self.validate_number(number) - results = self.object_list['results'] - items = [ - self.search_type_to_model(result.get('_id'), result.get('_type')) - for result in results - ] - return self._get_page(items, number, self) - - -class SearchModelPaginator(SearchPaginator): - - def __init__(self, object_list, per_page, model): - super().__init__(object_list, per_page) - self.model = model - - def page(self, number): - number = self.validate_number(number) - results = self.object_list['results'] - items = [ - self.model.load(result.get('_id')) - for result in results - ] - return self._get_page(items, number, self) - - -class SearchPagination(JSONAPIPagination): - - def __init__(self): - super().__init__() - self.paginator = None - - def paginate_queryset(self, queryset, request, view=None): - page_size = self.get_page_size(request) - if not page_size: - return None - - # Pagination requires an order by clause, especially when using Postgres. - # see: https://docs.djangoproject.com/en/1.10/topics/pagination/#required-arguments - if isinstance(queryset, QuerySet) and not queryset.ordered: - queryset = queryset.order_by(queryset.model._meta.pk.name) - - self.paginator = SearchPaginator(queryset, page_size) - model = getattr(request.parser_context['view'], 'model_class', None) - if model: - self.paginator = SearchModelPaginator(queryset, page_size, model) - - page_number = request.query_params.get(self.page_query_param, 1) - if page_number in self.last_page_strings: - page_number = self.paginator.num_pages - - try: - self.page = self.paginator.page(page_number) - except InvalidPage as exc: - msg = self.invalid_page_message.format( - page_number=page_number, message=str(exc), - ) - raise NotFound(msg) - - if self.paginator.num_pages > 1 and self.template is not None: - # The browsable API should display pagination controls. - self.display_page_controls = True - - self.request = request - return list(self.page) - - def get_search_field_url(self, field, query): - view_name = f'search:search-{field}' - return absolute_reverse( - view_name, - query_kwargs={ - 'q': query, - }, - kwargs={ - 'version': self.request.parser_context['kwargs']['version'], - }, - ) - - def get_search_field_total(self, field): - return self.paginator.object_list['counts'].get(field, 0) - - def get_search_field(self, field, query): - return OrderedDict([ - ( - 'related', OrderedDict([ - ('href', self.get_search_field_url(field, query)), - ( - 'meta', OrderedDict([ - ('total', self.get_search_field_total(field)), - ]), - ), - ]), - ), - ]) - - def get_response_dict(self, data, url): - if isinstance(self.paginator, SearchModelPaginator): - return super().get_response_dict(data, url) - else: - query = self.request.query_params.get('q', '*') - return OrderedDict([ - ('data', data), - ( - 'search_fields', OrderedDict([ - ('files', self.get_search_field('file', query)), - ('projects', self.get_search_field('project', query)), - ('components', self.get_search_field('component', query)), - ('registrations', self.get_search_field('registration', query)), - ('users', self.get_search_field('user', query)), - ('institutions', self.get_search_field('institution', query)), - ]), - ), - ( - 'meta', OrderedDict([ - ('total', self.page.paginator.count), - ('per_page', self.page.paginator.per_page), - ]), - ), - ( - 'links', OrderedDict([ - ('self', self.get_self_real_link(url)), - ('first', self.get_first_real_link(url)), - ('last', self.get_last_real_link(url)), - ('prev', self.get_previous_real_link(url)), - ('next', self.get_next_real_link(url)), - ]), - ), - ]) - - def get_response_dict_deprecated(self, data, url): - if isinstance(self.paginator, SearchModelPaginator): - return super().get_response_dict_deprecated(data, url) - else: - query = self.request.query_params.get('q', '*') - return OrderedDict([ - ('data', data), - ( - 'search_fields', OrderedDict([ - ('files', self.get_search_field('file', query)), - ('projects', self.get_search_field('project', query)), - ('components', self.get_search_field('component', query)), - ('registrations', self.get_search_field('registration', query)), - ('users', self.get_search_field('user', query)), - ('institutions', self.get_search_field('institution', query)), - ]), - ), - ( - 'links', OrderedDict([ - ('first', self.get_first_real_link(url)), - ('last', self.get_last_real_link(url)), - ('prev', self.get_previous_real_link(url)), - ('next', self.get_next_real_link(url)), - ( - 'meta', OrderedDict([ - ('total', self.page.paginator.count), - ('per_page', self.page.paginator.per_page), - ]), - ), - ]), - ), - ]) diff --git a/api/base/parsers.py b/api/base/parsers.py index d0ce368e6a5..11bff6b774c 100644 --- a/api/base/parsers.py +++ b/api/base/parsers.py @@ -3,7 +3,6 @@ import time import collections from jsonschema import validate, ValidationError, Draft7Validator -from django.core.exceptions import ImproperlyConfigured from rest_framework.parsers import JSONParser from rest_framework.exceptions import ParseError, NotAuthenticated @@ -307,52 +306,3 @@ def parse(self, stream, media_type=None, parser_context=None): raise JSONAPIException(detail='Signature has expired') return payload - -class SearchParser(JSONAPIParser): - - def parse(self, stream, media_type=None, parser_context=None): - try: - view = parser_context['view'] - except KeyError: - raise ImproperlyConfigured('SearchParser requires "view" context.') - data = super().parse(stream, media_type=media_type, parser_context=parser_context) - if not data: - raise JSONAPIException(detail='Invalid Payload') - - res = { - 'query': { - 'bool': {}, - }, - } - - sort = parser_context['request'].query_params.get('sort') - if sort: - res['sort'] = [{ - sort.lstrip('-'): { - 'order': 'desc' if sort.startswith('-') else 'asc', - }, - }] - - try: - q = data.pop('q') - except KeyError: - pass - else: - res['query']['bool'].update({ - 'must': { - 'query_string': { - 'query': q, - 'fields': view.search_fields, - }, - }, - }) - - if any(data.values()): - res['query']['bool'].update({'filter': []}) - for key, val in data.items(): - if val is not None: - if isinstance(val, list): - res['query']['bool']['filter'].append({'terms': {key: val}}) - else: - res['query']['bool']['filter'].append({'term': {key: val}}) - return res diff --git a/api/search/__init__.py b/api/search/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/api/search/permissions.py b/api/search/permissions.py deleted file mode 100644 index 7266e211d75..00000000000 --- a/api/search/permissions.py +++ /dev/null @@ -1,8 +0,0 @@ -from rest_framework.permissions import IsAuthenticatedOrReadOnly - -class IsAuthenticatedOrReadOnlyForSearch(IsAuthenticatedOrReadOnly): - def has_permission(self, request, view): - from api.search.views import BaseSearchView - if not isinstance(view, BaseSearchView): - return False - return request.method == 'POST' or super().has_permission(request, view) diff --git a/api/search/urls.py b/api/search/urls.py deleted file mode 100644 index ab8d62e5121..00000000000 --- a/api/search/urls.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.urls import re_path - -from api.search import views - -app_name = 'osf' - -urlpatterns = [ - re_path(r'^collections/$', views.SearchCollections.as_view(), name=views.SearchCollections.view_name), - - # not currently supported by v1, but should be supported by v2 - # re_path(r'^nodes/$', views.SearchProjects.as_view(), name=views.SearchProjects.view_name), -] diff --git a/api/search/views.py b/api/search/views.py deleted file mode 100644 index beb9d922c97..00000000000 --- a/api/search/views.py +++ /dev/null @@ -1,100 +0,0 @@ -from rest_framework import generics -from rest_framework.exceptions import ValidationError -from rest_framework.response import Response - -from api.base import permissions as base_permissions -from api.base.views import JSONAPIBaseView -from api.base.pagination import SearchPagination -from api.base.parsers import SearchParser -from api.base.settings import REST_FRAMEWORK, MAX_PAGE_SIZE -from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch -from api.collections.serializers import CollectionSubmissionSerializer - -from framework.auth.oauth_scopes import CoreScopes -from osf.models import CollectionSubmission - -from website.search import search -from website.search.exceptions import MalformedQueryError -from website.search.util import build_query -from api.base.filters import ElasticOSFOrderingFilter - - -class BaseSearchView(JSONAPIBaseView, generics.ListCreateAPIView): - - required_read_scopes = [CoreScopes.SEARCH] - required_write_scopes = [CoreScopes.NULL] - - permission_classes = ( - IsAuthenticatedOrReadOnlyForSearch, - base_permissions.TokenHasScope, - ) - - pagination_class = SearchPagination - filter_backends = [ElasticOSFOrderingFilter] - - @property - def search_fields(self): - # Should be overridden in subclasses to provide a list of keys found - # in the relevant elastic doc. - raise NotImplementedError - - def __init__(self): - super().__init__() - self.doc_type = getattr(self, 'doc_type', None) - - def get_parsers(self): - if self.request.method == 'POST': - return (SearchParser(),) - return super().get_parsers() - - def get_queryset(self, query=None): - page = int(self.request.query_params.get('page', '1')) - page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE) - start = (page - 1) * page_size - if query: - # Parser has built query, but needs paging info - query['from'] = start - query['size'] = page_size - else: - query = build_query(self.request.query_params.get('q', '*'), start=start, size=page_size) - try: - results = search.search(query, doc_type=self.doc_type, raw=True) - except MalformedQueryError as e: - raise ValidationError(e) - return results - - -class SearchCollections(BaseSearchView): - """ - """ - - model_class = CollectionSubmission - serializer_class = CollectionSubmissionSerializer - - doc_type = 'collectionSubmission' - view_category = 'search' - view_name = 'search-collected-metadata' - required_write_scopes = [CoreScopes.ADVANCED_SEARCH] - - @property - def search_fields(self): - return [ - 'abstract', - 'collectedType', - 'contributors.fullname', - 'status', - 'subjects', - 'provider', - 'title', - 'tags', - ] - - def create(self, request, *args, **kwargs): - # Override POST methods to behave like list, with header query parsing - queryset = self.filter_queryset(self.get_queryset(request.data)) - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) diff --git a/api_tests/base/test_views.py b/api_tests/base/test_views.py index 2d6b029b6ba..e5b0d23edeb 100644 --- a/api_tests/base/test_views.py +++ b/api_tests/base/test_views.py @@ -9,7 +9,6 @@ from framework.auth.oauth_scopes import CoreScopes from api.base.settings.defaults import API_BASE -from api.search.permissions import IsAuthenticatedOrReadOnlyForSearch from api.crossref.views import ParseCrossRefConfirmation from api.metrics.views import ( RawMetricsView, @@ -102,7 +101,7 @@ def test_does_not_exist_formatting(self): def test_view_classes_have_minimal_set_of_permissions_classes(self): base_permissions = [ TokenHasScope, - (IsAuthenticated, IsAuthenticatedOrReadOnly, IsAuthenticatedOrReadOnlyForSearch) + (IsAuthenticated, IsAuthenticatedOrReadOnly) ] for view in VIEW_CLASSES: if view in self.EXCLUDED_VIEWS: diff --git a/api_tests/search/__init__.py b/api_tests/search/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/api_tests/search/views/__init__.py b/api_tests/search/views/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/api_tests/search/views/test_views.py b/api_tests/search/views/test_views.py deleted file mode 100644 index 72cd936ecda..00000000000 --- a/api_tests/search/views/test_views.py +++ /dev/null @@ -1,420 +0,0 @@ -import pytest -import uuid -from unittest import mock - -from api.base.settings.defaults import API_BASE -from api_tests import utils -from framework.auth.cas import CasResponse -from framework.auth.core import Auth -from osf_tests.factories import ( - AuthUserFactory, - NodeFactory, - ProjectFactory, - RegistrationFactory, - InstitutionFactory, - CollectionFactory, - CollectionProviderFactory, - RegistrationProviderFactory, -) -from tests.utils import capture_notifications -from website import settings -from website.search import elastic_search -from website.search import search - -SCHEMA_VERSION = 2 - - -@pytest.mark.django_db -@pytest.mark.enable_search -@pytest.mark.enable_enqueue_task -class ApiSearchTestCase: - - @pytest.fixture(autouse=True) - def index(self): - settings.ELASTIC_INDEX = uuid.uuid4().hex - elastic_search.INDEX = settings.ELASTIC_INDEX - - search.create_index(elastic_search.INDEX) - yield - search.delete_index(elastic_search.INDEX) - - @pytest.fixture() - def user(self): - return AuthUserFactory() - - @pytest.fixture() - def institution(self): - return InstitutionFactory(name='Social Experiment') - - @pytest.fixture() - def collection_public(self, user): - return CollectionFactory(creator=user, provider=CollectionProviderFactory(), is_public=True, - status_choices=['', 'asdf', 'lkjh'], collected_type_choices=['', 'asdf', 'lkjh'], - issue_choices=['', '0', '1', '2'], volume_choices=['', '0', '1', '2'], - disease_choices=['illness'], data_type_choices=['realness'], - program_area_choices=['', 'asdf', 'lkjh'], grade_levels_choices=['super', 'cool']) - - @pytest.fixture() - def registration_collection(self, user): - return CollectionFactory(creator=user, provider=RegistrationProviderFactory(), is_public=True, - status_choices=['', 'asdf', 'lkjh'], collected_type_choices=['', 'asdf', 'lkjh']) - - @pytest.fixture() - def user_one(self): - user_one = AuthUserFactory(fullname='Kanye Omari West') - user_one.schools = [{ - 'degree': 'English', - 'institution': 'Chicago State University' - }] - user_one.jobs = [{ - 'title': 'Producer', - 'institution': 'GOOD Music, Inc.' - }] - user_one.save() - return user_one - - @pytest.fixture() - def user_two(self, institution): - user_two = AuthUserFactory(fullname='Chance The Rapper') - user_two.add_or_update_affiliated_institution(institution) - user_two.save() - return user_two - - @pytest.fixture() - def project(self, user_one): - project = ProjectFactory( - title='Graduation', - creator=user_one, - is_public=True) - project.update_search() - return project - - @pytest.fixture() - def project_public(self, user_one): - project_public = ProjectFactory( - title='The Life of Pablo', - creator=user_one, - is_public=True) - project_public.set_description( - 'Name one genius who ain\'t crazy', - auth=Auth(user_one), - save=True) - project_public.add_tag('Yeezus', auth=Auth(user_one), save=True) - return project_public - - @pytest.fixture() - def project_private(self, user_two): - return ProjectFactory(title='Coloring Book', creator=user_two) - - @pytest.fixture() - def component(self, user_one, project_public): - return NodeFactory( - parent=project_public, - title='Highlights', - description='', - creator=user_one, - is_public=True) - - @pytest.fixture() - def component_public(self, user_two, project_public): - component_public = NodeFactory( - parent=project_public, - title='Ultralight Beam', - creator=user_two, - is_public=True) - component_public.set_description( - 'This is my part, nobody else speak', - auth=Auth(user_two), - save=True) - component_public.add_tag('trumpets', auth=Auth(user_two), save=True) - return component_public - - @pytest.fixture() - def component_private(self, user_one, project_public): - return NodeFactory( - parent=project_public, - description='', - title='Wavves', - creator=user_one) - - @pytest.fixture() - def file_component(self, component, user_one): - return utils.create_test_file( - component, user_one, filename='Highlights.mp3') - - @pytest.fixture() - def file_public(self, component_public, user_one): - return utils.create_test_file( - component_public, - user_one, - filename='UltralightBeam.mp3') - - @pytest.fixture() - def file_private(self, component_private, user_one): - return utils.create_test_file( - component_private, user_one, filename='Wavves.mp3') - - -class TestSearchCollections(ApiSearchTestCase): - - def get_ids(self, data): - return [s['id'] for s in data] - - def post_payload(self, *args, **kwargs): - return { - 'data': { - 'attributes': kwargs - }, - 'type': 'search' - } - - @pytest.fixture() - def url_collection_search(self): - return f'/{API_BASE}search/collections/' - - @pytest.fixture() - def node_one(self, user): - return NodeFactory(title='Ismael Lo: Tajabone', creator=user, is_public=True) - - @pytest.fixture() - def registration_one(self, node_one): - return RegistrationFactory(project=node_one, is_public=True) - - @pytest.fixture() - def node_two(self, user): - return NodeFactory(title='Sambolera', creator=user, is_public=True) - - @pytest.fixture() - def registration_two(self, node_two): - return RegistrationFactory(project=node_two, is_public=True) - - @pytest.fixture() - def node_private(self, user): - return NodeFactory(title='Classified', creator=user) - - @pytest.fixture() - def registration_private(self, node_private): - return RegistrationFactory(project=node_private, is_public=False) - - @pytest.fixture() - def node_with_abstract(self, user): - node_with_abstract = NodeFactory(title='Sambolera', creator=user, is_public=True) - node_with_abstract.set_description( - 'Sambolera by Khadja Nin', - auth=Auth(user), - save=True) - return node_with_abstract - - @pytest.fixture() - def reg_with_abstract(self, node_with_abstract): - return RegistrationFactory(project=node_with_abstract, is_public=True) - - def test_search_collections( - self, app, url_collection_search, user, node_one, node_two, collection_public, - node_with_abstract, node_private, registration_collection, registration_one, registration_two, - registration_private, reg_with_abstract): - - with capture_notifications(): - collection_public.collect_object(node_one, user) - collection_public.collect_object(node_two, user) - collection_public.collect_object(node_private, user) - - registration_collection.collect_object(registration_one, user) - registration_collection.collect_object(registration_two, user) - registration_collection.collect_object(registration_private, user) - - # test_search_collections_no_auth - res = app.get(url_collection_search) - assert res.status_code == 200 - total = res.json['links']['meta']['total'] - num_results = len(res.json['data']) - assert total == 4 - assert num_results == 4 - actual_ids = self.get_ids(res.json['data']) - assert registration_private._id not in actual_ids - assert node_private._id not in actual_ids - - # test_search_collections_auth - res = app.get(url_collection_search, auth=user) - assert res.status_code == 200 - total = res.json['links']['meta']['total'] - num_results = len(res.json['data']) - assert total == 4 - assert num_results == 4 - actual_ids = self.get_ids(res.json['data']) - assert registration_private._id not in actual_ids - assert node_private._id not in actual_ids - - # test_search_collections_by_submission_title - url = '{}?q={}'.format(url_collection_search, 'Ismael') - res = app.get(url) - assert res.status_code == 200 - total = res.json['links']['meta']['total'] - num_results = len(res.json['data']) - assert node_one.title == registration_one.title == res.json['data'][0]['embeds']['guid']['data']['attributes']['title'] - assert total == num_results == 2 - - # test_search_collections_by_submission_abstract - with capture_notifications(): - collection_public.collect_object(node_with_abstract, user) - registration_collection.collect_object(reg_with_abstract, user) - url = '{}?q={}'.format(url_collection_search, 'KHADJA') - res = app.get(url) - assert res.status_code == 200 - total = res.json['links']['meta']['total'] - assert node_with_abstract.description == reg_with_abstract.description == res.json['data'][0]['embeds']['guid']['data']['attributes']['description'] - assert total == 2 - - # test_search_collections_no_results: - url = '{}?q={}'.format(url_collection_search, 'Wale Watu') - res = app.get(url) - assert res.status_code == 200 - total = res.json['links']['meta']['total'] - assert total == 0 - - def test_POST_search_collections( - self, app, url_collection_search, user, node_one, node_two, collection_public, - node_with_abstract, node_private, registration_collection, registration_one, - registration_two, registration_private, reg_with_abstract): - with capture_notifications(): - collection_public.collect_object(node_one, user, status='asdf', issue='0', volume='1', program_area='asdf') - collection_public.collect_object(node_two, user, collected_type='asdf', status='lkjh') - collection_public.collect_object(node_with_abstract, user, status='asdf', grade_levels='super') - collection_public.collect_object(node_private, user, status='asdf', collected_type='asdf') - - registration_collection.collect_object(registration_one, user, status='asdf') - registration_collection.collect_object(registration_two, user, collected_type='asdf', status='lkjh') - registration_collection.collect_object(reg_with_abstract, user, status='asdf') - registration_collection.collect_object(registration_private, user, status='asdf', collected_type='asdf') - - # test_search_empty - payload = self.post_payload() - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 6 - assert len(res.json['data']) == 6 - actual_ids = self.get_ids(res.json['data']) - assert registration_private._id not in actual_ids - assert node_private._id not in actual_ids - - # test_search_title_keyword - payload = self.post_payload(q='Ismael') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 2 - assert len(res.json['data']) == 2 - actual_ids = self.get_ids(res.json['data']) - assert registration_private._id not in actual_ids - assert node_private._id not in actual_ids - - # test_search_abstract_keyword - payload = self.post_payload(q='Khadja') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 2 - assert len(res.json['data']) == 2 - actual_ids = self.get_ids(res.json['data']) - assert node_with_abstract._id in actual_ids - assert reg_with_abstract._id in actual_ids - - # test_search_filter - payload = self.post_payload(status='asdf') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 4 - assert len(res.json['data']) == 4 - actual_ids = self.get_ids(res.json['data']) - assert registration_private._id not in actual_ids - assert node_private._id not in actual_ids - - payload = self.post_payload(status=['asdf', 'lkjh']) - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 6 - assert len(res.json['data']) == 6 - actual_ids = self.get_ids(res.json['data']) - assert registration_private._id not in actual_ids - assert node_private._id not in actual_ids - - payload = self.post_payload(collectedType='asdf') - res = app.post_json_api(url_collection_search, payload) - - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 2 - assert len(res.json['data']) == 2 - actual_ids = self.get_ids(res.json['data']) - assert node_two._id in actual_ids - assert registration_two._id in actual_ids - - payload = self.post_payload(status='asdf', issue='0', volume='1', programArea='asdf', collectedType='') - res = app.post_json_api(url_collection_search, payload) - - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 1 - assert len(res.json['data']) == 1 - actual_ids = self.get_ids(res.json['data']) - assert node_one._id in actual_ids - - payload = self.post_payload(gradeLevels='super') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 1 - - # test_search_abstract_keyword_and_filter - payload = self.post_payload(q='Khadja', status='asdf') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 2 - assert len(res.json['data']) == 2 - actual_ids = self.get_ids(res.json['data']) - assert node_with_abstract._id in actual_ids - assert reg_with_abstract._id in actual_ids - - # test_search_abstract_keyword_and_filter_provider - payload = self.post_payload(q='Khadja', status='asdf', provider=collection_public.provider._id) - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 1 - assert len(res.json['data']) == 1 - assert res.json['data'][0]['id'] == node_with_abstract._id - - def test_POST_search_collections_disease_data_type( - self, app, url_collection_search, user, node_one, node_two, collection_public, - node_with_abstract, node_private, registration_collection, registration_one, - registration_two, registration_private, reg_with_abstract): - - with capture_notifications(): - collection_public.collect_object(node_one, user, disease='illness', data_type='realness') - collection_public.collect_object(node_two, user, data_type='realness') - - payload = self.post_payload(disease='illness') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 1 - - payload = self.post_payload(dataType='realness') - res = app.post_json_api(url_collection_search, payload) - assert res.status_code == 200 - assert res.json['links']['meta']['total'] == 2 - assert len(res.json['data']) == 2 - - def test_POST_search_collections_scope(self, app, url_collection_search, user): - payload = self.post_payload(q='Collection') - - token_invalid = CasResponse( - authenticated=True, - user=user._id, - attributes={'accessTokenScope': ['osf.full_read']} - ) - with mock.patch('framework.auth.cas.CasClient.profile', return_value=token_invalid): - res = app.post_json_api(url_collection_search, payload, auth='some-invalid-token', expect_errors=True, auth_type='jwt') - assert res.status_code == 403 - - token_valid = CasResponse( - authenticated=True, - user=user._id, - attributes={'accessTokenScope': ['osf.full_read', 'osf.full_write']} - ) - with mock.patch('framework.auth.cas.CasClient.profile', return_value=token_valid): - res = app.post_json_api(url_collection_search, payload, auth='some-valid-token', auth_type='jwt') - assert res.status_code == 200 diff --git a/framework/auth/oauth_scopes.py b/framework/auth/oauth_scopes.py index f3f060a2571..2d121d61d57 100644 --- a/framework/auth/oauth_scopes.py +++ b/framework/auth/oauth_scopes.py @@ -204,8 +204,6 @@ class CoreScopes: READ_COLLECTION_SUBMISSION = 'read_collection_submission' WRITE_COLLECTION_SUBMISSION = 'write_collection_submission' - ADVANCED_SEARCH = 'advanced_search' - class ComposedScopes: """ @@ -336,7 +334,6 @@ class ComposedScopes: CoreScopes.CEDAR_METADATA_RECORD_READ, CoreScopes.MEETINGS_READ, CoreScopes.INSTITUTION_READ, - CoreScopes.SEARCH, CoreScopes.SCOPES_READ, CoreScopes.USERS_MESSAGE_READ_EMAIL )\ @@ -362,7 +359,6 @@ class ComposedScopes: CoreScopes.WRITE_COLLECTION_SUBMISSION_ACTION, CoreScopes.WRITE_COLLECTION_SUBMISSION, CoreScopes.USERS_MESSAGE_WRITE_EMAIL, - CoreScopes.ADVANCED_SEARCH ) # Admin permissions- includes functionality not intended for third-party use diff --git a/website/search/elastic_search.py b/website/search/elastic_search.py index ab9f339e0da..198bddc30cb 100644 --- a/website/search/elastic_search.py +++ b/website/search/elastic_search.py @@ -13,14 +13,10 @@ from framework.celery_tasks import app as celery_app from framework.database import paginated from osf.models import AbstractNode -from osf.models import OSFUser -from osf.models import BaseFileNode from osf.models import GuidMetadataRecord -from osf.models import Institution from osf.models import Preprint from osf.models import SpamStatus from addons.wiki.models import WikiPage -from osf.models import CollectionSubmission from osf.utils.sanitize import unescape_entities from osf.utils.workflows import CollectionSubmissionStates from website import settings @@ -44,16 +40,6 @@ 'group': 'Groups', } -DOC_TYPE_TO_MODEL = { - 'component': AbstractNode, - 'project': AbstractNode, - 'registration': AbstractNode, - 'user': OSFUser, - 'file': BaseFileNode, - 'institution': Institution, - 'preprint': Preprint, - 'collectionSubmission': CollectionSubmission, -} # Prevent tokenizing and stop word removal. NOT_ANALYZED_PROPERTY = {'type': 'string', 'index': 'not_analyzed'} From 5c9e978e9460c70a655e346abf2b59a8c21dffd8 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 9 Apr 2026 17:01:07 +0300 Subject: [PATCH 20/43] removed api/v2/search/ base route --- api/base/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/base/urls.py b/api/base/urls.py index 142e2df34c2..63984efd287 100644 --- a/api/base/urls.py +++ b/api/base/urls.py @@ -70,7 +70,6 @@ re_path(r'^requests/', include(('api.requests.urls', 'requests'), namespace='requests')), re_path(r'^resources/', include('api.resources.urls', namespace='resources')), re_path(r'^scopes/', include('api.scopes.urls', namespace='scopes')), - re_path(r'^search/', include('api.search.urls', namespace='search')), re_path(r'^sparse/', include('api.sparse.urls', namespace='sparse')), re_path(r'^subjects/', include('api.subjects.urls', namespace='subjects')), re_path(r'^subscriptions/', include('api.subscriptions.urls', namespace='subscriptions')), From 8a9243a87adf27af9540eeec574d7323ab1f3c42 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 9 Apr 2026 17:26:44 +0300 Subject: [PATCH 21/43] removed test related to legacy /explore/ route --- tests/test_misc_views.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_misc_views.py b/tests/test_misc_views.py index 3d1e0cad47c..989062ac77a 100644 --- a/tests/test_misc_views.py +++ b/tests/test_misc_views.py @@ -354,13 +354,6 @@ def test_get_node_parent_not_admin(self): assert children[0]['node']['id'] == child3._primary_key -class TestPublicViews(OsfTestCase): - - def test_explore(self): - res = self.app.get('/explore/', follow_redirects=True) - assert res.status_code == 200 - - class TestExternalAuthViews(OsfTestCase): def setUp(self): From dfe25a19dddd04f84241663fdc6769a0284618c6 Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Fri, 10 Apr 2026 14:59:10 +0300 Subject: [PATCH 22/43] Add required metadata validation --- osf/models/provider.py | 24 ++++ osf_tests/test_validate_required_metadata.py | 120 +++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 osf_tests/test_validate_required_metadata.py diff --git a/osf/models/provider.py b/osf/models/provider.py index f1dbdec65b3..82f808934db 100644 --- a/osf/models/provider.py +++ b/osf/models/provider.py @@ -257,6 +257,30 @@ def setup_share_source(self, provider_home_page): self.save() + def validate_required_metadata(self, osf_obj): + from jsonschema import validate, ValidationError as JsonSchemaValidationError + from osf.models.cedar_metadata import CedarMetadataRecord + + if not self.required_metadata_template: + return + + record = CedarMetadataRecord.objects.filter( + guid__in=osf_obj.guids.all(), + template=self.required_metadata_template, + is_published=True, + ).first() + + if record is None: + raise ValidationError( + f'Object must have a published CEDAR metadata record for the required template ' + f'"{self.required_metadata_template.schema_name}".' + ) + + try: + validate(record.metadata, self.required_metadata_template.template) + except JsonSchemaValidationError as e: + raise ValidationError(e.message) + class CollectionProvider(AbstractProvider): DEFAULT_SUBSCRIPTIONS = [ diff --git a/osf_tests/test_validate_required_metadata.py b/osf_tests/test_validate_required_metadata.py new file mode 100644 index 00000000000..73d4b166815 --- /dev/null +++ b/osf_tests/test_validate_required_metadata.py @@ -0,0 +1,120 @@ +import pytest +from faker import Faker +from django.core.exceptions import ValidationError + +from osf.models import CedarMetadataRecord, CedarMetadataTemplate +from osf_tests.factories import AuthUserFactory, PreprintFactory, PreprintProviderFactory + +fake = Faker() + +VALID_JSONSCHEMA = { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'type': 'object', + 'properties': { + 'title': {'type': 'string'}, + }, + 'required': ['title'], +} + + +@pytest.fixture() +def user(): + return AuthUserFactory() + + +@pytest.fixture() +def provider(): + return PreprintProviderFactory() + + +@pytest.fixture() +def cedar_template(): + return CedarMetadataTemplate.objects.create( + schema_name=fake.bs(), + cedar_id=fake.md5(), + template_version=1, + template=VALID_JSONSCHEMA, + active=True, + ) + + +@pytest.fixture() +def preprint(user, provider): + return PreprintFactory(creator=user, provider=provider) + + +@pytest.mark.django_db +class TestValidateRequiredMetadata: + + def test_no_required_template_passes(self, provider, preprint): + assert provider.required_metadata_template is None + provider.validate_required_metadata(preprint) + + def test_missing_record_raises(self, provider, cedar_template, preprint): + provider.required_metadata_template = cedar_template + provider.save() + + with pytest.raises(ValidationError, match='published CEDAR metadata record'): + provider.validate_required_metadata(preprint) + + def test_unpublished_record_raises(self, provider, cedar_template, preprint): + provider.required_metadata_template = cedar_template + provider.save() + + CedarMetadataRecord.objects.create( + guid=preprint.guids.first(), + template=cedar_template, + metadata={'title': 'My Preprint'}, + is_published=False, + ) + + with pytest.raises(ValidationError, match='published CEDAR metadata record'): + provider.validate_required_metadata(preprint) + + def test_published_valid_record_passes(self, provider, cedar_template, preprint): + provider.required_metadata_template = cedar_template + provider.save() + + CedarMetadataRecord.objects.create( + guid=preprint.guids.first(), + template=cedar_template, + metadata={'title': 'My Preprint'}, + is_published=True, + ) + + provider.validate_required_metadata(preprint) + + def test_published_invalid_record_raises(self, provider, cedar_template, preprint): + provider.required_metadata_template = cedar_template + provider.save() + + CedarMetadataRecord.objects.create( + guid=preprint.guids.first(), + template=cedar_template, + metadata={'title': 123}, + is_published=True, + ) + + with pytest.raises(ValidationError): + provider.validate_required_metadata(preprint) + + def test_record_for_wrong_template_raises(self, provider, cedar_template, preprint): + provider.required_metadata_template = cedar_template + provider.save() + + other_template = CedarMetadataTemplate.objects.create( + schema_name=fake.bs(), + cedar_id=fake.md5(), + template_version=1, + template=VALID_JSONSCHEMA, + active=True, + ) + CedarMetadataRecord.objects.create( + guid=preprint.guids.first(), + template=other_template, + metadata={'title': 'My Preprint'}, + is_published=True, + ) + + with pytest.raises(ValidationError, match='published CEDAR metadata record'): + provider.validate_required_metadata(preprint) From 4499aa8205d4759994f70b123d0f09f8a78a3e10 Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Wed, 8 Apr 2026 15:02:39 +0300 Subject: [PATCH 23/43] Add Required metadata template field to admin pages --- admin/base/utils.py | 7 +- admin/collection_providers/forms.py | 4 +- admin/collection_providers/views.py | 131 ++++++------------ admin/preprint_providers/forms.py | 5 +- admin/registration_providers/forms.py | 3 + .../views/test_collection_provider_detail.py | 4 +- .../views/test_preprint_provider_detail.py | 4 +- .../test_registration_provider_detail.py | 4 +- 8 files changed, 63 insertions(+), 99 deletions(-) diff --git a/admin/base/utils.py b/admin/base/utils.py index f0159204863..a612e06c185 100644 --- a/admin/base/utils.py +++ b/admin/base/utils.py @@ -1,7 +1,7 @@ """ Utility functions and classes """ -from osf.models import Subject, NodeLicense, Brand +from osf.models import Subject, NodeLicense, Brand, CedarMetadataTemplate from django.core.exceptions import ValidationError, PermissionDenied from django.urls import reverse @@ -116,6 +116,11 @@ def get_brand_choices(): brands = Brand.objects.all() return [no_default] + [(brand.id, brand.name) for brand in brands] +def get_cedar_template_choices(): + no_default = ('', '---------') + templates = CedarMetadataTemplate.objects.filter(active=True) + return [no_default] + [(t.id, t.schema_name) for t in templates] + def get_toplevel_subjects(): return Subject.objects.filter(parent__isnull=True, provider___id='osf').values_list('id', 'text') diff --git a/admin/collection_providers/forms.py b/admin/collection_providers/forms.py index 7d0e8dadf46..c190f9af9dd 100644 --- a/admin/collection_providers/forms.py +++ b/admin/collection_providers/forms.py @@ -4,7 +4,7 @@ from framework.utils import sanitize_html from osf.models import CollectionProvider -from admin.base.utils import get_nodelicense_choices, get_defaultlicense_choices, validate_slug +from admin.base.utils import get_nodelicense_choices, get_defaultlicense_choices, validate_slug, get_cedar_template_choices class CollectionProviderForm(forms.ModelForm): @@ -37,9 +37,11 @@ class Meta: def __init__(self, *args, **kwargs): nodelicense_choices = get_nodelicense_choices() defaultlicense_choices = get_defaultlicense_choices() + cedar_template_choices = get_cedar_template_choices() super().__init__(*args, **kwargs) self.fields['licenses_acceptable'].choices = nodelicense_choices self.fields['default_license'].choices = defaultlicense_choices + self.fields['required_metadata_template'].choices = cedar_template_choices def clean_description(self, *args, **kwargs): if not self.data.get('description'): diff --git a/admin/collection_providers/views.py b/admin/collection_providers/views.py index 779f7b89c04..ac9efd0e935 100644 --- a/admin/collection_providers/views.py +++ b/admin/collection_providers/views.py @@ -132,127 +132,78 @@ def get_context_data(self, *args, **kwargs): licenses_html += '' collection_provider_attributes['licenses_acceptable'] = licenses_html - primary_collection = collection_provider.primary_collection - # compile html list of collected_type_choices - collected_type_choices_html = '
    ' - for choice in primary_collection.collected_type_choices: - collected_type_choices_html += f'
  • {choice}
    • ' - collected_type_choices_html += '
' - kwargs['collected_type_choices'] = collected_type_choices_html - - # compile html list of status_choices - status_choices_html = '
    ' - for choice in primary_collection.status_choices: - status_choices_html += f'
  • {choice}
    • ' - status_choices_html += '
' - kwargs['status_choices'] = status_choices_html - - # compile html list of volume_choices - volume_choices_html = '
    ' - for choice in primary_collection.volume_choices: - volume_choices_html += f'
  • {choice}
    • ' - volume_choices_html += '
' - kwargs['volume_choices'] = volume_choices_html - - # compile html list of issue_choices - issue_choices_html = '
    ' - for choice in primary_collection.issue_choices: - issue_choices_html += f'
  • {choice}
    • ' - issue_choices_html += '
' - kwargs['issue_choices'] = issue_choices_html - - # compile html list of program_area_choices - program_area_choices_html = '
    ' - for choice in primary_collection.program_area_choices: - program_area_choices_html += f'
  • {choice}
    • ' - program_area_choices_html += '
' - kwargs['program_area_choices'] = program_area_choices_html - - # compile html list of school_type_choices - school_type_choices_html = '
    {choices}
'.format(choices=''.join( - f'
  • {choice}
    • ' for choice in primary_collection.school_type_choices - )) - kwargs['school_type_choices'] = school_type_choices_html - - # compile html list of study_design_choices - study_design_choices_html = '
      {choices}
    '.format(choices=''.join( - f'
  • {choice}
    • ' for choice in primary_collection.study_design_choices - )) - kwargs['study_design_choices'] = study_design_choices_html - - disease_choices_html = '
      {choices}
    '.format(choices=''.join( - f'
  • {choice}
    • ' for choice in primary_collection.disease_choices - )) - kwargs['disease_choices'] = disease_choices_html - - data_type_choices_html = '
      {choices}
    '.format(choices=''.join( - f'
  • {choice}
    • ' for choice in primary_collection.data_type_choices - )) - kwargs['data_type_choices'] = data_type_choices_html - - grade_levels_choices_html = '
      {choices}
    '.format(choices=''.join( - f'
  • {choice}
    • ' for choice in primary_collection.grade_levels_choices - )) - kwargs['grade_levels_choices'] = grade_levels_choices_html - - # get a dict of model fields so that we can set the initial value for the update form fields = model_to_dict(collection_provider) - fields['collected_type_choices'] = json.dumps(primary_collection.collected_type_choices) - fields['status_choices'] = json.dumps(primary_collection.status_choices) - fields['volume_choices'] = json.dumps(primary_collection.volume_choices) - fields['issue_choices'] = json.dumps(primary_collection.issue_choices) - fields['program_area_choices'] = json.dumps(primary_collection.program_area_choices) - fields = model_to_dict(collection_provider) - - fields['school_type_choices'] = json.dumps(primary_collection.school_type_choices) - fields['study_design_choices'] = json.dumps(primary_collection.study_design_choices) - fields['data_type_choices'] = json.dumps(primary_collection.data_type_choices) - fields['disease_choices'] = json.dumps(primary_collection.disease_choices) - fields['grade_levels_choices'] = json.dumps(primary_collection.grade_levels_choices) - - # compile html list of collected_type_choices if collection_provider.primary_collection: + primary_collection = collection_provider.primary_collection + + # compile html list of collected_type_choices collected_type_choices_html = '
      ' - for choice in collection_provider.primary_collection.collected_type_choices: + for choice in primary_collection.collected_type_choices: collected_type_choices_html += f'
    • {choice}
      • ' collected_type_choices_html += '
    ' kwargs['collected_type_choices'] = collected_type_choices_html # compile html list of status_choices status_choices_html = '
      ' - for choice in collection_provider.primary_collection.status_choices: + for choice in primary_collection.status_choices: status_choices_html += f'
    • {choice}
      • ' status_choices_html += '
    ' kwargs['status_choices'] = status_choices_html # compile html list of volume_choices volume_choices_html = '
      ' - for choice in collection_provider.primary_collection.volume_choices: + for choice in primary_collection.volume_choices: volume_choices_html += f'
    • {choice}
      • ' volume_choices_html += '
    ' kwargs['volume_choices'] = volume_choices_html # compile html list of issue_choices issue_choices_html = '
      ' - for choice in collection_provider.primary_collection.issue_choices: + for choice in primary_collection.issue_choices: issue_choices_html += f'
    • {choice}
      • ' issue_choices_html += '
    ' kwargs['issue_choices'] = issue_choices_html # compile html list of program_area_choices program_area_choices_html = '
      ' - for choice in collection_provider.primary_collection.program_area_choices: + for choice in primary_collection.program_area_choices: program_area_choices_html += f'
    • {choice}
      • ' program_area_choices_html += '
    ' kwargs['program_area_choices'] = program_area_choices_html - # get a dict of model fields so that we can set the initial value for the update form - fields['collected_type_choices'] = json.dumps(collection_provider.primary_collection.collected_type_choices) - fields['status_choices'] = json.dumps(collection_provider.primary_collection.status_choices) - fields['volume_choices'] = json.dumps(collection_provider.primary_collection.volume_choices) - fields['issue_choices'] = json.dumps(collection_provider.primary_collection.issue_choices) - fields['program_area_choices'] = json.dumps(collection_provider.primary_collection.program_area_choices) + # compile html list of school_type_choices + kwargs['school_type_choices'] = '
      {choices}
    '.format(choices=''.join( + f'
  • {choice}
    • ' for choice in primary_collection.school_type_choices + )) + + # compile html list of study_design_choices + kwargs['study_design_choices'] = '
      {choices}
    '.format(choices=''.join( + f'
  • {choice}
    • ' for choice in primary_collection.study_design_choices + )) + + kwargs['disease_choices'] = '
      {choices}
    '.format(choices=''.join( + f'
  • {choice}
    • ' for choice in primary_collection.disease_choices + )) + + kwargs['data_type_choices'] = '
      {choices}
    '.format(choices=''.join( + f'
  • {choice}
    • ' for choice in primary_collection.data_type_choices + )) + + kwargs['grade_levels_choices'] = '
      {choices}
    '.format(choices=''.join( + f'
  • {choice}
    • ' for choice in primary_collection.grade_levels_choices + )) + + fields['collected_type_choices'] = json.dumps(primary_collection.collected_type_choices) + fields['status_choices'] = json.dumps(primary_collection.status_choices) + fields['volume_choices'] = json.dumps(primary_collection.volume_choices) + fields['issue_choices'] = json.dumps(primary_collection.issue_choices) + fields['program_area_choices'] = json.dumps(primary_collection.program_area_choices) + fields['school_type_choices'] = json.dumps(primary_collection.school_type_choices) + fields['study_design_choices'] = json.dumps(primary_collection.study_design_choices) + fields['data_type_choices'] = json.dumps(primary_collection.data_type_choices) + fields['disease_choices'] = json.dumps(primary_collection.disease_choices) + fields['grade_levels_choices'] = json.dumps(primary_collection.grade_levels_choices) kwargs['form'] = CollectionProviderForm(initial=fields) @@ -272,7 +223,7 @@ def form_valid(self, form): if self.object.primary_collection: for choices_name in ['collected_type', 'status', 'issue', 'volume', 'program_area', 'school_type', 'study_design', 'data_type', 'disease', 'grade_levels']: _process_collection_choices(self.object, choices_name, form) - self.object.primary_collection.save() + self.object.primary_collection.save() return super().form_valid(form) def form_invalid(self, form): diff --git a/admin/preprint_providers/forms.py b/admin/preprint_providers/forms.py index cb7a0f1b1e9..cee414b4815 100644 --- a/admin/preprint_providers/forms.py +++ b/admin/preprint_providers/forms.py @@ -8,7 +8,8 @@ Subject ) from admin.base.utils import (get_subject_rules, get_toplevel_subjects, - get_nodelicense_choices, get_defaultlicense_choices, validate_slug) + get_nodelicense_choices, get_defaultlicense_choices, validate_slug, + get_cedar_template_choices) class PreprintProviderForm(forms.ModelForm): @@ -42,10 +43,12 @@ def __init__(self, *args, **kwargs): toplevel_choices = get_toplevel_subjects() nodelicense_choices = get_nodelicense_choices() defaultlicense_choices = get_defaultlicense_choices() + cedar_template_choices = get_cedar_template_choices() super().__init__(*args, **kwargs) self.fields['toplevel_subjects'].choices = toplevel_choices self.fields['licenses_acceptable'].choices = nodelicense_choices self.fields['default_license'].choices = defaultlicense_choices + self.fields['required_metadata_template'].choices = cedar_template_choices def clean_subjects_acceptable(self, *args, **kwargs): subject_ids = [_f for _f in self.data['subjects_chosen'].split(', ') if _f] diff --git a/admin/registration_providers/forms.py b/admin/registration_providers/forms.py index dffda09e455..3bae6aa9a09 100644 --- a/admin/registration_providers/forms.py +++ b/admin/registration_providers/forms.py @@ -7,6 +7,7 @@ get_defaultlicense_choices, validate_slug, get_brand_choices, + get_cedar_template_choices, ) @@ -41,10 +42,12 @@ def __init__(self, *args, **kwargs): nodelicense_choices = get_nodelicense_choices() defaultlicense_choices = get_defaultlicense_choices() brand_choices = get_brand_choices() + cedar_template_choices = get_cedar_template_choices() super().__init__(*args, **kwargs) self.fields['licenses_acceptable'].choices = nodelicense_choices self.fields['default_license'].choices = defaultlicense_choices self.fields['brand'].choices = brand_choices + self.fields['required_metadata_template'].choices = cedar_template_choices if kwargs.get('initial', None) and kwargs.get('initial').get('_id', None): provider = RegistrationProvider.load(kwargs.get('initial').get('_id')) self.fields['default_schema'].choices = provider.schemas.filter(visible=True, active=True).values_list('id', 'name') diff --git a/api_tests/providers/collections/views/test_collection_provider_detail.py b/api_tests/providers/collections/views/test_collection_provider_detail.py index a44c35f7c09..558089761cb 100644 --- a/api_tests/providers/collections/views/test_collection_provider_detail.py +++ b/api_tests/providers/collections/views/test_collection_provider_detail.py @@ -85,8 +85,8 @@ def test_required_metadata_template_is_null_by_default(self, app, provider, prov assert res.json['data']['relationships']['required_metadata_template']['data'] is None def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): - from osf.models import AbstractProvider - AbstractProvider.objects.filter(pk=provider.pk).update(required_metadata_template=cedar_template) + provider.required_metadata_template = cedar_template + provider.save() res = app.get(provider_url) assert res.status_code == 200 diff --git a/api_tests/providers/preprints/views/test_preprint_provider_detail.py b/api_tests/providers/preprints/views/test_preprint_provider_detail.py index 575ffc52f88..0be8eea8f3c 100644 --- a/api_tests/providers/preprints/views/test_preprint_provider_detail.py +++ b/api_tests/providers/preprints/views/test_preprint_provider_detail.py @@ -246,8 +246,8 @@ def test_required_metadata_template_is_null_by_default(self, app, provider, prov assert res.json['data']['relationships']['required_metadata_template']['data'] is None def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): - from osf.models import AbstractProvider - AbstractProvider.objects.filter(pk=provider.pk).update(required_metadata_template=cedar_template) + provider.required_metadata_template = cedar_template + provider.save() res = app.get(provider_url) assert res.status_code == 200 diff --git a/api_tests/providers/registrations/views/test_registration_provider_detail.py b/api_tests/providers/registrations/views/test_registration_provider_detail.py index 16f453decb8..8924eafc184 100644 --- a/api_tests/providers/registrations/views/test_registration_provider_detail.py +++ b/api_tests/providers/registrations/views/test_registration_provider_detail.py @@ -88,8 +88,8 @@ def test_required_metadata_template_is_null_by_default(self, app, provider, prov assert res.json['data']['relationships']['required_metadata_template']['data'] is None def test_required_metadata_template_when_set(self, app, provider, provider_url, cedar_template): - from osf.models import AbstractProvider - AbstractProvider.objects.filter(pk=provider.pk).update(required_metadata_template=cedar_template) + provider.required_metadata_template = cedar_template + provider.save() res = app.get(provider_url) assert res.status_code == 200 From e26fac7f4d1810bbc9f512f7e7402387d1d49645 Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Wed, 8 Apr 2026 19:45:06 +0300 Subject: [PATCH 24/43] fix tests --- api/providers/serializers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/providers/serializers.py b/api/providers/serializers.py index 8f7b1ef3af2..04826a003bf 100644 --- a/api/providers/serializers.py +++ b/api/providers/serializers.py @@ -132,6 +132,11 @@ class Meta: related_view_kwargs={'brand_id': ''}, ) + required_metadata_template = RelationshipField( + related_view='cedar-metadata-templates:cedar-metadata-template-detail', + related_view_kwargs={'template_id': ''}, + read_only=True, + ) filterable_fields = frozenset([ 'allow_submissions', 'allow_commenting', @@ -199,6 +204,11 @@ class Meta: related_view_kwargs={'provider_id': '<_id>'}, ) + required_metadata_template = RelationshipField( + related_view='cedar-metadata-templates:cedar-metadata-template-detail', + related_view_kwargs={'template_id': ''}, + ) + links = LinksField({ 'self': 'get_absolute_url', 'external_url': 'get_external_url', @@ -269,6 +279,11 @@ class Meta: related_view_kwargs={'brand_id': ''}, ) + required_metadata_template = RelationshipField( + related_view='cedar-metadata-templates:cedar-metadata-template-detail', + related_view_kwargs={'template_id': ''}, + ) + def get_preprints_url(self, obj): return absolute_reverse( 'providers:preprint-providers:preprints-list', kwargs={ From 05a44c57bb6ce5bbbf751a700c519e8558815cd2 Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Mon, 13 Apr 2026 08:24:33 -0400 Subject: [PATCH 25/43] add field, change admin form --- admin/cedar/forms.py | 4 ++-- osf/models/cedar_metadata.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/admin/cedar/forms.py b/admin/cedar/forms.py index f430be3cb4d..e7e4c6f5e33 100644 --- a/admin/cedar/forms.py +++ b/admin/cedar/forms.py @@ -1,4 +1,4 @@ -from django.forms import ModelForm, CharField, JSONField +from django.forms import ModelForm, CharField, JSONField, BooleanField from osf.models import CedarMetadataTemplate @@ -21,4 +21,4 @@ class CedarMetadataTemplateForm(ModelForm): class Meta: model = CedarMetadataTemplate - fields = ['schema_name', 'cedar_id', 'template_version', 'template', 'active'] + fields = ['schema_name', 'cedar_id', 'template_version', 'template', 'active', 'should_index_for_search'] diff --git a/osf/models/cedar_metadata.py b/osf/models/cedar_metadata.py index 6938301765b..df8d4cda4d3 100644 --- a/osf/models/cedar_metadata.py +++ b/osf/models/cedar_metadata.py @@ -10,6 +10,7 @@ class CedarMetadataTemplate(ObjectIDMixin, BaseModel): template = DateTimeAwareJSONField(default=dict) active = models.BooleanField(default=True) template_version = models.PositiveIntegerField() + should_index_for_search = models.BooleanField(default=False) class Meta: unique_together = ('cedar_id', 'template_version') From 903b2ce88b424e480ecbf5baeeb45aaaed9c990a Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Mon, 13 Apr 2026 08:24:49 -0400 Subject: [PATCH 26/43] add migration --- ...metadatatemplate_should_index_for_search.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py diff --git a/osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py b/osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py new file mode 100644 index 00000000000..134d51c5d89 --- /dev/null +++ b/osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.26 on 2026-04-13 12:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0038_abstractprovider_required_metadata_template'), + ] + + operations = [ + migrations.AddField( + model_name='cedarmetadatatemplate', + name='should_index_for_search', + field=models.BooleanField(default=False), + ), + ] From 8c702a00c9fa9ef20fa78d3eb0948eb618594e76 Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Mon, 13 Apr 2026 08:29:59 -0400 Subject: [PATCH 27/43] remove unused field --- admin/cedar/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/cedar/forms.py b/admin/cedar/forms.py index e7e4c6f5e33..f2eb43c6a9b 100644 --- a/admin/cedar/forms.py +++ b/admin/cedar/forms.py @@ -1,4 +1,4 @@ -from django.forms import ModelForm, CharField, JSONField, BooleanField +from django.forms import ModelForm, CharField, JSONField from osf.models import CedarMetadataTemplate From 7bc97eff4a605e02f7fd7e41f45e227176558ced Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Mon, 13 Apr 2026 17:20:11 +0300 Subject: [PATCH 28/43] Update imports --- osf/models/provider.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/osf/models/provider.py b/osf/models/provider.py index 82f808934db..dd32a88ff59 100644 --- a/osf/models/provider.py +++ b/osf/models/provider.py @@ -1,5 +1,6 @@ import json import requests +from jsonschema import validate as jsonschema_validate, ValidationError as JsonSchemaValidationError from django.apps import apps from django.contrib.postgres import fields @@ -20,6 +21,7 @@ from .brand import Brand from .citation import CitationStyle from .licenses import NodeLicense +from .cedar_metadata import CedarMetadataRecord from .storage import ProviderAssetFile from .subject import Subject from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField @@ -258,9 +260,6 @@ def setup_share_source(self, provider_home_page): self.save() def validate_required_metadata(self, osf_obj): - from jsonschema import validate, ValidationError as JsonSchemaValidationError - from osf.models.cedar_metadata import CedarMetadataRecord - if not self.required_metadata_template: return @@ -277,7 +276,7 @@ def validate_required_metadata(self, osf_obj): ) try: - validate(record.metadata, self.required_metadata_template.template) + jsonschema_validate(record.metadata, self.required_metadata_template.template) except JsonSchemaValidationError as e: raise ValidationError(e.message) From e70c4185a33d759e261c2bc8b8911b8e1018811d Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Tue, 14 Apr 2026 17:54:37 +0300 Subject: [PATCH 29/43] Add switch to enforce required_metadata_template --- api/collections/serializers.py | 13 ++++ api_tests/collections/test_views.py | 94 ++++++++++++++++++++++++++--- osf/features.yaml | 5 ++ osf/models/provider.py | 17 ++++++ 4 files changed, 121 insertions(+), 8 deletions(-) diff --git a/api/collections/serializers.py b/api/collections/serializers.py index decf6499146..2f24a2eb56a 100644 --- a/api/collections/serializers.py +++ b/api/collections/serializers.py @@ -1,7 +1,10 @@ +import waffle + from django.db import IntegrityError from rest_framework import exceptions from rest_framework import serializers as ser +from osf import features from osf.models import AbstractNode, Node, Collection, Guid, Registration from osf.exceptions import ValidationError, NodeStateError from api.base.serializers import LinksField, RelationshipField, LinkedNodesRelationshipSerializer, LinkedRegistrationsRelationshipSerializer, LinkedPreprintsRelationshipSerializer @@ -426,6 +429,11 @@ def create(self, validated_data): raise exceptions.ValidationError('"creator" must be specified.') if not (creator.has_perm('write_collection', collection) or (hasattr(guid.referent, 'has_permission') and guid.referent.has_permission(creator, WRITE))): raise exceptions.PermissionDenied('Must have write permission on either collection or collected object to collect.') + if waffle.switch_is_active(features.COLLECTION_SUBMISSION_WITH_CEDAR) and collection.provider_id: + try: + collection.provider.validate_required_metadata(guid.referent) + except ValidationError as e: + raise InvalidModelValueError(e.message) try: obj = collection.collect_object(guid.referent, creator, **validated_data) except ValidationError as e: @@ -462,6 +470,11 @@ def create(self, validated_data): raise exceptions.ValidationError('"creator" must be specified.') if not (creator.has_perm('write_collection', collection) or (hasattr(guid.referent, 'has_permission') and guid.referent.has_permission(creator, WRITE))): raise exceptions.PermissionDenied('Must have write permission on either collection or collected object to collect.') + if waffle.switch_is_active(features.COLLECTION_SUBMISSION_WITH_CEDAR) and collection.provider_id: + try: + collection.provider.validate_required_metadata(guid.referent) + except ValidationError as e: + raise InvalidModelValueError(e.message) try: obj = collection.collect_object(guid.referent, creator, **validated_data) except ValidationError as e: diff --git a/api_tests/collections/test_views.py b/api_tests/collections/test_views.py index cc04bb729f8..7ae088ab5a7 100644 --- a/api_tests/collections/test_views.py +++ b/api_tests/collections/test_views.py @@ -1,14 +1,24 @@ -import pytest from urllib.parse import urlparse +import pytest from django.utils.timezone import now +from waffle.testutils import override_switch from api.base.settings.defaults import API_BASE from api.taxonomies.serializers import subjects_as_relationships_version -from api_tests.subjects.mixins import UpdateSubjectsMixin, SubjectsFilterMixin, SubjectsListMixin, SubjectsRelationshipMixin +from api_tests.share._utils import mock_update_share +from api_tests.subjects.mixins import UpdateSubjectsMixin, SubjectsFilterMixin, SubjectsListMixin, \ + SubjectsRelationshipMixin +from api_tests.utils import disconnected_from_listeners from framework.auth.core import Auth +from osf import features +from osf.models import Collection, VersionedGuidMixin +from osf.utils.permissions import ADMIN, WRITE, READ +from osf.utils.sanitize import strip_html from osf_tests.factories import ( + CedarMetadataTemplateFactory, CollectionFactory, + CollectionProviderFactory, NodeFactory, RegistrationFactory, PreprintFactory, @@ -16,15 +26,9 @@ AuthUserFactory, SubjectFactory, ) -from osf.models import Collection, VersionedGuidMixin -from osf.utils.sanitize import strip_html -from osf.utils.permissions import ADMIN, WRITE, READ from website.project.signals import contributor_removed -from api_tests.utils import disconnected_from_listeners -from api_tests.share._utils import mock_update_share from website.views import find_bookmark_collection - url_collection_list = f'/{API_BASE}collections/' @@ -4384,6 +4388,80 @@ def test_filters(self, app, collection_with_one_collection_submission, collectio assert len(res.json['data']) == 1 +@pytest.mark.django_db +class TestCollectionSubmissionWithCedarSwitch: + + @pytest.fixture() + def cedar_template(self): + return CedarMetadataTemplateFactory( + schema_name='Test Schema', + cedar_id='https://cedar.example.com/template/1', + template_version=1, + ) + + @pytest.fixture() + def provider(self, cedar_template): + provider = CollectionProviderFactory() + provider.required_metadata_template = cedar_template + provider.save() + return provider + + @pytest.fixture() + def collection(self, user_one, provider): + c = CollectionFactory(creator=user_one) + c.provider = provider + c.save() + return c + + @pytest.fixture() + def collection_no_provider(self, user_one): + return CollectionFactory(creator=user_one) + + @pytest.fixture() + def project(self, user_one): + return ProjectFactory(creator=user_one) + + @pytest.fixture() + def url(self, collection): + return f'/{API_BASE}collections/{collection._id}/collected_metadata/' + + @pytest.fixture() + def url_no_provider(self, collection_no_provider): + return f'/{API_BASE}collections/{collection_no_provider._id}/collected_metadata/' + + @pytest.fixture() + def payload(self): + def make_collection_payload(**attributes): + return { + 'data': { + 'type': 'collected-metadata', + 'attributes': attributes, + } + } + return make_collection_payload + + def test_switch_active_no_provider_submission_succeeds(self, app, user_one, project, url_no_provider, payload): + with mock_update_share(): + with override_switch(features.COLLECTION_SUBMISSION_WITH_CEDAR, active=True): + res = app.post_json_api( + url_no_provider, + payload(guid=project._id), + auth=user_one.auth, + ) + assert res.status_code == 201 + + def test_switch_active_missing_cedar_record_submission_fails(self, app, user_one, project, url, payload): + with override_switch(features.COLLECTION_SUBMISSION_WITH_CEDAR, active=True): + res = app.post_json_api( + url, + payload(guid=project._id), + auth=user_one.auth, + expect_errors=True, + ) + assert res.status_code == 400 + assert 'CEDAR metadata record' in res.json['errors'][0]['detail'] + + class TestCollectedMetaSubjectFiltering(SubjectsFilterMixin): @pytest.fixture() def project_one(self, user): diff --git a/osf/features.yaml b/osf/features.yaml index cce490a25a4..9f826966eed 100644 --- a/osf/features.yaml +++ b/osf/features.yaml @@ -113,3 +113,8 @@ switches: name: populate_notification_types note: This is used to enable auto population of notification types. active: false + + - flag_name: COLLECTION_SUBMISSION_WITH_CEDAR + name: collection_submission_with_cedar + note: When active, enforces that objects submitted to a collection have a CEDAR metadata record matching the provider's required_metadata_template. + active: false diff --git a/osf/models/provider.py b/osf/models/provider.py index f1dbdec65b3..6adb107c4ee 100644 --- a/osf/models/provider.py +++ b/osf/models/provider.py @@ -165,6 +165,23 @@ def update_or_create_from_json(cls, provider_data, user): related_name='required_by_providers', ) + def validate_required_metadata(self, obj): + """ + Raises ValidationError if obj does not have a CedarMetadataRecord for + this provider's required_metadata_template. + Does nothing when required_metadata_template is not set. + """ + if not self.required_metadata_template_id: + return + guid = obj.guids.first() + if guid is None or not guid.cedar_metadata_records.filter( + template_id=self.required_metadata_template_id + ).exists(): + raise ValidationError( + f'Submitted object must have a CEDAR metadata record for template ' + f'"{self.required_metadata_template.schema_name}" to be submitted to this collection.' + ) + def __repr__(self): return ('(name={self.name!r}, default_license={self.default_license!r}, ' 'allow_submissions={self.allow_submissions!r}) with id {self.id!r}').format(self=self) From 3634ac5b371adbb5467776dd26525fc03c13d866 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 23 Apr 2026 21:07:14 +0300 Subject: [PATCH 30/43] added cedar metadata record creation/deletion in share for collection submission --- api/share/utils.py | 41 +++++++++++++++++++++++++++++ osf/models/collection_submission.py | 16 +++++++++++ 2 files changed, 57 insertions(+) diff --git a/api/share/utils.py b/api/share/utils.py index 583b148cb9e..189d3989e2b 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -4,6 +4,7 @@ """ from http import HTTPStatus import logging +from rdflib import Graph from django.apps import apps from celery.utils.time import get_exponential_backoff_interval @@ -14,6 +15,7 @@ from framework.encryption import ensure_bytes from framework.sentry import log_exception from osf.external.gravy_valet.exceptions import GVException +from osf.metadata.rdfutils import OSF from osf.metadata.osf_gathering import ( OsfmapPartition, pls_get_magic_metadata_basket, @@ -64,6 +66,45 @@ def _enqueue_update_share(osfresource): enqueue_task(task__update_share.s(_osfguid_value)) +@celery_app.task() +def share_update_cedar_metadata_record(osf_obj_guid, cedar_record): + referent = osf_obj_guid.referent + + graph = Graph() + full_metadata = { + '@id': referent.get_semantic_iri(), + OSF.hasCedarRecord: cedar_record.metadata, + } + graph.parse(data=full_metadata, format='json-ld') + + serialized_data = graph.serialize(format='turtle') + requests.post( + shtrove_ingest_url(), + params={ + 'record_identifier': f"CedarMetadataRecord:{cedar_record.guid._id}:{cedar_record.template.cedar_id}", + 'is_supplementary': True, + }, + headers={ + 'Content-Type': 'text/turtle; charset=utf-8', + **_shtrove_auth_headers(referent), + }, + data=ensure_bytes(serialized_data), + ) + + +@celery_app.task() +def share_delete_cedar_metadata_record(osf_obj_guid, cedar_record): + referent = osf_obj_guid.referent + + requests.delete( + shtrove_ingest_url(), + params={ + 'record_identifier': f"CedarMetadataRecord:{cedar_record.guid._id}:{cedar_record.template.cedar_id}", + }, + headers=_shtrove_auth_headers(referent), + ) + + @celery_app.task( bind=True, acks_late=True, diff --git a/osf/models/collection_submission.py b/osf/models/collection_submission.py index f2de5ba6610..69b6bf2c69c 100644 --- a/osf/models/collection_submission.py +++ b/osf/models/collection_submission.py @@ -4,11 +4,16 @@ from django.utils import timezone from django.utils.functional import cached_property from framework import sentry +from framework.celery_tasks.handlers import enqueue_task from framework.exceptions import PermissionsError from website.settings import DOMAIN from .base import BaseModel from .mixins import TaxonomizableMixin +from api.share.utils import ( + share_update_cedar_metadata_record, + share_delete_cedar_metadata_record +) from osf.utils.permissions import ADMIN from website.util import api_v2_url from website.search.exceptions import SearchUnavailableError @@ -461,6 +466,17 @@ def update_search(self): # It will automatically determine if a referent is part of the collection update_share(self.guid.referent) + for cedar_record in self.guid.cedar_metadata_records.filter( + is_published=True, + template__should_index_for_search=True + ): + enqueue_task(share_update_cedar_metadata_record.s(self.guid, cedar_record)) + + for cedar_record in self.guid.cedar_metadata_records.filter( + models.Q(is_published=False) | models.Q(template__should_index_for_search=True) + ): + enqueue_task(share_delete_cedar_metadata_record.s(self.guid, cedar_record)) + try: update_collected_metadata(self.guid._id, collection_id=self.collection.id) except SearchUnavailableError as e: From f52c484f6405ff17bde58e0ee4aef8c4d0ef0292 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Fri, 24 Apr 2026 13:23:20 +0300 Subject: [PATCH 31/43] removed validate_required_metadata redefinition --- osf/models/provider.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/osf/models/provider.py b/osf/models/provider.py index 8984abf9f55..dd32a88ff59 100644 --- a/osf/models/provider.py +++ b/osf/models/provider.py @@ -167,23 +167,6 @@ def update_or_create_from_json(cls, provider_data, user): related_name='required_by_providers', ) - def validate_required_metadata(self, obj): - """ - Raises ValidationError if obj does not have a CedarMetadataRecord for - this provider's required_metadata_template. - Does nothing when required_metadata_template is not set. - """ - if not self.required_metadata_template_id: - return - guid = obj.guids.first() - if guid is None or not guid.cedar_metadata_records.filter( - template_id=self.required_metadata_template_id - ).exists(): - raise ValidationError( - f'Submitted object must have a CEDAR metadata record for template ' - f'"{self.required_metadata_template.schema_name}" to be submitted to this collection.' - ) - def __repr__(self): return ('(name={self.name!r}, default_license={self.default_license!r}, ' 'allow_submissions={self.allow_submissions!r}) with id {self.id!r}').format(self=self) From ec8fb9f265b40f946d7892c8ac158198e10a0ee1 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Fri, 24 Apr 2026 19:36:24 +0300 Subject: [PATCH 32/43] enqueue cedar record update in share --- api/share/utils.py | 20 ++++++++++++++++---- osf/models/collection_submission.py | 4 ++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index 189d3989e2b..dcea21ca01f 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -67,8 +67,14 @@ def _enqueue_update_share(osfresource): @celery_app.task() -def share_update_cedar_metadata_record(osf_obj_guid, cedar_record): - referent = osf_obj_guid.referent +def share_update_cedar_metadata_record(guid_id, cedar_record_pk): + from osf.models import CedarMetadataRecord, Guid + + guid = Guid.load(guid_id) + referent = guid.referent + cedar_record = CedarMetadataRecord.objects.filter(pk=cedar_record_pk).first() + if not cedar_record: + return graph = Graph() full_metadata = { @@ -93,8 +99,14 @@ def share_update_cedar_metadata_record(osf_obj_guid, cedar_record): @celery_app.task() -def share_delete_cedar_metadata_record(osf_obj_guid, cedar_record): - referent = osf_obj_guid.referent +def share_delete_cedar_metadata_record(guid_id, cedar_record_pk): + from osf.models import CedarMetadataRecord, Guid + + guid = Guid.load(guid_id) + referent = guid.referent + cedar_record = CedarMetadataRecord.objects.filter(pk=cedar_record_pk).first() + if not cedar_record: + return requests.delete( shtrove_ingest_url(), diff --git a/osf/models/collection_submission.py b/osf/models/collection_submission.py index 69b6bf2c69c..f11cb589f8a 100644 --- a/osf/models/collection_submission.py +++ b/osf/models/collection_submission.py @@ -470,12 +470,12 @@ def update_search(self): is_published=True, template__should_index_for_search=True ): - enqueue_task(share_update_cedar_metadata_record.s(self.guid, cedar_record)) + enqueue_task(share_update_cedar_metadata_record.s(self.guid._id, cedar_record.pk)) for cedar_record in self.guid.cedar_metadata_records.filter( models.Q(is_published=False) | models.Q(template__should_index_for_search=True) ): - enqueue_task(share_delete_cedar_metadata_record.s(self.guid, cedar_record)) + enqueue_task(share_delete_cedar_metadata_record.s(self.guid._id, cedar_record.pk)) try: update_collected_metadata(self.guid._id, collection_id=self.collection.id) From 06ef8687d1cb004c7eff54c7434e0e26006ef276 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Mon, 27 Apr 2026 19:08:35 +0300 Subject: [PATCH 33/43] added focus_iri to shtrove POST request --- api/share/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/share/utils.py b/api/share/utils.py index dcea21ca01f..932bbb305d7 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -77,8 +77,9 @@ def share_update_cedar_metadata_record(guid_id, cedar_record_pk): return graph = Graph() + iri = referent.get_semantic_iri() full_metadata = { - '@id': referent.get_semantic_iri(), + '@id': iri, OSF.hasCedarRecord: cedar_record.metadata, } graph.parse(data=full_metadata, format='json-ld') @@ -87,6 +88,7 @@ def share_update_cedar_metadata_record(guid_id, cedar_record_pk): requests.post( shtrove_ingest_url(), params={ + 'focus_iri': iri, 'record_identifier': f"CedarMetadataRecord:{cedar_record.guid._id}:{cedar_record.template.cedar_id}", 'is_supplementary': True, }, From 52c8fedec22dfd80bef1ef77908d7ca07d893c48 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 30 Apr 2026 16:58:06 +0300 Subject: [PATCH 34/43] separate cedar record identifier to avoid potential issues --- api/share/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index 932bbb305d7..a634cedf1be 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -21,6 +21,7 @@ pls_get_magic_metadata_basket, ) from osf.metadata.serializers import get_metadata_serializer +from osf.models import CedarMetadataRecord from website import settings @@ -89,7 +90,7 @@ def share_update_cedar_metadata_record(guid_id, cedar_record_pk): shtrove_ingest_url(), params={ 'focus_iri': iri, - 'record_identifier': f"CedarMetadataRecord:{cedar_record.guid._id}:{cedar_record.template.cedar_id}", + 'record_identifier': _shtrove_cedar_record_identifier(cedar_record), 'is_supplementary': True, }, headers={ @@ -113,7 +114,7 @@ def share_delete_cedar_metadata_record(guid_id, cedar_record_pk): requests.delete( shtrove_ingest_url(), params={ - 'record_identifier': f"CedarMetadataRecord:{cedar_record.guid._id}:{cedar_record.template.cedar_id}", + 'record_identifier': _shtrove_cedar_record_identifier(cedar_record), }, headers=_shtrove_auth_headers(referent), ) @@ -234,6 +235,10 @@ def _shtrove_record_identifier(osf_item, osfmap_partition: OsfmapPartition): ) +def _shtrove_cedar_record_identifier(cedar_record: CedarMetadataRecord) -> str: + return f'{cedar_record.guid._id}/CedarMetadataRecord:{cedar_record.template.cedar_id}' + + def _shtrove_auth_headers(osf_item): _nonfile_item = ( osf_item.target From 13ccfd34928962f49bceffcdc67fe647cdfc915e Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 30 Apr 2026 17:06:29 +0300 Subject: [PATCH 35/43] separate cedar record serialization --- api/share/utils.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index a634cedf1be..73cabe1761c 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -67,6 +67,18 @@ def _enqueue_update_share(osfresource): enqueue_task(task__update_share.s(_osfguid_value)) +def cedar_record_to_turtle(referent, cedar_record): + graph = Graph() + iri = referent.get_semantic_iri() + full_metadata = { + '@id': iri, + OSF.hasCedarRecord: cedar_record.metadata, + } + graph.parse(data=full_metadata, format='json-ld') + + return graph.serialize(format='turtle') + + @celery_app.task() def share_update_cedar_metadata_record(guid_id, cedar_record_pk): from osf.models import CedarMetadataRecord, Guid @@ -77,19 +89,11 @@ def share_update_cedar_metadata_record(guid_id, cedar_record_pk): if not cedar_record: return - graph = Graph() - iri = referent.get_semantic_iri() - full_metadata = { - '@id': iri, - OSF.hasCedarRecord: cedar_record.metadata, - } - graph.parse(data=full_metadata, format='json-ld') - - serialized_data = graph.serialize(format='turtle') + serialized_data = cedar_record_to_turtle(referent, cedar_record) requests.post( shtrove_ingest_url(), params={ - 'focus_iri': iri, + 'focus_iri': referent.get_semantic_iri(), 'record_identifier': _shtrove_cedar_record_identifier(cedar_record), 'is_supplementary': True, }, From b1630a119c79c1b264a7d3957e1434223d8bf575 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 30 Apr 2026 21:16:48 +0300 Subject: [PATCH 36/43] added retry, moved tasks to task__update_share --- api/share/utils.py | 93 ++++++++------ osf/models/collection_submission.py | 16 --- osf_tests/test_collection_submission.py | 155 +++++++++++++++++++++++- 3 files changed, 209 insertions(+), 55 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index 73cabe1761c..53de276c576 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -7,6 +7,7 @@ from rdflib import Graph from django.apps import apps +from django.db.models import Q from celery.utils.time import get_exponential_backoff_interval import requests @@ -21,7 +22,6 @@ pls_get_magic_metadata_basket, ) from osf.metadata.serializers import get_metadata_serializer -from osf.models import CedarMetadataRecord from website import settings @@ -67,6 +67,29 @@ def _enqueue_update_share(osfresource): enqueue_task(task__update_share.s(_osfguid_value)) +def retry_shtrove_request(self_celery_task, _response): + try: + _response.raise_for_status() + except Exception as e: + log_exception(e) + if _response.status_code == HTTPStatus.TOO_MANY_REQUESTS: + retry_after = _response.headers.get('Retry-After') + try: + countdown = int(retry_after) + except (TypeError, ValueError): + retries = getattr(self_celery_task.request, 'retries', 0) + countdown = get_exponential_backoff_interval( + factor=4, + retries=retries, + maximum=2 * 60, + full_jitter=True, + ) + raise self_celery_task.retry(exc=e, countdown=countdown) + + if HTTPStatus(_response.status_code).is_server_error: + raise self_celery_task.retry(exc=e) + + def cedar_record_to_turtle(referent, cedar_record): graph = Graph() iri = referent.get_semantic_iri() @@ -79,8 +102,8 @@ def cedar_record_to_turtle(referent, cedar_record): return graph.serialize(format='turtle') -@celery_app.task() -def share_update_cedar_metadata_record(guid_id, cedar_record_pk): +@celery_app.task(bind=True) +def share_update_cedar_metadata_record(self, guid_id, cedar_record_pk): from osf.models import CedarMetadataRecord, Guid guid = Guid.load(guid_id) @@ -90,7 +113,7 @@ def share_update_cedar_metadata_record(guid_id, cedar_record_pk): return serialized_data = cedar_record_to_turtle(referent, cedar_record) - requests.post( + response = requests.post( shtrove_ingest_url(), params={ 'focus_iri': referent.get_semantic_iri(), @@ -103,25 +126,26 @@ def share_update_cedar_metadata_record(guid_id, cedar_record_pk): }, data=ensure_bytes(serialized_data), ) + retry_shtrove_request(self, response) -@celery_app.task() -def share_delete_cedar_metadata_record(guid_id, cedar_record_pk): +@celery_app.task(bind=True) +def share_delete_cedar_metadata_record(self, guid_id, cedar_record_pk): from osf.models import CedarMetadataRecord, Guid - guid = Guid.load(guid_id) referent = guid.referent cedar_record = CedarMetadataRecord.objects.filter(pk=cedar_record_pk).first() if not cedar_record: return - requests.delete( + response = requests.delete( shtrove_ingest_url(), params={ 'record_identifier': _shtrove_cedar_record_identifier(cedar_record), }, headers=_shtrove_auth_headers(referent), ) + retry_shtrove_request(self, response) @celery_app.task( @@ -154,36 +178,29 @@ def task__update_share(self, guid: str, is_backfill=False, osfmap_partition_name log_exception(e) raise self.retry(exc=e) - try: - _response.raise_for_status() - except Exception as e: - log_exception(e) - if _response.status_code == HTTPStatus.TOO_MANY_REQUESTS: - retry_after = _response.headers.get('Retry-After') - try: - countdown = int(retry_after) - except (TypeError, ValueError): - retries = getattr(self.request, 'retries', 0) - countdown = get_exponential_backoff_interval( - factor=4, - retries=retries, - maximum=2 * 60, - full_jitter=True, - ) - raise self.retry(exc=e, countdown=countdown) + retry_shtrove_request(self, _response) + # success response + if _is_deletion: + return - if HTTPStatus(_response.status_code).is_server_error: - raise self.retry(exc=e) - else: # success response - if not _is_deletion: - # enqueue followup task for supplementary metadata - _next_partition = _next_osfmap_partition(_osfmap_partition) - if _next_partition is not None: - task__update_share.delay( - guid, - is_backfill=is_backfill, - osfmap_partition_name=_next_partition.name, - ) + # enqueue followup task for supplementary metadata + _next_partition = _next_osfmap_partition(_osfmap_partition) + if _next_partition is not None: + task__update_share.delay( + guid, + is_backfill=is_backfill, + osfmap_partition_name=_next_partition.name, + ) + for cedar_record in _osfid_instance.cedar_metadata_records.filter( + is_published=True, + template__should_index_for_search=True, + ): + enqueue_task(share_update_cedar_metadata_record.s(_osfid_instance._id, cedar_record.pk)) + + for cedar_record in _osfid_instance.cedar_metadata_records.filter( + Q(is_published=False) | Q(template__should_index_for_search=False), + ): + enqueue_task(share_delete_cedar_metadata_record.s(_osfid_instance._id, cedar_record.pk)) def pls_send_trove_record(osf_item, *, is_backfill: bool, osfmap_partition: OsfmapPartition): @@ -239,7 +256,7 @@ def _shtrove_record_identifier(osf_item, osfmap_partition: OsfmapPartition): ) -def _shtrove_cedar_record_identifier(cedar_record: CedarMetadataRecord) -> str: +def _shtrove_cedar_record_identifier(cedar_record) -> str: return f'{cedar_record.guid._id}/CedarMetadataRecord:{cedar_record.template.cedar_id}' diff --git a/osf/models/collection_submission.py b/osf/models/collection_submission.py index f11cb589f8a..f2de5ba6610 100644 --- a/osf/models/collection_submission.py +++ b/osf/models/collection_submission.py @@ -4,16 +4,11 @@ from django.utils import timezone from django.utils.functional import cached_property from framework import sentry -from framework.celery_tasks.handlers import enqueue_task from framework.exceptions import PermissionsError from website.settings import DOMAIN from .base import BaseModel from .mixins import TaxonomizableMixin -from api.share.utils import ( - share_update_cedar_metadata_record, - share_delete_cedar_metadata_record -) from osf.utils.permissions import ADMIN from website.util import api_v2_url from website.search.exceptions import SearchUnavailableError @@ -466,17 +461,6 @@ def update_search(self): # It will automatically determine if a referent is part of the collection update_share(self.guid.referent) - for cedar_record in self.guid.cedar_metadata_records.filter( - is_published=True, - template__should_index_for_search=True - ): - enqueue_task(share_update_cedar_metadata_record.s(self.guid._id, cedar_record.pk)) - - for cedar_record in self.guid.cedar_metadata_records.filter( - models.Q(is_published=False) | models.Q(template__should_index_for_search=True) - ): - enqueue_task(share_delete_cedar_metadata_record.s(self.guid._id, cedar_record.pk)) - try: update_collected_metadata(self.guid._id, collection_id=self.collection.id) except SearchUnavailableError as e: diff --git a/osf_tests/test_collection_submission.py b/osf_tests/test_collection_submission.py index 2b661f00873..d6ab07c004a 100644 --- a/osf_tests/test_collection_submission.py +++ b/osf_tests/test_collection_submission.py @@ -1,3 +1,4 @@ +from unittest import mock import pytest from osf_tests.factories import ( @@ -8,11 +9,12 @@ from osf_tests.factories import NodeFactory, CollectionFactory, CollectionProviderFactory -from osf.models import CollectionSubmission, NotificationTypeEnum +from osf.models import CollectionSubmission, NotificationTypeEnum, CedarMetadataRecord, CedarMetadataTemplate from osf.utils.workflows import CollectionSubmissionStates from framework.exceptions import PermissionsError from api_tests.utils import UserRoles from django.utils import timezone +from website import settings from tests.utils import capture_notifications @@ -147,6 +149,38 @@ def configure_test_auth(node, user_role, provider=None): return user +@pytest.fixture() +def unmoderated_collection_submission_public(node, unmoderated_collection): + unmoderated_collection.is_public = True + unmoderated_collection.save() + collection_submission = CollectionSubmission( + guid=node.guids.first(), + collection=unmoderated_collection, + creator=node.creator, + ) + with capture_notifications(): + collection_submission.save() + assert not collection_submission.is_moderated + return collection_submission + + +@pytest.fixture() +def cedar_template_json(): + return {'t_key_1': 't_value_1', 't_key_2': 't_value_2', 't_key_3': 't_value_3'} + + +@pytest.fixture() +def cedar_template(cedar_template_json): + return CedarMetadataTemplate.objects.create( + schema_name='cedar_test_schema_name', + cedar_id='cedar_test_id', + template_version=1, + template=cedar_template_json, + active=True, + should_index_for_search=True + ) + + @pytest.mark.django_db class TestModeratedCollectionSubmission: @@ -574,3 +608,122 @@ def test_cancel_succeeds(self, node, hybrid_moderated_collection_submission): with capture_notifications(): hybrid_moderated_collection_submission.cancel(user=user, comment='Test Comment') assert hybrid_moderated_collection_submission.state == CollectionSubmissionStates.IN_PROGRESS + + +@pytest.mark.django_db +@pytest.mark.enable_enqueue_task +@mock.patch.object(settings, 'SHARE_ENABLED', True) +@mock.patch.object(settings, 'USE_CELERY', False) # run tasks synchronously +class TestCollectionSubmissionWithCedarRecord: + + @mock.patch('api.share.utils.pls_send_trove_record') + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_unindexable_template_and_unpublished_record_calls_records_deletion( + self, + mock_delete, + mock_create, + mock_pls, + unmoderated_collection_submission_public, + cedar_template, + cedar_template_json + ): + cedar_template.should_index_for_search = False + cedar_template.save() + CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template_json, + is_published=False, + ) + obj = mock.Mock() + obj.status_code = 200 + mock_pls.return_value = obj + unmoderated_collection_submission_public.save() + + assert not mock_create.s.called + assert mock_delete.s.called + + @mock.patch('api.share.utils.pls_send_trove_record') + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_indexable_template_and_unpublished_record_calls_records_deletion( + self, + mock_delete, + mock_create, + mock_pls, + unmoderated_collection_submission_public, + cedar_template, + cedar_template_json + ): + cedar_template.should_index_for_search = True + cedar_template.save() + CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template_json, + is_published=False, + ) + obj = mock.Mock() + obj.status_code = 200 + mock_pls.return_value = obj + unmoderated_collection_submission_public.save() + + assert not mock_create.s.called + assert mock_delete.s.called + + @mock.patch('api.share.utils.pls_send_trove_record') + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_unindexable_template_and_published_record_calls_records_deletion( + self, + mock_delete, + mock_create, + mock_pls, + unmoderated_collection_submission_public, + cedar_template, + cedar_template_json + ): + cedar_template.should_index_for_search = False + cedar_template.save() + CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template_json, + is_published=True, + ) + obj = mock.Mock() + obj.status_code = 200 + mock_pls.return_value = obj + unmoderated_collection_submission_public.save() + + assert not mock_create.s.called + assert mock_delete.s.called + + @mock.patch('api.share.utils.pls_send_trove_record') + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_indexable_template_and_published_record_call_shtrove( + self, + mock_delete, + mock_create, + mock_pls, + unmoderated_collection_submission_public, + cedar_template, + cedar_template_json + ): + cedar_template.should_index_for_search = True + cedar_template.save() + CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template_json, + is_published=True, + ) + obj = mock.Mock() + obj.status_code = 200 + mock_pls.return_value = obj + unmoderated_collection_submission_public.save() + + assert mock_create.s.called + assert not mock_delete.s.called From 3561ed0a93c8182c2190cf3d845f8045ec721c53 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Fri, 1 May 2026 17:56:11 +0300 Subject: [PATCH 37/43] added more tests, make delete cedar record function working without cedar record existence --- api/share/utils.py | 38 +++-- osf_tests/test_collection_submission.py | 198 +++++++++++++++++++++++- 2 files changed, 216 insertions(+), 20 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index 53de276c576..91912da9942 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -103,10 +103,10 @@ def cedar_record_to_turtle(referent, cedar_record): @celery_app.task(bind=True) -def share_update_cedar_metadata_record(self, guid_id, cedar_record_pk): - from osf.models import CedarMetadataRecord, Guid +def share_update_cedar_metadata_record(self, referent_id, cedar_record_pk): + from osf.models import Guid, CedarMetadataRecord - guid = Guid.load(guid_id) + guid = Guid.load(referent_id) referent = guid.referent cedar_record = CedarMetadataRecord.objects.filter(pk=cedar_record_pk).first() if not cedar_record: @@ -117,7 +117,7 @@ def share_update_cedar_metadata_record(self, guid_id, cedar_record_pk): shtrove_ingest_url(), params={ 'focus_iri': referent.get_semantic_iri(), - 'record_identifier': _shtrove_cedar_record_identifier(cedar_record), + 'record_identifier': _shtrove_cedar_record_identifier(cedar_record._id, cedar_record.template.cedar_id), 'is_supplementary': True, }, headers={ @@ -130,18 +130,18 @@ def share_update_cedar_metadata_record(self, guid_id, cedar_record_pk): @celery_app.task(bind=True) -def share_delete_cedar_metadata_record(self, guid_id, cedar_record_pk): - from osf.models import CedarMetadataRecord, Guid - guid = Guid.load(guid_id) - referent = guid.referent - cedar_record = CedarMetadataRecord.objects.filter(pk=cedar_record_pk).first() - if not cedar_record: - return - +def share_delete_cedar_metadata_record( + self, + cedar_referent___id, + cedar_record___id, + cedar_template_cedar_id, +): + from osf.models import Guid + referent = Guid.load(cedar_referent___id).referent response = requests.delete( shtrove_ingest_url(), params={ - 'record_identifier': _shtrove_cedar_record_identifier(cedar_record), + 'record_identifier': _shtrove_cedar_record_identifier(cedar_record___id, cedar_template_cedar_id), }, headers=_shtrove_auth_headers(referent), ) @@ -200,7 +200,13 @@ def task__update_share(self, guid: str, is_backfill=False, osfmap_partition_name for cedar_record in _osfid_instance.cedar_metadata_records.filter( Q(is_published=False) | Q(template__should_index_for_search=False), ): - enqueue_task(share_delete_cedar_metadata_record.s(_osfid_instance._id, cedar_record.pk)) + enqueue_task( + share_delete_cedar_metadata_record.s( + cedar_record.guid._id, + cedar_record._id, + cedar_record.template.cedar_id, + ), + ) def pls_send_trove_record(osf_item, *, is_backfill: bool, osfmap_partition: OsfmapPartition): @@ -256,8 +262,8 @@ def _shtrove_record_identifier(osf_item, osfmap_partition: OsfmapPartition): ) -def _shtrove_cedar_record_identifier(cedar_record) -> str: - return f'{cedar_record.guid._id}/CedarMetadataRecord:{cedar_record.template.cedar_id}' +def _shtrove_cedar_record_identifier(cedar_record___id, template_cedar_id) -> str: + return f'{cedar_record___id}/CedarMetadataRecord:{template_cedar_id}' def _shtrove_auth_headers(osf_item): diff --git a/osf_tests/test_collection_submission.py b/osf_tests/test_collection_submission.py index d6ab07c004a..32a33256bb2 100644 --- a/osf_tests/test_collection_submission.py +++ b/osf_tests/test_collection_submission.py @@ -13,6 +13,7 @@ from osf.utils.workflows import CollectionSubmissionStates from framework.exceptions import PermissionsError from api_tests.utils import UserRoles +from api.share.utils import cedar_record_to_turtle, _shtrove_cedar_record_identifier from django.utils import timezone from website import settings @@ -630,7 +631,7 @@ def test_unindexable_template_and_unpublished_record_calls_records_deletion( ): cedar_template.should_index_for_search = False cedar_template.save() - CedarMetadataRecord.objects.create( + record = CedarMetadataRecord.objects.create( guid=unmoderated_collection_submission_public.guid, template=cedar_template, metadata=cedar_template_json, @@ -643,6 +644,11 @@ def test_unindexable_template_and_unpublished_record_calls_records_deletion( assert not mock_create.s.called assert mock_delete.s.called + mock_delete.s.assert_called_with( + record.guid._id, + record._id, + record.template.cedar_id + ) @mock.patch('api.share.utils.pls_send_trove_record') @mock.patch('api.share.utils.share_update_cedar_metadata_record') @@ -658,7 +664,7 @@ def test_indexable_template_and_unpublished_record_calls_records_deletion( ): cedar_template.should_index_for_search = True cedar_template.save() - CedarMetadataRecord.objects.create( + record = CedarMetadataRecord.objects.create( guid=unmoderated_collection_submission_public.guid, template=cedar_template, metadata=cedar_template_json, @@ -671,6 +677,11 @@ def test_indexable_template_and_unpublished_record_calls_records_deletion( assert not mock_create.s.called assert mock_delete.s.called + mock_delete.s.assert_called_with( + record.guid._id, + record._id, + record.template.cedar_id + ) @mock.patch('api.share.utils.pls_send_trove_record') @mock.patch('api.share.utils.share_update_cedar_metadata_record') @@ -686,7 +697,7 @@ def test_unindexable_template_and_published_record_calls_records_deletion( ): cedar_template.should_index_for_search = False cedar_template.save() - CedarMetadataRecord.objects.create( + record = CedarMetadataRecord.objects.create( guid=unmoderated_collection_submission_public.guid, template=cedar_template, metadata=cedar_template_json, @@ -699,6 +710,11 @@ def test_unindexable_template_and_published_record_calls_records_deletion( assert not mock_create.s.called assert mock_delete.s.called + mock_delete.s.assert_called_with( + record.guid._id, + record._id, + record.template.cedar_id + ) @mock.patch('api.share.utils.pls_send_trove_record') @mock.patch('api.share.utils.share_update_cedar_metadata_record') @@ -714,7 +730,7 @@ def test_indexable_template_and_published_record_call_shtrove( ): cedar_template.should_index_for_search = True cedar_template.save() - CedarMetadataRecord.objects.create( + record = CedarMetadataRecord.objects.create( guid=unmoderated_collection_submission_public.guid, template=cedar_template, metadata=cedar_template_json, @@ -726,4 +742,178 @@ def test_indexable_template_and_published_record_call_shtrove( unmoderated_collection_submission_public.save() assert mock_create.s.called + mock_create.s.assert_called_with(unmoderated_collection_submission_public.guid._id, record.pk) assert not mock_delete.s.called + + def test_share_update_cedar_metadata_record(self, unmoderated_collection_submission_public, cedar_template): + metadata = { + '@context': { + 'pav': 'http://purl.org/pav/', + 'url': 'http://schema.org/url', + 'xsd': 'http://www.w3.org/2001/XMLSchema#', + 'name': 'http://schema.org/name', + 'oslc': 'http://open-services.net/ns/core#', + 'rdfs': 'http://www.w3.org/2000/01/rdf-schema#', + 'skos': 'http://www.w3.org/2004/02/skos/core#', + 'author': 'http://schema.org/author', + 'funder': 'https://schema.metadatacenter.org/properties/c35f0660-2072-46a3-8e0d-532e40d94919', + 'schema': 'http://schema.org/', + 'license': 'http://schema.org/license', + 'citation': 'http://schema.org/citation', + 'keywords': 'http://schema.org/keywords', + 'identifier': 'http://schema.org/identifier', + 'rdfs:label': { + '@type': 'xsd:string' + }, + 'description': 'http://schema.org/description', + 'schema:name': { + '@type': 'xsd:string' + }, + 'pav:createdBy': { + '@type': '@id' + }, + 'pav:createdOn': { + '@type': 'xsd:dateTime' + }, + 'skos:notation': { + '@type': 'xsd:string' + }, + 'oslc:modifiedBy': { + '@type': '@id' + }, + 'pav:derivedFrom': { + '@type': '@id' + }, + 'schema:isBasedOn': { + '@type': '@id' + }, + 'variableMeasured': 'http://schema.org/variableMeasured', + 'pav:lastUpdatedOn': { + '@type': 'xsd:dateTime' + }, + 'schema:description': { + '@type': 'xsd:string' + }, + 'About this template': 'https://repo.metadatacenter.org/template-fields/bc66544c-e100-439e-9e80-9b35537368e5' + }, + 'name': { + '@value': 'name' + }, + 'description': { + '@value': 'description' + }, + 'variableMeasured': [ + { + '@value': 'variable' + } + ], + 'author': [ + { + '@value': None + } + ], + 'citation': { + '@value': None + }, + 'license': { + '@value': None + }, + 'funder': [ + { + '@value': '1111111' + } + ], + 'url': {}, + 'keywords': { + '@value': None + }, + 'identifier': {} + } + record = CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=metadata, + is_published=True, + ) + result = cedar_record_to_turtle(record.guid.referent, record) + vocab_url = '' + schema_url = '' + schema_metadata_url = '' + urls_to_find = { + vocab_url: None, + schema_url: None, + schema_metadata_url: None + } + for url in urls_to_find.keys(): + urls_to_find[url] = result[result.index(url) - 3] + + # fetch urls from result and assign ns prefixes based on order of appearance in result to make test resilient to changes in order of namespace declaration in turtle output + ns1 = list(filter(lambda url: urls_to_find[url] == '1', urls_to_find.keys()))[0] + ns2 = list(filter(lambda url: urls_to_find[url] == '2', urls_to_find.keys()))[0] + ns3 = list(filter(lambda url: urls_to_find[url] == '3', urls_to_find.keys()))[0] + vocab_n = urls_to_find[vocab_url] + schema_n = urls_to_find[schema_url] + schema_metadata_n = urls_to_find[schema_metadata_url] + # compose expected result dynamically based on ordering of prefixes + # however ns attributes are strictly attached to specific prefix + expected = ( + f'@prefix ns1: {ns1} .\n' + f'@prefix ns2: {ns2} .\n' + f'@prefix ns3: {ns3} .\n\n' + f' ns{vocab_n}:hasCedarRecord [ ns{schema_n}:description "description" ;\n' + f' ns{schema_n}:identifier [ ] ;\n' + f' ns{schema_n}:name "name" ;\n' + f' ns{schema_n}:url [ ] ;\n' + f' ns{schema_n}:variableMeasured "variable" ;\n' + f' ns{schema_metadata_n}:c35f0660-2072-46a3-8e0d-532e40d94919 "1111111" ] .\n\n' + ) + + assert result == expected + + def test_cedar_record_identifier_on_create(self, unmoderated_collection_submission_public, cedar_template): + cedar_template.should_index_for_search = True + cedar_template.save() + + with mock.patch('api.share.utils.pls_send_trove_record'): + with mock.patch('api.share.utils.share_update_cedar_metadata_record'): + with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): + to_create_record = CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template.template, + is_published=True, + ) + + with mock.patch('api.share.utils.pls_send_trove_record'): + with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): + with mock.patch('api.share.utils._shtrove_cedar_record_identifier') as mock_identifier: + unmoderated_collection_submission_public.save() + mock_identifier.assert_called_with( + to_create_record._id, + to_create_record.template.cedar_id + ) + assert ( + _shtrove_cedar_record_identifier(to_create_record._id, to_create_record.template.cedar_id) == + f'{to_create_record._id}/CedarMetadataRecord:{to_create_record.template.cedar_id}' + ) + + def test_cedar_record_identifier_on_delete(self, unmoderated_collection_submission_public, cedar_template): + with mock.patch('api.share.utils.pls_send_trove_record'): + with mock.patch('api.share.utils.share_update_cedar_metadata_record'): + with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): + to_delete_record = CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template.template, + is_published=False, + ) + + with mock.patch('api.share.utils.pls_send_trove_record'): + with mock.patch('api.share.utils.share_update_cedar_metadata_record'): + with mock.patch('api.share.utils._shtrove_cedar_record_identifier') as mock_identifier: + unmoderated_collection_submission_public.save() + mock_identifier.assert_called_with(to_delete_record._id, to_delete_record.template.cedar_id) + assert ( + _shtrove_cedar_record_identifier(to_delete_record._id, to_delete_record.template.cedar_id) == + f'{to_delete_record._id}/CedarMetadataRecord:{to_delete_record.template.cedar_id}' + ) From ff4876f603fe5fec1002044ca36523dfe1fce052 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Mon, 4 May 2026 17:57:14 +0300 Subject: [PATCH 38/43] fixed tests --- api/share/utils.py | 3 +- api_tests/share/test_share_preprint.py | 12 +- osf_tests/test_collection_submission.py | 181 +++++++++++++++++------- 3 files changed, 142 insertions(+), 54 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index 91912da9942..b8e919f6f20 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -86,8 +86,7 @@ def retry_shtrove_request(self_celery_task, _response): ) raise self_celery_task.retry(exc=e, countdown=countdown) - if HTTPStatus(_response.status_code).is_server_error: - raise self_celery_task.retry(exc=e) + raise self_celery_task.retry(exc=e) def cedar_record_to_turtle(referent, cedar_record): diff --git a/api_tests/share/test_share_preprint.py b/api_tests/share/test_share_preprint.py index 118abf3105b..95993042532 100644 --- a/api_tests/share/test_share_preprint.py +++ b/api_tests/share/test_share_preprint.py @@ -134,12 +134,20 @@ def test_call_async_update_on_500_failure(self, mock_share_responses, preprint, with expect_preprint_ingest_request(mock_share_responses, preprint, count=5): preprint.update_search() - def test_no_call_async_update_on_400_failure(self, mock_share_responses, preprint, auth): + @mock.patch('api.share.utils.task__update_share.delay') + def test_no_call_async_update_on_400_failure(self, share_delay, mock_share_responses, preprint, auth): with capture_notifications(): mock_share_responses.replace(responses.POST, shtrove_ingest_url(), status=400) preprint.set_published(True, auth=auth, save=True) with expect_preprint_ingest_request(mock_share_responses, preprint, count=1, error_response=True): - preprint.update_search() + try: + preprint.update_search() + except Exception as err: + share_delay.assert_not_called() + assert str(err).startswith("Retry in 180s: HTTPError('400 Client Error:") + assert len(mock_share_responses.calls) == 1 + else: + pytest.fail('Expected Retry(HTTPError) to be raised') def test_delete_from_share(self, mock_share_responses): preprint = PreprintFactory() diff --git a/osf_tests/test_collection_submission.py b/osf_tests/test_collection_submission.py index 32a33256bb2..54d282b7690 100644 --- a/osf_tests/test_collection_submission.py +++ b/osf_tests/test_collection_submission.py @@ -1,5 +1,9 @@ +from contextlib import suppress from unittest import mock import pytest +from requests import Response +from requests.exceptions import HTTPError + from osf_tests.factories import ( AuthUserFactory, @@ -829,12 +833,14 @@ def test_share_update_cedar_metadata_record(self, unmoderated_collection_submiss }, 'identifier': {} } - record = CedarMetadataRecord.objects.create( - guid=unmoderated_collection_submission_public.guid, - template=cedar_template, - metadata=metadata, - is_published=True, - ) + with mock.patch('api.share.utils.pls_send_trove_record'): + record = CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=metadata, + is_published=True, + ) + result = cedar_record_to_turtle(record.guid.referent, record) vocab_url = '' schema_url = '' @@ -870,50 +876,125 @@ def test_share_update_cedar_metadata_record(self, unmoderated_collection_submiss assert result == expected - def test_cedar_record_identifier_on_create(self, unmoderated_collection_submission_public, cedar_template): - cedar_template.should_index_for_search = True - cedar_template.save() + @mock.patch('api.share.utils.pls_send_trove_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_cedar_record_identifier_on_create(self, mock_delete, mock_pls, unmoderated_collection_submission_public): + template = CedarMetadataTemplate.objects.create(schema_name='http://google.com', cedar_id='http26', template_version=1) + template.should_index_for_search = True + template.save() + with mock.patch('api.share.utils.share_update_cedar_metadata_record'): + to_create_record = CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=template, + metadata=template.template, + is_published=True, + ) - with mock.patch('api.share.utils.pls_send_trove_record'): - with mock.patch('api.share.utils.share_update_cedar_metadata_record'): - with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): - to_create_record = CedarMetadataRecord.objects.create( - guid=unmoderated_collection_submission_public.guid, - template=cedar_template, - metadata=cedar_template.template, - is_published=True, - ) + with mock.patch('api.share.utils.requests.post'): + with mock.patch('api.share.utils._shtrove_cedar_record_identifier') as mock_identifier: + unmoderated_collection_submission_public.save() + mock_identifier.assert_called_with( + to_create_record._id, + to_create_record.template.cedar_id + ) + assert ( + _shtrove_cedar_record_identifier(to_create_record._id, to_create_record.template.cedar_id) == + f'{to_create_record._id}/CedarMetadataRecord:http26' + ) - with mock.patch('api.share.utils.pls_send_trove_record'): - with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): - with mock.patch('api.share.utils._shtrove_cedar_record_identifier') as mock_identifier: - unmoderated_collection_submission_public.save() - mock_identifier.assert_called_with( - to_create_record._id, - to_create_record.template.cedar_id - ) - assert ( - _shtrove_cedar_record_identifier(to_create_record._id, to_create_record.template.cedar_id) == - f'{to_create_record._id}/CedarMetadataRecord:{to_create_record.template.cedar_id}' - ) - - def test_cedar_record_identifier_on_delete(self, unmoderated_collection_submission_public, cedar_template): - with mock.patch('api.share.utils.pls_send_trove_record'): - with mock.patch('api.share.utils.share_update_cedar_metadata_record'): - with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): - to_delete_record = CedarMetadataRecord.objects.create( - guid=unmoderated_collection_submission_public.guid, - template=cedar_template, - metadata=cedar_template.template, - is_published=False, - ) + @mock.patch('api.share.utils.pls_send_trove_record') + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + def test_cedar_record_identifier_on_delete(self, mock_update, mock_pls, unmoderated_collection_submission_public): + template = CedarMetadataTemplate.objects.create(schema_name='http://google.com', cedar_id='http25', template_version=1) + with mock.patch('api.share.utils.share_delete_cedar_metadata_record'): + to_delete_record = CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=template, + metadata=template.template, + is_published=False, + ) - with mock.patch('api.share.utils.pls_send_trove_record'): - with mock.patch('api.share.utils.share_update_cedar_metadata_record'): - with mock.patch('api.share.utils._shtrove_cedar_record_identifier') as mock_identifier: - unmoderated_collection_submission_public.save() - mock_identifier.assert_called_with(to_delete_record._id, to_delete_record.template.cedar_id) - assert ( - _shtrove_cedar_record_identifier(to_delete_record._id, to_delete_record.template.cedar_id) == - f'{to_delete_record._id}/CedarMetadataRecord:{to_delete_record.template.cedar_id}' - ) + with mock.patch('api.share.utils.requests.delete'): + with mock.patch('api.share.utils._shtrove_cedar_record_identifier') as mock_identifier: + unmoderated_collection_submission_public.save() + mock_identifier.assert_called_with(to_delete_record._id, to_delete_record.template.cedar_id) + assert ( + _shtrove_cedar_record_identifier(to_delete_record._id, to_delete_record.template.cedar_id) == + f'{to_delete_record._id}/CedarMetadataRecord:http25' + ) + + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_cedar_record_create_retry( + self, + mock_delete, + mock_create, + unmoderated_collection_submission_public, + cedar_template, + cedar_template_json + ): + cedar_template.should_index_for_search = True + cedar_template.save() + with mock.patch('api.share.utils.pls_send_trove_record') as mock_pls: + mock_pls.return_value = Response() + mock_pls.return_value.status_code = 400 + with suppress(Exception): + CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template_json, + is_published=True, + ) + + def mock_raise_for_status(*args, **kwargs): + raise HTTPError('Retry error') + + mock_pls.return_value = Response() + mock_pls.return_value.status_code = 400 + mock_pls.return_value.raise_for_status = mock_raise_for_status + try: + unmoderated_collection_submission_public.save() + except Exception as err: + assert str(err) == "Retry in 180s: HTTPError('Retry error')" + assert not mock_create.s.called + assert not mock_delete.s.called + else: + pytest.fail('Expected Retry(HTTPError) to be raised') + + @mock.patch('api.share.utils.share_update_cedar_metadata_record') + @mock.patch('api.share.utils.share_delete_cedar_metadata_record') + def test_cedar_record_delete_retry( + self, + mock_delete, + mock_create, + unmoderated_collection_submission_public, + cedar_template, + cedar_template_json + ): + cedar_template.should_index_for_search = False + cedar_template.save() + with mock.patch('api.share.utils.pls_send_trove_record') as mock_pls: + mock_pls.return_value = Response() + mock_pls.return_value.status_code = 400 + with suppress(Exception): + CedarMetadataRecord.objects.create( + guid=unmoderated_collection_submission_public.guid, + template=cedar_template, + metadata=cedar_template_json, + is_published=True, + ) + + def mock_raise_for_status(*args, **kwargs): + raise HTTPError('Retry error') + + mock_pls.return_value = Response() + mock_pls.return_value.status_code = 400 + mock_pls.return_value.raise_for_status = mock_raise_for_status + try: + unmoderated_collection_submission_public.save() + except Exception as err: + assert str(err) == "Retry in 180s: HTTPError('Retry error')" + assert not mock_create.s.called + assert not mock_delete.s.called + else: + pytest.fail('Expected Retry(HTTPError) to be raised') From c6d35f4cbc11c172638b1695df01ea63d5ed1868 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 6 May 2026 16:19:59 +0300 Subject: [PATCH 39/43] simplified turtle test and task__update_share --- api/share/utils.py | 37 +++++++++------- osf_tests/metadata/_utils.py | 8 ++++ .../metadata/test_serialized_metadata.py | 12 +----- osf_tests/test_collection_submission.py | 43 +++++-------------- 4 files changed, 42 insertions(+), 58 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index b8e919f6f20..1ddc0e85ac9 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -147,6 +147,25 @@ def share_delete_cedar_metadata_record( retry_shtrove_request(self, response) +def _schedule_cedar_record_updates(guid_instance): + for cedar_record in guid_instance.cedar_metadata_records.filter( + is_published=True, + template__should_index_for_search=True, + ): + enqueue_task(share_update_cedar_metadata_record.s(guid_instance._id, cedar_record.pk)) + + for cedar_record in guid_instance.cedar_metadata_records.filter( + Q(is_published=False) | Q(template__should_index_for_search=False), + ): + enqueue_task( + share_delete_cedar_metadata_record.s( + cedar_record.guid._id, + cedar_record._id, + cedar_record.template.cedar_id, + ), + ) + + @celery_app.task( bind=True, acks_late=True, @@ -190,22 +209,8 @@ def task__update_share(self, guid: str, is_backfill=False, osfmap_partition_name is_backfill=is_backfill, osfmap_partition_name=_next_partition.name, ) - for cedar_record in _osfid_instance.cedar_metadata_records.filter( - is_published=True, - template__should_index_for_search=True, - ): - enqueue_task(share_update_cedar_metadata_record.s(_osfid_instance._id, cedar_record.pk)) - - for cedar_record in _osfid_instance.cedar_metadata_records.filter( - Q(is_published=False) | Q(template__should_index_for_search=False), - ): - enqueue_task( - share_delete_cedar_metadata_record.s( - cedar_record.guid._id, - cedar_record._id, - cedar_record.template.cedar_id, - ), - ) + else: + _schedule_cedar_record_updates(_osfid_instance) def pls_send_trove_record(osf_item, *, is_backfill: bool, osfmap_partition: OsfmapPartition): diff --git a/osf_tests/metadata/_utils.py b/osf_tests/metadata/_utils.py index fb23bdb16c5..c8c778a1454 100644 --- a/osf_tests/metadata/_utils.py +++ b/osf_tests/metadata/_utils.py @@ -23,6 +23,14 @@ def assert_graphs_equal(actual_rdflib_graph, expected_rdflib_graph, label=''): )) +def assert_equivalent_turtle(actual_turtle, expected_turtle, label): + _actual = rdflib.Graph() + _actual.parse(data=actual_turtle, format='turtle') + _expected = rdflib.Graph() + _expected.parse(data=expected_turtle, format='turtle') + assert_graphs_equal(_actual, _expected, label=label) + + def _get_graph_and_focuses(triples): _graph = rdflib.Graph() _focuses = set() diff --git a/osf_tests/metadata/test_serialized_metadata.py b/osf_tests/metadata/test_serialized_metadata.py index ce59b830ba1..d3b38b540df 100644 --- a/osf_tests/metadata/test_serialized_metadata.py +++ b/osf_tests/metadata/test_serialized_metadata.py @@ -3,7 +3,6 @@ from unittest import mock import pytest -import rdflib from osf import models as osfdb from osf.metadata.osf_gathering import OsfmapPartition @@ -14,7 +13,7 @@ from osf.models.licenses import NodeLicense from api_tests.utils import create_test_file from osf_tests import factories -from osf_tests.metadata._utils import assert_graphs_equal +from osf_tests.metadata._utils import assert_equivalent_turtle from tests.base import OsfTestCase @@ -416,18 +415,11 @@ def _assert_expected_file(self, filename, actual_metadata): if filename.endswith('.turtle'): # HACK: because the turtle serializer may output things in different order # TODO: stable turtle serializer (or another primitive rdf serialization) - self._assert_equivalent_turtle(actual_metadata, _expected_metadata, filename) + assert_equivalent_turtle(actual_metadata, _expected_metadata, filename) else: # note: ignore trailing spaces self.assertEqual(actual_metadata.rstrip(), _expected_metadata.rstrip()) - def _assert_equivalent_turtle(self, actual_turtle, expected_turtle, filename): - _actual = rdflib.Graph() - _actual.parse(data=actual_turtle, format='turtle') - _expected = rdflib.Graph() - _expected.parse(data=expected_turtle, format='turtle') - assert_graphs_equal(_actual, _expected, label=filename) - # def _write_expected_file(self, filename, expected_metadata): # '''for updating expected metadata files from current serializers # (be careful to check that changes are, in fact, expected) diff --git a/osf_tests/test_collection_submission.py b/osf_tests/test_collection_submission.py index 54d282b7690..d3db5ca4bc8 100644 --- a/osf_tests/test_collection_submission.py +++ b/osf_tests/test_collection_submission.py @@ -4,7 +4,6 @@ from requests import Response from requests.exceptions import HTTPError - from osf_tests.factories import ( AuthUserFactory, ) @@ -14,6 +13,7 @@ from osf_tests.factories import NodeFactory, CollectionFactory, CollectionProviderFactory from osf.models import CollectionSubmission, NotificationTypeEnum, CedarMetadataRecord, CedarMetadataTemplate +from osf_tests.metadata._utils import assert_equivalent_turtle from osf.utils.workflows import CollectionSubmissionStates from framework.exceptions import PermissionsError from api_tests.utils import UserRoles @@ -842,39 +842,18 @@ def test_share_update_cedar_metadata_record(self, unmoderated_collection_submiss ) result = cedar_record_to_turtle(record.guid.referent, record) - vocab_url = '' - schema_url = '' - schema_metadata_url = '' - urls_to_find = { - vocab_url: None, - schema_url: None, - schema_metadata_url: None - } - for url in urls_to_find.keys(): - urls_to_find[url] = result[result.index(url) - 3] - - # fetch urls from result and assign ns prefixes based on order of appearance in result to make test resilient to changes in order of namespace declaration in turtle output - ns1 = list(filter(lambda url: urls_to_find[url] == '1', urls_to_find.keys()))[0] - ns2 = list(filter(lambda url: urls_to_find[url] == '2', urls_to_find.keys()))[0] - ns3 = list(filter(lambda url: urls_to_find[url] == '3', urls_to_find.keys()))[0] - vocab_n = urls_to_find[vocab_url] - schema_n = urls_to_find[schema_url] - schema_metadata_n = urls_to_find[schema_metadata_url] - # compose expected result dynamically based on ordering of prefixes - # however ns attributes are strictly attached to specific prefix expected = ( - f'@prefix ns1: {ns1} .\n' - f'@prefix ns2: {ns2} .\n' - f'@prefix ns3: {ns3} .\n\n' - f' ns{vocab_n}:hasCedarRecord [ ns{schema_n}:description "description" ;\n' - f' ns{schema_n}:identifier [ ] ;\n' - f' ns{schema_n}:name "name" ;\n' - f' ns{schema_n}:url [ ] ;\n' - f' ns{schema_n}:variableMeasured "variable" ;\n' - f' ns{schema_metadata_n}:c35f0660-2072-46a3-8e0d-532e40d94919 "1111111" ] .\n\n' + f'@prefix ns1: .\n' + f'@prefix ns2: .\n' + f'@prefix ns3: .\n\n' + f' ns1:hasCedarRecord [ ns2:description "description" ;\n' + f' ns2:identifier [ ] ;\n' + f' ns2:name "name" ;\n' + f' ns2:url [ ] ;\n' + f' ns2:variableMeasured "variable" ;\n' + f' ns3:c35f0660-2072-46a3-8e0d-532e40d94919 "1111111" ] .\n\n' ) - - assert result == expected + assert_equivalent_turtle(result, expected, 'test_share_update_cedar_metadata_record') @mock.patch('api.share.utils.pls_send_trove_record') @mock.patch('api.share.utils.share_delete_cedar_metadata_record') From db2313213257dcc0b44685c4fdc6427b29e5fe73 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Thu, 7 May 2026 17:56:57 +0300 Subject: [PATCH 40/43] use delay instead of signature for cedar tasks --- api/share/utils.py | 13 ++++------ osf_tests/test_collection_submission.py | 32 ++++++++++++------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/api/share/utils.py b/api/share/utils.py index 1ddc0e85ac9..5a0492eca8b 100644 --- a/api/share/utils.py +++ b/api/share/utils.py @@ -152,17 +152,14 @@ def _schedule_cedar_record_updates(guid_instance): is_published=True, template__should_index_for_search=True, ): - enqueue_task(share_update_cedar_metadata_record.s(guid_instance._id, cedar_record.pk)) - + share_update_cedar_metadata_record.delay(guid_instance._id, cedar_record.pk) for cedar_record in guid_instance.cedar_metadata_records.filter( Q(is_published=False) | Q(template__should_index_for_search=False), ): - enqueue_task( - share_delete_cedar_metadata_record.s( - cedar_record.guid._id, - cedar_record._id, - cedar_record.template.cedar_id, - ), + share_delete_cedar_metadata_record.delay( + cedar_record.guid._id, + cedar_record._id, + cedar_record.template.cedar_id, ) diff --git a/osf_tests/test_collection_submission.py b/osf_tests/test_collection_submission.py index d3db5ca4bc8..39bbce2e8dc 100644 --- a/osf_tests/test_collection_submission.py +++ b/osf_tests/test_collection_submission.py @@ -646,9 +646,9 @@ def test_unindexable_template_and_unpublished_record_calls_records_deletion( mock_pls.return_value = obj unmoderated_collection_submission_public.save() - assert not mock_create.s.called - assert mock_delete.s.called - mock_delete.s.assert_called_with( + assert not mock_create.delay.called + assert mock_delete.delay.called + mock_delete.delay.assert_called_with( record.guid._id, record._id, record.template.cedar_id @@ -679,9 +679,9 @@ def test_indexable_template_and_unpublished_record_calls_records_deletion( mock_pls.return_value = obj unmoderated_collection_submission_public.save() - assert not mock_create.s.called - assert mock_delete.s.called - mock_delete.s.assert_called_with( + assert not mock_create.delay.called + assert mock_delete.delay.called + mock_delete.delay.assert_called_with( record.guid._id, record._id, record.template.cedar_id @@ -712,9 +712,9 @@ def test_unindexable_template_and_published_record_calls_records_deletion( mock_pls.return_value = obj unmoderated_collection_submission_public.save() - assert not mock_create.s.called - assert mock_delete.s.called - mock_delete.s.assert_called_with( + assert not mock_create.delay.called + assert mock_delete.delay.called + mock_delete.delay.assert_called_with( record.guid._id, record._id, record.template.cedar_id @@ -745,9 +745,9 @@ def test_indexable_template_and_published_record_call_shtrove( mock_pls.return_value = obj unmoderated_collection_submission_public.save() - assert mock_create.s.called - mock_create.s.assert_called_with(unmoderated_collection_submission_public.guid._id, record.pk) - assert not mock_delete.s.called + assert mock_create.delay.called + mock_create.delay.assert_called_with(unmoderated_collection_submission_public.guid._id, record.pk) + assert not mock_delete.delay.called def test_share_update_cedar_metadata_record(self, unmoderated_collection_submission_public, cedar_template): metadata = { @@ -935,8 +935,8 @@ def mock_raise_for_status(*args, **kwargs): unmoderated_collection_submission_public.save() except Exception as err: assert str(err) == "Retry in 180s: HTTPError('Retry error')" - assert not mock_create.s.called - assert not mock_delete.s.called + assert not mock_create.delay.called + assert not mock_delete.delay.called else: pytest.fail('Expected Retry(HTTPError) to be raised') @@ -973,7 +973,7 @@ def mock_raise_for_status(*args, **kwargs): unmoderated_collection_submission_public.save() except Exception as err: assert str(err) == "Retry in 180s: HTTPError('Retry error')" - assert not mock_create.s.called - assert not mock_delete.s.called + assert not mock_create.delay.called + assert not mock_delete.delay.called else: pytest.fail('Expected Retry(HTTPError) to be raised') From 3056d1480216bc33bde71dad99ad93d7c386d2d9 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Tue, 12 May 2026 18:25:59 +0300 Subject: [PATCH 41/43] fixed migrations conflicts + share conflicts --- ...e.py => 0040_abstractprovider_required_metadata_template.py} | 2 +- ...py => 0041_cedarmetadatatemplate_should_index_for_search.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename osf/migrations/{0038_abstractprovider_required_metadata_template.py => 0040_abstractprovider_required_metadata_template.py} (91%) rename osf/migrations/{0039_cedarmetadatatemplate_should_index_for_search.py => 0041_cedarmetadatatemplate_should_index_for_search.py} (85%) diff --git a/osf/migrations/0038_abstractprovider_required_metadata_template.py b/osf/migrations/0040_abstractprovider_required_metadata_template.py similarity index 91% rename from osf/migrations/0038_abstractprovider_required_metadata_template.py rename to osf/migrations/0040_abstractprovider_required_metadata_template.py index f641e8741f3..d0e76c118e2 100644 --- a/osf/migrations/0038_abstractprovider_required_metadata_template.py +++ b/osf/migrations/0040_abstractprovider_required_metadata_template.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('osf', '0037_notification_refactor_post_release'), + ('osf', '0039_merge_20260427_1359'), ] operations = [ diff --git a/osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py b/osf/migrations/0041_cedarmetadatatemplate_should_index_for_search.py similarity index 85% rename from osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py rename to osf/migrations/0041_cedarmetadatatemplate_should_index_for_search.py index 134d51c5d89..8a4d5759401 100644 --- a/osf/migrations/0039_cedarmetadatatemplate_should_index_for_search.py +++ b/osf/migrations/0041_cedarmetadatatemplate_should_index_for_search.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('osf', '0038_abstractprovider_required_metadata_template'), + ('osf', '0040_abstractprovider_required_metadata_template'), ] operations = [ From 977d0b1b8d27168b7f45c761bb0dbbdee59a7df3 Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Wed, 13 May 2026 18:28:06 +0300 Subject: [PATCH 42/43] removed missed test case --- osf_tests/test_elastic_search.py | 85 -------------------------------- 1 file changed, 85 deletions(-) diff --git a/osf_tests/test_elastic_search.py b/osf_tests/test_elastic_search.py index 72866cfc031..e21ed1f44b2 100644 --- a/osf_tests/test_elastic_search.py +++ b/osf_tests/test_elastic_search.py @@ -1015,91 +1015,6 @@ def test_tag_aggregation(self): assert doc['key'] in tags -@pytest.mark.enable_search -@pytest.mark.enable_enqueue_task -class TestAddContributor(OsfTestCase): - # Tests of the search.search_contributor method - - def setUp(self): - self.name1 = 'Roger1 Taylor1' - self.name2 = 'John2 Deacon2' - self.name3 = 'j\xc3\xb3ebert3 Smith3' - self.name4 = 'B\xc3\xb3bbert4 Jones4' - - with run_celery_tasks(): - super().setUp() - self.user = factories.UserFactory(fullname=self.name1) - self.user3 = factories.UserFactory(fullname=self.name3) - - def test_unreg_users_dont_show_in_search(self): - unreg = factories.UnregUserFactory() - contribs = search.search_contributor(unreg.fullname) - assert len(contribs['users']) == 0 - - def test_unreg_users_do_show_on_projects(self): - with run_celery_tasks(): - unreg = factories.UnregUserFactory(fullname='Robert Paulson') - self.project = factories.ProjectFactory( - title='Glamour Rock', - creator=unreg, - is_public=True, - ) - results = query(unreg.fullname)['results'] - assert len(results) == 1 - - def test_search_fullname(self): - # Searching for full name yields exactly one result. - contribs = search.search_contributor(self.name1) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name2) - assert len(contribs['users']) == 0 - - def test_search_firstname(self): - # Searching for first name yields exactly one result. - contribs = search.search_contributor(self.name1.split(' ')[0]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name2.split(' ')[0]) - assert len(contribs['users']) == 0 - - def test_search_partial(self): - # Searching for part of first name yields exactly one - # result. - contribs = search.search_contributor(self.name1.split(' ')[0][:-1]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name2.split(' ')[0][:-1]) - assert len(contribs['users']) == 0 - - def test_search_fullname_special_character(self): - # Searching for a fullname with a special character yields - # exactly one result. - contribs = search.search_contributor(self.name3) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name4) - assert len(contribs['users']) == 0 - - def test_search_firstname_special_character(self): - # Searching for a first name with a special character yields - # exactly one result. - contribs = search.search_contributor(self.name3.split(' ')[0]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name4.split(' ')[0]) - assert len(contribs['users']) == 0 - - def test_search_partial_special_character(self): - # Searching for a partial name with a special character yields - # exactly one result. - contribs = search.search_contributor(self.name3.split(' ')[0][:-1]) - assert len(contribs['users']) == 1 - - contribs = search.search_contributor(self.name4.split(' ')[0][:-1]) - assert len(contribs['users']) == 0 - - @pytest.mark.enable_search @pytest.mark.enable_enqueue_task class TestProjectSearchResults(OsfTestCase): From 9ed5f23b0101f301ff78e15405ee499d0178512b Mon Sep 17 00:00:00 2001 From: Ihor Sokhan Date: Mon, 6 Jul 2026 17:42:41 +0300 Subject: [PATCH 43/43] removed duplicate method --- osf/models/provider.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/osf/models/provider.py b/osf/models/provider.py index 4a831e417dc..dd32a88ff59 100644 --- a/osf/models/provider.py +++ b/osf/models/provider.py @@ -208,24 +208,6 @@ def top_level_subjects(self): def readable_type(self): raise NotImplementedError - def validate_required_metadata(self, obj): - """ - Raises ValidationError if obj does not have a published CedarMetadataRecord for - this provider's required_metadata_template. - Does nothing when required_metadata_template is not set. - """ - if not self.required_metadata_template_id: - return - guid = obj.guids.first() - if guid is None or not guid.cedar_metadata_records.filter( - template_id=self.required_metadata_template_id, - is_published=True, - ).exists(): - raise ValidationError( - f'Submitted object must have a published CEDAR metadata record for template ' - f'"{self.required_metadata_template.schema_name}" to be submitted to this collection.' - ) - def get_asset_url(self, name): """ Helper that returns an associated ProviderAssetFile's url, or None