add task__reindex_resource_into_share to reindex into share guids with has_been_indexed None or False and public referents ...

Author: mkovalua

admin/share_reindex/views.py

@@ -3,11 +3,9 @@
 from django.shortcuts import redirect
 from django.views.generic import ListView, View
 from osf.models import Guid
-from django.db.models import F, Q
-from django.contrib.contenttypes.models import ContentType
-from osf.models import Registration, Preprint, Node, OSFUser
+from django.db.models import F
 from urllib.parse import urlencode
-
+from api.share.utils import get_not_indexed_guids_for_resource, task__reindex_resource_into_share
 
 class FailedShareIndexedGuidList(PermissionRequiredMixin, ListView):
     paginate_by = 25
@@ -18,28 +16,10 @@ class FailedShareIndexedGuidList(PermissionRequiredMixin, ListView):
 
     def get_queryset(self):
         resource_type = self.request.GET.get('type', 'projects')
-        resource_mapper = {
-            'projects': (Node, Q(is_public=True)),
-            'preprints': (Preprint, Q(is_public=True)),
-            'registries': (Registration, Q(is_public=True)),
-            'users': (OSFUser, Q(is_active=True))
-        }
-
-        resource_model, query = resource_mapper.get(resource_type)
-
-        node_type = ContentType.objects.get_for_model(resource_model)
-        public_node_ids = resource_model.objects.filter(query).values_list('id', flat=True)
-        # import pydevd_pycharm
-        # pydevd_pycharm.settrace('host.docker.internal', port=1234, stdout_to_server=True, stderr_to_server=True)
-        return Guid.objects.filter(
-            Q(has_been_indexed=False) | Q(has_been_indexed=None),
-            content_type=node_type,
-            object_id__in=public_node_ids
-        ).annotate(custom_id=F('_id'))
+        # use custom_id because _id fails to render in django template
+        return get_not_indexed_guids_for_resource(resource_type).annotate(custom_id=F('_id'))
 
     def get_context_data(self, **kwargs):
-        # import pydevd_pycharm
-        # pydevd_pycharm.settrace('host.docker.internal', port=1234, stdout_to_server=True, stderr_to_server=True)
         query_set = kwargs.pop('object_list', self.object_list)
         page_size = self.get_paginate_by(query_set)
         paginator, page, query_set, is_paginated = self.paginate_queryset(query_set, page_size)
@@ -58,8 +38,6 @@ def get_context_data(self, **kwargs):
         kwargs.setdefault('resource_guid_reindex', resource_type_guid_reindex.get(resource_type))
         status_msg = f'Reindex of {resource_type} started, please check in several minutes.' if self.request.GET.get('status') == 'indexing' else ''
         kwargs.setdefault('share_reindex_message', status_msg)
-        # import pydevd_pycharm
-        # pydevd_pycharm.settrace('host.docker.internal', port=1234, stdout_to_server=True, stderr_to_server=True)
         return super().get_context_data(**kwargs)
 
 
@@ -68,10 +46,9 @@ class FailedShareIndexedGuidReindex(PermissionRequiredMixin, View):
     raise_exception = True
 
     def post(self, request, *args, **kwargs):
-        # import pydevd_pycharm
-        # pydevd_pycharm.settrace('host.docker.internal', port=1234, stdout_to_server=True, stderr_to_server=True)
-        # 1. Get the guid from the URL string
         resource_type = self.kwargs.get('resource_type')
+        # reindex 100_000 guids in background task for specific resource_type and resource is public
+        task__reindex_resource_into_share.delay(resource_type, 100_000)
         base_url = reverse('share_reindex:list')
         query_string = urlencode({'type': resource_type, 'status': 'indexing'})
         return redirect(f"{base_url}?{query_string}")

api/share/utils.py

@@ -6,6 +6,9 @@
 import logging
 
 from django.apps import apps
+from django.db.models import Q
+from django.contrib.contenttypes.models import ContentType
+from website.settings import CeleryConfig
 from celery.utils.time import get_exponential_backoff_interval
 import requests
 
@@ -128,6 +131,32 @@ def task__update_share(self, guid: str, is_backfill=False, osfmap_partition_name
                 )
 
 
+@celery_app.task
+def task__reindex_resource_into_share(resource_type: str, limit: int):
+    guids = get_not_indexed_guids_for_resource(resource_type).values_list('_id', flat=True)[:limit].iterator()
+    for guid in guids:
+        task__update_share.apply_async(
+            kwargs={'guid': guid, 'is_backfill': True},
+            queue=CeleryConfig.task_low_queue,
+        )
+
+def get_not_indexed_guids_for_resource(resource_type: str):
+    from osf.models import Guid, Registration, Preprint, Node, OSFUser
+    resource_mapper = {
+        'projects': (Node, Q(is_public=True) & Q(deleted__isnull=True)),
+        'preprints': (Preprint, Q(is_public=True) & Q(is_published=True) & Q(deleted__isnull=True)),
+        'registries': (Registration, Q(is_public=True) & Q(deleted__isnull=True)),
+        'users': (OSFUser, Q(is_active=True) & Q(deleted__isnull=True)),
+    }
+    resource_model, query = resource_mapper.get(resource_type, 'projects')
+    node_type = ContentType.objects.get_for_model(resource_model)
+    public_node_ids = resource_model.objects.filter(query).values_list('id', flat=True)
+    return Guid.objects.filter(
+        Q(has_been_indexed=False) | Q(has_been_indexed=None),
+        content_type=node_type,
+        object_id__in=public_node_ids,
+    )
+
 def pls_send_trove_record(osf_item, *, is_backfill: bool, osfmap_partition: OsfmapPartition):
     try:
         _iri = osf_item.get_semantic_iri()