OpenConceptLab/ocl_issues#2388 | encoder_model param for reranking

core/concepts/views.py

@@ -18,6 +18,7 @@
 from core.bundles.models import Bundle
 from core.bundles.serializers import BundleSerializer
 from core.collections.documents import CollectionDocument
+from core.common import ERRBIT_LOGGER
 from core.common.constants import (
     HEAD, INCLUDE_INVERSE_MAPPINGS_PARAM, INCLUDE_RETIRED_PARAM, ACCESS_TYPE_NONE, LIMIT_PARAM, LIST_DEFAULT_LIMIT)
 from core.common.exceptions import Http400, Http403, Http409
@@ -994,7 +995,7 @@ class RerankConceptsListView(BaseAPIView):
     serializer_class = ConceptListSerializer
     permission_classes = (IsAuthenticated,)
 
-    def post(self, request, **kwargs):  # pylint: disable=unused-argument
+    def post(self, request, **kwargs):  # pylint: disable=unused-argument,too-many-return-statements
         user = self.request.user
         if user.is_mapper_waitlisted or not user.is_mapper_approved:
             return Response(
@@ -1006,6 +1007,12 @@ def post(self, request, **kwargs):  # pylint: disable=unused-argument
         name_key = self.request.data.get('name_key', None) or 'display_name'
         text = self.request.data.get('q', None)
         score_key = self.request.data.get('score_key', None)
+        encoder_model = self.request.data.get('encoder_model', None)
+        if encoder_model and encoder_model != settings.ENCODER_MODEL_NAME and not user.is_core_group:
+            return Response(
+                {'detail': 'You do not have permission to request for custom encoder models.'},
+                status=status.HTTP_403_FORBIDDEN
+            )
 
         if not isinstance(rows, list) or not rows:
             return Response(
@@ -1017,7 +1024,14 @@ def post(self, request, **kwargs):  # pylint: disable=unused-argument
                 {'detail': 'Missing "q" in request body.'},
                 status=status.HTTP_400_BAD_REQUEST
             )
-
-        return Response(
-            Reranker().rerank(hits=rows, name_key=name_key, txt=text, score_key=score_key, order_results=True)
-        )
+        try:
+            reranker = Reranker(model_name=encoder_model)
+            results = reranker.rerank(hits=rows, name_key=name_key, txt=text, score_key=score_key, order_results=True)
+            return Response(results)
+        except (ValueError, RuntimeError, OSError) as e:
+            return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST)
+        except Exception as e:
+            ERRBIT_LOGGER.log(e)
+            return Response(
+                {'detail': 'An error occurred while processing the rerank request.'},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR)

core/map_projects/migrations/0031_mapproject_encoder_model.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.5 on 2026-04-06 05:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('map_projects', '0030_auto_20260221_0250'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='mapproject',
+            name='encoder_model',
+            field=models.TextField(blank=True, default='BAAI/bge-reranker-v2-m3', null=True),
+        ),
+    ]

core/map_projects/models.py

@@ -1,6 +1,7 @@
 import json
 from collections import Counter
 
+from django.conf import settings
 from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.db import models, IntegrityError
@@ -38,6 +39,7 @@ class MapProject(BaseModel):
     candidates = models.JSONField(default=dict, null=True, blank=True)
     lookup_config = models.JSONField(default=dict, null=True, blank=True)
     analysis = models.JSONField(default=dict, null=True, blank=True)
+    encoder_model = models.TextField(null=True, blank=True, default=settings.ENCODER_MODEL_NAME)
 
     class Meta:
         db_table = 'map_projects'
@@ -193,14 +195,24 @@ def soft_delete(self):
 
     def clean(self):
         self.clean_filters()
-        if not self.include_retired:
-            self.include_retired = False
+        self.clean_include_retired()
+        self.clean_encoder_model()
+        self.clean_matches()
+
+    def clean_matches(self):
         if self.matches:
             try:
                 self.matches = json.loads(self.matches)
             except (json.JSONDecodeError, TypeError):
                 pass
 
+    def clean_include_retired(self):
+        if not self.include_retired:
+            self.include_retired = False
+
+    def clean_encoder_model(self):
+        self.encoder_model = self.encoder_model.strip() if self.encoder_model else settings.ENCODER_MODEL_NAME
+
     def clean_filters(self):
         if not self.filters:
             self.filters = {}

core/map_projects/serializers.py

@@ -22,7 +22,7 @@ class Meta:
             'created_by', 'updated_by', 'created_at', 'updated_at', 'url', 'is_active',
             'public_access', 'file', 'user_id', 'organization_id', 'description',
             'target_repo_url', 'include_retired', 'score_configuration',
-            'filters', 'candidates', 'algorithms', 'lookup_config', 'analysis'
+            'filters', 'candidates', 'algorithms', 'lookup_config', 'analysis', 'encoder_model'
         ]
 
     def prepare_object(self, validated_data, instance=None, file=None):
@@ -36,7 +36,8 @@ def prepare_object(self, validated_data, instance=None, file=None):
             instance.columns = columns
         for attr in [
             'name', 'description', 'extras', 'target_repo_url', 'include_retired',
-            'score_configuration', 'filters', 'candidates', 'algorithms', 'lookup_config', 'analysis'
+            'score_configuration', 'filters', 'candidates', 'algorithms', 'lookup_config', 'analysis',
+            'encoder_model'
         ]:
             setattr(instance, attr, validated_data.get(attr, get(instance, attr)))
         if not instance.id:
@@ -109,7 +110,8 @@ class Meta:
             'created_by', 'updated_by', 'created_at', 'updated_at', 'url', 'is_active',
             'owner', 'owner_type', 'owner_url', 'public_access',
             'target_repo_url', 'summary', 'logs', 'include_retired',
-            'score_configuration', 'filters', 'candidates', 'algorithms', 'lookup_config', 'analysis'
+            'score_configuration', 'filters', 'candidates', 'algorithms', 'lookup_config', 'analysis',
+            'encoder_model'
         ]
 
     def __init__(self, *args, **kwargs):

core/users/models.py

@@ -14,7 +14,7 @@
 from core.common.tasks import send_user_verification_email, send_user_reset_password_email
 from core.common.utils import web_url
 from core.users.constants import AUTH_GROUPS, MAPPER_WAITLIST_GROUP, STAFF_GROUP, SUPERADMIN_GROUP, GUEST_GROUP, \
-    MAPPER_APPROVED_GROUP
+    MAPPER_APPROVED_GROUP, CORE_USER_GROUP
 from .constants import USER_OBJECT_TYPE
 from ..common.checksums import ChecksumModel
 
@@ -243,6 +243,10 @@ def is_staff_group(self):
     def is_superadmin_group(self):
         return self.has_auth_group(SUPERADMIN_GROUP)
 
+    @property
+    def is_core_group(self):
+        return self.has_auth_group(CORE_USER_GROUP)
+
     @property
     def auth_headers(self):
         return {'Authorization': f'Token {self.get_token()}'}