Addressed @lucaro 's comments regarding priority-based dequeue of judgement validators. Untested.

Author: sauterl

backend/src/main/kotlin/dev/dres/api/rest/handler/judgement/DequeueJudgementHandler.kt

@@ -53,7 +53,7 @@ class DequeueJudgementHandler : AbstractJudgementHandler(),
     private fun nextRequest(ctx: Context): ApiJudgementRequest? {
         val evaluationManager = ctx.eligibleManagerForId<RunManager>()
         checkEligibility(ctx, evaluationManager)
-        val validator = evaluationManager.judgementValidators.find { it.hasOpen } ?: return null
+        val validator = evaluationManager.judgementValidators.sortedBy { it.priority }.find { it.hasOpen } ?: return null
         val next = validator.next() ?: return null
         val taskDescription = validator.taskTemplate.textualDescription()
         return ApiJudgementRequest(

backend/src/main/kotlin/dev/dres/api/rest/handler/judgement/DequeueVoteHandler.kt

@@ -52,7 +52,7 @@ class DequeueVoteHandler: AbstractJudgementHandler(), GetRestHandler<ApiJudgemen
     fun nextRequest(ctx: Context): ApiJudgementRequest? { //TODO needs adjustment to deal with answerSets
         val evaluationManager = ctx.eligibleManagerForId<RunManager>()
 
-        val validator = evaluationManager.judgementValidators.filterIsInstance<VoteValidator>().find { it.isActive } ?: return null
+        val validator = evaluationManager.judgementValidators.filterIsInstance<VoteValidator>().sortedBy { it.priority }.find { it.isActive } ?: return null
         val next = validator.next()
             ?: /* No submission awaiting judgement */
             return null

backend/src/main/kotlin/dev/dres/run/validation/interfaces/JudgementValidator.kt

@@ -31,6 +31,9 @@ interface JudgementValidator {
     /** The template of the task this validator belongs to */
     val taskTemplate: ApiTaskTemplate
 
+    /** The priority of this [JudgementValidator], higher priorities are represent by a higher number and consumers are expected to respect this */
+    val priority: Int
+
     /**
      * Retrieves and returns the next element that requires a verdict from this [JudgementValidator]'s internal queue.
      *

backend/src/main/kotlin/dev/dres/run/validation/judged/BasicJudgementValidator.kt

@@ -39,12 +39,23 @@ open class BasicJudgementValidator(
         private val counter = AtomicInteger()
         private const val judgementTimeout = 60_000 //ms until a request is re-scheduled
 
+        private const val defaultPriority = 0;
+
         /**
          * The key used to read the task type configuration order
          * The value expected under this key is LIFO, which results in LIFO ordering of the queue,
          * all other parameters or the absence of one results in the default FIFO behaviour.
          */
         private const val CONFIGURATION_ORDER_KEY = "JUDGEMENT.order"
+
+        /**
+         * The key used to read the task type configuration priority.
+         * The value is expected to be a number. Higher number represents higher priority.
+         * Consumers are expected to respect the {priority}
+         */
+        private const val CONFIGURATION_PRIORITY_KEY = "JUDGEMENT.priority"
+
+
     }
 
     /** The [BasicJudgementValidator]'s ID is simply an auto-incrementing number. */
@@ -53,6 +64,9 @@ open class BasicJudgementValidator(
     /** A [BasicJudgementValidator] is always deferring. */
     override val deferring: Boolean = true
 
+    /** The priority of this [JudgementValidator], higher priorities are represent by a higher number and consumers are expected to respect this */
+    override val priority = this.taskType.configuration[CONFIGURATION_PRIORITY_KEY]?.toInt() ?: defaultPriority
+
     /** Internal lock on relevant data structures. */
     private val updateLock = ReentrantReadWriteLock()