Dirt/pm 33527/multi delete task types

src/Admin/Jobs/JobsHostedService.cs

@@ -1,6 +1,8 @@
 using System.Runtime.InteropServices;
 using Bit.Admin.Auth.Jobs;
 using Bit.Admin.Tools.Jobs;
+using Bit.Core.Dirt.Services;
+using Bit.Core.Dirt.Services.Implementations;
 using Bit.Core.Jobs;
 using Bit.Core.Settings;
 using Quartz;
@@ -91,7 +93,7 @@ public override async Task StartAsync(CancellationToken cancellationToken)
         if (!_globalSettings.SelfHosted)
         {
             jobs.Add(new Tuple<Type, ITrigger>(typeof(AliveJob), everyTopOfTheHourTrigger));
-            jobs.Add(new Tuple<Type, ITrigger>(typeof(CleanUpOrganizationEventsJob), everyFiveMinutesTrigger));
+            jobs.Add(new Tuple<Type, ITrigger>(typeof(OrganizationDeleteTasksJob), everyFiveMinutesTrigger));
         }
 
         Jobs = jobs;
@@ -103,7 +105,8 @@ public static void AddJobsServices(IServiceCollection services, bool selfHosted)
         if (!selfHosted)
         {
             services.AddTransient<AliveJob>();
-            services.AddTransient<CleanUpOrganizationEventsJob>();
+            services.AddTransient<OrganizationDeleteTasksJob>();
+            services.AddTransient<IOrganizationDeleteTaskHandler, EventsCleanupOrganizationDeleteTaskHandler>();
         }
         services.AddTransient<DatabaseUpdateStatisticsJob>();
         services.AddTransient<DatabaseRebuildlIndexesJob>();

src/Admin/Jobs/OrganizationDeleteTasksJob.cs

@@ -0,0 +1,127 @@
+#nullable enable
+
+using Azure;
+using Bit.Core;
+using Bit.Core.Dirt.Enums;
+using Bit.Core.Dirt.Repositories;
+using Bit.Core.Dirt.Services;
+using Bit.Core.Jobs;
+using Bit.Core.Services;
+using Quartz;
+
+namespace Bit.Admin.Jobs;
+
+/// <summary>
+/// Drains the <c>OrganizationDeleteTask</c> queue: claims the next pending task and dispatches it to
+/// the <see cref="IOrganizationDeleteTaskHandler"/> registered for its type, deleting in bounded
+/// batches within a run budget so a large cleanup resumes across runs. All lease, progress, and
+/// error bookkeeping lives here; handlers only implement the per-type batch delete.
+/// </summary>
+public class OrganizationDeleteTasksJob : BaseJob
+{
+    private static readonly TimeSpan _runBudget = TimeSpan.FromMinutes(4);
+
+    // A missing handler is expected transiently while a rolling deploy is in flight, but a
+    // genuinely orphaned type (enqueued with no handler that will ever be deployed) stays
+    // unhandled long after any deploy completes. Below this age we log the miss at Warning;
+    // beyond it we escalate to Error so a stuck type can be alerted on rather than lost in
+    // steady Warning volume.
+    private static readonly TimeSpan _orphanedTaskEscalationThreshold = TimeSpan.FromHours(1);
+
+    private readonly IOrganizationDeleteTaskRepository _cleanupRepository;
+    private readonly IReadOnlyDictionary<OrganizationDeleteTaskType, IOrganizationDeleteTaskHandler> _handlers;
+    private readonly IFeatureService _featureService;
+
+    public OrganizationDeleteTasksJob(
+        IOrganizationDeleteTaskRepository cleanupRepository,
+        IEnumerable<IOrganizationDeleteTaskHandler> handlers,
+        IFeatureService featureService,
+        ILogger<OrganizationDeleteTasksJob> logger)
+        : base(logger)
+    {
+        _cleanupRepository = cleanupRepository;
+        // Throws at construction if two handlers claim the same type, failing fast on misconfiguration.
+        _handlers = handlers.ToDictionary(handler => handler.TaskType);
+        _featureService = featureService;
+    }
+
+    protected override async Task ExecuteJobAsync(IJobExecutionContext context)
+    {
+        if (!_featureService.IsEnabled(FeatureFlagKeys.OrganizationEventCleanup))
+        {
+            return;
+        }
+
+        var pending = await _cleanupRepository.ClaimNextPendingAsync();
+        if (pending is null)
+        {
+            return;
+        }
+
+        if (!_handlers.TryGetValue(pending.TaskType, out var handler))
+        {
+            // No handler is registered for this type. This is expected transiently when the server
+            // enqueuing tasks is ahead of this worker during a rolling deploy. We deliberately do NOT
+            // record a failure: doing so would burn the retry budget and could permanently abandon a
+            // task that only needs a newer worker. The claim lease expires and the task is reclaimed
+            // on a later run. Escalate to Error once it has been unhandled long enough that deploy
+            // skew is no longer a plausible explanation.
+            var unhandledFor = DateTime.UtcNow - pending.CreationDate;
+            var logLevel = unhandledFor >= _orphanedTaskEscalationThreshold ? LogLevel.Error : LogLevel.Warning;
+            _logger.Log(logLevel, Constants.BypassFiltersEventId,
+                "No handler registered for organization delete task type {TaskType} (task {TaskId}); leaving for retry. Unhandled for {UnhandledMinutes:N0} minutes.",
+                pending.TaskType, pending.Id, unhandledFor.TotalMinutes);
+            return;
+        }
+
+        _logger.LogInformation(Constants.BypassFiltersEventId,
+            "Starting {TaskType} cleanup for organization {OrganizationId} (task {TaskId})",
+            pending.TaskType, pending.OrganizationId, pending.Id);
+
+        var deadline = DateTime.UtcNow.Add(_runBudget);
+        var deleted = 0;
+        var totalDeleted = 0L;
+
+        try
+        {
+            while (DateTime.UtcNow < deadline && !context.CancellationToken.IsCancellationRequested)
+            {
+                deleted = await handler.DeleteBatchAsync(pending, context.CancellationToken);
+                if (deleted == 0)
+                {
+                    break;
+                }
+
+                await _cleanupRepository.UpdateProgressAsync(pending.Id, deleted);
+                totalDeleted += deleted;
+            }
+
+            if (deleted == 0)
+            {
+                await _cleanupRepository.UpdateCompletedAsync(pending.Id);
+                _logger.LogInformation(Constants.BypassFiltersEventId,
+                    "Completed {TaskType} cleanup for organization {OrganizationId}; deleted {Deleted} items this run",
+                    pending.TaskType, pending.OrganizationId, totalDeleted);
+            }
+            else
+            {
+                _logger.LogInformation(Constants.BypassFiltersEventId,
+                    "Paused {TaskType} cleanup for organization {OrganizationId}; deleted {Deleted} items this run, will resume",
+                    pending.TaskType, pending.OrganizationId, totalDeleted);
+            }
+        }
+        catch (Exception ex)
+        {
+            // Store a sanitized error, never ex.Message: Azure SDK messages can embed
+            // row-key identifiers (e.g. UserId=..., CipherId=...) that must not be persisted.
+            await _cleanupRepository.UpdateErrorAsync(pending.Id, BuildSanitizedError(ex));
+            throw;
+        }
+    }
+
+    private static string BuildSanitizedError(Exception ex) => ex switch
+    {
+        RequestFailedException rfe => $"{nameof(RequestFailedException)} (Status: {rfe.Status}, ErrorCode: {rfe.ErrorCode})",
+        _ => ex.GetType().FullName ?? ex.GetType().Name,
+    };
+}

src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs

@@ -77,7 +77,7 @@ public async Task DeleteAsync(Organization organization)
 
         await _sendFileStorageService.DeleteFilesForOrganizationAsync(organization.Id);
         await _cipherService.DeleteAttachmentsForOrganizationAsync(organization.Id);
-        await _organizationRepository.DeleteAndCreateDeleteTaskAsync(organization, OrganizationDeleteTaskType.EventsCleanup);
+        await _organizationRepository.DeleteAndCreateDeleteTasksAsync(organization, [OrganizationDeleteTaskType.EventsCleanup]);
         await _applicationCacheService.DeleteOrganizationAbilityAsync(organization.Id);
     }
 

src/Core/AdminConsole/Repositories/IOrganizationRepository.cs

@@ -88,12 +88,14 @@ public interface IOrganizationRepository : IRepository<Organization, Guid>
     Task InitializeOrganizationAsync(Organization organization, Func<DbConnection, DbTransaction, Task> confirmOwnerAction);
 
     /// <summary>
-    /// Deletes the organization and, within the same database transaction, enqueues an
-    /// <c>OrganizationDeleteTask</c> of the given type. This guarantees the deletion and the
-    /// cleanup-task record commit atomically, so durable downstream cleanup (e.g. purging
-    /// Table Storage event logs for GDPR) is never lost if the deletion succeeds.
+    /// Deletes the organization and, within the same database transaction, enqueues one
+    /// <c>OrganizationDeleteTask</c> per supplied task type. This guarantees the deletion and the
+    /// cleanup-task records commit atomically, so durable downstream cleanup (e.g. purging
+    /// Table Storage event logs for GDPR) is never lost if the deletion succeeds. Any team can
+    /// enqueue its own cleanup type by adding it to <paramref name="taskTypes"/> without changing
+    /// this signature. An empty collection deletes the organization without enqueuing any task.
     /// </summary>
     /// <param name="organization">The organization to delete.</param>
-    /// <param name="taskType">The type of cleanup task to enqueue.</param>
-    Task DeleteAndCreateDeleteTaskAsync(Organization organization, OrganizationDeleteTaskType taskType);
+    /// <param name="taskTypes">The cleanup task types to enqueue, one row created per type.</param>
+    Task DeleteAndCreateDeleteTasksAsync(Organization organization, IEnumerable<OrganizationDeleteTaskType> taskTypes);
 }

src/Core/Dirt/Services/IOrganizationDeleteTaskHandler.cs

@@ -0,0 +1,26 @@
+using Bit.Core.Dirt.Entities;
+using Bit.Core.Dirt.Enums;
+
+namespace Bit.Core.Dirt.Services;
+
+/// <summary>
+/// Handles the type-specific work for a single <see cref="OrganizationDeleteTask"/>. One
+/// implementation exists per <see cref="OrganizationDeleteTaskType"/>; the cleanup job resolves the
+/// matching handler by <see cref="TaskType"/> and drives it batch-by-batch. A team can add a new
+/// cleanup type by adding an enum value and registering a handler — no job changes required.
+/// </summary>
+public interface IOrganizationDeleteTaskHandler
+{
+    /// <summary>
+    /// The task type this handler is responsible for. Each type must have exactly one handler.
+    /// </summary>
+    OrganizationDeleteTaskType TaskType { get; }
+
+    /// <summary>
+    /// Deletes a single batch of data for the given task and returns the number of items deleted.
+    /// Return 0 when there is nothing left to delete, which signals completion. The job calls this
+    /// repeatedly within a bounded run budget, persisting progress between calls, so implementations
+    /// should delete a bounded amount per call and be safely resumable across runs.
+    /// </summary>
+    Task<int> DeleteBatchAsync(OrganizationDeleteTask task, CancellationToken cancellationToken);
+}

src/Core/Dirt/Services/Implementations/EventsCleanupOrganizationDeleteTaskHandler.cs

@@ -0,0 +1,24 @@
+using Bit.Core.Dirt.Entities;
+using Bit.Core.Dirt.Enums;
+using Bit.Core.Repositories;
+
+namespace Bit.Core.Dirt.Services.Implementations;
+
+/// <summary>
+/// Purges an organization's event logs (e.g. from Table Storage) after the organization is deleted,
+/// satisfying the <see cref="OrganizationDeleteTaskType.EventsCleanup"/> task type.
+/// </summary>
+public class EventsCleanupOrganizationDeleteTaskHandler : IOrganizationDeleteTaskHandler
+{
+    private readonly IEventRepository _eventRepository;
+
+    public EventsCleanupOrganizationDeleteTaskHandler(IEventRepository eventRepository)
+    {
+        _eventRepository = eventRepository;
+    }
+
+    public OrganizationDeleteTaskType TaskType => OrganizationDeleteTaskType.EventsCleanup;
+
+    public Task<int> DeleteBatchAsync(OrganizationDeleteTask task, CancellationToken cancellationToken)
+        => _eventRepository.DeleteManyByOrganizationIdAsync(task.OrganizationId);
+}

src/Infrastructure.Dapper/AdminConsole/Repositories/OrganizationRepository.cs

@@ -4,6 +4,7 @@
 using Bit.Core.AdminConsole.Enums.Provider;
 using Bit.Core.Auth.Entities;
 using Bit.Core.Billing.Organizations.Models;
+using Bit.Core.Dirt.Entities;
 using Bit.Core.Dirt.Enums;
 using Bit.Core.Entities;
 using Bit.Core.Models.Data.Organizations;
@@ -32,22 +33,37 @@ public OrganizationRepository(
     }
 
     public override Task DeleteAsync(Organization organization)
-        => DeleteInternalAsync(organization, null);
+        => DeleteInternalAsync(organization, []);
 
-    public Task DeleteAndCreateDeleteTaskAsync(Organization organization, OrganizationDeleteTaskType taskType)
-        => DeleteInternalAsync(organization, taskType);
+    public Task DeleteAndCreateDeleteTasksAsync(Organization organization,
+        IEnumerable<OrganizationDeleteTaskType> taskTypes)
+        => DeleteInternalAsync(organization, taskTypes);
 
-    private async Task DeleteInternalAsync(Organization organization, OrganizationDeleteTaskType? taskType)
+    private async Task DeleteInternalAsync(Organization organization,
+        IEnumerable<OrganizationDeleteTaskType> taskTypes)
     {
+        var creationDate = DateTime.UtcNow;
+        var deleteTasks = taskTypes
+            .Select(taskType =>
+            {
+                var task = new OrganizationDeleteTask
+                {
+                    OrganizationId = organization.Id,
+                    TaskType = taskType,
+                    CreationDate = creationDate,
+                };
+                task.SetNewId();
+                return task;
+            })
+            .ToList();
+
         using var connection = new SqlConnection(ConnectionString);
         await connection.ExecuteAsync(
             "[dbo].[Organization_DeleteById]",
             new
             {
                 organization.Id,
-                OrganizationDeleteTaskId = taskType.HasValue ? CoreHelpers.GenerateComb() : (Guid?)null,
-                OrganizationDeleteTaskType = (byte?)taskType,
-                OrganizationDeleteTaskCreationDate = taskType.HasValue ? DateTime.UtcNow : (DateTime?)null,
+                OrganizationDeleteTasks = deleteTasks.ToTvp(),
             },
             commandType: CommandType.StoredProcedure);
     }