From 285b531d88e231ae98cd72a8dcc17d7db3002933 Mon Sep 17 00:00:00 2001 From: Graham Walker Date: Wed, 1 Jul 2026 10:05:40 -0500 Subject: [PATCH] PM-33527 initial commit --- .../Jobs/CleanUpOrganizationEventsJob.cs | 97 -------- src/Admin/Jobs/JobsHostedService.cs | 7 +- src/Admin/Jobs/OrganizationDeleteTasksJob.cs | 127 ++++++++++ .../OrganizationDeleteCommand.cs | 2 +- .../Repositories/IOrganizationRepository.cs | 14 +- .../IOrganizationDeleteTaskHandler.cs | 26 ++ ...ntsCleanupOrganizationDeleteTaskHandler.cs | 24 ++ .../Repositories/OrganizationRepository.cs | 30 ++- src/Infrastructure.Dapper/DapperHelpers.cs | 16 ++ .../Repositories/OrganizationRepository.cs | 20 +- .../OrganizationDeleteTaskArray.sql | 4 + .../Organization_DeleteById.sql | 32 ++- ....cs => OrganizationDeleteTasksJobTests.cs} | 126 ++++++++-- .../OrganizationDeleteCommandTests.cs | 18 +- .../OrganizationDeleteTaskRepositoryTests.cs | 89 ++++++- ...OrganizationDeleteByIdMultiTaskEnqueue.sql | 235 ++++++++++++++++++ 16 files changed, 709 insertions(+), 158 deletions(-) delete mode 100644 src/Admin/Jobs/CleanUpOrganizationEventsJob.cs create mode 100644 src/Admin/Jobs/OrganizationDeleteTasksJob.cs create mode 100644 src/Core/Dirt/Services/IOrganizationDeleteTaskHandler.cs create mode 100644 src/Core/Dirt/Services/Implementations/EventsCleanupOrganizationDeleteTaskHandler.cs create mode 100644 src/Sql/dbo/Dirt/User Defined Types/OrganizationDeleteTaskArray.sql rename test/Admin.Test/Jobs/{CleanUpOrganizationEventsJobTests.cs => OrganizationDeleteTasksJobTests.cs} (51%) create mode 100644 util/Migrator/DbScripts/2026-06-30_00_OrganizationDeleteByIdMultiTaskEnqueue.sql diff --git a/src/Admin/Jobs/CleanUpOrganizationEventsJob.cs b/src/Admin/Jobs/CleanUpOrganizationEventsJob.cs deleted file mode 100644 index 9fd29541a251..000000000000 --- a/src/Admin/Jobs/CleanUpOrganizationEventsJob.cs +++ /dev/null @@ -1,97 +0,0 @@ -#nullable enable - -using Azure; -using Bit.Core; -using Bit.Core.Dirt.Repositories; -using Bit.Core.Jobs; -using Bit.Core.Repositories; -using Bit.Core.Services; -using Quartz; - -namespace Bit.Admin.Jobs; - -public class CleanUpOrganizationEventsJob : BaseJob -{ - private static readonly TimeSpan _runBudget = TimeSpan.FromMinutes(4); - - private readonly IOrganizationDeleteTaskRepository _cleanupRepository; - private readonly IEventRepository _eventRepository; - private readonly IFeatureService _featureService; - - public CleanUpOrganizationEventsJob( - IOrganizationDeleteTaskRepository cleanupRepository, - IEventRepository eventRepository, - IFeatureService featureService, - ILogger logger) - : base(logger) - { - _cleanupRepository = cleanupRepository; - _eventRepository = eventRepository; - _featureService = featureService; - } - - protected override async Task ExecuteJobAsync(IJobExecutionContext context) - { - if (!_featureService.IsEnabled(FeatureFlagKeys.OrganizationEventCleanup)) - { - return; - } - - var pending = await _cleanupRepository.ClaimNextPendingAsync(); - if (pending is null) - { - return; - } - - _logger.LogInformation(Constants.BypassFiltersEventId, - "Starting event cleanup for organization {OrganizationId} (cleanup {CleanupId})", - pending.OrganizationId, pending.Id); - - var deadline = DateTime.UtcNow.Add(_runBudget); - var deleted = 0; - var totalDeleted = 0L; - - try - { - while (DateTime.UtcNow < deadline && !context.CancellationToken.IsCancellationRequested) - { - deleted = await _eventRepository.DeleteManyByOrganizationIdAsync( - pending.OrganizationId); - if (deleted == 0) - { - break; - } - - await _cleanupRepository.UpdateProgressAsync(pending.Id, deleted); - totalDeleted += deleted; - } - - if (deleted == 0) - { - await _cleanupRepository.UpdateCompletedAsync(pending.Id); - _logger.LogInformation(Constants.BypassFiltersEventId, - "Completed event cleanup for organization {OrganizationId}; deleted {Deleted} events this run", - pending.OrganizationId, totalDeleted); - } - else - { - _logger.LogInformation(Constants.BypassFiltersEventId, - "Paused event cleanup for organization {OrganizationId}; deleted {Deleted} events this run, will resume", - pending.OrganizationId, totalDeleted); - } - } - catch (Exception ex) - { - // Store a sanitized error, never ex.Message: Azure SDK messages can embed - // row-key identifiers (e.g. UserId=..., CipherId=...) that must not be persisted. - await _cleanupRepository.UpdateErrorAsync(pending.Id, BuildSanitizedError(ex)); - throw; - } - } - - private static string BuildSanitizedError(Exception ex) => ex switch - { - RequestFailedException rfe => $"{nameof(RequestFailedException)} (Status: {rfe.Status}, ErrorCode: {rfe.ErrorCode})", - _ => ex.GetType().FullName ?? ex.GetType().Name, - }; -} diff --git a/src/Admin/Jobs/JobsHostedService.cs b/src/Admin/Jobs/JobsHostedService.cs index 216d24726494..099208bcf690 100644 --- a/src/Admin/Jobs/JobsHostedService.cs +++ b/src/Admin/Jobs/JobsHostedService.cs @@ -1,6 +1,8 @@ using System.Runtime.InteropServices; using Bit.Admin.Auth.Jobs; using Bit.Admin.Tools.Jobs; +using Bit.Core.Dirt.Services; +using Bit.Core.Dirt.Services.Implementations; using Bit.Core.Jobs; using Bit.Core.Settings; using Quartz; @@ -91,7 +93,7 @@ public override async Task StartAsync(CancellationToken cancellationToken) if (!_globalSettings.SelfHosted) { jobs.Add(new Tuple(typeof(AliveJob), everyTopOfTheHourTrigger)); - jobs.Add(new Tuple(typeof(CleanUpOrganizationEventsJob), everyFiveMinutesTrigger)); + jobs.Add(new Tuple(typeof(OrganizationDeleteTasksJob), everyFiveMinutesTrigger)); } Jobs = jobs; @@ -103,7 +105,8 @@ public static void AddJobsServices(IServiceCollection services, bool selfHosted) if (!selfHosted) { services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + services.AddTransient(); } services.AddTransient(); services.AddTransient(); diff --git a/src/Admin/Jobs/OrganizationDeleteTasksJob.cs b/src/Admin/Jobs/OrganizationDeleteTasksJob.cs new file mode 100644 index 000000000000..7196a2f53438 --- /dev/null +++ b/src/Admin/Jobs/OrganizationDeleteTasksJob.cs @@ -0,0 +1,127 @@ +#nullable enable + +using Azure; +using Bit.Core; +using Bit.Core.Dirt.Enums; +using Bit.Core.Dirt.Repositories; +using Bit.Core.Dirt.Services; +using Bit.Core.Jobs; +using Bit.Core.Services; +using Quartz; + +namespace Bit.Admin.Jobs; + +/// +/// Drains the OrganizationDeleteTask queue: claims the next pending task and dispatches it to +/// the registered for its type, deleting in bounded +/// batches within a run budget so a large cleanup resumes across runs. All lease, progress, and +/// error bookkeeping lives here; handlers only implement the per-type batch delete. +/// +public class OrganizationDeleteTasksJob : BaseJob +{ + private static readonly TimeSpan _runBudget = TimeSpan.FromMinutes(4); + + // A missing handler is expected transiently while a rolling deploy is in flight, but a + // genuinely orphaned type (enqueued with no handler that will ever be deployed) stays + // unhandled long after any deploy completes. Below this age we log the miss at Warning; + // beyond it we escalate to Error so a stuck type can be alerted on rather than lost in + // steady Warning volume. + private static readonly TimeSpan _orphanedTaskEscalationThreshold = TimeSpan.FromHours(1); + + private readonly IOrganizationDeleteTaskRepository _cleanupRepository; + private readonly IReadOnlyDictionary _handlers; + private readonly IFeatureService _featureService; + + public OrganizationDeleteTasksJob( + IOrganizationDeleteTaskRepository cleanupRepository, + IEnumerable handlers, + IFeatureService featureService, + ILogger logger) + : base(logger) + { + _cleanupRepository = cleanupRepository; + // Throws at construction if two handlers claim the same type, failing fast on misconfiguration. + _handlers = handlers.ToDictionary(handler => handler.TaskType); + _featureService = featureService; + } + + protected override async Task ExecuteJobAsync(IJobExecutionContext context) + { + if (!_featureService.IsEnabled(FeatureFlagKeys.OrganizationEventCleanup)) + { + return; + } + + var pending = await _cleanupRepository.ClaimNextPendingAsync(); + if (pending is null) + { + return; + } + + if (!_handlers.TryGetValue(pending.TaskType, out var handler)) + { + // No handler is registered for this type. This is expected transiently when the server + // enqueuing tasks is ahead of this worker during a rolling deploy. We deliberately do NOT + // record a failure: doing so would burn the retry budget and could permanently abandon a + // task that only needs a newer worker. The claim lease expires and the task is reclaimed + // on a later run. Escalate to Error once it has been unhandled long enough that deploy + // skew is no longer a plausible explanation. + var unhandledFor = DateTime.UtcNow - pending.CreationDate; + var logLevel = unhandledFor >= _orphanedTaskEscalationThreshold ? LogLevel.Error : LogLevel.Warning; + _logger.Log(logLevel, Constants.BypassFiltersEventId, + "No handler registered for organization delete task type {TaskType} (task {TaskId}); leaving for retry. Unhandled for {UnhandledMinutes:N0} minutes.", + pending.TaskType, pending.Id, unhandledFor.TotalMinutes); + return; + } + + _logger.LogInformation(Constants.BypassFiltersEventId, + "Starting {TaskType} cleanup for organization {OrganizationId} (task {TaskId})", + pending.TaskType, pending.OrganizationId, pending.Id); + + var deadline = DateTime.UtcNow.Add(_runBudget); + var deleted = 0; + var totalDeleted = 0L; + + try + { + while (DateTime.UtcNow < deadline && !context.CancellationToken.IsCancellationRequested) + { + deleted = await handler.DeleteBatchAsync(pending, context.CancellationToken); + if (deleted == 0) + { + break; + } + + await _cleanupRepository.UpdateProgressAsync(pending.Id, deleted); + totalDeleted += deleted; + } + + if (deleted == 0) + { + await _cleanupRepository.UpdateCompletedAsync(pending.Id); + _logger.LogInformation(Constants.BypassFiltersEventId, + "Completed {TaskType} cleanup for organization {OrganizationId}; deleted {Deleted} items this run", + pending.TaskType, pending.OrganizationId, totalDeleted); + } + else + { + _logger.LogInformation(Constants.BypassFiltersEventId, + "Paused {TaskType} cleanup for organization {OrganizationId}; deleted {Deleted} items this run, will resume", + pending.TaskType, pending.OrganizationId, totalDeleted); + } + } + catch (Exception ex) + { + // Store a sanitized error, never ex.Message: Azure SDK messages can embed + // row-key identifiers (e.g. UserId=..., CipherId=...) that must not be persisted. + await _cleanupRepository.UpdateErrorAsync(pending.Id, BuildSanitizedError(ex)); + throw; + } + } + + private static string BuildSanitizedError(Exception ex) => ex switch + { + RequestFailedException rfe => $"{nameof(RequestFailedException)} (Status: {rfe.Status}, ErrorCode: {rfe.ErrorCode})", + _ => ex.GetType().FullName ?? ex.GetType().Name, + }; +} diff --git a/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs b/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs index c405874eec63..6ede5398b432 100644 --- a/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs +++ b/src/Core/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommand.cs @@ -77,7 +77,7 @@ public async Task DeleteAsync(Organization organization) await _sendFileStorageService.DeleteFilesForOrganizationAsync(organization.Id); await _cipherService.DeleteAttachmentsForOrganizationAsync(organization.Id); - await _organizationRepository.DeleteAndCreateDeleteTaskAsync(organization, OrganizationDeleteTaskType.EventsCleanup); + await _organizationRepository.DeleteAndCreateDeleteTasksAsync(organization, [OrganizationDeleteTaskType.EventsCleanup]); await _applicationCacheService.DeleteOrganizationAbilityAsync(organization.Id); } diff --git a/src/Core/AdminConsole/Repositories/IOrganizationRepository.cs b/src/Core/AdminConsole/Repositories/IOrganizationRepository.cs index cbfe7e4fece2..46063e5372b2 100644 --- a/src/Core/AdminConsole/Repositories/IOrganizationRepository.cs +++ b/src/Core/AdminConsole/Repositories/IOrganizationRepository.cs @@ -88,12 +88,14 @@ public interface IOrganizationRepository : IRepository Task InitializeOrganizationAsync(Organization organization, Func confirmOwnerAction); /// - /// Deletes the organization and, within the same database transaction, enqueues an - /// OrganizationDeleteTask of the given type. This guarantees the deletion and the - /// cleanup-task record commit atomically, so durable downstream cleanup (e.g. purging - /// Table Storage event logs for GDPR) is never lost if the deletion succeeds. + /// Deletes the organization and, within the same database transaction, enqueues one + /// OrganizationDeleteTask per supplied task type. This guarantees the deletion and the + /// cleanup-task records commit atomically, so durable downstream cleanup (e.g. purging + /// Table Storage event logs for GDPR) is never lost if the deletion succeeds. Any team can + /// enqueue its own cleanup type by adding it to without changing + /// this signature. An empty collection deletes the organization without enqueuing any task. /// /// The organization to delete. - /// The type of cleanup task to enqueue. - Task DeleteAndCreateDeleteTaskAsync(Organization organization, OrganizationDeleteTaskType taskType); + /// The cleanup task types to enqueue, one row created per type. + Task DeleteAndCreateDeleteTasksAsync(Organization organization, IEnumerable taskTypes); } diff --git a/src/Core/Dirt/Services/IOrganizationDeleteTaskHandler.cs b/src/Core/Dirt/Services/IOrganizationDeleteTaskHandler.cs new file mode 100644 index 000000000000..15cedbad9d4c --- /dev/null +++ b/src/Core/Dirt/Services/IOrganizationDeleteTaskHandler.cs @@ -0,0 +1,26 @@ +using Bit.Core.Dirt.Entities; +using Bit.Core.Dirt.Enums; + +namespace Bit.Core.Dirt.Services; + +/// +/// Handles the type-specific work for a single . One +/// implementation exists per ; the cleanup job resolves the +/// matching handler by and drives it batch-by-batch. A team can add a new +/// cleanup type by adding an enum value and registering a handler — no job changes required. +/// +public interface IOrganizationDeleteTaskHandler +{ + /// + /// The task type this handler is responsible for. Each type must have exactly one handler. + /// + OrganizationDeleteTaskType TaskType { get; } + + /// + /// Deletes a single batch of data for the given task and returns the number of items deleted. + /// Return 0 when there is nothing left to delete, which signals completion. The job calls this + /// repeatedly within a bounded run budget, persisting progress between calls, so implementations + /// should delete a bounded amount per call and be safely resumable across runs. + /// + Task DeleteBatchAsync(OrganizationDeleteTask task, CancellationToken cancellationToken); +} diff --git a/src/Core/Dirt/Services/Implementations/EventsCleanupOrganizationDeleteTaskHandler.cs b/src/Core/Dirt/Services/Implementations/EventsCleanupOrganizationDeleteTaskHandler.cs new file mode 100644 index 000000000000..fad1be27c0aa --- /dev/null +++ b/src/Core/Dirt/Services/Implementations/EventsCleanupOrganizationDeleteTaskHandler.cs @@ -0,0 +1,24 @@ +using Bit.Core.Dirt.Entities; +using Bit.Core.Dirt.Enums; +using Bit.Core.Repositories; + +namespace Bit.Core.Dirt.Services.Implementations; + +/// +/// Purges an organization's event logs (e.g. from Table Storage) after the organization is deleted, +/// satisfying the task type. +/// +public class EventsCleanupOrganizationDeleteTaskHandler : IOrganizationDeleteTaskHandler +{ + private readonly IEventRepository _eventRepository; + + public EventsCleanupOrganizationDeleteTaskHandler(IEventRepository eventRepository) + { + _eventRepository = eventRepository; + } + + public OrganizationDeleteTaskType TaskType => OrganizationDeleteTaskType.EventsCleanup; + + public Task DeleteBatchAsync(OrganizationDeleteTask task, CancellationToken cancellationToken) + => _eventRepository.DeleteManyByOrganizationIdAsync(task.OrganizationId); +} diff --git a/src/Infrastructure.Dapper/AdminConsole/Repositories/OrganizationRepository.cs b/src/Infrastructure.Dapper/AdminConsole/Repositories/OrganizationRepository.cs index 78a3920eb85f..b9dc328e66b4 100644 --- a/src/Infrastructure.Dapper/AdminConsole/Repositories/OrganizationRepository.cs +++ b/src/Infrastructure.Dapper/AdminConsole/Repositories/OrganizationRepository.cs @@ -4,6 +4,7 @@ using Bit.Core.AdminConsole.Enums.Provider; using Bit.Core.Auth.Entities; using Bit.Core.Billing.Organizations.Models; +using Bit.Core.Dirt.Entities; using Bit.Core.Dirt.Enums; using Bit.Core.Entities; using Bit.Core.Models.Data.Organizations; @@ -32,22 +33,37 @@ public OrganizationRepository( } public override Task DeleteAsync(Organization organization) - => DeleteInternalAsync(organization, null); + => DeleteInternalAsync(organization, []); - public Task DeleteAndCreateDeleteTaskAsync(Organization organization, OrganizationDeleteTaskType taskType) - => DeleteInternalAsync(organization, taskType); + public Task DeleteAndCreateDeleteTasksAsync(Organization organization, + IEnumerable taskTypes) + => DeleteInternalAsync(organization, taskTypes); - private async Task DeleteInternalAsync(Organization organization, OrganizationDeleteTaskType? taskType) + private async Task DeleteInternalAsync(Organization organization, + IEnumerable taskTypes) { + var creationDate = DateTime.UtcNow; + var deleteTasks = taskTypes + .Select(taskType => + { + var task = new OrganizationDeleteTask + { + OrganizationId = organization.Id, + TaskType = taskType, + CreationDate = creationDate, + }; + task.SetNewId(); + return task; + }) + .ToList(); + using var connection = new SqlConnection(ConnectionString); await connection.ExecuteAsync( "[dbo].[Organization_DeleteById]", new { organization.Id, - OrganizationDeleteTaskId = taskType.HasValue ? CoreHelpers.GenerateComb() : (Guid?)null, - OrganizationDeleteTaskType = (byte?)taskType, - OrganizationDeleteTaskCreationDate = taskType.HasValue ? DateTime.UtcNow : (DateTime?)null, + OrganizationDeleteTasks = deleteTasks.ToTvp(), }, commandType: CommandType.StoredProcedure); } diff --git a/src/Infrastructure.Dapper/DapperHelpers.cs b/src/Infrastructure.Dapper/DapperHelpers.cs index 4384a6f7527b..160696b9e99e 100644 --- a/src/Infrastructure.Dapper/DapperHelpers.cs +++ b/src/Infrastructure.Dapper/DapperHelpers.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; using System.Reflection; +using Bit.Core.Dirt.Entities; using Bit.Core.Entities; using Bit.Core.Models.Data; using Dapper; @@ -155,11 +156,26 @@ public static class DapperHelpers ] ); + private static readonly DataTableBuilder _organizationDeleteTaskTableBuilder = new( + [ + t => t.Id, + t => t.TaskType, + t => t.CreationDate, + ] + ); + public static DataTable ToGuidIdArrayTVP(this IEnumerable ids) { return ids.ToArrayTVP("GuidId"); } + public static DataTable ToTvp(this IEnumerable tasks) + { + var table = _organizationDeleteTaskTableBuilder.Build(tasks ?? []); + table.SetTypeName("[dbo].[OrganizationDeleteTaskArray]"); + return table; + } + public static DataTable ToTwoGuidIdArrayTVP(this IEnumerable<(Guid id1, Guid id2)> values) { var table = new DataTable(); diff --git a/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs b/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs index 6a2fafefccb4..422f5041db07 100644 --- a/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs +++ b/src/Infrastructure.EntityFramework/AdminConsole/Repositories/OrganizationRepository.cs @@ -238,14 +238,14 @@ public async Task UpdateStorageAsync(Guid id) } public override Task DeleteAsync(Core.AdminConsole.Entities.Organization organization) - => DeleteInternalAsync(organization, null); + => DeleteInternalAsync(organization, []); - public Task DeleteAndCreateDeleteTaskAsync(Core.AdminConsole.Entities.Organization organization, - OrganizationDeleteTaskType taskType) - => DeleteInternalAsync(organization, taskType); + public Task DeleteAndCreateDeleteTasksAsync(Core.AdminConsole.Entities.Organization organization, + IEnumerable taskTypes) + => DeleteInternalAsync(organization, taskTypes); private async Task DeleteInternalAsync(Core.AdminConsole.Entities.Organization organization, - OrganizationDeleteTaskType? deleteTaskType) + IEnumerable deleteTaskTypes) { using (var scope = ServiceScopeFactory.CreateScope()) { @@ -314,14 +314,18 @@ await dbContext.OrganizationConnections.Where(oc => oc.OrganizationId == organiz var orgEntity = await dbContext.FindAsync(organization.Id); dbContext.Remove(orgEntity); - // Atomically enqueue the cleanup task within the same transaction as the + // Atomically enqueue the cleanup tasks within the same transaction as the // deletion, so durable downstream cleanup is never lost if the delete commits. - if (deleteTaskType.HasValue) + // One row is created per supplied task type. + var creationDate = DateTime.UtcNow; + foreach (var deleteTaskType in deleteTaskTypes) { var deleteTask = new Dirt.Models.OrganizationDeleteTask { OrganizationId = organization.Id, - TaskType = deleteTaskType.Value, + TaskType = deleteTaskType, + CreationDate = creationDate, + RevisionDate = creationDate, }; deleteTask.SetNewId(); await dbContext.OrganizationDeleteTasks.AddAsync(deleteTask); diff --git a/src/Sql/dbo/Dirt/User Defined Types/OrganizationDeleteTaskArray.sql b/src/Sql/dbo/Dirt/User Defined Types/OrganizationDeleteTaskArray.sql new file mode 100644 index 000000000000..e17030f121ac --- /dev/null +++ b/src/Sql/dbo/Dirt/User Defined Types/OrganizationDeleteTaskArray.sql @@ -0,0 +1,4 @@ +CREATE TYPE [dbo].[OrganizationDeleteTaskArray] AS TABLE ( + [Id] UNIQUEIDENTIFIER NOT NULL, + [TaskType] TINYINT NOT NULL, + [CreationDate] DATETIME2(7) NOT NULL); diff --git a/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql b/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql index 707e297b2b51..0e4b429b2b79 100644 --- a/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql +++ b/src/Sql/dbo/Stored Procedures/Organization_DeleteById.sql @@ -2,7 +2,8 @@ @Id UNIQUEIDENTIFIER, @OrganizationDeleteTaskId UNIQUEIDENTIFIER = NULL, @OrganizationDeleteTaskType TINYINT = NULL, - @OrganizationDeleteTaskCreationDate DATETIME2(7) = NULL + @OrganizationDeleteTaskCreationDate DATETIME2(7) = NULL, + @OrganizationDeleteTasks [dbo].[OrganizationDeleteTaskArray] READONLY WITH RECOMPILE AS BEGIN @@ -161,9 +162,32 @@ BEGIN WHERE [OrganizationId] = @Id - -- Atomically enqueue an OrganizationDeleteTask (e.g. for purging Table Storage - -- event logs) so downstream cleanup is durably recorded with the deletion. - IF @OrganizationDeleteTaskId IS NOT NULL + -- Atomically enqueue one or more OrganizationDeleteTasks (e.g. for purging Table + -- Storage event logs) so downstream cleanup is durably recorded with the deletion. + -- Preferred path: a set of tasks passed via table-valued parameter, letting any + -- number of teams enqueue their own cleanup type in the same transaction. + IF EXISTS (SELECT 1 FROM @OrganizationDeleteTasks) + BEGIN + INSERT INTO [dbo].[OrganizationDeleteTask] + ( + [Id], + [OrganizationId], + [TaskType], + [CreationDate], + [RevisionDate] + ) + SELECT + [Id], + @Id, + [TaskType], + [CreationDate], + [CreationDate] + FROM + @OrganizationDeleteTasks + END + -- Legacy single-task path. Retained so callers still running the previous server + -- version keep working during a rolling deployment; safe to remove once fully rolled. + ELSE IF @OrganizationDeleteTaskId IS NOT NULL BEGIN DECLARE @OrganizationDeleteTaskDate DATETIME2(7) = COALESCE(@OrganizationDeleteTaskCreationDate, SYSUTCDATETIME()) diff --git a/test/Admin.Test/Jobs/CleanUpOrganizationEventsJobTests.cs b/test/Admin.Test/Jobs/OrganizationDeleteTasksJobTests.cs similarity index 51% rename from test/Admin.Test/Jobs/CleanUpOrganizationEventsJobTests.cs rename to test/Admin.Test/Jobs/OrganizationDeleteTasksJobTests.cs index 1e41d7da4fb1..7641ba8b5433 100644 --- a/test/Admin.Test/Jobs/CleanUpOrganizationEventsJobTests.cs +++ b/test/Admin.Test/Jobs/OrganizationDeleteTasksJobTests.cs @@ -1,8 +1,9 @@ -using Bit.Admin.Jobs; +using Bit.Admin.Jobs; using Bit.Core; using Bit.Core.Dirt.Entities; +using Bit.Core.Dirt.Enums; using Bit.Core.Dirt.Repositories; -using Bit.Core.Repositories; +using Bit.Core.Dirt.Services; using Bit.Core.Services; using Microsoft.Extensions.Logging; using NSubstitute; @@ -11,23 +12,24 @@ namespace Admin.Test.Jobs; -public class CleanUpOrganizationEventsJobTests +public class OrganizationDeleteTasksJobTests { private readonly IOrganizationDeleteTaskRepository _cleanupRepository; - private readonly IEventRepository _eventRepository; + private readonly IOrganizationDeleteTaskHandler _handler; private readonly IFeatureService _featureService; - private readonly ILogger _logger; - private readonly CleanUpOrganizationEventsJob _sut; + private readonly ILogger _logger; + private readonly OrganizationDeleteTasksJob _sut; - public CleanUpOrganizationEventsJobTests() + public OrganizationDeleteTasksJobTests() { _cleanupRepository = Substitute.For(); - _eventRepository = Substitute.For(); + _handler = Substitute.For(); + _handler.TaskType.Returns(OrganizationDeleteTaskType.EventsCleanup); _featureService = Substitute.For(); - _logger = Substitute.For>(); - _sut = new CleanUpOrganizationEventsJob( + _logger = Substitute.For>(); + _sut = new OrganizationDeleteTasksJob( _cleanupRepository, - _eventRepository, + new[] { _handler }, _featureService, _logger); @@ -43,7 +45,7 @@ public async Task Execute_FeatureFlagOff_DoesNothing() await _sut.Execute(context); await _cleanupRepository.DidNotReceiveWithAnyArgs().ClaimNextPendingAsync(); - await _eventRepository.DidNotReceiveWithAnyArgs().DeleteManyByOrganizationIdAsync(default); + await _handler.DidNotReceiveWithAnyArgs().DeleteBatchAsync(default!, default); } [Fact] @@ -54,7 +56,68 @@ public async Task Execute_NoPendingCleanup_ReturnsEarly() await _sut.Execute(context); - await _eventRepository.DidNotReceiveWithAnyArgs().DeleteManyByOrganizationIdAsync(default); + await _handler.DidNotReceiveWithAnyArgs().DeleteBatchAsync(default!, default); + } + + [Fact] + public async Task Execute_NoHandlerForTaskType_LeavesForRetryWithoutRecordingFailure() + { + // SUT built with no handlers, so the claimed task's type has no dispatch target. + var sut = new OrganizationDeleteTasksJob( + _cleanupRepository, + Array.Empty(), + _featureService, + _logger); + var pending = CreatePending(); + _cleanupRepository.ClaimNextPendingAsync().Returns(pending); + var context = CreateContext(); + + await sut.Execute(context); + + // Deliberately does not burn the retry budget; the task is left for stale-lease reclaim. + await _cleanupRepository.DidNotReceiveWithAnyArgs().UpdateErrorAsync(default, default!); + await _cleanupRepository.DidNotReceiveWithAnyArgs().UpdateCompletedAsync(default); + await _cleanupRepository.DidNotReceiveWithAnyArgs().UpdateProgressAsync(default, default); + } + + [Fact] + public async Task Execute_NoHandlerForTaskType_RecentlyCreated_LogsWarningNotError() + { + var sut = new OrganizationDeleteTasksJob( + _cleanupRepository, + Array.Empty(), + _featureService, + _logger); + // Freshly enqueued: a missing handler is plausibly just rolling-deploy skew. + var pending = CreatePending(); + pending.CreationDate = DateTime.UtcNow; + _cleanupRepository.ClaimNextPendingAsync().Returns(pending); + var context = CreateContext(); + + await sut.Execute(context); + + AssertLogged(LogLevel.Warning); + AssertNotLogged(LogLevel.Error); + } + + [Fact] + public async Task Execute_NoHandlerForTaskType_UnhandledPastThreshold_EscalatesToError() + { + var sut = new OrganizationDeleteTasksJob( + _cleanupRepository, + Array.Empty(), + _featureService, + _logger); + // Unhandled for hours: deploy skew is no longer a plausible explanation, so escalate. + var pending = CreatePending(); + pending.CreationDate = DateTime.UtcNow.AddHours(-2); + _cleanupRepository.ClaimNextPendingAsync().Returns(pending); + var context = CreateContext(); + + await sut.Execute(context); + + AssertLogged(LogLevel.Error); + AssertNotLogged(LogLevel.Warning); } [Fact] @@ -62,15 +125,15 @@ public async Task Execute_DeletesRepeatedlyThenCompletes_WhenBatchReturnsZero() { var pending = CreatePending(); _cleanupRepository.ClaimNextPendingAsync().Returns(pending); - _eventRepository - .DeleteManyByOrganizationIdAsync(pending.OrganizationId) + _handler + .DeleteBatchAsync(pending, Arg.Any()) .Returns(2000, 2000, 500, 0); var context = CreateContext(); await _sut.Execute(context); - await _eventRepository.Received(4) - .DeleteManyByOrganizationIdAsync(pending.OrganizationId); + await _handler.Received(4) + .DeleteBatchAsync(pending, Arg.Any()); await _cleanupRepository.Received(2).UpdateProgressAsync(pending.Id, 2000); await _cleanupRepository.Received(1).UpdateProgressAsync(pending.Id, 500); await _cleanupRepository.Received(1).UpdateCompletedAsync(pending.Id); @@ -84,8 +147,8 @@ public async Task Execute_CancellationRequested_LeavesPending() _cleanupRepository.ClaimNextPendingAsync().Returns(pending); using var cts = new CancellationTokenSource(); - _eventRepository - .DeleteManyByOrganizationIdAsync(pending.OrganizationId) + _handler + .DeleteBatchAsync(pending, Arg.Any()) .Returns(_ => { cts.Cancel(); @@ -104,8 +167,8 @@ public async Task Execute_DeleteThrows_RecordsErrorAndDoesNotComplete() { var pending = CreatePending(); _cleanupRepository.ClaimNextPendingAsync().Returns(pending); - _eventRepository - .DeleteManyByOrganizationIdAsync(pending.OrganizationId) + _handler + .DeleteBatchAsync(pending, Arg.Any()) .Throws(new InvalidOperationException("boom")); var context = CreateContext(); @@ -124,8 +187,8 @@ public async Task Execute_DeleteThrows_DoesNotLeakRowKeyIdentifiersInError() _cleanupRepository.ClaimNextPendingAsync().Returns(pending); // Azure SDK messages can embed row-key identifiers; these must never be persisted. var leakyMessage = "The specified entity already exists. UserId=abc123, CipherId=def456"; - _eventRepository - .DeleteManyByOrganizationIdAsync(pending.OrganizationId) + _handler + .DeleteBatchAsync(pending, Arg.Any()) .Throws(new InvalidOperationException(leakyMessage)); var context = CreateContext(); @@ -136,10 +199,27 @@ await _cleanupRepository.Received(1).UpdateErrorAsync( Arg.Is(error => !error.Contains("UserId") && !error.Contains("CipherId"))); } + private void AssertLogged(LogLevel level) => + _logger.Received(1).Log( + level, + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>()); + + private void AssertNotLogged(LogLevel level) => + _logger.DidNotReceive().Log( + level, + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>()); + private static OrganizationDeleteTask CreatePending() => new() { Id = Guid.NewGuid(), OrganizationId = Guid.NewGuid(), + TaskType = OrganizationDeleteTaskType.EventsCleanup, CreationDate = DateTime.UtcNow, }; diff --git a/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs b/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs index 6b5a7f5643e9..773b7f97e83b 100644 --- a/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs +++ b/test/Core.Test/AdminConsole/OrganizationFeatures/Organizations/OrganizationDeleteCommandTests.cs @@ -36,8 +36,9 @@ public async Task Delete_Success(Organization organization, await cipherService.Received(1).DeleteAttachmentsForOrganizationAsync(organization.Id); // The deletion and the events-cleanup task enqueue happen atomically in one repository call. - await organizationRepository.Received(1).DeleteAndCreateDeleteTaskAsync( - organization, OrganizationDeleteTaskType.EventsCleanup); + await organizationRepository.Received(1).DeleteAndCreateDeleteTasksAsync( + organization, Arg.Is>( + taskTypes => taskTypes.SequenceEqual(new[] { OrganizationDeleteTaskType.EventsCleanup }))); await organizationRepository.DidNotReceive().DeleteAsync(organization); await applicationCacheService.Received(1).DeleteOrganizationAbilityAsync(organization.Id); } @@ -59,7 +60,7 @@ public async Task Delete_Fails_KeyConnector(Organization organization, SutProvid Assert.Contains("You cannot delete an Organization that is using Key Connector.", exception.Message); - await organizationRepository.DidNotReceiveWithAnyArgs().DeleteAndCreateDeleteTaskAsync(default, default); + await organizationRepository.DidNotReceiveWithAnyArgs().DeleteAndCreateDeleteTasksAsync(default, default); await applicationCacheService.DidNotReceiveWithAnyArgs().DeleteOrganizationAbilityAsync(default); } @@ -129,7 +130,8 @@ public async Task Delete_WhenFlagEnabled_HandlesBillingException( await sutProvider.Sut.DeleteAsync(organization); await sutProvider.GetDependency().Received(1) - .DeleteAndCreateDeleteTaskAsync(organization, OrganizationDeleteTaskType.EventsCleanup); + .DeleteAndCreateDeleteTasksAsync(organization, Arg.Is>( + taskTypes => taskTypes.SequenceEqual(new[] { OrganizationDeleteTaskType.EventsCleanup }))); } @@ -153,7 +155,8 @@ public async Task Delete_WhenFlagDisabled_HandlesBillingException( await sutProvider.Sut.DeleteAsync(organization); await sutProvider.GetDependency().Received(1) - .DeleteAndCreateDeleteTaskAsync(organization, OrganizationDeleteTaskType.EventsCleanup); + .DeleteAndCreateDeleteTasksAsync(organization, Arg.Is>( + taskTypes => taskTypes.SequenceEqual(new[] { OrganizationDeleteTaskType.EventsCleanup }))); } [Theory, PaidOrganizationCustomize, BitAutoData] @@ -171,7 +174,7 @@ public async Task Delete_WithFileSends_DeletesFilesBeforeDbRecords( .Returns(Task.CompletedTask) .AndDoes(_ => callOrder.Add("file")); sutProvider.GetDependency() - .DeleteAndCreateDeleteTaskAsync(organization, OrganizationDeleteTaskType.EventsCleanup) + .DeleteAndCreateDeleteTasksAsync(organization, Arg.Any>()) .Returns(Task.CompletedTask) .AndDoes(_ => callOrder.Add("db")); @@ -180,7 +183,8 @@ public async Task Delete_WithFileSends_DeletesFilesBeforeDbRecords( await sutProvider.GetDependency() .Received(1).DeleteFilesForOrganizationAsync(organization.Id); await sutProvider.GetDependency() - .Received(1).DeleteAndCreateDeleteTaskAsync(organization, OrganizationDeleteTaskType.EventsCleanup); + .Received(1).DeleteAndCreateDeleteTasksAsync(organization, Arg.Is>( + taskTypes => taskTypes.SequenceEqual(new[] { OrganizationDeleteTaskType.EventsCleanup }))); Assert.Equal(new[] { "file", "db" }, callOrder); } } diff --git a/test/Infrastructure.IntegrationTest/Dirt/Repositories/OrganizationDeleteTaskRepositoryTests.cs b/test/Infrastructure.IntegrationTest/Dirt/Repositories/OrganizationDeleteTaskRepositoryTests.cs index 4935a2d41826..1d007dbf598b 100644 --- a/test/Infrastructure.IntegrationTest/Dirt/Repositories/OrganizationDeleteTaskRepositoryTests.cs +++ b/test/Infrastructure.IntegrationTest/Dirt/Repositories/OrganizationDeleteTaskRepositoryTests.cs @@ -113,7 +113,7 @@ public async Task ClaimNextPendingAsync_FailureCountAtMax_RowNotClaimed( } [Theory, DatabaseData] - public async Task DeleteAndCreateDeleteTaskAsync_DeletesOrganizationAndEnqueuesTask( + public async Task DeleteAndCreateDeleteTasksAsync_DeletesOrganizationAndEnqueuesTask( IOrganizationRepository organizationRepository, Database database, IServiceProvider services) { var organization = await organizationRepository.CreateAsync(new Organization @@ -124,8 +124,8 @@ public async Task DeleteAndCreateDeleteTaskAsync_DeletesOrganizationAndEnqueuesT PrivateKey = "privatekey", }); - await organizationRepository.DeleteAndCreateDeleteTaskAsync( - organization, OrganizationDeleteTaskType.EventsCleanup); + await organizationRepository.DeleteAndCreateDeleteTasksAsync( + organization, [OrganizationDeleteTaskType.EventsCleanup]); // The organization is gone and the cleanup task was enqueued in the same transaction. Assert.Null(await organizationRepository.GetByIdAsync(organization.Id)); @@ -135,6 +135,34 @@ await organizationRepository.DeleteAndCreateDeleteTaskAsync( Assert.Null(task.CompletedDate); } + [Theory, DatabaseData] + public async Task DeleteAndCreateDeleteTasksAsync_MultipleTaskTypes_EnqueuesOneRowPerType( + IOrganizationRepository organizationRepository, Database database, IServiceProvider services) + { + var organization = await organizationRepository.CreateAsync(new Organization + { + Name = "Test Org", + BillingEmail = "test@example.com", + Plan = "Test", + PrivateKey = "privatekey", + }); + + // Only one task type exists today, so we exercise the multi-row paths (the TVP + // INSERT...SELECT on SqlServer and the foreach on EF) by enqueuing two elements: + // both must produce one distinct row per element, never collapsing or dropping rows. + await organizationRepository.DeleteAndCreateDeleteTasksAsync( + organization, + [OrganizationDeleteTaskType.EventsCleanup, OrganizationDeleteTaskType.EventsCleanup]); + + Assert.Null(await organizationRepository.GetByIdAsync(organization.Id)); + var tasks = await ListTasksByOrganizationIdAsync(services, database, organization.Id); + Assert.Equal(2, tasks.Count); + Assert.All(tasks, task => Assert.Equal(OrganizationDeleteTaskType.EventsCleanup, task.TaskType)); + Assert.All(tasks, task => Assert.Null(task.CompletedDate)); + // Each enqueued task must receive a unique primary key. + Assert.Equal(2, tasks.Select(task => task.Id).Distinct().Count()); + } + [Theory, DatabaseData] public async Task DeleteAsync_DoesNotEnqueueDeleteTask( IOrganizationRepository organizationRepository, Database database, IServiceProvider services) @@ -174,6 +202,28 @@ public async Task DeleteAsync_DoesNotEnqueueDeleteTask( .FirstOrDefaultAsync(t => t.OrganizationId == organizationId); } + /// + /// Reads all cleanup-task rows for an organization across providers, mirroring + /// but returning every row so the + /// multi-row enqueue path can be asserted. + /// + private static async Task> ListTasksByOrganizationIdAsync( + IServiceProvider services, Database database, Guid organizationId) + { + if (database.Type == SupportedDatabaseProviders.SqlServer && !database.UseEf) + { + return await QueryRowsByOrganizationIdAsync(database.ConnectionString, organizationId); + } + + using var scope = services.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var rows = await dbContext.OrganizationDeleteTasks + .AsNoTracking() + .Where(t => t.OrganizationId == organizationId) + .ToListAsync(); + return rows.Cast().ToList(); + } + private static async Task QueryRowAsync(string connectionString, Guid id) { await using var connection = new SqlConnection(connectionString); @@ -235,6 +285,39 @@ FROM [dbo].[OrganizationDeleteTask] }; } + private static async Task> QueryRowsByOrganizationIdAsync(string connectionString, Guid organizationId) + { + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = """ + SELECT [Id], [OrganizationId], [TaskType], [CreationDate], [RevisionDate], [StartDate], + [CompletedDate], [ItemsDeletedCount], [FailureCount], [LastError] + FROM [dbo].[OrganizationDeleteTask] + WHERE [OrganizationId] = @OrganizationId + """; + cmd.Parameters.AddWithValue("@OrganizationId", organizationId); + await using var reader = await cmd.ExecuteReaderAsync(); + var tasks = new List(); + while (await reader.ReadAsync()) + { + tasks.Add(new OrganizationDeleteTask + { + Id = reader.GetGuid(0), + OrganizationId = reader.GetGuid(1), + TaskType = (OrganizationDeleteTaskType)reader.GetByte(2), + CreationDate = reader.GetDateTime(3), + RevisionDate = reader.GetDateTime(4), + StartDate = reader.IsDBNull(5) ? null : reader.GetDateTime(5), + CompletedDate = reader.IsDBNull(6) ? null : reader.GetDateTime(6), + ItemsDeletedCount = reader.GetInt64(7), + FailureCount = reader.GetInt32(8), + LastError = reader.IsDBNull(9) ? null : reader.GetString(9), + }); + } + return tasks; + } + private static async Task BackdateRevisionDateAsync(string connectionString, Guid id, int minutes) { await using var connection = new SqlConnection(connectionString); diff --git a/util/Migrator/DbScripts/2026-06-30_00_OrganizationDeleteByIdMultiTaskEnqueue.sql b/util/Migrator/DbScripts/2026-06-30_00_OrganizationDeleteByIdMultiTaskEnqueue.sql new file mode 100644 index 000000000000..578805f2af80 --- /dev/null +++ b/util/Migrator/DbScripts/2026-06-30_00_OrganizationDeleteByIdMultiTaskEnqueue.sql @@ -0,0 +1,235 @@ +-- Generalize the atomic OrganizationDeleteTask enqueue in Organization_DeleteById so +-- any number of task types can be enqueued in the same transaction as the delete. +-- Adds a table-valued parameter; the existing scalar params are retained (defaulted) +-- so callers on the previous server version keep working during a rolling deployment. + +-- User-defined table type carrying the tasks to enqueue. +IF TYPE_ID('[dbo].[OrganizationDeleteTaskArray]') IS NULL +BEGIN + CREATE TYPE [dbo].[OrganizationDeleteTaskArray] AS TABLE ( + [Id] UNIQUEIDENTIFIER NOT NULL, + [TaskType] TINYINT NOT NULL, + [CreationDate] DATETIME2(7) NOT NULL); +END +GO + +CREATE OR ALTER PROCEDURE [dbo].[Organization_DeleteById] + @Id UNIQUEIDENTIFIER, + @OrganizationDeleteTaskId UNIQUEIDENTIFIER = NULL, + @OrganizationDeleteTaskType TINYINT = NULL, + @OrganizationDeleteTaskCreationDate DATETIME2(7) = NULL, + @OrganizationDeleteTasks [dbo].[OrganizationDeleteTaskArray] READONLY +WITH RECOMPILE +AS +BEGIN + SET NOCOUNT ON + + EXEC [dbo].[User_BumpAccountRevisionDateByOrganizationId] @Id + + DECLARE @BatchSize INT = 100 + WHILE @BatchSize > 0 + BEGIN + BEGIN TRANSACTION Organization_DeleteById_Ciphers + + DELETE TOP(@BatchSize) + FROM + [dbo].[Cipher] + WHERE + [UserId] IS NULL + AND [OrganizationId] = @Id + + SET @BatchSize = @@ROWCOUNT + + COMMIT TRANSACTION Organization_DeleteById_Ciphers + END + + BEGIN TRANSACTION Organization_DeleteById + + DELETE + FROM + [dbo].[AuthRequest] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[SsoUser] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[SsoConfig] + WHERE + [OrganizationId] = @Id + + DELETE CU + FROM + [dbo].[CollectionUser] CU + INNER JOIN + [dbo].[OrganizationUser] OU ON [CU].[OrganizationUserId] = [OU].[Id] + WHERE + [OU].[OrganizationId] = @Id + + DELETE AP + FROM + [dbo].[AccessPolicy] AP + INNER JOIN + [dbo].[OrganizationUser] OU ON [AP].[OrganizationUserId] = [OU].[Id] + WHERE + [OU].[OrganizationId] = @Id + + DELETE GU + FROM + [dbo].[GroupUser] GU + INNER JOIN + [dbo].[OrganizationUser] OU ON [GU].[OrganizationUserId] = [OU].[Id] + WHERE + [OU].[OrganizationId] = @Id + + DELETE + FROM + [dbo].[OrganizationUser] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[ProviderOrganization] + WHERE + [OrganizationId] = @Id + + EXEC [dbo].[OrganizationApiKey_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationConnection_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationSponsorship_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationDomain_OrganizationDeleted] @Id + EXEC [dbo].[OrganizationIntegration_OrganizationDeleted] @Id + + DELETE + FROM + [dbo].[Project] + WHERE + [OrganizationId] = @Id + + DELETE + FROM + [dbo].[Secret] + WHERE + [OrganizationId] = @Id + + DELETE AK + FROM + [dbo].[ApiKey] AK + INNER JOIN + [dbo].[ServiceAccount] SA ON [AK].[ServiceAccountId] = [SA].[Id] + WHERE + [SA].[OrganizationId] = @Id + + DELETE AP + FROM + [dbo].[AccessPolicy] AP + INNER JOIN + [dbo].[ServiceAccount] SA ON [AP].[GrantedServiceAccountId] = [SA].[Id] + WHERE + [SA].[OrganizationId] = @Id + + DELETE + FROM + [dbo].[ServiceAccount] + WHERE + [OrganizationId] = @Id + + -- Delete Notification Status + DELETE + NS + FROM + [dbo].[NotificationStatus] NS + INNER JOIN + [dbo].[Notification] N ON N.[Id] = NS.[NotificationId] + WHERE + N.[OrganizationId] = @Id + + -- Delete Notification + DELETE + FROM + [dbo].[Notification] + WHERE + [OrganizationId] = @Id + + -- Delete Organization Application + DELETE + FROM + [dbo].[OrganizationApplication] + WHERE + [OrganizationId] = @Id + + -- Delete Organization Report + DELETE + FROM + [dbo].[OrganizationReport] + WHERE + [OrganizationId] = @Id + + -- Delete Organization Owned Sends + DELETE + FROM + [dbo].[Send] + WHERE + [OrganizationId] = @Id + + -- Atomically enqueue one or more OrganizationDeleteTasks (e.g. for purging Table + -- Storage event logs) so downstream cleanup is durably recorded with the deletion. + -- Preferred path: a set of tasks passed via table-valued parameter, letting any + -- number of teams enqueue their own cleanup type in the same transaction. + IF EXISTS (SELECT 1 FROM @OrganizationDeleteTasks) + BEGIN + INSERT INTO [dbo].[OrganizationDeleteTask] + ( + [Id], + [OrganizationId], + [TaskType], + [CreationDate], + [RevisionDate] + ) + SELECT + [Id], + @Id, + [TaskType], + [CreationDate], + [CreationDate] + FROM + @OrganizationDeleteTasks + END + -- Legacy single-task path. Retained so callers still running the previous server + -- version keep working during a rolling deployment; safe to remove once fully rolled. + ELSE IF @OrganizationDeleteTaskId IS NOT NULL + BEGIN + DECLARE @OrganizationDeleteTaskDate DATETIME2(7) = COALESCE(@OrganizationDeleteTaskCreationDate, SYSUTCDATETIME()) + + INSERT INTO [dbo].[OrganizationDeleteTask] + ( + [Id], + [OrganizationId], + [TaskType], + [CreationDate], + [RevisionDate] + ) + VALUES + ( + @OrganizationDeleteTaskId, + @Id, + @OrganizationDeleteTaskType, + @OrganizationDeleteTaskDate, + @OrganizationDeleteTaskDate + ) + END + + DELETE + FROM + [dbo].[Organization] + WHERE + [Id] = @Id + + COMMIT TRANSACTION Organization_DeleteById +END +GO