diff --git a/NotificationService/GDPRNotificationMappingProcessor/.gitignore b/NotificationService/GDPRNotificationMappingProcessor/.gitignore new file mode 100644 index 0000000..ff5b00c --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/NotificationService/GDPRNotificationMappingProcessor/Constants.cs b/NotificationService/GDPRNotificationMappingProcessor/Constants.cs new file mode 100644 index 0000000..e74a38e --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/Constants.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace GDPRNotificationMappingProcessor +{ + + /// + /// class for constant literals. + /// + public sealed class Constants + { + /// + /// StorageType. + /// + public const string StorageType = "StorageType"; + + /// + /// Seconds to wait between attempts at polling the Azure KeyVault for changes in configuration. + /// + public const string KeyVaultConfigRefreshDurationSeconds = "KeyVaultConfigRefreshDurationSeconds"; + } +} diff --git a/NotificationService/GDPRNotificationMappingProcessor/GDPRNotificationMappingProcessor.cs b/NotificationService/GDPRNotificationMappingProcessor/GDPRNotificationMappingProcessor.cs new file mode 100644 index 0000000..2898b35 --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/GDPRNotificationMappingProcessor.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace GDPRNotificationMappingProcessor +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.WebJobs; + using Microsoft.Extensions.Configuration; + using Microsoft.OData.Edm.Vocabularies; + using Newtonsoft.Json; + using NotificationService.Common.Logger; + using NotificationService.Contracts; + using NotificationService.Contracts.Models.GDPR; + using NotificationService.Data; + using NotificationService.Data.Interfaces; + + /// + /// Function to process notification messages and create emailId and NotificationId mapping for GDPR scrubbing. + /// + public class GDPRNotificationMappingProcessor + { + /// + /// The logger. + /// + private readonly ILogger logger; + + /// + /// Instance of . + /// + private readonly IEmailNotificationRepository emailNotificationRepository; + + /// + /// Instance of Application Configuration. + /// + private readonly IConfiguration configuration; + + /// + /// Enum to specify type of database. + /// + private readonly StorageType repo; + + /// + /// Initializes a new instance of the class. + /// + /// The log. + /// The configuration. + /// The repositoryFactory. + public GDPRNotificationMappingProcessor( + ILogger logger, + IConfiguration configuration, + IRepositoryFactory repositoryFactory) + { + this.logger = logger; + this.configuration = configuration; + this.emailNotificationRepository = repositoryFactory.GetRepository(Enum.TryParse(this.configuration?[Constants.StorageType], out this.repo) ? this.repo : throw new Exception()); + } + + /// + /// Trigger method invoked when a notification item is added to the queue. + /// + /// Serialized queue item. + [FunctionName("GDPRNotificationMappingProcessor")] + public void Run([QueueTrigger("%GdprNotifEmailMapQueueName%", Connection = "AzureWebJobsStorage")]string message) + { + if (string.IsNullOrEmpty(message)) + { + return; + } + + this.logger.TraceInformation($"Started processing notificaiton to create emailid-notificationid mapping."); + NotificationMappingQueueItem item = JsonConvert.DeserializeObject(message); + NotificationType notifType = (NotificationType)Enum.Parse(typeof(NotificationType), item.NotificationType); + var payloadJson = item.Payload?.ToString(); + + if (string.IsNullOrEmpty(payloadJson)) + { + this.logger.TraceError($"Notification Payload for type {item.NotificationType} can't be null or empty"); + return; + } + + this.logger.TraceInformation($"processing notification mapping for notification type {item.NotificationType}"); + if (notifType == NotificationType.Mail) + { + var entities = JsonConvert.DeserializeObject>(payloadJson); + this.emailNotificationRepository.CreateEmailIdNotificationMappingForEmail(entities, item.ApplicationName); + } + else + { + var entities = JsonConvert.DeserializeObject>(payloadJson); + this.emailNotificationRepository.CreateEmailIdNotificationMappingForMeetingInvite(entities, item.ApplicationName); + } + + this.logger.TraceInformation($"Finished processing notificaiton for type {item.NotificationType} to create emailid-notificationid mapping."); + } + } +} diff --git a/NotificationService/GDPRNotificationMappingProcessor/GDPRNotificationMappingProcessor.csproj b/NotificationService/GDPRNotificationMappingProcessor/GDPRNotificationMappingProcessor.csproj new file mode 100644 index 0000000..9e60b93 --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/GDPRNotificationMappingProcessor.csproj @@ -0,0 +1,38 @@ + + + netcoreapp3.1 + v3 + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + Always + + + PreserveNewest + + + PreserveNewest + Never + + + diff --git a/NotificationService/GDPRNotificationMappingProcessor/Startup.cs b/NotificationService/GDPRNotificationMappingProcessor/Startup.cs new file mode 100644 index 0000000..3892ff7 --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/Startup.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Azure.Functions.Extensions.DependencyInjection; + +[assembly: FunctionsStartup(typeof(GDPRNotificationMappingProcessor.Startup))] + +namespace GDPRNotificationMappingProcessor +{ + using System; + using System.IO; + using System.Reflection; + using Microsoft.ApplicationInsights.AspNetCore; + using Microsoft.ApplicationInsights.DataContracts; + using Microsoft.ApplicationInsights.DependencyCollector; + using Microsoft.ApplicationInsights.Extensibility; + using Microsoft.Extensions.Configuration; + using Microsoft.Extensions.Configuration.AzureAppConfiguration; + using Microsoft.Extensions.Configuration.AzureKeyVault; + using Microsoft.Extensions.DependencyInjection; + using NotificationService.Common; + using NotificationService.Common.Configurations; + using NotificationService.Common.Logger; + using NotificationService.Data; + using NotificationService.Data.Interfaces; + using NotificationService.Data.Repositories; + + /// + /// Startup. + /// + public class Startup : FunctionsStartup + { + /// + /// Gets the maxDequeueCount from host.json. + /// + public static IConfigurationSection MaxDequeueCount { get; private set; } + + /// + /// Gets the application configuration. + /// + public IConfiguration Configuration { get; } + + /// + public override void Configure(IFunctionsHostBuilder builder) + { + var azureFuncConfig = builder?.Services?.BuildServiceProvider()?.GetService(); + var configBuilder = new ConfigurationBuilder(); + _ = configBuilder.AddConfiguration(azureFuncConfig); + var configFolder = Directory.GetParent(Assembly.GetExecutingAssembly().Location).Parent?.FullName; + _ = configBuilder.SetBasePath(configFolder); + _ = configBuilder.AddJsonFile("functionSettings.json"); + _ = configBuilder.AddEnvironmentVariables(); + + var configuration = configBuilder.Build(); + MaxDequeueCount = configuration.GetSection(ConfigConstants.MaxDequeueCountConfigKey); + + AzureKeyVaultConfigurationOptions azureKeyVaultConfigurationOptions = new AzureKeyVaultConfigurationOptions(configuration[ConfigConstants.KeyVaultUrlConfigKey]) + { + ReloadInterval = TimeSpan.FromSeconds(double.Parse(configuration[Constants.KeyVaultConfigRefreshDurationSeconds])), + }; + _ = configBuilder.AddAzureKeyVault(azureKeyVaultConfigurationOptions); + configuration = configBuilder.Build(); + _ = configBuilder.AddAzureAppConfiguration(options => + { + var settings = options.Connect(configuration[ConfigConstants.AzureAppConfigConnectionstringConfigKey]) + .Select(KeyFilter.Any, "Common").Select(KeyFilter.Any, "QueueProcessor"); + _ = settings.ConfigureRefresh(refreshOptions => + { + _ = refreshOptions.Register(key: configuration[ConfigConstants.ForceRefreshConfigKey], refreshAll: true, label: LabelFilter.Null); + }); + }); + + configuration = configBuilder.Build(); + + ITelemetryInitializer[] itm = new ITelemetryInitializer[1]; + var envInitializer = new EnvironmentInitializer + { + Service = configuration[AIConstants.ServiceConfigName], + ServiceLine = configuration[AIConstants.ServiceLineConfigName], + ServiceOffering = configuration[AIConstants.ServiceOfferingConfigName], + ComponentId = configuration[AIConstants.ComponentIdConfigName], + ComponentName = configuration[AIConstants.ComponentNameConfigName], + EnvironmentName = configuration[AIConstants.EnvironmentName], + IctoId = "IctoId", + }; + itm[0] = envInitializer; + LoggingConfiguration loggingConfiguration = new LoggingConfiguration + { + IsTraceEnabled = true, + TraceLevel = (SeverityLevel)Enum.Parse(typeof(SeverityLevel), configuration[ConfigConstants.AITraceLelelConfigKey]), + EnvironmentName = configuration[AIConstants.EnvironmentName], + }; + + var tconfig = TelemetryConfiguration.CreateDefault(); + tconfig.InstrumentationKey = configuration[ConfigConstants.AIInsrumentationConfigKey]; + + DependencyTrackingTelemetryModule depModule = new DependencyTrackingTelemetryModule(); + depModule.Initialize(tconfig); + + RequestTrackingTelemetryModule requestTrackingTelemetryModule = new RequestTrackingTelemetryModule(); + requestTrackingTelemetryModule.Initialize(tconfig); + + _ = builder.Services.AddSingleton(_ => new AILogger(loggingConfiguration, tconfig, itm)); + + StorageType storageType = (StorageType)Enum.Parse(typeof(StorageType), configuration?[ConfigConstants.StorageType]); + if (storageType == StorageType.DocumentDB) + { + _ = builder.Services.Configure(configuration.GetSection(ConfigConstants.CosmosDBConfigSectionKey)); + _ = builder.Services.Configure(s => s.Key = configuration[ConfigConstants.CosmosDBKeyConfigKey]); + _ = builder.Services.Configure(s => s.Uri = configuration[ConfigConstants.CosmosDBURIConfigKey]); + _ = builder.Services.AddScoped(); + _ = builder.Services.AddSingleton(); + _ = builder.Services.AddScoped(); + _ = builder.Services.AddScoped(s => s.GetService()); + } + + _ = builder.Services.Configure(configuration.GetSection(ConfigConstants.StorageAccountConfigSectionKey)); + _ = builder.Services.Configure(s => s.ConnectionString = configuration[ConfigConstants.StorageAccountConnectionStringConfigKey]); + _ = builder.Services.AddSingleton(configuration); + _ = builder.Services.AddScoped(); + _ = builder.Services.AddScoped(); + _ = builder.Services.AddScoped(s => s.GetService()); + _ = builder.Services.AddScoped(); + } + } +} diff --git a/NotificationService/GDPRNotificationMappingProcessor/functionSettings.json b/NotificationService/GDPRNotificationMappingProcessor/functionSettings.json new file mode 100644 index 0000000..a99506e --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/functionSettings.json @@ -0,0 +1,7 @@ +{ + "KeyVaultConfigRefreshDurationSeconds": "120", + "KeyVaultUrl": "__KeyVaultUrl__", + "AppConfig": { + "ForceRefresh": "ForceRefresh" + } +} \ No newline at end of file diff --git a/NotificationService/GDPRNotificationMappingProcessor/host.json b/NotificationService/GDPRNotificationMappingProcessor/host.json new file mode 100644 index 0000000..d767914 --- /dev/null +++ b/NotificationService/GDPRNotificationMappingProcessor/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "functionTimeout": "-1", + "extensions": { + "queues": { + "maxPollingInterval": "00:00:01", + "visibilityTimeout": "00:00:30", + "batchSize": 1, + "maxDequeueCount": 5 + } + } +} diff --git a/NotificationService/NotificationHandler/Startup.cs b/NotificationService/NotificationHandler/Startup.cs index 37c15b1..e8700e9 100644 --- a/NotificationService/NotificationHandler/Startup.cs +++ b/NotificationService/NotificationHandler/Startup.cs @@ -76,7 +76,8 @@ public void ConfigureServices(IServiceCollection services) s.GetService(), s.GetService(), s.GetService(), - s.GetService())) + s.GetService(), + s.GetService())) .AddScoped(s => new EmailHandlerManager( this.Configuration, diff --git a/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailHandlerManager.cs b/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailHandlerManager.cs index 87f7d0f..d21552c 100644 --- a/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailHandlerManager.cs +++ b/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailHandlerManager.cs @@ -12,6 +12,7 @@ namespace NotificationService.BusinessLibrary.Business.V1 using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; + using Newtonsoft.Json; using NotificationService.BusinessLibrary.Interfaces; using NotificationService.Common; using NotificationService.Common.Configurations; @@ -19,6 +20,7 @@ namespace NotificationService.BusinessLibrary.Business.V1 using NotificationService.Contracts; using NotificationService.Contracts.Entities; using NotificationService.Contracts.Models; + using NotificationService.Contracts.Models.GDPR; using NotificationService.Contracts.Models.Request; using NotificationService.Data; @@ -130,6 +132,8 @@ public async Task> QueueEmailNotifications(string ap this.logger.TraceVerbose($"Completed {nameof(this.cloudStorageClient.QueueCloudMessages)} method of {nameof(EmailHandlerManager)}.", traceProps); } + _ = Task.Run(async () => await this.emailManager.QueueEmailNotificaitionMapping(applicationName, entitiesToQueue, traceProps).ConfigureAwait(false)); + var responses = this.emailManager.NotificationEntitiesToResponse(notificationResponses, notificationItemEntities); this.logger.TraceInformation($"Completed {nameof(this.QueueEmailNotifications)} method of {nameof(EmailHandlerManager)}.", traceProps); result = true; @@ -151,16 +155,7 @@ public async Task> QueueEmailNotifications(string ap } } - /// - /// Queue email notification items. - /// - /// Application sourcing the email notification. - /// Array of email notification items. - /// - /// A representing the result of the asynchronous operation. - /// - /// Application Name cannot be null or empty. - applicationName. - /// meetingNotificationItems. + /// public async Task> QueueMeetingNotifications(string applicationName, MeetingNotificationItem[] meetingNotificationItems) { var stopwatch = new Stopwatch(); @@ -210,6 +205,8 @@ public async Task> QueueMeetingNotifications(string this.logger.TraceVerbose($"Completed {nameof(this.cloudStorageClient.QueueCloudMessages)} method of {nameof(EmailHandlerManager)}.", traceProps); } + _ = Task.Run(async () => await this.emailManager.QueueMeetingNotificactionMapping(applicationName, entitiesToQueue, traceProps).ConfigureAwait(false)); + var responses = this.emailManager.NotificationEntitiesToResponse(notificationResponses, notificationItemEntities); this.logger.TraceInformation($"Completed {nameof(this.QueueMeetingNotifications)} method of {nameof(EmailHandlerManager)}.", traceProps); result = true; diff --git a/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailManager.cs b/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailManager.cs index f4c3752..00e44a3 100644 --- a/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailManager.cs +++ b/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailManager.cs @@ -9,6 +9,7 @@ namespace NotificationService.BusinessLibrary using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; + using Newtonsoft.Json; using NotificationService.BusinessLibrary.Interfaces; using NotificationService.Common.Configurations; using NotificationService.Common.Logger; @@ -16,6 +17,7 @@ namespace NotificationService.BusinessLibrary using NotificationService.Contracts.Entities; using NotificationService.Contracts.Extensions; using NotificationService.Contracts.Models; + using NotificationService.Contracts.Models.GDPR; using NotificationService.Contracts.Models.Request; using NotificationService.Data; using NotificationService.Data.Interfaces; @@ -60,6 +62,11 @@ public class EmailManager : IEmailManager /// private readonly ITemplateMerge templateMerge; + /// + /// instance of . + /// + private readonly ICloudStorageClient cloudStorageClient; + /// /// Initializes a new instance of the class. /// @@ -73,7 +80,8 @@ public EmailManager( IRepositoryFactory repositoryFactory, ILogger logger, IMailTemplateManager templateManager, - ITemplateMerge templateMerge) + ITemplateMerge templateMerge, + ICloudStorageClient cloudStorageClient) { this.repositoryFactory = repositoryFactory; this.configuration = configuration; @@ -81,6 +89,7 @@ public EmailManager( this.logger = logger; this.templateManager = templateManager; this.templateMerge = templateMerge; + this.cloudStorageClient = cloudStorageClient; } /// @@ -313,5 +322,90 @@ public async Task> GetEmailNotificationsByDat { return await this.emailNotificationRepository.GetPendingOrFailedEmailNotificationsByDateRange(dateRange, applicationName, statusList).ConfigureAwait(false); } + + /// + public async Task QueueEmailNotificaitionMapping(string applicationName, List> notificationEntities, IDictionary traceProps) + { + this.logger.TraceInformation($"Started {nameof(this.QueueEmailNotificaitionMapping)} method of {nameof(EmailManager)}.", traceProps); + + if (notificationEntities == null || notificationEntities.Count == 0) + { + return; + } + + var isGdprEnabled = (bool)this.configuration.GetValue(typeof(bool), ConfigConstants.IsGDPREnabled); + if (!isGdprEnabled) + { + this.logger.TraceInformation($"GDPR scrub functionality is switched off Email Notification", traceProps); + return; + } + + var gdprMapQueue = this.configuration?[$"{ConfigConstants.StorageAccGdprMapQueueName}"]; + var cloudQueue = this.cloudStorageClient.GetCloudQueue(gdprMapQueue); + foreach (var item in notificationEntities) + { + var payload = item.Select(notificationItem => new EmailNotificationQueueItem() + { + BCC = notificationItem.BCC, + CC = notificationItem.CC, + From = notificationItem.From, + To = notificationItem.To, + NotificationId = notificationItem.NotificationId, + }); + var queueItem = new NotificationMappingQueueItem() + { + NotificationType = NotificationType.Mail.ToString(), + ApplicationName = applicationName, + Payload = payload, + }; + + var cloudMessage = JsonConvert.SerializeObject(queueItem); + await this.cloudStorageClient.QueueCloudMessages(cloudQueue, new List() { cloudMessage }).ConfigureAwait(false); + } + + this.logger.TraceInformation($"Finished {nameof(this.QueueEmailNotificaitionMapping)} method of {nameof(EmailManager)}.", traceProps); + } + + /// + public async Task QueueMeetingNotificactionMapping(string applicationName, List> notificationEntities, IDictionary traceProps) + { + this.logger.TraceInformation($"Started {nameof(this.QueueMeetingNotificactionMapping)} method of {nameof(EmailManager)}.", traceProps); + + if (notificationEntities == null || notificationEntities.Count == 0) + { + return; + } + + var isGdprEnabled = (bool)this.configuration.GetValue(typeof(bool), ConfigConstants.IsGDPREnabled); + if (!isGdprEnabled) + { + this.logger.TraceInformation($"GDPR scrub functionality is switched off for Meeting Notification", traceProps); + return; + } + + var gdprMapQueue = this.configuration?[$"{ConfigConstants.StorageAccGdprMapQueueName}"]; + var cloudQueue = this.cloudStorageClient.GetCloudQueue(gdprMapQueue); + foreach (var item in notificationEntities) + { + var payload = item.Select(notificationItem => new MeetingNotificationQueueItem() + { + From = notificationItem.From, + NotificationId = notificationItem.NotificationId, + RequiredAttendees = notificationItem.RequiredAttendees, + OptionalAttendees = notificationItem.OptionalAttendees, + }); + var queueItem = new NotificationMappingQueueItem() + { + NotificationType = NotificationType.Meet.ToString(), + ApplicationName = applicationName, + Payload = payload, + }; + + var cloudMessage = JsonConvert.SerializeObject(queueItem); + await this.cloudStorageClient.QueueCloudMessages(cloudQueue, new List() { cloudMessage }).ConfigureAwait(false); + } + + this.logger.TraceInformation($"Finished {nameof(this.QueueMeetingNotificactionMapping)} method of {nameof(EmailManager)}.", traceProps); + } } } diff --git a/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailServiceManager.cs b/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailServiceManager.cs index 7053427..31b9c91 100644 --- a/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailServiceManager.cs +++ b/NotificationService/NotificationService.BusinessLibrary/Business/v1/EmailServiceManager.cs @@ -180,6 +180,9 @@ public async Task> SendMeetingInvites(string applica IList notificationResponses = new List(); IList emailNotificationEntities = await this.emailManager.CreateMeetingNotificationEntities(applicationName, meetingInviteItems, NotificationItemStatus.Processing).ConfigureAwait(false); IList notificationEntities = await this.emailNotificationRepository.GetMeetingNotificationItemEntities(emailNotificationEntities.Select(e => e.NotificationId).ToList(), applicationName).ConfigureAwait(false); + + _ = Task.Run(async () => await this.emailManager.QueueMeetingNotificactionMapping(applicationName, new List>() { notificationEntities.ToList() }, null).ConfigureAwait(false)); + var notificationItemEntities = await this.ProcessMeetingNotificationEntities(applicationName, notificationEntities).ConfigureAwait(false); var responses = this.emailManager.NotificationEntitiesToResponse(notificationResponses, notificationItemEntities); this.logger.TraceInformation($"Finished {nameof(this.SendMeetingInvites)} method of {nameof(EmailServiceManager)}."); @@ -243,6 +246,9 @@ private async Task> SendNotificationsUsingPro this.logger.TraceInformation($"Started {nameof(this.SendNotificationsUsingProvider)} method of {nameof(EmailServiceManager)}."); IList emailNotificationEntities = await this.emailManager.CreateNotificationEntities(applicationName, emailNotificationItems, NotificationItemStatus.Processing).ConfigureAwait(false); IList notificationEntities = await this.emailNotificationRepository.GetEmailNotificationItemEntities(emailNotificationEntities.Select(e => e.NotificationId).ToList(), applicationName).ConfigureAwait(false); + + _ = Task.Run(async () => await this.emailManager.QueueEmailNotificaitionMapping(applicationName, new List>() { notificationEntities.ToList() }, null).ConfigureAwait(false) ); + var retEntities = await this.ProcessNotificationEntities(applicationName, notificationEntities).ConfigureAwait(false); this.logger.TraceInformation($"Finished {nameof(this.SendNotificationsUsingProvider)} method of {nameof(EmailServiceManager)}."); return retEntities; diff --git a/NotificationService/NotificationService.BusinessLibrary/Interfaces/IEmailManager.cs b/NotificationService/NotificationService.BusinessLibrary/Interfaces/IEmailManager.cs index 637daf7..86eb70d 100644 --- a/NotificationService/NotificationService.BusinessLibrary/Interfaces/IEmailManager.cs +++ b/NotificationService/NotificationService.BusinessLibrary/Interfaces/IEmailManager.cs @@ -73,5 +73,23 @@ public interface IEmailManager /// Status List of Notification items. /// A . Task> GetEmailNotificationsByDateRangeAndStatus(string applicationName, DateTimeRange dateRange, List statusList); + + /// + /// Queue Email Notification Messages to GDPR Mapping queue. + /// + /// Application Name. + /// notification entites to be queued. + /// telemetry trace properties to log. + /// A representing the asynchronous operation.> + Task QueueEmailNotificaitionMapping(string applicationName, List> notificationEntities, IDictionary traceProps); + + /// + /// Queue Meeting Notification Messages to GDPR Mapping queue. + /// + /// Application Name. + /// meeting notification entites to be queued. + /// telemetry trace properties to log. + /// A representing the asynchronous operation.> + Task QueueMeetingNotificactionMapping(string applicationName, List> notificationEntities, IDictionary traceProps); } } diff --git a/NotificationService/NotificationService.Common/Configurations/ConfigConstants.cs b/NotificationService/NotificationService.Common/Configurations/ConfigConstants.cs index 10ab15c..877d8e7 100644 --- a/NotificationService/NotificationService.Common/Configurations/ConfigConstants.cs +++ b/NotificationService/NotificationService.Common/Configurations/ConfigConstants.cs @@ -208,12 +208,31 @@ public static class ConfigConstants #pragma warning restore CA2211 // Non-constant fields should not be visible /// - /// A constant for DIrectSend SMTPServer config key from appsetting.json. + /// A constant for DirectSend SMTPServer config key from appsetting.json. /// #pragma warning disable CA2211 // Non-constant fields should not be visible #pragma warning disable SA1401 // Fields should be private public static string DirectSendSMTPServerConfigKey = $"{DirectSendSettingConfigSectionKey}:SmtpServer"; #pragma warning restore SA1401 // Fields should be private #pragma warning restore CA2211 // Non-constant fields should not be visible + + /// + /// A constant for GDPR queueName. + /// +#pragma warning disable CA2211 // Non-constant fields should not be visible +#pragma warning disable SA1401 // Fields should be private + public static string StorageAccGdprMapQueueName = $"{StorageAccountConfigSectionKey}:GdprMapQueueName"; +#pragma warning restore SA1401 // Fields should be private +#pragma warning restore CA2211 // Non-constant fields should not be visible + + /// + /// A constant for GDPR enabled flag. + /// +#pragma warning disable CA2211 // Non-constant fields should not be visible +#pragma warning disable SA1401 // Fields should be private + public static string IsGDPREnabled = "IsGDPREnabled"; +#pragma warning restore SA1401 // Fields should be private +#pragma warning restore CA2211 // Non-constant fields should not be visible + } } diff --git a/NotificationService/NotificationService.Common/Configurations/StorageAccountSetting.cs b/NotificationService/NotificationService.Common/Configurations/StorageAccountSetting.cs index 7c7d4bd..be314d6 100644 --- a/NotificationService/NotificationService.Common/Configurations/StorageAccountSetting.cs +++ b/NotificationService/NotificationService.Common/Configurations/StorageAccountSetting.cs @@ -37,5 +37,15 @@ public class StorageAccountSetting /// Gets or sets the Notification queue name. /// public string NotificationQueueName { get; set; } + + /// + /// Gets or sets the Email-Notification mapping TableName. + /// + public string EmailNotificationMapTableName { get; set; } + + /// + /// Gets or sets the Meeting-Notification mapping TableName. + /// + public string MeetingNotificationMapTableName { get; set; } } } diff --git a/NotificationService/NotificationService.Contracts/Entities/EmailNotificationMapEntity.cs b/NotificationService/NotificationService.Contracts/Entities/EmailNotificationMapEntity.cs new file mode 100644 index 0000000..b0185fd --- /dev/null +++ b/NotificationService/NotificationService.Contracts/Entities/EmailNotificationMapEntity.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NotificationService.Contracts.Entities +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using Microsoft.Azure.Cosmos.Table; + + /// + /// Storage Entity to store EmailId and NotificationId mapping for GDPR implementation. + /// + public class EmailNotificationMapEntity : TableEntity + { + /// + /// Gets or Sets ApplicationName. + /// + public string ApplicationName { get; set; } + + /// + /// Gets or sets a value indicating whether gets or Sets scrubbed status. + /// + public bool IsScrubbed { get; set; } + + /// + /// Gets or Sets CreateDateTime. + /// + public DateTime CreateDateTime { get; set; } + + /// + /// Gets or Sets UpdateDateTime. + /// + public DateTime UpdateDateTime { get; set; } + + /// + /// Comparer for EmailNotificaitonMapEntity. + /// + public class EmailNotificaitonMapEntityComparer : IEqualityComparer + { + /// + /// Equals comparison of two objects. + /// + /// first object to compare. + /// second object to compare. + /// Returns bool value as comparision result. + public bool Equals(EmailNotificationMapEntity obj1, EmailNotificationMapEntity obj2) + { + return obj1 != null && obj2 != null && obj1.RowKey == obj2.RowKey && obj1.PartitionKey == obj2.PartitionKey; + } + + /// + /// Generate HashCode for given Object. + /// + /// Object for which hashcode is generated. + /// Returns hascode for the input object. + public int GetHashCode([DisallowNull] EmailNotificationMapEntity obj) + { + if (obj == null) + { + return 0; + } + + int hash = 17; + if (!string.IsNullOrEmpty(obj.PartitionKey)) + { + hash = (hash * 23) ^ obj.PartitionKey.GetHashCode(StringComparison.InvariantCulture); + } + + if (!string.IsNullOrEmpty(obj.RowKey)) + { + hash = (hash * 23) ^ obj.RowKey.GetHashCode(StringComparison.InvariantCulture); + } + + return hash; + } + } + } +} diff --git a/NotificationService/NotificationService.Contracts/Models/GDPR/EmailNotificationQueueItem.cs b/NotificationService/NotificationService.Contracts/Models/GDPR/EmailNotificationQueueItem.cs new file mode 100644 index 0000000..ee240d8 --- /dev/null +++ b/NotificationService/NotificationService.Contracts/Models/GDPR/EmailNotificationQueueItem.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NotificationService.Contracts.Models.GDPR +{ + /// + /// Email Notification queue item entity. + /// + public class EmailNotificationQueueItem : NotificationQueueItem + { + /// + /// Gets or Sets To. + /// + public string To { get; set; } + + /// + /// Gets or Sets CC. + /// + public string CC { get; set; } + + /// + /// Gets or Sets BCC. + /// + public string BCC { get; set; } + } +} diff --git a/NotificationService/NotificationService.Contracts/Models/GDPR/MeetingNotificationQueueItem.cs b/NotificationService/NotificationService.Contracts/Models/GDPR/MeetingNotificationQueueItem.cs new file mode 100644 index 0000000..1c3d076 --- /dev/null +++ b/NotificationService/NotificationService.Contracts/Models/GDPR/MeetingNotificationQueueItem.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NotificationService.Contracts.Models.GDPR +{ + /// + /// Meeting Notification queue item entity. + /// + public class MeetingNotificationQueueItem : NotificationQueueItem + { + /// + /// Gets or Sets RequiredAttendees. + /// + public string RequiredAttendees { get; set; } + + /// + /// Gets or Sets OptionalAttendees. + /// + public string OptionalAttendees { get; set; } + } +} diff --git a/NotificationService/NotificationService.Contracts/Models/GDPR/NotificationMappingQueueItem.cs b/NotificationService/NotificationService.Contracts/Models/GDPR/NotificationMappingQueueItem.cs new file mode 100644 index 0000000..57ae242 --- /dev/null +++ b/NotificationService/NotificationService.Contracts/Models/GDPR/NotificationMappingQueueItem.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NotificationService.Contracts.Models.GDPR +{ + /// + /// Class for adding notification to queue for creating email-notification mapping. + /// + public class NotificationMappingQueueItem + { + /// + /// Gets or Sets Notification Type. + /// + public string NotificationType { get; set; } + + /// + /// Gets or Sets payload. + /// + public dynamic Payload { get; set; } + + /// + /// Gets or Sets Application Name. + /// + public string ApplicationName { get; set; } + } +} diff --git a/NotificationService/NotificationService.Contracts/Models/GDPR/NotificationQueueItem.cs b/NotificationService/NotificationService.Contracts/Models/GDPR/NotificationQueueItem.cs new file mode 100644 index 0000000..74e2fcc --- /dev/null +++ b/NotificationService/NotificationService.Contracts/Models/GDPR/NotificationQueueItem.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace NotificationService.Contracts.Models.GDPR +{ + /// + /// Notification queue base entity item. + /// + public class NotificationQueueItem + { + /// + /// Gets or Sets NotificaitonId. + /// + public string NotificationId { get; set; } + + /// + /// Gets or Sets From field. + /// + public string From { get; set; } + } +} diff --git a/NotificationService/NotificationService.Data/Interfaces/IEmailNotificationRepository.cs b/NotificationService/NotificationService.Data/Interfaces/IEmailNotificationRepository.cs index 5031cad..c2c4f7d 100644 --- a/NotificationService/NotificationService.Data/Interfaces/IEmailNotificationRepository.cs +++ b/NotificationService/NotificationService.Data/Interfaces/IEmailNotificationRepository.cs @@ -8,6 +8,7 @@ namespace NotificationService.Data using System.Threading.Tasks; using NotificationService.Contracts; using NotificationService.Contracts.Entities; + using NotificationService.Contracts.Models.GDPR; using NotificationService.Contracts.Models.Request; /// @@ -101,5 +102,19 @@ public interface IEmailNotificationRepository /// by default it is false. if false it will not populate body, attachments etc. /// A represents the return of the asynchronous operation. Task> GetPendingOrFailedEmailNotificationsByDateRange(DateTimeRange dateRange, string applicationName, List statusList, bool loadBody = false); + + /// + /// Create EmailId - NotificationId mapping in for email Notifications. + /// + /// List of notifications. + /// Application Name. + void CreateEmailIdNotificationMappingForEmail(IList notifications, string applicationName); + + /// + /// Create EmailId - NotificationId mapping in for meeting Notifications. + /// + /// List of notifications. + /// Application Name. + void CreateEmailIdNotificationMappingForMeetingInvite(IList notifications, string applicationName); } } diff --git a/NotificationService/NotificationService.Data/Repositories/EmailNotificationRepository.cs b/NotificationService/NotificationService.Data/Repositories/EmailNotificationRepository.cs index 7f99dec..c3e8ff7 100644 --- a/NotificationService/NotificationService.Data/Repositories/EmailNotificationRepository.cs +++ b/NotificationService/NotificationService.Data/Repositories/EmailNotificationRepository.cs @@ -18,6 +18,7 @@ namespace NotificationService.Data using NotificationService.Contracts; using NotificationService.Contracts.Entities; using NotificationService.Contracts.Extensions; + using NotificationService.Contracts.Models.GDPR; using NotificationService.Contracts.Models.Request; /// @@ -741,5 +742,12 @@ private Expression> GetMeeting return filterExpression; } + + /// + public void CreateEmailIdNotificationMappingForEmail(IList notifications, string applicationName) => throw new NotImplementedException(); + + /// + public void CreateEmailIdNotificationMappingForMeetingInvite(IList notifications, string applicationName) => throw new NotImplementedException(); + } } diff --git a/NotificationService/NotificationService.Data/Repositories/TableStorageEmailRepository.cs b/NotificationService/NotificationService.Data/Repositories/TableStorageEmailRepository.cs index 7cc3c04..36b3fc1 100644 --- a/NotificationService/NotificationService.Data/Repositories/TableStorageEmailRepository.cs +++ b/NotificationService/NotificationService.Data/Repositories/TableStorageEmailRepository.cs @@ -16,6 +16,7 @@ namespace NotificationService.Data.Repositories using NotificationService.Contracts; using NotificationService.Contracts.Entities; using NotificationService.Contracts.Extensions; + using NotificationService.Contracts.Models.GDPR; using NotificationService.Contracts.Models.Request; /// @@ -53,6 +54,11 @@ public class TableStorageEmailRepository : IEmailNotificationRepository /// private readonly IMailAttachmentRepository mailAttachmentRepository; + /// + /// MAX Retry for inserting records in storage table. + /// + private const int MaxRetries = 3; + /// /// Initializes a new instance of the class. /// @@ -485,6 +491,155 @@ private static string GetStatus(int status) return statusStr; } + /// + public void CreateEmailIdNotificationMappingForEmail(IList notifications, string applicationName) + { + var traceProps = new Dictionary(); + if (notifications == null || notifications.Count == 0) + { + return; + } + + traceProps[AIConstants.EmailNotificationCount] = notifications.Count + string.Empty; + traceProps[AIConstants.NotificationType] = NotificationType.Mail.ToString(); + + this.logger.TraceInformation($"Started {nameof(this.CreateEmailIdNotificationMappingForEmail)} method of {nameof(TableStorageEmailRepository)}", traceProps); + var tableName = this.storageAccountSetting?.EmailNotificationMapTableName; + + if (string.IsNullOrEmpty(tableName)) + { + this.logger.TraceError($"Storage Table not confifured for GDPR email to notification mapping.", traceProps); + throw new ArgumentNullException(tableName, "Storage Table not confifured for GDPR email to notification mapping."); + } + + var cloudTable = this.cloudStorageClient.GetCloudTable(tableName); + List list = new List(); + foreach (var notification in notifications) + { + this.CreateMappingEntity(list, notification.From, notification.NotificationId, applicationName); + this.CreateMappingEntity(list, notification.To, notification.NotificationId, applicationName); + this.CreateMappingEntity(list, notification.CC, notification.NotificationId, applicationName); + this.CreateMappingEntity(list, notification.BCC, notification.NotificationId, applicationName); + } + + var listWithoutDuplicates = list.ToHashSet(new EmailNotificationMapEntity.EmailNotificaitonMapEntityComparer()).ToList(); + IList tasks = new List(); + + foreach (var item in listWithoutDuplicates) + { + tasks.Add(Task.Run(() => { + int count = 0; + TableResult result = null; + do + { + count++; + try + { + TableOperation insertOperation = TableOperation.InsertOrMerge(item); + result = cloudTable.ExecuteAsync(insertOperation).GetAwaiter().GetResult(); + } catch (Exception ex) + { + this.logger.TraceInformation($"Exception while inserting email-notification mapping records {item.PartitionKey}, exception : {ex}"); + } + } while (result == null && count < MaxRetries); + })); + } + + _ = Task.WhenAll(tasks: tasks.ToArray()); + + this.logger.TraceInformation($"Finished {nameof(this.CreateEmailIdNotificationMappingForEmail)} method of {nameof(TableStorageEmailRepository)}", traceProps); + } + + /// + public void CreateEmailIdNotificationMappingForMeetingInvite(IList notifications, string applicationName) + { + var traceProps = new Dictionary(); + if (notifications == null || notifications.Count == 0) + { + return; + } + + traceProps[AIConstants.EmailNotificationCount] = notifications.Count + string.Empty; + traceProps[AIConstants.NotificationType] = NotificationType.Meet.ToString(); + + this.logger.TraceInformation($"Started {nameof(this.CreateEmailIdNotificationMappingForMeetingInvite)} method of {nameof(TableStorageEmailRepository)}", traceProps); + var tableName = this.storageAccountSetting?.MeetingNotificationMapTableName; + + if (string.IsNullOrEmpty(tableName)) + { + this.logger.TraceError($"Storage Table not confifured for GDPR meeting emails to notification mapping.", traceProps); + throw new ArgumentNullException(tableName, "Storage Table not confifured for GDPR meeting emails to notification mapping."); + } + + var cloudTable = this.cloudStorageClient.GetCloudTable(tableName); + List list = new List(); + foreach (var notification in notifications) + { + this.CreateMappingEntity(list, notification.From, notification.NotificationId, applicationName); + this.CreateMappingEntity(list, notification.RequiredAttendees, notification.NotificationId, applicationName); + this.CreateMappingEntity(list, notification.OptionalAttendees, notification.NotificationId, applicationName); + } + + var listWithoutDuplicates = list.ToHashSet(new EmailNotificationMapEntity.EmailNotificaitonMapEntityComparer()).ToList(); + + IList tasks = new List(); + + foreach (var item in listWithoutDuplicates) + { + tasks.Add(Task.Run(() => { + int count = 0; + TableResult result = null; + do + { + count++; + try + { + TableOperation insertOperation = TableOperation.InsertOrMerge(item); + result = cloudTable.ExecuteAsync(insertOperation).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + this.logger.TraceInformation($"Exception while inserting meeting email-notification mapping records {item.PartitionKey}, exception : {ex}"); + } + } while (result == null && count < MaxRetries); + })); + } + + _ = Task.WhenAll(tasks: tasks.ToArray()); + + this.logger.TraceInformation($"Finished {nameof(this.CreateEmailIdNotificationMappingForMeetingInvite)} method of {nameof(TableStorageEmailRepository)}", traceProps); + } + + + /// + /// Creates EmailId - notificationId mapping Entity. + /// + /// List of Entity instance. + /// comma-separated emailids. + /// notificationId. + private void CreateMappingEntity(List emailNotificationMapEntities, string emails, string notificationId, string applicationName) + { + if (string.IsNullOrEmpty(emails)) + { + return; + } + + emailNotificationMapEntities.AddRange(emails.Split(Common.ApplicationConstants.SplitCharacter, System.StringSplitOptions.RemoveEmptyEntries).Select(emailId => new EmailNotificationMapEntity() + { + PartitionKey = emailId, + RowKey = notificationId, + ApplicationName = applicationName, + CreateDateTime = DateTime.Now, + UpdateDateTime = DateTime.Now, + IsScrubbed = false, + })); + } + + /// + /// Create expression Filter. + /// + /// NotificationReportRequest instance. + /// Returns filter string. private string GetFilterExpression(NotificationReportRequest notificationReportRequest) { var filterSet = new HashSet(); diff --git a/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTests.cs b/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTests.cs index 174538d..aa7d7ab 100644 --- a/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTests.cs +++ b/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTests.cs @@ -6,6 +6,7 @@ namespace NotificationService.UnitTests.BusinessLibrary.V1.EmailManager using System; using System.Collections.Generic; using System.Threading.Tasks; + using Microsoft.Azure.Storage.Queue; using Microsoft.Extensions.Configuration; using Moq; using NotificationService.BusinessLibrary; @@ -25,6 +26,16 @@ namespace NotificationService.UnitTests.BusinessLibrary.V1.EmailManager /// public class EmailManagerTests { + /// + /// Application Name Constant. + /// + public const string ApplicationName = "TestApp"; + + /// + /// CloudQueue instance ref. + /// + private readonly Mock cloudQueue; + /// /// Initializes a new instance of the class. /// @@ -34,11 +45,25 @@ public EmailManagerTests() this.EncryptionService = new Mock(); this.TemplateManager = new Mock(); this.TemplateMerge = new Mock(); - this.Configuration = new Mock(); this.EmailNotificationRepo = new Mock(); + this.CloudStorageClient = new Mock(); this.RepositoryFactory = new Mock(); - _ = this.Configuration.Setup(x => x[ConfigConstants.StorageType]).Returns("StorageAccount"); + this.cloudQueue = new Mock(new Uri("https://test.azure.com/testqueue")); + + var testConfigValues = new Dictionary() + { + { ConfigConstants.StorageType, "StorageAccount" }, + { ConfigConstants.IsGDPREnabled, "false" }, + { ConfigConstants.StorageAccGdprMapQueueName, "TestQueue" }, + }; + + this.Configuration = new ConfigurationBuilder() + .AddInMemoryCollection(testConfigValues) + .Build(); + _ = this.RepositoryFactory.Setup(x => x.GetRepository(It.IsAny())).Returns(this.EmailNotificationRepo.Object); + _ = this.CloudStorageClient.Setup(x => x.GetCloudQueue(It.IsAny())).Returns(this.cloudQueue.Object); + this.CloudStorageClient.Setup(x => x.QueueCloudMessages(It.IsAny(), It.IsAny>(), It.IsAny())).Verifiable(); } /// @@ -54,7 +79,7 @@ public EmailManagerTests() /// /// Gets or sets Configuration. /// - public Mock Configuration { get; set; } + public IConfiguration Configuration { get; set; } /// /// Gets or sets Encryption Service Mock. @@ -87,6 +112,11 @@ public EmailManagerTests() /// public Mock EmailNotificationRepo { get; set; } + /// + /// Gets or sets cloutd Storage account mocked instance. + /// + public Mock CloudStorageClient { get; set; } + /// /// Notifications the entities to response tests. /// @@ -94,7 +124,7 @@ public EmailManagerTests() [Test] public async Task CreateMeetingNotificationEntitiesTests() { - var emailManager = new EmailManager(this.Configuration.Object, this.RepositoryFactory.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object); + var emailManager = new EmailManager(this.Configuration, this.RepositoryFactory.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object, this.CloudStorageClient.Object); var meetingNotificationItems = new List { new MeetingNotificationItem { EndDate = DateTime.UtcNow.AddHours(1), Start = DateTime.UtcNow.AddHours(1), End = DateTime.UtcNow }, @@ -111,7 +141,7 @@ public async Task CreateMeetingNotificationEntitiesTests() [Test] public void NotificationEntitiesToResponseTests() { - var emailManager = new EmailManager(this.Configuration.Object, this.RepositoryFactory.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object); + var emailManager = new EmailManager(this.Configuration, this.RepositoryFactory.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object, this.CloudStorageClient.Object); var meetingNotificationItems = new List { new MeetingNotificationItemEntity @@ -125,5 +155,77 @@ public void NotificationEntitiesToResponseTests() var meetingEntities = emailManager.NotificationEntitiesToResponse(new List(), meetingNotificationItems); Assert.IsTrue(meetingEntities.Count == 2); } + + /// + /// Queue EmailNotification for GDPR Mapping. + /// + /// A representing the asynchronous unit test. + [Test] + public async Task QueueEmailNotificaitionGDPRMappingTest_Success() + { + var emailNotificationItemEntities = new List>() + { + new List() + { + new EmailNotificationItemEntity() + { + Application = ApplicationName, + To = "test@abc.com", + From = "abc@contoso.com", + NotificationId = "1234", + Body = "Test Email", + }, + }, + }; + var testConfigValues = new Dictionary() + { + { ConfigConstants.StorageType, "StorageAccount" }, + { ConfigConstants.IsGDPREnabled, "true" }, + { ConfigConstants.StorageAccGdprMapQueueName, "TestQueue" }, + }; + + var config = new ConfigurationBuilder() + .AddInMemoryCollection(testConfigValues) + .Build(); + var emailManager = new EmailManager(config, this.RepositoryFactory.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object, this.CloudStorageClient.Object); + await emailManager.QueueEmailNotificaitionMapping(ApplicationName, emailNotificationItemEntities, null); + this.CloudStorageClient.Verify(x => x.GetCloudQueue(It.IsAny()), Times.AtLeastOnce); + } + + /// + /// Queue Meeting Notification for GDPR Mapping. + /// + /// A representing the asynchronous unit test. + [Test] + public async Task QueueMeetingNotificaitionGDPRMappingTest_Success() + { + var meetingNotificationItemEntities = new List>() + { + new List() + { + new MeetingNotificationItemEntity() + { + Application = ApplicationName, + RequiredAttendees = "test@abc.com", + From = "abc@contoso.com", + NotificationId = "1234", + Body = "Test Email", + }, + }, + }; + var testConfigValues = new Dictionary() + { + { ConfigConstants.StorageType, "StorageAccount" }, + { ConfigConstants.IsGDPREnabled, "true" }, + { ConfigConstants.StorageAccGdprMapQueueName, "TestQueue" }, + }; + + var config = new ConfigurationBuilder() + .AddInMemoryCollection(testConfigValues) + .Build(); + var emailManager = new EmailManager(config, this.RepositoryFactory.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object, this.CloudStorageClient.Object); + await emailManager.QueueMeetingNotificactionMapping(ApplicationName, meetingNotificationItemEntities, null); + this.CloudStorageClient.Verify(x => x.GetCloudQueue(It.IsAny()), Times.AtLeastOnce); + } } } diff --git a/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTestsBase.cs b/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTestsBase.cs index 836ffb1..e03027d 100644 --- a/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTestsBase.cs +++ b/NotificationService/NotificationService.UnitTests/BusinessLibrary/V1/EmailManager/EmailManagerTestsBase.cs @@ -315,7 +315,7 @@ protected void SetupTestBase() _ = this.TemplateMerge .Setup(tmr => tmr.CreateMailBodyUsingTemplate(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(mergedTemplate); - this.EmailManager = new EmailManager(this.Configuration, this.EmailNotificationRepository.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object); + this.EmailManager = new EmailManager(this.Configuration, this.EmailNotificationRepository.Object, this.Logger, this.TemplateManager.Object, this.TemplateMerge.Object, this.CloudStorageClient.Object); this.MSGraphNotificationProvider = new MSGraphNotificationProvider( this.Configuration, diff --git a/NotificationService/NotificationService.UnitTests/Data/Repositories/TableStorageRepositoryTests.cs b/NotificationService/NotificationService.UnitTests/Data/Repositories/TableStorageRepositoryTests.cs index d8ab0e9..af15522 100644 --- a/NotificationService/NotificationService.UnitTests/Data/Repositories/TableStorageRepositoryTests.cs +++ b/NotificationService/NotificationService.UnitTests/Data/Repositories/TableStorageRepositoryTests.cs @@ -14,10 +14,12 @@ namespace NotificationService.UnitTests.Data.Repositories using NotificationService.Common.Logger; using NotificationService.Contracts; using NotificationService.Contracts.Entities; + using NotificationService.Contracts.Models.GDPR; using NotificationService.Contracts.Models.Request; using NotificationService.Data; using NotificationService.Data.Repositories; using NUnit.Framework; + using Org.BouncyCastle.Math.EC.Rfc7748; /// /// Table Storage Repository Tests Class. @@ -44,6 +46,16 @@ public class TableStorageRepositoryTests /// private readonly string applicationName = "TestApp"; + /// + /// Instace of + /// + private readonly Mock cloudTable; + + /// + /// storageconfiguration options. + /// + IOptions storageConfigOptions; + /// /// DateRange object. /// @@ -66,8 +78,13 @@ public TableStorageRepositoryTests() this.cloudStorageClient = new Mock(); this.logger = new Mock(); this.mailAttachmentRepository = new Mock(); + this.cloudTable = new Mock(new Uri("https://test.azurestorage.com/testtable"), (TableClientConfiguration) null); this.meetingHistoryTable = new Mock(new Uri("http://unittests.localhost.com/FakeTable"), (TableClientConfiguration)null); + this.storageConfigOptions = Options.Create(new StorageAccountSetting { BlobContainerName = "Test", ConnectionString = "Test Con", MailTemplateTableName = "MailTemplate", EmailHistoryTableName = "EmailHistory", MeetingHistoryTableName = "MeetingHistory", NotificationQueueName = "test-queue", EmailNotificationMapTableName = "EmailMappingTable", MeetingNotificationMapTableName = "MeetingMappingTable" }); _ = this.cloudStorageClient.Setup(x => x.GetCloudTable("MeetingHistory")).Returns(this.meetingHistoryTable.Object); + _ = this.cloudStorageClient.Setup(x => x.GetCloudTable(It.IsAny())).Returns(this.cloudTable.Object); + TableResult res = new TableResult() { HttpStatusCode = 200}; + _ = this.cloudTable.Setup(x => x.ExecuteAsync(It.IsAny())).ReturnsAsync(res); } /// @@ -111,9 +128,11 @@ public async Task GetMeetingNotificationItemEntityTests() [Test] public async Task CreateMeetingNotificationItemEntitiesTests() { + this.meetingHistoryTable = new Mock(new Uri("http://unittests.localhost.com/FakeTable"), (TableClientConfiguration)null); + _ = this.cloudStorageClient.Setup(x => x.GetCloudTable("MeetingHistory")).Returns(this.meetingHistoryTable.Object); IList entities = new List { new MeetingNotificationItemEntity { NotificationId = "notificationId1", Application = "Application", RowKey = "notificationId1" }, new MeetingNotificationItemEntity { NotificationId = "notificationId2", Application = "Application", RowKey = "notificationId2" } }; _ = this.mailAttachmentRepository.Setup(e => e.UploadMeetingInvite(It.IsAny>(), It.IsAny())).Returns(Task.FromResult(entities)); - this.meetingHistoryTable.Setup(x => x.ExecuteBatchAsync(It.IsAny(), null, null)).Verifiable(); + this.meetingHistoryTable.Setup(x => x.ExecuteBatchAsync(It.IsAny())).Verifiable(); IOptions options = Options.Create(new StorageAccountSetting { BlobContainerName = "Test", ConnectionString = "Test Con", MailTemplateTableName = "MailTemplate", EmailHistoryTableName = "EmailHistory", MeetingHistoryTableName = "MeetingHistory", NotificationQueueName = "test-queue" }); var repo = new TableStorageEmailRepository(options, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); await repo.CreateMeetingNotificationItemEntities(entities, this.applicationName); @@ -177,5 +196,82 @@ public async Task GetEmailNotificationItemEntitiesBetweenDatesTests() result = await classUnderTest.GetPendingOrFailedEmailNotificationsByDateRange(this.dateRange, this.applicationName, null); Assert.IsNull(result); } + + /// + /// Create NotificationId and EmailId mapping for email Notifications. Test for Success. + /// + [Test] + public void CreateEmailIdNotificationMappingForEmail_Success() + { + this.storageConfigOptions.Value.EmailNotificationMapTableName = "EmailNotificationMapTable"; + var item = new List() { new EmailNotificationQueueItem() { To = "test1@contoso.com;test2@contoso.com", CC = "tst@abc.com", From = "abc@contosot.com", NotificationId = Guid.NewGuid().ToString() } }; + var classUnderTest = new TableStorageEmailRepository(this.storageConfigOptions, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); + classUnderTest.CreateEmailIdNotificationMappingForEmail(item, this.applicationName); + this.cloudStorageClient.Verify(x => x.GetCloudTable(It.IsAny()), Times.AtLeastOnce); + } + + /// + /// Create NotificationId and EmailId mapping for email Notifications. Test for Failures. + /// + [Test] + public void CreateEmailIdNotificationMappingForEmail_Failed() + { + var item = new List() { new EmailNotificationQueueItem() { To = "test1@contoso.com;test2@contoso.com", CC = "tst@abc.com", From = "abc@contosot.com", NotificationId = Guid.NewGuid().ToString() } }; + var classUnderTest = new TableStorageEmailRepository(this.storageConfigOptions, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); + + // Test for null EmailNotificationQueueItem list. + classUnderTest.CreateEmailIdNotificationMappingForEmail(null, this.applicationName); + + this.storageConfigOptions.Value.EmailNotificationMapTableName = null; + classUnderTest = new TableStorageEmailRepository(this.storageConfigOptions, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); + + // Test for Table does not exists exception. + try + { + classUnderTest.CreateEmailIdNotificationMappingForEmail(item, this.applicationName); + } catch(Exception ex) + { + Assert.IsTrue(ex is ArgumentNullException); + } + } + + /// + /// Create NotificationId and EmailId mapping for meeting Notifications. Test For Success. + /// + [Test] + public void CreateEmailIdNotificationForMeetingInvitesMapping_Success() + { + this.storageConfigOptions.Value.MeetingNotificationMapTableName = "MeetingNotificationMapTable"; + var item = new List() { new MeetingNotificationQueueItem() { RequiredAttendees = "test1@contoso.com;test2@contoso.com", OptionalAttendees = "tst@abc.com", From = "abc@contosot.com", NotificationId = Guid.NewGuid().ToString() } }; + var classUnderTest = new TableStorageEmailRepository(this.storageConfigOptions, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); + classUnderTest.CreateEmailIdNotificationMappingForMeetingInvite(item, this.applicationName); + this.cloudStorageClient.Verify(x => x.GetCloudTable(It.IsAny()), Times.AtLeastOnce); + } + + /// + /// Create NotificationId and EmailId mapping for meeting Notifications. Test For Failure. + /// + [Test] + public void CreateEmailIdNotificationForMeetingInvitesMapping_Failed() + { + var item = new List() { new MeetingNotificationQueueItem() { RequiredAttendees = "test1@contoso.com;test2@contoso.com", OptionalAttendees = "tst@abc.com", From = "abc@contosot.com", NotificationId = Guid.NewGuid().ToString() } }; + var classUnderTest = new TableStorageEmailRepository(this.storageConfigOptions, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); + + // Test for null EmailNotificationQueueItem list. + classUnderTest.CreateEmailIdNotificationMappingForMeetingInvite(null, this.applicationName); + + this.storageConfigOptions.Value.MeetingNotificationMapTableName = null; + classUnderTest = new TableStorageEmailRepository(this.storageConfigOptions, this.cloudStorageClient.Object, this.logger.Object, this.mailAttachmentRepository.Object); + + // Test for Table does not exists exception. + try + { + classUnderTest.CreateEmailIdNotificationMappingForMeetingInvite(item, this.applicationName); + } + catch (Exception ex) + { + Assert.IsTrue(ex is ArgumentNullException); + } + } } } diff --git a/NotificationService/NotificationService.sln b/NotificationService/NotificationService.sln index e3c0511..da44f7b 100644 --- a/NotificationService/NotificationService.sln +++ b/NotificationService/NotificationService.sln @@ -51,6 +51,8 @@ Project("{151D2E53-A2C4-4D7D-83FE-D05416EBD58E}") = "NotificationService.IaaC", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NotificationService.FunctionalTests", "NotificationService.FunctionalTests\NotificationService.FunctionalTests.csproj", "{7159519C-758B-4E95-B4E0-0DB8F1641162}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GDPRNotificationMappingProcessor", "GDPRNotificationMappingProcessor\GDPRNotificationMappingProcessor.csproj", "{F76A7CE6-2652-476F-A17B-BC2FA0E7943F}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution NotificationProviders\DirectSend.Shared\DirectSend.Shared.projitems*{63dd255d-4c9f-4ea7-9f3b-5f7b32657058}*SharedItemsImports = 13 @@ -118,6 +120,10 @@ Global {7159519C-758B-4E95-B4E0-0DB8F1641162}.Debug|Any CPU.Build.0 = Debug|Any CPU {7159519C-758B-4E95-B4E0-0DB8F1641162}.Release|Any CPU.ActiveCfg = Release|Any CPU {7159519C-758B-4E95-B4E0-0DB8F1641162}.Release|Any CPU.Build.0 = Release|Any CPU + {F76A7CE6-2652-476F-A17B-BC2FA0E7943F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F76A7CE6-2652-476F-A17B-BC2FA0E7943F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F76A7CE6-2652-476F-A17B-BC2FA0E7943F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F76A7CE6-2652-476F-A17B-BC2FA0E7943F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/NotificationService/NotificationService/Startup.cs b/NotificationService/NotificationService/Startup.cs index 3f83881..ef0fa73 100644 --- a/NotificationService/NotificationService/Startup.cs +++ b/NotificationService/NotificationService/Startup.cs @@ -95,7 +95,8 @@ public void ConfigureServices(IServiceCollection services) s.GetService(), s.GetService(), s.GetService(), - s.GetService())) + s.GetService(), + s.GetService())) .AddScoped(s => new EmailServiceManager(this.Configuration, s.GetService(), s.GetService(), s.GetService(), s.GetService(), s.GetService()))