Merge pull request Azure#81 from Azure/assistant-dalle

Add DALLE function

Author: MarcoABCardoso

gen-ai/Assistants/bot-in-a-box/README.md

@@ -63,17 +63,7 @@ azd up
 ```
 You will be prompted for a subscription, region and model information. Keep regional model availability when proceeding.
 
-3. Go to the Azure OpenAI Studio and create an Assistant with the tools you want to use. Alternatively, you can also use the API.
-
-> Important: currently, this application only supports Code Interpreter. Other tools will be implemented in the near future.
-
-![Assistant Creation](./readme_assets/assistant-creation.png)
-
-4. Add your newly created Assistant's ID in the AOAI_ASSISTANT_ID environment variable.
-
-![Add Assistant ID to environment](./readme_assets/assistant-id-variable.png)
-
-5. Test on Web Chat - go to your Azure Bot resource on the Azure portal and look for the Web Chat feature on the left side menu.
+3. Test on Web Chat - go to your Azure Bot resource on the Azure portal and look for the Web Chat feature on the left side menu.
 
 ![Test Web Chat](./readme_assets/assistant-test.png)
 
@@ -106,4 +96,13 @@ To deploy a Web Chat version of your app:
 - Add the secret to your App Service's environment variables, under the key DIRECT_LINE_SECRET;
 - Your bot will be available at https://APP_NAME.azurewebsites.net.
 
-Please note that doing so will make your bot public, unless you implement authentication / SSO.
+Please note that doing so will make your bot public, unless you implement authentication / SSO.
+
+### Creating custom functions
+
+To update function calling behavior or create your own functions, follow the steps below.
+
+- Go to [./src/Tools](./src/Tools/)
+- Create or update a JSON file with the function specification. You may also copy files from [./src/ToolsSamples](./src/ToolsSamples/) into [./src/Tools](./src/Tools/)
+- Go to `_Tools.cs` and create or update the method with the same name as the function's "name" field. **Use only lowercase characters and underscores for names**.
+- Redeploy with `azd up`. The Assistant definition will be updated automatically.

gen-ai/Assistants/bot-in-a-box/infra/main.bicep

@@ -21,7 +21,7 @@ param cosmosName string = ''
 param speechName string = ''
 
 param storageName string = ''
-param deployDalle3 bool = false
+param deployDalle3 bool
 param deploySpeech bool
 @description('Deploy Speech service?')
 
@@ -114,6 +114,7 @@ module m_app 'modules/appservice.bicep' = {
     openaiEndpoint: m_openai.outputs.openaiEndpoint
     openaiGPTModel: m_openai.outputs.openaiGPTModel
     openaiEmbeddingsModel: m_openai.outputs.openaiEmbeddingsModel
+    openaiDalleModel: m_openai.outputs.openaiDalleModel
     speechName: deploySpeech ? m_speech.outputs.speechName : ''
     speechEndpoint: deploySpeech ? m_speech.outputs.speechEndpoint : ''
     cosmosEndpoint: m_cosmos.outputs.cosmosEndpoint

gen-ai/Assistants/bot-in-a-box/infra/modules/appservice.bicep

@@ -7,6 +7,7 @@ param sku string = 'S1'
 param tags object = {}
 param openaiGPTModel string
 param openaiEmbeddingsModel string
+param openaiDalleModel string
 param speechName string
 var speechNames = !empty(speechName) ? [speechName] : []
 
@@ -87,6 +88,10 @@ resource appService 'Microsoft.Web/sites@2022-09-01' = {
           name: 'AOAI_ASSISTANT_ID'
           value: 'YOUR_ASSISTANT_ID'
         }
+        {
+          name: 'AOAI_DALLE_DEPLOYMENT'
+          value: openaiDalleModel
+        }
         {
           name: 'SPEECH_API_ENDPOINT'
           value: speechEndpoint

gen-ai/Assistants/bot-in-a-box/infra/modules/openai.bicep

@@ -64,7 +64,7 @@ resource adaEmbeddingsdeployment 'Microsoft.CognitiveServices/accounts/deploymen
 
 resource dalle3deployment 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = if (deployDalle3) {
   parent: openai
-  name: 'dall-e-3'
+  name: 'Dalle3'
   properties: {
     model: {
       format: 'OpenAI'
@@ -98,3 +98,4 @@ output openaiName string = openai.name
 output openaiEndpoint string = openai.properties.endpoint
 output openaiGPTModel string = gpt4deployment.name
 output openaiEmbeddingsModel string = adaEmbeddingsdeployment.name
+output openaiDalleModel string = deployDalle3 ? dalle3deployment.name : ''

gen-ai/Assistants/bot-in-a-box/src/AssistantBot.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <LangVersion>latest</LangVersion>
     <UserSecretsId>73c2a03b-aae7-4833-b3a8-c01744ad2b7e</UserSecretsId>
   </PropertyGroup>

gen-ai/Assistants/bot-in-a-box/src/Bots/AssistantBot.cs

@@ -116,7 +116,7 @@ public override async Task<List<string>> ProcessMessage(ConversationData convers
             {
                 if (run.Status == "requires_action")
                 {
-                    var tools = new Tools(conversationData, turnContext);
+                    var tools = new Tools(conversationData, turnContext, _aoaiClient);
                     var submitData = await tools.RunRequestedTools(run);
                     await _aoaiClient.SubmitToolOutputs(conversationData.ThreadId, run.Id, submitData);
                 }

gen-ai/Assistants/bot-in-a-box/src/Models/AOAIModels.cs

@@ -2,6 +2,7 @@
 
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
+using Azure.AI.OpenAI;
 
 namespace Models
 {
@@ -139,5 +140,45 @@ public class ThreadRun
         [JsonPropertyName("required_action")]
         public RequiredAction RequiredAction { get; set; }
     }
+    public class ImageGenerationInput
+    {
+        [JsonPropertyName("prompt")]
+        public string Prompt { get; set; }
+
+        [JsonPropertyName("size")]
+        public string Size { get; set; }
+
+        [JsonPropertyName("n")]
+        public int N { get; set; }
+    }
+    public class ImageGenerationOutput
+    {
+        [JsonPropertyName("id")]
+        public string Id { get; set; }
+
+        [JsonPropertyName("status")]
+        public string Status { get; set; }
+    }
+    public class ImageGenerationStatusResponse
+    {
+        [JsonPropertyName("result")]
+        public ImageGenerationResult Result { get; set; }
+        
+        [JsonPropertyName("status")]
+        public string Status { get; set; }
+    }
+    public class ImageGenerationResult
+    {
+        [JsonPropertyName("data")]
+        public List<GeneratedImage> Data { get; set; }
+    }
+    public class GeneratedImage
+    {
+        [JsonPropertyName("url")]
+        public string Url { get; set; }
+
+        [JsonPropertyName("revised_prompt")]
+        public string RevisedPrompt { get; set; }
+    }
 
 }

gen-ai/Assistants/bot-in-a-box/src/Services/AOAIClient.cs

@@ -5,7 +5,7 @@
 using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
-using Microsoft.Azure.Cosmos.Core;
+using Microsoft.IdentityModel.Tokens;
 using Models;
 
 namespace Services
@@ -14,11 +14,13 @@ public class AOAIClient
     {
         private readonly HttpClient _httpClient;
         private readonly string _accessKey;
-        public AOAIClient(HttpClient httpClient, Uri uriBase, string apiKey)
+        private readonly string _dalleDeployment;
+        public AOAIClient(HttpClient httpClient, Uri uriBase, string apiKey, string dalleDeployment)
         {
             httpClient.BaseAddress = uriBase;
             _httpClient = httpClient;
             _accessKey = apiKey;
+            _dalleDeployment = dalleDeployment;
         }
 
         public async Task<List<Assistant>> ListAssistants()
@@ -74,10 +76,30 @@ public async Task<HttpResponseMessage> GetFile(string fileId)
             var result = await SendRequest($"/files/{fileId}/content", HttpMethod.Get);
             return result;
         }
-
-        private async Task<HttpResponseMessage> SendRequest(string path, HttpMethod method, StringContent body = null)
+        public async Task<ImageGenerationResult> GenerateImages(ImageGenerationInput input)
+        {
+            if (_dalleDeployment.IsNullOrEmpty())
+                return await GenerateImagesV2(input);
+            else
+                return await GenerateImagesV3(_dalleDeployment, input);
+        }
+        public async Task<ImageGenerationResult> GenerateImagesV2(ImageGenerationInput input)
+        {
+            var output = await JsonRequest<ImageGenerationOutput>($"/images/generations:submit", HttpMethod.Post, new StringContent(JsonSerializer.Serialize(input), Encoding.UTF8, "application/json"), "2023-06-01-preview");
+            var response = await JsonRequest<ImageGenerationStatusResponse>($"/operations/images/{output.Id}", HttpMethod.Get, apiVersion: "2023-06-01-preview");
+            while (response.Status != "succeeded") {
+                response = await JsonRequest<ImageGenerationStatusResponse>($"/operations/images/{output.Id}", HttpMethod.Get, apiVersion: "2023-06-01-preview");
+                System.Threading.Thread.Sleep(5000);
+            }
+            return response.Result;
+        }
+        public async Task<ImageGenerationResult> GenerateImagesV3(string deploymentId, ImageGenerationInput input)
+        {
+            return await JsonRequest<ImageGenerationResult>($"/deployments/{deploymentId}/images/generations", HttpMethod.Post, new StringContent(JsonSerializer.Serialize(input), Encoding.UTF8, "application/json"), "2023-12-01-preview");
+        }
+        private async Task<HttpResponseMessage> SendRequest(string path, HttpMethod method, StringContent body = null, string apiVersion = "2024-02-15-preview")
         {
-            var url = "/openai" + path + "?api-version=2024-02-15-preview";
+            var url = "/openai" + path + "?api-version=" + apiVersion;
 
             var request = new HttpRequestMessage(method, url)
             {
@@ -91,9 +113,9 @@ private async Task<HttpResponseMessage> SendRequest(string path, HttpMethod meth
         }
 
 
-        private async Task<T> JsonRequest<T>(string path, HttpMethod method, StringContent body = null)
+        private async Task<T> JsonRequest<T>(string path, HttpMethod method, StringContent body = null, string apiVersion = "2024-02-15-preview")
         {
-            var response = await SendRequest(path, method, body);
+            var response = await SendRequest(path, method, body, apiVersion);
             var responseContent = await response.Content.ReadAsStringAsync();
             if (!response.IsSuccessStatusCode)
                 throw new Exception(responseContent);
@@ -102,9 +124,9 @@ private async Task<T> JsonRequest<T>(string path, HttpMethod method, StringConte
             return content;
         }
 
-        private async Task<HttpResponseMessage> SendRequest(string path, HttpMethod method, MultipartFormDataContent body)
+        private async Task<HttpResponseMessage> SendRequest(string path, HttpMethod method, MultipartFormDataContent body, string apiVersion = "2024-02-15-preview")
         {
-            var url = "/openai" + path + "?api-version=2024-02-15-preview";
+            var url = "/openai" + path + "?api-version=" + apiVersion;
 
             var request = new HttpRequestMessage(method, url)
             {
@@ -118,9 +140,9 @@ private async Task<HttpResponseMessage> SendRequest(string path, HttpMethod meth
         }
 
 
-        private async Task<T> JsonRequest<T>(string path, HttpMethod method, MultipartFormDataContent body)
+        private async Task<T> JsonRequest<T>(string path, HttpMethod method, MultipartFormDataContent body, string apiVersion = "2024-02-15-preview")
         {
-            var response = await SendRequest(path, method, body);
+            var response = await SendRequest(path, method, body, apiVersion);
 
             var responseContent = await response.Content.ReadAsStringAsync();
             if (!response.IsSuccessStatusCode)

gen-ai/Assistants/bot-in-a-box/src/Startup.cs

@@ -2,8 +2,6 @@
 // Licensed under the MIT License.
 
 using System;
-using Azure;
-using Azure.AI.OpenAI;
 using Azure.Identity;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -73,7 +71,7 @@ public void ConfigureServices(IServiceCollection services)
             // Create the Conversation state passing in the storage layer.
             var conversationState = new ConversationState(storage);
             services.AddSingleton(conversationState);
-            services.AddSingleton(new AOAIClient(new System.Net.Http.HttpClient(), new Uri(configuration.GetValue<string>("AOAI_API_ENDPOINT")), configuration.GetValue<string>("AOAI_API_KEY")));
+            services.AddSingleton(new AOAIClient(new System.Net.Http.HttpClient(), new Uri(configuration.GetValue<string>("AOAI_API_ENDPOINT")), configuration.GetValue<string>("AOAI_API_KEY"), configuration.GetValue<string>("AOAI_DALLE_DEPLOYMENT")));
             services.AddHttpClient();
             if (!configuration.GetValue<string>("SPEECH_API_ENDPOINT").IsNullOrEmpty())
                 services.AddSingleton(new SpeechService(new System.Net.Http.HttpClient(), configuration.GetValue<string>("SPEECH_API_ENDPOINT"), configuration.GetValue<string>("SPEECH_API_KEY")));

gen-ai/Assistants/bot-in-a-box/src/Tools/_Tools.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Text.Encodings.Web;
@@ -9,20 +8,24 @@
 using Microsoft.Bot.Schema;
 using Microsoft.BotBuilderSamples;
 using Models;
+using Services;
 
 public class Tools
 {
     private HttpClient client = new HttpClient();
+    private AOAIClient _aoaiClient;
     private ITurnContext _turnContext;
     private ConversationData _conversationData;
 
     public Tools(
         ConversationData conversationData,
-        ITurnContext<IMessageActivity> turnContext
+        ITurnContext<IMessageActivity> turnContext,
+        AOAIClient aoaiClient
     )
     {
         _conversationData = conversationData;
         _turnContext = turnContext;
+        _aoaiClient = aoaiClient;
     }
 
     public async Task<ToolOutputData> RunRequestedTools(ThreadRun run)
@@ -35,7 +38,7 @@ public async Task<ToolOutputData> RunRequestedTools(ThreadRun run)
         {
             var method = typeof(Tools).GetMethod(toolcall.Function.Name);
             var arguments = JsonSerializer.Deserialize<Dictionary<string, object>>(toolcall.Function.Arguments);
-            string output = await (Task<string>)method.Invoke(this, new object[]{arguments});
+            string output = await (Task<string>)method.Invoke(this, new object[] { arguments });
             var toolOutput = new ToolOutput
             {
                 ToolCallId = toolcall.Id,
@@ -94,6 +97,36 @@ public async Task<string> wikipedia_get_article(Dictionary<string, object> argum
             return await response.Content.ReadAsStringAsync();
         else
             return $"FAILED TO FETCH DATA FROM API. STATUS CODE {response.StatusCode}";
+    }
+    public async Task<string> dalle_generate_images(Dictionary<string, object> arguments)
+    {
+        var numImages = ((JsonElement)arguments["num_images"]).GetInt32();
+        var prompt = arguments["prompt"].ToString();
+        await _turnContext.SendActivityAsync($"Generating {numImages} images with the description \"{prompt}\"...");
+        var imageGenerationOutput = await _aoaiClient.GenerateImages(
+            new ImageGenerationInput()
+            {
+                Prompt = prompt,
+                Size = "1024x1024",
+                N = numImages
+            });
+
+        List<object> images = new();
+        foreach (GeneratedImage img in imageGenerationOutput.Data)
+            images.Add(new { type = "Image", url = img.Url });
+        object adaptiveCardJson = new
+        {
+            type = "AdaptiveCard",
+            version = "1.0",
+            body = images
+        };
 
+        var adaptiveCardAttachment = new Microsoft.Bot.Schema.Attachment()
+        {
+            ContentType = "application/vnd.microsoft.card.adaptive",
+            Content = adaptiveCardJson,
+        };
+        await _turnContext.SendActivityAsync(MessageFactory.Attachment(adaptiveCardAttachment));
+        return "Images were generated successfully and already sent to user. No need to embed or reference them into your next message.";
     }
 }