Skip to content

Commit cf498e6

Browse files
MarcoABCardosoMarcoABCardoso
authored
Merge pull request Azure#70 from Azure/assistant-extensibility
Assistant extensibility
2 parents 2255963 + 8e922d0 commit cf498e6

22 files changed

Lines changed: 336 additions & 172 deletions

‎gen-ai/Assistants/bot-in-a-box/azure.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,10 @@ services:
77
assistant-bot-app:
88
project: src
99
host: appservice
10-
language: dotnet
10+
language: dotnet
11+
hooks:
12+
postdeploy:
13+
shell: sh
14+
run: ./scripts/createAssistant.sh
15+
interactive: true
16+
continueOnError: false

‎gen-ai/Assistants/bot-in-a-box/infra/main.bicep‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,3 +137,7 @@ module m_bot 'modules/botservice.bicep' = {
137137

138138
output AZURE_RESOURCE_GROUP_ID string = resourceGroup.id
139139
output AZURE_RESOURCE_GROUP_NAME string = resourceGroup.name
140+
output AOAI_NAME string = m_openai.outputs.openaiName
141+
output AOAI_API_ENDPOINT string = m_openai.outputs.openaiEndpoint
142+
output APP_NAME string = m_app.outputs.appName
143+
output APP_HOSTNAME string = m_app.outputs.hostName

‎gen-ai/Assistants/bot-in-a-box/infra/modules/appservice.bicep‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,4 +160,5 @@ resource appService 'Microsoft.Web/sites@2022-09-01' = {
160160
}
161161
}
162162

163+
output appName string = appService.name
163164
output hostName string = appService.properties.defaultHostName

‎gen-ai/Assistants/bot-in-a-box/scripts/createAssistant.ps1‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
Write-Host "Loading azd .env file from current environment"
2+
foreach ($line in (& azd env get-values)) {
3+
if ($line -match "([^=]+)=(.*)") {
4+
$key = $matches[1]
5+
$value = $matches[2] -replace '^"|"$'
6+
[Environment]::SetEnvironmentVariable($key, $value)
7+
}
8+
}
9+
10+
$AOAI_API_KEY=az cognitiveservices account keys list -n $env:AOAI_NAME -g $env:AZURE_RESOURCE_GROUP_NAME --query key1 -o tsv
11+
$AOAI_ASSISTANT_NAME="assistant_in_a_box"
12+
$ASSISTANT_ID=((curl "$env:AOAI_API_ENDPOINT/openai/assistants?api-version=2024-02-15-preview" -H "api-key: $AOAI_API_KEY" | ConvertFrom-Json).data | Where-Object name -eq $AOAI_ASSISTANT_NAME).id
13+
if ( "$ASSISTANT_ID" == "null" )
14+
{ASSISTANT_ID=""}
15+
else
16+
{ASSISTANT_ID=/$ASSISTANT_ID}
17+
18+
echo "{
19+
`"name`":`"${AOAI_ASSISTANT_NAME}`",
20+
`"model`":`"gpt-4`",
21+
`"instructions`":`"`",
22+
`"tools`":[
23+
$(Get-ChildItem "./src/Tools" -Filter *.json |
24+
Foreach-Object {
25+
$content = Get-Content $_.FullName
26+
echo $content","
27+
})
28+
{}
29+
],
30+
`"file_ids`":[],
31+
`"metadata`":{}
32+
}" > tmp.json
33+
curl "$env:AOAI_API_ENDPOINT/openai/assistants$ASSISTANT_ID?api-version=2024-02-15-preview" \
34+
-H "api-key: $AOAI_API_KEY" \
35+
-H 'content-type: application/json' \
36+
-d @tmp.json
37+
rm tmp.json
38+
39+
$ASSISTANT_ID=((curl "$env:AOAI_API_ENDPOINT/openai/assistants?api-version=2024-02-15-preview" -H "api-key: $AOAI_API_KEY" | ConvertFrom-Json).data | Where-Object name -eq $AOAI_ASSISTANT_NAME).id
40+
41+
az webapp config appsettings set -g $AZURE_RESOURCE_GROUP_NAME -n $APP_NAME --settings AOAI_ASSISTANT_ID=$ASSISTANT_ID APP_URL=$APP_HOSTNAME

‎gen-ai/Assistants/bot-in-a-box/scripts/createAssistant.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
echo "Loading azd .env file from current environment..."
2+
3+
while IFS='=' read -r key value; do
4+
value=$(echo "$value" | sed 's/^"//' | sed 's/"$//')
5+
export "$key=$value"
6+
done <<EOF
7+
$(azd env get-values)
8+
EOF
9+
10+
AOAI_API_KEY=$(az cognitiveservices account keys list -n $AOAI_NAME -g $AZURE_RESOURCE_GROUP_NAME | jq -r .key1)
11+
AOAI_ASSISTANT_NAME="assistant_in_a_box"
12+
ASSISTANT_ID=$(curl "$AOAI_API_ENDPOINT/openai/assistants?api-version=2024-02-15-preview" \
13+
-H "api-key: $AOAI_API_KEY"|\
14+
jq -r '[.data[] | select( .name == "'$AOAI_ASSISTANT_NAME'")][0] | .id')
15+
if [ "$ASSISTANT_ID" == "null" ]; then
16+
ASSISTANT_ID=
17+
else
18+
ASSISTANT_ID=/$ASSISTANT_ID
19+
fi
20+
21+
echo '{
22+
"name":"'$AOAI_ASSISTANT_NAME'",
23+
"model":"gpt-4",
24+
"instructions":"",
25+
"tools":[
26+
'$(for each in ./src/Tools/*.json; do cat $each; echo ","; done)'
27+
{}
28+
],
29+
"file_ids":[],
30+
"metadata":{}
31+
}' > tmp.json
32+
curl "$AOAI_API_ENDPOINT/openai/assistants$ASSISTANT_ID?api-version=2024-02-15-preview" \
33+
-H "api-key: $AOAI_API_KEY" \
34+
-H 'content-type: application/json' \
35+
-d @tmp.json
36+
rm tmp.json
37+
38+
ASSISTANT_ID=$(curl "$AOAI_API_ENDPOINT/openai/assistants?api-version=2024-02-15-preview" \
39+
-H "api-key: $AOAI_API_KEY"|\
40+
jq -r '[.data[] | select( .name == "'$AOAI_ASSISTANT_NAME'")][0] | .id')
41+
42+
az webapp config appsettings set -g $AZURE_RESOURCE_GROUP_NAME -n $APP_NAME --settings AOAI_ASSISTANT_ID=$ASSISTANT_ID APP_URL=$APP_HOSTNAME

‎gen-ai/Assistants/bot-in-a-box/src/.editorconfig‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,6 @@
22
dotnet_diagnostic.SKEXP0011.severity = none
33
dotnet_diagnostic.SKEXP0060.severity = none
44
dotnet_diagnostic.SKEXP0061.severity = none
5+
6+
# IDE0055: Fix formatting
7+
dotnet_diagnostic.IDE0055.severity = none

‎gen-ai/Assistants/bot-in-a-box/src/AdapterWithErrorHandler.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public AdapterWithErrorHandler(BotFrameworkAuthentication auth, ILogger<IBotFram
2626
// Send a message to the user
2727
await turnContext.SendActivityAsync("The bot encountered an error or bug.");
2828
await turnContext.SendActivityAsync("To continue to run this bot, please fix the bot source code.");
29+
await turnContext.SendActivityAsync(exception.Message);
2930

3031
if (conversationState != null)
3132
{

‎gen-ai/Assistants/bot-in-a-box/src/Bots/AssistantBot.cs‎

Lines changed: 42 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@ namespace Microsoft.BotBuilderSamples
2424
{
2525
public class AssistantBot<T> : StateManagementBot<T> where T : Dialog
2626
{
27-
private string _aoaiModel;
2827
private string _aoaiAssistant;
2928
private readonly AOAIClient _aoaiClient;
3029
private readonly string _welcomeMessage;
3130
private readonly List<string> _suggestedQuestions;
31+
private readonly string _appUrl;
3232
private HttpClient client = new HttpClient();
3333

3434
public AssistantBot(
@@ -39,12 +39,12 @@ public AssistantBot(
3939
T dialog) :
4040
base(config, conversationState, userState, dialog)
4141
{
42-
_aoaiModel = config.GetValue<string>("AOAI_GPT_MODEL");
4342
_aoaiAssistant = config.GetValue<string>("AOAI_ASSISTANT_ID");
4443
_welcomeMessage = config.GetValue<string>("PROMPT_WELCOME_MESSAGE");
4544
_systemMessage = config.GetValue<string>("PROMPT_SYSTEM_MESSAGE");
46-
_suggestedQuestions = System.Text.Json.JsonSerializer.Deserialize<List<string>>(config.GetValue<string>("PROMPT_SUGGESTED_QUESTIONS"));
45+
_suggestedQuestions = JsonSerializer.Deserialize<List<string>>(config.GetValue<string>("PROMPT_SUGGESTED_QUESTIONS"));
4746
_aoaiClient = aoaiClient;
47+
_appUrl = config.GetValue("APP_URL", "http://localhost:3978");
4848
}
4949

5050
protected override async Task OnMembersAddedAsync(IList<ChannelAccount> membersAdded, ITurnContext<IConversationUpdateActivity> turnContext, CancellationToken cancellationToken)
@@ -62,7 +62,7 @@ await turnContext.SendActivityAsync(new Activity()
6262
});
6363
}
6464

65-
public override async Task<string> ProcessMessage(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext)
65+
public override async Task<List<string>> ProcessMessage(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext)
6666
{
6767
await turnContext.SendActivityAsync(new Activity(type: "typing"));
6868
if (conversationData.ThreadId.IsNullOrEmpty())
@@ -83,7 +83,7 @@ public override async Task<string> ProcessMessage(ConversationData conversationD
8383
conversationData.ThreadId = null;
8484
conversationData.History.Clear();
8585
conversationData.Attachments.Clear();
86-
return $"Thread {thread.Id} deleted.";
86+
return new List<string> { $"Thread {thread.Id} deleted." };
8787
}
8888

8989
// Add user message to thread
@@ -101,134 +101,57 @@ public override async Task<string> ProcessMessage(ConversationData conversationD
101101
});
102102

103103
// Wait until run completes
104-
while (run.Status != "completed")
104+
while (run.Status != "completed" && run.Status != "failed")
105105
{
106-
Console.WriteLine(JsonSerializer.Serialize(run));
107106
if (run.Status == "requires_action")
108107
{
109-
var submitData = new ToolOutputData()
110-
{
111-
ToolOutputs = new()
112-
};
113-
foreach (ToolCall toolcall in run.RequiredAction.SubmitToolOutputs.ToolCalls)
114-
{
115-
var arguments = System.Text.Json.JsonSerializer.Deserialize<Dictionary<string, object>>(toolcall.Function.Arguments);
116-
string output = "";
117-
switch (toolcall.Function.Name)
118-
{
119-
case "get_wikipedia_content":
120-
output = await GetWikipediaContent(conversationData, turnContext, arguments["page_title"].ToString());
121-
break;
122-
case "query_wikipedia":
123-
output = await QueryWikipedia(conversationData, turnContext, arguments["query"].ToString());
124-
break;
125-
case "mslearn_query_articles":
126-
output = await MslearnQueryArticles(conversationData, turnContext, arguments["query"].ToString());
127-
break;
128-
case "mslearn_get_article":
129-
output = await MslearnGetArticle(conversationData, turnContext, arguments["page_url"].ToString());
130-
break;
131-
case "bot_show_image":
132-
output = await BotShowImage(conversationData, turnContext, arguments["image_url"].ToString());
133-
break;
134-
default:
135-
output = "Function not found";
136-
break;
137-
}
138-
var toolOutput = new ToolOutput
139-
{
140-
ToolCallId = toolcall.Id,
141-
Output = output
142-
};
143-
submitData.ToolOutputs.Add(toolOutput);
144-
}
145-
108+
var tools = new Tools(conversationData, turnContext);
109+
var submitData = await tools.RunRequestedTools(run);
146110
await _aoaiClient.SubmitToolOutputs(conversationData.ThreadId, run.Id, submitData);
147111
}
148112
// await turnContext.SendActivityAsync($"The assistant is running...");
149113
System.Threading.Thread.Sleep(10000);
150114
run = await _aoaiClient.GetThreadRun(conversationData.ThreadId, run.Id);
151115
}
116+
if (run.Status == "failed") {
117+
await turnContext.SendActivityAsync("Something went wrong when running the assistant.");
118+
}
152119

153-
// Send back first message
120+
// Send back all messages written by the assistant since the last user message
121+
var responses = new List<string>();
154122
var messages = await _aoaiClient.ListThreadMessages(conversationData.ThreadId);
123+
var firstAssistantMessageIndex = messages.FindIndex(x => x.Role == "user") - 1;
155124

156-
return messages.First().Content.First().Text.Value;
157-
}
158-
public async Task<string> BotShowImage(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext, string imageUrl)
159-
{
160-
List<object> images = new();
161-
images.Add(new { type = "Image", url = imageUrl });
162-
object adaptiveCardJson = new
125+
for (var i = firstAssistantMessageIndex; i >= 0; i--)
163126
{
164-
type = "AdaptiveCard",
165-
version = "1.0",
166-
body = images
167-
};
168-
169-
var adaptiveCardAttachment = new Microsoft.Bot.Schema.Attachment()
170-
{
171-
ContentType = "application/vnd.microsoft.card.adaptive",
172-
Content = adaptiveCardJson,
173-
};
174-
await turnContext.SendActivityAsync(MessageFactory.Attachment(adaptiveCardAttachment));
175-
return "IMAGE SENT TO USER SUCCESSFULLY. DO NOT EMBED IT INTO YOUR RESPONSE.";
176-
}
177-
public async Task<string> MslearnQueryArticles(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext, string query)
178-
{
179-
await turnContext.SendActivityAsync($"Searching MS Learn for \"{query}\"...");
180-
HttpResponseMessage response = await client.GetAsync(
181-
$"https://learn.microsoft.com/api/search?search={UrlEncoder.Default.Encode(query)}&locale=en-us&$top=3"
182-
);
183-
if (response.IsSuccessStatusCode)
184-
return await response.Content.ReadAsStringAsync();
185-
else
186-
return $"FAILED TO FETCH DATA FROM API. STATUS CODE {response.StatusCode}";
187-
}
188-
public async Task<string> MslearnGetArticle(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext, string pageUrl)
189-
{
190-
if (!pageUrl.StartsWith("https://learn.microsoft.com/"))
191-
return "NOT ALLOWED TO FETCH DATA FROM API OUTSIDE OF LEARN.MICROSOFT.COM";
192-
await turnContext.SendActivityAsync($"Getting docs page \"{pageUrl}\"...");
193-
194-
var web = new HtmlWeb();
195-
var doc = web.Load(pageUrl);
196-
197-
return doc.GetElementbyId("main-column").InnerText;
198-
}
199-
public async Task<string> QueryWikipedia(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext, string query)
200-
{
201-
await turnContext.SendActivityAsync($"Searching Wikipedia for \"{query}\"...");
202-
HttpResponseMessage response = await client.GetAsync(
203-
$"https://en.wikipedia.org/w/api.php?action=opensearch&search={UrlEncoder.Default.Encode(query)}&limit=1"
204-
);
205-
if (response.IsSuccessStatusCode)
206-
return await response.Content.ReadAsStringAsync();
207-
else
208-
return $"FAILED TO FETCH DATA FROM API. STATUS CODE {response.StatusCode}";
209-
}
210-
public async Task<string> GetWikipediaContent(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext, string pageTitle)
211-
{
212-
await turnContext.SendActivityAsync($"Getting article \"{pageTitle}\"...");
213-
HttpResponseMessage response = await client.GetAsync(
214-
$"https://en.wikipedia.org/w/api.php?action=query&format=json&titles={UrlEncoder.Default.Encode(pageTitle)}&prop=extracts&explaintext"
215-
);
216-
if (response.IsSuccessStatusCode)
217-
return await response.Content.ReadAsStringAsync();
218-
else
219-
return $"FAILED TO FETCH DATA FROM API. STATUS CODE {response.StatusCode}";
220-
221-
}
127+
for (var j = messages[i].Content.Count() - 1; j >= 0; j--)
128+
{
129+
if (messages[i].Content[j].Type == "text")
130+
{
131+
responses.Add(messages[i].Content[j].Text.Value);
132+
await turnContext.SendActivityAsync(messages[i].Content[j].Text.Value);
133+
}
134+
if (messages[i].Content[j].Type == "image_file")
135+
{
136+
responses.Add($"Image (ID: {messages[i].Content[j].ImageFile.FileId})");
137+
List<object> images = [new { type = "Image", url = $"{_appUrl}/openai/files/{messages[i].Content[j].ImageFile.FileId}/content" }];
138+
object adaptiveCardJson = new
139+
{
140+
type = "AdaptiveCard",
141+
version = "1.0",
142+
body = images
143+
};
222144

223-
class QueryWikipediaArguments
224-
{
225-
[JsonPropertyName("query")]
226-
public string Query { get; set; }
227-
}
228-
class GetWikipediaContentArguments
229-
{
230-
[JsonPropertyName("page_title")]
231-
public string PageTitle { get; set; }
145+
var adaptiveCardAttachment = new Bot.Schema.Attachment()
146+
{
147+
ContentType = "application/vnd.microsoft.card.adaptive",
148+
Content = adaptiveCardJson,
149+
};
150+
await turnContext.SendActivityAsync(MessageFactory.Attachment(adaptiveCardAttachment));
151+
}
152+
}
153+
}
154+
return responses;
232155
}
233156
}
234157
}

‎gen-ai/Assistants/bot-in-a-box/src/Bots/StateManagementBot.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,6 @@ protected override async Task OnMessageActivityAsync(ITurnContext<IMessageActivi
6161
var userTokenClient = turnContext.TurnState.Get<UserTokenClient>();
6262

6363
// -- Special keywords
64-
// Clear conversation
6564
if (turnContext.Activity.Text != null)
6665
{
6766
if (turnContext.Activity.Text.ToLower() == "logout")
@@ -93,17 +92,17 @@ protected override async Task OnMessageActivityAsync(ITurnContext<IMessageActivi
9392

9493
conversationData.History.Add(new ConversationTurn { Role = "user", Message = turnContext.Activity.Text });
9594

96-
var replyText = await ProcessMessage(conversationData, turnContext);
95+
var replies = await ProcessMessage(conversationData, turnContext);
9796

98-
99-
conversationData.History.Add(new ConversationTurn { Role = "assistant", Message = replyText });
97+
foreach (var reply in replies) {
98+
conversationData.History.Add(new ConversationTurn { Role = "assistant", Message = reply });
99+
}
100100

101101
if (turnContext.Activity.Text == null || turnContext.Activity.Text.ToLower() == "")
102102
{
103103
return;
104104
}
105105

106-
await turnContext.SendActivityAsync(replyText);
107106

108107
conversationData.History = conversationData.History.GetRange(
109108
Math.Max(conversationData.History.Count - _max_messages, 0),
@@ -116,10 +115,10 @@ protected override async Task OnMessageActivityAsync(ITurnContext<IMessageActivi
116115

117116
}
118117

119-
public virtual async Task<string> ProcessMessage(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext)
118+
public virtual async Task<List<string>> ProcessMessage(ConversationData conversationData, ITurnContext<IMessageActivity> turnContext)
120119
{
121120
await turnContext.SendActivityAsync(JsonSerializer.Serialize(conversationData.History));
122-
return $"This chat now contains {conversationData.History.Count} messages";
121+
return new List<string>{ $"This chat now contains {conversationData.History.Count} messages" };
123122
}
124123

125124
public string FormatConversationHistory(ConversationData conversationData)

0 commit comments

Comments
 (0)