[Firebase AI] Add support for automatic functions (#1426)

* [Firebase AI] Add support for automatic functions

* Addressing gemini review feedback

* Update RequestOptions.cs

Author: a-maurice

docs/readme.md

@@ -116,6 +116,7 @@ Release Notes
       [migrate your apps to use Gemini Image models (the "Nano Banana" models)](https://firebase.google.com/docs/ai-logic/imagen-models-migration).
     - Firebase AI: Add support for the JsonSchema formatting.
     - Firebase AI: Add support for simplified object generation with GenerateObjectAsync.
+    - Firebase AI: Add support for automated function calling in Chat with AutoFunctionDeclaration.
     - Firebase AI: Add support for TemplateChatSession.
     - Functions: Rewrote internal serialization logic to C#. Removes dependency on internal C++ implementation.
 

firebaseai/src/Chat.cs

@@ -32,14 +32,18 @@ public class Chat
     private readonly GenerativeModel generativeModel;
     private readonly List<ModelContent> chatHistory;
 
+    private readonly Dictionary<string, BaseAutoFunctionDeclaration> autoFunctionDeclarations;
+    private readonly int autoFunctionTurnLimit;
+
     /// <summary>
     /// The previous content from the chat that has been successfully sent and received from the
     /// model. This will be provided to the model for each message sent as context for the discussion.
     /// </summary>
     public IReadOnlyList<ModelContent> History => chatHistory;
 
     // Note: No public constructor, get one through GenerativeModel.StartChat
-    private Chat(GenerativeModel model, IEnumerable<ModelContent> initialHistory)
+    private Chat(GenerativeModel model, IEnumerable<ModelContent> initialHistory,
+        IEnumerable<BaseAutoFunctionDeclaration> autoFunctions, int autoFunctionTurnLimit)
     {
       generativeModel = model;
 
@@ -51,15 +55,26 @@ private Chat(GenerativeModel model, IEnumerable<ModelContent> initialHistory)
       {
         chatHistory = new List<ModelContent>();
       }
+
+      if (autoFunctions != null && autoFunctions.Any())
+      {
+        autoFunctionDeclarations = autoFunctions.ToDictionary(afd => afd.Name);
+      }
+      else
+      {
+        autoFunctionDeclarations = null;
+      }
+      this.autoFunctionTurnLimit = autoFunctionTurnLimit;
     }
 
     /// <summary>
     /// Intended for internal use only.
     /// Use `GenerativeModel.StartChat` instead to ensure proper initialization and configuration of the `Chat`.
     /// </summary>
-    internal static Chat InternalCreateChat(GenerativeModel model, IEnumerable<ModelContent> initialHistory)
+    internal static Chat InternalCreateChat(GenerativeModel model, IEnumerable<ModelContent> initialHistory,
+        IEnumerable<BaseAutoFunctionDeclaration> autoFunctionDeclarations, int autoFunctionTurnLimit)
     {
-      return new Chat(model, initialHistory);
+      return new Chat(model, initialHistory, autoFunctionDeclarations, autoFunctionTurnLimit);
     }
 
     /// <summary>
@@ -142,70 +157,31 @@ public IAsyncEnumerable<GenerateContentResponse> SendMessageStreamAsync(
       return SendMessageStreamAsyncInternal(content, cancellationToken);
     }
 
-    private async Task<GenerateContentResponse> SendMessageAsyncInternal(
+    private Task<GenerateContentResponse> SendMessageAsyncInternal(
         IEnumerable<ModelContent> requestContent, CancellationToken cancellationToken = default)
     {
-      // Make sure that the requests are set to to role "user".
-      List<ModelContent> fixedRequests = requestContent.Select(FirebaseAIExtensions.ConvertToUser).ToList();
-      // Set up the context to send in the request
-      List<ModelContent> fullRequest = new(chatHistory);
-      fullRequest.AddRange(fixedRequests);
-
-      // Note: GenerateContentAsync can throw exceptions if there was a problem, but
-      // we allow it to just be passed back to the user.
-      GenerateContentResponse response = await generativeModel.GenerateContentAsync(fullRequest, cancellationToken);
-
-      // Only after getting a valid response, add both to the history for later.
-      // But either way pass the response along to the user.
-      if (response.Candidates.Any())
+      Task<GenerateContentResponse> generateContentFunc(List<ModelContent> fullRequest)
       {
-        ModelContent responseContent = response.Candidates.First().Content;
-
-        chatHistory.AddRange(fixedRequests);
-        chatHistory.Add(responseContent.ConvertToModel());
+        return generativeModel.GenerateContentAsync(fullRequest, cancellationToken);
       }
 
-      return response;
+      return ChatSessionHelpers.SendMessageAsync(chatHistory,
+          autoFunctionDeclarations, autoFunctionTurnLimit,
+          requestContent, generateContentFunc);
     }
 
-    private async IAsyncEnumerable<GenerateContentResponse> SendMessageStreamAsyncInternal(
+    private IAsyncEnumerable<GenerateContentResponse> SendMessageStreamAsyncInternal(
         IEnumerable<ModelContent> requestContent,
-        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+        CancellationToken cancellationToken = default)
     {
-      // Make sure that the requests are set to to role "user".
-      List<ModelContent> fixedRequests = requestContent.Select(FirebaseAIExtensions.ConvertToUser).ToList();
-      // Set up the context to send in the request
-      List<ModelContent> fullRequest = new(chatHistory);
-      fullRequest.AddRange(fixedRequests);
-
-      List<ModelContent> responseContents = new();
-      bool saveHistory = true;
-      // Note: GenerateContentStreamAsync can throw exceptions if there was a problem, but
-      // we allow it to just be passed back to the user.
-      await foreach (GenerateContentResponse response in
-          generativeModel.GenerateContentStreamAsync(fullRequest, cancellationToken))
+      IAsyncEnumerable<GenerateContentResponse> generateContentStreamFunc(List<ModelContent> fullRequest)
       {
-        // If the response had a problem, we still want to pass it along to the user for context,
-        // but we don't want to save the history anymore.
-        if (response.Candidates.Any())
-        {
-          ModelContent responseContent = response.Candidates.First().Content;
-          responseContents.Add(responseContent.ConvertToModel());
-        }
-        else
-        {
-          saveHistory = false;
-        }
-
-        yield return response;
+        return generativeModel.GenerateContentStreamAsync(fullRequest, cancellationToken);
       }
 
-      // After getting all the responses, and they were all valid, add everything to the history
-      if (saveHistory)
-      {
-        chatHistory.AddRange(fixedRequests);
-        chatHistory.AddRange(responseContents);
-      }
+      return ChatSessionHelpers.SendMessageStreamAsync(chatHistory,
+          autoFunctionDeclarations, autoFunctionTurnLimit,
+          requestContent, generateContentStreamFunc);
     }
   }
 

firebaseai/src/FunctionCalling.cs

@@ -17,6 +17,8 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reflection;
+using Firebase.AI.Internal;
 
 namespace Firebase.AI
 {
@@ -86,9 +88,9 @@ public FunctionDeclaration(string name, string description,
     internal Dictionary<string, object> ToJson()
     {
       var json = new Dictionary<string, object>() {
-      { "name", Name },
-      { "description", Description },
-    };
+        { "name", Name },
+        { "description", Description },
+      };
       // Only one of these will likely be set, but just check
       if (JsonParameters != null)
       {
@@ -103,6 +105,27 @@ internal Dictionary<string, object> ToJson()
     }
   }
 
+  /// <summary>
+  /// Structured representation of a function declaration, designed
+  /// to be automatically handled when using Chat, instead of requiring
+  /// manual handling.
+  /// 
+  /// See `FunctionDeclaration` for more information.
+  /// </summary>
+  public class AutoFunctionDeclaration : BaseAutoFunctionDeclaration
+  {
+    /// <summary>
+    /// Constructs a new `AutoFunctionDeclaration`
+    /// </summary>
+    /// <param name="callable">The delegate that will be called automatically when requested by the model.</param>
+    /// <param name="description">A brief description of the function.</param>
+    /// <param name="name">Optional name to use for the function, used to overwrite the delegate name.</param>
+    public AutoFunctionDeclaration(Delegate callable, string description,
+        string name = null)
+        : base(callable, description, name)
+    { }
+  }
+
   /// <summary>
   /// A tool that allows the generative model to connect to Google Search to access and incorporate
   /// up-to-date information from the web into its responses.
@@ -134,6 +157,7 @@ public readonly struct Tool
     // No public properties, on purpose since it is meant for user input only
 
     private List<FunctionDeclaration> FunctionDeclarations { get; }
+    internal List<AutoFunctionDeclaration> AutoFunctionDeclarations { get; }
     private GoogleSearch? GoogleSearch { get; }
     private CodeExecution? CodeExecution { get; }
     private UrlContext? UrlContext { get; }
@@ -146,6 +170,7 @@ public readonly struct Tool
     public Tool(params FunctionDeclaration[] functionDeclarations)
     {
       FunctionDeclarations = new List<FunctionDeclaration>(functionDeclarations);
+      AutoFunctionDeclarations = null;
       GoogleSearch = null;
       CodeExecution = null;
       UrlContext = null;
@@ -158,6 +183,34 @@ public Tool(params FunctionDeclaration[] functionDeclarations)
     public Tool(IEnumerable<FunctionDeclaration> functionDeclarations)
     {
       FunctionDeclarations = new List<FunctionDeclaration>(functionDeclarations);
+      AutoFunctionDeclarations = null;
+      GoogleSearch = null;
+      CodeExecution = null;
+      UrlContext = null;
+    }
+
+    /// <summary>
+    /// Creates a tool that allows the model to perform function calling.
+    /// </summary>
+    /// <param name="functionDeclarations">A list of `FunctionDeclarations` available to the model
+    ///   that can be used for function calling.</param>
+    public Tool(params AutoFunctionDeclaration[] functionDeclarations)
+    {
+      FunctionDeclarations = null;
+      AutoFunctionDeclarations = new List<AutoFunctionDeclaration>(functionDeclarations);
+      GoogleSearch = null;
+      CodeExecution = null;
+      UrlContext = null;
+    }
+    /// <summary>
+    /// Creates a tool that allows the model to perform function calling.
+    /// </summary>
+    /// <param name="functionDeclarations">A list of `FunctionDeclarations` available to the model
+    ///   that can be used for function calling.</param>
+    public Tool(IEnumerable<AutoFunctionDeclaration> functionDeclarations)
+    {
+      FunctionDeclarations = null;
+      AutoFunctionDeclarations = new List<AutoFunctionDeclaration>(functionDeclarations);
       GoogleSearch = null;
       CodeExecution = null;
       UrlContext = null;
@@ -171,6 +224,7 @@ public Tool(IEnumerable<FunctionDeclaration> functionDeclarations)
     public Tool(GoogleSearch googleSearch)
     {
       FunctionDeclarations = null;
+      AutoFunctionDeclarations = null;
       GoogleSearch = googleSearch;
       CodeExecution = null;
       UrlContext = null;
@@ -184,6 +238,7 @@ public Tool(GoogleSearch googleSearch)
     public Tool(CodeExecution codeExecution)
     {
       FunctionDeclarations = null;
+      AutoFunctionDeclarations = null;
       GoogleSearch = null;
       CodeExecution = codeExecution;
       UrlContext = null;
@@ -198,6 +253,7 @@ public Tool(CodeExecution codeExecution)
     public Tool(UrlContext urlContext)
     {
       FunctionDeclarations = null;
+      AutoFunctionDeclarations = null;
       GoogleSearch = null;
       CodeExecution = null;
       UrlContext = urlContext;
@@ -210,9 +266,18 @@ public Tool(UrlContext urlContext)
     internal Dictionary<string, object> ToJson()
     {
       var json = new Dictionary<string, object>();
+      List<Dictionary<string, object>> functionDeclarations = new();
       if (FunctionDeclarations != null && FunctionDeclarations.Any())
       {
-        json["functionDeclarations"] = FunctionDeclarations.Select(f => f.ToJson()).ToList();
+        functionDeclarations.AddRange(FunctionDeclarations.Select(f => f.ToJson()));
+      }
+      if (AutoFunctionDeclarations != null && AutoFunctionDeclarations.Any())
+      {
+        functionDeclarations.AddRange(AutoFunctionDeclarations.Select(f => f.ToJson()));
+      }
+      if (functionDeclarations.Count > 0)
+      {
+        json["functionDeclarations"] = functionDeclarations;
       }
       if (GoogleSearch.HasValue)
       {
@@ -342,4 +407,73 @@ internal Dictionary<string, object> ToJson()
     }
   }
 
+  /// <summary>
+  /// Attribute that can be attached to parameters to give them a description,
+  /// used when using `AutoFunctionDeclaration`.
+  /// </summary>
+  [AttributeUsage(AttributeTargets.Parameter)]
+  public class AutoFunctionDescriptionAttribute : Attribute
+  {
+    public AutoFunctionDescriptionAttribute(string description)
+    {
+      Description = description;
+    }
+
+    public string Description { get; set; } = null;
+  }
+
+#if !DOXYGEN
+  public abstract class BaseAutoFunctionDeclaration
+  {
+    internal string Name { get; }
+    internal string Description { get; }
+    internal JsonSchema Parameters { get; }
+
+    internal Delegate Callable { get; }
+
+    public BaseAutoFunctionDeclaration(Delegate callable, string description,
+        string name = null)
+    {
+      if (callable == null)
+      {
+        throw new ArgumentNullException(nameof(callable));
+      }
+      Name = name ?? callable.Method.Name;
+      Description = description;
+
+      Dictionary<string, JsonSchema> parameters = new();
+      List<string> optionalParameters = new();
+      // Construct the Parameters based on the given Delegate
+      foreach (var pInfo in callable.Method.GetParameters())
+      {
+        var attr = pInfo.GetCustomAttribute<AutoFunctionDescriptionAttribute>();
+        parameters.Add(pInfo.Name,
+            JsonSchema.FromType(pInfo.ParameterType, attr?.Description));
+        if (pInfo.HasDefaultValue)
+        {
+          optionalParameters.Add(pInfo.Name);
+        }
+      }
+      Parameters = JsonSchema.Object(parameters, optionalParameters);
+
+      Callable = callable;
+    }
+
+    /// <summary>
+    /// Intended for internal use only.
+    /// This method is used for serializing the object to JSON for the API request.
+    /// </summary>
+    internal Dictionary<string, object> ToJson()
+    {
+      var json = new Dictionary<string, object>() {
+        { "name", Name },
+        { "parametersJsonSchema", Parameters.ToJson() }
+      };
+      json.AddIfHasValue("description", Description);
+
+      return json;
+    }
+  }
+#endif // !DOXYGEN
+
 }

firebaseai/src/GenerativeModel.cs

@@ -255,7 +255,12 @@ public Chat StartChat(params ModelContent[] history)
     /// <param name="history">Initial content history to start with.</param>
     public Chat StartChat(IEnumerable<ModelContent> history)
     {
-      return Chat.InternalCreateChat(this, history);
+      // If we have auto functions, we pass them separately.
+      var autoFunctions = _tools?.Select(tool => tool.AutoFunctionDeclarations)
+          .Where(afd => afd != null)
+          .SelectMany(inner => inner);
+      return Chat.InternalCreateChat(this, history, autoFunctions,
+          _requestOptions?.AutoFunctionTurnLimit ?? RequestOptions.DefaultAutoFunctionTurnLimit);
     }
     #endregion