Added core project and integration for CLI to use core libraries

Author: AndrewMorgan1

TokenKit.sln

@@ -1,15 +1,17 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.14.36518.9 d17.14
+VisualStudioVersion = 17.14.36518.9
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{44399DF1-452C-412D-A5E4-8B13EB196561}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TokenKit", "src\TokenKit\TokenKit.csproj", "{74BBAD68-4EFF-4453-B5EA-183659224542}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TokenKit", "src\TokenKit\TokenKit.csproj", "{74BBAD68-4EFF-4453-B5EA-183659224542}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TokenKit.Tests", "tests\TokenKit.Tests\TokenKit.Tests.csproj", "{636A7797-0481-4825-89D1-1B294C132FD9}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TokenKit.Tests", "tests\TokenKit.Tests\TokenKit.Tests.csproj", "{636A7797-0481-4825-89D1-1B294C132FD9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TokenKit.Core", "src\TokenKit.Core\TokenKit.Core.csproj", "{89781812-0C24-4F01-B670-F2AA668A6220}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -25,13 +27,18 @@ Global
 		{636A7797-0481-4825-89D1-1B294C132FD9}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{636A7797-0481-4825-89D1-1B294C132FD9}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{636A7797-0481-4825-89D1-1B294C132FD9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{89781812-0C24-4F01-B670-F2AA668A6220}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{89781812-0C24-4F01-B670-F2AA668A6220}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{89781812-0C24-4F01-B670-F2AA668A6220}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{89781812-0C24-4F01-B670-F2AA668A6220}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{74BBAD68-4EFF-4453-B5EA-183659224542} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{636A7797-0481-4825-89D1-1B294C132FD9} = {44399DF1-452C-412D-A5E4-8B13EB196561}
+		{89781812-0C24-4F01-B670-F2AA668A6220} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {615E1888-FAAC-4D09-AF03-BA43CCD7E21E}

src/TokenKit.Core/Extensions/ServiceCollectionExtensions.cs

@@ -0,0 +1,21 @@
+using Microsoft.Extensions.DependencyInjection;
+using TokenKit.Core.Implementations;
+using TokenKit.Core.Interfaces;
+
+namespace TokenKit.Core.Extensions;
+
+public static class ServiceCollectionExtensions
+{
+    /// <summary>
+    /// Registers TokenKit.Core services. If <paramref name="jsonPath"/> is null,
+    /// the registry will look for models.data.json in AppContext.BaseDirectory.
+    /// </summary>
+    public static IServiceCollection AddTokenKitCore(this IServiceCollection services, string? jsonPath = null)
+    {
+        services.AddSingleton<IModelRegistry>(_ => new JsonModelRegistry(jsonPath));
+        services.AddSingleton<ICostEstimator, BasicCostEstimator>();
+        services.AddSingleton<ITokenKitCore, TokenKitCore>();
+        return services;
+    }
+}
+

src/TokenKit.Core/Implementations/BasicCostEstimator.cs

@@ -0,0 +1,13 @@
+using TokenKit.Core.Interfaces;
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Implementations;
+
+public sealed class BasicCostEstimator : ICostEstimator
+{
+    public decimal EstimateTotal(ModelInfo model, int tokenCount)
+    {
+        var perToken = (model.InputPricePer1K + model.OutputPricePer1K) / 1000m;
+        return Math.Round(tokenCount * perToken, 6);
+    }
+}

src/TokenKit.Core/Implementations/JsonModelRegistry.cs

@@ -0,0 +1,49 @@
+using System.Text.Json;
+using TokenKit.Core.Interfaces;
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Implementations;
+
+public sealed class JsonModelRegistry : IModelRegistry
+{
+    private readonly List<ModelInfo> _models;
+
+    /// <param name="path">
+    /// Optional custom path to models.data.json.
+    /// If null, resolves to: AppContext.BaseDirectory/models.data.json
+    /// </param>
+    public JsonModelRegistry(string? path = null)
+    {
+        var file = path ?? Path.Combine(AppContext.BaseDirectory, "models.data.json");
+
+        if (File.Exists(file))
+        {
+            try
+            {
+                var json = File.ReadAllText(file);
+                _models = JsonSerializer.Deserialize<List<ModelInfo>>(json) ?? new();
+            }
+            catch (Exception ex) when (ex is JsonException || ex is IOException)
+            {
+                // Gracefully handle corrupt or unreadable JSON
+                _models = new List<ModelInfo>();
+            }
+        }
+        else
+        {
+            _models = new List<ModelInfo>();
+        }
+    }
+
+    public ModelInfo? Get(string id) =>
+        _models.FirstOrDefault(m => string.Equals(m.Id, id, StringComparison.OrdinalIgnoreCase));
+
+    public IReadOnlyList<ModelInfo> GetAll(string? provider = null)
+    {
+        if (string.IsNullOrWhiteSpace(provider)) return _models;
+        return _models
+            .Where(m => string.Equals(m.Provider, provider, StringComparison.OrdinalIgnoreCase))
+            .ToList();
+    }
+}
+

src/TokenKit.Core/Implementations/SimpleTokenizerEngine.cs

@@ -0,0 +1,15 @@
+using TokenKit.Core.Interfaces;
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Implementations;
+
+public sealed class SimpleTokenizerEngine : ITokenizerEngine
+{
+    public string Name => "simple";
+
+    public int CountTokens(string text, ModelInfo model)
+        => string.IsNullOrWhiteSpace(text)
+            ? 0
+            : text.Split((char[])[' ', '\n', '\r', '\t'], StringSplitOptions.RemoveEmptyEntries).Length;
+}
+

src/TokenKit.Core/Implementations/TokenKitCore.cs

@@ -0,0 +1,70 @@
+using TokenKit.Core.Interfaces;
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Implementations;
+
+public sealed class TokenKitCore : ITokenKitCore
+{
+    private readonly IModelRegistry _registry;
+    private readonly IDictionary<string, ITokenizerEngine> _engines;
+    private readonly ICostEstimator _costs;
+
+    public TokenKitCore(IModelRegistry registry, IEnumerable<ITokenizerEngine> engines, ICostEstimator costs)
+    {
+        _registry = registry;
+        _engines = engines.ToDictionary(e => e.Name, StringComparer.OrdinalIgnoreCase);
+        _costs = costs;
+    }
+
+    public Task<IReadOnlyList<ModelInfo>> GetModelsAsync(string? provider = null, CancellationToken ct = default)
+        => Task.FromResult(_registry.GetAll(provider));
+
+    public Task<AnalyzeResponse> AnalyzeAsync(AnalyzeRequest req, CancellationToken ct = default)
+    {
+        var model = _registry.Get(req.ModelId) ?? throw new ArgumentException($"Unknown model '{req.ModelId}'.");
+        var engine = ResolveEngine(req.Engine);
+
+        var count = engine.CountTokens(req.Text, model);
+        var cost = _costs.EstimateTotal(model, count);
+        var valid = count <= model.MaxTokens;
+
+        return Task.FromResult(new AnalyzeResponse
+        {
+            Model = model.Id,
+            Provider = model.Provider,
+            Engine = engine.Name,
+            TokenCount = count,
+            EstimatedCost = cost,
+            Valid = valid,
+            Message = valid ? "OK" : $"Exceeds MaxTokens ({model.MaxTokens})."
+        });
+    }
+
+    public async Task<ValidateResponse> ValidateAsync(ValidateRequest req, CancellationToken ct = default)
+    {
+        var a = await AnalyzeAsync(new AnalyzeRequest
+        {
+            Text = req.Text,
+            ModelId = req.ModelId,
+            Engine = req.Engine
+        }, ct);
+
+        return new ValidateResponse
+        {
+            IsValid = a.Valid,
+            Message = a.Message,
+            TokenCount = a.TokenCount,
+            MaxTokens = _registry.Get(req.ModelId)!.MaxTokens
+        };
+    }
+
+    private ITokenizerEngine ResolveEngine(string? requested)
+    {
+        if (!string.IsNullOrWhiteSpace(requested) && _engines.TryGetValue(requested, out var eng))
+            return eng;
+
+        // default to first registered engine
+        return _engines.Values.First();
+    }
+}
+

src/TokenKit.Core/Interfaces/ICostEstimator.cs

@@ -0,0 +1,9 @@
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Interfaces;
+
+public interface ICostEstimator
+{
+    decimal EstimateTotal(ModelInfo model, int tokenCount);
+}
+

src/TokenKit.Core/Interfaces/IModelRegistry.cs

@@ -0,0 +1,10 @@
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Interfaces;
+
+public interface IModelRegistry
+{
+    ModelInfo? Get(string id);
+    IReadOnlyList<ModelInfo> GetAll(string? provider = null);
+}
+

src/TokenKit.Core/Interfaces/ITokenKitCore.cs

@@ -0,0 +1,11 @@
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Interfaces;
+
+public interface ITokenKitCore
+{
+    Task<AnalyzeResponse> AnalyzeAsync(AnalyzeRequest req, CancellationToken ct = default);
+    Task<ValidateResponse> ValidateAsync(ValidateRequest req, CancellationToken ct = default);
+    Task<IReadOnlyList<ModelInfo>> GetModelsAsync(string? provider = null, CancellationToken ct = default);
+}
+

src/TokenKit.Core/Interfaces/ITokenizerEngine.cs

@@ -0,0 +1,10 @@
+using TokenKit.Core.Models;
+
+namespace TokenKit.Core.Interfaces;
+
+public interface ITokenizerEngine
+{
+    string Name { get; }
+    int CountTokens(string text, ModelInfo model);
+}
+