Skip to content

Commit 5e32149

Browse files
author
AndrewMorgan1
committed
Added Missing code coverage tests
1 parent 5de47d0 commit 5e32149

3 files changed

Lines changed: 130 additions & 0 deletions

File tree

tests/TokenKit.Tests/Registry/JsonModelRegistryTests.cs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,5 +75,42 @@ public void GetAll_Should_Filter_By_Provider()
7575
Assert.Equal(2, openaiModels.Count);
7676
Assert.All(openaiModels, m => Assert.Equal("OpenAI", m.Provider));
7777
}
78+
79+
[Fact]
80+
public void Constructor_ShouldDeserializeValidJsonFile()
81+
{
82+
// Arrange
83+
var tempDir = Path.Combine(Path.GetTempPath(), $"TokenKitTests_{Guid.NewGuid()}");
84+
Directory.CreateDirectory(tempDir);
85+
var filePath = Path.Combine(tempDir, "models.data.json");
86+
87+
var models = new List<ModelInfo>
88+
{
89+
new() { Id = "gpt-4o", Provider = "OpenAI", MaxTokens = 100000, InputPricePer1K = 0.005m, OutputPricePer1K = 0.015m, Encoding = "cl100k_base" }
90+
};
91+
File.WriteAllText(filePath, System.Text.Json.JsonSerializer.Serialize(models));
92+
93+
// Act
94+
var registry = new JsonModelRegistry(filePath);
95+
var result = registry.GetAll();
96+
97+
// Assert
98+
Assert.Single(result);
99+
Assert.Equal("gpt-4o", result[0].Id);
100+
}
101+
102+
[Fact]
103+
public void Constructor_ShouldFallbackToEmptyList_WhenFileEmpty()
104+
{
105+
var tempDir = Path.Combine(Path.GetTempPath(), $"TokenKitTests_{Guid.NewGuid()}");
106+
Directory.CreateDirectory(tempDir);
107+
var filePath = Path.Combine(tempDir, "models.data.json");
108+
File.WriteAllText(filePath, ""); // empty file => Deserialize returns null, triggers ?? new()
109+
110+
var registry = new JsonModelRegistry(filePath);
111+
var result = registry.GetAll();
112+
113+
Assert.Empty(result);
114+
}
78115
}
79116
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
using TokenKit.Core.Implementations;
2+
using TokenKit.Core.Interfaces;
3+
using TokenKit.Core.Models;
4+
5+
namespace TokenKit.Tests.Services
6+
{
7+
public class SimpleTokenizerEngineTests
8+
{
9+
private readonly ITokenizerEngine _engine = new SimpleTokenizerEngine();
10+
private readonly ModelInfo _model = new() { Id = "test", Encoding = "none" };
11+
12+
[Fact]
13+
public void CountTokens_ShouldReturnZero_WhenTextIsNullOrWhitespace()
14+
{
15+
Assert.Equal(0, _engine.CountTokens(null!, _model)); // null
16+
Assert.Equal(0, _engine.CountTokens("", _model)); // empty
17+
Assert.Equal(0, _engine.CountTokens(" ", _model)); // whitespace
18+
Assert.Equal(0, _engine.CountTokens("\n\r\t", _model)); // control chars
19+
}
20+
21+
[Fact]
22+
public void CountTokens_ShouldReturnCorrectCount_ForSimpleText()
23+
{
24+
var text = "Hello world from TokenKit";
25+
var count = _engine.CountTokens(text, _model);
26+
27+
Assert.Equal(4, count); // 4 words
28+
}
29+
30+
[Fact]
31+
public void CountTokens_ShouldHandleMixedWhitespaceCorrectly()
32+
{
33+
var text = "Hello\tworld\nfrom TokenKit\rrocks";
34+
var count = _engine.CountTokens(text, _model);
35+
36+
Assert.Equal(5, count);
37+
}
38+
39+
[Fact]
40+
public void Name_ShouldBeSimple()
41+
{
42+
Assert.Equal("simple", _engine.Name);
43+
}
44+
}
45+
}

tests/TokenKit.Tests/Services/TokenizerTests.cs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,54 @@ public async Task InvalidEngine_ShouldFallbackToDefault()
9595
Assert.Equal("simple", result.Engine);
9696
}
9797

98+
[Fact]
99+
public async Task Analyze_ShouldThrow_WhenModelIsUnknown()
100+
{
101+
// Arrange: minimal Core setup with empty registry
102+
var services = new ServiceCollection();
103+
services.AddSingleton<IModelRegistry>(new InMemoryModelRegistry(Array.Empty<ModelInfo>()));
104+
services.AddSingleton<ITokenizerEngine, SimpleTextEncoder>();
105+
services.AddSingleton<ICostEstimator, BasicCostEstimator>();
106+
services.AddSingleton<ITokenKitCore, TokenKitCore>();
107+
108+
var core = services.BuildServiceProvider().GetRequiredService<ITokenKitCore>();
109+
110+
// Act & Assert
111+
var ex = await Assert.ThrowsAsync<ArgumentException>(async () =>
112+
{
113+
await core.AnalyzeAsync(new AnalyzeRequest
114+
{
115+
Text = "This should fail",
116+
ModelId = "non-existent-model"
117+
});
118+
});
119+
120+
Assert.Contains("Unknown model", ex.Message);
121+
}
122+
123+
[Fact]
124+
public async Task Validate_ShouldThrow_WhenModelIsUnknown()
125+
{
126+
var services = new ServiceCollection();
127+
services.AddSingleton<IModelRegistry>(new InMemoryModelRegistry(Array.Empty<ModelInfo>()));
128+
services.AddSingleton<ITokenizerEngine, SimpleTextEncoder>();
129+
services.AddSingleton<ICostEstimator, BasicCostEstimator>();
130+
services.AddSingleton<ITokenKitCore, TokenKitCore>();
131+
132+
var core = services.BuildServiceProvider().GetRequiredService<ITokenKitCore>();
133+
134+
var ex = await Assert.ThrowsAsync<ArgumentException>(async () =>
135+
{
136+
await core.ValidateAsync(new ValidateRequest
137+
{
138+
Text = "Some text",
139+
ModelId = "non-existent-model"
140+
});
141+
});
142+
143+
Assert.Contains("Unknown model", ex.Message);
144+
}
145+
98146
private sealed class InMemoryModelRegistry : IModelRegistry
99147
{
100148
private readonly List<ModelInfo> _models;

0 commit comments

Comments
 (0)