Skip to content

feat: add configurable max tokens support #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
antispam2002 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat: add configurable max tokens support #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
633b4e5
chore: add configurable max tokens support
Jun 6, 2026
0290529
chore: appliy parse fix
Jun 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Assets/settings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
// Define backends that don't need base URL configuration
const FIXED_URL_BACKENDS = ['openai', 'anthropic', 'openrouter', 'grok'];

// Define the default max token limit for requests
const DEFAULT_MAX_TOKENS = 1024;

// Define default feature to instruction mappings
const DEFAULT_FEATURE_MAPPINGS = {
'enhance-prompt': 'prompt',
Expand Down Expand Up @@ -74,6 +77,7 @@ async function loadSettings() {
serverSettings.backend ||
'ollama',
visionmodel: serverSettings.visionmodel || '',
max_tokens: Number(serverSettings.max_tokens) || DEFAULT_MAX_TOKENS,
linkChatAndVisionModels:
serverSettings.linkChatAndVisionModels !== false, // Default to true if not set
// Backends - merge using spread operator which does a "deep merge" of two objects
Expand Down Expand Up @@ -190,13 +194,15 @@ async function saveSettings(skipFeatureMappings = false) {
const visionTimeout = isLinked
? chatTimeout
: parseInt(document.getElementById('visionTimeout')?.value, 10);
const maxTokens = parseInt(document.getElementById('maxTokens')?.value, 10);
// Create settings object matching exact structure expected by C# DefaultSettings
const settings = {
// Core settings
backend: chatBackendId,
model: chatModel,
visionbackend: visionBackendId,
visionmodel: visionModel,
max_tokens: !isNaN(maxTokens) && maxTokens > 0 ? maxTokens : MP.settings.max_tokens,
linkChatAndVisionModels: isLinked,
backends: {
...MP.settings.backends,
Expand Down Expand Up @@ -1821,6 +1827,10 @@ function initSettingsModal() {
const defaultChatTimeout = currentBackend === 'ollama' || currentBackend === 'openaiapi' ? 120 : 60;
chatTimeoutInput.value = MP.settings.backends[currentBackend]?.timeout ?? defaultChatTimeout;
}
const maxTokensInput = document.getElementById('maxTokens');
if (maxTokensInput) {
maxTokensInput.value = MP.settings.max_tokens ?? DEFAULT_MAX_TOKENS;
}
const currentVisionBackend = MP.settings.visionbackend || 'ollama';
const currentVisionBackendRadio = document.getElementById(
`${currentVisionBackend}VisionBtn`
Expand Down
28 changes: 15 additions & 13 deletions BackendSchema.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ namespace Hartsy.Extensions.MagicPromptExtension;

public static class BackendSchema
{
public static readonly int DefaultMaxTokens = 1024;

public enum MessageType
{
Text,
Expand Down Expand Up @@ -37,7 +39,7 @@ public class MediaContent
/// <param name="model">Model name to use</param>
/// <param name="messageType">Type of message (Text or Vision)</param>
/// <returns>Returns an object with the schema type for the backend.</returns>
public static object GetSchemaType(string type, MessageContent content, string model, MessageType messageType = MessageType.Text, long seed = -1)
public static object GetSchemaType(string type, MessageContent content, string model, MessageType messageType = MessageType.Text, long seed = -1, int? maxTokens = null)
{
if (content == null || string.IsNullOrEmpty(model))
{
Expand All @@ -48,9 +50,9 @@ public static object GetSchemaType(string type, MessageContent content, string m
return type switch
{
"ollama" => OllamaRequestBody(content, model, messageType, seed),
"grok" => OpenAICompatibleRequestBody(content, model, messageType, preferPngForBase64: true, seed),
"openai" or "openaiapi" or "openrouter" => OpenAICompatibleRequestBody(content, model, messageType, preferPngForBase64: false, seed),
"anthropic" => AnthropicRequestBody(content, model, messageType),
"grok" => OpenAICompatibleRequestBody(content, model, messageType, preferPngForBase64: true, seed, maxTokens),
"openai" or "openaiapi" or "openrouter" => OpenAICompatibleRequestBody(content, model, messageType, preferPngForBase64: false, seed, maxTokens),
"anthropic" => AnthropicRequestBody(content, model, messageType, maxTokens),
_ => throw new ArgumentException($"Unsupported backend type: {type}")
};
}
Expand Down Expand Up @@ -139,7 +141,7 @@ private static object OllamaRequestBody(MessageContent content, string model, Me
}

/// <summary>Generates a request body for OpenAI and compatible backends.</summary>
private static object OpenAICompatibleRequestBody(MessageContent content, string model, MessageType messageType, bool preferPngForBase64, long seed = -1)
private static object OpenAICompatibleRequestBody(MessageContent content, string model, MessageType messageType, bool preferPngForBase64, long seed = -1, int? maxTokens = null)
{
List<object> messages = [];
// Add system message if instructions exist
Expand Down Expand Up @@ -178,7 +180,7 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
{
model,
messages = messages.ToArray(),
max_tokens = 1000,
max_tokens = maxTokens ?? DefaultMaxTokens,
temperature = 1.0,
stream = false,
seed
Expand All @@ -189,7 +191,7 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
{
model,
messages = messages.ToArray(),
max_tokens = 1000,
max_tokens = maxTokens ?? DefaultMaxTokens,
temperature = 1.0,
stream = false
};
Expand All @@ -202,7 +204,7 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
{
model,
messages = messages.ToArray(),
max_tokens = 1000,
max_tokens = maxTokens ?? DefaultMaxTokens,
temperature = 1.0,
stream = false,
seed
Expand All @@ -214,14 +216,14 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
model,
messages = messages.ToArray(),
temperature = 1.0,
max_tokens = 1000,
max_tokens = maxTokens ?? DefaultMaxTokens,
top_p = 0.9,
stream = false
};
}

/// <summary>Generates a request body for the Anthropic (Claude) API.</summary>
private static object AnthropicRequestBody(MessageContent content, string model, MessageType messageType)
private static object AnthropicRequestBody(MessageContent content, string model, MessageType messageType, int? maxTokens = null)
{
List<object> messages = [];
if (messageType == MessageType.Vision && content.Media?.Any() == true)
Expand Down Expand Up @@ -258,7 +260,7 @@ private static object AnthropicRequestBody(MessageContent content, string model,
model,
messages = messages.ToArray(),
system = content.Instructions,
max_tokens = 1024
max_tokens = maxTokens ?? DefaultMaxTokens
};
}
messages.Add(new { role = "user", content = content.Text });
Expand All @@ -267,7 +269,7 @@ private static object AnthropicRequestBody(MessageContent content, string model,
model,
messages = messages.ToArray(),
system = content.Instructions,
max_tokens = 1024
max_tokens = maxTokens ?? DefaultMaxTokens
};
}
}
}
6 changes: 5 additions & 1 deletion Tabs/Text2Image/MagicPrompt.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ <h6 class="settings-section-title">Chat LLM Settings</h6>
<label class="btn btn-outline-primary" for="grokLLMBtn">Grok (xAI)</label>
</div>
</div>
<div class="d-flex gap-3">
<div class="d-flex gap-3 flex-wrap">
<div class="form-group flex-grow-1" id="baseUrlContainer">
<label class="form-label">Base URL</label>
<input type="text" class="form-control" id="backendUrl" placeholder="Enter base URL">
Expand All @@ -190,6 +190,10 @@ <h6 class="settings-section-title">Chat LLM Settings</h6>
<label class="form-label">Timeout (s)</label>
<input type="number" class="form-control" id="chatTimeout" min="0" step="1" placeholder="seconds">
</div>
<div class="form-group" id="chatMaxTokensContainer" style="min-width: 140px;">
<label class="form-label">Max Tokens</label>
<input type="number" class="form-control" id="maxTokens" min="1" step="1" placeholder="tokens">
</div>
</div>
<div class="form-group">
<label class="form-label" for="modelSelect">Model</label>
Expand Down
9 changes: 8 additions & 1 deletion WebAPI/LLMAPICalls.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,14 @@ public static async Task<JObject> MagicPromptPhoneHome(JObject requestData, Sess
object requestBody;
try
{
requestBody = GetSchemaType(backend, messageContent, modelId, messageType, seed);
int maxTokens = BackendSchema.DefaultMaxTokens;
if (long.TryParse(settings["max_tokens"]?.ToString(), out long parsedMaxTokens)
&& parsedMaxTokens >= 1
&& parsedMaxTokens <= int.MaxValue)
{
maxTokens = (int)parsedMaxTokens;
}
requestBody = GetSchemaType(backend, messageContent, modelId, messageType, seed, maxTokens);
}
catch (ArgumentException ex)
{
Expand Down
2 changes: 2 additions & 0 deletions WebAPI/SessionSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Newtonsoft.Json.Linq;
using SwarmUI.Backends;
using SwarmUI.Core;
using SwarmUI.Utils;

Expand Down Expand Up @@ -82,6 +83,7 @@ public class SessionSettings : MagicPromptAPI
["visionbackend"] = "ollama",
["model"] = "llama3.2-vision:latest",
["visionmodel"] = "llama3.2-vision:latest",
["max_tokens"] = BackendSchema.DefaultMaxTokens,
["instructions"] = new JObject
{
["chat"] = "You are a chatbot named Hartsy. Come up with a random backstory as to why you were created and how you were made to help the user with Stable Diffusion. You will respond to any questions or chats in this character. You will include tips on how to make good prompts for stable diffusion. Never break character and randomly end your response with \"Thank you for choosing Hartsy!\"",
Expand Down