Skip to content

Commit 533ae73

Browse files
committed
Use 16k context for cagent new when using the dmr provider to limit memory usage and improve local model compatibility
Signed-off-by: Christopher Petito <chrisjpetito@gmail.com>
1 parent 5305b16 commit 533ae73

1 file changed

Lines changed: 18 additions & 2 deletions

File tree

internal/creator/agent.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,12 @@ func StreamCreateAgent(ctx context.Context, baseDir, prompt string, runConfig la
141141
fmt.Printf("Using default model: %s\n", modelName)
142142
}
143143

144+
// if the user provided a model override, lets use that by default for DMR
145+
// in the generated agentfile
146+
if providerName == "dmr" && modelName == "" {
147+
defaultModels["dmr"] = modelName
148+
}
149+
144150
// If not using a models gateway, avoid selecting a provider the user can't run
145151
var usableProviders []string
146152
if runConfig.ModelsGateway == "" {
@@ -157,12 +163,18 @@ func StreamCreateAgent(ctx context.Context, baseDir, prompt string, runConfig la
157163
usableProviders = append(usableProviders, "dmr")
158164
}
159165

166+
// Use 16k for DMR to limit memory costs
167+
maxTokens := 64000
168+
if providerName == "dmr" {
169+
maxTokens = 16000
170+
}
171+
160172
llm, err := provider.New(
161173
ctx,
162174
&latest.ModelConfig{
163175
Provider: providerName,
164176
Model: modelName,
165-
MaxTokens: 64000,
177+
MaxTokens: maxTokens,
166178
},
167179
environment.NewDefaultProvider(),
168180
options.WithGateway(runConfig.ModelsGateway),
@@ -179,13 +191,17 @@ func StreamCreateAgent(ctx context.Context, baseDir, prompt string, runConfig la
179191
// Provide soft guidance to prefer the selected providers
180192
instructions := agentBuilderInstructions + "\n\nPreferred model providers to use: " + strings.Join(usableProviders, ", ") + ". You must always use one or more of the following model configurations: \n"
181193
for _, provider := range usableProviders {
194+
suggestedMaxTokens := 64000
195+
if provider == "dmr" {
196+
suggestedMaxTokens = 16000
197+
}
182198
instructions += fmt.Sprintf(`
183199
version: "1"
184200
models:
185201
%s:
186202
provider: %s
187203
model: %s
188-
max_tokens: 64000\n`, provider, provider, defaultModels[provider])
204+
max_tokens: %d\n`, provider, provider, defaultModels[provider], suggestedMaxTokens)
189205
}
190206

191207
newTeam := team.New(

0 commit comments

Comments
 (0)