From d0beff9eda0b057dd83b0436e08d7e0000ebff2d Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Tue, 9 Jun 2026 11:25:30 +0200 Subject: [PATCH] Add WWDC26 Apple Intelligence APIs --- packages/apple-llm/README.md | 12 +- packages/apple-llm/ios/AppleLLM.mm | 17 +- packages/apple-llm/ios/AppleLLMError.swift | 4 +- packages/apple-llm/ios/AppleLLMImpl.swift | 584 +++++++++++++++-- .../apple-llm/src/AppleFoundationModels.ts | 24 + packages/apple-llm/src/NativeAppleLLM.ts | 4 + packages/apple-llm/src/ai-sdk.ts | 397 ++++++++++- packages/apple-llm/src/index.ts | 17 +- website/src/docs/apple/generating.md | 619 ++++++++---------- 9 files changed, 1275 insertions(+), 403 deletions(-) diff --git a/packages/apple-llm/README.md b/packages/apple-llm/README.md index 22e3c148..6da68299 100644 --- a/packages/apple-llm/README.md +++ b/packages/apple-llm/README.md @@ -3,7 +3,10 @@ A Vercel AI SDK provider for Apple Foundation Models, enabling access to Apple Intelligence in React Native applications. **Requirements:** -- iOS 26+ + +- iOS 26+ for text generation +- iOS 26.4+ for token counting and Image Playground generation +- iOS 27+ for image prompts, Private Cloud Compute, and Vision built-in tools - Apple Intelligence enabled device - Vercel AI SDK v5 - React Native New Architecture @@ -14,16 +17,21 @@ import { generateText } from 'ai' const answer = await generateText({ model: apple(), - prompt: 'What is the meaning of life?' + prompt: 'What is the meaning of life?', }) ``` ## Features - ✅ Text generation with Apple Foundation Models +- ✅ Image prompts on iOS 27+ +- ✅ Runtime model info and context-size metadata +- ✅ Private Cloud Compute language model selection on iOS 27+ - ✅ Structured outputs - ✅ Tool calling +- ✅ Vision OCR and barcode built-in tools on iOS 27+ - ✅ Streaming +- ✅ Image Playground generation through the AI SDK image API ## Documentation diff --git a/packages/apple-llm/ios/AppleLLM.mm b/packages/apple-llm/ios/AppleLLM.mm index 1b5d70d9..f6c6a936 100644 --- a/packages/apple-llm/ios/AppleLLM.mm +++ b/packages/apple-llm/ios/AppleLLM.mm @@ -115,7 +115,8 @@ - (void)generateText:(nonnull NSArray *)messages @"topP": options.topP().has_value() ? @(options.topP().value()) : [NSNull null], @"topK": options.topK().has_value() ? @(options.topK().value()) : [NSNull null], @"schema": options.schema() ?: [NSNull null], - @"tools": options.tools() ?: [NSNull null] + @"tools": options.tools() ?: [NSNull null], + @"providerOptions": options.providerOptions() ?: [NSNull null] }; auto callToolBlock = ^(NSString *toolId, NSString *arguments, void (^completion)(id, NSError *)) { @@ -131,6 +132,19 @@ - (void)countTokens:(nonnull NSString *)text [_llm countTokens:text resolve:resolve reject:reject]; } +- (void)getModelInfo:(NSString *)locale + model:(NSString *)model + resolve:(nonnull RCTPromiseResolveBlock)resolve + reject:(nonnull RCTPromiseRejectBlock)reject { + [_llm getModelInfo:locale model:model resolve:resolve reject:reject]; +} + +- (void)generateImages:(nonnull NSDictionary *)options + resolve:(nonnull RCTPromiseResolveBlock)resolve + reject:(nonnull RCTPromiseRejectBlock)reject { + [_llm generateImages:options resolve:resolve reject:reject]; +} + - (void)cancelStream:(nonnull NSString *)streamId { [_llm cancelStream:streamId]; } @@ -144,6 +158,7 @@ - (void)generateStream:(nonnull NSString *)streamId messages:(nonnull NSArray *) @"topK": options.topK().has_value() ? @(options.topK().value()) : [NSNull null], @"schema": options.schema() ?: [NSNull null], @"tools": options.tools() ?: [NSNull null], + @"providerOptions": options.providerOptions() ?: [NSNull null], }; auto callToolBlock = ^(NSString *toolId, NSString *arguments, void (^completion)(id, NSError *)) { diff --git a/packages/apple-llm/ios/AppleLLMError.swift b/packages/apple-llm/ios/AppleLLMError.swift index 8b9a7a36..630965e4 100644 --- a/packages/apple-llm/ios/AppleLLMError.swift +++ b/packages/apple-llm/ios/AppleLLMError.swift @@ -29,8 +29,8 @@ enum AppleLLMError: Error, LocalizedError { return "Generation error: \(message)" case .streamNotFound(let id): return "Stream with ID \(id) not found" - case .invalidMessage(let role): - return "Invalid message role '\(role)'. Supported roles are: system, user, assistant" + case .invalidMessage(let message): + return "Invalid message: \(message)" case .conflictingSamplingMethods: return "Cannot specify both topP and topK parameters simultaneously. Please use only one sampling method." case .invalidSchema(let message): diff --git a/packages/apple-llm/ios/AppleLLMImpl.swift b/packages/apple-llm/ios/AppleLLMImpl.swift index b1f0aed7..4abc7268 100644 --- a/packages/apple-llm/ios/AppleLLMImpl.swift +++ b/packages/apple-llm/ios/AppleLLMImpl.swift @@ -7,11 +7,18 @@ // import Foundation +import ImageIO import React #if canImport(FoundationModels) import FoundationModels #endif +#if canImport(ImagePlayground) +import ImagePlayground +#endif +#if compiler(>=6.3) && canImport(Vision) +import Vision +#endif public typealias ToolInvoker = @Sendable (String, String, @escaping (Any?, Error?) -> Void) -> Void @@ -40,6 +47,7 @@ public class AppleLLMImpl: NSObject { reject: @escaping (String, String, Error?) -> Void ) { #if canImport(FoundationModels) +#if compiler(>=6.3) if #available(iOS 26.4, *) { guard SystemLanguageModel.default.availability == .available else { reject( @@ -64,6 +72,102 @@ public class AppleLLMImpl: NSObject { #else let error = AppleLLMError.unsupportedOS reject("AppleLLM", error.localizedDescription, error) +#endif +#else + let error = AppleLLMError.unsupportedOS + reject("AppleLLM", error.localizedDescription, error) +#endif + } + + @objc + public func getModelInfo( + _ localeIdentifier: String?, + model requestedModel: String?, + resolve: @escaping (Any?) -> Void, + reject: @escaping (String, String, Error?) -> Void + ) { +#if canImport(FoundationModels) + if #available(iOS 26, *) { + let modelName = requestedModel ?? "system" + let locale = localeIdentifier.map(Locale.init(identifier:)) ?? Locale.current + + switch modelName { + case "system": + resolve(modelInfo(for: SystemLanguageModel.default, locale: locale, modelName: modelName)) + case "private-cloud-compute": +#if compiler(>=6.3) + if #available(iOS 27, *) { + let model = PrivateCloudComputeLanguageModel() + resolve(modelInfo(for: model, locale: locale, modelName: modelName)) + } else { + rejectWithAppleError(.unsupportedOS, reject: reject) + } +#else + rejectWithAppleError(.unsupportedOS, reject: reject) +#endif + default: + rejectWithAppleError(.invalidMessage("Unsupported model '\(modelName)'"), reject: reject) + } + } else { + rejectWithAppleError(.unsupportedOS, reject: reject) + } +#else + rejectWithAppleError(.unsupportedOS, reject: reject) +#endif + } + + @objc + public func generateImages( + _ options: [String: Any], + resolve: @escaping (Any?) -> Void, + reject: @escaping (String, String, Error?) -> Void + ) { +#if canImport(ImagePlayground) + if #available(iOS 26.4, *) { + Task { + do { + let creator = try await ImageCreator() + let concepts = try Self.createImagePlaygroundConcepts(from: options) + let style = Self.createImagePlaygroundStyle(from: options["style"] as? String) + let limit = max(1, min(options["n"] as? Int ?? 1, 4)) + var images: [String] = [] + +#if compiler(>=6.3) + let imageOptions = Self.createImagePlaygroundOptions(from: options) + for try await createdImage in creator.images( + for: concepts, + style: style, + options: imageOptions, + limit: limit + ) { + let data = try Self.pngData(from: createdImage.cgImage) + images.append(data.base64EncodedString()) + + if images.count >= limit { + break + } + } +#else + for try await createdImage in creator.images(for: concepts, style: style, limit: limit) { + let data = try Self.pngData(from: createdImage.cgImage) + images.append(data.base64EncodedString()) + + if images.count >= limit { + break + } + } +#endif + + resolve(images) + } catch { + reject("AppleLLM", error.localizedDescription, error) + } + } + } else { + rejectWithAppleError(.unsupportedOS, reject: reject) + } +#else + rejectWithAppleError(.unsupportedOS, reject: reject) #endif } @@ -84,30 +188,63 @@ public class AppleLLMImpl: NSObject { Task { do { - let tools = try self.createTools(from: options, toolInvoker: toolInvoker) - let (transcript, userPrompt) = try self.createTranscriptAndPrompt(from: messages, tools: tools) - - let session = LanguageModelSession.init( - model: SystemLanguageModel.default, - tools: tools, - transcript: transcript + let providerOptions = self.createProviderOptions(from: options) + let tools = try self.createTools(from: options, providerOptions: providerOptions, toolInvoker: toolInvoker) + let (transcript, userPrompt, promptAttachments) = try self.createTranscriptAndPrompt( + from: messages, + tools: tools ) - + + let session = try self.createSession(providerOptions: providerOptions, tools: tools, transcript: transcript) + let generationOptions = try self.createGenerationOptions(from: options) let generationSchema = try self.createGenerationSchema(from: options) do { - if let generationSchema { - let response = try await session.respond( - to: userPrompt, - schema: generationSchema, - includeSchemaInPrompt: true, - options: generationOptions - ) - resolve(response.toModelMessages()) + if promptAttachments.isEmpty { + if let generationSchema { + let response = try await session.respond( + to: userPrompt, + schema: generationSchema, + includeSchemaInPrompt: true, + options: generationOptions + ) + resolve(response.toModelMessages()) + } else { + let response = try await session.respond(to: userPrompt, options: generationOptions) + resolve(response.toModelMessages()) + } } else { - let response = try await session.respond(to: userPrompt, options: generationOptions) - resolve(response.toModelMessages()) +#if compiler(>=6.3) + if #available(iOS 27, *) { + let attachments = try self.createImageAttachments(from: promptAttachments) + if let generationSchema { + let response = try await session.respond( + schema: generationSchema, + includeSchemaInPrompt: true, + options: generationOptions + ) { + userPrompt + for attachment in attachments { + attachment + } + } + resolve(response.toModelMessages()) + } else { + let response = try await session.respond(options: generationOptions) { + userPrompt + for attachment in attachments { + attachment + } + } + resolve(response.toModelMessages()) + } + } else { + throw AppleLLMError.unsupportedOS + } +#else + throw AppleLLMError.unsupportedOS +#endif } } catch { if let appleError = self.mapToAppleLLMError(error, includeGenerationFallback: true) { @@ -151,34 +288,71 @@ public class AppleLLMImpl: NSObject { let task = Task { do { - let tools = try self.createTools(from: options, toolInvoker: toolInvoker) - let (transcript, userPrompt) = try self.createTranscriptAndPrompt(from: messages, tools: tools) - - let session = LanguageModelSession.init( - model: SystemLanguageModel.default, - tools: tools, - transcript: transcript + let providerOptions = self.createProviderOptions(from: options) + let tools = try self.createTools(from: options, providerOptions: providerOptions, toolInvoker: toolInvoker) + let (transcript, userPrompt, promptAttachments) = try self.createTranscriptAndPrompt( + from: messages, + tools: tools ) - + + let session = try self.createSession(providerOptions: providerOptions, tools: tools, transcript: transcript) + let generationOptions = try self.createGenerationOptions(from: options) let generationSchema = try self.createGenerationSchema(from: options) do { - if let generationSchema { - let responseStream = session.streamResponse( - to: userPrompt, - schema: generationSchema, - includeSchemaInPrompt: true, - options: generationOptions - ) - for try await chunk in responseStream { - onUpdate(streamId, String(describing: chunk.content)) + if promptAttachments.isEmpty { + if let generationSchema { + let responseStream = session.streamResponse( + to: userPrompt, + schema: generationSchema, + includeSchemaInPrompt: true, + options: generationOptions + ) + for try await chunk in responseStream { + onUpdate(streamId, String(describing: chunk.content)) + } + } else { + let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) + for try await chunk in responseStream { + onUpdate(streamId, chunk.content) + } } } else { - let responseStream = session.streamResponse(to: userPrompt, options: generationOptions) - for try await chunk in responseStream { - onUpdate(streamId, chunk.content) +#if compiler(>=6.3) + if #available(iOS 27, *) { + let attachments = try self.createImageAttachments(from: promptAttachments) + if let generationSchema { + let responseStream = session.streamResponse( + schema: generationSchema, + includeSchemaInPrompt: true, + options: generationOptions + ) { + userPrompt + for attachment in attachments { + attachment + } + } + for try await chunk in responseStream { + onUpdate(streamId, String(describing: chunk.content)) + } + } else { + let responseStream = session.streamResponse(options: generationOptions) { + userPrompt + for attachment in attachments { + attachment + } + } + for try await chunk in responseStream { + onUpdate(streamId, chunk.content) + } + } + } else { + throw AppleLLMError.unsupportedOS } +#else + throw AppleLLMError.unsupportedOS +#endif } if !Task.isCancelled { @@ -253,6 +427,105 @@ public class AppleLLMImpl: NSObject { } #if canImport(FoundationModels) + @available(iOS 26, *) + private func modelInfo(for model: SystemLanguageModel, locale: Locale, modelName: String) -> [String: Any] { + var info: [String: Any] = [ + "model": modelName, + "isAvailable": model.availability == .available, + "availability": availabilityString(model.availability), + "supportsLocale": model.supportsLocale(locale), + "supportedLanguages": model.supportedLanguages.map { String(describing: $0) }.sorted(), + "supportsTokenCounting": false, + "supportsImagePrompts": false, + "supportsPrivateCloudCompute": false, + "supportsDynamicProfiles": false, + "supportsVisionTools": false, + ] + +#if compiler(>=6.3) + if #available(iOS 26.4, *) { + info["contextSize"] = model.contextSize + info["supportsTokenCounting"] = true + } + + if #available(iOS 27, *) { + info["supportsImagePrompts"] = true + info["supportsDynamicProfiles"] = true + info["supportsVisionTools"] = true + } +#endif + + return info + } + +#if compiler(>=6.3) + @available(iOS 27, *) + private func modelInfo( + for model: PrivateCloudComputeLanguageModel, + locale: Locale, + modelName: String + ) -> [String: Any] { + return [ + "model": modelName, + "isAvailable": model.availability == .available, + "availability": availabilityString(model.availability), + "contextSize": model.contextSize, + "quotaUsage": String(describing: model.quotaUsage), + "supportsLocale": model.supportsLocale(locale), + "supportedLanguages": model.supportedLanguages.map { String(describing: $0) }.sorted(), + "supportsTokenCounting": false, + "supportsImagePrompts": true, + "supportsPrivateCloudCompute": true, + "supportsDynamicProfiles": true, + "supportsVisionTools": true, + ] + } +#endif + + private func availabilityString(_ availability: Any) -> String { + return String(describing: availability) + } + + @available(iOS 26, *) + private func createProviderOptions(from options: [String: Any]) -> [String: Any] { + guard let providerOptions = options["providerOptions"] as? [String: Any] else { + return [:] + } + + return providerOptions + } + + @available(iOS 26, *) + private func createSession( + providerOptions: [String: Any], + tools: [any Tool], + transcript: Transcript + ) throws -> LanguageModelSession { + let modelName = providerOptions["model"] as? String ?? "system" + + switch modelName { + case "system": + return LanguageModelSession( + model: SystemLanguageModel.default, + tools: tools, + transcript: transcript + ) + case "private-cloud-compute": +#if compiler(>=6.3) + if #available(iOS 27, *) { + return LanguageModelSession( + model: PrivateCloudComputeLanguageModel(), + tools: tools, + transcript: transcript + ) + } +#endif + throw AppleLLMError.unsupportedOS + default: + throw AppleLLMError.invalidMessage("Unsupported model '\(modelName)'") + } + } + @available(iOS 26, *) private func mapKnownAppleLLMError(_ error: Error) -> AppleLLMError? { if let appleError = error as? AppleLLMError { @@ -304,9 +577,13 @@ public class AppleLLMImpl: NSObject { } @available(iOS 26, *) - private func createTools(from options: [String: Any], toolInvoker: @escaping ToolInvoker) throws -> [any Tool] { + private func createTools( + from options: [String: Any], + providerOptions: [String: Any], + toolInvoker: @escaping ToolInvoker + ) throws -> [any Tool] { guard let toolsDict = options["tools"] as? [[String: Any]] else { - return [] + return try createBuiltInTools(from: providerOptions) } var tools: [any Tool] = [] @@ -328,15 +605,41 @@ public class AppleLLMImpl: NSObject { ) tools.append(tool) } + + tools.append(contentsOf: try createBuiltInTools(from: providerOptions)) return tools } + + @available(iOS 26, *) + private func createBuiltInTools(from providerOptions: [String: Any]) throws -> [any Tool] { + guard let toolNames = providerOptions["builtInTools"] as? [String], !toolNames.isEmpty else { + return [] + } + +#if compiler(>=6.3) && canImport(Vision) + if #available(iOS 27, *) { + return try toolNames.map { toolName in + switch toolName { + case "ocr": + return OCRTool() + case "barcode": + return BarcodeReaderTool() + default: + throw AppleLLMError.invalidMessage("Unsupported built-in tool '\(toolName)'") + } + } + } +#endif + + throw AppleLLMError.unsupportedOS + } - // TODO: - // • Investigate assetIDs parameter usage in Transcript.Response - // • Implement tool calling support @available(iOS 26, *) - private func createTranscriptAndPrompt(from messages: [[String: Any]], tools: [any Tool]) throws -> (Transcript, String) { + private func createTranscriptAndPrompt( + from messages: [[String: Any]], + tools: [any Tool] + ) throws -> (Transcript, String, [[String: Any]]) { guard !messages.isEmpty else { throw AppleLLMError.invalidMessage("Messages array cannot be empty") } @@ -347,6 +650,8 @@ public class AppleLLMImpl: NSObject { lastRole == "user" else { throw AppleLLMError.invalidMessage("Last message must be from user role") } + + let promptAttachments = lastMessage["attachments"] as? [[String: Any]] ?? [] var entries: [Transcript.Entry] = [] @@ -357,6 +662,10 @@ public class AppleLLMImpl: NSObject { let content = message["content"] as? String else { throw AppleLLMError.invalidMessage("Message must have role and content") } + + if let attachments = message["attachments"] as? [[String: Any]], !attachments.isEmpty { + throw AppleLLMError.invalidMessage("Image attachments are only supported on the final user prompt") + } let segment = Transcript.Segment.text( .init(content: content) @@ -376,11 +685,194 @@ public class AppleLLMImpl: NSObject { let response = Transcript.Response(assetIDs: [], segments: [segment]) entries.append(.response(response)) default: - throw AppleLLMError.invalidMessage(role) + throw AppleLLMError.invalidMessage("Unsupported role '\(role)'. Supported roles are: system, user, assistant") } } - return (Transcript(entries: entries), userPrompt) + return (Transcript(entries: entries), userPrompt, promptAttachments) + } + +#if compiler(>=6.3) + @available(iOS 27, *) + private func createImageAttachments( + from attachments: [[String: Any]] + ) throws -> [Attachment] { + try attachments.map { attachment in + guard let type = attachment["type"] as? String, type == "image" else { + throw AppleLLMError.invalidMessage("Unsupported attachment type") + } + + let imageURL = try Self.createImageURL(from: attachment) + var imageAttachment = Attachment(imageURL: imageURL) + + if let label = attachment["label"] as? String, !label.isEmpty { + imageAttachment = imageAttachment.label(label) + } + + return imageAttachment + } + } +#endif + + private static func createImageURL(from attachment: [String: Any]) throws -> URL { + if let urlString = attachment["url"] as? String, !urlString.isEmpty { + let url = urlString.hasPrefix("/") + ? URL(fileURLWithPath: urlString) + : URL(string: urlString) + + guard let url, url.isFileURL else { + throw AppleLLMError.invalidMessage("Image attachment URLs must be local file URLs") + } + + return url + } + + guard let dataValue = attachment["data"] as? String, !dataValue.isEmpty else { + throw AppleLLMError.invalidMessage("Image attachment must include url or data") + } + + let mediaType = attachment["mediaType"] as? String + let base64Payload = dataValue.contains(",") + ? String(dataValue.split(separator: ",", maxSplits: 1, omittingEmptySubsequences: false).last ?? "") + : dataValue + + guard let imageData = Data(base64Encoded: base64Payload) else { + throw AppleLLMError.invalidMessage("Image attachment data must be base64 encoded") + } + + let fileExtension = Self.fileExtension(forImageMediaType: mediaType) + let fileURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension(fileExtension) + + try imageData.write(to: fileURL, options: .atomic) + return fileURL + } + +#if canImport(ImagePlayground) + @available(iOS 26.4, *) + private static func createImagePlaygroundConcepts(from options: [String: Any]) throws -> [ImagePlaygroundConcept] { + var concepts: [ImagePlaygroundConcept] = [] + + if let prompt = options["prompt"] as? String, !prompt.isEmpty { + concepts.append(.text(prompt)) + } + + if let files = options["files"] as? [[String: Any]], let firstFile = files.first { + let attachment = try imageAttachmentFromImageModelFile(firstFile) + let url = try createImageURL(from: attachment) + + guard let imageConcept = ImagePlaygroundConcept.image(url) else { + throw AppleLLMError.invalidMessage("Image Playground file must resolve to a local image") + } + + concepts.append(imageConcept) + } + + guard !concepts.isEmpty else { + throw AppleLLMError.invalidMessage("Image Playground generation requires a prompt or image file") + } + + return concepts + } + + private static func imageAttachmentFromImageModelFile(_ file: [String: Any]) throws -> [String: Any] { + var attachment: [String: Any] = [ + "type": "image", + "mediaType": file["mediaType"] as? String ?? "image/png", + ] + + if let data = file["data"] as? String { + if data.hasPrefix("file://") || data.hasPrefix("/") { + attachment["url"] = data + } else { + attachment["data"] = data + } + } else { + throw AppleLLMError.invalidMessage("Image Playground file data must be a string") + } + + return attachment + } + + @available(iOS 26.4, *) + private static func createImagePlaygroundStyle(from style: String?) -> ImagePlaygroundStyle { + switch style { + case "animation": + return .animation + case "illustration": + return .illustration + case "sketch": + return .sketch +#if compiler(>=6.3) + case "any": + if #available(iOS 27, *) { + return .any + } + return .illustration + case "emoji": + if #available(iOS 27, *) { + return .emoji + } + return .illustration + case "externalProvider": + if #available(iOS 27, *) { + return .externalProvider + } + return .illustration +#endif + default: + return .illustration + } + } + +#if compiler(>=6.3) + @available(iOS 26.4, *) + private static func createImagePlaygroundOptions(from options: [String: Any]) -> ImagePlaygroundOptions { + var imageOptions = ImagePlaygroundOptions() + + switch options["personalization"] as? String { + case "disabled": + imageOptions.personalization = .disabled + case "enabled": + imageOptions.personalization = .enabled + default: + imageOptions.personalization = .automatic + } + + return imageOptions + } +#endif + + private static func pngData(from image: CGImage) throws -> Data { + let data = NSMutableData() + guard let destination = CGImageDestinationCreateWithData(data, "public.png" as CFString, 1, nil) else { + throw AppleLLMError.generationError("Failed to create PNG image destination") + } + + CGImageDestinationAddImage(destination, image, nil) + + guard CGImageDestinationFinalize(destination) else { + throw AppleLLMError.generationError("Failed to encode generated image") + } + + return data as Data + } +#endif + + private static func fileExtension(forImageMediaType mediaType: String?) -> String { + switch mediaType?.lowercased() { + case "image/jpeg", "image/jpg": + return "jpg" + case "image/heic": + return "heic" + case "image/webp": + return "webp" + case "image/gif": + return "gif" + default: + return "png" + } } @available(iOS 26, *) diff --git a/packages/apple-llm/src/AppleFoundationModels.ts b/packages/apple-llm/src/AppleFoundationModels.ts index 2cae01dc..54bab481 100644 --- a/packages/apple-llm/src/AppleFoundationModels.ts +++ b/packages/apple-llm/src/AppleFoundationModels.ts @@ -9,10 +9,23 @@ const tokenCountingUnavailableMessage = const nativeAppleLLM = NativeAppleLLM as Spec & { countTokens?: (text: string) => Promise + getModelInfo?: (locale?: string, model?: string) => Promise + generateImages?: (options: object) => Promise } const AppleFoundationModels: Spec = { isAvailable: () => NativeAppleLLM.isAvailable(), + getModelInfo: (locale?: string, model?: string) => { + if (typeof nativeAppleLLM.getModelInfo !== 'function') { + return Promise.reject( + new Error( + 'Apple Foundation Models model info is unavailable. Rebuild the native module with getModelInfo support.' + ) + ) + } + + return nativeAppleLLM.getModelInfo(locale, model) + }, countTokens: (text) => { if (typeof nativeAppleLLM.countTokens !== 'function') { return Promise.reject(new Error(tokenCountingUnavailableMessage)) @@ -20,6 +33,17 @@ const AppleFoundationModels: Spec = { return nativeAppleLLM.countTokens(text) }, + generateImages: (options) => { + if (typeof nativeAppleLLM.generateImages !== 'function') { + return Promise.reject( + new Error( + 'Apple Image Playground generation is unavailable. Rebuild the native module with generateImages support.' + ) + ) + } + + return nativeAppleLLM.generateImages(options) + }, generateText: (messages: AppleMessage[], options: AppleGenerationOptions) => NativeAppleLLM.generateText(messages, options), generateStream: ( diff --git a/packages/apple-llm/src/NativeAppleLLM.ts b/packages/apple-llm/src/NativeAppleLLM.ts index a5fe6e29..150f4fc7 100644 --- a/packages/apple-llm/src/NativeAppleLLM.ts +++ b/packages/apple-llm/src/NativeAppleLLM.ts @@ -8,6 +8,7 @@ import type { export interface AppleMessage { role: 'assistant' | 'system' | 'tool' | 'user' content: string + attachments?: UnsafeObject[] } export interface AppleGenerationOptions { @@ -17,6 +18,7 @@ export interface AppleGenerationOptions { topK?: number schema?: UnsafeObject tools?: UnsafeObject + providerOptions?: UnsafeObject } export type StreamUpdateEvent = { @@ -36,7 +38,9 @@ export type StreamErrorEvent = { export interface Spec extends TurboModule { isAvailable(): boolean + getModelInfo(locale?: string, model?: string): Promise countTokens(text: string): Promise + generateImages(options: UnsafeObject): Promise generateText( messages: AppleMessage[], options: AppleGenerationOptions diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index d47e6539..488cdfdd 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -2,8 +2,11 @@ import type { EmbeddingModelV3, EmbeddingModelV3CallOptions, EmbeddingModelV3Result, + ImageModelV3, + ImageModelV3CallOptions, LanguageModelV3, LanguageModelV3CallOptions, + LanguageModelV3Message, LanguageModelV3FunctionTool, LanguageModelV3Prompt, LanguageModelV3ProviderTool, @@ -30,25 +33,299 @@ import NativeAppleUtils from './NativeAppleUtils' type Tool = LanguageModelV3FunctionTool | LanguageModelV3ProviderTool type ToolDefinitionSet = Record +export type AppleLanguageModelId = 'system' | 'private-cloud-compute' +export type AppleBuiltInTool = 'ocr' | 'barcode' +export type AppleImageStyle = + | 'animation' + | 'any' + | 'emoji' + | 'externalProvider' + | 'illustration' + | 'sketch' +export type AppleImagePersonalization = 'automatic' | 'disabled' | 'enabled' + +export interface AppleProviderOptions { + model?: AppleLanguageModelId + builtInTools?: AppleBuiltInTool[] + context?: AppleContextOptions + style?: AppleImageStyle + personalization?: AppleImagePersonalization +} + +export interface AppleModelInfo { + model: AppleLanguageModelId + isAvailable: boolean + availability: string + supportsLocale: boolean + supportedLanguages: string[] + supportsTokenCounting: boolean + supportsImagePrompts: boolean + supportsPrivateCloudCompute: boolean + supportsDynamicProfiles: boolean + supportsVisionTools: boolean + contextSize?: number + quotaUsage?: string +} + +export interface AppleLanguageModelOptions { + model?: AppleLanguageModelId + availableTools?: ToolDefinitionSet + context?: AppleContextOptions +} + +export interface AppleContextOptions { + /** + * Keep the first system message and only the last N non-system messages. + */ + rollingWindowMessages?: number + /** + * Remove completed tool-call/tool-result entries from older context. + */ + dropCompletedToolCalls?: boolean +} + +type AppleMessageAttachment = { + type: 'image' + mediaType: string + data?: string + url?: string +} + +type AppleImageModelFile = { + mediaType: string + data: string +} + +function uint8ArrayToBase64(bytes: Uint8Array): string { + let binary = '' + const chunkSize = 0x8000 + for (let i = 0; i < bytes.length; i += chunkSize) { + const chunk = bytes.subarray(i, i + chunkSize) + binary += String.fromCharCode(...chunk) + } + return btoa(binary) +} + +function prepareImageAttachment( + part: Extract< + Extract< + LanguageModelV3Prompt[number], + { content: unknown[] } + >['content'][number], + { type: 'file' } + > +): AppleMessageAttachment { + const mediaType = part.mediaType.toLowerCase() + if (!mediaType.startsWith('image/')) { + throw new Error( + `Unsupported Apple Foundation Models file type: ${part.mediaType}` + ) + } + + const data = part.data as unknown + + if (data instanceof Uint8Array) { + return { + type: 'image', + mediaType, + data: uint8ArrayToBase64(data), + } + } + + const value = + data instanceof URL + ? data.toString() + : typeof data === 'string' + ? data + : null + + if (!value) { + throw new Error('Unsupported Apple Foundation Models image data') + } + + if (value.startsWith('data:')) { + return { + type: 'image', + mediaType, + data: value, + } + } + + return { + type: 'image', + mediaType, + url: value, + } +} + +function prepareImageModelFiles( + files: ImageModelV3CallOptions['files'] = [] +): AppleImageModelFile[] { + return files.map((file) => { + if (file.type === 'url') { + return { + mediaType: 'image/png', + data: file.url, + } + } + + const mediaType = file.mediaType.toLowerCase() + if (!mediaType.startsWith('image/')) { + throw new Error( + `Unsupported Apple Image Playground file type: ${file.mediaType}` + ) + } + + const data = file.data as unknown + const normalizedData = + data instanceof Uint8Array + ? uint8ArrayToBase64(data) + : data instanceof URL + ? data.toString() + : typeof data === 'string' + ? data + : null + + if (!normalizedData) { + throw new Error('Unsupported Apple Image Playground image data') + } + + return { + mediaType, + data: normalizedData, + } + }) +} + +function getAppleProviderOptions( + providerOptions: LanguageModelV3CallOptions['providerOptions'] | undefined +): AppleProviderOptions { + return (providerOptions?.apple ?? {}) as AppleProviderOptions +} + +function mergeAppleProviderOptions( + defaults: AppleLanguageModelOptions, + callOptions: AppleProviderOptions +): AppleProviderOptions { + return { + model: callOptions.model ?? defaults.model, + builtInTools: callOptions.builtInTools, + context: { + ...defaults.context, + ...callOptions.context, + }, + } +} + +export function trimAppleMessagesForContext( + messages: LanguageModelV3Prompt, + options: AppleContextOptions = {} +): LanguageModelV3Prompt { + let nextMessages = messages + + if (options.dropCompletedToolCalls) { + nextMessages = dropCompletedToolCallMessages(nextMessages) + } + + if (options.rollingWindowMessages && options.rollingWindowMessages > 0) { + const systemMessages = nextMessages.filter( + (message) => message.role === 'system' + ) + const conversationMessages = nextMessages.filter( + (message) => message.role !== 'system' + ) + nextMessages = [ + ...systemMessages, + ...conversationMessages.slice(-options.rollingWindowMessages), + ] + } + + return nextMessages +} + +function dropCompletedToolCallMessages( + messages: LanguageModelV3Prompt +): LanguageModelV3Prompt { + const lastToolRelatedIndex = findLastIndex(messages, isToolRelatedMessage) + + if (lastToolRelatedIndex === -1) { + return messages + } + + let latestToolExchangeStartIndex = lastToolRelatedIndex + while ( + latestToolExchangeStartIndex > 0 && + isToolRelatedMessage(messages[latestToolExchangeStartIndex - 1]) + ) { + latestToolExchangeStartIndex -= 1 + } + + return messages.filter((message, index) => { + if (index >= latestToolExchangeStartIndex) { + return true + } + + return !isToolRelatedMessage(message) + }) +} + +function findLastIndex( + items: T[], + predicate: (item: T, index: number) => boolean +) { + for (let index = items.length - 1; index >= 0; index -= 1) { + if (predicate(items[index], index)) { + return index + } + } + + return -1 +} + +function isToolRelatedMessage(message: LanguageModelV3Message) { + if (message.role === 'tool') { + return true + } + + if (message.role !== 'assistant') { + return false + } + + return message.content.some( + (part) => part.type === 'tool-call' || part.type === 'tool-result' + ) +} export function createAppleProvider({ availableTools, -}: { - availableTools?: ToolDefinitionSet -} = {}) { - const createLanguageModel = () => { - return new AppleLLMChatLanguageModel(availableTools) + model, + context, +}: AppleLanguageModelOptions = {}) { + const createLanguageModel = (options: AppleLanguageModelOptions = {}) => { + return new AppleLLMChatLanguageModel({ + availableTools: options.availableTools ?? availableTools, + model: options.model ?? model, + context: { + ...context, + ...options.context, + }, + }) } - const provider = function () { - return createLanguageModel() + const provider = function (options: AppleLanguageModelOptions = {}) { + return createLanguageModel(options) } provider.isAvailable = () => NativeAppleLLM.isAvailable() + provider.getModelInfo = (options: { locale?: string; model?: string } = {}) => + NativeAppleLLM.getModelInfo( + options.locale, + options.model + ) as Promise provider.languageModel = createLanguageModel provider.textEmbeddingModel = (options: AppleEmbeddingOptions = {}) => { return new AppleTextEmbeddingModel(options) } - provider.imageModel = () => { - throw new Error('Image generation models are not supported by Apple LLM') + provider.imageModel = (options: AppleImageModelOptions = {}) => { + return new AppleImageModel(options) } provider.transcriptionModel = (options: AppleTranscriptionOptions = {}) => { return new AppleTranscriptionModel(options) @@ -61,6 +338,47 @@ export function createAppleProvider({ export const apple = createAppleProvider() +export interface AppleImageModelOptions { + style?: AppleImageStyle + personalization?: AppleImagePersonalization +} + +class AppleImageModel implements ImageModelV3 { + readonly specificationVersion = 'v3' + readonly provider = 'apple' + readonly modelId = 'ImagePlayground' + readonly maxImagesPerCall = 4 + + private options: AppleImageModelOptions + + constructor(options: AppleImageModelOptions = {}) { + this.options = options + } + + async doGenerate(options: ImageModelV3CallOptions) { + const appleOptions = (options.providerOptions?.apple ?? + {}) as AppleImageModelOptions + const images = await NativeAppleLLM.generateImages({ + prompt: options.prompt, + n: options.n, + files: prepareImageModelFiles(options.files), + style: appleOptions.style ?? this.options.style, + personalization: + appleOptions.personalization ?? this.options.personalization, + }) + + return { + images, + warnings: [], + response: { + timestamp: new Date(), + modelId: this.modelId, + headers: undefined, + }, + } + } +} + export interface AppleTranscriptionOptions { language?: string } @@ -234,31 +552,48 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { readonly supportedUrls = {} readonly provider = 'apple' - readonly modelId = 'system-default' + readonly modelId: string private tools: ToolDefinitionSet = {} + private options: AppleLanguageModelOptions - constructor(availableTools: ToolDefinitionSet = {}) { - this.updateTools(availableTools) + constructor(options: AppleLanguageModelOptions = {}) { + this.options = options + this.modelId = options.model ?? 'system' + this.updateTools(options.availableTools ?? {}) } async prepare(): Promise {} private prepareMessages(messages: LanguageModelV3Prompt): AppleMessage[] { return messages.map((message): AppleMessage => { - const content = Array.isArray(message.content) - ? message.content.reduce((acc, part) => { - if (part.type === 'text') { - return acc + part.text - } - console.warn('Unsupported message content type:', part) + if (Array.isArray(message.content)) { + const attachments: AppleMessageAttachment[] = [] + const content = message.content.reduce((acc, part) => { + if (part.type === 'text') { + return acc + part.text + } + + if (part.type === 'file') { + attachments.push(prepareImageAttachment(part)) return acc - }, '') - : message.content + } + + throw new Error( + `Unsupported Apple Foundation Models message content type: ${JSON.stringify(part)}` + ) + }, '') + + return { + role: message.role, + content, + attachments, + } + } return { role: message.role, - content, + content: message.content, } }) } @@ -296,7 +631,14 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { } async doGenerate(options: LanguageModelV3CallOptions) { - const messages = this.prepareMessages(options.prompt) + const appleOptions = mergeAppleProviderOptions( + this.options, + getAppleProviderOptions(options.providerOptions) + ) + const prompt = appleOptions.context + ? trimAppleMessagesForContext(options.prompt, appleOptions.context) + : options.prompt + const messages = this.prepareMessages(prompt) const tools = this.prepareTools(options.tools) for (const tool of tools) { @@ -310,6 +652,7 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { topP: options.topP, topK: options.topK, tools, + providerOptions: appleOptions as unknown as Record, schema: options.responseFormat?.type === 'json' ? options.responseFormat.schema @@ -363,7 +706,14 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { } async doStream(options: LanguageModelV3CallOptions) { - const messages = this.prepareMessages(options.prompt) + const appleOptions = mergeAppleProviderOptions( + this.options, + getAppleProviderOptions(options.providerOptions) + ) + const prompt = appleOptions.context + ? trimAppleMessagesForContext(options.prompt, appleOptions.context) + : options.prompt + const messages = this.prepareMessages(prompt) const tools = this.prepareTools(options.tools) if (typeof ReadableStream === 'undefined') { @@ -477,6 +827,7 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { topP: options.topP, topK: options.topK, tools, + providerOptions: appleOptions as unknown as Record, schema, }) } catch (error) { diff --git a/packages/apple-llm/src/index.ts b/packages/apple-llm/src/index.ts index 0e73c5e5..b94cb78d 100644 --- a/packages/apple-llm/src/index.ts +++ b/packages/apple-llm/src/index.ts @@ -1,4 +1,19 @@ -export { apple, createAppleProvider } from './ai-sdk' +export { + apple, + createAppleProvider, + trimAppleMessagesForContext, +} from './ai-sdk' +export type { + AppleBuiltInTool, + AppleContextOptions, + AppleImagePersonalization, + AppleImageModelOptions, + AppleImageStyle, + AppleLanguageModelId, + AppleLanguageModelOptions, + AppleModelInfo, + AppleProviderOptions, +} from './ai-sdk' export { default as AppleFoundationModels } from './AppleFoundationModels' export type { AppleLLMError, AppleLLMErrorCode } from './errors' export { AppleLLMErrorCodes } from './errors' diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 07fda3c6..77f00992 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -1,345 +1,366 @@ # Generating -You can generate response using Apple Foundation Models with the Vercel AI SDK's `generateText` or `generateObject` function. +Use `@react-native-ai/apple` with the Vercel AI SDK to call Apple Foundation +Models from React Native. The provider keeps the common text, object, +streaming, tool, and image APIs aligned with AI SDK v5, and exposes a small +Apple-specific surface for runtime model information and Apple-only options. ## Requirements -- **iOS 26+** - Apple Foundation Models is available in iOS 26 or later -- **Apple Intelligence enabled device** - Device must support Apple Intelligence +- iOS 26+ for Apple Foundation Models text generation. +- iOS 26.4+ for token counting and Image Playground image generation. +- iOS 27+ for multimodal image prompts, Private Cloud Compute language models, + Dynamic Profiles, and Vision built-in tools. +- An Apple Intelligence-capable device with Apple Intelligence enabled. +- React Native New Architecture. + +Some Apple Intelligence symbols are SDK-gated as well as OS-gated. If you build +with an older Xcode SDK that does not expose a newer Apple API, the provider +keeps that feature unavailable instead of failing native compilation. ## Text Generation ```typescript -import { apple } from '@react-native-ai/apple'; -import { generateText } from 'ai'; +import { apple } from '@react-native-ai/apple' +import { generateText } from 'ai' + +const result = await generateText({ + model: apple(), + prompt: 'Explain quantum computing in simple terms.', +}) + +console.log(result.text) +``` + +Configure the model with the normal AI SDK generation options: +```typescript const result = await generateText({ model: apple(), - prompt: 'Explain quantum computing in simple terms' -}); + prompt: 'Write a concise product announcement.', + temperature: 0.6, + maxTokens: 300, + topP: 0.9, +}) ``` +Do not pass both `topP` and `topK` in the same request. Apple Foundation Models +supports one sampling strategy at a time. + ## Streaming ```typescript -import { streamText } from 'ai'; -import { apple } from '@react-native-ai/apple'; +import { apple } from '@react-native-ai/apple' +import { streamText } from 'ai' -const { textStream } = await streamText({ +const result = streamText({ model: apple(), - prompt: 'Write me a short essay on the meaning of life' -}); + prompt: 'Draft a short release note.', +}) -for await (const delta of textStream) { - console.log(delta); +for await (const delta of result.textStream) { + console.log(delta) } ``` -> [!NOTE] -> Streaming objects is currently not supported. +Streaming structured objects are not currently supported by this provider. ## Structured Output -Generate structured data that conforms to a specific schema: +Use `generateObject` when you want Apple guided generation through the AI SDK: ```typescript -import { generateObject } from 'ai'; -import { apple } from '@react-native-ai/apple'; -import { z } from 'zod'; - -const schema = z.object({ - name: z.string(), - age: z.number().int().min(0).max(150), - email: z.string().email(), - occupation: z.string() -}); +import { apple } from '@react-native-ai/apple' +import { generateObject } from 'ai' +import { z } from 'zod' const result = await generateObject({ model: apple(), - prompt: 'Generate a user profile for a software developer', - schema -}); + prompt: 'Create a compact user profile for a software developer.', + schema: z.object({ + name: z.string(), + role: z.string(), + seniority: z.enum(['junior', 'mid', 'senior']), + }), +}) -console.log(result.object); -// { name: string, age: number, email: string, occupation: string } +console.log(result.object) ``` -## Tool Calling - -Enable Apple Foundation Models to use custom tools in your React Native applications. - -### Important Apple-Specific Behavior - -Tools are executed by Apple, not the Vercel AI SDK, which means: - -- **No AI SDK callbacks**: `maxSteps`, `onStepStart`, and `onStepFinish` will not be executed -- **Pre-register all tools**: You must pass all tools to `createAppleProvider` upfront -- **Empty toolCallId**: Apple doesn't provide tool call IDs, so they will be empty strings +Supported schema shapes include objects, arrays, strings, numbers, booleans, +and enums. String formats, regular expressions, and unions are not currently +mapped to Apple Foundation Models. -### Setup +## Availability And Model Info -All tools must be registered ahead of time with Apple provider. To do so, you must create one by calling `createAppleProvider`: +Check availability before showing Apple-only UI: ```typescript -import { createAppleProvider } from '@react-native-ai/apple'; -import { generateText, tool } from 'ai'; -import { z } from 'zod'; +import { apple } from '@react-native-ai/apple' -const getWeather = tool({ - description: 'Get current weather information', - inputSchema: z.object({ - city: z.string() - }), - execute: async ({ city }) => { - return `Weather in ${city}: Sunny, 25°C`; - } -}); - -const apple = createAppleProvider({ - availableTools: { - getWeather - } -}); +if (!apple.isAvailable()) { + // Show fallback UI or use another provider. +} ``` -If you want to change the tools at runtime, you can do it as follows: +For capability-aware UI, ask native for the active model metadata: ```typescript -const apple = createAppleProvider({ - availableTools: { - getWeather - } -}); -const model = apple(); +const info = await apple.getModelInfo({ locale: 'en-US' }) -model.updateTools({ - getWeather, - getDate -}); +console.log(info.isAvailable) +console.log(info.contextSize) +console.log(info.supportsImagePrompts) +console.log(info.supportsVisionTools) ``` -### Basic Tool Usage +`getModelInfo` returns availability, locale support, supported languages, +context size when Apple exposes it, and feature flags for token counting, +image prompts, Private Cloud Compute, Dynamic Profiles, and Vision built-in +tools. + +## Private Cloud Compute -Then, generate output like with any other Vercel AI SDK provider: +The default `apple()` model uses `SystemLanguageModel.default`. On iOS 27 and +newer, choose Apple's Private Cloud Compute language model with provider +options: ```typescript const result = await generateText({ model: apple(), - prompt: 'What is the weather in Paris?', - tools: { - getWeather - } -}); + prompt: 'Analyze this longer task with more reasoning.', + providerOptions: { + apple: { + model: 'private-cloud-compute', + }, + }, +}) +``` + +You can also create a provider with a default model: + +```typescript +import { createAppleProvider } from '@react-native-ai/apple' + +const apple = createAppleProvider({ + model: 'private-cloud-compute', +}) ``` -### Inspecting Tool Calls +Call `apple.getModelInfo({ model: 'private-cloud-compute' })` before enabling +this path. It requires iOS 27 APIs and may report quota usage. + +## Image Prompts -You can inspect tool calls and their results after generation: +On iOS 27 and newer, Apple Foundation Models can analyze images alongside text +prompts. Pass images through the AI SDK message format: ```typescript +import { apple } from '@react-native-ai/apple' +import { generateText } from 'ai' + const result = await generateText({ model: apple(), - prompt: 'What is the weather in Paris?', - tools: { getWeather } -}); - -// Inspect tool calls made during generation -console.log(result.toolCalls); -// Example: [{ toolCallId: '<< redacted >>', toolName: 'getWeather', input: '{"city":"Paris"}' }] + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe this image for accessibility.' }, + { + type: 'file', + mediaType: 'image/jpeg', + data: 'file:///path/to/photo.jpg', + }, + ], + }, + ], +}) -// Inspect tool results returned -console.log(result.toolResults); -// Example: [{ toolCallId: '<< redacted >>', toolName: 'getWeather', result: 'Weather in Paris: Sunny, 25°C' }] +console.log(result.text) ``` -### Tool calling with structured output +The provider accepts local file URLs, absolute file paths, base64 image data, +and image data URLs. Remote HTTP URLs are not sent directly to Apple; download +them first and pass a local file URL or base64 payload. + +Image attachments are currently supported on the final user prompt. Passing +image parts on older OS versions throws `AppleLLMErrorCodes.UnsupportedOS`. + +## Image Generation -You can also use [`experimental_output`](https://v5.ai-sdk.dev/docs/reference/ai-sdk-core/generate-text#experimental_output) to generate structured output with `generateText`. This is useful when you want to perform tool calls at the same time. +Use the AI SDK image API with Apple's Image Playground framework: ```typescript -const response = await generateText({ - model: apple(), - system: `Help the person with getting weather information.`, - prompt: 'What is the weather in Wroclaw?', - tools: { - getWeather, - }, - experimental_output: Output.object({ - schema: z.object({ - weather: z.string(), - city: z.string(), - }), +import { apple } from '@react-native-ai/apple' +import { generateImage } from 'ai' + +const result = await generateImage({ + model: apple.imageModel({ + style: 'illustration', + personalization: 'disabled', }), + prompt: 'A friendly robot watering balcony herbs', + n: 1, }) + +const base64Png = result.images[0].base64 ``` -### Supported features +The provider uses `ImageCreator` on iOS 26.4 and newer and returns PNG images. +Supported styles are `animation`, `illustration`, and `sketch`; iOS 27 adds +`any`, `emoji`, and `externalProvider`. Apple supports at most one source image +concept for generation, so pass no more than one file. Personalization is +applied when the app is built with an SDK that exposes `ImagePlaygroundOptions`. -We aim to cover most of the OpenAI supported formats, including the following: +Per-call Apple options can override model defaults: -- **Objects**: `z.object({})` with nested properties -- **Arrays**: `z.array()` with `minItems` and `maxItems` constraints -- **Strings**: `z.string()` -- **Numbers**: `z.number()` with `minimum`, `maximum`, `exclusiveMinimum`, `exclusiveMaximum` -- **Booleans**: `z.boolean()` -- **Enums**: `z.enum([])` for string and number values +```typescript +const result = await generateImage({ + model: apple.imageModel(), + prompt: 'A hand-drawn icon for a notes app', + providerOptions: { + apple: { + style: 'sketch', + personalization: 'disabled', + }, + }, +}) +``` -The following features are currently not supported due to underlying model limitations: +## Tool Calling -- **String formats**: `email()`, `url()`, `uuid()`, `datetime()` etc. -- **Regular expressions**: Due to a -- **Unions**: `z.union()`, `z.discriminatedUnion()` +Apple executes tools inside the Foundation Models session. Register JavaScript +tools up front so native can call them by name: -## Availability Check +```typescript +import { createAppleProvider } from '@react-native-ai/apple' +import { generateText, tool } from 'ai' +import { z } from 'zod' -Always check if Apple Intelligence is available before using the provider: +const getWeather = tool({ + description: 'Get current weather information.', + inputSchema: z.object({ + city: z.string(), + }), + execute: async ({ city }) => `Weather in ${city}: sunny`, +}) -```typescript -import { apple } from '@react-native-ai/apple'; +const apple = createAppleProvider({ + availableTools: { + getWeather, + }, +}) -if (!apple.isAvailable()) { - // Handle fallback logic - return; -} +const result = await generateText({ + model: apple(), + prompt: 'What is the weather in Paris?', + tools: { + getWeather, + }, +}) ``` -## Context Window - -Apple Foundation Models have a fixed context window of 4096 tokens. This limit applies to the full request context, including system instructions, previous conversation messages, tool definitions, schemas, and the current user prompt. +Important Apple-specific behavior: -The `maxTokens` option only limits how many tokens the model can generate in its response. It does not increase the available context window or reserve enough room for a long prompt. +- Tool calls are provider-executed, so AI SDK step callbacks such as + `maxSteps`, `onStepStart`, and `onStepFinish` do not run for native Apple + tool execution. +- Apple does not provide stable tool call IDs; the provider returns empty IDs. +- You can update the registered tools on a model instance with + `model.updateTools(...)` before generation. -If the full context is too large, Apple may fail generation with a context-window overflow error. The provider does not automatically estimate tokens, remove messages from your prompt, or retry the request, because token estimates can vary by language and different apps need different memory strategies. Handle this at the application level by catching the error and choosing the recovery behavior that fits your product: +## Vision Built-In Tools -- Start a new conversation without the previous transcript, which is Apple's recommended baseline after this error -- Keep a sliding window of recent messages -- Summarize older messages and include the summary instead of the full transcript -- Ask the user to shorten the prompt or start a new chat +iOS 27 adds Foundation Models integration with Vision's OCR and barcode tools. +Enable them through Apple provider options: ```typescript -import { - AppleLLMErrorCodes, - type AppleLLMError, - apple, -} from '@react-native-ai/apple'; -import { generateText } from 'ai'; - -try { - const result = await generateText({ - model: apple(), - messages - }); -} catch (error) { - const appleError = error as AppleLLMError; +const result = await generateText({ + model: apple(), + messages, + providerOptions: { + apple: { + builtInTools: ['ocr', 'barcode'], + }, + }, +}) +``` - if (appleError.code === AppleLLMErrorCodes.ContextWindowExceeded) { - // Apply your app's recovery strategy here. - // For example: retry with fewer messages or start a new chat. - } +These tools require iOS 27. On older systems the provider throws +`AppleLLMErrorCodes.UnsupportedOS`. - throw error; -} -``` +## Context Window -For streaming calls, use `fullStream` when you need to inspect provider error parts: +Apple's available context depends on the model and OS version. Prefer runtime +metadata over hard-coded values: ```typescript -import { - AppleLLMErrorCodes, - type AppleLLMError, - apple, -} from '@react-native-ai/apple'; -import { streamText } from 'ai'; +const info = await apple.getModelInfo() +console.log(info.contextSize) +``` -const result = streamText({ - model: apple(), - messages -}); +The provider exposes two light context policies inspired by Apple's Dynamic +Profile history utilities: -for await (const part of result.fullStream) { - if (part.type === 'error') { - const error = part.error as AppleLLMError; +```typescript +import { createAppleProvider } from '@react-native-ai/apple' - if (error.code === AppleLLMErrorCodes.ContextWindowExceeded) { - // Apply your app's recovery strategy here. - } - } -} +const apple = createAppleProvider({ + context: { + rollingWindowMessages: 12, + dropCompletedToolCalls: true, + }, +}) ``` -If you only consume `textStream`, pass `onError` to `streamText`. The AI SDK does not emit error parts through the text-only stream, so capture the error there and handle it after the stream finishes: +You can also set them per request: ```typescript -import { - AppleLLMErrorCodes, - type AppleLLMError, - apple, -} from '@react-native-ai/apple'; -import { streamText } from 'ai'; - -let streamError: unknown; -const result = streamText({ +const result = await generateText({ model: apple(), messages, - onError: ({ error }) => { - streamError = error; + providerOptions: { + apple: { + context: { + rollingWindowMessages: 8, + dropCompletedToolCalls: true, + }, + }, }, -}); - -for await (const delta of result.textStream) { - console.log(delta); -} - -if (streamError) { - const error = streamError as AppleLLMError; - - if (error.code === AppleLLMErrorCodes.ContextWindowExceeded) { - // Apply your app's recovery strategy here. - } - - throw streamError; -} +}) ``` -## Public Error Codes +`rollingWindowMessages` keeps system messages and the last N non-system +messages. `dropCompletedToolCalls` removes older completed tool-call/tool-result +entries while keeping the newest tool-related exchange. For larger apps, +summarize older turns at the application layer and include the summary as a +system or user message. -Apple Foundation Models exposes a deliberate public error-code surface through `AppleLLMError.code`. Use `error.code` for application control flow and treat `error.message` as display/debug text rather than a compatibility contract. +Use low-level token counting as an estimate, not a fit guarantee: -Only documented `AppleLLMErrorCodes.*` values are stable: - -- `AppleLLMErrorCodes.ModelUnavailable` / `MODEL_UNAVAILABLE` -- `AppleLLMErrorCodes.UnsupportedOS` / `UNSUPPORTED_OS` -- `AppleLLMErrorCodes.GenerationError` / `GENERATION_ERROR` -- `AppleLLMErrorCodes.InvalidMessage` / `INVALID_MESSAGE` -- `AppleLLMErrorCodes.ConflictingSamplingMethods` / `CONFLICTING_SAMPLING_METHODS` -- `AppleLLMErrorCodes.InvalidSchema` / `INVALID_SCHEMA` -- `AppleLLMErrorCodes.ToolCallError` / `TOOL_CALL_ERROR` -- `AppleLLMErrorCodes.UnknownToolCallError` / `UNKNOWN_TOOL_CALL_ERROR` -- `AppleLLMErrorCodes.ContextWindowExceeded` / `CONTEXT_WINDOW_EXCEEDED` +```typescript +import { AppleFoundationModels } from '@react-native-ai/apple' -These codes mean: +const tokenCount = await AppleFoundationModels.countTokens( + 'Summarize this text in three bullet points.' +) +``` -- `MODEL_UNAVAILABLE`: Apple Intelligence is unavailable on the current device or configuration. -- `UNSUPPORTED_OS`: the current runtime does not support the required Apple Foundation Models API. -- `GENERATION_ERROR`: Foundation Models generation failed for an uncategorized provider-side reason. -- `INVALID_MESSAGE`: the prompt/message structure is invalid for this provider. -- `CONFLICTING_SAMPLING_METHODS`: both `topP` and `topK` were provided. -- `INVALID_SCHEMA`: the provided schema or tool schema cannot be used by Apple Foundation Models. -- `TOOL_CALL_ERROR`: a tool execution failed. -- `UNKNOWN_TOOL_CALL_ERROR`: tool execution finished in an unusable or unexpected state. -- `CONTEXT_WINDOW_EXCEEDED`: the request exceeded the model context window. +The full request also includes instructions, tool definitions, schema text, +attachments, and generated output budget. If Apple reports a context overflow, +trim or summarize and retry. -Errors that do not have a recognized public Apple LLM code may still be thrown, but they are plain `Error` values and are not part of the stable Apple provider API. +## Error Handling -### Handling `generateText` errors +Use public error codes for app control flow: ```typescript import { AppleLLMErrorCodes, type AppleLLMError, apple, -} from '@react-native-ai/apple'; -import { generateText } from 'ai'; +} from '@react-native-ai/apple' +import { generateText } from 'ai' try { await generateText({ @@ -347,27 +368,25 @@ try { messages, }) } catch (error) { - if ( - error instanceof Error && - 'code' in error && - error.code === AppleLLMErrorCodes.ModelUnavailable - ) { - // Show fallback UI or disable Apple-specific features. + const appleError = error as AppleLLMError + + if (appleError.code === AppleLLMErrorCodes.ContextWindowExceeded) { + // Retry with a smaller transcript or a summary. } throw error } ``` -### Handling streaming errors with `fullStream` +For streaming, inspect `fullStream` when you need provider error parts: ```typescript import { AppleLLMErrorCodes, type AppleLLMError, apple, -} from '@react-native-ai/apple'; -import { streamText } from 'ai'; +} from '@react-native-ai/apple' +import { streamText } from 'ai' const result = streamText({ model: apple(), @@ -375,107 +394,51 @@ const result = streamText({ }) for await (const part of result.fullStream) { - if ( - part.type === 'error' && - part.error instanceof Error && - 'code' in part.error && - part.error.code === AppleLLMErrorCodes.InvalidSchema - ) { - // Handle invalid schema input. - } -} -``` - -### Handling streaming errors with `textStream` and `onError` - -```typescript -import { - AppleLLMErrorCodes, - type AppleLLMError, - apple, -} from '@react-native-ai/apple'; -import { streamText } from 'ai'; - -let streamError: unknown -const result = streamText({ - model: apple(), - messages, - onError: ({ error }) => { - streamError = error - }, -}) - -for await (const delta of result.textStream) { - console.log(delta) -} + if (part.type === 'error') { + const error = part.error as AppleLLMError -if ( - streamError instanceof Error && - 'code' in streamError && - streamError.code === AppleLLMErrorCodes.ToolCallError -) { - // Handle tool failure. + if (error.code === AppleLLMErrorCodes.ModelUnavailable) { + // Show fallback UI. + } + } } ``` -## Available Options +Stable public codes are: -Configure model behavior with generation options: +- `MODEL_UNAVAILABLE` +- `UNSUPPORTED_OS` +- `GENERATION_ERROR` +- `INVALID_MESSAGE` +- `CONFLICTING_SAMPLING_METHODS` +- `INVALID_SCHEMA` +- `TOOL_CALL_ERROR` +- `UNKNOWN_TOOL_CALL_ERROR` +- `CONTEXT_WINDOW_EXCEEDED` -- `temperature` (0-1): Controls randomness. Higher values = more creative, lower = more focused -- `maxTokens`: Maximum number of tokens to generate -- `topP` (0-1): Nucleus sampling threshold -- `topK`: Top-K sampling parameter +## Direct Native API -You can pass selected options with either `generateText` or `generateObject` as follows: +Prefer the AI SDK models for generation. Use `AppleFoundationModels` only when +you need low-level native access: ```typescript -import { apple } from '@react-native-ai/apple'; -import { generateText } from 'ai'; - -const result = await generateText({ - model: apple(), - prompt: 'Write a creative story', - temperature: 0.8, - maxTokens: 500, - topP: 0.9, -}); -``` - -## Direct API Access - -For advanced use cases, you can access the native Apple Foundation Models API directly: - -### AppleFoundationModels - -```tsx -import { AppleFoundationModels } from '@react-native-ai/apple' - -// Check if Apple Intelligence is available -const isAvailable = AppleFoundationModels.isAvailable() - -// Generate text responses -const messages = [{ role: 'user', content: 'Hello' }] -const options = { temperature: 0.7, maxTokens: 100 } - -const result = await AppleFoundationModels.generateText(messages, options) -``` - -On iOS 26.4 and newer, you can also count the number of tokens in a string -before sending it to the model: - -```tsx import { AppleFoundationModels } from '@react-native-ai/apple' -const tokenCount = await AppleFoundationModels.countTokens( - 'Summarize this text in three bullet points.' +const info = await AppleFoundationModels.getModelInfo('en-US', 'system') +const tokens = await AppleFoundationModels.countTokens('A short prompt') +const result = await AppleFoundationModels.generateText( + [{ role: 'user', content: 'Hello' }], + { temperature: 0.7, maxTokens: 100 } ) ``` -Token counting is useful for estimating prompt size, but it is not a complete -guarantee that a generation request will fit in the model context window. The -full context also includes instructions, previous messages in the transcript, -tools, schemas, and generated output. +Low-level `generateImages` is available for advanced callers, but +`apple.imageModel()` is the recommended Image Playground integration because it +matches the AI SDK image API. + +## Apple Documentation -The maximum context window size for Apple's Foundation models is 4096 tokens -per session. More information can be found [here](https://developer.apple.com/documentation/technotes/tn3193-managing-the-on-device-foundation-model-s-context-window). +- [Apple Intelligence](https://developer.apple.com/apple-intelligence/) +- [Foundation Models](https://developer.apple.com/documentation/FoundationModels) +- [Image Playground](https://developer.apple.com/documentation/imageplayground) +- [Managing the on-device foundation model's context window](https://developer.apple.com/documentation/technotes/tn3193-managing-the-on-device-foundation-model-s-context-window)