From 389e7eeacc833ce8b8fd9aef146364a9592b5faa Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Thu, 21 May 2026 16:18:29 +0200 Subject: [PATCH 1/3] feat: support ocr for images using llm Signed-off-by: Lukas Schaefer --- lib/AppInfo/Application.php | 1 + lib/TaskProcessing/ImageToTextOcrProvider.php | 192 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 lib/TaskProcessing/ImageToTextOcrProvider.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 2804e5fd..636f32f4 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -148,6 +148,7 @@ public function register(IRegistrationContext $context): void { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class); } if ($isUsingOpenAI || $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled') === '1') { + $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ImageToTextOcrProvider::class); $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class); } } diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php new file mode 100644 index 00000000..44e7a23b --- /dev/null +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -0,0 +1,192 @@ +openAiAPIService->getServiceName(); + } + + public function getTaskTypeId(): string { + return ImageToTextOpticalCharacterRecognition::ID; + } + + public function getExpectedRuntime(): int { + return $this->openAiAPIService->getExpTextProcessingTime(); + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + public function getOptionalInputShape(): array { + return [ + 'max_tokens' => new ShapeDescriptor( + $this->l->t('Maximum output words'), + $this->l->t('The maximum number of words/tokens that can be generated in the output.'), + EShapeType::Number + ), + 'model' => new ShapeDescriptor( + $this->l->t('Model'), + $this->l->t('The model used to generate the output'), + EShapeType::Enum + ), + ]; + } + + public function getOptionalInputShapeEnumValues(): array { + return [ + 'model' => $this->openAiAPIService->getModelEnumValues($this->userId), + ]; + } + + public function getOptionalInputShapeDefaults(): array { + $adminModel = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + return [ + 'max_tokens' => $this->openAiSettingsService->getMaxTokens(), + 'model' => $adminModel, + ]; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return []; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process(?string $userId, array $input, callable $reportProgress): array { + if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { + throw new RuntimeException('Must support chat completion endpoint'); + } + + if (!isset($input['input']) || !is_array($input['input'])) { + throw new RuntimeException('Invalid file list'); + } + if (count($input['input']) === 0) { + throw new RuntimeException('Invalid file list'); + } + if (count($input['input']) > 500) { + throw new RuntimeException('Too many files given. Max is 500'); + } + + if (isset($input['model']) && is_string($input['model'])) { + $model = $input['model']; + } else { + $model = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + } + + $maxTokens = null; + if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { + $maxTokens = $input['max_tokens']; + } + + $fileSize = 0; + $outputs = []; + $systemPrompt = 'Extract all visible text from the image. Return only the extracted text without additional commentary. Preserve the original language of the text.'; + $userPrompt = 'Extract all text from this image.'; + + foreach ($input['input'] as $i => $file) { + if (!$file instanceof File || !$file->isReadable()) { + throw new RuntimeException('Invalid input file'); + } + $fileSize += intval($file->getSize()); + if ($fileSize > 50 * 1000 * 1000) { + throw new UserFacingProcessingException('Filesize of input files too large. Max is 50MB', userFacingMessage: $this->l->t('Filesize of input files too large. Max is 50MB')); + } + + $fileType = $file->getMimeType(); + if (!str_starts_with($fileType, 'image/')) { + throw new UserFacingProcessingException('Only supports image file types' . $fileType, userFacingMessage: $this->l->t('Only supports image file types')); + } + if ($this->openAiAPIService->isUsingOpenAi()) { + $validFileTypes = [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + ]; + if (!in_array($fileType, $validFileTypes, true)) { + throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); + } + } + + $inputFile = base64_encode(stream_get_contents($file->fopen('rb'))); + $history = [ + json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $fileType . ';base64,' . $inputFile, + ], + ], + ], + ]), + ]; + + try { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; + + if (count($messages) === 0) { + throw new RuntimeException('No result in OpenAI/LocalAI response.'); + } + + $outputs[] = array_pop($messages); + $reportProgress(($i + 1) / count($input['input'])); + } catch (\Exception $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + } + + return ['output' => $outputs]; + } +} From 51b3e516ceab4e76f8e14a051f55d3c4eb85a1ce Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 6 Jul 2026 17:12:04 -0400 Subject: [PATCH 2/3] Support pdfs by converting them to images Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/ImageToTextOcrProvider.php | 157 +++++++++++++----- 1 file changed, 118 insertions(+), 39 deletions(-) diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php index 44e7a23b..3bc91f46 100644 --- a/lib/TaskProcessing/ImageToTextOcrProvider.php +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -9,11 +9,11 @@ namespace OCA\OpenAi\TaskProcessing; +use Imagick; use OCA\OpenAi\AppInfo\Application; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; use OCP\Files\File; -use OCP\IAppConfig; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\UserFacingProcessingException; @@ -30,7 +30,6 @@ public function __construct( private OpenAiSettingsService $openAiSettingsService, private IL10N $l, private LoggerInterface $logger, - private IAppConfig $appConfig, private ?string $userId, ) { } @@ -128,10 +127,12 @@ public function process(?string $userId, array $input, callable $reportProgress) $fileSize = 0; $outputs = []; + $fileCount = (float)count($input['input']); $systemPrompt = 'Extract all visible text from the image. Return only the extracted text without additional commentary. Preserve the original language of the text.'; $userPrompt = 'Extract all text from this image.'; - foreach ($input['input'] as $i => $file) { + $fileIndex = 0; + foreach ($input['input'] as $file) { if (!$file instanceof File || !$file->isReadable()) { throw new RuntimeException('Invalid input file'); } @@ -141,52 +142,130 @@ public function process(?string $userId, array $input, callable $reportProgress) } $fileType = $file->getMimeType(); - if (!str_starts_with($fileType, 'image/')) { - throw new UserFacingProcessingException('Only supports image file types' . $fileType, userFacingMessage: $this->l->t('Only supports image file types')); - } - if ($this->openAiAPIService->isUsingOpenAi()) { - $validFileTypes = [ - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/webp', - ]; - if (!in_array($fileType, $validFileTypes, true)) { - throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); - } - } - $inputFile = base64_encode(stream_get_contents($file->fopen('rb'))); - $history = [ - json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:' . $fileType . ';base64,' . $inputFile, + if ($fileType === 'application/pdf') { + $outputForFile = ''; + $imagickProbe = $this->getImagickProbe($file); + $pagesRead = 0; + foreach ($this->imagickPdfToJpegBase64($imagickProbe['image']) as $base64Image) { + $pagesRead++; + $history = [json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . $base64Image, + ], ], ], - ], - ]), - ]; + ])]; + try { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; - try { - $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $messages = $completion['messages']; + if (count($messages) === 0) { + $this->logger->warning('No result in OpenAI/LocalAI response.'); + $outputs[] = ''; + continue; + } - if (count($messages) === 0) { - throw new RuntimeException('No result in OpenAI/LocalAI response.'); + $outputForFile .= array_pop($messages) . "\n\n"; + $reportProgress(((float)$fileIndex + (float)($pagesRead / $imagickProbe['count'])) / $fileCount); + } catch (\Exception $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + } + $outputs[] = $outputForFile; + $reportProgress(((float)$fileIndex + 1.0) / $fileCount); + } elseif (str_starts_with($fileType, 'image/')) { + if ($this->openAiAPIService->isUsingOpenAi()) { + $validFileTypes = [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + ]; + if (!in_array($fileType, $validFileTypes, true)) { + throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); + } } - $outputs[] = array_pop($messages); - $reportProgress(($i + 1) / count($input['input'])); - } catch (\Exception $e) { - $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); - throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + $base64Image = base64_encode(stream_get_contents($file->fopen('rb'))); + $history = [ + json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $fileType . ';base64,' . $base64Image, + ], + ], + ], + ]), + ]; + try { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; + $reportProgress(((float)$fileIndex + 1.0) / $fileCount); + + if (count($messages) === 0) { + $this->logger->warning('No result in OpenAI/LocalAI response.'); + $outputs[] = ''; + continue; + } + + $outputs[] = array_pop($messages); + } catch (\Exception $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + } else { + throw new UserFacingProcessingException('Only supports image and pdf file types' . $fileType, userFacingMessage: $this->l->t('Only supports image and pdf file types')); } + $fileIndex++; } return ['output' => $outputs]; } + + /** + * @return array{count: int, image: Imagick} + */ + private function getImagickProbe(File $file): array { + if (!extension_loaded('imagick')) { + throw new RuntimeException('Imagick extension not available can not process PDF'); + } + if (empty(Imagick::queryFormats('PDF'))) { + throw new RuntimeException('Imagick has no PDF support (Ghostscript missing or blocked by policy.xml)'); + } + + $pdfContent = $file->getContent(); + $probe = new Imagick(); + $probe->setResolution(200, 200); + $probe->readImageBlob($pdfContent); + return ['count' => $probe->getNumberImages(), 'image' => $probe]; + } + /** + * @return \Generator + */ + private function imagickPdfToJpegBase64(Imagick $im, int $maxPages = 100): \Generator { + $pageCount = 0; + foreach ($im as $page) { + if ($pageCount >= $maxPages) { + break; + } + $page = $page->getImage(); + $page->setImageBackgroundColor('white'); + $page = $page->flattenImages(); + $page->setImageFormat('jpeg'); + $page->setImageCompressionQuality(85); + yield base64_encode($page->getImageBlob()); + $page->clear(); + $pageCount++; + } + $im->clear(); + } } From e27f1d80403e246d87b7f06406d92deb5d50a175 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 6 Jul 2026 18:23:51 -0400 Subject: [PATCH 3/3] Add batching to requests Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/ImageToTextOcrProvider.php | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php index 3bc91f46..d21f5197 100644 --- a/lib/TaskProcessing/ImageToTextOcrProvider.php +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -143,23 +143,29 @@ public function process(?string $userId, array $input, callable $reportProgress) $fileType = $file->getMimeType(); + // handle pdf files by converting to jpeg and then passing to the api if ($fileType === 'application/pdf') { $outputForFile = ''; $imagickProbe = $this->getImagickProbe($file); $pagesRead = 0; - foreach ($this->imagickPdfToJpegBase64($imagickProbe['image']) as $base64Image) { - $pagesRead++; - $history = [json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:image/jpeg;base64,' . $base64Image, + $history = []; + foreach ($this->imagickPdfToJpegBase64($imagickProbe['image'], 5) as $base64Image) { + $pagesRead += count($base64Image); + $history = []; + // Batches 5 in one request to speed up the process + foreach ($base64Image as $image) { + $history[] = json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . $image, + ], ], ], - ], - ])]; + ]); + } try { $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); $messages = $completion['messages']; @@ -249,10 +255,11 @@ private function getImagickProbe(File $file): array { return ['count' => $probe->getNumberImages(), 'image' => $probe]; } /** - * @return \Generator + * @return \Generator> */ - private function imagickPdfToJpegBase64(Imagick $im, int $maxPages = 100): \Generator { + private function imagickPdfToJpegBase64(Imagick $im, int $batchSize, int $maxPages = 500): \Generator { $pageCount = 0; + $batch = []; foreach ($im as $page) { if ($pageCount >= $maxPages) { break; @@ -262,9 +269,17 @@ private function imagickPdfToJpegBase64(Imagick $im, int $maxPages = 100): \Gene $page = $page->flattenImages(); $page->setImageFormat('jpeg'); $page->setImageCompressionQuality(85); - yield base64_encode($page->getImageBlob()); + $batch[] = base64_encode($page->getImageBlob()); $page->clear(); $pageCount++; + // Read in batchs to avoid memory issues + if (count($batch) >= $batchSize) { + yield $batch; + $batch = []; + } + } + if (count($batch) > 0) { + yield $batch; } $im->clear(); }