Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import org.apache.texera.amber.operator.huggingFace.codegen.{
CodegenContext,
ImageTaskCodegen,
MediaGenCodegen,
PythonCodegenBase,
HuggingFaceCodegenBase,
QaRankingCodegen,
TaskCodegen,
TextGenCodegen
Expand All @@ -47,12 +47,12 @@ import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
* `TaskCodegen` implementations registered in `registeredCodegens`.
*
* The Python script that runs at execution time is assembled by
* `PythonCodegenBase.render(ctx, codegen)`, which composes the shared
* `HuggingFaceCodegenBase.render(ctx, codegen)`, which composes the shared
* provider-fallback / request-loop infrastructure with the per-task
* payload + parse snippets supplied by the selected `TaskCodegen`.
*
* User-provided string fields are typed as [[EncodableString]] so the
* `pyb"..."` macro inside `PythonCodegenBase` emits them as
* `pyb"..."` macro inside `HuggingFaceCodegenBase` emits them as
* base64-decoded expressions at runtime instead of raw Python literals —
* this is what allows the operator to satisfy
* `PythonCodeRawInvalidTextSpec`'s contract that arbitrary `@JsonProperty`
Expand Down Expand Up @@ -246,7 +246,7 @@ class HuggingFaceInferenceOpDesc extends PythonOperatorDescriptor {
sentencesColumn = safeSentencesColumn
)

PythonCodegenBase.render(ctx, codegenForTask(safeTask))
HuggingFaceCodegenBase.render(ctx, codegenForTask(safeTask))
}

override def operatorInfo: OperatorInfo =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui
* loading, audio MIME inference, media-URL fetching, etc.) will be added
* in subsequent PRs as the corresponding task families land.
*/
object PythonCodegenBase {
object HuggingFaceCodegenBase {

def render(ctx: CodegenContext, codegen: TaskCodegen): String = {
val payload = codegen.payloadPython(ctx)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ package org.apache.texera.amber.operator.huggingFace.codegen
* zero-shot-image-classification, image-text-to-text, image-to-image.
*
* Per-row `current_image_bytes` is resolved upstream in
* [[PythonCodegenBase]]'s `process_table` (either from the operator's
* [[HuggingFaceCodegenBase]]'s `process_table` (either from the operator's
* uploaded image or from `INPUT_IMAGE_COLUMN`). The image helpers
* (`_read_image_input`, `_compress_image_bytes`, `_image_input_as_base64`,
* `_read_binary_value`, `_looks_like_html`, `_html_to_image_bytes`,
* `_extract_json_arg`) live in PythonCodegenBase alongside the per-task
* `_extract_json_arg`) live in HuggingFaceCodegenBase alongside the per-task
* tuples (`image_only_tasks`, `image_prompt_tasks`, `image_tasks`).
*/
object ImageTaskCodegen extends TaskCodegen {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
* Inputs the dispatcher passes through to each TaskCodegen.
*
* User-provided string fields are typed as [[EncodableString]] so the
* `pyb"..."` macro in [[PythonCodegenBase]] emits them as base64-decoded
* `pyb"..."` macro in [[HuggingFaceCodegenBase]] emits them as base64-decoded
* runtime expressions rather than raw Python string literals — required to
* pass `PythonCodeRawInvalidTextSpec`'s leakage check.
*/
Expand Down Expand Up @@ -57,7 +57,7 @@ final case class CodegenContext(
* `*Codegen` objects and adding them to that map.
*
* Snippets returned by these methods are Python source spliced into the
* shared template assembled by [[PythonCodegenBase.render]]. Snippets must
* shared template assembled by [[HuggingFaceCodegenBase.render]]. Snippets must
* NOT directly inline user-provided strings — reference the per-instance
* attributes `self.HF_API_TOKEN`, `self.MODEL_ID`, `self.PROMPT_COLUMN`,
* etc. that the base class initializes from `CodegenContext` via the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class PythonCodegenBaseSpec extends AnyFlatSpec with Matchers {
class HuggingFaceCodegenBaseSpec extends AnyFlatSpec with Matchers {

// A stand-in TaskCodegen whose payload/parse snippets are distinctive
// sentinels. Because payloadPython/parsePython return plain Python source
Expand Down Expand Up @@ -59,8 +59,8 @@ class PythonCodegenBaseSpec extends AnyFlatSpec with Matchers {
safeTemp = safeTemp
)

"PythonCodegenBase.render" should "emit the ProcessTableOperator skeleton and shared helpers" in {
val out = PythonCodegenBase.render(makeCtx(), StubCodegen)
"HuggingFaceCodegenBase.render" should "emit the ProcessTableOperator skeleton and shared helpers" in {
val out = HuggingFaceCodegenBase.render(makeCtx(), StubCodegen)
out should include("class ProcessTableOperator(UDFTableOperator):")
out should include("def open(self):")
out should include("def process_table(")
Expand All @@ -73,7 +73,7 @@ class PythonCodegenBaseSpec extends AnyFlatSpec with Matchers {
}

it should "splice the per-task codegen's payload and parse snippets into the template" in {
val out = PythonCodegenBase.render(makeCtx(), StubCodegen)
val out = HuggingFaceCodegenBase.render(makeCtx(), StubCodegen)
out should include("STUB_PAYLOAD_zX7q42")
out should include("STUB_PARSE_zX7q42")
}
Expand All @@ -82,8 +82,8 @@ class PythonCodegenBaseSpec extends AnyFlatSpec with Matchers {
// The same context routed through two different codegens must yield the
// spliced fragments of whichever codegen is passed, proving the base is a
// pure host for the codegen's snippets.
val stub = PythonCodegenBase.render(makeCtx(), StubCodegen)
val real = PythonCodegenBase.render(makeCtx(), TextGenCodegen)
val stub = HuggingFaceCodegenBase.render(makeCtx(), StubCodegen)
val real = HuggingFaceCodegenBase.render(makeCtx(), TextGenCodegen)
stub should include("STUB_PAYLOAD_zX7q42")
real should not include "STUB_PAYLOAD_zX7q42"
// TextGenCodegen's real payload/parse markers appear only in the real run.
Expand All @@ -96,13 +96,14 @@ class PythonCodegenBaseSpec extends AnyFlatSpec with Matchers {
}

it should "interpolate the numeric context fields as raw Python literals" in {
val out = PythonCodegenBase.render(makeCtx(safeMaxTokens = 512, safeTemp = 0.9), StubCodegen)
val out =
HuggingFaceCodegenBase.render(makeCtx(safeMaxTokens = 512, safeTemp = 0.9), StubCodegen)
out should include("self.MAX_NEW_TOKENS = 512")
out should include("self.TEMPERATURE = 0.9")
}

it should "assign user-provided strings via runtime base64 decode expressions, not raw literals" in {
val out = PythonCodegenBase.render(makeCtx(), StubCodegen)
val out = HuggingFaceCodegenBase.render(makeCtx(), StubCodegen)
// Every user-supplied string field open() assigns is set through the safe
// decode helper. This covers all EncodableString context fields, including
// the result/task and per-task (image/audio/context/labels/sentences)
Expand All @@ -125,7 +126,7 @@ class PythonCodegenBaseSpec extends AnyFlatSpec with Matchers {
it should "never leak raw user-provided string values into the generated source" in {
// Sentinels contain underscores, which base64 output cannot contain, so a
// literal match here can only mean the raw value leaked past the encoder.
val out = PythonCodegenBase.render(
val out = HuggingFaceCodegenBase.render(
makeCtx(
hfApiToken = "MARKER_TOKEN_zXyq42",
modelId = "MARKER_MODEL_zXyq42",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import { CommonModule } from "@angular/common";
import { FieldType, FieldTypeConfig } from "@ngx-formly/core";
import { NzButtonModule } from "ng-zorro-antd/button";

// Keep in sync with PythonCodegenBase._compress_image_bytes(max_bytes) on the backend:
// Keep in sync with HuggingFaceCodegenBase._compress_image_bytes(max_bytes) on the backend:
// the uploaded data URL must stay within the size the inference helpers expect.
const MAX_DATA_URL_LENGTH = 45000;
const INITIAL_MAX_DIMENSION = 512;
Expand Down
Loading