diff --git a/client/client.go b/client/client.go index a63c8134..5e2437ef 100644 --- a/client/client.go +++ b/client/client.go @@ -118,11 +118,13 @@ type API interface { CreateAITaskBuilderDataset(workspaceID string, payload CreateAITaskBuilderDatasetPayload) (*CreateAITaskBuilderDatasetResponse, error) CreateAITaskBuilderCollection(payload model.CreateAITaskBuilderCollection) (*CreateAITaskBuilderCollectionResponse, error) GetAITaskBuilderBatch(batchID string) (*GetAITaskBuilderBatchResponse, error) + GetAITaskBuilderDataset(datasetID string) (*GetAITaskBuilderDatasetResponse, error) UpdateAITaskBuilderBatch(params UpdateBatchParams) (*UpdateAITaskBuilderBatchResponse, error) GetAITaskBuilderBatchStatus(batchID string) (*GetAITaskBuilderBatchStatusResponse, error) GetAITaskBuilderBatches(workspaceID string) (*GetAITaskBuilderBatchesResponse, error) GetAITaskBuilderResponses(batchID string) (*GetAITaskBuilderResponsesResponse, error) GetAITaskBuilderTasks(batchID string) (*GetAITaskBuilderTasksResponse, error) + GetAITaskBuilderTaskGroups(batchID string) (*GetAITaskBuilderTaskGroupsResponse, error) InitiateBatchExport(batchID string) (*BatchExportResponse, error) GetBatchExportStatus(batchID, exportID string) (*BatchExportResponse, error) SyncAITaskBuilderBatch(batchID string) (*AITaskBuilderBatchSyncResponse, error) @@ -1384,6 +1386,18 @@ func (c *Client) GetAITaskBuilderTasks(batchID string) (*GetAITaskBuilderTasksRe return &response, nil } +// GetAITaskBuilderTaskGroups will return the task group IDs for an AI Task Builder batch. +func (c *Client) GetAITaskBuilderTaskGroups(batchID string) (*GetAITaskBuilderTaskGroupsResponse, error) { + var response GetAITaskBuilderTaskGroupsResponse + + url := fmt.Sprintf("/api/v1/data-collection/batches/%s/task-groups", batchID) + _, err := c.Execute(http.MethodGet, url, nil, &response) + if err != nil { + return nil, fmt.Errorf("unable to fulfil request %s: %s", url, err) + } + return &response, nil +} + // InitiateBatchExport starts a batch export job via POST. // Returns "generating" + ExportID (202) if a new job was enqueued, // or "complete" + URL immediately (200) if a valid export already exists. @@ -1463,6 +1477,18 @@ func (c *Client) GetAITaskBuilderBatchSyncStatus(batchID, syncID string) (*AITas return &response, nil } +// GetAITaskBuilderDataset will return an AI Task Builder dataset by ID. +func (c *Client) GetAITaskBuilderDataset(datasetID string) (*GetAITaskBuilderDatasetResponse, error) { + var response GetAITaskBuilderDatasetResponse + + url := fmt.Sprintf("/api/v1/data-collection/datasets/%s", datasetID) + _, err := c.Execute(http.MethodGet, url, nil, &response) + if err != nil { + return nil, fmt.Errorf("unable to fulfil request %s: %s", url, err) + } + return &response, nil +} + // GetAITaskBuilderDatasetStatus will return the status of an AI Task Builder dataset. func (c *Client) GetAITaskBuilderDatasetStatus(datasetID string) (*GetAITaskBuilderDatasetStatusResponse, error) { var response GetAITaskBuilderDatasetStatusResponse diff --git a/client/responses.go b/client/responses.go index cd8126fc..d22b69da 100644 --- a/client/responses.go +++ b/client/responses.go @@ -262,6 +262,26 @@ type GetAITaskBuilderResponsesResponse struct { // The API returns a simple array of task ID strings. type GetAITaskBuilderTasksResponse []string +// GetAITaskBuilderTaskGroupsResponse is the response for the get AI Task Builder task groups endpoint. +// The API returns a simple array of task group ID strings. +type GetAITaskBuilderTaskGroupsResponse []string + +// GetAITaskBuilderDatasetResponse is the response for the get AI Task Builder dataset endpoint. +type GetAITaskBuilderDatasetResponse struct { + ID string `json:"id"` + Name string `json:"name"` + CreatedAt string `json:"created_at"` + CreatedBy string `json:"created_by"` + WorkspaceID string `json:"workspace_id"` + TotalDatapointCount int `json:"total_datapoint_count"` + SchemaVersion int `json:"schema_version"` + Status model.DatasetStatus `json:"status,omitempty"` + Filename *string `json:"filename,omitempty"` + HasPredeterminedGroupingID *bool `json:"has_predetermined_grouping_id,omitempty"` + Schema *DatasetSchema `json:"schema"` + Imports []model.DatasetImportJob `json:"imports"` +} + // GetAITaskBuilderDatasetStatusResponse is the response for the get AI Task Builder dataset status endpoint. type GetAITaskBuilderDatasetStatusResponse struct { Status model.DatasetStatus `json:"status"` diff --git a/cmd/aitaskbuilder/batch_preview.go b/cmd/aitaskbuilder/batch_preview.go new file mode 100644 index 00000000..814f2d53 --- /dev/null +++ b/cmd/aitaskbuilder/batch_preview.go @@ -0,0 +1,106 @@ +package aitaskbuilder + +import ( + "errors" + "fmt" + "io" + + "github.com/pkg/browser" + "github.com/prolific-oss/cli/client" + "github.com/spf13/cobra" +) + +// BrowserOpener is a function type for opening URLs in a browser. +// This allows for dependency injection in tests. +type BrowserOpener func(url string) error + +// DefaultBrowserOpener uses the system browser to open URLs. +var DefaultBrowserOpener BrowserOpener = browser.OpenURL + +// BatchPreviewOptions is the options for the batch preview command. +type BatchPreviewOptions struct { + Args []string + BatchID string + BrowserOpener BrowserOpener +} + +// NewBatchPreviewCommand creates a new `batch preview` command to open a batch +// preview in the browser. +func NewBatchPreviewCommand(c client.API, w io.Writer) *cobra.Command { + return NewBatchPreviewCommandWithOpener(c, w, DefaultBrowserOpener) +} + +// NewBatchPreviewCommandWithOpener creates a new `batch preview` command with a +// custom browser opener. This is useful for testing to avoid opening actual +// browser windows. +func NewBatchPreviewCommandWithOpener(c client.API, w io.Writer, browserOpener BrowserOpener) *cobra.Command { + var opts BatchPreviewOptions + opts.BrowserOpener = browserOpener + + cmd := &cobra.Command{ + Use: "preview ", + Args: cobra.ExactArgs(1), + Short: "Preview a batch in the browser", + Long: `Preview a batch in the browser + +Opens the batch's first task group in your default web browser so you can +preview it before launch.`, + Example: ` +Preview an AI Task Builder batch in the browser: +$ prolific aitaskbuilder batch preview + `, + RunE: func(cmd *cobra.Command, args []string) error { + opts.Args = args + opts.BatchID = args[0] + + err := renderAITaskBuilderBatchPreview(c, opts, w) + if err != nil { + return fmt.Errorf("error: %s", err) + } + + return nil + }, + } + + return cmd +} + +// renderAITaskBuilderBatchPreview validates access to the batch, resolves a task +// group, builds the preview URL, prints it, and attempts to open it in a browser. +func renderAITaskBuilderBatchPreview(c client.API, opts BatchPreviewOptions, w io.Writer) error { + if opts.BatchID == "" { + return errors.New(ErrBatchIDRequired) + } + + // Fetch batch to validate access + _, err := c.GetAITaskBuilderBatch(opts.BatchID) + if err != nil { + return fmt.Errorf("failed to get batch: %s", err) + } + + taskGroups, err := c.GetAITaskBuilderTaskGroups(opts.BatchID) + if err != nil { + return fmt.Errorf("failed to get task groups: %s", err) + } + if len(*taskGroups) == 0 { + return fmt.Errorf("%s %s", ErrNoTaskGroupsFound, opts.BatchID) + } + + taskGroupID := (*taskGroups)[0] + + // Build the preview URL and display it + previewURL := GetBatchPreviewURL(opts.BatchID, taskGroupID) + fmt.Fprintln(w, "Opening batch preview in browser...") + fmt.Fprintln(w) + fmt.Fprintln(w, previewURL) + + // Attempt to open the browser - don't fail if it doesn't work + // (e.g., in headless/CI environments) + if opts.BrowserOpener != nil { + if err := opts.BrowserOpener(previewURL); err != nil { + fmt.Fprintln(w, "(Browser did not open automatically - use the URL above)") + } + } + + return nil +} diff --git a/cmd/aitaskbuilder/batch_preview_test.go b/cmd/aitaskbuilder/batch_preview_test.go new file mode 100644 index 00000000..9a27d24c --- /dev/null +++ b/cmd/aitaskbuilder/batch_preview_test.go @@ -0,0 +1,206 @@ +package aitaskbuilder_test + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "os" + "testing" + + "github.com/golang/mock/gomock" + "github.com/prolific-oss/cli/client" + "github.com/prolific-oss/cli/cmd/aitaskbuilder" + "github.com/prolific-oss/cli/mock_client" + "github.com/prolific-oss/cli/model" +) + +// noOpBrowserOpener is a no-op browser opener for testing. +func noOpBrowserOpener(url string) error { + return nil +} + +const testTaskGroupID = "task-group-1" + +func TestNewBatchPreviewCommand(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, os.Stdout, noOpBrowserOpener) + + use := "preview " + short := "Preview a batch in the browser" + + if cmd.Use != use { + t.Fatalf("expected use: %s; got %s", use, cmd.Use) + } + + if cmd.Short != short { + t.Fatalf("expected short: %s; got %s", short, cmd.Short) + } +} + +func TestBatchPreviewRequiresBatchID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, os.Stdout, noOpBrowserOpener) + err := cmd.Args(cmd, []string{}) + + if err == nil { + t.Fatal("expected error when batch-id is missing") + } +} + +func TestBatchPreviewAcceptsPositionalBatchID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + batchID := testBatchUUID + taskGroupID := testTaskGroupID + + response := client.GetAITaskBuilderBatchResponse{ + AITaskBuilderBatch: model.AITaskBuilderBatch{ID: batchID}, + } + taskGroups := client.GetAITaskBuilderTaskGroupsResponse{taskGroupID} + + c.EXPECT().GetAITaskBuilderBatch(gomock.Eq(batchID)).Return(&response, nil).Times(1) + c.EXPECT().GetAITaskBuilderTaskGroups(gomock.Eq(batchID)).Return(&taskGroups, nil).Times(1) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, os.Stdout, noOpBrowserOpener) + if err := cmd.RunE(cmd, []string{batchID}); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestBatchPreviewCallsGetBatchAndTaskGroups(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + batchID := testBatchUUID + taskGroupID := testTaskGroupID + + response := client.GetAITaskBuilderBatchResponse{ + AITaskBuilderBatch: model.AITaskBuilderBatch{ID: batchID}, + } + taskGroups := client.GetAITaskBuilderTaskGroupsResponse{taskGroupID} + + c.EXPECT().GetAITaskBuilderBatch(gomock.Eq(batchID)).Return(&response, nil).Times(1) + c.EXPECT().GetAITaskBuilderTaskGroups(gomock.Eq(batchID)).Return(&taskGroups, nil).Times(1) + + var b bytes.Buffer + writer := bufio.NewWriter(&b) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, writer, noOpBrowserOpener) + err := cmd.RunE(cmd, []string{batchID}) + writer.Flush() + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } +} + +func TestBatchPreviewReturnsErrorOnBatchClientError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + batchID := "an-invalid-batch-id" + + c.EXPECT().GetAITaskBuilderBatch(gomock.Eq(batchID)).Return(nil, errors.New("batch not found")).Times(1) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, os.Stdout, noOpBrowserOpener) + err := cmd.RunE(cmd, []string{batchID}) + + expected := "error: failed to get batch: batch not found" + if err == nil || err.Error() != expected { + t.Fatalf("expected error %q, got %v", expected, err) + } +} + +func TestBatchPreviewReturnsErrorWhenNoTaskGroups(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + batchID := testBatchUUID + + response := client.GetAITaskBuilderBatchResponse{ + AITaskBuilderBatch: model.AITaskBuilderBatch{ID: batchID}, + } + taskGroups := client.GetAITaskBuilderTaskGroupsResponse{} + + c.EXPECT().GetAITaskBuilderBatch(gomock.Eq(batchID)).Return(&response, nil).Times(1) + c.EXPECT().GetAITaskBuilderTaskGroups(gomock.Eq(batchID)).Return(&taskGroups, nil).Times(1) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, os.Stdout, noOpBrowserOpener) + err := cmd.RunE(cmd, []string{batchID}) + + expected := fmt.Sprintf("error: %s %s", aitaskbuilder.ErrNoTaskGroupsFound, batchID) + if err == nil || err.Error() != expected { + t.Fatalf("expected error %q, got %v", expected, err) + } +} + +func TestBatchPreviewOutputContainsURL(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + + batchID := testBatchUUID + taskGroupID := testTaskGroupID + + response := client.GetAITaskBuilderBatchResponse{ + AITaskBuilderBatch: model.AITaskBuilderBatch{ID: batchID}, + } + taskGroups := client.GetAITaskBuilderTaskGroupsResponse{taskGroupID} + + c.EXPECT().GetAITaskBuilderBatch(gomock.Eq(batchID)).Return(&response, nil).Times(1) + c.EXPECT().GetAITaskBuilderTaskGroups(gomock.Eq(batchID)).Return(&taskGroups, nil).Times(1) + + var b bytes.Buffer + writer := bufio.NewWriter(&b) + + cmd := aitaskbuilder.NewBatchPreviewCommandWithOpener(c, writer, noOpBrowserOpener) + err := cmd.RunE(cmd, []string{batchID}) + writer.Flush() + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + + output := b.String() + expectedStrings := []string{ + "Opening batch preview in browser", + "data-collection-tool/batches/" + batchID + "/task-groups/" + taskGroupID, + "preview=true", + } + + for _, expected := range expectedStrings { + if !bytes.Contains([]byte(output), []byte(expected)) { + t.Errorf("expected output to contain %q, got: %s", expected, output) + } + } +} + +func TestGetBatchPreviewPath(t *testing.T) { + path := aitaskbuilder.GetBatchPreviewPath("batch-123", "task-group-456") + + expected := "data-collection-tool/batches/batch-123/task-groups/task-group-456?preview=true" + if path != expected { + t.Fatalf("expected path %q, got %q", expected, path) + } +} + +func TestGetBatchPreviewURL(t *testing.T) { + url := aitaskbuilder.GetBatchPreviewURL("batch-123", "task-group-456") + + expected := "https://app.prolific.com/data-collection-tool/batches/batch-123/task-groups/task-group-456?preview=true" + if url != expected { + t.Fatalf("expected URL %q, got %q", expected, url) + } +} diff --git a/cmd/aitaskbuilder/batch_setup_test.go b/cmd/aitaskbuilder/batch_setup_test.go index 09eb4cbb..6f98802b 100644 --- a/cmd/aitaskbuilder/batch_setup_test.go +++ b/cmd/aitaskbuilder/batch_setup_test.go @@ -155,7 +155,7 @@ func TestNewBatchSetupCommandMissingRequiredFlags(t *testing.T) { }, { name: "missing dataset-id flag", - args: []string{"--batch-id", "01954894-65b3-779e-aaf6-348698e23634", "--tasks-per-group", "3"}, + args: []string{"--batch-id", testBatchUUID, "--tasks-per-group", "3"}, expectedErr: `required flag(s) "dataset-id" not set`, }, { diff --git a/cmd/aitaskbuilder/batches.go b/cmd/aitaskbuilder/batches.go index 8593cff7..674f4317 100644 --- a/cmd/aitaskbuilder/batches.go +++ b/cmd/aitaskbuilder/batches.go @@ -1,9 +1,11 @@ package aitaskbuilder import ( + "fmt" "io" "github.com/prolific-oss/cli/client" + "github.com/prolific-oss/cli/config" "github.com/spf13/cobra" ) @@ -27,7 +29,18 @@ func NewBatchesCommand(client client.API, w io.Writer) *cobra.Command { NewGetBatchesListCommand(client, w), NewGetResponsesCommand(client, w), NewBatchTasksCommand(client, w), + NewBatchPreviewCommand(client, w), ) return cmd } + +// GetBatchPreviewPath returns the URL path to a batch preview, agnostic of domain. +func GetBatchPreviewPath(batchID, taskGroupID string) string { + return fmt.Sprintf("data-collection-tool/batches/%s/task-groups/%s?preview=true", batchID, taskGroupID) +} + +// GetBatchPreviewURL returns the full URL to a batch preview using configuration. +func GetBatchPreviewURL(batchID, taskGroupID string) string { + return fmt.Sprintf("%s/%s", config.GetApplicationURL(), GetBatchPreviewPath(batchID, taskGroupID)) +} diff --git a/cmd/aitaskbuilder/constants.go b/cmd/aitaskbuilder/constants.go index 5d3204a4..a3385fa8 100644 --- a/cmd/aitaskbuilder/constants.go +++ b/cmd/aitaskbuilder/constants.go @@ -17,6 +17,7 @@ const ( ErrFilePathRequired = "file path is required" ErrInstructionInputRequired = "either instructions file (-f) or JSON string (-j) must be provided" ErrNameRequired = "name is required" + ErrNoTaskGroupsFound = "no task groups found for batch" ErrSchemaInvalidJSON = "schema contains invalid JSON" ErrSchemaStrictSetInBoth = "cannot set strict in both --schema and --strict" ErrStrictRequiresSchema = "--strict requires --schema" diff --git a/cmd/aitaskbuilder/create_dataset.go b/cmd/aitaskbuilder/create_dataset.go index fdc9b5e7..cc6f65ed 100644 --- a/cmd/aitaskbuilder/create_dataset.go +++ b/cmd/aitaskbuilder/create_dataset.go @@ -44,15 +44,16 @@ passes it through unchanged. The value is the full schema object, for example: "fields": { "question": { "type": "text", "label": "Question" }, "image": { "type": "image_url" }, + "audio": { "type": "audio_url" }, "source": { "type": "metadata" }, "group": { "type": "task_group_id" } } } -Field types are text, image_url, metadata, and task_group_id (at most one). By -default schemas are created with "strict": false. Use --strict to enable strict -mode when the schema JSON does not already set "strict" (passing --strict -alongside a schema that sets "strict" is an error). See +Field types are text, image_url, audio_url, metadata, and task_group_id (at +most one). By default schemas are created with "strict": false. Use --strict +to enable strict mode when the schema JSON does not already set "strict" +(passing --strict alongside a schema that sets "strict" is an error). See docs/examples/dataset-schema.json for a full example. Schemas are only accepted for workspaces where the typed-dataset feature is diff --git a/cmd/aitaskbuilder/dataset_schema.go b/cmd/aitaskbuilder/dataset_schema.go index ef388d48..115a72ab 100644 --- a/cmd/aitaskbuilder/dataset_schema.go +++ b/cmd/aitaskbuilder/dataset_schema.go @@ -18,6 +18,7 @@ var validDatasetSchemaFieldTypes = map[string]bool{ "image_url": true, "metadata": true, "task_group_id": true, + "audio_url": true, } // rawDatasetSchema mirrors DatasetSchema but distinguishes an absent "strict" @@ -54,7 +55,7 @@ func resolveDatasetSchema(schemaInput string, strict, strictSet bool) (*client.D taskGroupIDCount := 0 for key, field := range parsed.Fields { if !validDatasetSchemaFieldTypes[field.Type] { - return nil, fmt.Errorf("field %q has invalid type %q; must be one of text, image_url, metadata, task_group_id", key, field.Type) + return nil, fmt.Errorf("field %q has invalid type %q; must be one of text, image_url, metadata, task_group_id, audio_url", key, field.Type) } if field.Type == "task_group_id" { taskGroupIDCount++ diff --git a/cmd/aitaskbuilder/dataset_schema_test.go b/cmd/aitaskbuilder/dataset_schema_test.go index 38a43caf..9934ede8 100644 --- a/cmd/aitaskbuilder/dataset_schema_test.go +++ b/cmd/aitaskbuilder/dataset_schema_test.go @@ -18,6 +18,7 @@ func TestResolveDatasetSchemaInlineValid(t *testing.T) { "fields": { "question": { "type": "text", "label": "Question" }, "image": { "type": "image_url" }, + "audio": { "type": "audio_url" }, "source": { "type": "metadata" }, "group": { "type": "task_group_id" } } @@ -33,8 +34,11 @@ func TestResolveDatasetSchemaInlineValid(t *testing.T) { if schema.Strict == nil || !*schema.Strict { t.Fatal("expected strict to be true from JSON") } - if len(schema.Fields) != 4 { - t.Fatalf("expected 4 fields; got %d", len(schema.Fields)) + if len(schema.Fields) != 5 { + t.Fatalf("expected 5 fields; got %d", len(schema.Fields)) + } + if schema.Fields["audio"].Type != "audio_url" { + t.Fatalf("unexpected audio field: %+v", schema.Fields["audio"]) } if schema.Fields["question"].Type != "text" || schema.Fields["question"].Label != "Question" { t.Fatalf("unexpected question field: %+v", schema.Fields["question"]) @@ -154,7 +158,7 @@ func TestResolveDatasetSchemaInvalidFieldType(t *testing.T) { if !strings.Contains(msg, `"q"`) || !strings.Contains(msg, `"number"`) { t.Fatalf("expected error to name field and type; got %v", err) } - if !strings.Contains(msg, "text, image_url, metadata, task_group_id") { + if !strings.Contains(msg, "text, image_url, metadata, task_group_id, audio_url") { t.Fatalf("expected error to list allowed types; got %v", err) } } diff --git a/cmd/aitaskbuilder/get_batch_test.go b/cmd/aitaskbuilder/get_batch_test.go index 8b17db4f..b8bde491 100644 --- a/cmd/aitaskbuilder/get_batch_test.go +++ b/cmd/aitaskbuilder/get_batch_test.go @@ -16,6 +16,9 @@ import ( "github.com/prolific-oss/cli/model" ) +// testBatchUUID is a shared batch ID used across multiple aitaskbuilder tests. +const testBatchUUID = "01954894-65b3-779e-aaf6-348698e23634" + func TestNewGetBatchCommand(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -40,7 +43,7 @@ func TestNewGetBatchCommandCallsAPI(t *testing.T) { defer ctrl.Finish() c := mock_client.NewMockAPI(ctrl) - batchID := "01954894-65b3-779e-aaf6-348698e23634" + batchID := testBatchUUID createdAt, _ := time.Parse(time.RFC3339, "2025-02-27T18:03:59.795Z") response := client.GetAITaskBuilderBatchResponse{ @@ -84,7 +87,7 @@ func TestNewGetBatchCommandCallsAPI(t *testing.T) { writer.Flush() expected := `AI Task Builder Batch Details: -ID: 01954894-65b3-779e-aaf6-348698e23634 +ID: ` + batchID + ` Name: Test Batch Status: UNINITIALISED Total Task Count: 0 diff --git a/cmd/aitaskbuilder/get_dataset_status_test.go b/cmd/aitaskbuilder/get_dataset_status_test.go index 991f91ec..64c9d7b8 100644 --- a/cmd/aitaskbuilder/get_dataset_status_test.go +++ b/cmd/aitaskbuilder/get_dataset_status_test.go @@ -19,7 +19,12 @@ import ( func setupMockClient(t *testing.T) *mock_client.MockAPI { ctrl := gomock.NewController(t) t.Cleanup(func() { ctrl.Finish() }) - return mock_client.NewMockAPI(ctrl) + c := mock_client.NewMockAPI(ctrl) + c.EXPECT(). + GetAITaskBuilderDataset(gomock.Any()). + Return(&client.GetAITaskBuilderDatasetResponse{}, nil). + AnyTimes() + return c } func TestNewGetDatasetStatusCommand(t *testing.T) { diff --git a/cmd/aitaskbuilder/upload_dataset.go b/cmd/aitaskbuilder/upload_dataset.go index 2ce801ff..2477a565 100644 --- a/cmd/aitaskbuilder/upload_dataset.go +++ b/cmd/aitaskbuilder/upload_dataset.go @@ -1,11 +1,15 @@ package aitaskbuilder import ( + "bufio" "context" + "encoding/csv" + "encoding/json" "errors" "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "slices" @@ -27,6 +31,15 @@ const ( // DatasetUploadPollSleep is the sleep function used between dataset import status polls. var DatasetUploadPollSleep func(time.Duration) = time.Sleep +var validAudioURLFileExtensions = map[string]bool{ + ".aac": true, + ".m4a": true, + ".mp3": true, + ".wav": true, +} + +const supportedAudioURLFileExtensions = ".aac, .m4a, .mp3, .wav" + // DatasetUploadOptions are the options for uploading to an AI Task Builder dataset. type DatasetUploadOptions struct { Args []string @@ -110,6 +123,15 @@ func uploadDatasetFile(client client.API, opts DatasetUploadOptions, w io.Writer return err } + dataset, err := client.GetAITaskBuilderDataset(opts.DatasetID) + if err != nil { + return fmt.Errorf("failed to get dataset: %w", err) + } + + if err := validateAudioURLFields(opts.FilePath, uploadRequest.Format, dataset.Schema); err != nil { + return err + } + fmt.Fprintf(w, "Getting upload URL for dataset %s and file %s...\n", opts.DatasetID, uploadRequest.UploadFilename) // Get upload URL from API @@ -257,6 +279,164 @@ func uploadFileToPresignedURL(filePath, uploadURL, method, contentType string) e return nil } +func validateAudioURLFields(filePath string, format model.DatasetImportFormat, schema *client.DatasetSchema) error { + if schema == nil { + return nil + } + + audioFields := make(map[string]struct{}) + for fieldName, field := range schema.Fields { + if field.Type == "audio_url" { + audioFields[fieldName] = struct{}{} + } + } + + if len(audioFields) == 0 { + return nil + } + + switch format { + case model.DatasetImportFormatCSV: + return validateAudioURLFieldsInCSV(filePath, audioFields) + case model.DatasetImportFormatJSONL: + return ValidateAudioURLFieldsInJSONL(filePath, audioFields) + default: + return nil + } +} + +func validateAudioURLFieldsInCSV(filePath string, audioFields map[string]struct{}) error { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", filePath, err) + } + defer file.Close() + + reader := csv.NewReader(file) + headers, err := reader.Read() + if err != nil { + return fmt.Errorf("failed to read CSV header from %s: %w", filePath, err) + } + + audioColumnIndexes := make(map[int]string) + for idx, header := range headers { + fieldName := strings.TrimSpace(header) + if _, ok := audioFields[fieldName]; ok { + audioColumnIndexes[idx] = fieldName + } + } + + if len(audioColumnIndexes) == 0 { + return nil + } + + recordIndex := 1 + for { + record, err := reader.Read() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("failed to read CSV record %d from %s: %w", recordIndex, filePath, err) + } + + for idx, fieldName := range audioColumnIndexes { + if idx >= len(record) { + continue + } + + if err := validateAudioURLValue(recordIndex, fieldName, record[idx]); err != nil { + return err + } + } + + recordIndex++ + } +} + +func ValidateAudioURLFieldsInJSONL(filePath string, audioFields map[string]struct{}) error { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", filePath, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 10*1024*1024) + + recordIndex := 1 + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + recordIndex++ + continue + } + + record := make(map[string]any) + if err := json.Unmarshal([]byte(line), &record); err != nil { + return fmt.Errorf("failed to parse JSONL record %d from %s: %w", recordIndex, filePath, err) + } + + for fieldName := range audioFields { + value, ok := record[fieldName] + if !ok || value == nil { + continue + } + + valueString, ok := value.(string) + if !ok { + return fmt.Errorf( + "record %d field %s: audio URL must be a string ending with one of %s", + recordIndex, + fieldName, + supportedAudioURLFileExtensions, + ) + } + + if err := validateAudioURLValue(recordIndex, fieldName, valueString); err != nil { + return err + } + } + + recordIndex++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("failed to read JSONL file %s: %w", filePath, err) + } + + return nil +} + +func validateAudioURLValue(recordIndex int, fieldName, value string) error { + trimmedValue := strings.TrimSpace(value) + if trimmedValue == "" { + return nil + } + + if !hasSupportedAudioURLExtension(trimmedValue) { + return fmt.Errorf( + "record %d field %s: audio URL %q must end with one of %s", + recordIndex, + fieldName, + trimmedValue, + supportedAudioURLFileExtensions, + ) + } + + return nil +} + +func hasSupportedAudioURLExtension(value string) bool { + parsedURL, err := url.ParseRequestURI(value) + if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" { + return false + } + + extension := strings.ToLower(filepath.Ext(parsedURL.Path)) + return validAudioURLFileExtensions[extension] +} + func waitForDatasetImport( client client.API, datasetID, importID string, diff --git a/cmd/aitaskbuilder/upload_dataset_test.go b/cmd/aitaskbuilder/upload_dataset_test.go index 929daf50..18e4acdb 100644 --- a/cmd/aitaskbuilder/upload_dataset_test.go +++ b/cmd/aitaskbuilder/upload_dataset_test.go @@ -16,6 +16,7 @@ import ( "github.com/golang/mock/gomock" "github.com/prolific-oss/cli/client" "github.com/prolific-oss/cli/cmd/aitaskbuilder" + "github.com/prolific-oss/cli/mock_client" "github.com/prolific-oss/cli/model" ) @@ -248,6 +249,60 @@ func TestDatasetUploadCommandFormatOverrideAppendsExtension(t *testing.T) { } } +func TestDatasetUploadCommandRejectsUnsupportedCSVAudioExtension(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "dataset.csv") + if err := os.WriteFile(filePath, []byte("question,clip\nhello,https://example.com/audio.txt\n"), 0o600); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mock_client.NewMockAPI(ctrl) + c.EXPECT(). + GetAITaskBuilderDataset(gomock.Eq("dataset-audio")). + Return(&client.GetAITaskBuilderDatasetResponse{ + Schema: &client.DatasetSchema{ + Fields: map[string]client.DatasetSchemaField{ + "clip": {Type: "audio_url"}, + }, + }, + }, nil). + Times(1) + + cmd := aitaskbuilder.NewDatasetUploadCommand(c, os.Stdout) + _ = cmd.Flags().Set("dataset-id", "dataset-audio") + _ = cmd.Flags().Set("file", filePath) + + err := cmd.RunE(cmd, nil) + if err == nil { + t.Fatal("expected invalid audio URL extension error") + } + + if !strings.Contains(err.Error(), `must end with one of .aac, .m4a, .mp3, .wav`) { + t.Fatalf("expected supported extensions in error, got %v", err) + } +} + +func TestValidateAudioURLFieldsInJSONLRejectsUnsupportedAudioExtension(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "dataset.jsonl") + if err := os.WriteFile(filePath, []byte("{\"clip\":\"https://example.com/audio.mov\"}\n"), 0o600); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + audioFields := map[string]struct{}{ + "clip": {}, + } + + err := aitaskbuilder.ValidateAudioURLFieldsInJSONL(filePath, audioFields) + if err == nil { + t.Fatal("expected invalid audio URL extension error") + } + + if !strings.Contains(err.Error(), "record 1 field clip") { + t.Fatalf("expected record location in error, got %v", err) + } +} + func TestDatasetUploadCommandRendersPartialImportSummary(t *testing.T) { defer setDatasetUploadPollSleepForTesting(func(time.Duration) {})() diff --git a/contract_test/contract_test.go b/contract_test/contract_test.go index 7d2d748d..444fadc7 100644 --- a/contract_test/contract_test.go +++ b/contract_test/contract_test.go @@ -199,7 +199,7 @@ var operations = []operation{ c.CreateAITaskBuilderDataset("ws-id", client.CreateAITaskBuilderDatasetPayload{Name: "t"}) }}, {operationID: "get-dataset-upload-url", call: func(c *client.Client) { c.GetAITaskBuilderDatasetUploadURL("ds-id", "data.jsonl") }}, - {operationID: "get-task-builder-dataset", skip: "OUTOFSCOPE: no CLI command for retrieving a specific dataset by ID"}, + {operationID: "get-task-builder-dataset", call: func(c *client.Client) { c.GetAITaskBuilderDataset("ds-id") }}, {operationID: "get-task-builder-dataset-status", call: func(c *client.Client) { c.GetAITaskBuilderDatasetStatus("ds-id") }}, {operationID: "get-dataset-import-status", call: func(c *client.Client) { c.GetAITaskBuilderDatasetImportStatus("ds-id", "import-id") @@ -275,6 +275,7 @@ var operations = []operation{ }) }}, {operationID: "get-project-studies", call: func(c *client.Client) { c.GetStudies("", "proj-id") }}, + {operationID: "delete-project-study", skip: "OUTOFSCOPE: no CLI command for deleting a study"}, {operationID: "get-study", call: func(c *client.Client) { c.GetStudy("study-id") }}, {operationID: "delete-study", skip: "OUTOFSCOPE: no CLI command for deleting a study"}, {operationID: "update-study", call: func(c *client.Client) { diff --git a/docs/examples/dataset-schema.json b/docs/examples/dataset-schema.json index fbac5a67..df374bd5 100644 --- a/docs/examples/dataset-schema.json +++ b/docs/examples/dataset-schema.json @@ -3,6 +3,7 @@ "fields": { "question": { "type": "text", "label": "Question" }, "image": { "type": "image_url", "label": "Reference image" }, + "audio": { "type": "audio_url", "label": "Reference audio" }, "source": { "type": "metadata" }, "group": { "type": "task_group_id" } } diff --git a/mock_client/mock_client.go b/mock_client/mock_client.go index 13a4e172..6fcce556 100644 --- a/mock_client/mock_client.go +++ b/mock_client/mock_client.go @@ -480,6 +480,21 @@ func (mr *MockAPIMockRecorder) GetAITaskBuilderBatches(workspaceID interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAITaskBuilderBatches", reflect.TypeOf((*MockAPI)(nil).GetAITaskBuilderBatches), workspaceID) } +// GetAITaskBuilderDataset mocks base method. +func (m *MockAPI) GetAITaskBuilderDataset(datasetID string) (*client.GetAITaskBuilderDatasetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAITaskBuilderDataset", datasetID) + ret0, _ := ret[0].(*client.GetAITaskBuilderDatasetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAITaskBuilderDataset indicates an expected call of GetAITaskBuilderDataset. +func (mr *MockAPIMockRecorder) GetAITaskBuilderDataset(datasetID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAITaskBuilderDataset", reflect.TypeOf((*MockAPI)(nil).GetAITaskBuilderDataset), datasetID) +} + // GetAITaskBuilderDatasetImportStatus mocks base method. func (m *MockAPI) GetAITaskBuilderDatasetImportStatus(datasetID, importID string) (*client.GetAITaskBuilderDatasetImportStatusResponse, error) { m.ctrl.T.Helper() @@ -540,6 +555,21 @@ func (mr *MockAPIMockRecorder) GetAITaskBuilderResponses(batchID interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAITaskBuilderResponses", reflect.TypeOf((*MockAPI)(nil).GetAITaskBuilderResponses), batchID) } +// GetAITaskBuilderTaskGroups mocks base method. +func (m *MockAPI) GetAITaskBuilderTaskGroups(batchID string) (*client.GetAITaskBuilderTaskGroupsResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAITaskBuilderTaskGroups", batchID) + ret0, _ := ret[0].(*client.GetAITaskBuilderTaskGroupsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAITaskBuilderTaskGroups indicates an expected call of GetAITaskBuilderTaskGroups. +func (mr *MockAPIMockRecorder) GetAITaskBuilderTaskGroups(batchID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAITaskBuilderTaskGroups", reflect.TypeOf((*MockAPI)(nil).GetAITaskBuilderTaskGroups), batchID) +} + // GetAITaskBuilderTasks mocks base method. func (m *MockAPI) GetAITaskBuilderTasks(batchID string) (*client.GetAITaskBuilderTasksResponse, error) { m.ctrl.T.Helper() diff --git a/scripts/manual-tests/test_audio_batch_preview.go b/scripts/manual-tests/test_audio_batch_preview.go new file mode 100644 index 00000000..5d2629c4 --- /dev/null +++ b/scripts/manual-tests/test_audio_batch_preview.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +const defaultWorkspaceID = "679271425fe00981084a5f58" + +var fieldPattern = regexp.MustCompile(`(?m)^([A-Za-z ]+):\s*(\S+)$`) + +func main() { + repoRoot, err := repoRoot() + if err != nil { + fatal(err) + } + + workspaceID := defaultWorkspaceID + if len(os.Args) > 1 { + workspaceID = os.Args[1] + } + + cliBinary := filepath.Join(os.TempDir(), "prolific-cli") + if err := buildCLI(repoRoot, cliBinary); err != nil { + fatal(err) + } + + schema := `{"fields":{"question":{"type":"text","label":"Question"},"clip":{"type":"audio_url","label":"Audio clip"}}}` + batchItems := `[{"rows":[{"columns":[{"items":[{"type":"dataset_field","field":"question"},{"type":"dataset_field","field":"clip"},{"type":"free_text","description":"Please describe what you heard."}]}]}]}]` + + // Create a V4 dataset whose schema includes an audio_url field. + output, err := run(repoRoot, cliBinary, + "aitaskbuilder", "dataset", "create", + "-n", "Audio URL Test Dataset", + "-w", workspaceID, + "--strict", + "--schema", schema, + ) + if err != nil { + fatal(err) + } + + datasetID, err := extractField(output, "ID") + if err != nil { + fatal(err) + } + fmt.Printf("Created dataset: %s\n", datasetID) + + csvPath := filepath.Join(os.TempDir(), "audio-dataset.csv") + csvContents := strings.Join([]string{ + "question,clip", + `"What emotion is being expressed?","https://www.soundhelix.com/examples/mp3/SoundHelix-Song-1.mp3"`, + `"Transcribe the spoken word.","https://www.w3schools.com/html/horse.mp3"`, + "", + }, "\n") + if err := os.WriteFile(csvPath, []byte(csvContents), 0o600); err != nil { + fatal(fmt.Errorf("failed to write CSV fixture: %w", err)) + } + + invalidCSVPath := filepath.Join(os.TempDir(), "audio-dataset-invalid.csv") + invalidCSVContents := strings.Join([]string{ + "question,clip", + `"This row should fail validation.","https://example.com/not-audio.txt"`, + "", + }, "\n") + if err := os.WriteFile(invalidCSVPath, []byte(invalidCSVContents), 0o600); err != nil { + fatal(fmt.Errorf("failed to write invalid CSV fixture: %w", err)) + } + + // Attempt an invalid upload first to confirm audio_url extension validation rejects non-audio URLs. + if _, err := runAllowFailure(repoRoot, cliBinary, "aitaskbuilder", "dataset", "upload", "-d", datasetID, "-f", invalidCSVPath); err == nil { + fatal(fmt.Errorf("expected invalid audio URL upload to fail")) + } else { + fmt.Println("Confirmed invalid audio URL upload was rejected.") + } + + // Upload valid sample data containing supported audio URL extensions. + if _, err := run(repoRoot, cliBinary, "aitaskbuilder", "dataset", "upload", "-d", datasetID, "-f", csvPath); err != nil { + fatal(err) + } + + // Check dataset status for extra visibility while running the manual flow. + _, _ = runAllowFailure(repoRoot, cliBinary, "aitaskbuilder", "dataset", "check", "-d", datasetID) + + // Create a batch linked to the dataset and define the participant layout via batch_items. + // batch_items replaces the deprecated standalone instructions endpoint, and this + // manual flow intentionally references the audio_url dataset field even though + // the checked-in OpenAPI file has not caught up with the backend yet. + output, err = run(repoRoot, cliBinary, + "aitaskbuilder", "batch", "create", + "-n", "Audio URL Test Batch", + "-w", workspaceID, + "-d", datasetID, + "--task-name", "Audio Review Task", + "--task-introduction", "Listen to the audio clip and answer the question.", + "--task-steps", "1. Listen to the clip\\n2. Answer the question", + "--batch-items-json", batchItems, + ) + if err != nil { + fatal(err) + } + + batchID, err := extractField(output, "ID") + if err != nil { + fatal(err) + } + fmt.Printf("Created batch: %s\n", batchID) + + // Set up the batch so the persisted preview route has a task group to open. + if _, err := run(repoRoot, cliBinary, + "aitaskbuilder", "batch", "setup", + "-b", batchID, + "-d", datasetID, + "--tasks-per-group", "1", + ); err != nil { + fatal(err) + } + + // Preview the batch through the researcher preview URL flow. + if _, err := run(repoRoot, cliBinary, "aitaskbuilder", "batch", "preview", batchID); err != nil { + fatal(err) + } + + fmt.Println("\nDone. Review output above for correctness.") +} + +func repoRoot() (string, error) { + _, filePath, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("failed to resolve script path") + } + + return filepath.Abs(filepath.Join(filepath.Dir(filePath), "..", "..")) +} + +func buildCLI(repoRoot, cliBinary string) error { + fmt.Println("Building CLI...") + cmd := exec.CommandContext(context.Background(), "go", "build", "-o", cliBinary, ".") + _, err := runWithCheck(true, repoRoot, "go", cmd) + return err +} + +func run(repoRoot, cliBinary string, args ...string) (string, error) { + // #nosec G702 -- cliBinary is the local binary built by this script, not shell-expanded user input. + cmd := exec.CommandContext(context.Background(), cliBinary, args...) + return runWithCheck(true, repoRoot, cliBinary, cmd) +} + +func runAllowFailure(repoRoot, cliBinary string, args ...string) (string, error) { + // #nosec G702 -- cliBinary is the local binary built by this script, not shell-expanded user input. + cmd := exec.CommandContext(context.Background(), cliBinary, args...) + return runWithCheck(false, repoRoot, cliBinary, cmd) +} + +func runWithCheck(check bool, repoRoot, command string, cmd *exec.Cmd) (string, error) { + fmt.Printf("\n$ %s %s\n", command, strings.Join(cmd.Args[1:], " ")) + + cmd.Dir = repoRoot + cmd.Env = os.Environ() + + output, err := cmd.CombinedOutput() + text := string(output) + fmt.Print(text) + + if err != nil && check { + return text, fmt.Errorf("command failed: %w", err) + } + + return text, err +} + +func extractField(output, label string) (string, error) { + matches := fieldPattern.FindAllStringSubmatch(output, -1) + for _, match := range matches { + if len(match) >= 3 && match[1] == label { + return match[2], nil + } + } + + return "", fmt.Errorf("could not find %q in output", label) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +}