diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index e6ee8871a..ddc34be53 100755 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -87b666fa172b01444d306112309b6109c096f98b \ No newline at end of file +4811189a6ca18d0632b06ad1d44b696a0a17dbe4 \ No newline at end of file diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md old mode 100644 new mode 100755 index 2319a5a25..269157906 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -15,3 +15,11 @@ ### Internal Changes ### API Changes +* Add `create_stream()`, `delete_stream()`, `get_stream()`, `list_streams()` and `update_stream()` methods for [w.feature_engineering](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/ml/feature_engineering.html) workspace-level service. +* Add `parameters` field for `databricks.sdk.service.jobs.PipelineTask`. +* Add `pipeline_task` field for `databricks.sdk.service.jobs.ResolvedValues`. +* Add `parameters` field for `databricks.sdk.service.pipelines.CreatePipeline`. +* Add `parameters` field for `databricks.sdk.service.pipelines.EditPipeline`. +* Add `parameters` field for `databricks.sdk.service.pipelines.GetPipelineResponse`. +* [Breaking] Remove `catalog_id` field for `databricks.sdk.service.postgres.CatalogCatalogStatus`. +* [Breaking] Remove `synced_table_id` field for `databricks.sdk.service.postgres.SyncedTableSyncedTableStatus`. \ No newline at end of file diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index 8b0619011..60e2a294d 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -9231,7 +9231,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Securable: class SecurableKind(Enum): - """Latest kind: CONNECTION_ICEBERG_REST_OAUTH_M2M = 336; Next id: 337""" + """Latest kind: CONNECTION_GOOGLE_CLOUD_LAKEHOUSE_SERVICE_ACCOUNT = 340; Next id: 341""" TABLE_DB_STORAGE = "TABLE_DB_STORAGE" TABLE_DELTA = "TABLE_DELTA" diff --git a/databricks/sdk/service/disasterrecovery.py b/databricks/sdk/service/disasterrecovery.py index 34e894d7c..a0c9e1b12 100755 --- a/databricks/sdk/service/disasterrecovery.py +++ b/databricks/sdk/service/disasterrecovery.py @@ -689,7 +689,10 @@ def list_failover_groups( :param parent: str The parent resource. Format: accounts/{account_id}. :param page_size: int (optional) - Maximum number of failover groups to return per page. Default: 50, maximum: 100. + Maximum number of failover groups to return per page: - when set to a value greater than 0, the page + length is the minimum of this value and a server configured value; - when set to 0 or unset, the + page length is set to a server configured value (recommended); - when set to a value less than 0, an + invalid parameter error is returned. :param page_token: str (optional) Page token received from a previous ListFailoverGroups call. Provide this to retrieve the subsequent page. @@ -725,7 +728,10 @@ def list_stable_urls( :param parent: str The parent resource. Format: accounts/{account_id}. :param page_size: int (optional) - Maximum number of stable URLs to return per page. Default: 50, maximum: 100. + Maximum number of stable URLs to return per page: - when set to a value greater than 0, the page + length is the minimum of this value and a server configured value; - when set to 0 or unset, the + page length is set to a server configured value (recommended); - when set to a value less than 0, an + invalid parameter error is returned. :param page_token: str (optional) Page token received from a previous ListStableUrls call. Provide this to retrieve the subsequent page. diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 0880526d4..aa2258a41 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -3992,6 +3992,10 @@ class PipelineTask: full_refresh_selection: Optional[List[str]] = None """A list of tables to update with fullRefresh.""" + parameters: Optional[Dict[str, str]] = None + """Key/value-map of parameters passed to the pipeline execution. Limited to 10k characters in + total.""" + refresh_flow_selection: Optional[List[str]] = None """Flow names to selectively refresh. These are unioned with other selective refresh options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh.""" @@ -4009,6 +4013,8 @@ def as_dict(self) -> dict: body["full_refresh"] = self.full_refresh if self.full_refresh_selection: body["full_refresh_selection"] = [v for v in self.full_refresh_selection] + if self.parameters: + body["parameters"] = self.parameters if self.pipeline_id is not None: body["pipeline_id"] = self.pipeline_id if self.refresh_flow_selection: @@ -4026,6 +4032,8 @@ def as_shallow_dict(self) -> dict: body["full_refresh"] = self.full_refresh if self.full_refresh_selection: body["full_refresh_selection"] = self.full_refresh_selection + if self.parameters: + body["parameters"] = self.parameters if self.pipeline_id is not None: body["pipeline_id"] = self.pipeline_id if self.refresh_flow_selection: @@ -4042,6 +4050,7 @@ def from_dict(cls, d: Dict[str, Any]) -> PipelineTask: return cls( full_refresh=d.get("full_refresh", None), full_refresh_selection=d.get("full_refresh_selection", None), + parameters=d.get("parameters", None), pipeline_id=d.get("pipeline_id", None), refresh_flow_selection=d.get("refresh_flow_selection", None), refresh_selection=d.get("refresh_selection", None), @@ -4629,6 +4638,32 @@ def from_dict(cls, d: Dict[str, Any]) -> ResolvedParamPairValues: return cls(parameters=d.get("parameters", None)) +@dataclass +class ResolvedPipelineTaskValues: + parameters: Optional[Dict[str, str]] = None + """Key/value-map of parameters passed to the pipeline execution. Limited to 10k characters in + total.""" + + def as_dict(self) -> dict: + """Serializes the ResolvedPipelineTaskValues into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.parameters: + body["parameters"] = self.parameters + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ResolvedPipelineTaskValues into a shallow dictionary of its immediate attributes.""" + body = {} + if self.parameters: + body["parameters"] = self.parameters + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ResolvedPipelineTaskValues: + """Deserializes the ResolvedPipelineTaskValues from a dictionary.""" + return cls(parameters=d.get("parameters", None)) + + @dataclass class ResolvedPythonWheelTaskValues: named_parameters: Optional[Dict[str, str]] = None @@ -4721,6 +4756,8 @@ class ResolvedValues: notebook_task: Optional[ResolvedNotebookTaskValues] = None + pipeline_task: Optional[ResolvedPipelineTaskValues] = None + python_wheel_task: Optional[ResolvedPythonWheelTaskValues] = None run_job_task: Optional[ResolvedRunJobTaskValues] = None @@ -4744,6 +4781,8 @@ def as_dict(self) -> dict: body["dbt_task"] = self.dbt_task.as_dict() if self.notebook_task: body["notebook_task"] = self.notebook_task.as_dict() + if self.pipeline_task: + body["pipeline_task"] = self.pipeline_task.as_dict() if self.python_wheel_task: body["python_wheel_task"] = self.python_wheel_task.as_dict() if self.run_job_task: @@ -4769,6 +4808,8 @@ def as_shallow_dict(self) -> dict: body["dbt_task"] = self.dbt_task if self.notebook_task: body["notebook_task"] = self.notebook_task + if self.pipeline_task: + body["pipeline_task"] = self.pipeline_task if self.python_wheel_task: body["python_wheel_task"] = self.python_wheel_task if self.run_job_task: @@ -4792,6 +4833,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ResolvedValues: condition_task=_from_dict(d, "condition_task", ResolvedConditionTaskValues), dbt_task=_from_dict(d, "dbt_task", ResolvedDbtTaskValues), notebook_task=_from_dict(d, "notebook_task", ResolvedNotebookTaskValues), + pipeline_task=_from_dict(d, "pipeline_task", ResolvedPipelineTaskValues), python_wheel_task=_from_dict(d, "python_wheel_task", ResolvedPythonWheelTaskValues), run_job_task=_from_dict(d, "run_job_task", ResolvedRunJobTaskValues), simulation_task=_from_dict(d, "simulation_task", ResolvedParamPairValues), diff --git a/databricks/sdk/service/ml.py b/databricks/sdk/service/ml.py index bec83e273..fa24972f9 100755 --- a/databricks/sdk/service/ml.py +++ b/databricks/sdk/service/ml.py @@ -1497,6 +1497,83 @@ def from_dict(cls, d: Dict[str, Any]) -> DeltaTableSource: ) +@dataclass +class DirectMtlsConfig: + """Direct connection configs for mTLS, as Kafka Connections do not support mTLS yet (XTA-18030). + Temporarily used until UC Kafka Connections gain mTLS support.""" + + bootstrap_servers: str + """A comma-separated list of host:port pairs for the Kafka bootstrap servers.""" + + mtls_config: MtlsConfig + """Mutual-TLS authentication configuration.""" + + def as_dict(self) -> dict: + """Serializes the DirectMtlsConfig into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.bootstrap_servers is not None: + body["bootstrap_servers"] = self.bootstrap_servers + if self.mtls_config: + body["mtls_config"] = self.mtls_config.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the DirectMtlsConfig into a shallow dictionary of its immediate attributes.""" + body = {} + if self.bootstrap_servers is not None: + body["bootstrap_servers"] = self.bootstrap_servers + if self.mtls_config: + body["mtls_config"] = self.mtls_config + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> DirectMtlsConfig: + """Deserializes the DirectMtlsConfig from a dictionary.""" + return cls( + bootstrap_servers=d.get("bootstrap_servers", None), mtls_config=_from_dict(d, "mtls_config", MtlsConfig) + ) + + +@dataclass +class DirectSchemas: + """Schema definitions provided directly on the Stream, as opposed to referencing a schema registry. + In a future milestone, we will support schema registries through a UC Connection.""" + + key_schema: Optional[SchemaConfig] = None + """Schema for the message key. This is only used for Kafka streams. For Kafka, at least one of + payload_schema or key_schema must be specified.""" + + payload_schema: Optional[SchemaConfig] = None + """Schema for the message payload. For Kafka, this is the value schema. Unless the platform + supports another schema (e.g. keys for Kafka), this must be specified.""" + + def as_dict(self) -> dict: + """Serializes the DirectSchemas into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.key_schema: + body["key_schema"] = self.key_schema.as_dict() + if self.payload_schema: + body["payload_schema"] = self.payload_schema.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the DirectSchemas into a shallow dictionary of its immediate attributes.""" + body = {} + if self.key_schema: + body["key_schema"] = self.key_schema + if self.payload_schema: + body["payload_schema"] = self.payload_schema + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> DirectSchemas: + """Deserializes the DirectSchemas from a dictionary.""" + return cls( + key_schema=_from_dict(d, "key_schema", SchemaConfig), + payload_schema=_from_dict(d, "payload_schema", SchemaConfig), + ) + + @dataclass class EntityColumn: name: str @@ -2902,6 +2979,113 @@ def from_dict(cls, d: Dict[str, Any]) -> HttpUrlSpecWithoutSecret: return cls(enable_ssl_verification=d.get("enable_ssl_verification", None), url=d.get("url", None)) +@dataclass +class IngestionConfig: + """Configuration for the Databricks-managed ingestion pipeline. Groups the ingestion destination + (required) and optional backfill source.""" + + ingestion_destination: IngestionDestination + """Destination for the Databricks-managed Delta table that holds an offline copy of the streaming + data for querying and training. This table contains both 1) forward-filled data from the Stream + and 2) backfilled data from the BackfillSource (if provided). This table is created and managed + by Databricks and is deleted when the Stream is deleted.""" + + backfill_job_id: Optional[int] = None + """The ID of the Databricks Job that performs the historical backfill of the ingestion Delta table.""" + + backfill_source: Optional[BackfillSource] = None + """A user-provided source for backfilling data. Historical data is used when creating a training + set from streaming features linked to this Stream. The backfill data stored in this location + will be copied into the ingestion table for offline querying and training. The schema for this + source must match exactly that of the key and payload schemas specified for this Stream.""" + + deduplication_columns: Optional[List[str]] = None + """Column paths used to identify duplicate rows during ingestion; only one row per distinct + combination of these values is kept. Use dot notation for nested fields (e.g. `value.user_id`). + Empty list means every column is compared.""" + + ingestion_job_id: Optional[int] = None + """The ID of the Databricks Job that performs the forward-fill ingestion.""" + + ingestion_pipeline_id: Optional[str] = None + """The ID of the SDP pipeline that continuously copies new events from the streaming source into + the ingestion Delta table.""" + + def as_dict(self) -> dict: + """Serializes the IngestionConfig into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.backfill_job_id is not None: + body["backfill_job_id"] = self.backfill_job_id + if self.backfill_source: + body["backfill_source"] = self.backfill_source.as_dict() + if self.deduplication_columns: + body["deduplication_columns"] = [v for v in self.deduplication_columns] + if self.ingestion_destination: + body["ingestion_destination"] = self.ingestion_destination.as_dict() + if self.ingestion_job_id is not None: + body["ingestion_job_id"] = self.ingestion_job_id + if self.ingestion_pipeline_id is not None: + body["ingestion_pipeline_id"] = self.ingestion_pipeline_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the IngestionConfig into a shallow dictionary of its immediate attributes.""" + body = {} + if self.backfill_job_id is not None: + body["backfill_job_id"] = self.backfill_job_id + if self.backfill_source: + body["backfill_source"] = self.backfill_source + if self.deduplication_columns: + body["deduplication_columns"] = self.deduplication_columns + if self.ingestion_destination: + body["ingestion_destination"] = self.ingestion_destination + if self.ingestion_job_id is not None: + body["ingestion_job_id"] = self.ingestion_job_id + if self.ingestion_pipeline_id is not None: + body["ingestion_pipeline_id"] = self.ingestion_pipeline_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> IngestionConfig: + """Deserializes the IngestionConfig from a dictionary.""" + return cls( + backfill_job_id=d.get("backfill_job_id", None), + backfill_source=_from_dict(d, "backfill_source", BackfillSource), + deduplication_columns=d.get("deduplication_columns", None), + ingestion_destination=_from_dict(d, "ingestion_destination", IngestionDestination), + ingestion_job_id=d.get("ingestion_job_id", None), + ingestion_pipeline_id=d.get("ingestion_pipeline_id", None), + ) + + +@dataclass +class IngestionDestination: + """Destination for the Databricks-managed Delta table that holds an offline copy of the streaming + data for querying and training.""" + + delta_table_name: Optional[str] = None + """The full three-part name (catalog, schema, name) of the Delta table to be created for ingestion.""" + + def as_dict(self) -> dict: + """Serializes the IngestionDestination into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.delta_table_name is not None: + body["delta_table_name"] = self.delta_table_name + return body + + def as_shallow_dict(self) -> dict: + """Serializes the IngestionDestination into a shallow dictionary of its immediate attributes.""" + body = {} + if self.delta_table_name is not None: + body["delta_table_name"] = self.delta_table_name + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> IngestionDestination: + """Deserializes the IngestionDestination from a dictionary.""" + return cls(delta_table_name=d.get("delta_table_name", None)) + + @dataclass class InputTag: """Tag for a dataset input.""" @@ -3190,6 +3374,93 @@ def from_dict(cls, d: Dict[str, Any]) -> KafkaSource: ) +@dataclass +class KafkaStreamConfig: + """Kafka-specific configuration for a Stream.""" + + subscription_mode: KafkaSubscriptionMode + """Options to configure which Kafka topics to pull data from.""" + + extra_options: Optional[Dict[str, str]] = None + """Miscellaneous source options. Accepted keys are source options or Kafka consumer options + (kafka.*), validated against an allow-list at request time. All auth configuration goes through + the underlying UC Connection(s) or configs and should not be stored here.""" + + def as_dict(self) -> dict: + """Serializes the KafkaStreamConfig into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.extra_options: + body["extra_options"] = self.extra_options + if self.subscription_mode: + body["subscription_mode"] = self.subscription_mode.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the KafkaStreamConfig into a shallow dictionary of its immediate attributes.""" + body = {} + if self.extra_options: + body["extra_options"] = self.extra_options + if self.subscription_mode: + body["subscription_mode"] = self.subscription_mode + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> KafkaStreamConfig: + """Deserializes the KafkaStreamConfig from a dictionary.""" + return cls( + extra_options=d.get("extra_options", None), + subscription_mode=_from_dict(d, "subscription_mode", KafkaSubscriptionMode), + ) + + +@dataclass +class KafkaSubscriptionMode: + """Subscription mode for Kafka topic selection, matching standard Spark Structured Streaming + options.""" + + assign: Optional[str] = None + """A JSON string that contains the specific topic-partitions to consume from. For example, for + '{"topicA":[0,1],"topicB":[2,4]}', topicA's 0'th and 1st partitions will be consumed from.""" + + subscribe: Optional[str] = None + """A comma-separated list of Kafka topics to read from. For example, 'topicA,topicB,topicC'.""" + + subscribe_pattern: Optional[str] = None + """A regular expression matching topics to subscribe to. For example, 'topic.*' will subscribe to + all topics starting with 'topic'.""" + + def as_dict(self) -> dict: + """Serializes the KafkaSubscriptionMode into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.assign is not None: + body["assign"] = self.assign + if self.subscribe is not None: + body["subscribe"] = self.subscribe + if self.subscribe_pattern is not None: + body["subscribe_pattern"] = self.subscribe_pattern + return body + + def as_shallow_dict(self) -> dict: + """Serializes the KafkaSubscriptionMode into a shallow dictionary of its immediate attributes.""" + body = {} + if self.assign is not None: + body["assign"] = self.assign + if self.subscribe is not None: + body["subscribe"] = self.subscribe + if self.subscribe_pattern is not None: + body["subscribe_pattern"] = self.subscribe_pattern + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> KafkaSubscriptionMode: + """Deserializes the KafkaSubscriptionMode from a dictionary.""" + return cls( + assign=d.get("assign", None), + subscribe=d.get("subscribe", None), + subscribe_pattern=d.get("subscribe_pattern", None), + ) + + @dataclass class LastFunction: """Returns the last value.""" @@ -3616,6 +3887,40 @@ def from_dict(cls, d: Dict[str, Any]) -> ListRegistryWebhooks: ) +@dataclass +class ListStreamsResponse: + """Response to a ListStreamsRequest.""" + + next_page_token: Optional[str] = None + """Pagination token to request the next page of results for this query.""" + + streams: Optional[List[Stream]] = None + """List of Streams.""" + + def as_dict(self) -> dict: + """Serializes the ListStreamsResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + if self.streams: + body["streams"] = [v.as_dict() for v in self.streams] + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ListStreamsResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + if self.streams: + body["streams"] = self.streams + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ListStreamsResponse: + """Deserializes the ListStreamsResponse from a dictionary.""" + return cls(next_page_token=d.get("next_page_token", None), streams=_repeated_dict(d, "streams", Stream)) + + @dataclass class ListTransitionRequestsResponse: requests: Optional[List[Activity]] = None @@ -6660,6 +6965,214 @@ def from_dict(cls, d: Dict[str, Any]) -> StddevSampFunction: return cls(input=d.get("input", None)) +@dataclass +class Stream: + """A Stream is a governed UC entity representing an external streaming data source. The + source_config oneof determines the streaming platform source (e.g. Kafka, Kinesis, etc.).""" + + name: str + """Full three-part (catalog.schema.stream) name of the stream.""" + + source_config: StreamSourceConfig + """Source-specific configuration. Determines the streaming platform source.""" + + connection_config: StreamConnectionConfig + """Specifies how to connect and authenticate to the stream platform.""" + + schema_config: StreamSchemaConfig + """Schema definitions for the stream. Currently only direct schemas are supported. In a future + milestone, we will support schema registries through a UC Connection.""" + + ingestion_config: IngestionConfig + """Configuration for streaming data ingestion: the managed table storing an offline copy of forward + fill data and optional historical backfill.""" + + browse_only: Optional[bool] = None + """Indicates whether the principal is limited to retrieving metadata for the associated object + through the BROWSE privilege when include_browse is enabled in the request.""" + + create_time: Optional[Timestamp] = None + """Time at which this Stream was created.""" + + created_by: Optional[str] = None + """Username of the Stream creator.""" + + description: Optional[str] = None + """User-provided description.""" + + update_time: Optional[Timestamp] = None + """Time at which this Stream was last modified.""" + + updated_by: Optional[str] = None + """Username of user who last modified the Stream.""" + + def as_dict(self) -> dict: + """Serializes the Stream into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.browse_only is not None: + body["browse_only"] = self.browse_only + if self.connection_config: + body["connection_config"] = self.connection_config.as_dict() + if self.create_time is not None: + body["create_time"] = self.create_time.ToJsonString() + if self.created_by is not None: + body["created_by"] = self.created_by + if self.description is not None: + body["description"] = self.description + if self.ingestion_config: + body["ingestion_config"] = self.ingestion_config.as_dict() + if self.name is not None: + body["name"] = self.name + if self.schema_config: + body["schema_config"] = self.schema_config.as_dict() + if self.source_config: + body["source_config"] = self.source_config.as_dict() + if self.update_time is not None: + body["update_time"] = self.update_time.ToJsonString() + if self.updated_by is not None: + body["updated_by"] = self.updated_by + return body + + def as_shallow_dict(self) -> dict: + """Serializes the Stream into a shallow dictionary of its immediate attributes.""" + body = {} + if self.browse_only is not None: + body["browse_only"] = self.browse_only + if self.connection_config: + body["connection_config"] = self.connection_config + if self.create_time is not None: + body["create_time"] = self.create_time + if self.created_by is not None: + body["created_by"] = self.created_by + if self.description is not None: + body["description"] = self.description + if self.ingestion_config: + body["ingestion_config"] = self.ingestion_config + if self.name is not None: + body["name"] = self.name + if self.schema_config: + body["schema_config"] = self.schema_config + if self.source_config: + body["source_config"] = self.source_config + if self.update_time is not None: + body["update_time"] = self.update_time + if self.updated_by is not None: + body["updated_by"] = self.updated_by + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Stream: + """Deserializes the Stream from a dictionary.""" + return cls( + browse_only=d.get("browse_only", None), + connection_config=_from_dict(d, "connection_config", StreamConnectionConfig), + create_time=_timestamp(d, "create_time"), + created_by=d.get("created_by", None), + description=d.get("description", None), + ingestion_config=_from_dict(d, "ingestion_config", IngestionConfig), + name=d.get("name", None), + schema_config=_from_dict(d, "schema_config", StreamSchemaConfig), + source_config=_from_dict(d, "source_config", StreamSourceConfig), + update_time=_timestamp(d, "update_time"), + updated_by=d.get("updated_by", None), + ) + + +@dataclass +class StreamConnectionConfig: + """Specifies how to connect and authenticate to the stream platform.""" + + direct_mtls_config: Optional[DirectMtlsConfig] = None + """Direct mTLS configuration for stream platform access. This is only used in the short term until + UC Kafka Connections support mTLS (XTA-18030). Once UC Kafka Connections support mTLS, this will + be deprecated.""" + + uc_connection_name: Optional[str] = None + """Name of an existing UC Connection for stream platform access. Must be the correct type for the + streaming platform (e.g. a Kafka Connection for a Kafka Stream).""" + + def as_dict(self) -> dict: + """Serializes the StreamConnectionConfig into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.direct_mtls_config: + body["direct_mtls_config"] = self.direct_mtls_config.as_dict() + if self.uc_connection_name is not None: + body["uc_connection_name"] = self.uc_connection_name + return body + + def as_shallow_dict(self) -> dict: + """Serializes the StreamConnectionConfig into a shallow dictionary of its immediate attributes.""" + body = {} + if self.direct_mtls_config: + body["direct_mtls_config"] = self.direct_mtls_config + if self.uc_connection_name is not None: + body["uc_connection_name"] = self.uc_connection_name + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> StreamConnectionConfig: + """Deserializes the StreamConnectionConfig from a dictionary.""" + return cls( + direct_mtls_config=_from_dict(d, "direct_mtls_config", DirectMtlsConfig), + uc_connection_name=d.get("uc_connection_name", None), + ) + + +@dataclass +class StreamSchemaConfig: + """Schema definitions for the stream. Currently only direct schemas are supported. In a future + milestone, we will support schema registries through a UC Connection.""" + + direct_schemas: Optional[DirectSchemas] = None + """Schema definitions provided directly on the Stream.""" + + def as_dict(self) -> dict: + """Serializes the StreamSchemaConfig into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.direct_schemas: + body["direct_schemas"] = self.direct_schemas.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the StreamSchemaConfig into a shallow dictionary of its immediate attributes.""" + body = {} + if self.direct_schemas: + body["direct_schemas"] = self.direct_schemas + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> StreamSchemaConfig: + """Deserializes the StreamSchemaConfig from a dictionary.""" + return cls(direct_schemas=_from_dict(d, "direct_schemas", DirectSchemas)) + + +@dataclass +class StreamSourceConfig: + """Source-specific configuration. Determines the streaming platform source.""" + + kafka_stream_config: Optional[KafkaStreamConfig] = None + """Configuration for Apache Kafka streams.""" + + def as_dict(self) -> dict: + """Serializes the StreamSourceConfig into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.kafka_stream_config: + body["kafka_stream_config"] = self.kafka_stream_config.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the StreamSourceConfig into a shallow dictionary of its immediate attributes.""" + body = {} + if self.kafka_stream_config: + body["kafka_stream_config"] = self.kafka_stream_config + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> StreamSourceConfig: + """Deserializes the StreamSourceConfig from a dictionary.""" + return cls(kafka_stream_config=_from_dict(d, "kafka_stream_config", KafkaStreamConfig)) + + @dataclass class StreamingMode: """The streaming mode configuration for a streaming materialization pipeline.""" @@ -8787,6 +9300,29 @@ def create_materialized_feature(self, materialized_feature: MaterializedFeature) res = self._api.do("POST", "/api/2.0/feature-engineering/materialized-features", body=body, headers=headers) return MaterializedFeature.from_dict(res) + def create_stream(self, stream: Stream) -> Stream: + """Create a Stream, a governed UC entity representing an external streaming data source. + + :param stream: :class:`Stream` + The Stream to create. + + :returns: :class:`Stream` + """ + + body = stream.as_dict() + query = {} + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do("POST", "/api/2.0/feature-engineering/streams", body=body, headers=headers) + return Stream.from_dict(res) + def delete_feature(self, full_name: str): """Delete a Feature. @@ -8847,6 +9383,25 @@ def delete_materialized_feature(self, materialized_feature_id: str): "DELETE", f"/api/2.0/feature-engineering/materialized-features/{materialized_feature_id}", headers=headers ) + def delete_stream(self, name: str): + """Delete a Stream by its full three-part name (catalog.schema.stream). + + :param name: str + Full three-part name (catalog.schema.stream) of the Stream to delete. + + + """ + + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + self._api.do("DELETE", f"/api/2.0/feature-engineering/streams/{name}", headers=headers) + def get_feature(self, full_name: str) -> Feature: """Get a Feature. @@ -8910,6 +9465,26 @@ def get_materialized_feature(self, materialized_feature_id: str) -> Materialized ) return MaterializedFeature.from_dict(res) + def get_stream(self, name: str) -> Stream: + """Get a Stream by its full three-part name (catalog.schema.stream). + + :param name: str + Full three-part name (catalog.schema.stream) of the Stream to get. + + :returns: :class:`Stream` + """ + + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do("GET", f"/api/2.0/feature-engineering/streams/{name}", headers=headers) + return Stream.from_dict(res) + def list_features( self, catalog_name: str, schema_name: str, *, page_size: Optional[int] = None, page_token: Optional[str] = None ) -> Iterator[Feature]: @@ -9034,6 +9609,45 @@ def list_materialized_features( return query["page_token"] = json["next_page_token"] + def list_streams( + self, *, page_size: Optional[int] = None, page_token: Optional[str] = None, parent: Optional[str] = None + ) -> Iterator[Stream]: + """List Streams under a given catalog.schema parent. + + :param page_size: int (optional) + The maximum number of results to return. + :param page_token: str (optional) + Pagination token to go to the next page based on a previous query. + :param parent: str (optional) + Two-part name (catalog.schema) of the parent under which to list Streams. + + :returns: Iterator over :class:`Stream` + """ + + query = {} + if page_size is not None: + query["page_size"] = page_size + if page_token is not None: + query["page_token"] = page_token + if parent is not None: + query["parent"] = parent + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + while True: + json = self._api.do("GET", "/api/2.0/feature-engineering/streams", query=query, headers=headers) + if "streams" in json: + for v in json["streams"]: + yield Stream.from_dict(v) + if "next_page_token" not in json or not json["next_page_token"]: + return + query["page_token"] = json["next_page_token"] + def update_feature(self, full_name: str, feature: Feature, update_mask: str) -> Feature: """Update a Feature. @@ -9143,6 +9757,37 @@ def update_materialized_feature( ) return MaterializedFeature.from_dict(res) + def update_stream(self, name: str, stream: Stream, update_mask: FieldMask) -> Stream: + """Update a Stream. Only fields listed in `update_mask` are mutated. + + :param name: str + Full three-part (catalog.schema.stream) name of the stream. + :param stream: :class:`Stream` + The Stream to update. + :param update_mask: FieldMask + The list of fields to update. + + :returns: :class:`Stream` + """ + + body = stream.as_dict() + query = {} + if update_mask is not None: + query["update_mask"] = update_mask.ToJsonString() + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do( + "PATCH", f"/api/2.0/feature-engineering/streams/{name}", query=query, body=body, headers=headers + ) + return Stream.from_dict(res) + class FeatureStoreAPI: """A feature store is a centralized repository that enables data scientists to find and share features. Using diff --git a/databricks/sdk/service/pipelines.py b/databricks/sdk/service/pipelines.py index 3281820bd..055a4e876 100755 --- a/databricks/sdk/service/pipelines.py +++ b/databricks/sdk/service/pipelines.py @@ -841,6 +841,10 @@ class GetPipelineResponse: name: Optional[str] = None """A human friendly identifier for the pipeline, taken from the `spec`.""" + parameters: Optional[Dict[str, str]] = None + """Key/value map of default parameters to use for pipeline execution. Maximum total size: 10k + characters (JSON format)""" + pipeline_id: Optional[str] = None """The ID of the pipeline.""" @@ -879,6 +883,8 @@ def as_dict(self) -> dict: body["latest_updates"] = [v.as_dict() for v in self.latest_updates] if self.name is not None: body["name"] = self.name + if self.parameters: + body["parameters"] = self.parameters if self.pipeline_id is not None: body["pipeline_id"] = self.pipeline_id if self.run_as: @@ -912,6 +918,8 @@ def as_shallow_dict(self) -> dict: body["latest_updates"] = self.latest_updates if self.name is not None: body["name"] = self.name + if self.parameters: + body["parameters"] = self.parameters if self.pipeline_id is not None: body["pipeline_id"] = self.pipeline_id if self.run_as: @@ -937,6 +945,7 @@ def from_dict(cls, d: Dict[str, Any]) -> GetPipelineResponse: last_modified=d.get("last_modified", None), latest_updates=_repeated_dict(d, "latest_updates", UpdateStateInfo), name=d.get("name", None), + parameters=d.get("parameters", None), pipeline_id=d.get("pipeline_id", None), run_as=_from_dict(d, "run_as", RunAs), run_as_user_name=d.get("run_as_user_name", None), @@ -5184,6 +5193,7 @@ def create( libraries: Optional[List[PipelineLibrary]] = None, name: Optional[str] = None, notifications: Optional[List[Notifications]] = None, + parameters: Optional[Dict[str, str]] = None, photon: Optional[bool] = None, restart_window: Optional[RestartWindow] = None, root_path: Optional[str] = None, @@ -5241,6 +5251,9 @@ def create( Friendly identifier for this pipeline. :param notifications: List[:class:`Notifications`] (optional) List of notification settings for this pipeline. + :param parameters: Dict[str,str] (optional) + Key/value map of default parameters to use for pipeline execution. Maximum total size: 10k + characters (JSON format) :param photon: bool (optional) Whether Photon is enabled for this pipeline. :param restart_window: :class:`RestartWindow` (optional) @@ -5312,6 +5325,8 @@ def create( body["name"] = name if notifications is not None: body["notifications"] = [v.as_dict() for v in notifications] + if parameters is not None: + body["parameters"] = parameters if photon is not None: body["photon"] = photon if restart_window is not None: @@ -5771,6 +5786,7 @@ def update( libraries: Optional[List[PipelineLibrary]] = None, name: Optional[str] = None, notifications: Optional[List[Notifications]] = None, + parameters: Optional[Dict[str, str]] = None, photon: Optional[bool] = None, restart_window: Optional[RestartWindow] = None, root_path: Optional[str] = None, @@ -5831,6 +5847,9 @@ def update( Friendly identifier for this pipeline. :param notifications: List[:class:`Notifications`] (optional) List of notification settings for this pipeline. + :param parameters: Dict[str,str] (optional) + Key/value map of default parameters to use for pipeline execution. Maximum total size: 10k + characters (JSON format) :param photon: bool (optional) Whether Photon is enabled for this pipeline. :param restart_window: :class:`RestartWindow` (optional) @@ -5902,6 +5921,8 @@ def update( body["name"] = name if notifications is not None: body["notifications"] = [v.as_dict() for v in notifications] + if parameters is not None: + body["parameters"] = parameters if photon is not None: body["photon"] = photon if restart_window is not None: diff --git a/databricks/sdk/service/postgres.py b/databricks/sdk/service/postgres.py index 579f56af4..b98ce575d 100755 --- a/databricks/sdk/service/postgres.py +++ b/databricks/sdk/service/postgres.py @@ -487,9 +487,6 @@ class CatalogCatalogStatus: Format: projects/{project_id}/branches/{branch_id}.""" - catalog_id: Optional[str] = None - """Part of the resource name.""" - postgres_database: Optional[str] = None """The name of the Postgres database associated with the catalog.""" @@ -503,8 +500,6 @@ def as_dict(self) -> dict: body = {} if self.branch is not None: body["branch"] = self.branch - if self.catalog_id is not None: - body["catalog_id"] = self.catalog_id if self.postgres_database is not None: body["postgres_database"] = self.postgres_database if self.project is not None: @@ -516,8 +511,6 @@ def as_shallow_dict(self) -> dict: body = {} if self.branch is not None: body["branch"] = self.branch - if self.catalog_id is not None: - body["catalog_id"] = self.catalog_id if self.postgres_database is not None: body["postgres_database"] = self.postgres_database if self.project is not None: @@ -529,7 +522,6 @@ def from_dict(cls, d: Dict[str, Any]) -> CatalogCatalogStatus: """Deserializes the CatalogCatalogStatus from a dictionary.""" return cls( branch=d.get("branch", None), - catalog_id=d.get("catalog_id", None), postgres_database=d.get("postgres_database", None), project=d.get("project", None), ) @@ -2866,9 +2858,6 @@ class SyncedTableSyncedTableStatus: provisioning_phase: Optional[ProvisioningPhase] = None """The current phase of the data synchronization pipeline.""" - synced_table_id: Optional[str] = None - """Part of the resource name.""" - unity_catalog_provisioning_state: Optional[ProvisioningInfoState] = None """The provisioning state of the synced table entity in Unity Catalog.""" @@ -2893,8 +2882,6 @@ def as_dict(self) -> dict: body["project"] = self.project if self.provisioning_phase is not None: body["provisioning_phase"] = self.provisioning_phase.value - if self.synced_table_id is not None: - body["synced_table_id"] = self.synced_table_id if self.unity_catalog_provisioning_state is not None: body["unity_catalog_provisioning_state"] = self.unity_catalog_provisioning_state.value return body @@ -2920,8 +2907,6 @@ def as_shallow_dict(self) -> dict: body["project"] = self.project if self.provisioning_phase is not None: body["provisioning_phase"] = self.provisioning_phase - if self.synced_table_id is not None: - body["synced_table_id"] = self.synced_table_id if self.unity_catalog_provisioning_state is not None: body["unity_catalog_provisioning_state"] = self.unity_catalog_provisioning_state return body @@ -2939,7 +2924,6 @@ def from_dict(cls, d: Dict[str, Any]) -> SyncedTableSyncedTableStatus: pipeline_id=d.get("pipeline_id", None), project=d.get("project", None), provisioning_phase=_enum(d, "provisioning_phase", ProvisioningPhase), - synced_table_id=d.get("synced_table_id", None), unity_catalog_provisioning_state=_enum(d, "unity_catalog_provisioning_state", ProvisioningInfoState), ) diff --git a/databricks/sdk/service/sql.py b/databricks/sdk/service/sql.py index 7ddfa6316..aa910ad7c 100755 --- a/databricks/sdk/service/sql.py +++ b/databricks/sdk/service/sql.py @@ -10461,6 +10461,7 @@ def start(self, id: str) -> Wait[GetWarehouseResponse]: headers = { "Accept": "application/json", + "Content-Type": "application/json", } cfg = self._api._cfg @@ -10486,6 +10487,7 @@ def stop(self, id: str) -> Wait[GetWarehouseResponse]: headers = { "Accept": "application/json", + "Content-Type": "application/json", } cfg = self._api._cfg diff --git a/databricks/sdk/service/supervisoragents.py b/databricks/sdk/service/supervisoragents.py index 39f081dd4..708b72183 100755 --- a/databricks/sdk/service/supervisoragents.py +++ b/databricks/sdk/service/supervisoragents.py @@ -619,9 +619,10 @@ def from_dict(cls, d: Dict[str, Any]) -> SupervisorAgentPermissionsDescription: class Tool: tool_type: str """Tool type. Must be one of: "genie_space", "knowledge_assistant", "uc_function", "uc_connection", - "app", "volume", "dashboard", "serving_endpoint", "table", "vector_search_index", "catalog", - "schema", "supervisor_agent", "web_search". The legacy values "lakeview_dashboard" and - "uc_table" are also accepted and remain equivalent to "dashboard" and "table" respectively.""" + "uc_mcp", "app", "volume", "dashboard", "serving_endpoint", "table", "vector_search_index", + "catalog", "schema", "supervisor_agent", "web_search", "skill". The legacy values + "lakeview_dashboard" and "uc_table" are also accepted and remain equivalent to "dashboard" and + "table" respectively.""" app: Optional[App] = None @@ -852,7 +853,7 @@ def create_supervisor_agent(self, supervisor_agent: SupervisorAgent) -> Supervis def create_tool(self, parent: str, tool: Tool, tool_id: str) -> Tool: """Creates a Tool under a Supervisor Agent. Specify one of "genie_space", "knowledge_assistant", "uc_function", "uc_connection", "app", "volume", "dashboard", "table", "vector_search_index", - "catalog", "schema", "supervisor_agent", "web_search" in the request body. The legacy values + "catalog", "schema", "supervisor_agent", "web_search", "skill" in the request body. The legacy values "lakeview_dashboard" and "uc_table" are also accepted and remain equivalent to "dashboard" and "table" respectively. diff --git a/databricks/sdk/service/vectorsearch.py b/databricks/sdk/service/vectorsearch.py index 113af521c..3a8d82c3c 100755 --- a/databricks/sdk/service/vectorsearch.py +++ b/databricks/sdk/service/vectorsearch.py @@ -1174,8 +1174,11 @@ def from_dict(cls, d: Dict[str, Any]) -> QueryVectorIndexResponse: @dataclass class RerankerConfig: model: Optional[str] = None + """Reranker identifier: - When model_type=BASE/UNSPECIFIED: must be "databricks_reranker". - When + model_type=FINETUNED: the Model Serving endpoint name hosting a finetuned reranker.""" parameters: Optional[RerankerConfigRerankerParameters] = None + """Parameters that control how the reranker processes the query results.""" def as_dict(self) -> dict: """Serializes the RerankerConfig into a dictionary suitable for use as a JSON request body.""" diff --git a/docs/account/disasterrecovery/disaster_recovery.rst b/docs/account/disasterrecovery/disaster_recovery.rst index 517726ff1..9b91b3825 100755 --- a/docs/account/disasterrecovery/disaster_recovery.rst +++ b/docs/account/disasterrecovery/disaster_recovery.rst @@ -111,7 +111,10 @@ :param parent: str The parent resource. Format: accounts/{account_id}. :param page_size: int (optional) - Maximum number of failover groups to return per page. Default: 50, maximum: 100. + Maximum number of failover groups to return per page: - when set to a value greater than 0, the page + length is the minimum of this value and a server configured value; - when set to 0 or unset, the + page length is set to a server configured value (recommended); - when set to a value less than 0, an + invalid parameter error is returned. :param page_token: str (optional) Page token received from a previous ListFailoverGroups call. Provide this to retrieve the subsequent page. @@ -126,7 +129,10 @@ :param parent: str The parent resource. Format: accounts/{account_id}. :param page_size: int (optional) - Maximum number of stable URLs to return per page. Default: 50, maximum: 100. + Maximum number of stable URLs to return per page: - when set to a value greater than 0, the page + length is the minimum of this value and a server configured value; - when set to 0 or unset, the + page length is set to a server configured value (recommended); - when set to a value less than 0, an + invalid parameter error is returned. :param page_token: str (optional) Page token received from a previous ListStableUrls call. Provide this to retrieve the subsequent page. diff --git a/docs/account/oauth2/service_principal_secrets.rst b/docs/account/oauth2/service_principal_secrets.rst old mode 100644 new mode 100755 index e95b779f0..f8e0b247f --- a/docs/account/oauth2/service_principal_secrets.rst +++ b/docs/account/oauth2/service_principal_secrets.rst @@ -15,7 +15,6 @@ [Authentication using OAuth tokens for service principals]: https://docs.databricks.com/dev-tools/authentication-oauth.html [Databricks Terraform Provider]: https://github.com/databricks/terraform-provider-databricks/blob/master/docs/index.md#authenticating-with-service-principal - .. py:method:: create(service_principal_id: str [, lifetime: Optional[str]]) -> CreateServicePrincipalSecretResponse diff --git a/docs/account/settings/network_connectivity.rst b/docs/account/settings/network_connectivity.rst index 5e1db5aed..9cc92a8d0 100755 --- a/docs/account/settings/network_connectivity.rst +++ b/docs/account/settings/network_connectivity.rst @@ -11,7 +11,6 @@ Azure Private Link. See [configure serverless secure connectivity]. [configure serverless secure connectivity]: https://learn.microsoft.com/azure/databricks/security/network/serverless-network-security - .. py:method:: create_network_connectivity_configuration(network_connectivity_config: CreateNetworkConnectivityConfiguration) -> NetworkConnectivityConfiguration diff --git a/docs/dbdataclasses/catalog.rst b/docs/dbdataclasses/catalog.rst index 48a97e428..d8baff997 100755 --- a/docs/dbdataclasses/catalog.rst +++ b/docs/dbdataclasses/catalog.rst @@ -1548,7 +1548,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: SecurableKind - Latest kind: CONNECTION_ICEBERG_REST_OAUTH_M2M = 336; Next id: 337 + Latest kind: CONNECTION_GOOGLE_CLOUD_LAKEHOUSE_SERVICE_ACCOUNT = 340; Next id: 341 .. py:attribute:: TABLE_DB_STORAGE :value: "TABLE_DB_STORAGE" diff --git a/docs/dbdataclasses/jobs.rst b/docs/dbdataclasses/jobs.rst index 1245ea0b6..5fc8275a9 100755 --- a/docs/dbdataclasses/jobs.rst +++ b/docs/dbdataclasses/jobs.rst @@ -666,6 +666,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ResolvedPipelineTaskValues + :members: + :undoc-members: + .. autoclass:: ResolvedPythonWheelTaskValues :members: :undoc-members: diff --git a/docs/dbdataclasses/ml.rst b/docs/dbdataclasses/ml.rst index f606e874b..ffb95b157 100755 --- a/docs/dbdataclasses/ml.rst +++ b/docs/dbdataclasses/ml.rst @@ -250,6 +250,14 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: DirectMtlsConfig + :members: + :undoc-members: + +.. autoclass:: DirectSchemas + :members: + :undoc-members: + .. autoclass:: EntityColumn :members: :undoc-members: @@ -467,6 +475,14 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: IngestionConfig + :members: + :undoc-members: + +.. autoclass:: IngestionDestination + :members: + :undoc-members: + .. autoclass:: InputTag :members: :undoc-members: @@ -491,6 +507,14 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: KafkaStreamConfig + :members: + :undoc-members: + +.. autoclass:: KafkaSubscriptionMode + :members: + :undoc-members: + .. autoclass:: LastFunction :members: :undoc-members: @@ -539,6 +563,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ListStreamsResponse + :members: + :undoc-members: + .. autoclass:: ListTransitionRequestsResponse :members: :undoc-members: @@ -1058,6 +1086,22 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: Stream + :members: + :undoc-members: + +.. autoclass:: StreamConnectionConfig + :members: + :undoc-members: + +.. autoclass:: StreamSchemaConfig + :members: + :undoc-members: + +.. autoclass:: StreamSourceConfig + :members: + :undoc-members: + .. autoclass:: StreamingMode :members: :undoc-members: diff --git a/docs/workspace/catalog/resource_quotas.rst b/docs/workspace/catalog/resource_quotas.rst old mode 100644 new mode 100755 index ae293fc11..14a4fca4a --- a/docs/workspace/catalog/resource_quotas.rst +++ b/docs/workspace/catalog/resource_quotas.rst @@ -10,7 +10,6 @@ limits. For more information on resource quotas see the [Unity Catalog documentation]. [Unity Catalog documentation]: https://docs.databricks.com/en/data-governance/unity-catalog/index.html#resource-quotas - .. py:method:: get_quota(parent_securable_type: str, parent_full_name: str, quota_name: str) -> GetQuotaResponse diff --git a/docs/workspace/compute/instance_profiles.rst b/docs/workspace/compute/instance_profiles.rst old mode 100644 new mode 100755 index 7a5192bd5..5a874108c --- a/docs/workspace/compute/instance_profiles.rst +++ b/docs/workspace/compute/instance_profiles.rst @@ -9,7 +9,6 @@ buckets] using instance profiles for more information. [Secure access to S3 buckets]: https://docs.databricks.com/administration-guide/cloud-configurations/aws/instance-profiles.html - .. py:method:: add(instance_profile_arn: str [, iam_role_arn: Optional[str], is_meta_instance_profile: Optional[bool], skip_validation: Optional[bool]]) diff --git a/docs/workspace/ml/feature_engineering.rst b/docs/workspace/ml/feature_engineering.rst index 1d4de90c1..1e85ea9a8 100755 --- a/docs/workspace/ml/feature_engineering.rst +++ b/docs/workspace/ml/feature_engineering.rst @@ -46,6 +46,16 @@ :returns: :class:`MaterializedFeature` + .. py:method:: create_stream(stream: Stream) -> Stream + + Create a Stream, a governed UC entity representing an external streaming data source. + + :param stream: :class:`Stream` + The Stream to create. + + :returns: :class:`Stream` + + .. py:method:: delete_feature(full_name: str) Delete a Feature. @@ -77,6 +87,16 @@ + .. py:method:: delete_stream(name: str) + + Delete a Stream by its full three-part name (catalog.schema.stream). + + :param name: str + Full three-part name (catalog.schema.stream) of the Stream to delete. + + + + .. py:method:: get_feature(full_name: str) -> Feature Get a Feature. @@ -108,6 +128,16 @@ :returns: :class:`MaterializedFeature` + .. py:method:: get_stream(name: str) -> Stream + + Get a Stream by its full three-part name (catalog.schema.stream). + + :param name: str + Full three-part name (catalog.schema.stream) of the Stream to get. + + :returns: :class:`Stream` + + .. py:method:: list_features(catalog_name: str, schema_name: str [, page_size: Optional[int], page_token: Optional[str]]) -> Iterator[Feature] List Features. @@ -153,6 +183,20 @@ :returns: Iterator over :class:`MaterializedFeature` + .. py:method:: list_streams( [, page_size: Optional[int], page_token: Optional[str], parent: Optional[str]]) -> Iterator[Stream] + + List Streams under a given catalog.schema parent. + + :param page_size: int (optional) + The maximum number of results to return. + :param page_token: str (optional) + Pagination token to go to the next page based on a previous query. + :param parent: str (optional) + Two-part name (catalog.schema) of the parent under which to list Streams. + + :returns: Iterator over :class:`Stream` + + .. py:method:: update_feature(full_name: str, feature: Feature, update_mask: str) -> Feature Update a Feature. @@ -199,4 +243,18 @@ pipeline_state field can be updated. :returns: :class:`MaterializedFeature` + + + .. py:method:: update_stream(name: str, stream: Stream, update_mask: FieldMask) -> Stream + + Update a Stream. Only fields listed in `update_mask` are mutated. + + :param name: str + Full three-part (catalog.schema.stream) name of the stream. + :param stream: :class:`Stream` + The Stream to update. + :param update_mask: FieldMask + The list of fields to update. + + :returns: :class:`Stream` \ No newline at end of file diff --git a/docs/workspace/oauth2/service_principal_secrets_proxy.rst b/docs/workspace/oauth2/service_principal_secrets_proxy.rst old mode 100644 new mode 100755 index 929c8fa72..33c48eef8 --- a/docs/workspace/oauth2/service_principal_secrets_proxy.rst +++ b/docs/workspace/oauth2/service_principal_secrets_proxy.rst @@ -16,7 +16,6 @@ [Authentication using OAuth tokens for service principals]: https://docs.databricks.com/dev-tools/authentication-oauth.html [Databricks Terraform Provider]: https://github.com/databricks/terraform-provider-databricks/blob/master/docs/index.md#authenticating-with-service-principal - .. py:method:: create(service_principal_id: str [, lifetime: Optional[str]]) -> CreateServicePrincipalSecretResponse diff --git a/docs/workspace/pipelines/pipelines.rst b/docs/workspace/pipelines/pipelines.rst index b4c1fc8b2..70c8e007c 100755 --- a/docs/workspace/pipelines/pipelines.rst +++ b/docs/workspace/pipelines/pipelines.rst @@ -109,7 +109,7 @@ :returns: :class:`ClonePipelineResponse` - .. py:method:: create( [, allow_duplicate_names: Optional[bool], budget_policy_id: Optional[str], catalog: Optional[str], channel: Optional[str], clusters: Optional[List[PipelineCluster]], configuration: Optional[Dict[str, str]], continuous: Optional[bool], deployment: Optional[PipelineDeployment], development: Optional[bool], dry_run: Optional[bool], edition: Optional[str], environment: Optional[PipelinesEnvironment], event_log: Optional[EventLogSpec], filters: Optional[Filters], gateway_definition: Optional[IngestionGatewayPipelineDefinition], id: Optional[str], ingestion_definition: Optional[IngestionPipelineDefinition], libraries: Optional[List[PipelineLibrary]], name: Optional[str], notifications: Optional[List[Notifications]], photon: Optional[bool], restart_window: Optional[RestartWindow], root_path: Optional[str], run_as: Optional[RunAs], schema: Optional[str], serverless: Optional[bool], storage: Optional[str], tags: Optional[Dict[str, str]], target: Optional[str], trigger: Optional[PipelineTrigger], usage_policy_id: Optional[str]]) -> CreatePipelineResponse + .. py:method:: create( [, allow_duplicate_names: Optional[bool], budget_policy_id: Optional[str], catalog: Optional[str], channel: Optional[str], clusters: Optional[List[PipelineCluster]], configuration: Optional[Dict[str, str]], continuous: Optional[bool], deployment: Optional[PipelineDeployment], development: Optional[bool], dry_run: Optional[bool], edition: Optional[str], environment: Optional[PipelinesEnvironment], event_log: Optional[EventLogSpec], filters: Optional[Filters], gateway_definition: Optional[IngestionGatewayPipelineDefinition], id: Optional[str], ingestion_definition: Optional[IngestionPipelineDefinition], libraries: Optional[List[PipelineLibrary]], name: Optional[str], notifications: Optional[List[Notifications]], parameters: Optional[Dict[str, str]], photon: Optional[bool], restart_window: Optional[RestartWindow], root_path: Optional[str], run_as: Optional[RunAs], schema: Optional[str], serverless: Optional[bool], storage: Optional[str], tags: Optional[Dict[str, str]], target: Optional[str], trigger: Optional[PipelineTrigger], usage_policy_id: Optional[str]]) -> CreatePipelineResponse Usage: @@ -190,6 +190,9 @@ Friendly identifier for this pipeline. :param notifications: List[:class:`Notifications`] (optional) List of notification settings for this pipeline. + :param parameters: Dict[str,str] (optional) + Key/value map of default parameters to use for pipeline execution. Maximum total size: 10k + characters (JSON format) :param photon: bool (optional) Whether Photon is enabled for this pipeline. :param restart_window: :class:`RestartWindow` (optional) @@ -493,7 +496,7 @@ .. py:method:: stop_and_wait(pipeline_id: str, timeout: datetime.timedelta = 0:20:00) -> GetPipelineResponse - .. py:method:: update(pipeline_id: str [, allow_duplicate_names: Optional[bool], budget_policy_id: Optional[str], catalog: Optional[str], channel: Optional[str], clusters: Optional[List[PipelineCluster]], configuration: Optional[Dict[str, str]], continuous: Optional[bool], deployment: Optional[PipelineDeployment], development: Optional[bool], edition: Optional[str], environment: Optional[PipelinesEnvironment], event_log: Optional[EventLogSpec], expected_last_modified: Optional[int], filters: Optional[Filters], gateway_definition: Optional[IngestionGatewayPipelineDefinition], id: Optional[str], ingestion_definition: Optional[IngestionPipelineDefinition], libraries: Optional[List[PipelineLibrary]], name: Optional[str], notifications: Optional[List[Notifications]], photon: Optional[bool], restart_window: Optional[RestartWindow], root_path: Optional[str], run_as: Optional[RunAs], schema: Optional[str], serverless: Optional[bool], storage: Optional[str], tags: Optional[Dict[str, str]], target: Optional[str], trigger: Optional[PipelineTrigger], usage_policy_id: Optional[str]]) + .. py:method:: update(pipeline_id: str [, allow_duplicate_names: Optional[bool], budget_policy_id: Optional[str], catalog: Optional[str], channel: Optional[str], clusters: Optional[List[PipelineCluster]], configuration: Optional[Dict[str, str]], continuous: Optional[bool], deployment: Optional[PipelineDeployment], development: Optional[bool], edition: Optional[str], environment: Optional[PipelinesEnvironment], event_log: Optional[EventLogSpec], expected_last_modified: Optional[int], filters: Optional[Filters], gateway_definition: Optional[IngestionGatewayPipelineDefinition], id: Optional[str], ingestion_definition: Optional[IngestionPipelineDefinition], libraries: Optional[List[PipelineLibrary]], name: Optional[str], notifications: Optional[List[Notifications]], parameters: Optional[Dict[str, str]], photon: Optional[bool], restart_window: Optional[RestartWindow], root_path: Optional[str], run_as: Optional[RunAs], schema: Optional[str], serverless: Optional[bool], storage: Optional[str], tags: Optional[Dict[str, str]], target: Optional[str], trigger: Optional[PipelineTrigger], usage_policy_id: Optional[str]]) Usage: @@ -593,6 +596,9 @@ Friendly identifier for this pipeline. :param notifications: List[:class:`Notifications`] (optional) List of notification settings for this pipeline. + :param parameters: Dict[str,str] (optional) + Key/value map of default parameters to use for pipeline execution. Maximum total size: 10k + characters (JSON format) :param photon: bool (optional) Whether Photon is enabled for this pipeline. :param restart_window: :class:`RestartWindow` (optional) diff --git a/docs/workspace/sql/statement_execution.rst b/docs/workspace/sql/statement_execution.rst index 77f65c858..b8e12e388 100755 --- a/docs/workspace/sql/statement_execution.rst +++ b/docs/workspace/sql/statement_execution.rst @@ -87,7 +87,6 @@ [Apache Arrow Columnar]: https://arrow.apache.org/overview/ [Databricks SQL Statement Execution API tutorial]: https://docs.databricks.com/sql/api/sql-execution-tutorial.html - .. py:method:: cancel_execution(statement_id: str) diff --git a/docs/workspace/supervisoragents/supervisor_agents.rst b/docs/workspace/supervisoragents/supervisor_agents.rst index a58be386c..9e35ef725 100755 --- a/docs/workspace/supervisoragents/supervisor_agents.rst +++ b/docs/workspace/supervisoragents/supervisor_agents.rst @@ -32,7 +32,7 @@ Creates a Tool under a Supervisor Agent. Specify one of "genie_space", "knowledge_assistant", "uc_function", "uc_connection", "app", "volume", "dashboard", "table", "vector_search_index", - "catalog", "schema", "supervisor_agent", "web_search" in the request body. The legacy values + "catalog", "schema", "supervisor_agent", "web_search", "skill" in the request body. The legacy values "lakeview_dashboard" and "uc_table" are also accepted and remain equivalent to "dashboard" and "table" respectively. diff --git a/docs/workspace/tags/tag_policies.rst b/docs/workspace/tags/tag_policies.rst old mode 100644 new mode 100755 index 0c335d8ac..726c0b6ab --- a/docs/workspace/tags/tag_policies.rst +++ b/docs/workspace/tags/tag_policies.rst @@ -10,7 +10,6 @@ [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/tag_policy - .. py:method:: create_tag_policy(tag_policy: TagPolicy) -> TagPolicy