From 0b848e4c9cc8420b373c237858470b73beb66639 Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Sun, 19 Jul 2026 16:15:31 +0300 Subject: [PATCH 1/8] [MINOR][PYSPARK][DOCS] Fixed some links in documentation. ### What changes were proposed in this pull request? Fixed some links in DataFrame and Pandas-On-Spark-DataFrame classes. ### Why are the changes needed? The documentation will be more comprehensive. ### Does this PR introduce any user-facing change? No. This is a documentation-only change. ### How was this patch tested? Manual review. ### Was this patch authored or co-authored using generative AI tooling? No. --- .../reference/pyspark.sql/core_classes.rst | 3 +++ python/pyspark/pandas/frame.py | 24 ++++++++++--------- python/pyspark/sql/__init__.py | 3 +++ python/pyspark/sql/dataframe.py | 4 +++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/python/docs/source/reference/pyspark.sql/core_classes.rst b/python/docs/source/reference/pyspark.sql/core_classes.rst index af38ef1c4f5ee..1ba4c62994b74 100644 --- a/python/docs/source/reference/pyspark.sql/core_classes.rst +++ b/python/docs/source/reference/pyspark.sql/core_classes.rst @@ -38,6 +38,9 @@ Core Classes DataFrameReader DataFrameWriter DataFrameWriterV2 + MergeIntoWriter + DataStreamReader + DataStreamWriter UDFRegistration UDTFRegistration udf.UserDefinedFunction diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py index 4b9ce84b3b7ab..5bdc665a15e7e 100644 --- a/python/pyspark/pandas/frame.py +++ b/python/pyspark/pandas/frame.py @@ -390,6 +390,19 @@ class DataFrame(Frame, Generic[T]): pandas-on-Spark DataFrame that corresponds to pandas DataFrame logically. This holds Spark DataFrame internally. + .. versionadded:: 3.2.0 + + .. versionchanged:: 3.4.0 + Since 3.4.0, it deals with `data` and `index` in this approach: + 1, when `data` is a distributed dataset (Internal DataFrame/Spark DataFrame/ + pandas-on-Spark DataFrame/pandas-on-Spark Series), it will first parallelize + the `index` if necessary, and then try to combine the `data` and `index`; + Note that if `data` and `index` doesn't have the same anchor, then + `compute.ops_on_diff_frames` should be turned on; + 2, when `data` is a local dataset (Pandas DataFrame/numpy ndarray/list/etc), + it will first collect the `index` to driver if necessary, and then apply + the `pandas.DataFrame(...)` creation internally; + :ivar _internal: an internal immutable Frame to manage metadata. :type _internal: InternalFrame @@ -409,17 +422,6 @@ class DataFrame(Frame, Generic[T]): copy : boolean, default False Copy data from inputs. Only affects DataFrame / 2d ndarray input - .. versionchanged:: 3.4.0 - Since 3.4.0, it deals with `data` and `index` in this approach: - 1, when `data` is a distributed dataset (Internal DataFrame/Spark DataFrame/ - pandas-on-Spark DataFrame/pandas-on-Spark Series), it will first parallelize - the `index` if necessary, and then try to combine the `data` and `index`; - Note that if `data` and `index` doesn't have the same anchor, then - `compute.ops_on_diff_frames` should be turned on; - 2, when `data` is a local dataset (Pandas DataFrame/numpy ndarray/list/etc), - it will first collect the `index` to driver if necessary, and then apply - the `pandas.DataFrame(...)` creation internally; - Examples -------- Constructing DataFrame from a dictionary. diff --git a/python/pyspark/sql/__init__.py b/python/pyspark/sql/__init__.py index 117f57f29d3fc..20e3345949981 100644 --- a/python/pyspark/sql/__init__.py +++ b/python/pyspark/sql/__init__.py @@ -49,6 +49,7 @@ from pyspark.sql.group import GroupedData from pyspark.sql.observation import Observation from pyspark.sql.readwriter import DataFrameReader, DataFrameWriter, DataFrameWriterV2 +from pyspark.sql.streaming import DataStreamReader, DataStreamWriter from pyspark.sql.merge import MergeIntoWriter from pyspark.sql.window import Window, WindowSpec from pyspark.sql.pandas.group_ops import PandasCogroupedOps @@ -77,6 +78,8 @@ "DataFrameWriter", "DataFrameWriterV2", "MergeIntoWriter", + "DataStreamReader", + "DataStreamWriter", "PandasCogroupedOps", "is_remote", ] diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 6c4d32ea1797f..766e91b5f4198 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1206,6 +1206,8 @@ def hint( :class:`DataFrame` Hinted DataFrame + .. note:: See also https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select-hints.html + Examples -------- >>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")], schema=["age", "name"]) @@ -6482,7 +6484,7 @@ def pandas_api( See Also -------- - pyspark.pandas.frame.DataFrame.to_spark + pyspark.pandas.DataFrame.to_spark Examples -------- From 6848b333fc2eef4040bf65cfe545c612093c1a90 Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Mon, 20 Jul 2026 02:46:09 +0300 Subject: [PATCH 2/8] add links, See also, Returns sections --- python/pyspark/sql/functions/builtin.py | 799 ++++++++++++++++++++++-- python/pyspark/sql/pandas/functions.py | 2 +- python/pyspark/sql/tvf.py | 28 +- 3 files changed, 762 insertions(+), 67 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 2c2345c1a58a4..a0995b44c9d31 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -52,6 +52,7 @@ MapType, NumericType, _from_numpy_type, + VariantType ) # Keep UserDefinedFunction import for backwards compatible import; moved in SPARK-22409 @@ -322,6 +323,10 @@ def col(col: str) -> Column: :class:`~pyspark.sql.Column` the corresponding column instance. + See Also + -------- + :meth:`pyspark.sql.functions.column` + Examples -------- >>> col('x') @@ -555,6 +560,10 @@ def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: left : :class:`~pyspark.sql.Column` or column name right : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- Example 1: Integer plus Integer. @@ -635,6 +644,14 @@ def try_avg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.avg` + Examples -------- Example 1: Calculating the average age @@ -695,6 +712,10 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: right : :class:`~pyspark.sql.Column` or column name divisor + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- Example 1: Integer divided by Integer. @@ -807,6 +828,10 @@ def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: right : :class:`~pyspark.sql.Column` or column name multiplier + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- Example 1: Integer multiplied by Integer. @@ -869,6 +894,10 @@ def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: left : :class:`~pyspark.sql.Column` or column name right : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- Example 1: Integer minus Integer. @@ -949,6 +978,14 @@ def try_sum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.sum` + Examples -------- Example 1: Calculating the sum of values in a column @@ -1646,6 +1683,7 @@ def sum(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.try_sum` :meth:`pyspark.sql.functions.min` :meth:`pyspark.sql.functions.max` :meth:`pyspark.sql.functions.avg` @@ -1710,6 +1748,7 @@ def avg(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.try_avg` :meth:`pyspark.sql.functions.min` :meth:`pyspark.sql.functions.max` :meth:`pyspark.sql.functions.sum` @@ -2965,6 +3004,10 @@ def e() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -3276,6 +3319,10 @@ def pi() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> import pyspark.sql.functions as sf @@ -4739,6 +4786,10 @@ def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: x : :class:`~pyspark.sql.Column` or column name the independent variable. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.regr_avgx` @@ -5190,6 +5241,10 @@ def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: x : :class:`~pyspark.sql.Column` or column name the independent variable. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.regr_avgx` @@ -6257,6 +6312,10 @@ def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) :class:`~pyspark.sql.Column` the base rased to the power the argument. + See Also + -------- + :meth:`pyspark.sql.functions.power` + Examples -------- >>> from pyspark.sql import functions as sf @@ -7096,6 +7155,10 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: :class:`~pyspark.sql.Column` first value of the group. + See Also + -------- + :meth:`pyspark.sql.functions.first_value` + Examples -------- >>> from pyspark.sql import functions as sf @@ -7438,6 +7501,10 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: :class:`~pyspark.sql.Column` last value of the group. + See Also + -------- + :meth:`pyspark.sql.functions.last_value` + Examples -------- >>> from pyspark.sql import functions as sf @@ -7484,7 +7551,7 @@ def monotonically_increasing_id() -> Column: ----- The function is non-deterministic because its result depends on partition IDs. - As an example, consider a :class:`DataFrame` with two partitions, each with 3 records. + As an example, consider a :class:`~DataFrame` with two partitions, each with 3 records. This expression would return the following IDs: 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. @@ -7539,7 +7606,12 @@ def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - value from first column or second if first is NaN . + value from first column or second if first is NaN. + + See Also + -------- + :meth:`pyspark.sql.functions.nvl` + :meth:`pyspark.sql.functions.nvl2` Examples -------- @@ -9387,6 +9459,11 @@ def current_timezone() -> Column: See Also -------- :meth:`pyspark.sql.functions.convert_timezone` + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.curdate` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.localtimestamp` + :meth:`pyspark.sql.functions.current_time` Examples -------- @@ -9444,8 +9521,12 @@ def current_time(precision: Optional[int] = None) -> Column: See Also -------- + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.curdate` :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.localtimestamp` :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.current_timezone` Examples -------- @@ -9497,6 +9578,8 @@ def current_timestamp() -> Column: :meth:`pyspark.sql.functions.curdate` :meth:`pyspark.sql.functions.current_date` :meth:`pyspark.sql.functions.localtimestamp` + :meth:`pyspark.sql.functions.current_time` + :meth:`pyspark.sql.functions.current_timezone` Examples -------- @@ -9710,9 +9793,15 @@ def year(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -9801,9 +9890,15 @@ def quarter(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -9892,10 +9987,15 @@ def month(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.monthname` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -9979,10 +10079,21 @@ def dayofweek(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10060,10 +10171,21 @@ def dayofmonth(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Returns ------- @@ -10154,14 +10276,15 @@ def day(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.dayname` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.weekofyear` :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10244,11 +10367,22 @@ def dayofyear(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` :meth:`pyspark.sql.functions.weekofyear` :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10340,9 +10474,15 @@ def hour(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10417,9 +10557,15 @@ def minute(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Returns ------- @@ -10504,9 +10650,15 @@ def second(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10579,10 +10731,21 @@ def weekofyear(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` :meth:`pyspark.sql.functions.dayofmonth` :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10662,11 +10825,21 @@ def weekday(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` :meth:`pyspark.sql.functions.weekofyear` :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10827,7 +11000,21 @@ def dayname(col: "ColumnOrName") -> Column: See Also -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` :meth:`pyspark.sql.functions.monthname` Examples @@ -10917,8 +11104,14 @@ def extract(field: Column, source: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10972,8 +11165,14 @@ def date_part(field: Column, source: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -12121,7 +12320,7 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + col: :class:`~pyspark.sql.Column` or column name column values to convert. format: literal string, optional format to use to convert timestamp values. @@ -12188,6 +12387,17 @@ def xpath(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12224,6 +12434,17 @@ def xpath_boolean(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12260,6 +12481,17 @@ def xpath_double(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12296,6 +12528,17 @@ def xpath_number(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> import pyspark.sql.functions as sf @@ -12333,6 +12576,17 @@ def xpath_float(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12369,6 +12623,17 @@ def xpath_int(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12405,6 +12670,18 @@ def xpath_long(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12441,6 +12718,17 @@ def xpath_short(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -12476,6 +12764,17 @@ def xpath_string(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + xml: :class:`~pyspark.sql.Column` or column name + column with xml values. + path: literal string (Column is not yet supported) + xpath to extract + + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -13701,6 +14000,10 @@ def to_unix_timestamp( format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert UNIX timestamp values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.to_date` @@ -13774,6 +14077,10 @@ def to_timestamp_ltz( format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampType` timestamp values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.to_date` @@ -13845,6 +14152,10 @@ def to_timestamp_ntz( format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampNTZType` timestamp values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.to_date` @@ -13907,10 +14218,18 @@ def current_catalog() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- - :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -13933,9 +14252,12 @@ def current_path() -> Column: See Also -------- + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_database` :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.current_database` Examples -------- @@ -13956,10 +14278,18 @@ def current_database() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_catalog` :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -13980,10 +14310,18 @@ def current_schema() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_catalog` :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -14004,10 +14342,18 @@ def current_user() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.user` :meth:`pyspark.sql.functions.session_user` + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -14028,6 +14374,10 @@ def user() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.current_user` @@ -14052,6 +14402,10 @@ def session_user() -> Column: .. versionadded:: 4.0.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.user` @@ -14082,6 +14436,10 @@ def uuid(seed: Optional[Union[Column, int]] = None) -> Column: seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- Example 1: Generate UUIDs with random seed @@ -15331,10 +15689,11 @@ def instr( See Also -------- - :meth:`pyspark.sql.functions.locate` :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring` :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.functions.position` + :meth:`pyspark.sql.Column.substr` Examples -------- @@ -15612,6 +15971,7 @@ def substring( :meth:`pyspark.sql.functions.locate` :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.functions.position` :meth:`pyspark.sql.Column.substr` Examples @@ -15710,6 +16070,7 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: :meth:`pyspark.sql.functions.locate` :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.position` :meth:`pyspark.sql.Column.substr` Examples @@ -15862,6 +16223,7 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring` :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.functions.position` :meth:`pyspark.sql.Column.substr` Examples @@ -17214,6 +17576,10 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: :class:`~pyspark.sql.Column` replaced value. + See Also + -------- + :meth:`pyspark.sql.functions.replace` + Examples -------- >>> from pyspark.sql import functions as sf @@ -17245,11 +17611,15 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C Parameters ---------- - col : :class:`~pyspark.sql.Column` or str + col: :class:`~pyspark.sql.Column` or str Input column or strings. - format : :class:`~pyspark.sql.Column` or str, optional + format: :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.try_to_binary` @@ -17260,14 +17630,14 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C >>> import pyspark.sql.functions as sf >>> df = spark.createDataFrame([("abc",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() + >>> df.select(sf.to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() [Row(r=b'abc')] Example 2: Convert string to a timestamp without encoding specified >>> import pyspark.sql.functions as sf >>> df = spark.createDataFrame([("414243",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() + >>> df.select(sf.to_binary(df.e).alias('r')).collect() [Row(r=b'ABC')] """ if format is not None: @@ -17298,7 +17668,7 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: 'PR': Only allowed at the end of the format string; specifies that the result string will be wrapped by angle brackets if the input value is negative. If `col` is a datetime, `format` shall be a valid datetime pattern, see - Patterns. + https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html. If `col` is a binary, it is converted to a string in one of the formats: 'base64': a base 64 string. 'hex': a string in the hexadecimal format. @@ -17313,6 +17683,10 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([(78.12,)], ["e"]) @@ -17344,7 +17718,7 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: 'PR': Only allowed at the end of the format string; specifies that the result string will be wrapped by angle brackets if the input value is negative. If `col` is a datetime, `format` shall be a valid datetime pattern, see - Patterns. + https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html. If `col` is a binary, it is converted to a string in one of the formats: 'base64': a base 64 string. 'hex': a string in the hexadecimal format. @@ -17359,6 +17733,10 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([(78.12,)], ["e"]) @@ -17400,6 +17778,10 @@ def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.try_to_number` @@ -17432,6 +17814,10 @@ def replace( A column of string, If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("ABCabc", "abc", "DEF",)], ["a", "b", "c"]) @@ -17467,6 +17853,10 @@ def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnO partNum : :class:`~pyspark.sql.Column` or column name A column of string, requested part of the split (1-based). + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.sentences` @@ -17573,6 +17963,10 @@ def try_parse_url( :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + See Also + -------- + :meth:`pyspark.sql.functions.parse_url` + Examples -------- Example 1: Extracting the query part from a URL @@ -17689,6 +18083,10 @@ def parse_url( :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + See Also + -------- + :meth:`pyspark.sql.functions.try_parse_url` + Examples -------- Example 1: Extracting the query part from a URL @@ -17781,6 +18179,10 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: cols : :class:`~pyspark.sql.Column` or str column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.format_string` @@ -17899,6 +18301,10 @@ def try_url_decode(str: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + See Also + -------- + :meth:`pyspark.sql.functions.url_decode` + Examples -------- Example 1: Decoding a URL-encoded string @@ -18023,6 +18429,10 @@ def position( start : :class:`~pyspark.sql.Column` or str, optional A column of string, start position. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> import pyspark.sql.functions as sf @@ -18066,6 +18476,10 @@ def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: suffix : :class:`~pyspark.sql.Column` or str A column of string, the suffix. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) @@ -18104,6 +18518,10 @@ def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: prefix : :class:`~pyspark.sql.Column` or str A column of string, the prefix. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) @@ -18139,6 +18557,14 @@ def char(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or str Input column or strings. + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.chr` + Examples -------- >>> import pyspark.sql.functions as sf @@ -18166,6 +18592,10 @@ def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: trim : :class:`~pyspark.sql.Column` or str, optional The trim string characters to trim, the default value is a single space + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("SSparkSQLS", "SL", )], ['a', 'b']) @@ -18196,6 +18626,10 @@ def char_length(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.character_length` @@ -18228,6 +18662,10 @@ def character_length(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.char_length` @@ -18259,6 +18697,14 @@ def chr(n: "ColumnOrName") -> Column: n : :class:`~pyspark.sql.Column` or column name target column to compute on. + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.char` + Examples -------- >>> import pyspark.sql.functions as sf @@ -18296,6 +18742,10 @@ def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.to_binary` @@ -18354,6 +18804,10 @@ def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.to_number` @@ -18406,6 +18860,10 @@ def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: right : :class:`~pyspark.sql.Column` or str The input column or strings to find, may be NULL. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("Spark SQL", "Spark")], ['a', 'b']) @@ -18443,6 +18901,10 @@ def elt(*inputs: "ColumnOrName") -> Column: inputs : :class:`~pyspark.sql.Column` or str Input columns or strings. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([(1, "scala", "java")], ['a', 'b', 'c']) @@ -18471,6 +18933,10 @@ def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: str_array : :class:`~pyspark.sql.Column` or str The comma-delimited list. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("ab", "abc,b,ab,c,def")], ['a', 'b']) @@ -18511,6 +18977,10 @@ def like( If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) @@ -18561,6 +19031,10 @@ def ilike( If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) @@ -18592,6 +19066,10 @@ def lcase(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.lower` @@ -18623,6 +19101,10 @@ def ucase(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.upper` @@ -18657,6 +19139,14 @@ def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: len : :class:`~pyspark.sql.Column` or str Input column or strings, the leftmost `len`. + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.right` + Examples -------- >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) @@ -18681,6 +19171,14 @@ def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: len : :class:`~pyspark.sql.Column` or str Input column or strings, the rightmost `len`. + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.left` + Examples -------- >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) @@ -18763,6 +19261,16 @@ def collate(col: "ColumnOrName", collation: str) -> Column: ------- :class:`~pyspark.sql.Column` A new column of string type, where each value has the specified collation. + + See Also + -------- + :meth:`pyspark.sql.functions.collation` + + Examples + -------- + >>> df = spark.createDataFrame([("abc",)], ["c"]) + >>> df.select(sf.collate(df.c, 'SYSTEM.BUILTIN.UTF8_BINARY').alias('r')).collect() + [Row(r='abc')] """ from pyspark.sql.classic.column import _to_java_column @@ -19753,6 +20261,10 @@ def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: extraction : index to check for in array or key to check for in map + Returns + ------- + :class:`~pyspark.sql.Column` + Notes ----- The position is not zero based, but 1 based index. @@ -21284,6 +21796,10 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column: :class:`~pyspark.sql.Column` string representation of given JSON object value. + See Also + -------- + :meth:`pyspark.sql.functions.parse_json` + Examples -------- Example 1: Extract a json object from json string @@ -21381,7 +21897,7 @@ def from_json( options: Optional[Mapping[str, str]] = None, ) -> Column: """ - Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType` + Parses a column containing a JSON string into a :class:`MapType` with :class:`~StringType` as keys type, :class:`StructType` or :class:`ArrayType` with the specified schema. Returns `null`, in the case of an unparsable string. @@ -21500,7 +22016,7 @@ def try_parse_json( col: "ColumnOrName", ) -> Column: """ - Parses a column containing a JSON string into a :class:`VariantType`. Returns None if a string + Parses a column containing a JSON string into a :class:`~VariantType`. Returns None if a string contains an invalid JSON value. .. versionadded:: 4.0.0 @@ -21515,6 +22031,10 @@ def try_parse_json( :class:`~pyspark.sql.Column` a new column of VariantType. + See Also + -------- + :meth:`pyspark.sql.functions.parse_json` + Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''}, {'json': '''{a : 1}'''} ]) @@ -21595,6 +22115,10 @@ def parse_json( :class:`~pyspark.sql.Column` a new column of VariantType. + See Also + -------- + :meth:`pyspark.sql.functions.try_parse_json` + Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) @@ -21993,6 +22517,10 @@ def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> :class:`~pyspark.sql.Column` a column of `targetType` representing the extracted result + See Also + -------- + :meth:`pyspark.sql.functions.try_variant_get` + Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) @@ -22040,6 +22568,10 @@ def try_variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str :class:`~pyspark.sql.Column` a column of `targetType` representing the extracted result + See Also + -------- + :meth:`pyspark.sql.functions.variant_get` + Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) @@ -22564,7 +23096,7 @@ def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]] @_try_remote_functions def to_xml(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - Converts a column containing a :class:`StructType` into a XML string. + Converts a column containing a :class:`~StructType` into a XML string. Throws an exception, in the case of an unsupported type. .. versionadded:: 4.0.0 @@ -22703,7 +23235,7 @@ def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]] @_try_remote_functions def to_csv(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - CSV Function: Converts a column containing a :class:`StructType` into a CSV string. + CSV Function: Converts a column containing a :class:`~StructType` into a CSV string. Throws an exception, in the case of an unsupported type. .. versionadded:: 3.0.0 @@ -22810,6 +23342,12 @@ def size(col: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` length of the array/map. + See Also + -------- + :meth:`pyspark.sql.functions.size` + :meth:`pyspark.sql.functions.array_size` + :meth:`pyspark.sql.functions.cardinality` + Examples -------- >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data']) @@ -23007,6 +23545,11 @@ def array_size(col: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new column that contains the size of each array. + See Also + -------- + :meth:`pyspark.sql.functions.size` + :meth:`pyspark.sql.functions.cardinality` + Examples -------- Example 1: Basic usage with integer array @@ -23089,6 +23632,11 @@ def cardinality(col: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` length of the array/map. + See Also + -------- + :meth:`pyspark.sql.functions.size` + :meth:`pyspark.sql.functions.array_size` + Examples -------- >>> import pyspark.sql.functions as sf @@ -25155,7 +25703,7 @@ def years(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + method of the :class:`~DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25199,7 +25747,7 @@ def months(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + method of the :class:`~DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25243,7 +25791,7 @@ def days(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + method of the :class:`~DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25287,7 +25835,7 @@ def hours(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + method of the :class:`~DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25835,6 +26383,10 @@ def time_from_seconds(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name Seconds since midnight (0 to 86399.999999). + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -25861,6 +26413,10 @@ def time_from_millis(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name Milliseconds since midnight (0 to 86399999). + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -25887,6 +26443,10 @@ def time_from_micros(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name Microseconds since midnight (0 to 86399999999). + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -25913,6 +26473,10 @@ def time_to_seconds(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name TIME value to convert. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -25939,6 +26503,10 @@ def time_to_millis(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name TIME value to convert. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -25965,6 +26533,10 @@ def time_to_micros(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name TIME value to convert. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -27081,7 +27653,7 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + method of the :class:`~DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -27111,28 +27683,35 @@ def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None The optional endianness of the output WKB, 'NDR' for little-endian (default) or 'XDR' for big-endian. - Examples - -------- + Returns + ------- + :class:`~pyspark.sql.Column` + Examples + ------- Example 1: Getting WKB from GEOGRAPHY. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] Example 2: Getting WKB from GEOMETRY. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] Example 3: Getting WKB (little-endian) from GEOGRAPHY. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb'), 'NDR'))).collect() [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] Example 4: Getting WKB (big-endian) from GEOMETRY. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb'), 'XDR'))).collect() @@ -27156,6 +27735,10 @@ def st_geogfromwkb(wkb: "ColumnOrName") -> Column: wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOGRAPHY value. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -27181,6 +27764,10 @@ def st_geomfromwkb( srid : :class:`~pyspark.sql.Column` or int, optional The optional SRID value of the geometry. Default is 0. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -27209,16 +27796,21 @@ def st_setsrid(geo: "ColumnOrName", srid: Union["ColumnOrName", int]) -> Column: srid : :class:`~pyspark.sql.Column` or int An INTEGER representing the new SRID of the geospatial value. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- - Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'), 4326)], ['wkb', 'srid']) # noqa >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geogfromwkb('wkb'), 'srid'))).collect() [Row(st_srid(st_setsrid(st_geogfromwkb(wkb), srid))=4326)] Example 2: Setting the SRID on GEOMETRY with SRID as an integer literal. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geomfromwkb('wkb'), 4326))).collect() @@ -27240,16 +27832,21 @@ def st_srid(geo: "ColumnOrName") -> Column: geo : :class:`~pyspark.sql.Column` or str A geospatial value, either a GEOGRAPHY or a GEOMETRY. + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- - Example 1: Getting the SRID of GEOGRAPHY. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.st_srid(sf.st_geogfromwkb('wkb'))).collect() [Row(st_srid(st_geogfromwkb(wkb))=4326)] Example 2: Getting the SRID of GEOMETRY. + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.st_srid(sf.st_geomfromwkb('wkb'))).collect() @@ -29930,6 +30527,14 @@ def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or str col2 : :class:`~pyspark.sql.Column` or str + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.nullif` + Examples -------- >>> import pyspark.sql.functions as sf @@ -29956,6 +30561,10 @@ def isnotnull(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -29992,6 +30601,10 @@ def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -30027,6 +30640,14 @@ def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.ifnull` + Examples -------- >>> import pyspark.sql.functions as sf @@ -30061,6 +30682,10 @@ def nullifzero(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> import pyspark.sql.functions as sf @@ -30096,6 +30721,15 @@ def nvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.nanvl` + :meth:`pyspark.sql.functions.nvl2` + Examples -------- >>> import pyspark.sql.functions as sf @@ -30132,6 +30766,15 @@ def nvl2(col1: "ColumnOrName", col2: "ColumnOrName", col3: "ColumnOrName") -> Co col2 : :class:`~pyspark.sql.Column` or column name col3 : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + + See Also + -------- + :meth:`pyspark.sql.functions.nanvl` + :meth:`pyspark.sql.functions.nvl` + Examples -------- >>> import pyspark.sql.functions as sf @@ -30166,6 +30809,10 @@ def zeroifnull(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> import pyspark.sql.functions as sf @@ -30607,6 +31254,10 @@ def sha(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.sha1` @@ -30632,6 +31283,10 @@ def input_file_block_length() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.input_file_name` @@ -30665,6 +31320,10 @@ def input_file_block_start() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.input_file_name` @@ -30705,6 +31364,10 @@ def reflect(*cols: "ColumnOrName") -> Column: and the second element should be a Column representing literal string for the method name, and the remaining are input arguments (Columns or column names) to the Java method. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.java_method` @@ -30740,6 +31403,10 @@ def java_method(*cols: "ColumnOrName") -> Column: and the second element should be a Column representing literal string for the method name, and the remaining are input arguments (Columns or column names) to the Java method. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.reflect` @@ -30838,6 +31505,10 @@ def version() -> Column: .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -30862,6 +31533,10 @@ def typeof(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + Examples -------- >>> from pyspark.sql import functions as sf @@ -30944,6 +31619,10 @@ def bitmap_bit_position(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input column. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.bitmap_bucket_number` @@ -30977,6 +31656,10 @@ def bitmap_bucket_number(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input column. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31011,6 +31694,10 @@ def bitmap_construct_agg(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input column will most likely be bitmap_bit_position(). + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31047,6 +31734,10 @@ def bitmap_count(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input bitmap. + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31076,6 +31767,15 @@ def bitmap_or_agg(col: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The input column should be bitmaps created from bitmap_construct_agg(). + + Returns + ------- + :class:`~pyspark.sql.Column` + See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31084,11 +31784,6 @@ def bitmap_or_agg(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.bitmap_count` :meth:`pyspark.sql.functions.bitmap_and_agg` - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). - Examples -------- >>> from pyspark.sql import functions as sf @@ -31197,7 +31892,7 @@ def udf( returnType : :class:`pyspark.sql.types.DataType` or str, optional the return type of the user-defined function. The value can be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - Defaults to :class:`StringType`. + Defaults to :class:`~StringType`. useArrow : bool, optional whether to use Arrow to optimize the (de)serialization. When it is None, the Spark config "spark.sql.execution.pythonUDF.arrow.enabled" takes effect. diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py index b14b10b44859e..4b66998299964 100644 --- a/python/pyspark/sql/pandas/functions.py +++ b/python/pyspark/sql/pandas/functions.py @@ -74,7 +74,7 @@ def arrow_udf(f=None, returnType=None, functionType=None): the return type of the user-defined function. The value can be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. functionType : int, optional - an enum value in :class:`pyspark.sql.functions.ArrowUDFType`. + an enum value in :class:`~pyspark.sql.functions.ArrowUDFType`. Default: SCALAR. This parameter exists for compatibility. Using Python type hints is encouraged. diff --git a/python/pyspark/sql/tvf.py b/python/pyspark/sql/tvf.py index 90874578a3562..57b0f525675fd 100644 --- a/python/pyspark/sql/tvf.py +++ b/python/pyspark/sql/tvf.py @@ -59,7 +59,7 @@ def range( Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -101,7 +101,7 @@ def explode(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -185,7 +185,7 @@ def explode_outer(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -238,7 +238,7 @@ def inline(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -301,7 +301,7 @@ def inline_outer(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -344,7 +344,7 @@ def json_tuple(self, input: Column, *fields: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -394,7 +394,7 @@ def posexplode(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -437,7 +437,7 @@ def posexplode_outer(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -490,7 +490,7 @@ def stack(self, n: Column, *fields: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -525,7 +525,7 @@ def collations(self) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -546,7 +546,7 @@ def sql_keywords(self) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -574,7 +574,7 @@ def variant_explode(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -639,7 +639,7 @@ def variant_explode_outer(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -681,7 +681,7 @@ def python_worker_logs(self) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- From 5a48090ab8c868aef821a9d94b9dcf8335fb9b26 Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Mon, 20 Jul 2026 02:50:14 +0300 Subject: [PATCH 3/8] removed DataStreamReader, DataStreamWriter from init.py --- python/pyspark/sql/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/pyspark/sql/__init__.py b/python/pyspark/sql/__init__.py index 20e3345949981..117f57f29d3fc 100644 --- a/python/pyspark/sql/__init__.py +++ b/python/pyspark/sql/__init__.py @@ -49,7 +49,6 @@ from pyspark.sql.group import GroupedData from pyspark.sql.observation import Observation from pyspark.sql.readwriter import DataFrameReader, DataFrameWriter, DataFrameWriterV2 -from pyspark.sql.streaming import DataStreamReader, DataStreamWriter from pyspark.sql.merge import MergeIntoWriter from pyspark.sql.window import Window, WindowSpec from pyspark.sql.pandas.group_ops import PandasCogroupedOps @@ -78,8 +77,6 @@ "DataFrameWriter", "DataFrameWriterV2", "MergeIntoWriter", - "DataStreamReader", - "DataStreamWriter", "PandasCogroupedOps", "is_remote", ] From 60aab1294d120b1525765014b6f8582731843f6f Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Mon, 20 Jul 2026 03:25:43 +0300 Subject: [PATCH 4/8] Removed block 20264-20267, made links to spark.apache.org as hyperlink --- python/pyspark/sql/functions/builtin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index a0995b44c9d31..5144e0ec2f35c 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -17668,7 +17668,7 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: 'PR': Only allowed at the end of the format string; specifies that the result string will be wrapped by angle brackets if the input value is negative. If `col` is a datetime, `format` shall be a valid datetime pattern, see - https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html. + `Datetime pattern `. If `col` is a binary, it is converted to a string in one of the formats: 'base64': a base 64 string. 'hex': a string in the hexadecimal format. @@ -17718,7 +17718,7 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: 'PR': Only allowed at the end of the format string; specifies that the result string will be wrapped by angle brackets if the input value is negative. If `col` is a datetime, `format` shall be a valid datetime pattern, see - https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html. + `Datetime pattern `. If `col` is a binary, it is converted to a string in one of the formats: 'base64': a base 64 string. 'hex': a string in the hexadecimal format. @@ -20261,10 +20261,6 @@ def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: extraction : index to check for in array or key to check for in map - Returns - ------- - :class:`~pyspark.sql.Column` - Notes ----- The position is not zero based, but 1 based index. From 2a70036b87e73906d123191741dd0e93754bce10 Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Mon, 20 Jul 2026 23:24:22 +0300 Subject: [PATCH 5/8] Revert python\pyspark\sql\functions\builtin.py --- python/pyspark/sql/functions/builtin.py | 801 ++---------------------- 1 file changed, 55 insertions(+), 746 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 5144e0ec2f35c..45eba0a51adf9 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -52,7 +52,6 @@ MapType, NumericType, _from_numpy_type, - VariantType ) # Keep UserDefinedFunction import for backwards compatible import; moved in SPARK-22409 @@ -323,10 +322,6 @@ def col(col: str) -> Column: :class:`~pyspark.sql.Column` the corresponding column instance. - See Also - -------- - :meth:`pyspark.sql.functions.column` - Examples -------- >>> col('x') @@ -560,10 +555,6 @@ def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: left : :class:`~pyspark.sql.Column` or column name right : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- Example 1: Integer plus Integer. @@ -644,14 +635,6 @@ def try_avg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.avg` - Examples -------- Example 1: Calculating the average age @@ -712,10 +695,6 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: right : :class:`~pyspark.sql.Column` or column name divisor - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- Example 1: Integer divided by Integer. @@ -828,10 +807,6 @@ def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: right : :class:`~pyspark.sql.Column` or column name multiplier - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- Example 1: Integer multiplied by Integer. @@ -894,10 +869,6 @@ def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: left : :class:`~pyspark.sql.Column` or column name right : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- Example 1: Integer minus Integer. @@ -978,14 +949,6 @@ def try_sum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.sum` - Examples -------- Example 1: Calculating the sum of values in a column @@ -1683,7 +1646,6 @@ def sum(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.try_sum` :meth:`pyspark.sql.functions.min` :meth:`pyspark.sql.functions.max` :meth:`pyspark.sql.functions.avg` @@ -1748,7 +1710,6 @@ def avg(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.try_avg` :meth:`pyspark.sql.functions.min` :meth:`pyspark.sql.functions.max` :meth:`pyspark.sql.functions.sum` @@ -3004,10 +2965,6 @@ def e() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -3319,10 +3276,6 @@ def pi() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> import pyspark.sql.functions as sf @@ -4786,10 +4739,6 @@ def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: x : :class:`~pyspark.sql.Column` or column name the independent variable. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.regr_avgx` @@ -5241,10 +5190,6 @@ def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: x : :class:`~pyspark.sql.Column` or column name the independent variable. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.regr_avgx` @@ -6312,10 +6257,6 @@ def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) :class:`~pyspark.sql.Column` the base rased to the power the argument. - See Also - -------- - :meth:`pyspark.sql.functions.power` - Examples -------- >>> from pyspark.sql import functions as sf @@ -7155,10 +7096,6 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: :class:`~pyspark.sql.Column` first value of the group. - See Also - -------- - :meth:`pyspark.sql.functions.first_value` - Examples -------- >>> from pyspark.sql import functions as sf @@ -7501,10 +7438,6 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: :class:`~pyspark.sql.Column` last value of the group. - See Also - -------- - :meth:`pyspark.sql.functions.last_value` - Examples -------- >>> from pyspark.sql import functions as sf @@ -7551,7 +7484,7 @@ def monotonically_increasing_id() -> Column: ----- The function is non-deterministic because its result depends on partition IDs. - As an example, consider a :class:`~DataFrame` with two partitions, each with 3 records. + As an example, consider a :class:`DataFrame` with two partitions, each with 3 records. This expression would return the following IDs: 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. @@ -7606,12 +7539,7 @@ def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - value from first column or second if first is NaN. - - See Also - -------- - :meth:`pyspark.sql.functions.nvl` - :meth:`pyspark.sql.functions.nvl2` + value from first column or second if first is NaN . Examples -------- @@ -9459,11 +9387,6 @@ def current_timezone() -> Column: See Also -------- :meth:`pyspark.sql.functions.convert_timezone` - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.curdate` - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.localtimestamp` - :meth:`pyspark.sql.functions.current_time` Examples -------- @@ -9521,12 +9444,8 @@ def current_time(precision: Optional[int] = None) -> Column: See Also -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.curdate` :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.localtimestamp` :meth:`pyspark.sql.functions.current_timestamp` - :meth:`pyspark.sql.functions.current_timezone` Examples -------- @@ -9578,8 +9497,6 @@ def current_timestamp() -> Column: :meth:`pyspark.sql.functions.curdate` :meth:`pyspark.sql.functions.current_date` :meth:`pyspark.sql.functions.localtimestamp` - :meth:`pyspark.sql.functions.current_time` - :meth:`pyspark.sql.functions.current_timezone` Examples -------- @@ -9793,15 +9710,9 @@ def year(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -9890,15 +9801,9 @@ def quarter(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -9987,15 +9892,10 @@ def month(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.monthname` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10079,21 +9979,10 @@ def dayofweek(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.weekofyear` Examples -------- @@ -10171,21 +10060,10 @@ def dayofmonth(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekofyear` Returns ------- @@ -10276,15 +10154,14 @@ def day(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayname` :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofweek` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` + :meth:`pyspark.sql.functions.weekday` Examples -------- @@ -10367,22 +10244,11 @@ def dayofyear(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.dayofmonth` :meth:`pyspark.sql.functions.weekofyear` :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10474,15 +10340,9 @@ def hour(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10557,15 +10417,9 @@ def minute(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Returns ------- @@ -10650,15 +10504,9 @@ def second(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10731,21 +10579,10 @@ def weekofyear(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.dayofweek` :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofweek` :meth:`pyspark.sql.functions.dayofmonth` :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -10825,21 +10662,11 @@ def weekday(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` :meth:`pyspark.sql.functions.weekofyear` :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.dayofmonth` :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` + :meth:`pyspark.sql.functions.dayofmonth` Examples -------- @@ -11000,21 +10827,7 @@ def dayname(col: "ColumnOrName") -> Column: See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` :meth:`pyspark.sql.functions.monthname` Examples @@ -11104,14 +10917,8 @@ def extract(field: Column, source: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -11165,14 +10972,8 @@ def date_part(field: Column, source: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.monthname` Examples -------- @@ -12320,7 +12121,7 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non Parameters ---------- - col: :class:`~pyspark.sql.Column` or column name + col : :class:`~pyspark.sql.Column` or column name column values to convert. format: literal string, optional format to use to convert timestamp values. @@ -12387,17 +12188,6 @@ def xpath(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12434,17 +12224,6 @@ def xpath_boolean(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12481,17 +12260,6 @@ def xpath_double(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12528,17 +12296,6 @@ def xpath_number(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> import pyspark.sql.functions as sf @@ -12576,17 +12333,6 @@ def xpath_float(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12623,17 +12369,6 @@ def xpath_int(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12670,18 +12405,6 @@ def xpath_long(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12718,17 +12441,6 @@ def xpath_short(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -12764,17 +12476,6 @@ def xpath_string(xml: "ColumnOrName", path: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - xml: :class:`~pyspark.sql.Column` or column name - column with xml values. - path: literal string (Column is not yet supported) - xpath to extract - - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -14000,10 +13701,6 @@ def to_unix_timestamp( format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert UNIX timestamp values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.to_date` @@ -14077,10 +13774,6 @@ def to_timestamp_ltz( format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampType` timestamp values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.to_date` @@ -14152,10 +13845,6 @@ def to_timestamp_ntz( format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampNTZType` timestamp values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.to_date` @@ -14218,18 +13907,10 @@ def current_catalog() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.current_user` - :meth:`pyspark.sql.functions.session_user` - :meth:`pyspark.sql.functions.current_schema` :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_path` + :meth:`pyspark.sql.functions.current_schema` Examples -------- @@ -14252,12 +13933,9 @@ def current_path() -> Column: See Also -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.current_user` - :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_schema` :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_schema` Examples -------- @@ -14278,18 +13956,10 @@ def current_database() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.current_user` - :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_catalog` :meth:`pyspark.sql.functions.current_schema` - :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -14310,18 +13980,10 @@ def current_schema() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.current_user` - :meth:`pyspark.sql.functions.session_user` :meth:`pyspark.sql.functions.current_catalog` :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -14342,18 +14004,10 @@ def current_user() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.user` :meth:`pyspark.sql.functions.session_user` - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_schema` - :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_path` Examples -------- @@ -14374,10 +14028,6 @@ def user() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.current_user` @@ -14402,10 +14052,6 @@ def session_user() -> Column: .. versionadded:: 4.0.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.user` @@ -14436,10 +14082,6 @@ def uuid(seed: Optional[Union[Column, int]] = None) -> Column: seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- Example 1: Generate UUIDs with random seed @@ -15689,11 +15331,10 @@ def instr( See Also -------- + :meth:`pyspark.sql.functions.locate` :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring` :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.functions.position` - :meth:`pyspark.sql.Column.substr` Examples -------- @@ -15971,7 +15612,6 @@ def substring( :meth:`pyspark.sql.functions.locate` :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.functions.position` :meth:`pyspark.sql.Column.substr` Examples @@ -16070,7 +15710,6 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: :meth:`pyspark.sql.functions.locate` :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.position` :meth:`pyspark.sql.Column.substr` Examples @@ -16223,7 +15862,6 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: :meth:`pyspark.sql.functions.substr` :meth:`pyspark.sql.functions.substring` :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.functions.position` :meth:`pyspark.sql.Column.substr` Examples @@ -17576,10 +17214,6 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: :class:`~pyspark.sql.Column` replaced value. - See Also - -------- - :meth:`pyspark.sql.functions.replace` - Examples -------- >>> from pyspark.sql import functions as sf @@ -17611,15 +17245,11 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C Parameters ---------- - col: :class:`~pyspark.sql.Column` or str + col : :class:`~pyspark.sql.Column` or str Input column or strings. - format: :class:`~pyspark.sql.Column` or str, optional + format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.try_to_binary` @@ -17630,14 +17260,14 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C >>> import pyspark.sql.functions as sf >>> df = spark.createDataFrame([("abc",)], ["e"]) - >>> df.select(sf.to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() + >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() [Row(r=b'abc')] Example 2: Convert string to a timestamp without encoding specified >>> import pyspark.sql.functions as sf >>> df = spark.createDataFrame([("414243",)], ["e"]) - >>> df.select(sf.to_binary(df.e).alias('r')).collect() + >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() [Row(r=b'ABC')] """ if format is not None: @@ -17668,7 +17298,7 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: 'PR': Only allowed at the end of the format string; specifies that the result string will be wrapped by angle brackets if the input value is negative. If `col` is a datetime, `format` shall be a valid datetime pattern, see - `Datetime pattern `. + Patterns. If `col` is a binary, it is converted to a string in one of the formats: 'base64': a base 64 string. 'hex': a string in the hexadecimal format. @@ -17683,10 +17313,6 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([(78.12,)], ["e"]) @@ -17718,7 +17344,7 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: 'PR': Only allowed at the end of the format string; specifies that the result string will be wrapped by angle brackets if the input value is negative. If `col` is a datetime, `format` shall be a valid datetime pattern, see - `Datetime pattern `. + Patterns. If `col` is a binary, it is converted to a string in one of the formats: 'base64': a base 64 string. 'hex': a string in the hexadecimal format. @@ -17733,10 +17359,6 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([(78.12,)], ["e"]) @@ -17778,10 +17400,6 @@ def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.try_to_number` @@ -17814,10 +17432,6 @@ def replace( A column of string, If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("ABCabc", "abc", "DEF",)], ["a", "b", "c"]) @@ -17853,10 +17467,6 @@ def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnO partNum : :class:`~pyspark.sql.Column` or column name A column of string, requested part of the split (1-based). - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.sentences` @@ -17963,10 +17573,6 @@ def try_parse_url( :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. - See Also - -------- - :meth:`pyspark.sql.functions.parse_url` - Examples -------- Example 1: Extracting the query part from a URL @@ -18083,10 +17689,6 @@ def parse_url( :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. - See Also - -------- - :meth:`pyspark.sql.functions.try_parse_url` - Examples -------- Example 1: Extracting the query part from a URL @@ -18179,10 +17781,6 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: cols : :class:`~pyspark.sql.Column` or str column names or :class:`~pyspark.sql.Column`\\s to be used in formatting - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.format_string` @@ -18301,10 +17899,6 @@ def try_url_decode(str: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. - See Also - -------- - :meth:`pyspark.sql.functions.url_decode` - Examples -------- Example 1: Decoding a URL-encoded string @@ -18429,10 +18023,6 @@ def position( start : :class:`~pyspark.sql.Column` or str, optional A column of string, start position. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> import pyspark.sql.functions as sf @@ -18476,10 +18066,6 @@ def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: suffix : :class:`~pyspark.sql.Column` or str A column of string, the suffix. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) @@ -18518,10 +18104,6 @@ def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: prefix : :class:`~pyspark.sql.Column` or str A column of string, the prefix. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) @@ -18557,14 +18139,6 @@ def char(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or str Input column or strings. - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.chr` - Examples -------- >>> import pyspark.sql.functions as sf @@ -18589,12 +18163,8 @@ def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. - trim : :class:`~pyspark.sql.Column` or str, optional - The trim string characters to trim, the default value is a single space - - Returns - ------- - :class:`~pyspark.sql.Column` + trim : :class:`~pyspark.sql.Column` or str, optional + The trim string characters to trim, the default value is a single space Examples -------- @@ -18626,10 +18196,6 @@ def char_length(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.character_length` @@ -18662,10 +18228,6 @@ def character_length(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.char_length` @@ -18697,14 +18259,6 @@ def chr(n: "ColumnOrName") -> Column: n : :class:`~pyspark.sql.Column` or column name target column to compute on. - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.char` - Examples -------- >>> import pyspark.sql.functions as sf @@ -18742,10 +18296,6 @@ def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.to_binary` @@ -18804,10 +18354,6 @@ def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.to_number` @@ -18860,10 +18406,6 @@ def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: right : :class:`~pyspark.sql.Column` or str The input column or strings to find, may be NULL. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("Spark SQL", "Spark")], ['a', 'b']) @@ -18901,10 +18443,6 @@ def elt(*inputs: "ColumnOrName") -> Column: inputs : :class:`~pyspark.sql.Column` or str Input columns or strings. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([(1, "scala", "java")], ['a', 'b', 'c']) @@ -18933,10 +18471,6 @@ def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: str_array : :class:`~pyspark.sql.Column` or str The comma-delimited list. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("ab", "abc,b,ab,c,def")], ['a', 'b']) @@ -18977,10 +18511,6 @@ def like( If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) @@ -19031,10 +18561,6 @@ def ilike( If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) @@ -19066,10 +18592,6 @@ def lcase(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.lower` @@ -19101,10 +18623,6 @@ def ucase(str: "ColumnOrName") -> Column: str : :class:`~pyspark.sql.Column` or str Input column or strings. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.upper` @@ -19139,14 +18657,6 @@ def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: len : :class:`~pyspark.sql.Column` or str Input column or strings, the leftmost `len`. - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.right` - Examples -------- >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) @@ -19171,14 +18681,6 @@ def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: len : :class:`~pyspark.sql.Column` or str Input column or strings, the rightmost `len`. - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.left` - Examples -------- >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) @@ -19261,16 +18763,6 @@ def collate(col: "ColumnOrName", collation: str) -> Column: ------- :class:`~pyspark.sql.Column` A new column of string type, where each value has the specified collation. - - See Also - -------- - :meth:`pyspark.sql.functions.collation` - - Examples - -------- - >>> df = spark.createDataFrame([("abc",)], ["c"]) - >>> df.select(sf.collate(df.c, 'SYSTEM.BUILTIN.UTF8_BINARY').alias('r')).collect() - [Row(r='abc')] """ from pyspark.sql.classic.column import _to_java_column @@ -21792,10 +21284,6 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column: :class:`~pyspark.sql.Column` string representation of given JSON object value. - See Also - -------- - :meth:`pyspark.sql.functions.parse_json` - Examples -------- Example 1: Extract a json object from json string @@ -21893,7 +21381,7 @@ def from_json( options: Optional[Mapping[str, str]] = None, ) -> Column: """ - Parses a column containing a JSON string into a :class:`MapType` with :class:`~StringType` + Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType` as keys type, :class:`StructType` or :class:`ArrayType` with the specified schema. Returns `null`, in the case of an unparsable string. @@ -22012,7 +21500,7 @@ def try_parse_json( col: "ColumnOrName", ) -> Column: """ - Parses a column containing a JSON string into a :class:`~VariantType`. Returns None if a string + Parses a column containing a JSON string into a :class:`VariantType`. Returns None if a string contains an invalid JSON value. .. versionadded:: 4.0.0 @@ -22027,10 +21515,6 @@ def try_parse_json( :class:`~pyspark.sql.Column` a new column of VariantType. - See Also - -------- - :meth:`pyspark.sql.functions.parse_json` - Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''}, {'json': '''{a : 1}'''} ]) @@ -22111,10 +21595,6 @@ def parse_json( :class:`~pyspark.sql.Column` a new column of VariantType. - See Also - -------- - :meth:`pyspark.sql.functions.try_parse_json` - Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) @@ -22513,10 +21993,6 @@ def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> :class:`~pyspark.sql.Column` a column of `targetType` representing the extracted result - See Also - -------- - :meth:`pyspark.sql.functions.try_variant_get` - Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) @@ -22564,10 +22040,6 @@ def try_variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str :class:`~pyspark.sql.Column` a column of `targetType` representing the extracted result - See Also - -------- - :meth:`pyspark.sql.functions.variant_get` - Examples -------- >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) @@ -23092,7 +22564,7 @@ def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]] @_try_remote_functions def to_xml(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - Converts a column containing a :class:`~StructType` into a XML string. + Converts a column containing a :class:`StructType` into a XML string. Throws an exception, in the case of an unsupported type. .. versionadded:: 4.0.0 @@ -23231,7 +22703,7 @@ def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]] @_try_remote_functions def to_csv(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - CSV Function: Converts a column containing a :class:`~StructType` into a CSV string. + CSV Function: Converts a column containing a :class:`StructType` into a CSV string. Throws an exception, in the case of an unsupported type. .. versionadded:: 3.0.0 @@ -23338,12 +22810,6 @@ def size(col: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` length of the array/map. - See Also - -------- - :meth:`pyspark.sql.functions.size` - :meth:`pyspark.sql.functions.array_size` - :meth:`pyspark.sql.functions.cardinality` - Examples -------- >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data']) @@ -23541,11 +23007,6 @@ def array_size(col: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new column that contains the size of each array. - See Also - -------- - :meth:`pyspark.sql.functions.size` - :meth:`pyspark.sql.functions.cardinality` - Examples -------- Example 1: Basic usage with integer array @@ -23628,11 +23089,6 @@ def cardinality(col: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` length of the array/map. - See Also - -------- - :meth:`pyspark.sql.functions.size` - :meth:`pyspark.sql.functions.array_size` - Examples -------- >>> import pyspark.sql.functions as sf @@ -25699,7 +25155,7 @@ def years(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the :class:`~DataFrameWriterV2`. + method of the `DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25743,7 +25199,7 @@ def months(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the :class:`~DataFrameWriterV2`. + method of the `DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25787,7 +25243,7 @@ def days(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the :class:`~DataFrameWriterV2`. + method of the `DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -25831,7 +25287,7 @@ def hours(col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the :class:`~DataFrameWriterV2`. + method of the `DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -26379,10 +25835,6 @@ def time_from_seconds(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name Seconds since midnight (0 to 86399.999999). - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -26409,10 +25861,6 @@ def time_from_millis(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name Milliseconds since midnight (0 to 86399999). - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -26439,10 +25887,6 @@ def time_from_micros(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name Microseconds since midnight (0 to 86399999999). - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -26469,10 +25913,6 @@ def time_to_seconds(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name TIME value to convert. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -26499,10 +25939,6 @@ def time_to_millis(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name TIME value to convert. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -26529,10 +25965,6 @@ def time_to_micros(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name TIME value to convert. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -27649,7 +27081,7 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: ----- This function can be used only in combination with :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the :class:`~DataFrameWriterV2`. + method of the `DataFrameWriterV2`. """ from pyspark.sql.functions import partitioning @@ -27679,35 +27111,28 @@ def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None The optional endianness of the output WKB, 'NDR' for little-endian (default) or 'XDR' for big-endian. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples - ------- - Example 1: Getting WKB from GEOGRAPHY. + -------- + Example 1: Getting WKB from GEOGRAPHY. >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] Example 2: Getting WKB from GEOMETRY. - >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] Example 3: Getting WKB (little-endian) from GEOGRAPHY. - >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb'), 'NDR'))).collect() [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] Example 4: Getting WKB (big-endian) from GEOMETRY. - >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb'), 'XDR'))).collect() @@ -27731,10 +27156,6 @@ def st_geogfromwkb(wkb: "ColumnOrName") -> Column: wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOGRAPHY value. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -27760,10 +27181,6 @@ def st_geomfromwkb( srid : :class:`~pyspark.sql.Column` or int, optional The optional SRID value of the geometry. Default is 0. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -27792,21 +27209,16 @@ def st_setsrid(geo: "ColumnOrName", srid: Union["ColumnOrName", int]) -> Column: srid : :class:`~pyspark.sql.Column` or int An INTEGER representing the new SRID of the geospatial value. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- - Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. + Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'), 4326)], ['wkb', 'srid']) # noqa >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geogfromwkb('wkb'), 'srid'))).collect() [Row(st_srid(st_setsrid(st_geogfromwkb(wkb), srid))=4326)] Example 2: Setting the SRID on GEOMETRY with SRID as an integer literal. - >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geomfromwkb('wkb'), 4326))).collect() @@ -27828,21 +27240,16 @@ def st_srid(geo: "ColumnOrName") -> Column: geo : :class:`~pyspark.sql.Column` or str A geospatial value, either a GEOGRAPHY or a GEOMETRY. - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- - Example 1: Getting the SRID of GEOGRAPHY. + Example 1: Getting the SRID of GEOGRAPHY. >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.st_srid(sf.st_geogfromwkb('wkb'))).collect() [Row(st_srid(st_geogfromwkb(wkb))=4326)] Example 2: Getting the SRID of GEOMETRY. - >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa >>> df.select(sf.st_srid(sf.st_geomfromwkb('wkb'))).collect() @@ -30523,14 +29930,6 @@ def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or str col2 : :class:`~pyspark.sql.Column` or str - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.nullif` - Examples -------- >>> import pyspark.sql.functions as sf @@ -30557,10 +29956,6 @@ def isnotnull(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -30597,10 +29992,6 @@ def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -30636,14 +30027,6 @@ def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.ifnull` - Examples -------- >>> import pyspark.sql.functions as sf @@ -30678,10 +30061,6 @@ def nullifzero(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> import pyspark.sql.functions as sf @@ -30717,15 +30096,6 @@ def nvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.nanvl` - :meth:`pyspark.sql.functions.nvl2` - Examples -------- >>> import pyspark.sql.functions as sf @@ -30762,15 +30132,6 @@ def nvl2(col1: "ColumnOrName", col2: "ColumnOrName", col3: "ColumnOrName") -> Co col2 : :class:`~pyspark.sql.Column` or column name col3 : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.nanvl` - :meth:`pyspark.sql.functions.nvl` - Examples -------- >>> import pyspark.sql.functions as sf @@ -30805,10 +30166,6 @@ def zeroifnull(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> import pyspark.sql.functions as sf @@ -31250,10 +30607,6 @@ def sha(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.sha1` @@ -31279,10 +30632,6 @@ def input_file_block_length() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.input_file_name` @@ -31316,10 +30665,6 @@ def input_file_block_start() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.input_file_name` @@ -31360,10 +30705,6 @@ def reflect(*cols: "ColumnOrName") -> Column: and the second element should be a Column representing literal string for the method name, and the remaining are input arguments (Columns or column names) to the Java method. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.java_method` @@ -31399,10 +30740,6 @@ def java_method(*cols: "ColumnOrName") -> Column: and the second element should be a Column representing literal string for the method name, and the remaining are input arguments (Columns or column names) to the Java method. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.reflect` @@ -31501,10 +30838,6 @@ def version() -> Column: .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -31529,10 +30862,6 @@ def typeof(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name - Returns - ------- - :class:`~pyspark.sql.Column` - Examples -------- >>> from pyspark.sql import functions as sf @@ -31615,10 +30944,6 @@ def bitmap_bit_position(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input column. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.bitmap_bucket_number` @@ -31652,10 +30977,6 @@ def bitmap_bucket_number(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input column. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31690,10 +31011,6 @@ def bitmap_construct_agg(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input column will most likely be bitmap_bit_position(). - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31730,10 +31047,6 @@ def bitmap_count(col: "ColumnOrName") -> Column: col : :class:`~pyspark.sql.Column` or column name The input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31763,15 +31076,6 @@ def bitmap_or_agg(col: "ColumnOrName") -> Column: .. versionadded:: 3.5.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). - - Returns - ------- - :class:`~pyspark.sql.Column` - See Also -------- :meth:`pyspark.sql.functions.bitmap_bit_position` @@ -31780,6 +31084,11 @@ def bitmap_or_agg(col: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.bitmap_count` :meth:`pyspark.sql.functions.bitmap_and_agg` + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The input column should be bitmaps created from bitmap_construct_agg(). + Examples -------- >>> from pyspark.sql import functions as sf @@ -31888,7 +31197,7 @@ def udf( returnType : :class:`pyspark.sql.types.DataType` or str, optional the return type of the user-defined function. The value can be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - Defaults to :class:`~StringType`. + Defaults to :class:`StringType`. useArrow : bool, optional whether to use Arrow to optimize the (de)serialization. When it is None, the Spark config "spark.sql.execution.pythonUDF.arrow.enabled" takes effect. @@ -32534,4 +31843,4 @@ def _test() -> None: if __name__ == "__main__": - _test() + _test() \ No newline at end of file From 34c8c513a641b8eadfecf5adffcc1dcbda86a16c Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Mon, 20 Jul 2026 23:31:13 +0300 Subject: [PATCH 6/8] Revert python\pyspark\sql\functions\builtin.py --- python/pyspark/sql/functions/builtin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 45eba0a51adf9..2c2345c1a58a4 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -31843,4 +31843,4 @@ def _test() -> None: if __name__ == "__main__": - _test() \ No newline at end of file + _test() From 579e2c08f1bf0b0ba9aee0e77a21e2f445bc6354 Mon Sep 17 00:00:00 2001 From: Artem Kozhin Date: Mon, 20 Jul 2026 23:35:32 +0300 Subject: [PATCH 7/8] Added a hyperlink to the Hints guide in DataFrame.hint() --- python/pyspark/sql/dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 766e91b5f4198..825922edb07f9 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1206,7 +1206,7 @@ def hint( :class:`DataFrame` Hinted DataFrame - .. note:: See also https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select-hints.html + .. note:: See also `Hints ` Examples -------- From 6baf78b011a81e331894a8f20de0a5440b2a0465 Mon Sep 17 00:00:00 2001 From: art000109 <59031761+art000109@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:36:39 +0300 Subject: [PATCH 8/8] Update python/pyspark/sql/dataframe.py Thank you Co-authored-by: Hyukjin Kwon --- python/pyspark/sql/dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 825922edb07f9..508e83f84d4b7 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1206,7 +1206,7 @@ def hint( :class:`DataFrame` Hinted DataFrame - .. note:: See also `Hints ` + .. note:: See also `Hints `_ Examples --------