diff --git a/python/docs/source/reference/pyspark.sql/core_classes.rst b/python/docs/source/reference/pyspark.sql/core_classes.rst index af38ef1c4f5ee..1ba4c62994b74 100644 --- a/python/docs/source/reference/pyspark.sql/core_classes.rst +++ b/python/docs/source/reference/pyspark.sql/core_classes.rst @@ -38,6 +38,9 @@ Core Classes DataFrameReader DataFrameWriter DataFrameWriterV2 + MergeIntoWriter + DataStreamReader + DataStreamWriter UDFRegistration UDTFRegistration udf.UserDefinedFunction diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py index 4b9ce84b3b7ab..5bdc665a15e7e 100644 --- a/python/pyspark/pandas/frame.py +++ b/python/pyspark/pandas/frame.py @@ -390,6 +390,19 @@ class DataFrame(Frame, Generic[T]): pandas-on-Spark DataFrame that corresponds to pandas DataFrame logically. This holds Spark DataFrame internally. + .. versionadded:: 3.2.0 + + .. versionchanged:: 3.4.0 + Since 3.4.0, it deals with `data` and `index` in this approach: + 1, when `data` is a distributed dataset (Internal DataFrame/Spark DataFrame/ + pandas-on-Spark DataFrame/pandas-on-Spark Series), it will first parallelize + the `index` if necessary, and then try to combine the `data` and `index`; + Note that if `data` and `index` doesn't have the same anchor, then + `compute.ops_on_diff_frames` should be turned on; + 2, when `data` is a local dataset (Pandas DataFrame/numpy ndarray/list/etc), + it will first collect the `index` to driver if necessary, and then apply + the `pandas.DataFrame(...)` creation internally; + :ivar _internal: an internal immutable Frame to manage metadata. :type _internal: InternalFrame @@ -409,17 +422,6 @@ class DataFrame(Frame, Generic[T]): copy : boolean, default False Copy data from inputs. Only affects DataFrame / 2d ndarray input - .. versionchanged:: 3.4.0 - Since 3.4.0, it deals with `data` and `index` in this approach: - 1, when `data` is a distributed dataset (Internal DataFrame/Spark DataFrame/ - pandas-on-Spark DataFrame/pandas-on-Spark Series), it will first parallelize - the `index` if necessary, and then try to combine the `data` and `index`; - Note that if `data` and `index` doesn't have the same anchor, then - `compute.ops_on_diff_frames` should be turned on; - 2, when `data` is a local dataset (Pandas DataFrame/numpy ndarray/list/etc), - it will first collect the `index` to driver if necessary, and then apply - the `pandas.DataFrame(...)` creation internally; - Examples -------- Constructing DataFrame from a dictionary. diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 6c4d32ea1797f..825922edb07f9 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1206,6 +1206,8 @@ def hint( :class:`DataFrame` Hinted DataFrame + .. note:: See also `Hints ` + Examples -------- >>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")], schema=["age", "name"]) @@ -6482,7 +6484,7 @@ def pandas_api( See Also -------- - pyspark.pandas.frame.DataFrame.to_spark + pyspark.pandas.DataFrame.to_spark Examples -------- diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py index b14b10b44859e..4b66998299964 100644 --- a/python/pyspark/sql/pandas/functions.py +++ b/python/pyspark/sql/pandas/functions.py @@ -74,7 +74,7 @@ def arrow_udf(f=None, returnType=None, functionType=None): the return type of the user-defined function. The value can be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. functionType : int, optional - an enum value in :class:`pyspark.sql.functions.ArrowUDFType`. + an enum value in :class:`~pyspark.sql.functions.ArrowUDFType`. Default: SCALAR. This parameter exists for compatibility. Using Python type hints is encouraged. diff --git a/python/pyspark/sql/tvf.py b/python/pyspark/sql/tvf.py index 90874578a3562..57b0f525675fd 100644 --- a/python/pyspark/sql/tvf.py +++ b/python/pyspark/sql/tvf.py @@ -59,7 +59,7 @@ def range( Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -101,7 +101,7 @@ def explode(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -185,7 +185,7 @@ def explode_outer(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -238,7 +238,7 @@ def inline(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -301,7 +301,7 @@ def inline_outer(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -344,7 +344,7 @@ def json_tuple(self, input: Column, *fields: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -394,7 +394,7 @@ def posexplode(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -437,7 +437,7 @@ def posexplode_outer(self, collection: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -490,7 +490,7 @@ def stack(self, n: Column, *fields: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` See Also -------- @@ -525,7 +525,7 @@ def collations(self) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -546,7 +546,7 @@ def sql_keywords(self) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -574,7 +574,7 @@ def variant_explode(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -639,7 +639,7 @@ def variant_explode_outer(self, input: Column) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples -------- @@ -681,7 +681,7 @@ def python_worker_logs(self) -> DataFrame: Returns ------- - :class:`DataFrame` + :class:`~pyspark.sql.DataFrame` Examples --------