Update docstrings so that cross references work in online docs. Also switch from autosummary to autoapi in sphinx for building API reference documents

Author: timsaucer

docs/source/api.rst

@@ -55,6 +55,7 @@
     "sphinx.ext.napoleon",
     "myst_parser",
     "IPython.sphinxext.ipython_directive",
+    "autoapi.extension",
 ]
 
 source_suffix = {
@@ -80,23 +81,36 @@
 
 autosummary_generate = True
 
+autoapi_dirs = ["../../python"]
+autoapi_ignore = ["*tests*"]
+autoapi_member_order = "groupwise"
+suppress_warnings = ["autoapi.python_import_resolution"]
+autoapi_keep_files = True
+autoapi_python_class_content = "both"
 
-def autodoc_skip_member(app, what, name, obj, skip, options):
-    exclude_functions = "__init__"
-    exclude_classes = ("Expr", "DataFrame")
 
-    class_name = ""
-    if hasattr(obj, "__qualname__"):
-        if obj.__qualname__ is not None:
-            class_name = obj.__qualname__.split(".")[0]
+def autoapi_skip_member_fn(app, what, name, obj, skip, options):
+    skip_contents = [
+        # Re-exports
+        ("class", "datafusion.DataFrame"),
+        ("class", "datafusion.SessionContext"),
+        ("module", "datafusion.common"),
+        # Deprecated
+        ("class", "datafusion.substrait.serde"),
+        ("class", "datafusion.substrait.plan"),
+        ("class", "datafusion.substrait.producer"),
+        ("class", "datafusion.substrait.consumer"),
+        ("method", "datafusion.context.SessionContext.tables"),
+        ("method", "datafusion.dataframe.DataFrame.unnest_column"),
+    ]
+    if (what, name) in skip_contents:
+        skip = True
 
-    should_exclude = name in exclude_functions and class_name in exclude_classes
+    return skip
 
-    return True if should_exclude else None
 
-
-def setup(app):
-    app.connect("autodoc-skip-member", autodoc_skip_member)
+def setup(sphinx):
+    sphinx.connect("autoapi-skip-member", autoapi_skip_member_fn)
 
 
 # -- Options for HTML output -------------------------------------------------
@@ -106,9 +120,7 @@ def setup(app):
 #
 html_theme = "pydata_sphinx_theme"
 
-html_theme_options = {
-    "use_edit_page_button": True,
-}
+html_theme_options = {"use_edit_page_button": False, "show_toc_level": 2}
 
 html_context = {
     "github_user": "apache",

docs/source/api/dataframe.rst

@@ -104,5 +104,3 @@ Example
    :hidden:
    :maxdepth: 1
    :caption: API
-
-   api

docs/source/api/execution_context.rst

@@ -15,6 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
+.. _user_guide_concepts:
+
 Concepts
 ========
 

docs/source/api/expression.rst

@@ -15,6 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
+.. _expressions:
+
 Expressions
 ===========