Refactoring: Initialize dlt_filesystem package inline

NOTICE

@@ -8,11 +8,11 @@ Copyright 2022-2025 ScaleVector
 Licensed under the Apache License, Version 2.0
 
 The following directories contain code from dlt-hub:
+- dlt_filesystem
 - omniload/source/airtable/
 - omniload/source/asana/
 - omniload/source/chess/
 - omniload/source/facebook_ads/
-- omniload/source/filesystem/
 - omniload/source/freshdesk/
 - omniload/source/github/
 - omniload/source/google_ads/

pyproject.toml

@@ -155,7 +155,7 @@ optional-dependencies.ibm-db = [
 # explicitly here (msgpack, cbor2, lxml, pyyaml). lxml and pyyaml are commonly present
 # transitively but are declared so the extra is self-contained. Note: iterabledata's import
 # package name is the generic `iterable`, which can shadow a same-named local package.
-# See omniload/source/filesystem/format/iterable_codec.py.
+# See dlt_filesystem/source/format/iterable_codec.py.
 optional-dependencies.iterable = [
   "cbor2<7",
   "iterabledata>=1.0.15,<2",
@@ -248,6 +248,18 @@ lint.extend-ignore = [
   # Probable insecure usage of temporary file or directory
   "S108",
 ]
+lint.per-file-ignores."src/dlt_filesystem/source/*" = [
+  # line-too-long
+  "E501",
+]
+lint.per-file-ignores."src/dlt_filesystem/target/{local.py,util.py}" = [
+  # line-too-long
+  "E501",
+]
+lint.per-file-ignores."src/dlt_filesystem/testing/{stub.py}" = [
+  # line-too-long
+  "E501",
+]
 lint.per-file-ignores."src/omniload/api.py" = [
   # line-too-long
   "E501",

src/dlt_filesystem/error.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+
+class MissingConnectorOption(Exception):
+    def __init__(self, option, connector):
+        super().__init__(f"{option} is required to connect to {connector}")
+
+
+class InvalidBlobTableError(Exception):
+    def __init__(self, source):
+        super().__init__(
+            f"Invalid source table for {source} "
+            "Ensure that the table is in the format {bucket-name}/{file glob}"
+        )

src/omniload/source/filesystem/adapter.py → src/dlt_filesystem/source/adapter.py

@@ -23,7 +23,7 @@
 from dlt.sources.filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files
 from fsspec import AbstractFileSystem
 
-from omniload.source.filesystem.format.readers import (
+from dlt_filesystem.source.format.readers import (
     ReadersSource,
     read_bson,
     read_cbor,

src/omniload/source/filesystem/api.py → src/dlt_filesystem/source/api.py

@@ -1,5 +1,5 @@
-from omniload.source.filesystem.impl.local import LocalFilesystemSource
-from omniload.source.filesystem.impl.remote import (
+from dlt_filesystem.source.impl.local import LocalFilesystemSource
+from dlt_filesystem.source.impl.remote import (
     AzureSource,
     GCSSource,
     S3Source,

src/omniload/source/filesystem/format/helpers.py → src/dlt_filesystem/source/format/helpers.py

@@ -18,7 +18,7 @@
 
 from dlt.common.typing import TDataItem
 
-from omniload.source.filesystem.format.settings import DEFAULT_CHUNK_SIZE
+from dlt_filesystem.source.format.settings import DEFAULT_CHUNK_SIZE
 
 
 def add_columns(columns: List[str], rows: List[List[Any]]) -> List[Dict[str, Any]]:

src/omniload/source/filesystem/format/iterable_codec.py → src/dlt_filesystem/source/format/iterable_codec.py

@@ -9,7 +9,7 @@
 
 - **Streaming via `iterabledata`** (e.g. MessagePack). `iterabledata` (PyPI ``iterabledata``,
   import package ``iterable``) exposes a uniform per-format class that takes a ``stream=`` file
-  object and yields record dicts via ``read_bulk(n)``. omniload feeds it the already-
+  object and yields record dicts via ``read_bulk(n)``. `dlt-filesystem` feeds it the already-
   authenticated ``file_obj.open()`` handle, so remote sources keep flowing through dlt's fsspec
   layer and iterabledata's own cloud-storage / credential path is never touched. Note a
   streaming wire format like MessagePack carries no length prefix, so a truncated tail reads as
@@ -18,7 +18,7 @@
 - **Whole-file direct decode** (e.g. CBOR). Some formats are whole-file rather than streaming,
   and iterabledata's wrapper for them wraps its decode in a bare ``except Exception`` that
   yields nothing, so a truncated / corrupt file would load as zero rows with **no error**
-  (silent data loss). For those, omniload decodes the bytes with the format's own library
+  (silent data loss). For those, `dlt-filesystem` decodes the bytes with the format's own library
   directly (``eager_decoder``) so decode errors propagate. This also matches the project's
   per-format routing policy: route to iterabledata only where it is the better path.
 
@@ -53,7 +53,7 @@
 from dlt.common.utils import map_nested_values_in_place
 from dlt.sources.filesystem import FileItemDict
 
-from omniload.source.filesystem.error import (
+from dlt_filesystem.source.error import (
     MissingDecoderError,
     MissingReaderOptionError,
 )
@@ -349,9 +349,10 @@ def _xml_eager_decode(data: bytes, options: dict) -> Iterator[Any]:
 
 # Lexically sorted by format name. XML and YAML both use the whole-file `eager_decoder` seam
 # (never an iterabledata class): iterabledata's XML parser resolves entities and can't be locked
-# down through its API, and its YAML wrapper is eager and swallows parse errors, so omniload owns
-# both decodes -- a safe lxml parse for XML, `yaml.safe_load_all` for YAML. See
-# `docs/getting-started/file-format-routing.md`.
+# down through its API, and its YAML wrapper is eager and swallows parse errors, so
+# `dlt-filesystem` owns both decodes -- a safe lxml parse for XML, `yaml.safe_load_all` for YAML.
+# See `docs/getting-started/file-format-routing.md`.
+# TODO: Adjust `pip_hint` values after breaking out into dedicated package.
 FORMAT_TO_ITERABLE: dict[str, IterableFormat] = {
     "cbor": IterableFormat(
         decoder_dist="cbor2",

src/omniload/source/filesystem/format/readers.py → src/dlt_filesystem/source/format/readers.py

@@ -19,10 +19,10 @@
 from dlt.sources import DltResource, DltSource, TDataItems
 from dlt.sources.filesystem import FileItemDict
 
-from omniload.codec.python import cast_kwargs_to_signature
-from omniload.source.filesystem.format.helpers import fetch_arrow, fetch_json
-from omniload.source.filesystem.format.iterable_codec import read_via_iterable
-from omniload.source.filesystem.format.settings import DEFAULT_CHUNK_SIZE
+from dlt_filesystem.source.format.helpers import fetch_arrow, fetch_json
+from dlt_filesystem.source.format.iterable_codec import read_via_iterable
+from dlt_filesystem.source.format.settings import DEFAULT_CHUNK_SIZE
+from dlt_filesystem.util.python import cast_kwargs_to_signature
 
 
 def _polars_csv_symbols() -> Dict[str, Any]:
@@ -346,7 +346,7 @@ def read_bson(
     import bson
     from dlt.common.utils import map_nested_values_in_place
 
-    from omniload.source.filesystem.format.bson_codec import convert_bson_objs
+    from dlt_filesystem.source.format.bson_codec import convert_bson_objs
 
     for file_obj in items:
         with file_obj.open() as f:

src/omniload/source/filesystem/format/registry.py → src/dlt_filesystem/source/format/registry.py

@@ -1,4 +1,4 @@
-from omniload.source.filesystem.error import UnsupportedEndpointError
+from dlt_filesystem.source.error import UnsupportedEndpointError
 
 # Formats whose reader ships with the base install (core dependencies).
 BASE_FILE_FORMATS: dict[str, str] = {
@@ -48,7 +48,7 @@ def advertised_file_formats() -> tuple[str, ...]:
     importable, so a base install doesn't advertise a format that would fail with an install
     hint (the reader still routes such a format and raises that hint if it is used).
     """
-    from omniload.source.filesystem.format.iterable_codec import (
+    from dlt_filesystem.source.format.iterable_codec import (
         installed_iterable_formats,
     )
 

src/omniload/source/filesystem/impl/local.py → src/dlt_filesystem/source/impl/local.py

@@ -1,15 +1,15 @@
 import os
 
-from omniload.error import MissingValueError
-from omniload.source.filesystem.base import FilesystemSource
-from omniload.source.filesystem.error import UnsupportedEndpointError
-from omniload.source.filesystem.format.registry import supported_file_format_message
-from omniload.source.filesystem.impl.util import (
+from dlt_filesystem.error import MissingConnectorOption
+from dlt_filesystem.source.base import FilesystemSource
+from dlt_filesystem.source.error import UnsupportedEndpointError
+from dlt_filesystem.source.format.registry import supported_file_format_message
+from dlt_filesystem.source.impl.util import (
     _is_absolute_local,
     _split_dir_glob,
     _url_path_to_local,
 )
-from omniload.source.filesystem.router import (
+from dlt_filesystem.source.router import (
     determine_endpoint,
     parse_fragment,
 )
@@ -50,7 +50,7 @@ def dlt_source(self, uri: str, table: str, **kwargs):
         if not spec:
             spec = table.strip()
         if not spec:
-            raise MissingValueError("path", "file URI")
+            raise MissingConnectorOption("path", "file URI")
 
         # Strip the trailing #fragment (format hint and/or #key=value reader
         # hints) before splitting into dir/glob, so file://feed.dat#csv and
@@ -79,8 +79,8 @@ def dlt_source(self, uri: str, table: str, **kwargs):
 
         fs = fsspec.filesystem("file")
 
-        from omniload.source.filesystem.adapter import resource_for_reader
-        from omniload.source.filesystem.model import FilesystemReference
+        from dlt_filesystem.source.adapter import resource_for_reader
+        from dlt_filesystem.source.model import FilesystemReference
 
         # Pass the plain absolute directory (not a hand-built file:// URL). dlt's
         # glob_files routes a local path through make_file_url/make_local_path, which is

src/omniload/source/filesystem/impl/remote.py → src/dlt_filesystem/source/impl/remote.py

@@ -3,17 +3,17 @@
 from typing import Any, Dict
 from urllib.parse import parse_qs, urlparse
 
-from omniload.error import InvalidBlobTableError, MissingValueError
-from omniload.source.filesystem.base import FilesystemSource
-from omniload.source.filesystem.error import UnsupportedEndpointError
-from omniload.source.filesystem.format.registry import supported_file_format_message
-from omniload.source.filesystem.router import (
+from dlt_filesystem.error import InvalidBlobTableError, MissingConnectorOption
+from dlt_filesystem.source.base import FilesystemSource
+from dlt_filesystem.source.error import UnsupportedEndpointError
+from dlt_filesystem.source.format.registry import supported_file_format_message
+from dlt_filesystem.source.router import (
     blob_hints,
     determine_endpoint,
     parse_fragment,
     parse_uri,
 )
-from omniload.util.auth import AzureBlobAuth, parse_azure_blob_auth
+from dlt_filesystem.util.auth import AzureBlobAuth, parse_azure_blob_auth
 
 
 class GCSSource(FilesystemSource):
@@ -35,21 +35,23 @@ def dlt_source(self, uri: str, table: str, **kwargs):
         credentials_path = params.get("credentials_path")
         credentials_base64 = params.get("credentials_base64")
         credentials_available = any(
-            map(
+            map(  # noqa: C417
                 lambda x: x is not None,
                 [credentials_path, credentials_base64],
             )
         )
         if credentials_available is False:
-            raise MissingValueError("credentials_path or credentials_base64", "GCS")
+            raise MissingConnectorOption(
+                "credentials_path or credentials_base64", "GCS"
+            )
 
         credentials = None
         if credentials_path:
             credentials = credentials_path[0]
         else:
             credentials = json.loads(base64.b64decode(credentials_base64[0]).decode())  # type: ignore
 
-        # There's a compatiblity issue between google-auth, dlt and gcsfs
+        # There's a compatibility issue between google-auth, dlt and gcsfs
         # that makes it difficult to use google.oauth2.service_account.Credentials
         # (The RECOMMENDED way of passing service account credentials)
         # directly with gcsfs. As a workaround, we construct the GCSFileSystem
@@ -69,8 +71,8 @@ def dlt_source(self, uri: str, table: str, **kwargs):
                 f"Failed to parse endpoint from path: {path_to_file}"
             ) from e
 
-        from omniload.source.filesystem.adapter import resource_for_reader
-        from omniload.source.filesystem.model import FilesystemReference
+        from dlt_filesystem.source.adapter import resource_for_reader
+        from dlt_filesystem.source.model import FilesystemReference
 
         return resource_for_reader(
             FilesystemReference(
@@ -128,8 +130,8 @@ def dlt_source(self, uri: str, table: str, **kwargs):
                 f"Failed to parse endpoint from path: {path_to_file}"
             ) from e
 
-        from omniload.source.filesystem.adapter import resource_for_reader
-        from omniload.source.filesystem.model import FilesystemReference
+        from dlt_filesystem.source.adapter import resource_for_reader
+        from dlt_filesystem.source.model import FilesystemReference
 
         return resource_for_reader(
             FilesystemReference(
@@ -211,8 +213,8 @@ def dlt_source(self, uri: str, table: str, **kwargs):
                 f"Failed to parse endpoint from path: {path_to_file}"
             ) from e
 
-        from omniload.source.filesystem.adapter import resource_for_reader
-        from omniload.source.filesystem.model import FilesystemReference
+        from dlt_filesystem.source.adapter import resource_for_reader
+        from dlt_filesystem.source.model import FilesystemReference
 
         return resource_for_reader(
             FilesystemReference(
@@ -231,7 +233,7 @@ def dlt_source(self, uri: str, table: str, **kwargs):
         parsed_uri = urlparse(uri)
         host = parsed_uri.hostname
         if not host:
-            raise MissingValueError("host", "SFTP URI")
+            raise MissingConnectorOption("host", "SFTP URI")
         port = parsed_uri.port or 22
         username = parsed_uri.username
         password = parsed_uri.password
@@ -268,8 +270,8 @@ def dlt_source(self, uri: str, table: str, **kwargs):
         except Exception as e:
             raise ValueError(f"Failed to parse endpoint from path: {table}") from e
 
-        from omniload.source.filesystem.adapter import resource_for_reader
-        from omniload.source.filesystem.model import FilesystemReference
+        from dlt_filesystem.source.adapter import resource_for_reader
+        from dlt_filesystem.source.model import FilesystemReference
 
         return resource_for_reader(
             FilesystemReference(

src/omniload/source/filesystem/router.py → src/dlt_filesystem/source/router.py

@@ -9,7 +9,7 @@
 from dlt.extract import DltResource
 from fsspec import AbstractFileSystem
 
-from omniload.source.filesystem.format.registry import (
+from dlt_filesystem.source.format.registry import (
     FORMAT_TO_READER,
     reader_for_format,
 )

src/omniload/target/filesystem/api.py → src/dlt_filesystem/target/api.py

@@ -1,5 +1,5 @@
-from omniload.target.filesystem.local import LocalFilesystemDestination
-from omniload.target.filesystem.remote import (
+from dlt_filesystem.target.local import LocalFilesystemDestination
+from dlt_filesystem.target.remote import (
     AzureDestination,
     GCSDestination,
     S3Destination,

src/omniload/target/filesystem/local.py → src/dlt_filesystem/target/local.py

@@ -1,6 +1,6 @@
 """Write CSV / JSONL / Parquet files to the local filesystem via ``file://``.
 
-This is the write-side twin of ``omniload.source.filesystem`` ``LocalFilesystemSource``
+This is the write-side twin of ``dlt_filesystem.source`` ``LocalFilesystemSource``
 (the ``file://`` source). It mirrors that URI grammar exactly: everything after
 ``file://`` is a filesystem path (never an RFC-8089 host), relative paths resolve
 against the working directory, and the output format is taken from the file
@@ -21,10 +21,10 @@
 import tempfile
 from pathlib import Path
 
-from omniload.target.filesystem.registry import writer_for_format
-from omniload.target.filesystem.util import _resolve_output_target, _strip_dlt_columns
-from omniload.target.model import DEFAULT_DATASET_NAME
-from omniload.util.loader import load_dlt_file
+from dlt_filesystem.target.model import DEFAULT_DATASET_NAME
+from dlt_filesystem.target.registry import writer_for_format
+from dlt_filesystem.target.util import _resolve_output_target, _strip_dlt_columns
+from dlt_filesystem.util.loader import load_dlt_file
 
 
 class LocalFilesystemDestination:

src/dlt_filesystem/target/model.py

@@ -0,0 +1 @@
+DEFAULT_DATASET_NAME = "public"

src/omniload/target/filesystem/registry.py → src/dlt_filesystem/target/registry.py

@@ -1,8 +1,9 @@
-# csv_headless is a read-only concept (parsing a header-less CSV); writing always emits a
-# header, so the write side supports the plain-format subset of FORMAT_TO_READER.
+# csv_headless is a read-only concept (parsing a header-less CSV);
+# writing always emits a header, so the write side supports the
+# plain-format subset of FORMAT_TO_READER.
 from typing import Callable
 
-from omniload.target.filesystem.writer import write_csv, write_jsonl, write_parquet
+from dlt_filesystem.target.writer import write_csv, write_jsonl, write_parquet
 
 FORMAT_TO_WRITER: dict[str, Callable[[str, list[dict]], None]] = {
     "csv": write_csv,