mdio-python/tests/unit/v1/test_dataset_builder_add_variable.py at 763c5c05ac79cf7d688580ae01927d84ebd2b99b · TGSAI/mdio-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
"""Tests the schema v1 dataset_builder.add_variable() public API."""

import pytest

from mdio.builder.dataset_builder import MDIODatasetBuilder
from mdio.builder.dataset_builder import _BuilderState
from mdio.builder.schemas.chunk_grid import RegularChunkGrid
from mdio.builder.schemas.chunk_grid import RegularChunkShape
from mdio.builder.schemas.compressors import Blosc
from mdio.builder.schemas.compressors import BloscCname
from mdio.builder.schemas.dtype import ScalarType
from mdio.builder.schemas.v1.stats import CenteredBinHistogram
from mdio.builder.schemas.v1.stats import SummaryStatistics
from mdio.builder.schemas.v1.units import LengthUnitEnum
from mdio.builder.schemas.v1.units import LengthUnitModel
from mdio.builder.schemas.v1.variable import CoordinateMetadata
from mdio.builder.schemas.v1.variable import VariableMetadata

from .helpers import validate_builder
from .helpers import validate_variable


def test_add_variable_no_coords() -> None:
    """Test adding variable. Check the state transition and validate required parameters.."""
    builder = MDIODatasetBuilder("test_dataset")
    validate_builder(builder, _BuilderState.INITIAL, n_dims=0, n_coords=0, n_var=0)

    # Validate: Must add at least one dimension before adding variables
    msg = "Must add at least one dimension before adding variables"
    with pytest.raises(ValueError, match=msg):
        builder.add_variable("amplitude", dimensions=("speed",), data_type=ScalarType.FLOAT32)

    # Add dimension before we can add a data variable
    builder.add_dimension("inline", 100)
    builder.add_dimension("crossline", 200)
    builder.add_dimension("depth", 300)

    # Validate: required parameters must be preset
    bad_name = None
    with pytest.raises(ValueError, match="'name' must be a non-empty string"):
        builder.add_variable(bad_name, dimensions=("speed",), data_type=ScalarType.FLOAT32)
    with pytest.raises(ValueError, match="'name' must be a non-empty string"):
        builder.add_variable("", dimensions=("speed",), data_type=ScalarType.FLOAT32)
    with pytest.raises(ValueError, match="'dimensions' must be a non-empty list"):
        builder.add_variable("bad_amplitude", dimensions=None, data_type=ScalarType.FLOAT32)
    with pytest.raises(ValueError, match="'dimensions' must be a non-empty list"):
        builder.add_variable("bad_amplitude", dimensions=(), data_type=ScalarType.FLOAT32)

    # Validate: Add a variable using non-existent dimensions is not allowed
    msg = "Pre-existing dimension named 'il' is not found"
    with pytest.raises(ValueError, match=msg):
        builder.add_variable("bad_amplitude", dimensions=("il", "xl", "depth"), data_type=ScalarType.FLOAT32)

    # Add a variable without coordinates
    builder.add_variable("amplitude", dimensions=("inline", "crossline", "depth"), data_type=ScalarType.FLOAT32)
    validate_builder(builder, _BuilderState.HAS_VARIABLES, n_dims=3, n_coords=0, n_var=1)
    validate_variable(
        builder,
        "amplitude",
        dims=[("inline", 100), ("crossline", 200), ("depth", 300)],
        coords=None,
        dtype=ScalarType.FLOAT32,
    )

    # Validate: adding a variable with the same name twice is not allowed
    msg = "Adding variable with the same name twice is not allowed"
    with pytest.raises(ValueError, match=msg):
        builder.add_variable("amplitude", dimensions=("inline", "crossline", "depth"), data_type=ScalarType.FLOAT32)


def test_add_variable_with_coords() -> None:
    """Test adding variable. Check the state transition and validate required parameters.."""
    builder = MDIODatasetBuilder("test_dataset")
    builder.add_dimension("inline", 100)
    builder.add_dimension("crossline", 200)
    builder.add_dimension("depth", 300)

    # Add dimension coordinates before we can add a data variable
    builder.add_coordinate("inline", dimensions=("inline",), data_type=ScalarType.UINT32)
    builder.add_coordinate("crossline", dimensions=("crossline",), data_type=ScalarType.UINT32)

    # Validate: adding a variable with a coordinate that has not been pre-created is not allowed
    msg = "Pre-existing coordinate named 'depth' is not found"
    with pytest.raises(ValueError, match=msg):
        builder.add_variable(
            "ampl",
            dimensions=("inline", "crossline", "depth"),
            coordinates=("inline", "crossline", "depth"),
            data_type=ScalarType.FLOAT32,
        )

    # Add a variable with pre-defined dimension coordinates
    builder.add_variable(
        "ampl",
        dimensions=("inline", "crossline", "depth"),
        coordinates=("inline", "crossline"),
        data_type=ScalarType.FLOAT32,
    )
    validate_builder(builder, _BuilderState.HAS_VARIABLES, n_dims=3, n_coords=2, n_var=3)
    validate_variable(
        builder,
        "ampl",
        dims=[("inline", 100), ("crossline", 200), ("depth", 300)],
        coords=["inline", "crossline"],
        dtype=ScalarType.FLOAT32,
    )

    # Add non-dim coordinates (e.g., 2D coordinates)
    builder.add_coordinate("cdp_x", dimensions=("inline", "crossline"), data_type=ScalarType.FLOAT32)
    builder.add_coordinate("cdp_y", dimensions=("inline", "crossline"), data_type=ScalarType.FLOAT32)

    # Add a variable with pre-defined dimension and non-dimension coordinates
    builder.add_variable(
        "ampl2",
        dimensions=("inline", "crossline", "depth"),
        coordinates=("inline", "crossline", "cdp_x", "cdp_y"),
        data_type=ScalarType.FLOAT32,
    )
    validate_builder(builder, _BuilderState.HAS_VARIABLES, n_dims=3, n_coords=4, n_var=6)
    validate_variable(
        builder,
        "ampl2",
        dims=[("inline", 100), ("crossline", 200), ("depth", 300)],
        coords=["inline", "crossline", "cdp_x", "cdp_y"],
        dtype=ScalarType.FLOAT32,
    )


def test_add_variable_with_defaults() -> None:
    """Test adding variable with default arguments."""
    builder = MDIODatasetBuilder("test_dataset")
    # Add dimensions before we can add a data variables
    builder.add_dimension("inline", 100)
    builder.add_dimension("crossline", 200)
    builder.add_dimension("depth", 300)
    # Add dimension coordinates
    depth_metadata = CoordinateMetadata(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))
    builder.add_coordinate("inline", dimensions=("inline",), data_type=ScalarType.UINT32)
    builder.add_coordinate("crossline", dimensions=("crossline",), data_type=ScalarType.UINT32)
    builder.add_coordinate("depth", dimensions=("depth",), data_type=ScalarType.UINT32, metadata=depth_metadata)

    # Add data variable using defaults
    builder.add_variable("ampl", dimensions=("inline", "crossline", "depth"), data_type=ScalarType.FLOAT32)
    validate_builder(builder, _BuilderState.HAS_VARIABLES, n_dims=3, n_coords=3, n_var=4)
    v = validate_variable(
        builder,
        "ampl",
        dims=[("inline", 100), ("crossline", 200), ("depth", 300)],
        coords=None,
        dtype=ScalarType.FLOAT32,
    )
    assert v.long_name is None  # Default value
    assert v.compressor is None  # Default value
    assert v.coordinates is None  # Default value
    assert v.metadata is None  # Default value


def test_add_variable_full_parameters() -> None:
    """Test adding variable with full parameters."""
    builder = MDIODatasetBuilder("test_dataset")
    # Add dimensions before we can add a data variables
    builder.add_dimension("inline", 100)
    builder.add_dimension("crossline", 200)
    builder.add_dimension("depth", 300)
    # Add dimension coordinates
    builder.add_coordinate("inline", dimensions=("inline",), data_type=ScalarType.UINT32)
    builder.add_coordinate("crossline", dimensions=("crossline",), data_type=ScalarType.UINT32)
    builder.add_coordinate("depth", dimensions=("depth",), data_type=ScalarType.UINT32)
    # Add coordinates before we can add a data variable
    builder.add_coordinate("cdp_x", dimensions=("inline", "crossline"), data_type=ScalarType.FLOAT64)
    builder.add_coordinate("cdp_y", dimensions=("inline", "crossline"), data_type=ScalarType.FLOAT64)

    # Add data variable with full parameters
    histogram = CenteredBinHistogram(bin_centers=[1, 2], counts=[10, 15])
    stats = SummaryStatistics(count=100, sum=1215.1, sum_squares=125.12, min=5.61, max=10.84, histogram=histogram)
    metadata = VariableMetadata(
        units_v1=LengthUnitModel(length=LengthUnitEnum.FOOT),
        attributes={"MGA": 51, "UnitSystem": "Imperial"},
        chunk_grid=RegularChunkGrid(configuration=RegularChunkShape(chunk_shape=(20,))),
        stats_v1=stats,
        crs="EPSG:32610",
        bin_inl=12.5,
        bin_xl=25.0,
    )
    builder.add_variable(
        "ampl",
        long_name="Amplitude (dimensionless)",
        dimensions=("inline", "crossline", "depth"),
        data_type=ScalarType.FLOAT32,
        compressor=Blosc(cname=BloscCname.zstd),
        coordinates=("inline", "crossline", "depth", "cdp_x", "cdp_y"),
        metadata=metadata,
    )
    validate_builder(builder, _BuilderState.HAS_VARIABLES, n_dims=3, n_coords=5, n_var=6)
    v = validate_variable(
        builder,
        "ampl",
        dims=[("inline", 100), ("crossline", 200), ("depth", 300)],
        coords=["inline", "crossline", "depth", "cdp_x", "cdp_y"],
        dtype=ScalarType.FLOAT32,
    )
    assert v.long_name == "Amplitude (dimensionless)"
    assert isinstance(v.compressor, Blosc)
    assert v.compressor.cname == BloscCname.zstd
    assert len(v.coordinates) == 5
    assert v.metadata.stats_v1.count == 100
    assert isinstance(v.metadata, VariableMetadata)
    assert v.metadata.units_v1.length == LengthUnitEnum.FOOT
    assert v.metadata.attributes["MGA"] == 51
    assert v.metadata.attributes["UnitSystem"] == "Imperial"
    assert v.metadata.chunk_grid.name == "regular"
    assert v.metadata.chunk_grid.configuration.chunk_shape == (20,)
    assert v.metadata.stats_v1.count == 100
    assert v.metadata.stats_v1.sum == 1215.1
    assert v.metadata.stats_v1.sum_squares == 125.12
    assert v.metadata.stats_v1.min == 5.61
    assert v.metadata.stats_v1.max == 10.84
    assert v.metadata.stats_v1.histogram.bin_centers == [1, 2]
    assert v.metadata.stats_v1.histogram.counts == [10, 15]
    assert v.metadata.crs == "EPSG:32610"
    assert v.metadata.bin_inl == 12.5
    assert v.metadata.bin_xl == 25.0