diff --git a/.github/workflows/CDA-testing.yml b/.github/workflows/CDA-testing.yml index c66bbe4b..00d85e81 100644 --- a/.github/workflows/CDA-testing.yml +++ b/.github/workflows/CDA-testing.yml @@ -11,6 +11,11 @@ jobs: integration-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.13'] + steps: - uses: actions/checkout@v6 @@ -29,7 +34,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v6 with: - python-version: '3.9.X' + python-version: ${{ matrix.python-version }} # Use actions-poetry to handle installation - name: Install Poetry and Dependencies @@ -45,7 +50,8 @@ jobs: id: cache-poetry-venv with: path: .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-py${{ matrix.python-version }}-poetry-${{ hashFiles('poetry.lock') + }} # Install dependencies only if cache is missed - name: Install dependencies diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e51f9129..c2cf089f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -14,7 +14,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v6 with: - python-version: '3.x' + python-version: '3.13' # Unlike the code-check workflow, this job requires the dev dependencies to be # installed to make sure we have the necessary, tools, stub files, etc. diff --git a/compose_files/sql/users.sql b/compose_files/sql/users.sql index bf009138..f9368274 100644 --- a/compose_files/sql/users.sql +++ b/compose_files/sql/users.sql @@ -1,58 +1,56 @@ set define on define OFFICE_EROC=&1 +declare + type office_list_t is table of varchar2(16); + + procedure add_full_admin(p_user varchar2, p_offices office_list_t) is + begin + for i in 1 .. p_offices.count loop + cwms_sec.add_cwms_user(p_user, NULL, p_offices(i)); + cwms_sec.add_user_to_group(p_user, 'All Users', p_offices(i)); + cwms_sec.add_user_to_group(p_user, 'CWMS Users', p_offices(i)); + cwms_sec.add_user_to_group(p_user, 'TS ID Creator', p_offices(i)); + cwms_sec.add_user_to_group(p_user, 'CWMS User Admins', p_offices(i)); + cwms_sec.add_user_to_group(p_user, 'CWMS PD Users', p_offices(i)); + end loop; + end; begin cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','All Users', 'HQ'); cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','All Users', 'SPK'); + cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','All Users', 'MVP'); + cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','All Users', 'LRL'); cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','CWMS Users', 'HQ'); cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','CWMS User Admins', 'HQ'); - + cwms_sec.add_user_to_group('&&OFFICE_EROC.webtest','CWMS PD Users', 'HQ'); - cwms_sec.add_cwms_user('l2hectest',NULL,'SPK'); - cwms_sec.update_edipi('l2hectest',1234567890); - cwms_sec.add_user_to_group('l2hectest','All Users', 'SPK'); - cwms_sec.add_user_to_group('l2hectest','CWMS Users', 'SPK'); - cwms_sec.add_user_to_group('l2hectest','TS ID Creator','SPK'); - cwms_sec.add_user_to_group('l2hectest','CWMS User Admins', 'SPK'); + add_full_admin('l2hectest', office_list_t('SPK')); + cwms_sec.update_edipi('l2hectest', 1234567890); - cwms_sec.add_cwms_user('l1hectest',NULL,'SPL'); + cwms_sec.add_cwms_user('l1hectest', NULL, 'SPL'); -- intentionally no extra permissions. - --cwms_sec.add_user_to_group('l2hectest','CWMS Users', 'SPL'); - cwms_sec.add_cwms_user('m5hectest',NULL,'SWT'); + cwms_sec.add_cwms_user('m5hectest', NULL, 'SWT'); cwms_sec.add_user_to_group('m5hectest','All Users', 'SWT'); cwms_sec.add_user_to_group('m5hectest','CWMS Users', 'SWT'); execute immediate 'grant execute on cwms_20.cwms_upass to web_user'; - cwms_sec.add_cwms_user('m5testadmin', NULL, 'LRL'); cwms_sec.add_user_to_group('m5testadmin','All Users', 'LRL'); cwms_sec.add_user_to_group('m5testadmin','CWMS Users', 'LRL'); cwms_sec.add_user_to_group('m5testadmin','CWMS User Admins', 'LRL'); + cwms_sec.add_user_to_group('m5testadmin','CWMS PD Users', 'LRL'); + + add_full_admin('q0hectest', office_list_t('LRL', 'SPK', 'MVP')); - cwms_sec.add_cwms_user('q0hectest', NULL, 'LRL'); - cwms_sec.add_user_to_group('q0hectest','All Users', 'LRL'); - cwms_sec.add_user_to_group('q0hectest','CWMS Users', 'LRL'); - cwms_sec.add_user_to_group('q0hectest','TS ID Creator','LRL'); - cwms_sec.add_user_to_group('q0hectest','CWMS User Admins', 'LRL'); - - cwms_sec.add_cwms_user('q0hectest', NULL, 'SPK'); - cwms_sec.add_user_to_group('q0hectest','All Users', 'SPK'); - cwms_sec.add_user_to_group('q0hectest','CWMS Users', 'SPK'); - cwms_sec.add_user_to_group('q0hectest','TS ID Creator','SPK'); - cwms_sec.add_user_to_group('q0hectest','CWMS User Admins', 'SPK'); - - cwms_sec.add_cwms_user('q0hectest', NULL, 'MVP'); - cwms_sec.add_user_to_group('q0hectest','All Users', 'MVP'); - cwms_sec.add_user_to_group('q0hectest','CWMS Users', 'MVP'); - cwms_sec.add_user_to_group('q0hectest','TS ID Creator','MVP'); - cwms_sec.add_user_to_group('q0hectest','CWMS User Admins', 'MVP'); - - insert into cwms_20.at_api_keys (userid, key_name, apikey) values ('Q0HECTEST', 'testkey', '0123456789abcdef0123456789abcdef'); - insert into cwms_20.at_api_keys (userid, key_name, apikey) values ('L2HECTEST', 'testkey2', '1234567890abcdef1234567890abcdef'); + insert into cwms_20.at_api_keys (userid, key_name, apikey, created, expires) values ('Q0HECTEST', 'testkey', 'ak1_CYflBX6c$argon2id$v=19$m=19456,t=2,p=1$8Wh8X9m+O81UvrbCJ/eOFQ$+E0Rp3jhjduIHxaqmzx+OLR43B3HdcMuDyn8cO5/69s', sysdate, sysdate + 365); + -- key is ak1_CYflBX6cQOHlJkkcsA6NPvJ7npm1kynzfUsa45ncIPcGNewkcvK2ounQN8MaDj8Wkc8o0HiZvLkETpGrGkl3OvJD9Nt0vQCIPLBeqQiLGBQHsPDZmk1gEkVCzubSyfKy31bagcf0jrajn6zCcRAv1tpMpnucFCkUwCpTYwNCfCnPkqukNVpOyTv7I2II8NIxBQmQOZPc09yOrKPkQpj1sHM4NNxIcUfTZrPpidT1QGjhfVaaWW1AiqodkxXPxlTqvuRLz9bL + + insert into cwms_20.at_api_keys (userid, key_name, apikey, created, expires) values ('L2HECTEST', 'testkey2', 'ak1_3rF3RXlB$argon2id$v=19$m=19456,t=2,p=1$xxL2ItUkn3gC1LT5F8Wb0g$2az8A0GpJVVhbaccD4ICVWvnM2uoKzU652r9jemE9qg', sysdate, sysdate + 365); + -- key is ak1_3rF3RXlBRpWiWNdBhrX5LwNnbwyPX7J9MDro8b3aoRVp2bM4FRZPsOZOoFtjVuwt9bNnniWqzIaCfTzKNUqdp43ItSk7oFLdqf05gVhM5UtaLZa2BN7KNdb7hSxYu6FQJkt8haSQK3swUaC9qlRFLIPMerbjxxIF8UnuQ7Oe54uyiN7JJaaHErI0m7qo7ir2bkxHvC0aWw9UkT9Z8RKfWeaQBIizqZnicqmXgsekLvqwkZ2jJrLUw180aFr5g7rCEsRtVtRE -- Non-admin API key for the L1 with reduced permissions -- Used by CDA user-management tests that verify 403 handling - insert into cwms_20.at_api_keys (userid, key_name, apikey) values ('L1HECTEST', 'non_admin_test_key', 'fedcba9876543210fedcba9876543210'); - + insert into cwms_20.at_api_keys (userid, key_name, apikey, created, expires) values ('L1HECTEST', 'non_admin_test_key', 'ak1_SZUNxN3n$argon2id$v=19$m=19456,t=2,p=1$KAlZGdgEboEHvEVcpNGD0g$pPCCkQfOx8v5HNpwPadJNUHGI0I40a7HgcZ7JsTE6T0', sysdate, sysdate + 365); + -- key is ak1_SZUNxN3nDx0NpUfOJxpOuGqbqRKdYjbx86x6YVISLb9DiBi3Io5o6T6UFvkHknjIRnRO6oQfA1q6rP4XRDYMH9Hlr4ndffL6NjxPUaBZLSnqukV0uGuZKOUWBB04L5SyloJniOHkFe6ymvB9tzeziGYwzrDv3k6lzacG9vftHkCHB1QbjwwCC0sDkFvuwCe9qnyx5us11qL0YAfKXhe0fBCA2TmNDz8WXfw1HfBnAKx6WD7KqHngplWu4miOvkNverxFmAdJ end; / quit; diff --git a/tests/cda/conftest.py b/tests/cda/conftest.py index 8a8d00b6..c03288cf 100644 --- a/tests/cda/conftest.py +++ b/tests/cda/conftest.py @@ -10,7 +10,10 @@ def pytest_addoption(parser): parser.addoption( "--api_key", action="store", - default=os.getenv("CDA_API_KEY", "0123456789abcdef0123456789abcdef"), + default=os.getenv( + "CDA_API_KEY", + "ak1_CYflBX6cQOHlJkkcsA6NPvJ7npm1kynzfUsa45ncIPcGNewkcvK2ounQN8MaDj8Wkc8o0HiZvLkETpGrGkl3OvJD9Nt0vQCIPLBeqQiLGBQHsPDZmk1gEkVCzubSyfKy31bagcf0jrajn6zCcRAv1tpMpnucFCkUwCpTYwNCfCnPkqukNVpOyTv7I2II8NIxBQmQOZPc09yOrKPkQpj1sHM4NNxIcUfTZrPpidT1QGjhfVaaWW1AiqodkxXPxlTqvuRLz9bL", + ), help="Set a custom API key for the CWMS API", ) parser.addoption( @@ -22,14 +25,18 @@ def pytest_addoption(parser): parser.addoption( "--user_admin_api_key", action="store", - default=os.getenv("CDA_USER_ADMIN_API_KEY", "0123456789abcdef0123456789abcdef"), + default=os.getenv( + "CDA_USER_ADMIN_API_KEY", + "ak1_CYflBX6cQOHlJkkcsA6NPvJ7npm1kynzfUsa45ncIPcGNewkcvK2ounQN8MaDj8Wkc8o0HiZvLkETpGrGkl3OvJD9Nt0vQCIPLBeqQiLGBQHsPDZmk1gEkVCzubSyfKy31bagcf0jrajn6zCcRAv1tpMpnucFCkUwCpTYwNCfCnPkqukNVpOyTv7I2II8NIxBQmQOZPc09yOrKPkQpj1sHM4NNxIcUfTZrPpidT1QGjhfVaaWW1AiqodkxXPxlTqvuRLz9bL", + ), help="Set an admin API key for CDA user-management tests", ) parser.addoption( "--user_non_admin_api_key", action="store", default=os.getenv( - "CDA_USER_NON_ADMIN_API_KEY", "fedcba9876543210fedcba9876543210" + "CDA_USER_NON_ADMIN_API_KEY", + "ak1_SZUNxN3nDx0NpUfOJxpOuGqbqRKdYjbx86x6YVISLb9DiBi3Io5o6T6UFvkHknjIRnRO6oQfA1q6rP4XRDYMH9Hlr4ndffL6NjxPUaBZLSnqukV0uGuZKOUWBB04L5SyloJniOHkFe6ymvB9tzeziGYwzrDv3k6lzacG9vftHkCHB1QbjwwCC0sDkFvuwCe9qnyx5us11qL0YAfKXhe0fBCA2TmNDz8WXfw1HfBnAKx6WD7KqHngplWu4miOvkNverxFmAdJ", ), help="Set a non-admin API key for CDA user authorization tests", ) diff --git a/tests/cda/timeseries/timeseries_CDA_test.py b/tests/cda/timeseries/timeseries_CDA_test.py index c267e06b..83abc050 100644 --- a/tests/cda/timeseries/timeseries_CDA_test.py +++ b/tests/cda/timeseries/timeseries_CDA_test.py @@ -92,8 +92,25 @@ END = DT + timedelta(minutes=5) +def _cleanup_tsids_and_location(): + for ts_id in TSIDS: + try: + cwms.delete_timeseries_identifier( + ts_id=ts_id, office_id=TEST_OFFICE, delete_method="DELETE_ALL" + ) + except Exception as e: + print(f"Failed to delete tsid {ts_id}: {e}") + try: + cwms.delete_location(TEST_LOCATION_ID, TEST_OFFICE, cascade_delete=True) + except Exception as e: + print(f"Failed to delete location {TEST_LOCATION_ID}: {e}") + + @pytest.fixture(scope="module", autouse=True) def setup_data(): + # Clean up any leftover state from a prior aborted run before starting. + _cleanup_tsids_and_location() + location = { "name": TEST_LOCATION_ID, "latitude": 40.0, @@ -111,25 +128,24 @@ def setup_data(): } cwms.store_location(location) - ts_json = { - "name": TEST_TSID_DELETE, - "office-id": TEST_OFFICE, - "units": "ft", - "values": [[EPOCH_MS, 99, 0]], - } - ts.store_timeseries(ts_json) - yield - for ts_id in TSIDS: - try: - cwms.delete_timeseries_identifier( - ts_id=ts_id, office_id=TEST_OFFICE, delete_method="DELETE_ALL" - ) - except Exception as e: - print(f"Failed to delete tsid {ts_id}: {e}") + ts.store_timeseries( + { + "name": TEST_TSID_DELETE, + "office-id": TEST_OFFICE, + "units": "ft", + "values": [[EPOCH_MS, 99, 0]], + } + ) + # Warmup GET against /cwms-data/timeseries. The first GET on this + # endpoint after a cold connection intermittently trips a CDA bug + # (SESSION_OFFICE_ID_NOT_SET → ORA-01422 for multi-office API-key + # users). Absorbing the failure here keeps individual tests clean. try: - cwms.delete_location(TEST_LOCATION_ID, TEST_OFFICE, cascade_delete=True) + ts.get_timeseries(TEST_TSID_DELETE, TEST_OFFICE, begin=BEGIN, end=END) except Exception as e: - print(f"Failed to delete location {TEST_LOCATION_ID}: {e}") + print(f"Warmup GET failed (expected on cold CDA connection): {e}") + yield + _cleanup_tsids_and_location() @pytest.fixture(autouse=True) diff --git a/tests/mock/ratings/rating_spec_test.py b/tests/mock/ratings/rating_spec_test.py deleted file mode 100644 index 35424825..00000000 --- a/tests/mock/ratings/rating_spec_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2024 -# United States Army Corps of Engineers - Hydrologic Engineering Center (USACE/HEC) -# All Rights Reserved. USACE PROPRIETARY/CONFIDENTIAL. -# Source may not be released without written approval from HEC - -import pandas as pd -import pytest -import pytz - -import cwms.api -import cwms.ratings.ratings_spec as ratings_spec -from tests._test_utils import read_resource_file - -_MOCK_ROOT = "https://mockwebserver.cwms.gov" -_RAT_SPEC_JSON = read_resource_file("rating_spec.json") -_RAT_SPECS_JSON = read_resource_file("rating_specs.json") - - -@pytest.fixture(autouse=True) -def init_session(): - cwms.api.init_session(api_root=_MOCK_ROOT) - - -def test_get_rating_spec_default(requests_mock): - requests_mock.get( - f"{_MOCK_ROOT}" - "/ratings/spec/TestRating.Stage;Flow.USGS-EXSA-TEST.USGS-NWIS-TEST?office=MVP", - json=_RAT_SPEC_JSON, - ) - - rating_id = "TestRating.Stage;Flow.USGS-EXSA-TEST.USGS-NWIS-TEST?office=MVP" - office_id = "MVP" - - spec = ratings_spec.get_rating_spec(rating_id=rating_id, office_id=office_id) - - assert spec.json == _RAT_SPEC_JSON - assert type(spec.df) is pd.DataFrame - - -def test_get_rating_specs_default(requests_mock): - requests_mock.get( - f"{_MOCK_ROOT}" "/ratings/spec?office=MVP", - json=_RAT_SPECS_JSON, - ) - - office_id = "MVP" - - specs = ratings_spec.get_rating_specs(office_id=office_id) - - assert specs.json == _RAT_SPECS_JSON - assert type(specs.df) is pd.DataFrame - assert specs.df.shape[0] == 4 diff --git a/tests/mock/ratings/rating_template_test.py b/tests/mock/ratings/rating_template_test.py deleted file mode 100644 index d7b4f079..00000000 --- a/tests/mock/ratings/rating_template_test.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2024 -# United States Army Corps of Engineers - Hydrologic Engineering Center (USACE/HEC) -# All Rights Reserved. USACE PROPRIETARY/CONFIDENTIAL. -# Source may not be released without written approval from HEC - -import pandas as pd -import pytest -import pytz - -import cwms.api -import cwms.ratings.ratings_template as ratings_template -from tests._test_utils import read_resource_file - -_MOCK_ROOT = "https://mockwebserver.cwms.gov" -_RAT_TEMP_JSON = read_resource_file("rating_template.json") -_RAT_TEMPS_JSON = read_resource_file("rating_templates.json") - - -@pytest.fixture(autouse=True) -def init_session(): - cwms.api.init_session(api_root=_MOCK_ROOT) - - -def test_get_rating_template_default(requests_mock): - requests_mock.get( - f"{_MOCK_ROOT}" "/ratings/template/Stage;Flow.USGS-EXSA-TEST?office=MVP", - json=_RAT_TEMP_JSON, - ) - - rating_temp = "Stage;Flow.USGS-EXSA-TEST" - office_id = "MVP" - - template = ratings_template.get_rating_template( - template_id=rating_temp, office_id=office_id - ) - - assert template.json == _RAT_TEMP_JSON - assert type(template.df) is pd.DataFrame - - -def test_get_rating_templates_default(requests_mock): - requests_mock.get( - f"{_MOCK_ROOT}" "/ratings/template?office=MVP", - json=_RAT_TEMPS_JSON, - ) - - office_id = "MVP" - - templates = ratings_template.get_rating_templates(office_id=office_id) - - assert templates.json == _RAT_TEMPS_JSON - assert type(templates.df) is pd.DataFrame - assert templates.df.shape[0] == 5