diff --git a/hertzbeat-collector/hertzbeat-collector-cloudspanner/pom.xml b/hertzbeat-collector/hertzbeat-collector-cloudspanner/pom.xml new file mode 100644 index 00000000000..db343dfcf1c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-cloudspanner/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-collector + 2.0-SNAPSHOT + + + hertzbeat-collector-cloudspanner + ${project.artifactId} + + + ${java.version} + ${java.version} + UTF-8 + + + + + org.apache.hertzbeat + hertzbeat-collector-common + provided + + + org.springframework.boot + spring-boot-starter + provided + + + + com.google.cloud + google-cloud-spanner-jdbc + + + org.springframework.boot + spring-boot-starter-test + test + + + diff --git a/hertzbeat-collector/hertzbeat-collector-cloudspanner/src/main/java/org/apache/hertzbeat/collector/collect/cloudspanner/CloudSpannerDriverLoader.java b/hertzbeat-collector/hertzbeat-collector-cloudspanner/src/main/java/org/apache/hertzbeat/collector/collect/cloudspanner/CloudSpannerDriverLoader.java new file mode 100644 index 00000000000..5413521cf27 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-cloudspanner/src/main/java/org/apache/hertzbeat/collector/collect/cloudspanner/CloudSpannerDriverLoader.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.cloudspanner; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.CommandLineRunner; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; + +/** + * Force-loads the Cloud Spanner JDBC driver at startup to prevent SPI + * concurrent deadlock when multiple threads first access the driver + * simultaneously. Mirrors the pattern in JdbcSpiLoader for the drivers + * bundled in hertzbeat-collector-basic. + */ +@Service +@Slf4j +@Order(value = Ordered.HIGHEST_PRECEDENCE) +public class CloudSpannerDriverLoader implements CommandLineRunner { + + @Override + public void run(String... args) { + log.info("loading Cloud Spanner JDBC driver"); + try { + Class.forName("com.google.cloud.spanner.jdbc.JdbcDriver"); + } catch (Exception e) { + log.error("failed to load Cloud Spanner JDBC driver: {}", e.getMessage()); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-cloudspanner/src/test/java/org/apache/hertzbeat/collector/collect/cloudspanner/CloudSpannerDriverLoaderTest.java b/hertzbeat-collector/hertzbeat-collector-cloudspanner/src/test/java/org/apache/hertzbeat/collector/collect/cloudspanner/CloudSpannerDriverLoaderTest.java new file mode 100644 index 00000000000..7edc5d3a0dd --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-cloudspanner/src/test/java/org/apache/hertzbeat/collector/collect/cloudspanner/CloudSpannerDriverLoaderTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.cloudspanner; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.sql.DriverManager; +import org.junit.jupiter.api.Test; + +/** + * Test case for {@link CloudSpannerDriverLoader} + */ +class CloudSpannerDriverLoaderTest { + + @Test + void spannerDriverMustBeRegisteredAfterLoad() throws Exception { + new CloudSpannerDriverLoader().run(); + assertNotNull( + DriverManager.getDriver( + "jdbc:cloudspanner:/projects/p/instances/i/databases/d"), + "Cloud Spanner JDBC driver must be registered after CloudSpannerDriverLoader runs"); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index 2e0a08d2166..960dddfa7cb 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -127,6 +127,11 @@ mssql-jdbc test + + org.apache.hertzbeat + hertzbeat-collector-cloudspanner + ${hertzbeat.version} + diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/spanner/SpannerJdbcTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/spanner/SpannerJdbcTemplateIntegrationTest.java new file mode 100644 index 00000000000..9d01064709b --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/spanner/SpannerJdbcTemplateIntegrationTest.java @@ -0,0 +1,689 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.spanner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.cloud.spanner.Dialect; +import com.google.cloud.spanner.InstanceConfigId; +import com.google.cloud.spanner.InstanceId; +import com.google.cloud.spanner.InstanceInfo; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +/** + * Integration test for app-cloudspanner.yml (Google Standard SQL dialect) and + * app-cloudspanner-pg.yml (PostgreSQL dialect) against the official Cloud Spanner + * emulator. A single emulator container hosts both databases; the GSS database + * is created via autoConfigEmulator and the PG database via the Spanner Java client. + * SPANNER_SYS statistics tables are not supported by the emulator and are therefore + * not covered here. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class SpannerJdbcTemplateIntegrationTest { + + private static final String EMULATOR_IMAGE = + "gcr.io/cloud-spanner-emulator/emulator:1.5.53"; + private static final int GRPC_PORT = 9010; + + private static final String PROJECT = "test-project"; + private static final String INSTANCE = "test-instance"; + private static final String DATABASE = "test-db"; + private static final String PG_DATABASE = "pg-test-db"; + + // SPANNER_SYS statistics views are not available in the emulator + private static final java.util.Set EMULATOR_UNSUPPORTED_GROUPS = + java.util.Set.of("query_stats", "txn_stats", "lock_stats", "table_sizes"); + + private GenericContainer emulator; + private String jdbcUrl; + private String pgJdbcUrl; + private List templateMetrics; + private List pgTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue( + DockerClientFactory.instance().isDockerAvailable(), + "Docker/Podman is required for integration tests"); + + new CollectStrategyFactory().run(); + templateMetrics = loadTemplate().getMetrics(); + pgTemplateMetrics = loadPgTemplate().getMetrics(); + + emulator = new GenericContainer<>(DockerImageName.parse(EMULATOR_IMAGE)) + .withExposedPorts(GRPC_PORT) + .waitingFor(Wait.forListeningPort()); + emulator.start(); + + provisionEmulator(); + + jdbcUrl = String.format( + "jdbc:cloudspanner://%s:%d/projects/%s/instances/%s/databases/%s" + + ";usePlainText=true;autoConfigEmulator=true", + emulator.getHost(), + emulator.getMappedPort(GRPC_PORT), + PROJECT, INSTANCE, DATABASE); + createSchema(); + seedData(); + + pgJdbcUrl = String.format( + "jdbc:cloudspanner://%s:%d/projects/%s/instances/%s/databases/%s" + + ";usePlainText=true;autoConfigEmulator=true", + emulator.getHost(), + emulator.getMappedPort(GRPC_PORT), + PROJECT, INSTANCE, PG_DATABASE); + createPgSchema(); + seedPgData(); + } + + @AfterAll + void tearDown() { + if (emulator != null) { + emulator.stop(); + } + } + + // ── Contract: metric groups must exist ── + + @Test + @DisplayName("tables metric group must be defined") + void tablesShouldBeDefined() { + assertTrue(templateMetrics.stream() + .anyMatch(m -> "tables".equals(m.getName())), + "app-cloudspanner.yml must define a tables metric group"); + } + + @Test + @DisplayName("indexes metric group must be defined") + void indexesShouldBeDefined() { + assertTrue(templateMetrics.stream() + .anyMatch(m -> "indexes".equals(m.getName())), + "app-cloudspanner.yml must define an indexes metric group"); + } + + @Test + @DisplayName("database_options metric group must be defined") + void databaseOptionsShouldBeDefined() { + assertTrue(templateMetrics.stream() + .anyMatch(m -> "database_options".equals(m.getName())), + "app-cloudspanner.yml must define a database_options metric group"); + } + + // ── Field type contracts ── + + @Test + @DisplayName("tables fields must all be type string") + void tablesFieldTypesShouldBeStrings() { + Metrics m = findMetric("tables"); + for (Metrics.Field f : m.getFields()) { + assertEquals(1, f.getType(), + "tables." + f.getField() + " must be type 1 (string)"); + } + } + + @Test + @DisplayName("indexes fields must all be type string") + void indexesFieldTypesShouldBeStrings() { + Metrics m = findMetric("indexes"); + for (Metrics.Field f : m.getFields()) { + assertEquals(1, f.getType(), + "indexes." + f.getField() + " must be type 1 (string)"); + } + } + + @Test + @DisplayName("database_options fields must all be type string") + void databaseOptionsFieldTypesShouldBeStrings() { + Metrics m = findMetric("database_options"); + for (Metrics.Field f : m.getFields()) { + assertEquals(1, f.getType(), + "database_options." + f.getField() + " must be type 1 (string)"); + } + } + + // ── Every metric group must collect successfully ── + + @TestFactory + @DisplayName("all metric groups must collect successfully against the emulator") + Stream shouldCollectAllMetricGroups() { + return templateMetrics.stream() + .filter(tmpl -> !EMULATOR_UNSUPPORTED_GROUPS.contains(tmpl.getName())) + .map(tmpl -> DynamicContainer.dynamicContainer( + tmpl.getName(), + Stream.of( + DynamicTest.dynamicTest("collects without error", + () -> { + CollectRep.MetricsData data = + collect(materialize(tmpl)); + assertEquals( + CollectRep.Code.SUCCESS, + data.getCode(), + tmpl.getName() + " failed: " + + data.getMsg()); + assertEquals( + tmpl.getFields().size(), + data.getFieldsCount(), + tmpl.getName() + + " field count mismatch"); + })))); + } + + // ── tables-specific assertions ── + + @TestFactory + @DisplayName("tables metric group edge cases") + Stream tablesEdgeCases() { + return Stream.of( + DynamicTest.dynamicTest( + "returns at least one row", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("tables"))); + assertTrue(data.getValuesCount() > 0, + "tables must return at least one row"); + }), + DynamicTest.dynamicTest( + "test_table must appear", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("tables"))); + boolean found = false; + for (int i = 0; i < data.getValuesCount(); i++) { + if ("test_table".equalsIgnoreCase( + data.getValues().get(i).getColumns(0))) { + found = true; + break; + } + } + assertTrue(found, + "tables must include test_table"); + }), + DynamicTest.dynamicTest( + "spanner_state must be COMMITTED for all rows", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("tables"))); + for (int i = 0; i < data.getValuesCount(); i++) { + assertEquals("COMMITTED", + data.getValues().get(i).getColumns(2), + "spanner_state must be COMMITTED"); + } + })); + } + + // ── indexes-specific assertions ── + + @TestFactory + @DisplayName("indexes metric group edge cases") + Stream indexesEdgeCases() { + return Stream.of( + DynamicTest.dynamicTest( + "returns at least one row", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("indexes"))); + assertTrue(data.getValuesCount() > 0, + "indexes must return at least one row"); + }), + DynamicTest.dynamicTest( + "secondary index idx_test_name must appear", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("indexes"))); + boolean found = false; + for (int i = 0; i < data.getValuesCount(); i++) { + if ("idx_test_name".equalsIgnoreCase( + data.getValues().get(i).getColumns(1))) { + found = true; + break; + } + } + assertTrue(found, + "indexes must include idx_test_name"); + }), + DynamicTest.dynamicTest( + "is_unique must be Y or N for all rows", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("indexes"))); + for (int i = 0; i < data.getValuesCount(); i++) { + String v = data.getValues().get(i).getColumns(4); + assertTrue("Y".equals(v) || "N".equals(v), + "is_unique must be Y/N, got: " + v); + } + }), + DynamicTest.dynamicTest( + "is_null_filtered must be Y or N for all rows", + () -> { + CollectRep.MetricsData data = + collect(materialize(findMetric("indexes"))); + for (int i = 0; i < data.getValuesCount(); i++) { + String v = data.getValues().get(i).getColumns(5); + assertTrue("Y".equals(v) || "N".equals(v), + "is_null_filtered must be Y/N, got: " + v); + } + })); + } + + // ── database_options-specific assertions ── + + @Test + @DisplayName("database_options must return at least one row") + void databaseOptionsShouldReturnRows() throws Exception { + CollectRep.MetricsData data = + collect(materialize(findMetric("database_options"))); + assertTrue(data.getValuesCount() > 0, + "database_options must return at least one row"); + } + + @Test + @DisplayName("database_dialect option must be GOOGLE_STANDARD_SQL") + void databaseDialectShouldBeGoogleSql() throws Exception { + CollectRep.MetricsData data = + collect(materialize(findMetric("database_options"))); + boolean found = false; + for (int i = 0; i < data.getValuesCount(); i++) { + if ("database_dialect".equalsIgnoreCase( + data.getValues().get(i).getColumns(0))) { + assertEquals("GOOGLE_STANDARD_SQL", + data.getValues().get(i).getColumns(1), + "database_dialect must be GOOGLE_STANDARD_SQL"); + found = true; + break; + } + } + assertTrue(found, "database_options must include database_dialect"); + } + + private void createSchema() throws Exception { + try (Connection conn = DriverManager.getConnection(jdbcUrl); + Statement stmt = conn.createStatement()) { + stmt.execute(""" + CREATE TABLE test_table ( + id INT64 NOT NULL, + name STRING(100) NOT NULL, + value FLOAT64 + ) PRIMARY KEY (id)"""); + stmt.execute( + "CREATE INDEX idx_test_name ON test_table (name)"); + } + } + + private void seedData() throws Exception { + try (Connection conn = DriverManager.getConnection(jdbcUrl); + var ps = conn.prepareStatement( + "INSERT INTO test_table (id, name, value) VALUES (?, ?, ?)")) { + ps.setLong(1, 1); + ps.setString(2, "alpha"); + ps.setDouble(3, 1.0); + ps.executeUpdate(); + ps.setLong(1, 2); + ps.setString(2, "beta"); + ps.setDouble(3, 2.0); + ps.executeUpdate(); + ps.setLong(1, 3); + ps.setString(2, "gamma"); + ps.setDouble(3, 3.0); + ps.executeUpdate(); + } + } + + // ── Collection helpers ── + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L).tenantId(1L).app("cloudspanner") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { }); + var dispatch = new CapturingCollectDataDispatch(); + var collector = new MetricsCollect( + metric, new StubTimeout(timerTask), dispatch, + "collector-test", List.of()); + collector.run(); + assertNotNull(dispatch.metricsData, + metric.getName() + " should dispatch metrics data"); + return dispatch.metricsData; + } + + private Metrics materialize(Metrics templateMetric) { + Metrics m = JsonUtil.fromJson( + JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbc = m.getJdbc(); + jdbc.setUrl(jdbcUrl); + jdbc.setTimeout("30000"); + jdbc.setReuseConnection("false"); + if (m.getAliasFields() == null || m.getAliasFields().isEmpty()) { + m.setAliasFields( + m.getFields().stream() + .map(Metrics.Field::getField) + .collect(Collectors.toList())); + } + return m; + } + + private Metrics findMetric(String name) { + return templateMetrics.stream() + .filter(m -> name.equals(m.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException( + "Metric group not found: " + name)); + } + + private Job loadTemplate() throws Exception { + Path path = Path.of( + "..", "..", "hertzbeat-manager", "src", "main", + "resources", "define", "app-cloudspanner.yml") + .toAbsolutePath().normalize(); + try (Reader r = Files.newBufferedReader(path)) { + return new Yaml().loadAs(r, Job.class); + } + } + + // ── Emulator provisioning ── + + private void provisionEmulator() throws Exception { + SpannerOptions opts = SpannerOptions.newBuilder() + .setEmulatorHost(emulator.getHost() + ":" + + emulator.getMappedPort(GRPC_PORT)) + .setProjectId(PROJECT) + .build(); + try (Spanner spanner = opts.getService()) { + spanner.getInstanceAdminClient() + .createInstance(InstanceInfo + .newBuilder(InstanceId.of(PROJECT, INSTANCE)) + .setDisplayName(INSTANCE) + .setInstanceConfigId( + InstanceConfigId.of(PROJECT, "emulator-config")) + .setNodeCount(1) + .build()) + .get(); + var dbAdmin = spanner.getDatabaseAdminClient(); + dbAdmin.createDatabase(INSTANCE, DATABASE, List.of()).get(); + dbAdmin.createDatabase(INSTANCE, + "CREATE DATABASE \"" + PG_DATABASE + "\"", + Dialect.POSTGRESQL, List.of()).get(); + } + } + + private void createPgSchema() throws Exception { + try (Connection conn = DriverManager.getConnection(pgJdbcUrl); + Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE test_table (" + + "id bigint PRIMARY KEY," + + " name varchar(100) NOT NULL," + + " value float8" + + ")"); + stmt.execute("CREATE INDEX idx_test_name ON test_table (name)"); + } + } + + private void seedPgData() throws Exception { + try (Connection conn = DriverManager.getConnection(pgJdbcUrl); + var ps = conn.prepareStatement( + "INSERT INTO test_table (id, name, value)" + + " VALUES (?, ?, ?)")) { + ps.setLong(1, 1); + ps.setString(2, "alpha"); + ps.setDouble(3, 1.0); + ps.executeUpdate(); + ps.setLong(1, 2); + ps.setString(2, "beta"); + ps.setDouble(3, 2.0); + ps.executeUpdate(); + ps.setLong(1, 3); + ps.setString(2, "gamma"); + ps.setDouble(3, 3.0); + ps.executeUpdate(); + } + } + + private Job loadPgTemplate() throws Exception { + Path path = Path.of( + "..", "..", "hertzbeat-manager", "src", "main", + "resources", "define", "app-cloudspanner-pg.yml") + .toAbsolutePath().normalize(); + try (Reader r = Files.newBufferedReader(path)) { + return new Yaml().loadAs(r, Job.class); + } + } + + private Metrics materializePg(Metrics templateMetric) { + Metrics m = JsonUtil.fromJson( + JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbc = m.getJdbc(); + jdbc.setUrl(pgJdbcUrl); + jdbc.setTimeout("30000"); + jdbc.setReuseConnection("false"); + if (m.getAliasFields() == null || m.getAliasFields().isEmpty()) { + m.setAliasFields( + m.getFields().stream() + .map(Metrics.Field::getField) + .collect(Collectors.toList())); + } + return m; + } + + private Metrics findPgMetric(String name) { + return pgTemplateMetrics.stream() + .filter(m -> name.equals(m.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException( + "PG metric group not found: " + name)); + } + + // ── PG dialect contract tests ── + + @Test + @DisplayName("PG tables metric group must be defined") + void pgTablesShouldBeDefined() { + assertTrue(pgTemplateMetrics.stream() + .anyMatch(m -> "tables".equals(m.getName())), + "app-cloudspanner-pg.yml must define a tables metric group"); + } + + @Test + @DisplayName("PG indexes metric group must be defined") + void pgIndexesShouldBeDefined() { + assertTrue(pgTemplateMetrics.stream() + .anyMatch(m -> "indexes".equals(m.getName())), + "app-cloudspanner-pg.yml must define an indexes metric group"); + } + + @Test + @DisplayName("PG database_options metric group must be defined") + void pgDatabaseOptionsShouldBeDefined() { + assertTrue(pgTemplateMetrics.stream() + .anyMatch(m -> "database_options".equals(m.getName())), + "app-cloudspanner-pg.yml must define a database_options metric group"); + } + + // ── PG dialect collection tests ── + + @TestFactory + @DisplayName("all PG metric groups must collect successfully against the emulator") + Stream shouldCollectAllPgMetricGroups() { + return pgTemplateMetrics.stream() + .filter(tmpl -> !EMULATOR_UNSUPPORTED_GROUPS.contains(tmpl.getName())) + .map(tmpl -> DynamicContainer.dynamicContainer( + "pg-" + tmpl.getName(), + Stream.of(DynamicTest.dynamicTest( + "collects without error", + () -> { + CollectRep.MetricsData data = + collect(materializePg(tmpl)); + assertEquals( + CollectRep.Code.SUCCESS, + data.getCode(), + tmpl.getName() + " failed: " + + data.getMsg()); + assertEquals( + tmpl.getFields().size(), + data.getFieldsCount(), + tmpl.getName() + + " field count mismatch"); + })))); + } + + @Test + @DisplayName("PG database_dialect option must be POSTGRESQL") + void pgDatabaseDialectShouldBePostgresql() throws Exception { + CollectRep.MetricsData data = + collect(materializePg(findPgMetric("database_options"))); + boolean found = false; + for (int i = 0; i < data.getValuesCount(); i++) { + if ("database_dialect".equalsIgnoreCase( + data.getValues().get(i).getColumns(0))) { + assertEquals("POSTGRESQL", + data.getValues().get(i).getColumns(1), + "database_dialect must be POSTGRESQL"); + found = true; + break; + } + } + assertTrue(found, + "database_options must include database_dialect"); + } + + @Test + @DisplayName("PG test_table must appear in tables metric group") + void pgTestTableShouldAppear() throws Exception { + CollectRep.MetricsData data = + collect(materializePg(findPgMetric("tables"))); + assertTrue(data.getValuesCount() > 0, + "PG tables must return at least one row"); + boolean found = false; + for (int i = 0; i < data.getValuesCount(); i++) { + if ("test_table".equalsIgnoreCase( + data.getValues().get(i).getColumns(0))) { + found = true; + break; + } + } + assertTrue(found, "PG tables must include test_table"); + } + + @Test + @DisplayName("PG idx_test_name must appear in indexes metric group") + void pgIndexShouldAppear() throws Exception { + CollectRep.MetricsData data = + collect(materializePg(findPgMetric("indexes"))); + assertTrue(data.getValuesCount() > 0, + "PG indexes must return at least one row"); + boolean found = false; + for (int i = 0; i < data.getValuesCount(); i++) { + if ("idx_test_name".equalsIgnoreCase( + data.getValues().get(i).getColumns(1))) { + found = true; + break; + } + } + assertTrue(found, "PG indexes must include idx_test_name"); + } + + // ── Inner types ── + + private static final class CapturingCollectDataDispatch + implements CollectDataDispatch { + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, + CollectRep.MetricsData data) { + this.metricsData = data; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, + List list) { + if (list != null && !list.isEmpty()) { + this.metricsData = list.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/pom.xml b/hertzbeat-collector/pom.xml index f1e1454623c..05a489e4636 100644 --- a/hertzbeat-collector/pom.xml +++ b/hertzbeat-collector/pom.xml @@ -43,6 +43,7 @@ hertzbeat-collector-nebulagraph hertzbeat-collector-rocketmq hertzbeat-collector-kafka + hertzbeat-collector-cloudspanner diff --git a/hertzbeat-manager/src/main/resources/define/app-cloudspanner-pg.yml b/hertzbeat-manager/src/main/resources/define/app-cloudspanner-pg.yml new file mode 100644 index 00000000000..8a0576620d6 --- /dev/null +++ b/hertzbeat-manager/src/main/resources/define/app-cloudspanner-pg.yml @@ -0,0 +1,463 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +category: db +app: cloudspanner-pg +name: + zh-CN: Google Cloud Spanner (PostgreSQL) + en-US: Google Cloud Spanner (PostgreSQL) + ja-JP: Google Cloud Spanner (PostgreSQL) +help: + zh-CN: > + Hertzbeat 通过 JDBC 对使用 PostgreSQL 方言的 Google Cloud Spanner 数据库进行监控。 + 仅适用于创建时指定了 PostgreSQL 方言的数据库。 + 请分别填写项目 ID、实例 ID 和数据库名称，Hertzbeat 将自动构建连接字符串。 + 如需附加 JDBC 连接属性（例如服务账号凭据），请在"额外参数"字段中以英文分号开头填写： + ;credentials=/path/to/key.json + en-US: > + Hertzbeat monitors Google Cloud Spanner databases using the PostgreSQL dialect via JDBC. + Use this template only for databases created with the PostgreSQL dialect. + Provide the project ID, instance ID, and database name separately — Hertzbeat + builds the connection URL automatically. + To add extra JDBC connection properties (e.g. a service account key file), enter + them in the Extra Params field starting with a semicolon: + ;credentials=/path/to/key.json + ja-JP: > + Hertzbeat は JDBC を使用して PostgreSQL 方言の Google Cloud Spanner データベースを監視します。 + PostgreSQL 方言で作成されたデータベースにのみ使用してください。 + プロジェクト ID、インスタンス ID、データベース名を個別に入力してください。 + 接続 URL は Hertzbeat が自動的に構築します。 +helpLink: + zh-CN: https://cloud.google.com/spanner/docs/use-oss-jdbc + en-US: https://cloud.google.com/spanner/docs/use-oss-jdbc + ja-JP: https://cloud.google.com/spanner/docs/use-oss-jdbc + +params: + - field: host + name: + zh-CN: 实例 ID + en-US: Instance ID + ja-JP: インスタンス ID + type: host + required: true + placeholder: 'my-instance' + - field: project + name: + zh-CN: 项目 ID + en-US: Project ID + ja-JP: プロジェクト ID + type: text + required: true + placeholder: 'my-gcp-project' + - field: database + name: + zh-CN: 数据库名 + en-US: Database Name + ja-JP: データベース名 + type: text + required: true + placeholder: 'my-database' + - field: params + name: + zh-CN: 额外参数 + en-US: Extra Params + ja-JP: 追加パラメータ + type: text + required: false + placeholder: ';credentials=/path/to/key.json' + - field: timeout + name: + zh-CN: 查询超时时间(ms) + en-US: Query Timeout(ms) + ja-JP: クエリタイムアウト（ｍｓ） + type: number + range: '[400,200000]' + required: true + defaultValue: 6000 + - field: reuseConnection + name: + zh-CN: 复用连接 + en-US: Reuse Connection + ja-JP: 接続再利用 + type: boolean + required: false + defaultValue: true + +metrics: + - name: tables + i18n: + zh-CN: 数据表 + en-US: Tables + ja-JP: テーブル + priority: 0 + fields: + - field: table_name + type: 1 + label: true + i18n: + zh-CN: 表名 + en-US: Table Name + ja-JP: テーブル名 + - field: table_type + type: 1 + i18n: + zh-CN: 表类型 + en-US: Table Type + ja-JP: テーブルタイプ + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT table_name, table_type + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name + + - name: indexes + i18n: + zh-CN: 索引 + en-US: Indexes + ja-JP: インデックス + priority: 1 + fields: + - field: table_name + type: 1 + label: true + i18n: + zh-CN: 表名 + en-US: Table Name + ja-JP: テーブル名 + - field: index_name + type: 1 + label: true + i18n: + zh-CN: 索引名 + en-US: Index Name + ja-JP: インデックス名 + - field: index_type + type: 1 + i18n: + zh-CN: 索引类型 + en-US: Index Type + ja-JP: インデックスタイプ + - field: index_state + type: 1 + i18n: + zh-CN: 索引状态 + en-US: Index State + ja-JP: インデックス状態 + - field: is_unique + type: 1 + i18n: + zh-CN: 是否唯一 + en-US: Is Unique + ja-JP: ユニーク + - field: is_null_filtered + type: 1 + i18n: + zh-CN: 是否过滤 NULL + en-US: Is Null Filtered + ja-JP: NULL フィルタ + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT table_name, index_name, index_type, index_state, + CASE WHEN is_unique = 'YES' THEN 'Y' ELSE 'N' END AS is_unique, + CASE WHEN is_null_filtered = 'YES' THEN 'Y' ELSE 'N' END AS is_null_filtered + FROM information_schema.indexes + WHERE table_schema = 'public' + ORDER BY table_name, index_name + + - name: database_options + i18n: + zh-CN: 数据库配置 + en-US: Database Options + ja-JP: データベースオプション + priority: 2 + fields: + - field: option_name + type: 1 + label: true + i18n: + zh-CN: 配置项 + en-US: Option Name + ja-JP: オプション名 + - field: option_value + type: 1 + i18n: + zh-CN: 配置值 + en-US: Option Value + ja-JP: オプション値 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT option_name, option_value + FROM information_schema.database_options + ORDER BY option_name + + - name: query_stats + i18n: + zh-CN: 查询统计（近1分钟） + en-US: Query Stats (last minute) + ja-JP: クエリ統計（直近1分） + priority: 3 + fields: + - field: interval_end + type: 1 + i18n: + zh-CN: 统计截止时间 + en-US: Interval End + ja-JP: 集計終了時刻 + - field: execution_count + type: 0 + i18n: + zh-CN: 执行次数 + en-US: Execution Count + ja-JP: 実行回数 + - field: avg_latency_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均延迟(s) + en-US: Avg Latency(s) + ja-JP: 平均レイテンシ(s) + - field: avg_rows + type: 0 + i18n: + zh-CN: 平均返回行数 + en-US: Avg Rows + ja-JP: 平均返却行数 + - field: avg_rows_scanned + type: 0 + i18n: + zh-CN: 平均扫描行数 + en-US: Avg Rows Scanned + ja-JP: 平均スキャン行数 + - field: avg_cpu_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均 CPU 时间(s) + en-US: Avg CPU(s) + ja-JP: 平均 CPU 時間(s) + - field: all_failed_execution_count + type: 0 + i18n: + zh-CN: 失败次数 + en-US: Failed Count + ja-JP: 失敗回数 + - field: timed_out_execution_count + type: 0 + i18n: + zh-CN: 超时次数 + en-US: Timed Out Count + ja-JP: タイムアウト回数 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT interval_end::text AS interval_end, + execution_count, + avg_latency_seconds, + avg_rows, + avg_rows_scanned, + avg_cpu_seconds, + all_failed_execution_count, + timed_out_execution_count + FROM SPANNER_SYS.QUERY_STATS_TOTAL_MINUTE + ORDER BY interval_end DESC + LIMIT 1 + + - name: txn_stats + i18n: + zh-CN: 事务统计（近1分钟） + en-US: Transaction Stats (last minute) + ja-JP: トランザクション統計（直近1分） + priority: 4 + fields: + - field: interval_end + type: 1 + i18n: + zh-CN: 统计截止时间 + en-US: Interval End + ja-JP: 集計終了時刻 + - field: commit_attempt_count + type: 0 + i18n: + zh-CN: 提交尝试次数 + en-US: Commit Attempts + ja-JP: コミット試行回数 + - field: commit_abort_count + type: 0 + i18n: + zh-CN: 中止次数 + en-US: Commit Aborts + ja-JP: コミット中止回数 + - field: commit_retry_count + type: 0 + i18n: + zh-CN: 重试次数 + en-US: Commit Retries + ja-JP: コミット再試行回数 + - field: commit_failed_precondition_count + type: 0 + i18n: + zh-CN: 前置条件失败次数 + en-US: Failed Precondition Count + ja-JP: 前提条件失敗回数 + - field: avg_total_latency_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均总延迟(s) + en-US: Avg Total Latency(s) + ja-JP: 平均総レイテンシ(s) + - field: avg_commit_latency_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均提交延迟(s) + en-US: Avg Commit Latency(s) + ja-JP: 平均コミットレイテンシ(s) + - field: avg_participants + type: 0 + i18n: + zh-CN: 平均参与者数 + en-US: Avg Participants + ja-JP: 平均参加者数 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT interval_end::text AS interval_end, + commit_attempt_count, + commit_abort_count, + commit_retry_count, + commit_failed_precondition_count, + avg_total_latency_seconds, + avg_commit_latency_seconds, + avg_participants + FROM SPANNER_SYS.TXN_STATS_TOTAL_MINUTE + ORDER BY interval_end DESC + LIMIT 1 + + - name: lock_stats + i18n: + zh-CN: 锁等待统计（近1分钟） + en-US: Lock Stats (last minute) + ja-JP: ロック待機統計（直近1分） + priority: 5 + fields: + - field: interval_end + type: 1 + i18n: + zh-CN: 统计截止时间 + en-US: Interval End + ja-JP: 集計終了時刻 + - field: total_lock_wait_seconds + type: 0 + unit: s + i18n: + zh-CN: 总锁等待时间(s) + en-US: Total Lock Wait(s) + ja-JP: 合計ロック待機時間(s) + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT interval_end::text AS interval_end, + total_lock_wait_seconds + FROM SPANNER_SYS.LOCK_STATS_TOTAL_MINUTE + ORDER BY interval_end DESC + LIMIT 1 + + - name: table_sizes + i18n: + zh-CN: 表空间统计（近1小时） + en-US: Table Sizes (last hour) + ja-JP: テーブルサイズ統計（直近1時間） + priority: 6 + fields: + - field: table_name + type: 1 + label: true + i18n: + zh-CN: 表名 + en-US: Table Name + ja-JP: テーブル名 + - field: used_bytes + type: 0 + unit: B + i18n: + zh-CN: 已用字节 + en-US: Used Bytes + ja-JP: 使用バイト数 + - field: used_ssd_bytes + type: 0 + unit: B + i18n: + zh-CN: SSD 已用字节 + en-US: Used SSD Bytes + ja-JP: SSD 使用バイト数 + - field: used_hdd_bytes + type: 0 + unit: B + i18n: + zh-CN: HDD 已用字节 + en-US: Used HDD Bytes + ja-JP: HDD 使用バイト数 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT table_name, + used_bytes, + used_ssd_bytes, + used_hdd_bytes + FROM SPANNER_SYS.TABLE_SIZES_STATS_1HOUR + WHERE interval_end = ( + SELECT MAX(interval_end) FROM SPANNER_SYS.TABLE_SIZES_STATS_1HOUR + ) + ORDER BY table_name diff --git a/hertzbeat-manager/src/main/resources/define/app-cloudspanner.yml b/hertzbeat-manager/src/main/resources/define/app-cloudspanner.yml new file mode 100644 index 00000000000..d1283a8d49d --- /dev/null +++ b/hertzbeat-manager/src/main/resources/define/app-cloudspanner.yml @@ -0,0 +1,474 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +category: db +app: cloudspanner +name: + zh-CN: Google Cloud Spanner + en-US: Google Cloud Spanner + ja-JP: Google Cloud Spanner +help: + zh-CN: > + Hertzbeat 通过 JDBC 对 Google Cloud Spanner（Google Standard SQL 方言）进行监控。 + 请分别填写项目 ID、实例 ID 和数据库名称，Hertzbeat 将自动构建连接字符串。 + 如需附加 JDBC 连接属性（例如服务账号凭据），请在"额外参数"字段中以英文分号开头填写， + 例如：;credentials=/path/to/key.json + en-US: > + Hertzbeat monitors Google Cloud Spanner (Google Standard SQL dialect) via JDBC. + Provide the project ID, instance ID, and database name separately — Hertzbeat + builds the connection URL automatically. + To add extra JDBC connection properties (e.g. a service account key file), enter + them in the Extra Params field starting with a semicolon: + ;credentials=/path/to/key.json + ja-JP: > + Hertzbeat は JDBC を使用して Google Cloud Spanner（Google Standard SQL 方言）を監視します。 + プロジェクト ID、インスタンス ID、データベース名を個別に入力してください。 + 接続 URL は Hertzbeat が自動的に構築します。 + 追加の JDBC 接続プロパティ（サービスアカウントキーなど）が必要な場合は、 + 「追加パラメータ」フィールドにセミコロン始まりで入力してください。 +helpLink: + zh-CN: https://cloud.google.com/spanner/docs/use-oss-jdbc + en-US: https://cloud.google.com/spanner/docs/use-oss-jdbc + ja-JP: https://cloud.google.com/spanner/docs/use-oss-jdbc + +params: + - field: host + name: + zh-CN: 实例 ID + en-US: Instance ID + ja-JP: インスタンス ID + type: host + required: true + placeholder: 'my-instance' + - field: project + name: + zh-CN: 项目 ID + en-US: Project ID + ja-JP: プロジェクト ID + type: text + required: true + placeholder: 'my-gcp-project' + - field: database + name: + zh-CN: 数据库名 + en-US: Database Name + ja-JP: データベース名 + type: text + required: true + placeholder: 'my-database' + - field: params + name: + zh-CN: 额外参数 + en-US: Extra Params + ja-JP: 追加パラメータ + type: text + required: false + placeholder: ';credentials=/path/to/key.json' + - field: timeout + name: + zh-CN: 查询超时时间(ms) + en-US: Query Timeout(ms) + ja-JP: クエリタイムアウト（ｍｓ） + type: number + range: '[400,200000]' + required: true + defaultValue: 6000 + - field: reuseConnection + name: + zh-CN: 复用连接 + en-US: Reuse Connection + ja-JP: 接続再利用 + type: boolean + required: false + defaultValue: true + +metrics: + - name: tables + i18n: + zh-CN: 数据表 + en-US: Tables + ja-JP: テーブル + priority: 0 + fields: + - field: table_name + type: 1 + label: true + i18n: + zh-CN: 表名 + en-US: Table Name + ja-JP: テーブル名 + - field: table_type + type: 1 + i18n: + zh-CN: 表类型 + en-US: Table Type + ja-JP: テーブルタイプ + - field: spanner_state + type: 1 + i18n: + zh-CN: Spanner 状态 + en-US: Spanner State + ja-JP: Spannerの状態 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT TABLE_NAME AS table_name, + TABLE_TYPE AS table_type, + SPANNER_STATE AS spanner_state + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = '' + ORDER BY TABLE_NAME + + - name: indexes + i18n: + zh-CN: 索引 + en-US: Indexes + ja-JP: インデックス + priority: 1 + fields: + - field: table_name + type: 1 + label: true + i18n: + zh-CN: 表名 + en-US: Table Name + ja-JP: テーブル名 + - field: index_name + type: 1 + label: true + i18n: + zh-CN: 索引名 + en-US: Index Name + ja-JP: インデックス名 + - field: index_type + type: 1 + i18n: + zh-CN: 索引类型 + en-US: Index Type + ja-JP: インデックスタイプ + - field: index_state + type: 1 + i18n: + zh-CN: 索引状态 + en-US: Index State + ja-JP: インデックス状態 + - field: is_unique + type: 1 + i18n: + zh-CN: 是否唯一 + en-US: Is Unique + ja-JP: ユニーク + - field: is_null_filtered + type: 1 + i18n: + zh-CN: 是否过滤 NULL + en-US: Is Null Filtered + ja-JP: NULL フィルタ + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT TABLE_NAME AS table_name, + INDEX_NAME AS index_name, + INDEX_TYPE AS index_type, + INDEX_STATE AS index_state, + CASE WHEN IS_UNIQUE THEN 'Y' ELSE 'N' END AS is_unique, + CASE WHEN IS_NULL_FILTERED THEN 'Y' ELSE 'N' END AS is_null_filtered + FROM INFORMATION_SCHEMA.INDEXES + WHERE TABLE_SCHEMA = '' + ORDER BY TABLE_NAME, INDEX_NAME + + - name: database_options + i18n: + zh-CN: 数据库配置 + en-US: Database Options + ja-JP: データベースオプション + priority: 2 + fields: + - field: option_name + type: 1 + label: true + i18n: + zh-CN: 配置项 + en-US: Option Name + ja-JP: オプション名 + - field: option_value + type: 1 + i18n: + zh-CN: 配置值 + en-US: Option Value + ja-JP: オプション値 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT OPTION_NAME AS option_name, + OPTION_VALUE AS option_value + FROM INFORMATION_SCHEMA.DATABASE_OPTIONS + ORDER BY OPTION_NAME + + - name: query_stats + i18n: + zh-CN: 查询统计（近1分钟） + en-US: Query Stats (last minute) + ja-JP: クエリ統計（直近1分） + priority: 3 + fields: + - field: interval_end + type: 1 + i18n: + zh-CN: 统计截止时间 + en-US: Interval End + ja-JP: 集計終了時刻 + - field: execution_count + type: 0 + i18n: + zh-CN: 执行次数 + en-US: Execution Count + ja-JP: 実行回数 + - field: avg_latency_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均延迟(s) + en-US: Avg Latency(s) + ja-JP: 平均レイテンシ(s) + - field: avg_rows + type: 0 + i18n: + zh-CN: 平均返回行数 + en-US: Avg Rows + ja-JP: 平均返却行数 + - field: avg_rows_scanned + type: 0 + i18n: + zh-CN: 平均扫描行数 + en-US: Avg Rows Scanned + ja-JP: 平均スキャン行数 + - field: avg_cpu_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均 CPU 时间(s) + en-US: Avg CPU(s) + ja-JP: 平均 CPU 時間(s) + - field: all_failed_execution_count + type: 0 + i18n: + zh-CN: 失败次数 + en-US: Failed Count + ja-JP: 失敗回数 + - field: timed_out_execution_count + type: 0 + i18n: + zh-CN: 超时次数 + en-US: Timed Out Count + ja-JP: タイムアウト回数 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT CAST(INTERVAL_END AS STRING) AS interval_end, + EXECUTION_COUNT AS execution_count, + AVG_LATENCY_SECONDS AS avg_latency_seconds, + AVG_ROWS AS avg_rows, + AVG_ROWS_SCANNED AS avg_rows_scanned, + AVG_CPU_SECONDS AS avg_cpu_seconds, + ALL_FAILED_EXECUTION_COUNT AS all_failed_execution_count, + TIMED_OUT_EXECUTION_COUNT AS timed_out_execution_count + FROM SPANNER_SYS.QUERY_STATS_TOTAL_MINUTE + ORDER BY INTERVAL_END DESC + LIMIT 1 + + - name: txn_stats + i18n: + zh-CN: 事务统计（近1分钟） + en-US: Transaction Stats (last minute) + ja-JP: トランザクション統計（直近1分） + priority: 4 + fields: + - field: interval_end + type: 1 + i18n: + zh-CN: 统计截止时间 + en-US: Interval End + ja-JP: 集計終了時刻 + - field: commit_attempt_count + type: 0 + i18n: + zh-CN: 提交尝试次数 + en-US: Commit Attempts + ja-JP: コミット試行回数 + - field: commit_abort_count + type: 0 + i18n: + zh-CN: 中止次数 + en-US: Commit Aborts + ja-JP: コミット中止回数 + - field: commit_retry_count + type: 0 + i18n: + zh-CN: 重试次数 + en-US: Commit Retries + ja-JP: コミット再試行回数 + - field: commit_failed_precondition_count + type: 0 + i18n: + zh-CN: 前置条件失败次数 + en-US: Failed Precondition Count + ja-JP: 前提条件失敗回数 + - field: avg_total_latency_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均总延迟(s) + en-US: Avg Total Latency(s) + ja-JP: 平均総レイテンシ(s) + - field: avg_commit_latency_seconds + type: 0 + unit: s + i18n: + zh-CN: 平均提交延迟(s) + en-US: Avg Commit Latency(s) + ja-JP: 平均コミットレイテンシ(s) + - field: avg_participants + type: 0 + i18n: + zh-CN: 平均参与者数 + en-US: Avg Participants + ja-JP: 平均参加者数 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT CAST(INTERVAL_END AS STRING) AS interval_end, + COMMIT_ATTEMPT_COUNT AS commit_attempt_count, + COMMIT_ABORT_COUNT AS commit_abort_count, + COMMIT_RETRY_COUNT AS commit_retry_count, + COMMIT_FAILED_PRECONDITION_COUNT AS commit_failed_precondition_count, + AVG_TOTAL_LATENCY_SECONDS AS avg_total_latency_seconds, + AVG_COMMIT_LATENCY_SECONDS AS avg_commit_latency_seconds, + AVG_PARTICIPANTS AS avg_participants + FROM SPANNER_SYS.TXN_STATS_TOTAL_MINUTE + ORDER BY INTERVAL_END DESC + LIMIT 1 + + - name: lock_stats + i18n: + zh-CN: 锁等待统计（近1分钟） + en-US: Lock Stats (last minute) + ja-JP: ロック待機統計（直近1分） + priority: 5 + fields: + - field: interval_end + type: 1 + i18n: + zh-CN: 统计截止时间 + en-US: Interval End + ja-JP: 集計終了時刻 + - field: total_lock_wait_seconds + type: 0 + unit: s + i18n: + zh-CN: 总锁等待时间(s) + en-US: Total Lock Wait(s) + ja-JP: 合計ロック待機時間(s) + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT CAST(INTERVAL_END AS STRING) AS interval_end, + TOTAL_LOCK_WAIT_SECONDS AS total_lock_wait_seconds + FROM SPANNER_SYS.LOCK_STATS_TOTAL_MINUTE + ORDER BY INTERVAL_END DESC + LIMIT 1 + + - name: table_sizes + i18n: + zh-CN: 表空间统计（近1小时） + en-US: Table Sizes (last hour) + ja-JP: テーブルサイズ統計（直近1時間） + priority: 6 + fields: + - field: table_name + type: 1 + label: true + i18n: + zh-CN: 表名 + en-US: Table Name + ja-JP: テーブル名 + - field: used_bytes + type: 0 + unit: B + i18n: + zh-CN: 已用字节 + en-US: Used Bytes + ja-JP: 使用バイト数 + - field: used_ssd_bytes + type: 0 + unit: B + i18n: + zh-CN: SSD 已用字节 + en-US: Used SSD Bytes + ja-JP: SSD 使用バイト数 + - field: used_hdd_bytes + type: 0 + unit: B + i18n: + zh-CN: HDD 已用字节 + en-US: Used HDD Bytes + ja-JP: HDD 使用バイト数 + protocol: jdbc + jdbc: + url: 'jdbc:cloudspanner:/projects/^_^project^_^/instances/^_^host^_^/databases/^_^database^_^^_^params^_^' + platform: cloudspanner + timeout: ^_^timeout^_^ + reuseConnection: ^_^reuseConnection^_^ + queryType: multiRow + sql: >- + SELECT TABLE_NAME AS table_name, + USED_BYTES AS used_bytes, + USED_SSD_BYTES AS used_ssd_bytes, + USED_HDD_BYTES AS used_hdd_bytes + FROM SPANNER_SYS.TABLE_SIZES_STATS_1HOUR + WHERE INTERVAL_END = ( + SELECT MAX(INTERVAL_END) FROM SPANNER_SYS.TABLE_SIZES_STATS_1HOUR + ) + ORDER BY TABLE_NAME diff --git a/hertzbeat-warehouse/pom.xml b/hertzbeat-warehouse/pom.xml index e65b30d5367..666a6cf8c90 100644 --- a/hertzbeat-warehouse/pom.xml +++ b/hertzbeat-warehouse/pom.xml @@ -117,6 +117,10 @@ com.google.protobuf protobuf-java + + io.grpc + grpc-netty-shaded + diff --git a/pom.xml b/pom.xml index c4d5ec7e95f..a19451886a3 100644 --- a/pom.xml +++ b/pom.xml @@ -136,6 +136,7 @@ 8.4.0 1.4.1 12.10.2.jre11 + 2.34.0 21.5.0.0 4.6.1 0.4.6 @@ -403,6 +404,20 @@ mssql-jdbc ${mssql-jdbc.version} + + + com.google.cloud + google-cloud-spanner-jdbc + ${google-cloud-spanner-jdbc.version} + + + + io.grpc + grpc-netty-shaded + 1.70.0 + com.oracle.database.jdbc