test(grpc-suite): add post-upgrade gRPC surface coverage

.github/workflows/tests.yaml

@@ -48,6 +48,15 @@ jobs:
         uses: ./.github/actions/setup-ubuntu
       - run: make build-contracts test-full
 
+  grpc-surface:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup environment
+        uses: ./.github/actions/setup-ubuntu
+      - run: make build-contracts test-grpc-surface
+
   coverage:
     runs-on: ubuntu-latest
     steps:

cmd/akash/cmd/root.go

@@ -79,6 +79,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig sdkutil.EncodingConfig)
 		rosettaCmd.RosettaCommand(encodingConfig.InterfaceRegistry, encodingConfig.Codec),
 		pruning.Cmd(ac.newApp, home),
 		snapshot.Cmd(ac.newApp),
+		sdkserver.NewRollbackCmd(ac.newApp, home),
 		testnetCmd(app.ModuleBasics(), banktypes.GenesisBalancesIterator{}),
 		PrepareGenesisCmd(app.DefaultHome, app.ModuleBasics()),
 		testnetify.GetCmd(ac.newTestnetApp),

make/test-integration.mk

@@ -22,6 +22,21 @@ test-full: wasmvm-libs
 test-integration:
 	$(GO_TEST) -v -tags="e2e.integration" -ldflags '$(ldflags)' ./tests/e2e/...
 
+# test-grpc-surface runs the exhaustive gRPC transaction/query suite against an
+# in-process single-validator network (the fast, standalone-compilable mirror of
+# the post-upgrade verification that runs against a testnetify-forked node).
+.PHONY: test-grpc-surface
+test-grpc-surface: wasmvm-libs
+	$(GO_TEST) -v -tags="e2e.integration" -ldflags '$(ldflags)' -timeout 30m ./tests/fullsurface/... -args -grpc-suite-mode=all
+
+.PHONY: test-grpc-surface-tx
+test-grpc-surface-tx: wasmvm-libs
+	$(GO_TEST) -v -tags="e2e.integration" -ldflags '$(ldflags)' -timeout 30m ./tests/fullsurface/... -args -grpc-suite-mode=tx
+
+.PHONY: test-grpc-surface-query
+test-grpc-surface-query: wasmvm-libs
+	$(GO_TEST) -v -tags="e2e.integration" -ldflags '$(ldflags)' -timeout 10m ./tests/fullsurface/... -args -grpc-suite-mode=query
+
 .PHONY: test-coverage
 test-coverage: wasmvm-libs
 	$(GO_TEST) $(BUILD_FLAGS) -coverprofile=coverage.txt \

make/test-upgrade.mk

@@ -84,11 +84,11 @@ test-reset:
 	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --uto=$(UPGRADE_TO) --snapshot-url=$(SNAPSHOT_URL) --chain-meta=$(CHAIN_METADATA_URL) --max-validators=$(MAX_VALIDATORS) clean
 	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --uto=$(UPGRADE_TO) --snapshot-url=$(SNAPSHOT_URL) --gbv=$(GENESIS_BINARY_VERSION) --chain-meta=$(CHAIN_METADATA_URL) bins
 	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --uto=$(UPGRADE_TO) --snapshot-url=$(SNAPSHOT_URL) --chain-meta=$(CHAIN_METADATA_URL) keys
-	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --state-config=$(STATE_CONFIG) --snapshot-url=$(SNAPSHOT_URL) --chain-meta=$(CHAIN_METADATA_URL) --max-validators=$(MAX_VALIDATORS) prepare-state
+	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --state-config=$(STATE_CONFIG) --snapshot-url=$(SNAPSHOT_URL) --chain-meta=$(CHAIN_METADATA_URL) --uto=$(UPGRADE_TO) --max-validators=$(MAX_VALIDATORS) prepare-state
 
 .PHONY: prepare-state
 prepare-state:
-	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --state-config=$(STATE_CONFIG) --chain-meta=$(CHAIN_METADATA_URL) --max-validators=$(MAX_VALIDATORS) prepare-state
+	$(ROOT_DIR)/script/upgrades.sh --workdir=$(AP_RUN_DIR) --config="$(PWD)/config.json" --state-config=$(STATE_CONFIG) --snapshot-url=$(SNAPSHOT_URL) --chain-meta=$(CHAIN_METADATA_URL) --uto=$(UPGRADE_TO) --max-validators=$(MAX_VALIDATORS) prepare-state
 
 .PHONY: bins
 bins:

script/upgrades.sh

@@ -438,6 +438,13 @@ function prepare_state() {
 		cosmovisor_dir=$valdir/cosmovisor
 		genesis_bin=$cosmovisor_dir/genesis/bin
 
+		rm -rf "$cosmovisor_dir/current"
+		rm -f "$cosmovisor_dir/upgrades/${UPGRADE_TO}/upgrade-info.json"
+		pushd "$(pwd)"
+		cd "$cosmovisor_dir"
+		ln -snf genesis current
+		popd
+
 		genesis_file=${valdir}/config/genesis.json
 		addrbook_file=${valdir}/config/genesis.json
 		rm -f "$genesis_file"
@@ -491,11 +498,13 @@ function prepare_state() {
 			cd "${valdir}"
 
 			echo "Unpacking snapshot from $snap_file..."
+			rm -rf data
 
 			# shellcheck disable=SC2086
 			(pv -petrafb -i 5 "$snap_file" | eval "$tar_cmd") 2>&1 | stdbuf -o0 tr '\r' '\n'
 
 			rm -f upgrade-info.json
+			rm -f data/upgrade-info.json
 
 			popd
 		else
@@ -520,8 +529,72 @@ function prepare_state() {
 
 	rvaldir=$validators_dir/.akash0
 
+	if [[ -n "$UPGRADE_TO" && -x "$rvaldir/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" ]]; then
+		echo "rolling snapshot back before testnetify"
+		# Testnetify must commit one fork-state block before the scheduled upgrade
+		# height. Cached snapshots can sit at the last pre-upgrade height, so roll
+		# back twice to start one block earlier.
+		"$rvaldir/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" rollback --home "$rvaldir" --hard
+		"$rvaldir/cosmovisor/upgrades/$UPGRADE_TO/bin/akash" rollback --home "$rvaldir" --hard
+		cat >"$rvaldir/data/priv_validator_state.json" <<EOL
+{
+  "height": "0",
+  "round": 0,
+  "step": 0
+}
+EOL
+	fi
+
 	echo "testnetifying state"
-	$AKASH testnetify --home="$rvaldir" --testnet-rootdir="$validators_dir" --testnet-config="${STATE_CONFIG}" --yes
+	$AKASH testnetify --home="$rvaldir" --testnet-rootdir="$validators_dir" --testnet-config="${STATE_CONFIG}" --yes &
+	rpid=$!
+
+	local testnetify_wait
+	local testnetify_timeout
+	testnetify_wait=0
+	testnetify_timeout=${TESTNETIFY_TIMEOUT_SECONDS:-600}
+	while kill -0 "$rpid" 2>/dev/null; do
+		if [[ -f "$rvaldir/data/upgrade-info.json" ]]; then
+			kill "$rpid" 2>/dev/null || true
+			wait "$rpid" || true
+			rpid=
+			break
+		fi
+		if [[ $testnetify_wait -ge $testnetify_timeout ]]; then
+			kill "$rpid" 2>/dev/null || true
+			wait "$rpid" || true
+			echoerr "testnetify did not complete within ${testnetify_timeout}s"
+			return 1
+		fi
+		sleep 1
+		testnetify_wait=$((testnetify_wait + 1))
+	done
+	if [[ -n "$rpid" ]]; then
+		if ! wait "$rpid"; then
+			if [[ ! -f "$rvaldir/data/upgrade-info.json" ]]; then
+				return 1
+			fi
+		fi
+	fi
+
+	if [[ -f "$rvaldir/data/upgrade-info.json" ]]; then
+		local rollback_upgrade
+		rollback_upgrade=$UPGRADE_TO
+		if [[ -z "$rollback_upgrade" ]]; then
+			rollback_upgrade=$(jq -r '.name' "$rvaldir/data/upgrade-info.json")
+		fi
+
+		echo "testnetify reached upgrade height; rolling back pending upgrade block"
+		"$rvaldir/cosmovisor/upgrades/$rollback_upgrade/bin/akash" rollback --home "$rvaldir" --hard
+		rm -f "$rvaldir/data/upgrade-info.json"
+		cat >"$rvaldir/data/priv_validator_state.json" <<EOL
+{
+  "height": "0",
+  "round": 0,
+  "step": 0
+}
+EOL
+	fi
 
 	if [[ $MAX_VALIDATORS -gt 1 ]]; then
 		echo "starting testnet validator"

tests/fullsurface/fullsurface_test.go

@@ -0,0 +1,92 @@
+//go:build e2e.integration
+
+// Package fullsurface hosts the in-process driver for the exhaustive gRPC
+// transaction/query suite (grpcsuite). It spins up a single-validator
+// testutil/network and runs the same suite that the post-upgrade worker runs
+// against a testnetify-forked node, giving a fast (minutes) iteration and per-PR
+// CI signal without the full upgrade cycle.
+//
+// It lives in its own package (not tests/e2e) so it compiles standalone against
+// the pinned SDK; the existing tests/e2e integration tests currently only build
+// against the workspace-local chain-sdk.
+package fullsurface
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cosmos/cosmos-sdk/codec"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	govtypes "github.com/cosmos/cosmos-sdk/x/gov/types"
+	govv1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1"
+
+	"pkg.akt.dev/node/v2/tests/upgrade/grpcsuite"
+	"pkg.akt.dev/node/v2/testutil"
+	"pkg.akt.dev/node/v2/testutil/network"
+)
+
+var grpcSuiteMode = flag.String("grpc-suite-mode", string(grpcsuite.RunModeAll), "grpcsuite run mode: all, tx, or query")
+
+func TestFullSurfaceGRPC(t *testing.T) {
+	mode, err := grpcsuite.ParseRunMode(*grpcSuiteMode)
+	require.NoError(t, err)
+
+	// Short gov voting period + low min deposit so the suite's gov fast-path (used
+	// for every MsgUpdateParams and other gov-gated messages) completes quickly.
+	// The post-upgrade harness gets the equivalent via tests/upgrade/testnet.json.
+	cfg := network.DefaultConfig(testutil.NewTestNetworkFixture,
+		network.WithInterceptState(func(cdc codec.Codec, moduleName string, state json.RawMessage) json.RawMessage {
+			if moduleName != govtypes.ModuleName {
+				return nil
+			}
+			var gs govv1.GenesisState
+			cdc.MustUnmarshalJSON(state, &gs)
+			vp := 8 * time.Second
+			ep := 6 * time.Second
+			gs.Params.VotingPeriod = &vp
+			gs.Params.ExpeditedVotingPeriod = &ep
+			gs.Params.MinDeposit = sdk.NewCoins(sdk.NewInt64Coin("uakt", 10_000_000))
+			return cdc.MustMarshalJSON(&gs)
+		}),
+	)
+	cfg.NumValidators = 1
+
+	net := network.New(t, cfg)
+	defer net.Cleanup()
+
+	_, err = net.WaitForHeightWithTimeout(2, 30*time.Second)
+	require.NoError(t, err)
+
+	val := net.Validators[0]
+	require.NotEmpty(t, val.AppConfig.GRPC.Address, "gRPC server must be enabled")
+
+	root, err := filepath.Abs("../..")
+	require.NoError(t, err)
+
+	env := grpcsuite.Env{
+		GRPCEndpoint: val.AppConfig.GRPC.Address,
+		ChainID:      cfg.ChainID,
+		RepoRoot:     root,
+		Cdc:          cfg.Codec,
+		InterfaceReg: cfg.InterfaceRegistry,
+		TxConfig:     cfg.TxConfig,
+		Amino:        cfg.LegacyAmino,
+		Keyring:      val.ClientCtx.Keyring,
+		Funder:       "node0", // validator key name in the keyring
+		FunderAddr:   val.Address,
+		BondDenom:    cfg.BondDenom,
+		GasPrices:    "0.025uakt",
+		// Enforce full coverage: fail if any in-scope tx or query is not
+		// exercised (a new RPC added by a future upgrade turns this red).
+		RequireFullCoverage: true,
+		Mode:                mode,
+	}
+
+	grpcsuite.Run(context.Background(), t, env)
+}

tests/upgrade/grpcsuite/README.md

@@ -0,0 +1,87 @@
+# grpcsuite — exhaustive post-upgrade gRPC tx/query verification
+
+`grpcsuite` exercises **every in-scope Akash and mounted Cosmos SDK transaction,
+plus every in-scope query over the chain's gRPC API** after a network upgrade, so
+that a passing run means the upgraded chain's API surface is verified accurate.
+CLI is explicitly out of scope.
+
+It runs in two places against the **same** code:
+
+| Driver | Build tag | Target | Speed | Purpose |
+| --- | --- | --- | --- | --- |
+| `tests/upgrade` universal worker (`grpcsurface_worker_test.go`) | `e2e.upgrade` | testnetify-forked, freshly-upgraded validator | slow (full upgrade) | **acceptance path** — runs after every upgrade |
+| `tests/fullsurface` (`fullsurface_test.go`) | `e2e.integration` | in-process single-validator `testutil/network` | minutes | fast local iteration + per-PR CI |
+
+Run the fast path: `make test-grpc-surface`.
+
+For narrower local debugging:
+- `make test-grpc-surface-tx` runs the authored tx packs and gates tx coverage.
+  Packs still call query RPCs for setup and assertions.
+- `make test-grpc-surface-query` runs the dynamic query smoke sweep and gates
+  query coverage without mutating chain state.
+
+The acceptance path runs automatically inside `make -C tests/upgrade test` (the
+existing `network-upgrade` CI job), because the suite is registered as the
+**universal post-upgrade worker** (runs for every upgrade name).
+
+## How it works
+
+- **All checks run over gRPC.** Queries route through a gRPC-backed
+  `client.Context` (`WithGRPCClient`); transactions are signed locally and
+  broadcast via the cosmos `tx.ServiceClient`, then polled for inclusion — nothing
+  touches Comet RPC. See `grpcconn.go`, `txbroadcast.go`.
+- **Dynamic discovery + coverage gate (`coverage.go`).** The binary drives *what*
+  is tested: gRPC server reflection lists the live Query services; the
+  InterfaceRegistry lists the registered `Msg` implementations, restricted to the
+  active served version. The gate fails if any in-scope Msg/query was never
+  exercised — so a new RPC added by a future upgrade turns CI red until a case
+  exists. This is what keeps "test every single one" self-maintaining.
+- **Dynamic query smoke sweep (`smoke.go`).** Fires an empty request at every
+  discovered query method, auto-covering the entire query surface and failing on
+  any advertised-but-`Unimplemented` method. Authored query cases add correctness
+  with real inputs.
+- **Authored, dependency-ordered packs (`pack_*.go`).** A valid tx needs real
+  prior state (lease ⇐ bid ⇐ order ⇐ deployment+provider), so transactions are
+  authored scenarios, not fuzzed. Packs run in order and thread created handles
+  through the shared `World`.
+- **Governance fast-path (`gov.go`).** Gov-gated messages (every `MsgUpdateParams`,
+  `MsgFundVault`, etc.) are batched into one proposal, voted through by the
+  funder (who holds ~all voting power on a testnetify fork / single-validator
+  net), and recorded once passed. Requires a short voting period (set in
+  `tests/upgrade/testnet.json` and the in-process driver).
+
+## Adding a module pack
+
+1. Create `pack_<module>.go` implementing `Pack` (`Name`, `Available`, `Run`).
+2. Gate `Available` on `d.HasModule("akash.<module>.<version>")` (Query service).
+3. In `Run`, fund accounts via `s.FundAccountDefault`, build msgs with the SDK
+   types, broadcast with `s.BroadcastOK` (happy path) or `s.BroadcastExpectErr`
+   (negative / disabled), query with the generated `NewQueryClient(s.Conn)`.
+4. For gov-gated msgs, build them with `Authority: s.GovAuthority()` and pass via
+   `s.PassGovProposal(...)`.
+5. Publish handles other packs need via `s.World.Set`; read with `s.World.Get`.
+6. Register the pack in `pack.go` in dependency order.
+
+The deployment, provider and gov-params packs are worked examples.
+
+## Status
+
+**Full in-scope coverage** — run `make test-grpc-surface` and read the
+`coverage:` line:
+- **Queries: 130/130 in-scope methods** (smoke sweep + authored typed cases).
+- **Transactions: 93/93 in-scope messages.** Akash coverage includes deployment,
+  provider, market, audit, escrow, cert, oracle, bme and module params. Cosmos SDK
+  coverage includes auth, authz, bank, consensus, distribution, evidence,
+  feegrant, gov v1, legacy gov v1beta1, mint, slashing, staking, upgrade and
+  vesting. CosmWasm coverage includes store, instantiate, execute, migrate, admin,
+  code config, pin/unpin and wasm params.
+
+`RequireFullCoverage` is `true` in both drivers, so the gate **fails** if any
+in-scope tx or query stops being exercised — e.g. when a future upgrade adds a new
+Akash or mounted Cosmos SDK RPC, until a case is authored for it.
+
+The suite targets exactly the surface the running `main` binary serves: discovery
+is driven by gRPC reflection + the interface registry, and every pack is
+reflection-gated (`Available`/`HasModule`). A pack whose module is not served by
+the binary under test is skipped automatically, so the suite always matches the
+active surface with no manual bookkeeping.