Skip to content

Commit b8eef52

Browse files
haojinminghaojinming
authored
[close #108] Br debug checksum cmd (#114)
* implement debug checksum cmd Signed-off-by: haojinming <jinming.hao@pingcap.com> * add api version check Signed-off-by: haojinming <jinming.hao@pingcap.com> add warning Signed-off-by: haojinming <jinming.hao@pingcap.com> revert changes Signed-off-by: haojinming <jinming.hao@pingcap.com> * move common functions Signed-off-by: haojinming <jinming.hao@pingcap.com>
1 parent 93bccd0 commit b8eef52

11 files changed

Lines changed: 217 additions & 103 deletions

File tree

‎br/cmd/br/debug.go‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,23 @@ package main
44

55
import (
66
"context"
7+
"crypto/tls"
78
"path"
89
"reflect"
10+
"strings"
911

1012
"github.com/gogo/protobuf/proto"
1113
"github.com/pingcap/errors"
1214
"github.com/pingcap/log"
1315
"github.com/spf13/cobra"
16+
"github.com/tikv/migration/br/pkg/checksum"
17+
"github.com/tikv/migration/br/pkg/conn"
1418
"github.com/tikv/migration/br/pkg/metautil"
19+
"github.com/tikv/migration/br/pkg/pdutil"
1520
"github.com/tikv/migration/br/pkg/task"
1621
"github.com/tikv/migration/br/pkg/utils"
1722
"github.com/tikv/migration/br/pkg/version/build"
23+
pd "github.com/tikv/pd/client"
1824
)
1925

2026
// NewDebugCommand return a debug subcommand.
@@ -51,10 +57,9 @@ func newCheckSumCommand() *cobra.Command {
5157
Short: "check the backup data",
5258
Args: cobra.NoArgs,
5359
RunE: func(cmd *cobra.Command, _ []string) error {
54-
return errors.Errorf("checksum is unsupported")
60+
return runRawChecksumCommand(cmd, "RawChecksum")
5561
},
5662
}
57-
command.Hidden = true
5863
return command
5964
}
6065

@@ -77,6 +82,7 @@ func newBackupMetaValidateCommand() *cobra.Command {
7782
},
7883
}
7984
command.Flags().Uint64("offset", 0, "the offset of table id alloctor")
85+
command.Hidden = true
8086
return command
8187
}
8288

@@ -223,3 +229,55 @@ func setPDConfigCommand() *cobra.Command {
223229
}
224230
return pdConfigCmd
225231
}
232+
233+
func runRawChecksumCommand(command *cobra.Command, cmdName string) error {
234+
cfg := task.Config{LogProgress: HasLogFile()}
235+
err := cfg.ParseFromFlags(command.Flags())
236+
if err != nil {
237+
command.SilenceUsage = false
238+
return errors.Trace(err)
239+
}
240+
241+
ctx := GetDefaultContext()
242+
pdAddress := strings.Join(cfg.PD, ",")
243+
securityOption := pd.SecurityOption{}
244+
var tlsConf *tls.Config = nil
245+
if cfg.TLS.IsEnabled() {
246+
securityOption.CAPath = cfg.TLS.CA
247+
securityOption.CertPath = cfg.TLS.Cert
248+
securityOption.KeyPath = cfg.TLS.Key
249+
tlsConf, err = cfg.TLS.ToTLSConfig()
250+
if err != nil {
251+
return errors.Trace(err)
252+
}
253+
}
254+
pdCtrl, err := pdutil.NewPdController(ctx, pdAddress, tlsConf, securityOption)
255+
if err != nil {
256+
return errors.Trace(err)
257+
}
258+
storageAPIVersion, err := conn.GetTiKVApiVersion(ctx, pdCtrl.GetPDClient(), tlsConf)
259+
if err != nil {
260+
return errors.Trace(err)
261+
}
262+
_, _, backupMeta, err := task.ReadBackupMeta(ctx, metautil.MetaFile, &cfg)
263+
if err != nil {
264+
return errors.Trace(err)
265+
}
266+
fileChecksum, keyRanges := task.CalcChecksumAndRangeFromBackupMeta(ctx, backupMeta, storageAPIVersion)
267+
if !task.CheckBackupAPIVersion(storageAPIVersion, backupMeta.ApiVersion) {
268+
return errors.Errorf("Unsupported api version, storage:%s, backup meta:%s.",
269+
storageAPIVersion.String(), backupMeta.ApiVersion.String())
270+
}
271+
checksumMethod := checksum.StorageChecksumCommand
272+
if storageAPIVersion != backupMeta.ApiVersion {
273+
checksumMethod = checksum.StorageScanCommand
274+
}
275+
276+
executor := checksum.NewExecutor(keyRanges, cfg.PD, pdCtrl.GetPDClient(), storageAPIVersion,
277+
cfg.ChecksumConcurrency)
278+
err = checksum.Run(ctx, cmdName, executor, checksumMethod, fileChecksum)
279+
if err != nil {
280+
return errors.Trace(err)
281+
}
282+
return nil
283+
}

‎br/pkg/backup/client.go‎

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ import (
66
"context"
77
"crypto/tls"
88
"encoding/hex"
9-
"encoding/json"
109
"fmt"
1110
"io"
12-
"io/ioutil"
13-
"net/http"
1411
"os"
1512
"strings"
1613
"sync"
@@ -66,14 +63,6 @@ const (
6663
RegionUnit ProgressUnit = "region"
6764
)
6865

69-
type StorageConfig struct {
70-
APIVersion int `json:"api-version"`
71-
EnableTTL bool `json:"enable-ttl"`
72-
}
73-
type StoreConfig struct {
74-
Storage StorageConfig `json:"storage"`
75-
}
76-
7766
// Client is a client instructs TiKV how to do a backup.
7867
type Client struct {
7968
mgr ClientMgr
@@ -91,7 +80,7 @@ func NewBackupClient(ctx context.Context, mgr ClientMgr, config *tls.Config) (*C
9180
log.Info("new backup client")
9281
pdClient := mgr.GetPDClient()
9382
clusterID := pdClient.GetClusterID(ctx)
94-
curAPIVer, err := GetCurrentTiKVApiVersion(ctx, mgr.GetPDClient(), config)
83+
curAPIVer, err := conn.GetTiKVApiVersion(ctx, mgr.GetPDClient(), config)
9584
if err != nil {
9685
return nil, errors.Trace(err)
9786
}
@@ -142,53 +131,6 @@ func (bc *Client) GetTS(ctx context.Context, duration time.Duration, ts uint64)
142131
return backupTS, nil
143132
}
144133

145-
func GetCurrentTiKVApiVersion(ctx context.Context, pdClient pd.Client, tlsConf *tls.Config) (kvrpcpb.APIVersion, error) {
146-
allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, conn.SkipTiFlash)
147-
if err != nil {
148-
return kvrpcpb.APIVersion_V1, errors.Trace(err)
149-
} else if len(allStores) == 0 {
150-
return kvrpcpb.APIVersion_V1, errors.New("store are empty")
151-
}
152-
schema := "http"
153-
httpClient := http.Client{}
154-
if tlsConf != nil {
155-
httpClient = http.Client{
156-
Transport: &http.Transport{TLSClientConfig: tlsConf},
157-
}
158-
schema = "https"
159-
}
160-
url := fmt.Sprintf("%s://%s/config", schema, allStores[0].StatusAddress)
161-
resp, err := httpClient.Get(url)
162-
if err != nil {
163-
return kvrpcpb.APIVersion_V1, errors.Trace(err)
164-
}
165-
defer resp.Body.Close()
166-
body, err := ioutil.ReadAll(resp.Body)
167-
if err != nil {
168-
return kvrpcpb.APIVersion_V1, errors.Trace(err)
169-
}
170-
var cfg StoreConfig
171-
if err := json.Unmarshal(body, &cfg); err != nil {
172-
return kvrpcpb.APIVersion_V1, errors.Trace(err)
173-
}
174-
var apiVersion kvrpcpb.APIVersion
175-
if cfg.Storage.APIVersion == 0 { // in old version without apiversion config. it's APIV1.
176-
apiVersion = kvrpcpb.APIVersion_V1
177-
} else if cfg.Storage.APIVersion == 1 {
178-
if cfg.Storage.EnableTTL {
179-
apiVersion = kvrpcpb.APIVersion_V1TTL
180-
} else {
181-
apiVersion = kvrpcpb.APIVersion_V1
182-
}
183-
} else if cfg.Storage.APIVersion == 2 {
184-
apiVersion = kvrpcpb.APIVersion_V2
185-
} else {
186-
errMsg := fmt.Sprintf("Invalid apiversion %d", cfg.Storage.APIVersion)
187-
return kvrpcpb.APIVersion_V1, errors.New(errMsg)
188-
}
189-
return apiVersion, nil
190-
}
191-
192134
func (bc *Client) GetCurAPIVersion() kvrpcpb.APIVersion {
193135
return bc.curAPIVer
194136
}

‎br/pkg/backup/client_test.go‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
. "github.com/pingcap/check"
1212
backuppb "github.com/pingcap/kvproto/pkg/brpb"
1313
"github.com/pingcap/kvproto/pkg/errorpb"
14-
"github.com/pingcap/kvproto/pkg/kvrpcpb"
1514
"github.com/tikv/client-go/v2/oracle"
1615
"github.com/tikv/client-go/v2/testutils"
1716
"github.com/tikv/client-go/v2/tikv"
@@ -226,31 +225,3 @@ func (r *testBackup) TestCheckBackupIsLocked(c *C) {
226225
err = backup.CheckBackupStorageIsLocked(ctx, r.storage)
227226
c.Assert(err, ErrorMatches, "backup lock file and sst file exist in(.+)")
228227
}
229-
230-
func (r *testBackup) TestGetCurrentTiKVApiVersion(c *C) {
231-
ctx := context.Background()
232-
233-
httpmock.Activate()
234-
defer httpmock.DeactivateAndReset()
235-
// Exact URL match
236-
httpmock.RegisterResponder("GET", `=~^/config`,
237-
httpmock.NewStringResponder(200, `{"storage":{"api-version":1, "enable-ttl":false}}`))
238-
239-
apiVer, err := backup.GetCurrentTiKVApiVersion(ctx, r.mockPDClient, nil)
240-
c.Assert(err, IsNil)
241-
c.Assert(apiVer, Equals, kvrpcpb.APIVersion_V1)
242-
243-
httpmock.RegisterResponder("GET", `=~^/config`,
244-
httpmock.NewStringResponder(200, `{"storage":{"api-version":1, "enable-ttl":true}}`))
245-
246-
apiVer, err = backup.GetCurrentTiKVApiVersion(ctx, r.mockPDClient, nil)
247-
c.Assert(err, IsNil)
248-
c.Assert(apiVer, Equals, kvrpcpb.APIVersion_V1TTL)
249-
250-
httpmock.RegisterResponder("GET", `=~^/config`,
251-
httpmock.NewStringResponder(200, `{"storage":{"api-version":2, "enable-ttl":true}}`))
252-
253-
apiVer, err = backup.GetCurrentTiKVApiVersion(ctx, r.mockPDClient, nil)
254-
c.Assert(err, IsNil)
255-
c.Assert(apiVer, Equals, kvrpcpb.APIVersion_V2)
256-
}

‎br/pkg/checksum/executor.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,9 @@ func (exec *Executor) Execute(
333333

334334
func Run(ctx context.Context, cmdName string,
335335
executor *Executor, method StorageChecksumMethod, expect Checksum) error {
336+
if executor.apiVersion != kvrpcpb.APIVersion_V1 {
337+
fmt.Printf("\033[1;37;41m%s\033[0m\n", "Warning: TiKV cluster is TTL enabled, checksum may be mismatch if some data expired during backup/restore.")
338+
}
336339
glue := new(gluetikv.Glue)
337340
updateCh := glue.StartProgress(ctx, cmdName+" Checksum", int64(len(executor.keyRanges)), false)
338341
progressCallBack := func(unit backup.ProgressUnit) {

‎br/pkg/conn/conn.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ package conn
55
import (
66
"context"
77
"crypto/tls"
8+
"encoding/json"
89
"fmt"
10+
"io/ioutil"
11+
"net/http"
912
"os"
1013
"sync"
1114
"time"
@@ -14,8 +17,10 @@ import (
1417
"github.com/pingcap/errors"
1518
"github.com/pingcap/failpoint"
1619
backuppb "github.com/pingcap/kvproto/pkg/brpb"
20+
"github.com/pingcap/kvproto/pkg/kvrpcpb"
1721
"github.com/pingcap/kvproto/pkg/metapb"
1822
"github.com/pingcap/log"
23+
"github.com/pingcap/tidb/br/pkg/conn"
1924
"github.com/tikv/client-go/v2/tikv"
2025
"github.com/tikv/client-go/v2/txnkv/txnlock"
2126
berrors "github.com/tikv/migration/br/pkg/errors"
@@ -416,3 +421,58 @@ func (mgr *Mgr) Close() {
416421

417422
mgr.PdController.Close()
418423
}
424+
425+
type StorageConfig struct {
426+
APIVersion int `json:"api-version"`
427+
EnableTTL bool `json:"enable-ttl"`
428+
}
429+
type StoreConfig struct {
430+
Storage StorageConfig `json:"storage"`
431+
}
432+
433+
func GetTiKVApiVersion(ctx context.Context, pdClient pd.Client, tlsConf *tls.Config) (kvrpcpb.APIVersion, error) {
434+
allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, conn.SkipTiFlash)
435+
if err != nil {
436+
return kvrpcpb.APIVersion_V1, errors.Trace(err)
437+
} else if len(allStores) == 0 {
438+
return kvrpcpb.APIVersion_V1, errors.New("store are empty")
439+
}
440+
schema := "http"
441+
httpClient := http.Client{}
442+
if tlsConf != nil {
443+
httpClient = http.Client{
444+
Transport: &http.Transport{TLSClientConfig: tlsConf},
445+
}
446+
schema = "https"
447+
}
448+
url := fmt.Sprintf("%s://%s/config", schema, allStores[0].StatusAddress)
449+
resp, err := httpClient.Get(url)
450+
if err != nil {
451+
return kvrpcpb.APIVersion_V1, errors.Trace(err)
452+
}
453+
defer resp.Body.Close()
454+
body, err := ioutil.ReadAll(resp.Body)
455+
if err != nil {
456+
return kvrpcpb.APIVersion_V1, errors.Trace(err)
457+
}
458+
var cfg StoreConfig
459+
if err := json.Unmarshal(body, &cfg); err != nil {
460+
return kvrpcpb.APIVersion_V1, errors.Trace(err)
461+
}
462+
var apiVersion kvrpcpb.APIVersion
463+
if cfg.Storage.APIVersion == 0 { // in old version without apiversion config. it's APIV1.
464+
apiVersion = kvrpcpb.APIVersion_V1
465+
} else if cfg.Storage.APIVersion == 1 {
466+
if cfg.Storage.EnableTTL {
467+
apiVersion = kvrpcpb.APIVersion_V1TTL
468+
} else {
469+
apiVersion = kvrpcpb.APIVersion_V1
470+
}
471+
} else if cfg.Storage.APIVersion == 2 {
472+
apiVersion = kvrpcpb.APIVersion_V2
473+
} else {
474+
errMsg := fmt.Sprintf("Invalid apiversion %d", cfg.Storage.APIVersion)
475+
return kvrpcpb.APIVersion_V1, errors.New(errMsg)
476+
}
477+
return apiVersion, nil
478+
}

‎br/pkg/conn/conn_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import (
66
"context"
77
"testing"
88

9+
"github.com/jarcoal/httpmock"
910
"github.com/pingcap/errors"
1011
"github.com/pingcap/failpoint"
12+
"github.com/pingcap/kvproto/pkg/kvrpcpb"
1113
"github.com/pingcap/kvproto/pkg/metapb"
1214
"github.com/stretchr/testify/require"
1315
"github.com/tikv/migration/br/pkg/pdutil"
@@ -269,3 +271,45 @@ func TestGetConnOnCanceledContext(t *testing.T) {
269271
require.Error(t, err)
270272
require.Contains(t, err.Error(), "context canceled")
271273
}
274+
275+
type mockPDClient struct {
276+
pd.Client
277+
}
278+
279+
func (c *mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) {
280+
store := &metapb.Store{
281+
Id: 0,
282+
Address: "127.0.0.1",
283+
}
284+
return []*metapb.Store{store}, nil
285+
}
286+
287+
func TestGetTiKVApiVersion(t *testing.T) {
288+
ctx := context.Background()
289+
290+
mockPdClient := mockPDClient{}
291+
292+
httpmock.Activate()
293+
defer httpmock.DeactivateAndReset()
294+
// Exact URL match
295+
httpmock.RegisterResponder("GET", `=~^/config`,
296+
httpmock.NewStringResponder(200, `{"storage":{"api-version":1, "enable-ttl":false}}`))
297+
298+
apiVer, err := GetTiKVApiVersion(ctx, &mockPdClient, nil)
299+
require.Equal(t, err, nil)
300+
require.Equal(t, apiVer, kvrpcpb.APIVersion_V1)
301+
302+
httpmock.RegisterResponder("GET", `=~^/config`,
303+
httpmock.NewStringResponder(200, `{"storage":{"api-version":1, "enable-ttl":true}}`))
304+
305+
apiVer, err = GetTiKVApiVersion(ctx, &mockPdClient, nil)
306+
require.Equal(t, err, nil)
307+
require.Equal(t, apiVer, kvrpcpb.APIVersion_V1TTL)
308+
309+
httpmock.RegisterResponder("GET", `=~^/config`,
310+
httpmock.NewStringResponder(200, `{"storage":{"api-version":2, "enable-ttl":true}}`))
311+
312+
apiVer, err = GetTiKVApiVersion(ctx, &mockPdClient, nil)
313+
require.Equal(t, err, nil)
314+
require.Equal(t, apiVer, kvrpcpb.APIVersion_V2)
315+
}

0 commit comments

Comments
 (0)