From 8a0848e331eb03dbe69bb69fae20461d05d0dc5b Mon Sep 17 00:00:00 2001 From: abing <410174833@qq.com> Date: Thu, 9 Nov 2023 14:08:24 +0800 Subject: [PATCH 1/5] sui check --- action/action_check_aggregation.go | 2 +- action/action_move_lint.go | 195 +++++++++++++++++++++++++++++ aptos-check.yml | 13 +- consts/consts.go | 3 + consts/contract_check_enum.go | 1 + executor/executor.go | 4 +- 6 files changed, 209 insertions(+), 9 deletions(-) create mode 100644 action/action_move_lint.go diff --git a/action/action_check_aggregation.go b/action/action_check_aggregation.go index d38f00a..7381199 100644 --- a/action/action_check_aggregation.go +++ b/action/action_check_aggregation.go @@ -123,7 +123,7 @@ func (a *CheckAggregationAction) Hook() (*model.ActionResult, error) { styleGuideValidationsReportRaw.Context = contractCheckResultDetailsList checkResultList = append(checkResultList, styleGuideValidationsReportRaw) } - if strings.Contains(result, consts.ContractSecurityAnalysisReport.Name) || strings.Contains(result, consts.FormalSpecificationAndVerificationReport.Name) { + if strings.Contains(result, consts.ContractSecurityAnalysisReport.Name) || strings.Contains(result, consts.FormalSpecificationAndVerificationReport.Name) || strings.Contains(result, consts.SuiContractSecurityAnalysisReport.Name) { var securityAnalysisReport model.ContractCheckResult[[]model.ContractStyleGuideValidationsReportDetails] err := json.Unmarshal(file, &securityAnalysisReport) if err != nil { diff --git a/action/action_move_lint.go b/action/action_move_lint.go new file mode 100644 index 0000000..a253ae1 --- /dev/null +++ b/action/action_move_lint.go @@ -0,0 +1,195 @@ +package action + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + path2 "path" + "strconv" + "strings" + + "github.com/hamster-shared/aline-engine/consts" + "github.com/hamster-shared/aline-engine/logger" + "github.com/hamster-shared/aline-engine/model" + "github.com/hamster-shared/aline-engine/output" +) + +// MoveLint MoveLint Sui contract check +type MoveLint struct { + path string + ctx context.Context + output *output.Output +} + +func NewMoveLint(step model.Step, ctx context.Context, output *output.Output) *MoveLint { + return &MoveLint{ + path: step.With["path"], + ctx: ctx, + output: output, + } +} + +func (m *MoveLint) Pre() error { + return nil +} + +func (m *MoveLint) Hook() (*model.ActionResult, error) { + + stack := m.ctx.Value(STACK).(map[string]interface{}) + + workdir, ok := stack["workdir"].(string) + if !ok { + return nil, errors.New("workdir is empty") + } + jobName, ok := stack["name"].(string) + if !ok { + return nil, errors.New("get job name error") + } + jobId, ok := stack["id"].(string) + if !ok { + return nil, errors.New("get job id error") + } + userHomeDir, err := os.UserHomeDir() + if err != nil { + logger.Errorf("Failed to get home directory, the file will be saved to the current directory, err is %s", err.Error()) + userHomeDir = "." + } + + codePath := path2.Join(workdir, m.path) + destDir := path2.Join(userHomeDir, consts.ArtifactoryDir, jobName, consts.CheckName, jobId, consts.MoveLintCheckOutputDir) + _, err = os.Stat(destDir) + if os.IsNotExist(err) { + err := os.MkdirAll(destDir, os.ModePerm) + if err != nil { + return nil, err + } + } + commandTemplate := consts.MoveLintCheck + command := fmt.Sprintf(commandTemplate, codePath) + fields := strings.Fields(command) + out, err := m.ExecuteCommand(fields, workdir) + if out == "" && err != nil { + return nil, err + } + dest := path2.Join(destDir, consts.MoveLintCheckOutputDir+consts.SuffixType) + create, err := os.Create(dest) + if err != nil { + return nil, err + } + _, err = create.WriteString(out) + if err != nil { + return nil, err + } + create.Close() + + m.path = destDir + id, err := strconv.Atoi(jobId) + if err != nil { + return nil, err + } + actionResult := model.ActionResult{ + Artifactorys: nil, + Reports: []model.Report{ + { + Id: id, + Url: "", + Type: 2, + }, + }, + } + return &actionResult, err +} + +func (m *MoveLint) Post() error { + dest := path2.Join(m.path, consts.MoveLintCheckOutputDir+consts.SuffixType) + fileByte, err := os.ReadFile(dest) + if err != nil { + return err + } + successFlag := true + var checkResultDetailsList []model.ContractCheckResultDetails[[]model.ContractStyleGuideValidationsReportDetails] + var total int + startIndex := strings.Index(string(fileByte), "[{") + var moveLintJsonList []MoveLintJson + var resultString string + if startIndex != -1 { + resultString = string(fileByte)[startIndex:] + err := json.Unmarshal([]byte(resultString), &moveLintJsonList) + if err != nil { + return err + } + } + + fileToMoveLintJsonMap := make(map[string][]MoveLintJson) + for _, moveLintJson := range moveLintJsonList { + fileToMoveLintJsonMap[moveLintJson.File] = append(fileToMoveLintJsonMap[moveLintJson.File], moveLintJson) + } + + for file, moveLintJsons := range fileToMoveLintJsonMap { + var suiCheckReportDetailsList []model.ContractStyleGuideValidationsReportDetails + for _, moveLintJson := range moveLintJsons { + var suiCheckReportDetails model.ContractStyleGuideValidationsReportDetails + suiCheckReportDetails.Level = moveLintJson.Level + suiCheckReportDetails.Line = string(rune(moveLintJson.Lines[0])) + suiCheckReportDetails.Note = moveLintJson.Title + ": " + moveLintJson.Verbose + suiCheckReportDetailsList = append(suiCheckReportDetailsList, suiCheckReportDetails) + successFlag = false + } + contractCheckResultDetails := model.NewContractCheckResultDetails(file, len(suiCheckReportDetailsList), suiCheckReportDetailsList) + total = total + contractCheckResultDetails.Issue + checkResultDetailsList = append(checkResultDetailsList, contractCheckResultDetails) + } + + var result string + if successFlag { + result = consts.CheckSuccess.Result + } else { + result = consts.CheckFail.Result + } + checkResult := model.NewContractCheckResult(consts.SuiContractSecurityAnalysisReport.Name, result, consts.SuiContractSecurityAnalysisReport.Tool, checkResultDetailsList, total) + create, err := os.Create(path2.Join(m.path, consts.CheckResult)) + fmt.Println(checkResult) + if err != nil { + return err + } + marshal, err := json.Marshal(checkResult) + if err != nil { + return err + } + _, err = create.WriteString(string(marshal)) + if err != nil { + return err + } + create.Close() + return nil +} + +func (m *MoveLint) ExecuteCommand(commands []string, workdir string) (string, error) { + c := exec.CommandContext(m.ctx, commands[0], commands[1:]...) // mac linux + c.Dir = workdir + logger.Debugf("execute move-lint check command: %s", strings.Join(commands, " ")) + m.output.WriteCommandLine(strings.Join(commands, " ")) + out, err := c.CombinedOutput() + fmt.Println(string(out)) + m.output.WriteCommandLine(string(out)) + if err != nil { + m.output.WriteLine(err.Error()) + } + return string(out), err +} + +type MoveLintJson struct { + No int `json:"no"` + Wiki string `json:"wiki"` + Title string `json:"title"` + Verbose string `json:"verbose"` + Level string `json:"level"` + Description string `json:"description"` + File string `json:"file"` + Start int `json:"start"` + End int `json:"end"` + Lines []int `json:"lines"` +} diff --git a/aptos-check.yml b/aptos-check.yml index c15e9c4..9a4615a 100644 --- a/aptos-check.yml +++ b/aptos-check.yml @@ -6,25 +6,24 @@ stages: - name: git-clone uses: git-checkout with: - url: https://github.com/hamster-template/aptos-token-staking.git + url: https://github.com/daixiang11/hello_word.git branch: main - Check Aptos : + Check Sui: needs: - Initialization steps: - - name: aptos-install + - name: install run: | yarn install - - name: aptos-check - uses: aptos-check + - name: sui-check + uses: sui-check with: path: - cachePath: /Users/abing/.move:/root/.move Output Results: needs: - - Check Aptos + - Check Sui steps: - name: check-aggregation uses: check-aggregation diff --git a/consts/consts.go b/consts/consts.go index 66428ad..bcec3a6 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -58,6 +58,9 @@ const ( MoveProve = "Move Prove" MoveProveCheckOutputDir = "move-prover" MoveProveCheck = "docker run --rm %s -v %s:/tmp hamstershare/aptoslabs-tools:aptos-node-v1.3.3 aptos move prove --package-dir %s %s" + MoveLint = "Move-Lint" + MoveLintCheckOutputDir = "move-lint" + MoveLintCheck = " docker run --rm -v %s:/tmp hamstershare/rust-tools:1.73 move-lint -j -p /tmp" ) var InkUrlMap = map[string]string{ diff --git a/consts/contract_check_enum.go b/consts/contract_check_enum.go index a6fafba..37cd6c3 100644 --- a/consts/contract_check_enum.go +++ b/consts/contract_check_enum.go @@ -17,6 +17,7 @@ var ( ContractMethodsPropertiesReport = contractCheckResult("Contract Methods Properties Report", "sol-profiler") ContractStyleGuideValidationsReport = contractCheckResult("Code Quality Analysis Report", "Solhint") ContractSecurityAnalysisReport = contractCheckResult("Security Analysis Report", "Mythril") + SuiContractSecurityAnalysisReport = contractCheckResult("Security Analysis Report", "Move Lint") FrontEndCheckReport = contractCheckResult("Static analysis report", "ESLint") EthGasCheckReport = contractCheckResult("Gas Usage Analysis Report", "eth-gas-reporter") FormalSpecificationAndVerificationReport = contractCheckResult("Formal Specification and Verification Report", "Move Prove") diff --git a/executor/executor.go b/executor/executor.go index d9e8a50..79290ed 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -244,12 +244,14 @@ func (e *Executor) Execute(id int, job *model.Job) error { ah = action.NewCheckAggregationAction(step, ctx, jobWrapper.Output) } else if step.Uses == "deploy-ink-contract" { ah = action.NewInkAction(step, ctx, jobWrapper.Output) - } else if step.Uses == "frontend-check" { + } else if step.Uses == "frontend- check" { ah = action.NewEslintAction(step, ctx, jobWrapper.Output) } else if step.Uses == "eth-gas-reporter" { ah = action.NewEthGasReporterAction(step, ctx, jobWrapper.Output) } else if step.Uses == "aptos-check" { ah = action.NewMoveProverAction(step, ctx, jobWrapper.Output) + } else if step.Uses == "sui-check" { + ah = action.NewMoveLint(step, ctx, jobWrapper.Output) } else if step.Uses == "workdir" { ah = action.NewWorkdirAction(step, ctx, jobWrapper.Output) } else if step.Uses == "openai" { From d19527b0220016ced9306b4ed59a7c5851e3cf03 Mon Sep 17 00:00:00 2001 From: abing <410174833@qq.com> Date: Mon, 11 Dec 2023 19:48:07 +0800 Subject: [PATCH 2/5] sui check --- action/action_move_lint.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/action/action_move_lint.go b/action/action_move_lint.go index a253ae1..66d0231 100644 --- a/action/action_move_lint.go +++ b/action/action_move_lint.go @@ -74,6 +74,9 @@ func (m *MoveLint) Hook() (*model.ActionResult, error) { if out == "" && err != nil { return nil, err } + if strings.Contains(out, "Error {") { + return nil, errors.New(out) + } dest := path2.Join(destDir, consts.MoveLintCheckOutputDir+consts.SuffixType) create, err := os.Create(dest) if err != nil { From f39499bc3e9827fc7beff6afb4e1e942659f3528 Mon Sep 17 00:00:00 2001 From: abing <410174833@qq.com> Date: Tue, 12 Dec 2023 14:11:36 +0800 Subject: [PATCH 3/5] sui check --- action/action_move_lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action/action_move_lint.go b/action/action_move_lint.go index 66d0231..dc5df5e 100644 --- a/action/action_move_lint.go +++ b/action/action_move_lint.go @@ -136,7 +136,7 @@ func (m *MoveLint) Post() error { for _, moveLintJson := range moveLintJsons { var suiCheckReportDetails model.ContractStyleGuideValidationsReportDetails suiCheckReportDetails.Level = moveLintJson.Level - suiCheckReportDetails.Line = string(rune(moveLintJson.Lines[0])) + suiCheckReportDetails.Line = strconv.Itoa(moveLintJson.Lines[0]) suiCheckReportDetails.Note = moveLintJson.Title + ": " + moveLintJson.Verbose suiCheckReportDetailsList = append(suiCheckReportDetailsList, suiCheckReportDetails) successFlag = false From dfc2bc9584c56ff7416bff1a5fc7198c93bfb21e Mon Sep 17 00:00:00 2001 From: mohaijiang Date: Thu, 14 Dec 2023 16:20:49 +0800 Subject: [PATCH 4/5] fix pipeline step error --- executor/executor.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index 79290ed..fb1f882 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -192,7 +192,7 @@ func (e *Executor) Execute(id int, job *model.Job) error { stageWapper.StartTime = time.Now() jobWrapper.Stages[index] = stageWapper jobWrapper.Output.NewStage(stageWapper.Name) - jober.SaveJobDetail(jobWrapper.Name, jobWrapper) + _ = jober.SaveJobDetail(jobWrapper.Name, jobWrapper) for index, step := range stageWapper.Stage.Steps { var ah action.ActionHandler @@ -205,7 +205,7 @@ func (e *Executor) Execute(id int, job *model.Job) error { } stageWapper.Stage.Steps[index].StartTime = time.Now() stageWapper.Stage.Steps[index].Status = model.STATUS_RUNNING - jober.SaveJobDetail(jobWrapper.Name, jobWrapper) + _ = jober.SaveJobDetail(jobWrapper.Name, jobWrapper) actionContext := aline_context.NewActionContext(step, ctx, jobWrapper.Output) // 如果 step 超时，则调用 cancel，在这里存储该 job 的计时器 // 每次新 step 时，都会重新设置该计时器，所以不需要存储到底是哪个 step @@ -276,10 +276,11 @@ func (e *Executor) Execute(id int, job *model.Job) error { } else { stageWapper.Stage.Steps[index].Status = model.STATUS_SUCCESS } - err := jober.SaveJobDetail(jobWrapper.Name, jobWrapper) + _ = jober.SaveJobDetail(jobWrapper.Name, jobWrapper) if err != nil { - logger.Error("SaveJobDetail error: ", err) + break } + } if err != nil { @@ -290,7 +291,7 @@ func (e *Executor) Execute(id int, job *model.Job) error { dataTime := time.Since(stageWapper.StartTime) stageWapper.Duration = dataTime.Milliseconds() jobWrapper.Stages[index] = stageWapper - jober.SaveJobDetail(jobWrapper.Name, jobWrapper) + _ = jober.SaveJobDetail(jobWrapper.Name, jobWrapper) logger.Info("}") if err != nil { cancel() From b286131a6b5358d08aee5a77c5bc5647ccf42db0 Mon Sep 17 00:00:00 2001 From: abing <410174833@qq.com> Date: Tue, 20 Feb 2024 17:03:45 +0800 Subject: [PATCH 5/5] icp identity --- action/action_icp_build.go | 34 ++++++++-------- action/action_icp_deploy.go | 79 ++++++++++++++++++------------------- 2 files changed, 55 insertions(+), 58 deletions(-) diff --git a/action/action_icp_build.go b/action/action_icp_build.go index 8abefb2..6f86437 100644 --- a/action/action_icp_build.go +++ b/action/action_icp_build.go @@ -53,27 +53,27 @@ func (a *ICPBuildAction) Hook() (*model.ActionResult, error) { icNetwork = "local" } - locker, err := utils.Lock() - if err != nil { - return nil, err - } - - defer utils.Unlock(locker) - - a.ac.WriteLine(fmt.Sprintf("use identity: %s", a.userId)) - cmd := exec.Command(DFX_BIN, "identity", "use", a.userId) - cmd.Dir = workdir - output, err := cmd.CombinedOutput() - a.ac.WriteLine(string(output)) - if err != nil { - return nil, err - } + //locker, err := utils.Lock() + //if err != nil { + // return nil, err + //} + // + //defer utils.Unlock(locker) + + //a.ac.WriteLine(fmt.Sprintf("use identity: %s", a.userId)) + //cmd := exec.Command(DFX_BIN, "identity", "use", a.userId) + //cmd.Dir = workdir + //output, err := cmd.CombinedOutput() + //a.ac.WriteLine(string(output)) + //if err != nil { + // return nil, err + //} actionResult := &model.ActionResult{} - cmd = exec.Command(DFX_BIN, "build", "--check", "--network", icNetwork) + cmd := exec.Command(DFX_BIN, "build", "--check", "--network", icNetwork, "--identity", a.userId) cmd.Dir = workdir - output, err = cmd.CombinedOutput() + output, err := cmd.CombinedOutput() a.ac.WriteLine(string(output)) if err != nil { return nil, err diff --git a/action/action_icp_deploy.go b/action/action_icp_deploy.go index 91b0eb8..eb4e2b0 100644 --- a/action/action_icp_deploy.go +++ b/action/action_icp_deploy.go @@ -58,7 +58,6 @@ func (a *ICPDeployAction) Pre() error { workdir := a.ac.GetWorkdir() cacheDir := path.Join(workdir, ".dfx") - //fmt.Println(path) homeDir, _ := os.UserHomeDir() toCacheDir := path.Join(homeDir, "pipelines/jobs", a.ac.GetJobName(), ".dfx") @@ -73,23 +72,22 @@ func (a *ICPDeployAction) Pre() error { if icNetwork == "" { icNetwork = "local" } - dfxBin := "/usr/local/bin/dfx" - - locker, err := utils.Lock() - if err != nil { - return err - } - defer utils.Unlock(locker) - - cmd := exec.Command(dfxBin, "identity", "use", a.userId, "-qq") - logger.Info("execute: ", strings.Join(cmd.Args, " ")) - cmd.Dir = workdir - output, err := cmd.CombinedOutput() - logger.Info(string(output)) - if err != nil { - return err - } + //locker, err := utils.Lock() + //if err != nil { + // return err + //} + // + //defer utils.Unlock(locker) + // + //cmd := exec.Command(DFX_BIN, "identity", "use", a.userId, "-qq") + //logger.Info("execute: ", strings.Join(cmd.Args, " ")) + //cmd.Dir = workdir + //output, err := cmd.CombinedOutput() + //logger.Info(string(output)) + //if err != nil { + // return err + //} var dfxJson DFXJson if err := json.Unmarshal([]byte(a.dfxJson), &dfxJson); err != nil { @@ -107,7 +105,7 @@ func (a *ICPDeployAction) Pre() error { } for canisterId, _ := range dfxJson.Canisters { - cmd := exec.Command(dfxBin, "canister", "create", canisterId, "--network", icNetwork, "--with-cycles", "300000000000") + cmd := exec.Command(DFX_BIN, "canister", "create", canisterId, "--network", icNetwork, "--with-cycles", "300000000000", "--identity", a.userId) logger.Info("execute: ", strings.Join(cmd.Args, " ")) cmd.Dir = workdir logger.Infof("execute create canister command: %s", cmd) @@ -126,7 +124,7 @@ func (a *ICPDeployAction) Pre() error { } logger.Info("write dfx.json: ", a.dfxJson) - err = os.WriteFile(path.Join(workdir, "dfx.json"), []byte(a.dfxJson), 0644) + err := os.WriteFile(path.Join(workdir, "dfx.json"), []byte(a.dfxJson), 0644) if err != nil { logger.Error("write dfx.json error:", err) return err @@ -154,31 +152,30 @@ func (a *ICPDeployAction) Hook() (*model.ActionResult, error) { if icNetwork == "" { icNetwork = "local" } - dfxBin := "/usr/local/bin/dfx" - locker, err := utils.Lock() - if err != nil { - return nil, err - } - - defer utils.Unlock(locker) - - cmd := exec.Command(dfxBin, "identity", "use", a.userId) - logger.Info("execute: ", strings.Join(cmd.Args, " ")) - cmd.Dir = workdir - output, err := cmd.CombinedOutput() - if err != nil { - return nil, err - } - logger.Info(string(output)) + //locker, err := utils.Lock() + //if err != nil { + // return nil, err + //} + // + //defer utils.Unlock(locker) + // + //cmd := exec.Command(DFX_BIN, "identity", "use", a.userId) + //logger.Info("execute: ", strings.Join(cmd.Args, " ")) + //cmd.Dir = workdir + //output, err := cmd.CombinedOutput() + //if err != nil { + // return nil, err + //} + //logger.Info(string(output)) actionResult := &model.ActionResult{} if a.deployCmd { - cmd = exec.Command(dfxBin, "deploy", "--network", icNetwork, "--with-cycles", "300000000000") + cmd := exec.Command(DFX_BIN, "deploy", "--network", icNetwork, "--with-cycles", "300000000000", "--identity", a.userId) logger.Info("execute: ", strings.Join(cmd.Args, " ")) cmd.Dir = workdir logger.Infof("execute deploy canister command: %s", cmd) - output, err = cmd.CombinedOutput() + output, err := cmd.CombinedOutput() if err != nil { logger.Error("execute deploy fail:", err) a.ac.WriteLine(string(output)) @@ -191,7 +188,7 @@ func (a *ICPDeployAction) Hook() (*model.ActionResult, error) { urls := analyzeURL(string(output)) for key, value := range urls { - cmd = exec.Command(dfxBin, "canister", "id", key, "--network", icNetwork, "-qq") + cmd = exec.Command(DFX_BIN, "canister", "id", key, "--network", icNetwork, "-qq", "--identity", a.userId) cmd.Dir = workdir output, err = cmd.CombinedOutput() logger.Info(string(output)) @@ -218,9 +215,9 @@ func (a *ICPDeployAction) Hook() (*model.ActionResult, error) { } for canisterName, _ := range dfxJson.Canisters { - cmd = exec.Command(dfxBin, "canister", "id", canisterName, "--network", icNetwork, "-qq") + cmd := exec.Command(DFX_BIN, "canister", "id", canisterName, "--network", icNetwork, "-qq", "--identity", a.userId) cmd.Dir = workdir - output, err = cmd.CombinedOutput() + output, err := cmd.CombinedOutput() logger.Info(string(output)) canisterId := strings.TrimSpace(string(output)) @@ -229,7 +226,7 @@ func (a *ICPDeployAction) Hook() (*model.ActionResult, error) { } fmt.Println("canisterName : ", canisterName) - cmd := exec.Command(dfxBin, "canister", "install", canisterName, "--yes", "--mode=reinstall", "--network", icNetwork) + cmd = exec.Command(DFX_BIN, "canister", "install", canisterName, "--yes", "--mode=reinstall", "--network", icNetwork, "--identity", a.userId) logger.Info("execute: ", strings.Join(cmd.Args, " ")) cmd.Dir = workdir output, err = cmd.CombinedOutput()