From 772819b8ad44b3f4709b6473ec6a340f3b8c1481 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Mon, 31 Jul 2023 13:19:53 +0200 Subject: [PATCH 01/19] Add repo find command --- cmd/repo/command.go | 16 +++ cmd/repo/find/command.go | 65 ++++++++++++ cmd/repo/find/error.go | 30 ++++++ cmd/repo/find/flag.go | 20 ++++ cmd/repo/find/runner.go | 145 ++++++++++++++++++++++++++ go.mod | 1 + go.sum | 3 + pkg/githubclient/client.go | 4 + pkg/githubclient/client_repository.go | 12 +-- 9 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 cmd/repo/find/command.go create mode 100644 cmd/repo/find/error.go create mode 100644 cmd/repo/find/flag.go create mode 100644 cmd/repo/find/runner.go diff --git a/cmd/repo/command.go b/cmd/repo/command.go index d390a209a..6b45e89c4 100644 --- a/cmd/repo/command.go +++ b/cmd/repo/command.go @@ -8,6 +8,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/giantswarm/devctl/cmd/repo/find" "github.com/giantswarm/devctl/cmd/repo/setup" ) @@ -35,6 +36,20 @@ func New(config Config) (*cobra.Command, error) { var err error + var findCmd *cobra.Command + { + c := find.Config{ + Logger: config.Logger, + Stderr: config.Stderr, + Stdout: config.Stdout, + } + + findCmd, err = find.New(c) + if err != nil { + return nil, microerror.Mask(err) + } + } + var setupCmd *cobra.Command { c := setup.Config{ @@ -67,6 +82,7 @@ func New(config Config) (*cobra.Command, error) { f.Init(c) + c.AddCommand(findCmd) c.AddCommand(setupCmd) return c, nil diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go new file mode 100644 index 000000000..ae63c9913 --- /dev/null +++ b/cmd/repo/find/command.go @@ -0,0 +1,65 @@ +package find + +import ( + "io" + "os" + + "github.com/giantswarm/microerror" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +const ( + name = "find" + description = `Find repositories based on very specific criteria + +The --what flag allows to specify what search criteria should be used. When combining several critaria, +a repository will be returned when it's macthing at least one criteria (boolean OR). + +Note: archived repositories are always excluded. + +Criteria: + +- README_OLD_CIRCLECI_BAGDE - A /README.md file is present, containing an outdated CircleCI badge. +- NO_CODEOWNERS - No /README-md file is present. +- DEFAULT_BRANCH_MASTER - The default branch is named 'master'. +` +) + +type Config struct { + Logger *logrus.Logger + Stderr io.Writer + Stdout io.Writer +} + +func New(config Config) (*cobra.Command, error) { + if config.Logger == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config) + } + if config.Stderr == nil { + config.Stderr = os.Stderr + } + if config.Stdout == nil { + config.Stdout = os.Stdout + } + + f := &flag{} + + r := &runner{ + flag: f, + logger: config.Logger, + stderr: config.Stderr, + stdout: config.Stdout, + } + + c := &cobra.Command{ + Use: name, + Short: description, + Long: description, + RunE: r.Run, + } + + f.Init(c) + + return c, nil +} diff --git a/cmd/repo/find/error.go b/cmd/repo/find/error.go new file mode 100644 index 000000000..07a89a5e5 --- /dev/null +++ b/cmd/repo/find/error.go @@ -0,0 +1,30 @@ +package find + +import "github.com/giantswarm/microerror" + +var invalidConfigError = µerror.Error{ + Kind: "invalidConfigError", +} + +// IsInvalidConfig asserts invalidConfigError. +func IsInvalidConfig(err error) bool { + return microerror.Cause(err) == invalidConfigError +} + +var invalidFlagError = µerror.Error{ + Kind: "invalidFlagError", +} + +// IsInvalidFlag asserts invalidFlagError. +func IsInvalidFlag(err error) bool { + return microerror.Cause(err) == invalidFlagError +} + +var envVarNotFoundError = µerror.Error{ + Kind: "envVarNotFoundError", +} + +// IsEnvVarNotFound asserts envVarNotFoundError. +func IsEnvVarNotFound(err error) bool { + return microerror.Cause(err) == envVarNotFoundError +} diff --git a/cmd/repo/find/flag.go b/cmd/repo/find/flag.go new file mode 100644 index 000000000..7ac0b9c33 --- /dev/null +++ b/cmd/repo/find/flag.go @@ -0,0 +1,20 @@ +package find + +import "github.com/spf13/cobra" + +type flag struct { + What []string + + IncludeArchived bool + IncludeFork bool +} + +func (f *flag) Init(cmd *cobra.Command) { + cmd.PersistentFlags().BoolVar(&f.IncludeArchived, "include-archived", false, "Also return archived repositories.") + cmd.PersistentFlags().BoolVar(&f.IncludeFork, "include-fork", false, "Also return giantswarm forks of repositories.") + cmd.PersistentFlags().StringSliceVar(&f.What, "what", []string{}, "What repos to find. See full help for all available criteria.") +} + +func (f *flag) Validate() error { + return nil +} diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go new file mode 100644 index 000000000..1ac0ef79e --- /dev/null +++ b/cmd/repo/find/runner.go @@ -0,0 +1,145 @@ +package find + +import ( + "context" + b64 "encoding/base64" + "fmt" + "io" + "os" + "strings" + + "github.com/giantswarm/devctl/pkg/githubclient" + "github.com/giantswarm/microerror" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "k8s.io/utils/strings/slices" +) + +const ( + githubTokenEnvVar = "GITHUB_TOKEN" + githubOrg = "giantswarm" +) + +type runner struct { + flag *flag + logger *logrus.Logger + stdout io.Writer + stderr io.Writer +} + +func (r *runner) Run(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + err := r.flag.Validate() + if err != nil { + return microerror.Mask(err) + } + + err = r.run(ctx, cmd, args) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) error { + token, found := os.LookupEnv(githubTokenEnvVar) + if !found { + return microerror.Maskf(envVarNotFoundError, "environement variable %#q was not found", githubTokenEnvVar) + } + + if len(r.flag.What) < 1 { + return microerror.Maskf(invalidConfigError, "no search criteria specified via --what flag") + } + + c := githubclient.Config{ + Logger: r.logger, + AccessToken: token, + } + + client, err := githubclient.New(c) + if err != nil { + return microerror.Mask(err) + } + + repos, err := client.ListRepositories(ctx, githubOrg) + if err != nil { + return microerror.Mask(err) + } + + realClient := client.GetUnderlyingClient(ctx) + + matchingReposCount := 0 + + for i, repo := range repos { + matched := []string{} + + repoMetadata, err := client.GetRepository(ctx, githubOrg, repo.Name) + if err != nil { + return microerror.Mask(err) + } + + if !r.flag.IncludeArchived && repoMetadata.GetArchived() { + // Skip archived repos + continue + } + + if !r.flag.IncludeFork && *repoMetadata.Fork { + // Skip fork repos + continue + } + + // Check for matching criteria + + if slices.Contains(r.flag.What, "README_OLD_CIRCLECI_BAGDE") { + fileContent, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "/README.md", nil) + if err == nil { + decodedContent, _ := b64.StdEncoding.DecodeString(*fileContent.Content) + if strings.Contains(string(decodedContent), fmt.Sprintf("https://circleci.com/gh/%s", githubOrg)) { + output := " /README.md has old CircleCI badge (README_OLD_CIRCLECI_BAGDE)\n" + if repoMetadata.Fork != nil && *repoMetadata.Fork { + output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + } + output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) + output += fmt.Sprintf(" Badge code via https://app.circleci.com/settings/project/github/%s/%s/status-badges)\n", githubOrg, repo.Name) + matched = append(matched, output) + } + } + } + + if slices.Contains(r.flag.What, "NO_CODEOWNERS") { + _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "/CODEOWNERS", nil) + if err != nil { + output := " /CODEOWNERS file not present (NO_CODEOWNERS)\n" + if repoMetadata.Fork != nil && *repoMetadata.Fork { + output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + } + matched = append(matched, output) + } + } + + if slices.Contains(r.flag.What, "DEFAULT_BRANCH_MASTER") { + if repoMetadata.GetDefaultBranch() == "master" { + output := " Default branch is 'master'\n" + if repoMetadata.Fork != nil && *repoMetadata.Fork { + output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + } + matched = append(matched, output) + } + } + + if len(matched) > 0 { + matchingReposCount++ + fmt.Printf("\n(%d of %d) https://github.com/%s/%s\n", i, len(repos), githubOrg, repo.Name) + for _, item := range matched { + fmt.Print(item) + } + } + + } + + fmt.Printf("Found %d matching non-archived repositoiries\n", matchingReposCount) + + return nil +} diff --git a/go.mod b/go.mod index 615cadf26..3f4054156 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( golang.org/x/net v0.10.0 golang.org/x/oauth2 v0.8.0 k8s.io/apimachinery v0.20.12 + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index dfd28c241..ace3cc489 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,7 @@ github.com/rhysd/go-github-selfupdate v1.2.3/go.mod h1:mp/N8zj6jFfBQy/XMYoWsmfzx github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -307,6 +308,8 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/pkg/githubclient/client.go b/pkg/githubclient/client.go index 3707405e9..c3e3a10c2 100644 --- a/pkg/githubclient/client.go +++ b/pkg/githubclient/client.go @@ -54,3 +54,7 @@ func (c *Client) getUnderlyingClient(ctx context.Context) *github.Client { return client } + +func (c *Client) GetUnderlyingClient(ctx context.Context) *github.Client { + return c.getUnderlyingClient(ctx) +} diff --git a/pkg/githubclient/client_repository.go b/pkg/githubclient/client_repository.go index 630dc4390..8b0b06b58 100644 --- a/pkg/githubclient/client_repository.go +++ b/pkg/githubclient/client_repository.go @@ -12,22 +12,18 @@ import ( ) func (c *Client) ListRepositories(ctx context.Context, owner string) ([]Repository, error) { - c.logger.Infof("listing repositories for owner %#q", owner) - underlyingClient := c.getUnderlyingClient(ctx) var repos []Repository { opt := &github.RepositoryListByOrgOptions{ ListOptions: github.ListOptions{ - PerPage: 500, + PerPage: 100, }, Type: "all", } for pageCnt := 0; ; pageCnt++ { - c.logger.Infof("listing page %d of repositories for owner %#q", pageCnt, owner) - pageRepos, resp, err := underlyingClient.Repositories.ListByOrg(ctx, owner, opt) if err != nil { return nil, microerror.Mask(err) @@ -46,19 +42,13 @@ func (c *Client) ListRepositories(ctx context.Context, owner string) ([]Reposito repos = append(repos, r) } - - c.logger.Infof("listed page %d of %d repositories for owner %#q", pageCnt, len(pageRepos), owner) } } - c.logger.Infof("listed %d repositories for owner %#q", len(repos), owner) - return repos, nil } func (c *Client) GetRepository(ctx context.Context, owner, repo string) (*github.Repository, error) { - c.logger.Infof("get repository details for \"%s/%s\"", owner, repo) - underlyingClient := c.getUnderlyingClient(ctx) repository, response, err := underlyingClient.Repositories.Get(ctx, owner, repo) From 1161b130ce2c6d03c7ded2b80b475b08556cdfef Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Mon, 31 Jul 2023 13:23:31 +0200 Subject: [PATCH 02/19] Upgrade go-github to v53 --- cmd/repo/setup/runner.go | 2 +- go.mod | 8 +++++--- go.sum | 17 ++++++++++++----- pkg/githubclient/client.go | 2 +- pkg/githubclient/client_content.go | 2 +- pkg/githubclient/client_repository.go | 2 +- pkg/githubclient/error_internal.go | 2 +- pkg/githubclient/types.go | 2 +- 8 files changed, 23 insertions(+), 14 deletions(-) diff --git a/cmd/repo/setup/runner.go b/cmd/repo/setup/runner.go index 0efb9f53a..3fb018c12 100644 --- a/cmd/repo/setup/runner.go +++ b/cmd/repo/setup/runner.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/giantswarm/microerror" - "github.com/google/go-github/v44/github" + "github.com/google/go-github/v53/github" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/go.mod b/go.mod index 3f4054156..9e1fc140e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/giantswarm/devctl -go 1.18 +go 1.19 require ( github.com/Masterminds/semver/v3 v3.2.1 @@ -10,7 +10,7 @@ require ( github.com/giantswarm/microerror v0.4.0 github.com/giantswarm/micrologger v0.6.0 github.com/giantswarm/release-operator/v3 v3.2.0 - github.com/google/go-github/v44 v44.1.0 + github.com/google/go-github/v53 v53.2.0 github.com/pelletier/go-toml v1.9.5 github.com/rhysd/go-github-selfupdate v1.2.3 github.com/sirupsen/logrus v1.9.0 @@ -24,6 +24,8 @@ require ( ) require ( + github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/giantswarm/k8smetadata v0.6.0 // indirect github.com/go-kit/log v0.2.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -46,7 +48,7 @@ require ( github.com/stretchr/testify v1.7.2 // indirect github.com/tcnksm/go-gitconfig v0.1.2 // indirect github.com/ulikunitz/xz v0.5.9 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.7.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index ace3cc489..dd0a572ca 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -10,8 +12,12 @@ github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdn github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -80,11 +86,11 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= -github.com/google/go-github/v44 v44.1.0 h1:shWPaufgdhr+Ad4eo/pZv9ORTxFpsxPEPEuuXAKIQGA= -github.com/google/go-github/v44 v44.1.0/go.mod h1:iWn00mWcP6PRWHhXm0zuFJ8wbEjE5AGO5D5HXYM4zgw= +github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= +github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -179,8 +185,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -231,6 +237,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/githubclient/client.go b/pkg/githubclient/client.go index c3e3a10c2..5d75945bc 100644 --- a/pkg/githubclient/client.go +++ b/pkg/githubclient/client.go @@ -4,7 +4,7 @@ import ( "context" "github.com/giantswarm/microerror" - "github.com/google/go-github/v44/github" + "github.com/google/go-github/v53/github" "github.com/sirupsen/logrus" "golang.org/x/oauth2" ) diff --git a/pkg/githubclient/client_content.go b/pkg/githubclient/client_content.go index 961d3f0dd..47dc308e0 100644 --- a/pkg/githubclient/client_content.go +++ b/pkg/githubclient/client_content.go @@ -4,7 +4,7 @@ import ( "context" "github.com/giantswarm/microerror" - "github.com/google/go-github/v44/github" + "github.com/google/go-github/v53/github" ) func (c *Client) GetFile(ctx context.Context, owner, repo, path, ref string) (RepositoryFile, error) { diff --git a/pkg/githubclient/client_repository.go b/pkg/githubclient/client_repository.go index 8b0b06b58..1b28ffeb4 100644 --- a/pkg/githubclient/client_repository.go +++ b/pkg/githubclient/client_repository.go @@ -8,7 +8,7 @@ import ( "regexp" "github.com/giantswarm/microerror" - "github.com/google/go-github/v44/github" + "github.com/google/go-github/v53/github" ) func (c *Client) ListRepositories(ctx context.Context, owner string) ([]Repository, error) { diff --git a/pkg/githubclient/error_internal.go b/pkg/githubclient/error_internal.go index 234201ba8..d333aaf6b 100644 --- a/pkg/githubclient/error_internal.go +++ b/pkg/githubclient/error_internal.go @@ -1,6 +1,6 @@ package githubclient -import "github.com/google/go-github/v44/github" +import "github.com/google/go-github/v53/github" func isGithub404(err error) bool { if err == nil { diff --git a/pkg/githubclient/types.go b/pkg/githubclient/types.go index f63c188fc..f4e36aaaa 100644 --- a/pkg/githubclient/types.go +++ b/pkg/githubclient/types.go @@ -4,7 +4,7 @@ import ( "time" "github.com/giantswarm/microerror" - "github.com/google/go-github/v44/github" + "github.com/google/go-github/v53/github" ) type Repository struct { From 20b923e215a1179745644f293e7b9247c91065f1 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Fri, 4 Aug 2023 11:40:23 +0200 Subject: [PATCH 03/19] Resolve merge conflict --- cmd/repo/find/command.go | 11 +++++--- cmd/repo/find/runner.go | 55 +++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index ae63c9913..2de102fd1 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -1,3 +1,5 @@ +// Package find provides the 'repo find' command, which helps to discover +// GitHub repositories with certain features, or certain features missing. package find import ( @@ -20,9 +22,12 @@ Note: archived repositories are always excluded. Criteria: -- README_OLD_CIRCLECI_BAGDE - A /README.md file is present, containing an outdated CircleCI badge. -- NO_CODEOWNERS - No /README-md file is present. -- DEFAULT_BRANCH_MASTER - The default branch is named 'master'. +- DEFAULT_BRANCH_MASTER - The default branch is named 'master'. +- HAS_DOCS_DIR - Has a directory named 'docs' on the root level. +- HAS_PR_TEMPLATE_IN_DOCS - Has the file docs/pull_request_template.md (which is not the desired location). +- NO_CODEOWNERS - The /CODEOWNERS file is not present. +- NO_DESCRIPTION - Repository description is missing. +- README_OLD_CIRCLECI_BAGDE - An outdated CircleCI badge is present in the README. ` ) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 1ac0ef79e..24e1a1cb0 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -18,6 +18,14 @@ import ( const ( githubTokenEnvVar = "GITHUB_TOKEN" githubOrg = "giantswarm" + + // Criteria names + critHasDocsDirectory = "HAS_DOCS_DIR" + critHasPrTemplateInDocs = "HAS_PR_TEMPLATE_IN_DOCS" + critReadmeHasOldCircleCiBadge = "README_OLD_CIRCLECI_BAGDE" + critNoCodeownersFile = "NO_CODEOWNERS" + critNoDescription = "NO_DESCRIPTION" + critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" ) type runner struct { @@ -92,12 +100,33 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err // Check for matching criteria - if slices.Contains(r.flag.What, "README_OLD_CIRCLECI_BAGDE") { - fileContent, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "/README.md", nil) + if slices.Contains(r.flag.What, critHasDocsDirectory) { + _, items, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "docs", nil) + if err == nil { + output := fmt.Sprintf(" /docs directory exists (%s)\n", critHasDocsDirectory) + for _, item := range items { + path := item.GetPath() + ftype := item.GetType() + output += fmt.Sprintf(" %s %s\n", path, ftype) + } + matched = append(matched, output) + } + } + + if slices.Contains(r.flag.What, critHasPrTemplateInDocs) { + _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "docs/pull_request_template.md", nil) + if err == nil { + output := fmt.Sprintf(" /docs/pull_request_template.md file exists (%s)\n", critHasPrTemplateInDocs) + matched = append(matched, output) + } + } + + if slices.Contains(r.flag.What, critReadmeHasOldCircleCiBadge) { + fileContent, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "README.md", nil) if err == nil { decodedContent, _ := b64.StdEncoding.DecodeString(*fileContent.Content) if strings.Contains(string(decodedContent), fmt.Sprintf("https://circleci.com/gh/%s", githubOrg)) { - output := " /README.md has old CircleCI badge (README_OLD_CIRCLECI_BAGDE)\n" + output := fmt.Sprintf(" /README.md has old CircleCI badge (%s)\n", critReadmeHasOldCircleCiBadge) if repoMetadata.Fork != nil && *repoMetadata.Fork { output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } @@ -108,10 +137,10 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } - if slices.Contains(r.flag.What, "NO_CODEOWNERS") { - _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "/CODEOWNERS", nil) + if slices.Contains(r.flag.What, critNoCodeownersFile) { + _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "CODEOWNERS", nil) if err != nil { - output := " /CODEOWNERS file not present (NO_CODEOWNERS)\n" + output := fmt.Sprintf(" /CODEOWNERS file not present (%s)\n", critNoCodeownersFile) if repoMetadata.Fork != nil && *repoMetadata.Fork { output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } @@ -119,9 +148,16 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } - if slices.Contains(r.flag.What, "DEFAULT_BRANCH_MASTER") { + if slices.Contains(r.flag.What, critNoDescription) { + if repoMetadata.GetDescription() == "" { + output := fmt.Sprintf(" Repository has no description (%s)\n", critNoDescription) + matched = append(matched, output) + } + } + + if slices.Contains(r.flag.What, critDefaultBranchMaster) { if repoMetadata.GetDefaultBranch() == "master" { - output := " Default branch is 'master'\n" + output := fmt.Sprintf(" Default branch is 'master' (%s)\n", critDefaultBranchMaster) if repoMetadata.Fork != nil && *repoMetadata.Fork { output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } @@ -129,6 +165,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } + // Print output per repo if len(matched) > 0 { matchingReposCount++ fmt.Printf("\n(%d of %d) https://github.com/%s/%s\n", i, len(repos), githubOrg, repo.Name) @@ -139,7 +176,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } - fmt.Printf("Found %d matching non-archived repositoiries\n", matchingReposCount) + fmt.Printf("\nFound %d matching non-archived repositoiries\n", matchingReposCount) return nil } From 6c72ad11ffd41529ba6a72fc593ee8cc2651d20e Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 8 Aug 2023 09:34:48 +0200 Subject: [PATCH 04/19] Improve help --- cmd/repo/find/command.go | 44 ++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index 2de102fd1..d2c9c35fa 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -18,16 +18,48 @@ const ( The --what flag allows to specify what search criteria should be used. When combining several critaria, a repository will be returned when it's macthing at least one criteria (boolean OR). +Example: + + devctl repo find --what NO_DESCRIPTION + Note: archived repositories are always excluded. Criteria: -- DEFAULT_BRANCH_MASTER - The default branch is named 'master'. -- HAS_DOCS_DIR - Has a directory named 'docs' on the root level. -- HAS_PR_TEMPLATE_IN_DOCS - Has the file docs/pull_request_template.md (which is not the desired location). -- NO_CODEOWNERS - The /CODEOWNERS file is not present. -- NO_DESCRIPTION - Repository description is missing. -- README_OLD_CIRCLECI_BAGDE - An outdated CircleCI badge is present in the README. + DEFAULT_BRANCH_MASTER + + The default branch is named 'master'. We want to rename these to 'main'. + + HAS_DOCS_DIR + + The repo has a directory named 'docs' on the root level. This is the place + where we want to store technical documentation for the repo. + + HAS_PR_TEMPLATE_IN_DOCS + + Has the file docs/pull_request_template.md, which is not the desired location. + We want this to be in .github/pull_request_template.md. + + NO_CODEOWNERS + + There is no CODEOWNERS file in the root folder, which means that the repository + has no owner. We want repos to have owners, ideally. + + NO_DESCRIPTION + + Repository description is missing. We want a meaningful description to be present, + to understand easily what the repository is about, even from lists. + + NO_README + + Repository has no README.md file in the root folder. We want thjis to be present, + to have some basic info and documentation available. + + README_OLD_CIRCLECI_BAGDE + + There is an outdated (broken) CircleCI badge is present in the README. This should + better get replaced by an up-to-date one, as it can otherwise not fulfil its purpose, + and broken images never make a good impression. ` ) From 87f1241fded456d4484694d05c4154dc333d2392 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 8 Aug 2023 09:35:14 +0200 Subject: [PATCH 05/19] Add README_OLD_GODOC_LINK and NO_README criteria --- cmd/repo/find/runner.go | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 24e1a1cb0..8c782dd49 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -23,8 +23,10 @@ const ( critHasDocsDirectory = "HAS_DOCS_DIR" critHasPrTemplateInDocs = "HAS_PR_TEMPLATE_IN_DOCS" critReadmeHasOldCircleCiBadge = "README_OLD_CIRCLECI_BAGDE" + critReadmeHasOldGodocLink = "README_OLD_GODOC_LINK" critNoCodeownersFile = "NO_CODEOWNERS" critNoDescription = "NO_DESCRIPTION" + critNoReadme = "NO_README" critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" ) @@ -98,6 +100,8 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err continue } + //fmt.Printf("Permissions: %#v\n", repoMetadata.Permissions) + // Check for matching criteria if slices.Contains(r.flag.What, critHasDocsDirectory) { @@ -121,20 +125,38 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } - if slices.Contains(r.flag.What, critReadmeHasOldCircleCiBadge) { - fileContent, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "README.md", nil) - if err == nil { + if slices.Contains(r.flag.What, critReadmeHasOldCircleCiBadge) || slices.Contains(r.flag.What, critNoReadme) || slices.Contains(r.flag.What, critReadmeHasOldGodocLink) { + fileContent, _, resp, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "README.md", nil) + + if (slices.Contains(r.flag.What, critReadmeHasOldCircleCiBadge) || slices.Contains(r.flag.What, critReadmeHasOldGodocLink)) && err == nil { decodedContent, _ := b64.StdEncoding.DecodeString(*fileContent.Content) - if strings.Contains(string(decodedContent), fmt.Sprintf("https://circleci.com/gh/%s", githubOrg)) { - output := fmt.Sprintf(" /README.md has old CircleCI badge (%s)\n", critReadmeHasOldCircleCiBadge) - if repoMetadata.Fork != nil && *repoMetadata.Fork { - output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + + if slices.Contains(r.flag.What, critReadmeHasOldCircleCiBadge) { + if strings.Contains(string(decodedContent), fmt.Sprintf("https://circleci.com/gh/%s", githubOrg)) { + output := fmt.Sprintf(" /README.md has old CircleCI badge (%s)\n", critReadmeHasOldCircleCiBadge) + if repoMetadata.Fork != nil && *repoMetadata.Fork { + output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + } + output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) + output += fmt.Sprintf(" Badge code via https://app.circleci.com/settings/project/github/%s/%s/status-badges)\n", githubOrg, repo.Name) + matched = append(matched, output) + } + } + + if slices.Contains(r.flag.What, critReadmeHasOldGodocLink) { + if strings.Contains(string(decodedContent), "godoc.org") { + output := fmt.Sprintf(" /README.md has link to godoc.org (%s)\n", critReadmeHasOldGodocLink) + output += fmt.Sprintf(" Should be https://pkg.go.dev/github.com/%s/%s\n", githubOrg, repo.Name) + output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) + matched = append(matched, output) } - output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) - output += fmt.Sprintf(" Badge code via https://app.circleci.com/settings/project/github/%s/%s/status-badges)\n", githubOrg, repo.Name) - matched = append(matched, output) } } + + if slices.Contains(r.flag.What, critNoReadme) && err != nil && resp.StatusCode == 404 { + output := fmt.Sprintf(" /README.md not present (%s)\n", critNoReadme) + matched = append(matched, output) + } } if slices.Contains(r.flag.What, critNoCodeownersFile) { From 318d9ffae3d9219cd77a758fb3609a6de570bfda Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Wed, 9 Aug 2023 09:27:46 +0200 Subject: [PATCH 06/19] Add --must-have-codeowners flag --- cmd/repo/find/command.go | 2 +- cmd/repo/find/flag.go | 4 +++- cmd/repo/find/runner.go | 29 ++++++++++++++++------------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index d2c9c35fa..5dd3c9d8c 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -20,7 +20,7 @@ a repository will be returned when it's macthing at least one criteria (boolean Example: - devctl repo find --what NO_DESCRIPTION + devctl repo find --what NO_DESCRIPTION --must-have-codeowners Note: archived repositories are always excluded. diff --git a/cmd/repo/find/flag.go b/cmd/repo/find/flag.go index 7ac0b9c33..baa79dd02 100644 --- a/cmd/repo/find/flag.go +++ b/cmd/repo/find/flag.go @@ -3,7 +3,8 @@ package find import "github.com/spf13/cobra" type flag struct { - What []string + What []string + MustHaveCodeowners bool IncludeArchived bool IncludeFork bool @@ -13,6 +14,7 @@ func (f *flag) Init(cmd *cobra.Command) { cmd.PersistentFlags().BoolVar(&f.IncludeArchived, "include-archived", false, "Also return archived repositories.") cmd.PersistentFlags().BoolVar(&f.IncludeFork, "include-fork", false, "Also return giantswarm forks of repositories.") cmd.PersistentFlags().StringSliceVar(&f.What, "what", []string{}, "What repos to find. See full help for all available criteria.") + cmd.PersistentFlags().BoolVar(&f.MustHaveCodeowners, "must-have-codeowners", false, "Only return repositories that have CODEOWNERS.") } func (f *flag) Validate() error { diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 8c782dd49..3cbf727ba 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -100,10 +100,24 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err continue } - //fmt.Printf("Permissions: %#v\n", repoMetadata.Permissions) - // Check for matching criteria + if slices.Contains(r.flag.What, critNoCodeownersFile) || r.flag.MustHaveCodeowners { + _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "CODEOWNERS", nil) + if err != nil { + if r.flag.MustHaveCodeowners { + // Skip repo without it + continue + } + + output := fmt.Sprintf(" /CODEOWNERS file not present (%s)\n", critNoCodeownersFile) + if repoMetadata.Fork != nil && *repoMetadata.Fork { + output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + } + matched = append(matched, output) + } + } + if slices.Contains(r.flag.What, critHasDocsDirectory) { _, items, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "docs", nil) if err == nil { @@ -159,17 +173,6 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } - if slices.Contains(r.flag.What, critNoCodeownersFile) { - _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "CODEOWNERS", nil) - if err != nil { - output := fmt.Sprintf(" /CODEOWNERS file not present (%s)\n", critNoCodeownersFile) - if repoMetadata.Fork != nil && *repoMetadata.Fork { - output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) - } - matched = append(matched, output) - } - } - if slices.Contains(r.flag.What, critNoDescription) { if repoMetadata.GetDescription() == "" { output := fmt.Sprintf(" Repository has no description (%s)\n", critNoDescription) From 06a1850a9f08c1e618b45c17b8d3c42b31b99517 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 15 Aug 2023 15:57:13 +0200 Subject: [PATCH 07/19] Add BAD_CODEOWNERS criterium --- cmd/repo/find/command.go | 4 ++++ cmd/repo/find/runner.go | 30 ++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index 5dd3c9d8c..e6df3d157 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -40,6 +40,10 @@ Criteria: Has the file docs/pull_request_template.md, which is not the desired location. We want this to be in .github/pull_request_template.md. + BAD_CODEOWNERS + + Finds repositories with errors in CODEOWNERS files. + NO_CODEOWNERS There is no CODEOWNERS file in the root folder, which means that the repository diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 3cbf727ba..4e57fcb12 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -25,6 +25,7 @@ const ( critReadmeHasOldCircleCiBadge = "README_OLD_CIRCLECI_BAGDE" critReadmeHasOldGodocLink = "README_OLD_GODOC_LINK" critNoCodeownersFile = "NO_CODEOWNERS" + critCodeownersErrors = "BAD_CODOWNERS" critNoDescription = "NO_DESCRIPTION" critNoReadme = "NO_README" critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" @@ -82,13 +83,14 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err matchingReposCount := 0 - for i, repo := range repos { + for _, repo := range repos { matched := []string{} repoMetadata, err := client.GetRepository(ctx, githubOrg, repo.Name) if err != nil { return microerror.Mask(err) } + defaultBranch := repoMetadata.GetDefaultBranch() if !r.flag.IncludeArchived && repoMetadata.GetArchived() { // Skip archived repos @@ -106,7 +108,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "CODEOWNERS", nil) if err != nil { if r.flag.MustHaveCodeowners { - // Skip repo without it + // Skip repo without CODEOWNERS file continue } @@ -118,6 +120,26 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } + if slices.Contains(r.flag.What, critCodeownersErrors) { + errs, resp, err := realClient.Repositories.GetCodeownersErrors(ctx, githubOrg, repo.Name) + if err != nil { + if resp != nil && resp.StatusCode == 404 { + // CODEOWNERS not found in this repo. Do nothing. + } else { + return microerror.Mask(err) + } + } + + if errs != nil && len(errs.Errors) > 0 { + output := fmt.Sprintf(" Errors found in CODEOWNERS files (%s)\n", critCodeownersErrors) + for _, item := range errs.Errors { + messageFirstLine := strings.Split(item.Message, "\n")[0] + output += fmt.Sprintf(" https://github.com/%s/%s/blob/%s/%s#L%d - %q\n", githubOrg, repo.Name, defaultBranch, item.Path, item.Line, messageFirstLine) + } + matched = append(matched, output) + } + } + if slices.Contains(r.flag.What, critHasDocsDirectory) { _, items, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "docs", nil) if err == nil { @@ -151,7 +173,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err if repoMetadata.Fork != nil && *repoMetadata.Fork { output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } - output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) + output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, defaultBranch) output += fmt.Sprintf(" Badge code via https://app.circleci.com/settings/project/github/%s/%s/status-badges)\n", githubOrg, repo.Name) matched = append(matched, output) } @@ -193,7 +215,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err // Print output per repo if len(matched) > 0 { matchingReposCount++ - fmt.Printf("\n(%d of %d) https://github.com/%s/%s\n", i, len(repos), githubOrg, repo.Name) + fmt.Printf("\n- [ ] https://github.com/%s/%s\n", githubOrg, repo.Name) for _, item := range matched { fmt.Print(item) } From 6742ccc23301dce3602c6fddfdec3a4f1023a7c2 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 15 Aug 2023 17:29:43 +0200 Subject: [PATCH 08/19] Group imports --- cmd/repo/find/runner.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 4e57fcb12..292032f25 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -8,11 +8,12 @@ import ( "os" "strings" - "github.com/giantswarm/devctl/pkg/githubclient" "github.com/giantswarm/microerror" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/utils/strings/slices" + + "github.com/giantswarm/devctl/pkg/githubclient" ) const ( @@ -112,9 +113,9 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err continue } - output := fmt.Sprintf(" /CODEOWNERS file not present (%s)\n", critNoCodeownersFile) + output := fmt.Sprintf(" - /CODEOWNERS file not present (%s)\n", critNoCodeownersFile) if repoMetadata.Fork != nil && *repoMetadata.Fork { - output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + output += fmt.Sprintf(" - Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } matched = append(matched, output) } @@ -131,10 +132,10 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } if errs != nil && len(errs.Errors) > 0 { - output := fmt.Sprintf(" Errors found in CODEOWNERS files (%s)\n", critCodeownersErrors) + output := fmt.Sprintf(" - Errors found in CODEOWNERS files (%s)\n", critCodeownersErrors) for _, item := range errs.Errors { messageFirstLine := strings.Split(item.Message, "\n")[0] - output += fmt.Sprintf(" https://github.com/%s/%s/blob/%s/%s#L%d - %q\n", githubOrg, repo.Name, defaultBranch, item.Path, item.Line, messageFirstLine) + output += fmt.Sprintf(" - https://github.com/%s/%s/blob/%s/%s#L%d - %q\n", githubOrg, repo.Name, defaultBranch, item.Path, item.Line, messageFirstLine) } matched = append(matched, output) } @@ -143,11 +144,11 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err if slices.Contains(r.flag.What, critHasDocsDirectory) { _, items, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "docs", nil) if err == nil { - output := fmt.Sprintf(" /docs directory exists (%s)\n", critHasDocsDirectory) + output := fmt.Sprintf(" - /docs directory exists (%s)\n", critHasDocsDirectory) for _, item := range items { path := item.GetPath() ftype := item.GetType() - output += fmt.Sprintf(" %s %s\n", path, ftype) + output += fmt.Sprintf(" - %s %s\n", path, ftype) } matched = append(matched, output) } @@ -156,7 +157,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err if slices.Contains(r.flag.What, critHasPrTemplateInDocs) { _, _, _, err := realClient.Repositories.GetContents(ctx, githubOrg, repo.Name, "docs/pull_request_template.md", nil) if err == nil { - output := fmt.Sprintf(" /docs/pull_request_template.md file exists (%s)\n", critHasPrTemplateInDocs) + output := fmt.Sprintf(" - /docs/pull_request_template.md file exists (%s)\n", critHasPrTemplateInDocs) matched = append(matched, output) } } @@ -169,44 +170,44 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err if slices.Contains(r.flag.What, critReadmeHasOldCircleCiBadge) { if strings.Contains(string(decodedContent), fmt.Sprintf("https://circleci.com/gh/%s", githubOrg)) { - output := fmt.Sprintf(" /README.md has old CircleCI badge (%s)\n", critReadmeHasOldCircleCiBadge) + output := fmt.Sprintf(" - /README.md has old CircleCI badge (%s)\n", critReadmeHasOldCircleCiBadge) if repoMetadata.Fork != nil && *repoMetadata.Fork { output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } - output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, defaultBranch) - output += fmt.Sprintf(" Badge code via https://app.circleci.com/settings/project/github/%s/%s/status-badges)\n", githubOrg, repo.Name) + output += fmt.Sprintf(" - Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, defaultBranch) + output += fmt.Sprintf(" - Badge code via https://app.circleci.com/settings/project/github/%s/%s/status-badges)\n", githubOrg, repo.Name) matched = append(matched, output) } } if slices.Contains(r.flag.What, critReadmeHasOldGodocLink) { if strings.Contains(string(decodedContent), "godoc.org") { - output := fmt.Sprintf(" /README.md has link to godoc.org (%s)\n", critReadmeHasOldGodocLink) - output += fmt.Sprintf(" Should be https://pkg.go.dev/github.com/%s/%s\n", githubOrg, repo.Name) - output += fmt.Sprintf(" Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) + output := fmt.Sprintf(" - /README.md has link to godoc.org (%s)\n", critReadmeHasOldGodocLink) + output += fmt.Sprintf(" - Should be https://pkg.go.dev/github.com/%s/%s\n", githubOrg, repo.Name) + output += fmt.Sprintf(" - Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) matched = append(matched, output) } } } if slices.Contains(r.flag.What, critNoReadme) && err != nil && resp.StatusCode == 404 { - output := fmt.Sprintf(" /README.md not present (%s)\n", critNoReadme) + output := fmt.Sprintf(" - /README.md not present (%s)\n", critNoReadme) matched = append(matched, output) } } if slices.Contains(r.flag.What, critNoDescription) { if repoMetadata.GetDescription() == "" { - output := fmt.Sprintf(" Repository has no description (%s)\n", critNoDescription) + output := fmt.Sprintf(" - Repository has no description (%s)\n", critNoDescription) matched = append(matched, output) } } if slices.Contains(r.flag.What, critDefaultBranchMaster) { if repoMetadata.GetDefaultBranch() == "master" { - output := fmt.Sprintf(" Default branch is 'master' (%s)\n", critDefaultBranchMaster) + output := fmt.Sprintf(" - Default branch is 'master' (%s)\n", critDefaultBranchMaster) if repoMetadata.Fork != nil && *repoMetadata.Fork { - output += fmt.Sprintf(" Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + output += fmt.Sprintf(" - Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) } matched = append(matched, output) } From 9c22e8758e04a2b8d022768743d190293e309c00 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 15 Aug 2023 17:36:59 +0200 Subject: [PATCH 09/19] Silence gosec --- cmd/repo/find/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 292032f25..9ac1b4a6c 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -17,6 +17,7 @@ import ( ) const ( + //nolint:gosec githubTokenEnvVar = "GITHUB_TOKEN" githubOrg = "giantswarm" From 5ede522f86270f9b5db14b3881989643ffff5f6e Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Wed, 16 Aug 2023 09:57:32 +0200 Subject: [PATCH 10/19] Add help for README_OLD_GODOC_LINK --- cmd/repo/find/command.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index e6df3d157..3ea026ebc 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -64,6 +64,10 @@ Criteria: There is an outdated (broken) CircleCI badge is present in the README. This should better get replaced by an up-to-date one, as it can otherwise not fulfil its purpose, and broken images never make a good impression. + + README_OLD_GODOC_LINK + + The README contains an outdated godoc.org link. Should be pkg.go.dev nowadays. ` ) From ffa56af80fb10b53dc37b5d95dc9b81f1f319256 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Wed, 16 Aug 2023 09:57:52 +0200 Subject: [PATCH 11/19] Replace function call with variable --- cmd/repo/find/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 9ac1b4a6c..8cd9a3154 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -185,7 +185,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err if strings.Contains(string(decodedContent), "godoc.org") { output := fmt.Sprintf(" - /README.md has link to godoc.org (%s)\n", critReadmeHasOldGodocLink) output += fmt.Sprintf(" - Should be https://pkg.go.dev/github.com/%s/%s\n", githubOrg, repo.Name) - output += fmt.Sprintf(" - Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, repoMetadata.GetDefaultBranch()) + output += fmt.Sprintf(" - Edit via https://github.com/%s/%s/edit/%s/README.md\n", githubOrg, repo.Name, defaultBranch) matched = append(matched, output) } } @@ -205,7 +205,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } if slices.Contains(r.flag.What, critDefaultBranchMaster) { - if repoMetadata.GetDefaultBranch() == "master" { + if defaultBranch == "master" { output := fmt.Sprintf(" - Default branch is 'master' (%s)\n", critDefaultBranchMaster) if repoMetadata.Fork != nil && *repoMetadata.Fork { output += fmt.Sprintf(" - Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) From 3401cabfbe337aae61411f35a516370835c4cc99 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Thu, 17 Aug 2023 10:46:20 +0200 Subject: [PATCH 12/19] Add NO_DEPENDENCY_GRAPH criterion --- cmd/repo/find/command.go | 5 +++++ cmd/repo/find/runner.go | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index 3ea026ebc..f5cbacffb 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -49,6 +49,11 @@ Criteria: There is no CODEOWNERS file in the root folder, which means that the repository has no owner. We want repos to have owners, ideally. + NO_DEPENDENCY_GRAPH + + Returns repos where the dependency graph feature is inactive. We want it to be + active. + NO_DESCRIPTION Repository description is missing. We want a meaningful description to be present, diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 8cd9a3154..ac44cb542 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/giantswarm/microerror" + "github.com/google/go-github/v53/github" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/utils/strings/slices" @@ -31,6 +32,7 @@ const ( critNoDescription = "NO_DESCRIPTION" critNoReadme = "NO_README" critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" + critNoDependencyGraph = "NO_DEPENDENCY_GRAPH" ) type runner struct { @@ -214,6 +216,25 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } + if slices.Contains(r.flag.What, critNoDependencyGraph) { + path := fmt.Sprintf("/repos/%s/%s/dependency-graph/sbom", githubOrg, repo.Name) + req, err := realClient.NewRequest("GET", path, nil, github.WithVersion("2022-11-28")) + if err != nil { + return microerror.Mask(err) + } + + resp, err := realClient.Do(context.Background(), req, nil) + if err != nil { + if resp != nil && resp.StatusCode == 404 { + output := fmt.Sprintf(" - Dependency graph not active (%s)\n", critNoDependencyGraph) + output += fmt.Sprintf(" - Enable it here: https://github.com/%s/%s/network/dependencies\n", githubOrg, repo.Name) + matched = append(matched, output) + } else { + return microerror.Mask(err) + } + } + } + // Print output per repo if len(matched) > 0 { matchingReposCount++ From e6560d6f457879e150cbc151f3fdeeb07c2be730 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Thu, 14 Sep 2023 11:13:07 +0200 Subject: [PATCH 13/19] Make renovate ignore github.com/imdario/mergo --- pkg/gen/input/renovate/internal/file/renovate.json.template | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/gen/input/renovate/internal/file/renovate.json.template b/pkg/gen/input/renovate/internal/file/renovate.json.template index 1d5e54057..b5dfd52eb 100644 --- a/pkg/gen/input/renovate/internal/file/renovate.json.template +++ b/pkg/gen/input/renovate/internal/file/renovate.json.template @@ -57,9 +57,10 @@ ".github/workflows/pre_commit_*.yaml" ], "ignoreDeps": [ + "actions/setup-go", "architect", - "zricethezav/gitleaks-action", - "actions/setup-go" + "github.com/imdario/mergo", + "zricethezav/gitleaks-action" ], "regexManagers": [ { From 3b1bfda840c10d46815af7d55ad9867f2d60f58f Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Thu, 14 Sep 2023 11:16:17 +0200 Subject: [PATCH 14/19] Revert unwanted change --- pkg/gen/input/renovate/internal/file/renovate.json.template | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/gen/input/renovate/internal/file/renovate.json.template b/pkg/gen/input/renovate/internal/file/renovate.json.template index b5dfd52eb..1d5e54057 100644 --- a/pkg/gen/input/renovate/internal/file/renovate.json.template +++ b/pkg/gen/input/renovate/internal/file/renovate.json.template @@ -57,10 +57,9 @@ ".github/workflows/pre_commit_*.yaml" ], "ignoreDeps": [ - "actions/setup-go", "architect", - "github.com/imdario/mergo", - "zricethezav/gitleaks-action" + "zricethezav/gitleaks-action", + "actions/setup-go" ], "regexManagers": [ { From 2837fc0fd1092e5979306d2c2210ba0338a217b1 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Fri, 22 Sep 2023 11:09:00 +0200 Subject: [PATCH 15/19] Add criterium NO_DEPENDABOT_ALERTS --- cmd/repo/find/command.go | 5 +++++ cmd/repo/find/runner.go | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index f5cbacffb..8328f61fc 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -49,6 +49,11 @@ Criteria: There is no CODEOWNERS file in the root folder, which means that the repository has no owner. We want repos to have owners, ideally. + NO_DEPENDABOT_ALERTS + + Dependabot security alerts are disabled. Note: this is independent from + version updates via Dependabot. + NO_DEPENDENCY_GRAPH Returns repos where the dependency graph feature is inactive. We want it to be diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index ac44cb542..53f4e6adf 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -5,6 +5,7 @@ import ( b64 "encoding/base64" "fmt" "io" + "net/http" "os" "strings" @@ -33,6 +34,7 @@ const ( critNoReadme = "NO_README" critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" critNoDependencyGraph = "NO_DEPENDENCY_GRAPH" + critDependabotAlertsDisabled = "NO_DEPENDABOT_ALERTS" ) type runner struct { @@ -124,10 +126,26 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } + if slices.Contains(r.flag.What, critDependabotAlertsDisabled) { + enabled, _, err := realClient.Repositories.GetVulnerabilityAlerts(ctx, githubOrg, repo.Name) + if err != nil { + return microerror.Mask(err) + } + + if !enabled { + output := fmt.Sprintf(" - Dependabot security alerts disabled (%s)\n", critDependabotAlertsDisabled) + output += fmt.Sprintf(" - Go to https://github.com/%s/%s/security to enable", githubOrg, repo.Name) + if repoMetadata.Fork != nil && *repoMetadata.Fork { + output += fmt.Sprintf(" - Note: this repo is a fork of %s\n", repoMetadata.GetForksURL()) + } + matched = append(matched, output) + } + } + if slices.Contains(r.flag.What, critCodeownersErrors) { errs, resp, err := realClient.Repositories.GetCodeownersErrors(ctx, githubOrg, repo.Name) if err != nil { - if resp != nil && resp.StatusCode == 404 { + if resp != nil && resp.StatusCode == http.StatusNotFound { // CODEOWNERS not found in this repo. Do nothing. } else { return microerror.Mask(err) From 291ad7e423dde82bcd65f136a8e21a6b6ab29af2 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Fri, 22 Sep 2023 11:09:11 +0200 Subject: [PATCH 16/19] Sort repository list by name instead of creation date --- pkg/githubclient/client_repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/githubclient/client_repository.go b/pkg/githubclient/client_repository.go index 1b28ffeb4..21db822dd 100644 --- a/pkg/githubclient/client_repository.go +++ b/pkg/githubclient/client_repository.go @@ -20,7 +20,7 @@ func (c *Client) ListRepositories(ctx context.Context, owner string) ([]Reposito ListOptions: github.ListOptions{ PerPage: 100, }, - + Sort: "full_name", Type: "all", } for pageCnt := 0; ; pageCnt++ { From 2b2479331e2dd9e489dff38c2ef7a2d3f1ae56f6 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Wed, 27 Sep 2023 15:46:12 +0200 Subject: [PATCH 17/19] WIP: Fetch info about renovate app --- cmd/repo/find/runner.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 53f4e6adf..3c5f08bcf 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -35,6 +35,8 @@ const ( critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" critNoDependencyGraph = "NO_DEPENDENCY_GRAPH" critDependabotAlertsDisabled = "NO_DEPENDABOT_ALERTS" + + renovateAppID = 2740 ) type runner struct { @@ -79,13 +81,27 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err if err != nil { return microerror.Mask(err) } + realClient := client.GetUnderlyingClient(ctx) - repos, err := client.ListRepositories(ctx, githubOrg) + // Renovate App + installations, _, err := realClient.Organizations.ListInstallations(ctx, githubOrg, &github.ListOptions{PerPage: 100}) if err != nil { return microerror.Mask(err) } - realClient := client.GetUnderlyingClient(ctx) + fmt.Printf("Found %d installations\n", *installations.TotalCount) + + for _, inst := range installations.Installations { + if *inst.AppID == renovateAppID { + repoSelection := inst.GetRepositorySelection() + fmt.Printf("Renovate installation: repository selection %s", repoSelection) + } + } + + repos, err := client.ListRepositories(ctx, githubOrg) + if err != nil { + return microerror.Mask(err) + } matchingReposCount := 0 From 98ed7710d66a4d6bf6ce9964699b826bcfb956e3 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Wed, 27 Sep 2023 17:46:31 +0200 Subject: [PATCH 18/19] Remove app installation code again --- cmd/repo/find/runner.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 3c5f08bcf..27d5855fd 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -35,8 +35,6 @@ const ( critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" critNoDependencyGraph = "NO_DEPENDENCY_GRAPH" critDependabotAlertsDisabled = "NO_DEPENDABOT_ALERTS" - - renovateAppID = 2740 ) type runner struct { @@ -83,21 +81,6 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } realClient := client.GetUnderlyingClient(ctx) - // Renovate App - installations, _, err := realClient.Organizations.ListInstallations(ctx, githubOrg, &github.ListOptions{PerPage: 100}) - if err != nil { - return microerror.Mask(err) - } - - fmt.Printf("Found %d installations\n", *installations.TotalCount) - - for _, inst := range installations.Installations { - if *inst.AppID == renovateAppID { - repoSelection := inst.GetRepositorySelection() - fmt.Printf("Renovate installation: repository selection %s", repoSelection) - } - } - repos, err := client.ListRepositories(ctx, githubOrg) if err != nil { return microerror.Mask(err) From a5526d02532f4fd399f2d4654064c44062751141 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 31 Oct 2023 11:28:45 +0100 Subject: [PATCH 19/19] Rename crit, handle error --- cmd/repo/find/command.go | 4 +++ cmd/repo/find/runner.go | 64 ++++++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/cmd/repo/find/command.go b/cmd/repo/find/command.go index 8328f61fc..6f66df88f 100644 --- a/cmd/repo/find/command.go +++ b/cmd/repo/find/command.go @@ -49,6 +49,10 @@ Criteria: There is no CODEOWNERS file in the root folder, which means that the repository has no owner. We want repos to have owners, ideally. + BAD_DEFAULT_BRANCH_PROTECTION_GO + + The Go repo has no effective protection rules for the default branch. + NO_DEPENDABOT_ALERTS Dependabot security alerts are disabled. Note: this is independent from diff --git a/cmd/repo/find/runner.go b/cmd/repo/find/runner.go index 27d5855fd..1fdb613bf 100644 --- a/cmd/repo/find/runner.go +++ b/cmd/repo/find/runner.go @@ -24,17 +24,18 @@ const ( githubOrg = "giantswarm" // Criteria names - critHasDocsDirectory = "HAS_DOCS_DIR" - critHasPrTemplateInDocs = "HAS_PR_TEMPLATE_IN_DOCS" - critReadmeHasOldCircleCiBadge = "README_OLD_CIRCLECI_BAGDE" - critReadmeHasOldGodocLink = "README_OLD_GODOC_LINK" - critNoCodeownersFile = "NO_CODEOWNERS" - critCodeownersErrors = "BAD_CODOWNERS" - critNoDescription = "NO_DESCRIPTION" - critNoReadme = "NO_README" - critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" - critNoDependencyGraph = "NO_DEPENDENCY_GRAPH" - critDependabotAlertsDisabled = "NO_DEPENDABOT_ALERTS" + critHasDocsDirectory = "HAS_DOCS_DIR" + critHasPrTemplateInDocs = "HAS_PR_TEMPLATE_IN_DOCS" + critReadmeHasOldCircleCiBadge = "README_OLD_CIRCLECI_BAGDE" + critReadmeHasOldGodocLink = "README_OLD_GODOC_LINK" + critNoCodeownersFile = "NO_CODEOWNERS" + critCodeownersErrors = "BAD_CODOWNERS" + critNoDescription = "NO_DESCRIPTION" + critNoReadme = "NO_README" + critDefaultBranchMaster = "DEFAULT_BRANCH_MASTER" + critNoDependencyGraph = "NO_DEPENDENCY_GRAPH" + critDependabotAlertsDisabled = "NO_DEPENDABOT_ALERTS" + critBadDefaultBranchProtectionGo = "BAD_DEFAULT_BRANCH_PROTECTION_GO" ) type runner struct { @@ -63,7 +64,7 @@ func (r *runner) Run(cmd *cobra.Command, args []string) error { func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) error { token, found := os.LookupEnv(githubTokenEnvVar) if !found { - return microerror.Maskf(envVarNotFoundError, "environement variable %#q was not found", githubTokenEnvVar) + return microerror.Maskf(envVarNotFoundError, "environment variable %#q was not found", githubTokenEnvVar) } if len(r.flag.What) < 1 { @@ -252,6 +253,45 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err } } + if slices.Contains(r.flag.What, critBadDefaultBranchProtectionGo) && repoMetadata.GetLanguage() == "Go" { + protection, resp, err := realClient.Repositories.GetBranchProtection(ctx, githubOrg, repo.Name, defaultBranch) + output := "" + + if err != nil { + // Branch protection not in place + if resp != nil && resp.StatusCode == http.StatusNotFound { + output += fmt.Sprintf(" - Default branch has no protection rules (%s)\n", critBadDefaultBranchProtectionGo) + output += fmt.Sprintf(" - Default branch name is %q\n", defaultBranch) + output += fmt.Sprintf(" - Configure protection here: https://github.com/%s/%s/settings/branches\n", githubOrg, repo.Name) + } else { + return microerror.Mask(err) + } + matched = append(matched, output) + } else { + // Branch protection details + if protection.RequiredStatusChecks == nil || protection.RequiredPullRequestReviews == nil { + output += fmt.Sprintf(" - Default branch protection not sufficient (%s)\n", critBadDefaultBranchProtectionGo) + if protection.RequiredStatusChecks == nil { + output += " - Status checks are not required to pass before merging\n" + } + if protection.RequiredPullRequestReviews == nil { + output += " - PR reviews are not required\n" + } + + output += fmt.Sprintf(" - Configure protection here: https://github.com/%s/%s/settings/branches\n", githubOrg, repo.Name) + + matched = append(matched, output) + } + + if protection.RequiredStatusChecks != nil && len(protection.RequiredStatusChecks.Checks) == 0 { + output += fmt.Sprintf(" - Default branch protection not sufficient (%s)\n", critBadDefaultBranchProtectionGo) + output += " - None of the checks is required\n" + output += fmt.Sprintf(" - Fix this here: https://github.com/%s/%s/settings/branches\n", githubOrg, repo.Name) + matched = append(matched, output) + } + } + } + // Print output per repo if len(matched) > 0 { matchingReposCount++