Skip to content

Commit dccb5a6

Browse files
danskmtdanskmt
committed
fix: use GAF network stack for license downloads with retry and caching
1 parent fe92ed4 commit dccb5a6

4 files changed

Lines changed: 229 additions & 28 deletions

File tree

‎.circleci/config.yml‎

Lines changed: 50 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,8 @@ commands:
186186
command: |
187187
# load Windows $Env:Path values of tools necessary to execute the build (like Go)
188188
if [ -f "/c/tools-cache/<< pipeline.parameters.windows_bash_env_script >>" ]; then source "/c/tools-cache/<< pipeline.parameters.windows_bash_env_script >>"; fi
189-
make << parameters.make_target >> GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >> STATIC_NODE_BINARY=true CGO_ENABLED=0
189+
LICENSE_CACHE_DIR="" && [ -d "cliv2/internal/embedded/_data/licenses" ] && LICENSE_CACHE_DIR="$(pwd)/cliv2/internal/embedded/_data/licenses"
190+
make << parameters.make_target >> GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >> STATIC_NODE_BINARY=true CGO_ENABLED=0 LICENSE_CACHE_DIR="$LICENSE_CACHE_DIR"
190191
- unless:
191192
condition: << parameters.static >>
192193
steps:
@@ -199,7 +200,8 @@ commands:
199200
command: |
200201
# load Windows $Env:Path values of tools necessary to execute the build (like Go)
201202
if [ -f "/c/tools-cache/<< pipeline.parameters.windows_bash_env_script >>" ]; then source "/c/tools-cache/<< pipeline.parameters.windows_bash_env_script >>"; fi
202-
make << parameters.make_target >> GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >> STATIC_NODE_BINARY=false CGO_ENABLED=1
203+
LICENSE_CACHE_DIR="" && [ -d "cliv2/internal/embedded/_data/licenses" ] && LICENSE_CACHE_DIR="$(pwd)/cliv2/internal/embedded/_data/licenses"
204+
make << parameters.make_target >> GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >> STATIC_NODE_BINARY=false CGO_ENABLED=1 LICENSE_CACHE_DIR="$LICENSE_CACHE_DIR"
203205
install-go:
204206
parameters:
205207
go_os:
@@ -603,6 +605,12 @@ workflows:
603605
requires:
604606
- prepare-build
605607

608+
- prepare-licenses:
609+
name: prepare-licenses
610+
context: devex_cli_docker_hub
611+
requires:
612+
- prepare-build
613+
606614
- build-artifact:
607615
name: build linux amd64
608616
context:
@@ -615,7 +623,7 @@ workflows:
615623
go_download_base_url: << pipeline.parameters.go_download_base_url >>
616624
executor: docker-amd64-xl
617625
requires:
618-
- prepare-build
626+
- prepare-licenses
619627

620628
- build-artifact:
621629
name: build linux static amd64
@@ -630,7 +638,7 @@ workflows:
630638
make_target: build-experimental
631639
executor: docker-amd64-xl
632640
requires:
633-
- prepare-build
641+
- prepare-licenses
634642

635643
- build-artifact:
636644
name: build linux fips amd64
@@ -644,7 +652,7 @@ workflows:
644652
make_target: build-fips
645653
executor: docker-amd64-xl
646654
requires:
647-
- prepare-build
655+
- prepare-licenses
648656

649657
- build-artifact:
650658
name: build linux arm64
@@ -658,7 +666,7 @@ workflows:
658666
go_download_base_url: << pipeline.parameters.go_download_base_url >>
659667
executor: docker-arm64-xl
660668
requires:
661-
- prepare-build
669+
- prepare-licenses
662670

663671
- build-artifact:
664672
name: build linux static arm64
@@ -673,7 +681,7 @@ workflows:
673681
make_target: build-experimental
674682
executor: docker-arm64-xl
675683
requires:
676-
- prepare-build
684+
- prepare-licenses
677685

678686
- build-artifact:
679687
name: build linux fips arm64
@@ -687,7 +695,7 @@ workflows:
687695
make_target: build-fips
688696
executor: docker-arm64-xl
689697
requires:
690-
- prepare-build
698+
- prepare-licenses
691699

692700
- build-artifact:
693701
name: build alpine amd64
@@ -701,7 +709,7 @@ workflows:
701709
executor: docker-amd64-xl
702710
c_compiler: /usr/bin/musl-gcc
703711
requires:
704-
- prepare-build
712+
- prepare-licenses
705713

706714
- build-artifact:
707715
name: build alpine arm64
@@ -715,7 +723,7 @@ workflows:
715723
executor: docker-arm64-xl
716724
c_compiler: /usr/bin/musl-gcc
717725
requires:
718-
- prepare-build
726+
- prepare-licenses
719727

720728
- build-artifact:
721729
name: build macOS amd64
@@ -729,7 +737,7 @@ workflows:
729737
- snyk-macos-signing
730738
- iac-cli
731739
requires:
732-
- prepare-build
740+
- prepare-licenses
733741

734742
- build-artifact:
735743
name: build macOS arm64
@@ -743,7 +751,7 @@ workflows:
743751
- snyk-macos-signing
744752
- iac-cli
745753
requires:
746-
- prepare-build
754+
- prepare-licenses
747755

748756
- build-artifact:
749757
name: build windows amd64
@@ -759,7 +767,7 @@ workflows:
759767
- snyk-windows-signing
760768
- iac-cli
761769
requires:
762-
- prepare-build
770+
- prepare-licenses
763771

764772
- acceptance-tests:
765773
name: acceptance-tests linux static arm64
@@ -1487,6 +1495,35 @@ jobs:
14871495
- binary-releases/snyk-fix.tgz
14881496
- binary-releases/snyk-protect.tgz
14891497

1498+
prepare-licenses:
1499+
executor: docker-amd64
1500+
steps:
1501+
- prepare-workspace
1502+
- install-go:
1503+
go_os: linux
1504+
go_target_os: linux
1505+
go_arch: amd64
1506+
base_url: << pipeline.parameters.go_download_base_url >>
1507+
extraction_path: '.'
1508+
- restore_cache:
1509+
name: Restoring license cache
1510+
keys:
1511+
- license-cache-v1-{{ checksum "cliv2/go.sum" }}
1512+
- run:
1513+
name: Prepare 3rd party licenses
1514+
working_directory: ./cliv2
1515+
command: |
1516+
GOOS=linux GOARCH=amd64 go run scripts/prepare_licenses.go
1517+
- save_cache:
1518+
name: Saving license cache
1519+
key: license-cache-v1-{{ checksum "cliv2/go.sum" }}
1520+
paths:
1521+
- cliv2/internal/embedded/_data/licenses
1522+
- persist_to_workspace:
1523+
root: .
1524+
paths:
1525+
- cliv2/internal/embedded/_data/licenses
1526+
14901527
build-artifact:
14911528
parameters:
14921529
go_os:

‎cliv2/Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ bindir = $(exec_prefix)/bin
3434
WORKING_DIR = $(CURDIR)
3535
BUILD_DIR = $(WORKING_DIR)/_bin
3636
CACHE_DIR = $(WORKING_DIR)/_cache
37+
LICENSE_CACHE_DIR ?=
3738
SRCS = $(shell find $(WORKING_DIR) -type f -name '*.go')
3839

3940
# load cached variables if available
@@ -198,7 +199,7 @@ $(WORKING_DIR)/internal/httpauth/generated/spnego_generated_mock.go:
198199

199200
$(CACHE_DIR)/prepare-3rd-party-licenses:
200201
@echo "$(LOG_PREFIX) Preparing 3rd Party Licenses"
201-
@GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) $(GOCMD) run scripts/prepare_licenses.go > $(CACHE_DIR)/prepare-3rd-party-licenses 2> $(CACHE_DIR)/prepare-3rd-party-licenses.log \
202+
@GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) LICENSE_CACHE_DIR=$(LICENSE_CACHE_DIR) $(GOCMD) run scripts/prepare_licenses.go > $(CACHE_DIR)/prepare-3rd-party-licenses 2> $(CACHE_DIR)/prepare-3rd-party-licenses.log \
202203
|| (cat $(CACHE_DIR)/prepare-3rd-party-licenses.log && exit 1)
203204
@echo "$(LOG_PREFIX) 3rd party licenses: ok"
204205

‎cliv2/scripts/prepare_licenses.go‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,47 @@ import (
88
"os/exec"
99
"path/filepath"
1010
"strings"
11-
"time"
11+
12+
"github.com/snyk/go-application-framework/pkg/configuration"
13+
"github.com/snyk/go-application-framework/pkg/networking"
14+
"github.com/snyk/go-application-framework/pkg/networking/middleware"
1215
)
1316

1417
// licensesEmbeddedDir is the cliv2-relative tree where go-licenses and manual downloads write.
1518
var licensesEmbeddedDir = filepath.Join(".", "internal", "embedded", "_data", "licenses")
1619

20+
const maxDownloadAttempts = 5
21+
1722
func log(msg string) {
1823
fmt.Fprintln(os.Stderr, msg)
1924
}
2025

26+
var newHTTPClient = func() *http.Client {
27+
cfg := configuration.NewWithOpts(configuration.WithAutomaticEnv())
28+
cfg.Set(middleware.ConfigurationKeyRequestAttempts, maxDownloadAttempts)
29+
30+
na := networking.NewNetworkAccess(cfg)
31+
na.AddHeaderField("User-Agent", "Snyk-CLI-Build/1.0")
32+
33+
if token := os.Getenv("GITHUB_TOKEN"); token != "" {
34+
na.AddHeaderField("Authorization", "Bearer "+token)
35+
}
36+
37+
return na.GetUnauthorizedHttpClient()
38+
}
39+
2140
func main() {
2241
log("Preparing 3rd party licenses...")
2342

43+
if src := os.Getenv("LICENSE_CACHE_DIR"); src != "" {
44+
if err := copyLicenseCache(src, licensesEmbeddedDir); err != nil {
45+
log(fmt.Sprintf("Error restoring cached licenses: %v", err))
46+
os.Exit(1)
47+
}
48+
log("Restored licenses from cache, skipping downloads.")
49+
return
50+
}
51+
2452
goBinPath := filepath.Join(mustGetwd(), "_cache")
2553
if err := os.Setenv("GOBIN", goBinPath); err != nil {
2654
log(fmt.Sprintf("Error setting GOBIN: %v", err))
@@ -50,6 +78,8 @@ func main() {
5078
os.Exit(1)
5179
}
5280

81+
client := newHTTPClient()
82+
5383
log("Downloading manual licenses...")
5484
manualLicenses := []struct{ url, pkg string }{
5585
{"https://raw.githubusercontent.com/davecgh/go-spew/master/LICENSE", "github.com/davecgh/go-spew"},
@@ -58,7 +88,7 @@ func main() {
5888
{"https://go.dev/LICENSE?m=text", "go.dev"},
5989
}
6090
for _, lic := range manualLicenses {
61-
if err := manualLicenseDownload(lic.url, lic.pkg); err != nil {
91+
if err := manualLicenseDownload(client, lic.url, lic.pkg); err != nil {
6292
log(fmt.Sprintf("Error downloading license: %v", err))
6393
os.Exit(1)
6494
}
@@ -72,7 +102,28 @@ func main() {
72102
log("Done preparing 3rd party licenses.")
73103
}
74104

75-
func manualLicenseDownload(url, packageName string) error {
105+
func copyLicenseCache(src, dst string) error {
106+
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
107+
if err != nil {
108+
return err
109+
}
110+
rel, err := filepath.Rel(src, path)
111+
if err != nil {
112+
return err
113+
}
114+
target := filepath.Join(dst, rel)
115+
if info.IsDir() {
116+
return os.MkdirAll(target, 0o755)
117+
}
118+
data, err := os.ReadFile(path)
119+
if err != nil {
120+
return err
121+
}
122+
return os.WriteFile(target, data, info.Mode())
123+
})
124+
}
125+
126+
func manualLicenseDownload(client *http.Client, url, packageName string) error {
76127
folderPath := filepath.Join(licensesEmbeddedDir, packageName)
77128
licenseFile := filepath.Join(folderPath, "LICENSE")
78129

@@ -86,14 +137,7 @@ func manualLicenseDownload(url, packageName string) error {
86137
return fmt.Errorf("creating directory for %s: %w", packageName, err)
87138
}
88139

89-
req, err := http.NewRequest(http.MethodGet, url, nil)
90-
if err != nil {
91-
return fmt.Errorf("creating request for %s: %w", packageName, err)
92-
}
93-
req.Header.Set("User-Agent", "Snyk-CLI-Build/1.0")
94-
95-
client := &http.Client{Timeout: 30 * time.Second}
96-
resp, err := client.Do(req)
140+
resp, err := client.Get(url)
97141
if err != nil {
98142
return fmt.Errorf("downloading license for %s: %w", packageName, err)
99143
}

0 commit comments

Comments
 (0)