diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index 211901ec3c6..cf60b1bd51d 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -67,11 +67,15 @@ func listBitbucketWorkspaces( err errors.Error, ) { var res *http.Response + // /user/permissions/workspaces was removed and /workspaces deprecated by + // Bitbucket CHANGE-2770; /user/workspaces lists the current user's workspaces + // and is the supported replacement. res, err = apiClient.Get( - "/user/permissions/workspaces", + "/user/workspaces", url.Values{ - "sort": {"workspace.slug"}, - "fields": {"values.workspace.slug,values.workspace.name,pagelen,page,size"}, + // No sort/fields: /user/workspaces rejects sort=workspace.slug with + // HTTP 400 "Invalid field name". The bare call returns the nested + // workspace objects we need; WorkspaceResponse ignores extra fields. "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, @@ -98,9 +102,9 @@ func listBitbucketWorkspaces( for _, r := range resBody.Values { children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: r.Workspace.Slug, - Name: r.Workspace.Name, - FullName: r.Workspace.Name, + Id: r.GroupId(), + Name: r.GroupName(), + FullName: r.GroupName(), }) } return @@ -152,6 +156,10 @@ func listBitbucketRepos( return } +// searchBitbucketRepos searches repositories by name across the user's workspaces. +// The cross-workspace GET /repositories?role=member was removed by Bitbucket +// CHANGE-2770, so we enumerate workspaces and query the workspace-scoped +// GET /repositories/{workspace} endpoint for each, aggregating up to PageSize hits. func searchBitbucketRepos( apiClient plugin.ApiClient, params *dsmodels.DsRemoteApiScopeSearchParams, @@ -159,39 +167,84 @@ func searchBitbucketRepos( children []dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo], err errors.Error, ) { - var res *http.Response - res, err = apiClient.Get( - "/repositories", - url.Values{ - "sort": {"name"}, - "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, - "role": {"member"}, - "q": {fmt.Sprintf(`full_name~"%s"`, params.Search)}, - "page": {fmt.Sprintf("%v", params.Page)}, - "pagelen": {fmt.Sprintf("%v", params.PageSize)}, - }, - nil, - ) - if err != nil { - return nil, err + pageSize := params.PageSize + if pageSize == 0 { + pageSize = 100 } - var resBody models.ReposResponse - err = api.UnmarshalResponse(res, &resBody) + + workspaces, err := listAllBitbucketWorkspaces(apiClient) if err != nil { - return + return nil, err } - for _, r := range resBody.Values { - children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ - Type: api.RAS_ENTRY_TYPE_SCOPE, - Id: r.FullName, - Name: r.Name, - FullName: r.FullName, - Data: r.ConvertApiScope(), - }) + + for _, workspace := range workspaces { + if len(children) >= pageSize { + break + } + var res *http.Response + res, err = apiClient.Get( + fmt.Sprintf("/repositories/%s", workspace), + url.Values{ + "sort": {"name"}, + "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, + "q": {fmt.Sprintf(`name~"%s"`, params.Search)}, + "pagelen": {fmt.Sprintf("%v", pageSize)}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.ReposResponse + err = api.UnmarshalResponse(res, &resBody) + if err != nil { + return + } + for _, r := range resBody.Values { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: r.FullName, + Name: r.Name, + FullName: r.FullName, + Data: r.ConvertApiScope(), + }) + } } return } +// listAllBitbucketWorkspaces returns every workspace slug accessible to the +// authenticated user, following pagination of GET /2.0/user/workspaces. +func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Error) { + var slugs []string + for page := 1; ; page++ { + res, err := apiClient.Get( + "/user/workspaces", + url.Values{ + // No sort/fields (see listBitbucketWorkspaces): /user/workspaces + // returns 400 on sort=workspace.slug. + "page": {fmt.Sprintf("%v", page)}, + "pagelen": {"100"}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.WorkspaceResponse + if err = api.UnmarshalResponse(res, &resBody); err != nil { + return nil, err + } + for _, w := range resBody.Values { + slugs = append(slugs, w.GroupId()) + } + if len(resBody.Values) == 0 || page*resBody.Pagelen >= resBody.Size { + break + } + } + return slugs, nil +} + // RemoteScopes list all available scopes on the remote server // @Summary list all available scopes on the remote server // @Description list all available scopes on the remote server diff --git a/backend/plugins/bitbucket/models/repo.go b/backend/plugins/bitbucket/models/repo.go index 799b549b923..b33df3615b6 100644 --- a/backend/plugins/bitbucket/models/repo.go +++ b/backend/plugins/bitbucket/models/repo.go @@ -117,17 +117,15 @@ type WorkspaceResponse struct { Values []GroupResponse `json:"values"` } +// GroupResponse maps an entry from GET /2.0/user/workspaces, the supported +// replacement after Bitbucket CHANGE-2770 removed the cross-workspace +// GET /2.0/user/permissions/workspaces and deprecated GET /2.0/workspaces. +// Each value nests the workspace slug/name under a "workspace" object. type GroupResponse struct { - //Type string `json:"type"` - //Permission string `json:"permission"` - //LastAccessed time.Time `json:"last_accessed"` - //AddedOn time.Time `json:"added_on"` Workspace WorkspaceItem `json:"workspace"` } type WorkspaceItem struct { - //Type string `json:"type"` - //Uuid string `json:"uuid"` Slug string `json:"slug" group:"id"` Name string `json:"name" group:"name"` } diff --git a/backend/plugins/bitbucket/tasks/api_common.go b/backend/plugins/bitbucket/tasks/api_common.go index 352f41cb952..ad21c9b1e65 100644 --- a/backend/plugins/bitbucket/tasks/api_common.go +++ b/backend/plugins/bitbucket/tasks/api_common.go @@ -238,7 +238,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error { if res.StatusCode == http.StatusUnauthorized { return errors.Unauthorized.New("authentication failed, please check your AccessToken") } - if res.StatusCode == http.StatusNotFound { + // 404: repo has no issue tracker. 410 Gone: Bitbucket has sunset the issue + // tracker/wiki API, so the endpoint is permanently removed for the repo. + // Both mean "nothing to collect" — skip gracefully instead of retrying. + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusGone { return api.ErrIgnoreAndContinue } return nil diff --git a/backend/plugins/bitbucket/tasks/api_common_test.go b/backend/plugins/bitbucket/tasks/api_common_test.go new file mode 100644 index 00000000000..9d15e52e9c4 --- /dev/null +++ b/backend/plugins/bitbucket/tasks/api_common_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestIgnoreHTTPStatus404(t *testing.T) { + cases := []struct { + name string + statusCode int + wantIgnore bool // expect ErrIgnoreAndContinue (graceful skip, no retry) + wantErr bool // expect a real error + }{ + {"404 no issue tracker -> ignore", http.StatusNotFound, true, false}, + {"410 issue tracker sunset -> ignore", http.StatusGone, true, false}, + {"401 unauthorized -> error", http.StatusUnauthorized, false, true}, + {"200 ok -> continue", http.StatusOK, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ignoreHTTPStatus404(&http.Response{StatusCode: tc.statusCode}) + switch { + case tc.wantIgnore: + assert.Equal(t, api.ErrIgnoreAndContinue, err) + case tc.wantErr: + assert.Error(t, err) + default: + assert.NoError(t, err) + } + }) + } +}