From a2a7abd5d0530bc5a02854823d0b3c26bb769d2b Mon Sep 17 00:00:00 2001 From: Kenneth Wong Date: Wed, 10 Jun 2026 16:26:48 +0800 Subject: [PATCH 1/4] fix(bitbucket): migrate off cross-workspace APIs removed by CHANGE-2770 - listBitbucketWorkspaces: replace GET /user/permissions/workspaces with the supported GET /workspaces (lists the current user's workspaces). Workspace slug/name now parsed from the top level of each value instead of a nested "workspace" object. - searchBitbucketRepos: replace cross-workspace GET /repositories?role=member with per-workspace GET /repositories/{workspace}, enumerating the user's workspaces first and aggregating up to PageSize matches. - models: flatten GroupResponse to match the /workspaces response shape. --- backend/plugins/bitbucket/api/remote_api.go | 115 ++++++++++++++------ backend/plugins/bitbucket/models/repo.go | 17 +-- 2 files changed, 88 insertions(+), 44 deletions(-) diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index 211901ec3c6..78f1ddd75c9 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -67,11 +67,13 @@ func listBitbucketWorkspaces( err errors.Error, ) { var res *http.Response + // /user/permissions/workspaces was removed by Bitbucket CHANGE-2770; /workspaces + // lists the current user's workspaces and is the supported replacement. res, err = apiClient.Get( - "/user/permissions/workspaces", + "/workspaces", url.Values{ - "sort": {"workspace.slug"}, - "fields": {"values.workspace.slug,values.workspace.name,pagelen,page,size"}, + "sort": {"slug"}, + "fields": {"values.slug,values.name,pagelen,page,size"}, "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, @@ -98,9 +100,9 @@ func listBitbucketWorkspaces( for _, r := range resBody.Values { children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: r.Workspace.Slug, - Name: r.Workspace.Name, - FullName: r.Workspace.Name, + Id: r.Slug, + Name: r.Name, + FullName: r.Name, }) } return @@ -152,6 +154,10 @@ func listBitbucketRepos( return } +// searchBitbucketRepos searches repositories by name across the user's workspaces. +// The cross-workspace GET /repositories?role=member was removed by Bitbucket +// CHANGE-2770, so we enumerate workspaces and query the workspace-scoped +// GET /repositories/{workspace} endpoint for each, aggregating up to PageSize hits. func searchBitbucketRepos( apiClient plugin.ApiClient, params *dsmodels.DsRemoteApiScopeSearchParams, @@ -159,39 +165,84 @@ func searchBitbucketRepos( children []dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo], err errors.Error, ) { - var res *http.Response - res, err = apiClient.Get( - "/repositories", - url.Values{ - "sort": {"name"}, - "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, - "role": {"member"}, - "q": {fmt.Sprintf(`full_name~"%s"`, params.Search)}, - "page": {fmt.Sprintf("%v", params.Page)}, - "pagelen": {fmt.Sprintf("%v", params.PageSize)}, - }, - nil, - ) - if err != nil { - return nil, err + pageSize := params.PageSize + if pageSize == 0 { + pageSize = 100 } - var resBody models.ReposResponse - err = api.UnmarshalResponse(res, &resBody) + + workspaces, err := listAllBitbucketWorkspaces(apiClient) if err != nil { - return + return nil, err } - for _, r := range resBody.Values { - children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ - Type: api.RAS_ENTRY_TYPE_SCOPE, - Id: r.FullName, - Name: r.Name, - FullName: r.FullName, - Data: r.ConvertApiScope(), - }) + + for _, workspace := range workspaces { + if len(children) >= pageSize { + break + } + var res *http.Response + res, err = apiClient.Get( + fmt.Sprintf("/repositories/%s", workspace), + url.Values{ + "sort": {"name"}, + "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, + "q": {fmt.Sprintf(`name~"%s"`, params.Search)}, + "pagelen": {fmt.Sprintf("%v", pageSize)}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.ReposResponse + err = api.UnmarshalResponse(res, &resBody) + if err != nil { + return + } + for _, r := range resBody.Values { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: r.FullName, + Name: r.Name, + FullName: r.FullName, + Data: r.ConvertApiScope(), + }) + } } return } +// listAllBitbucketWorkspaces returns every workspace slug accessible to the +// authenticated user, following pagination of GET /2.0/workspaces. +func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Error) { + var slugs []string + for page := 1; ; page++ { + res, err := apiClient.Get( + "/workspaces", + url.Values{ + "sort": {"slug"}, + "fields": {"values.slug,pagelen,page,size"}, + "page": {fmt.Sprintf("%v", page)}, + "pagelen": {"100"}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.WorkspaceResponse + if err = api.UnmarshalResponse(res, &resBody); err != nil { + return nil, err + } + for _, w := range resBody.Values { + slugs = append(slugs, w.Slug) + } + if len(resBody.Values) == 0 || page*resBody.Pagelen >= resBody.Size { + break + } + } + return slugs, nil +} + // RemoteScopes list all available scopes on the remote server // @Summary list all available scopes on the remote server // @Description list all available scopes on the remote server diff --git a/backend/plugins/bitbucket/models/repo.go b/backend/plugins/bitbucket/models/repo.go index 799b549b923..d0785cbba1e 100644 --- a/backend/plugins/bitbucket/models/repo.go +++ b/backend/plugins/bitbucket/models/repo.go @@ -117,27 +117,20 @@ type WorkspaceResponse struct { Values []GroupResponse `json:"values"` } +// GroupResponse maps an entry from GET /2.0/workspaces. The cross-workspace +// GET /2.0/user/permissions/workspaces was removed by Bitbucket CHANGE-2770, +// so slug/name now live at the top level instead of under a nested workspace. type GroupResponse struct { - //Type string `json:"type"` - //Permission string `json:"permission"` - //LastAccessed time.Time `json:"last_accessed"` - //AddedOn time.Time `json:"added_on"` - Workspace WorkspaceItem `json:"workspace"` -} - -type WorkspaceItem struct { - //Type string `json:"type"` - //Uuid string `json:"uuid"` Slug string `json:"slug" group:"id"` Name string `json:"name" group:"name"` } func (p GroupResponse) GroupId() string { - return p.Workspace.Slug + return p.Slug } func (p GroupResponse) GroupName() string { - return p.Workspace.Name + return p.Name } type ReposResponse struct { From 68b71862dbe7df12a586f34a306ab8c013f6bde0 Mon Sep 17 00:00:00 2001 From: Kenneth Wong Date: Sat, 11 Jul 2026 04:02:38 +0800 Subject: [PATCH 2/4] fix(bitbucket): list workspaces via supported /user/workspaces (CHANGE-2770) GET /2.0/workspaces is also deprecated, not just /user/permissions/workspaces. The supported replacement for enumerating the current user's workspaces is GET /2.0/user/workspaces, which nests slug/name under a workspace object. - remote_api.go: listBitbucketWorkspaces / listAllBitbucketWorkspaces now call /user/workspaces with workspace.* fields and read via GroupId()/GroupName(). - models/repo.go: re-nest GroupResponse under WorkspaceItem to match the /user/workspaces response shape. Repo listing stays on the supported GET /2.0/repositories/{workspace}. --- backend/plugins/bitbucket/api/remote_api.go | 27 +++++++++++---------- backend/plugins/bitbucket/models/repo.go | 15 ++++++++---- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index 78f1ddd75c9..4862dafb47c 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -67,13 +67,14 @@ func listBitbucketWorkspaces( err errors.Error, ) { var res *http.Response - // /user/permissions/workspaces was removed by Bitbucket CHANGE-2770; /workspaces - // lists the current user's workspaces and is the supported replacement. + // /user/permissions/workspaces was removed and /workspaces deprecated by + // Bitbucket CHANGE-2770; /user/workspaces lists the current user's workspaces + // and is the supported replacement. res, err = apiClient.Get( - "/workspaces", + "/user/workspaces", url.Values{ - "sort": {"slug"}, - "fields": {"values.slug,values.name,pagelen,page,size"}, + "sort": {"workspace.slug"}, + "fields": {"values.workspace.slug,values.workspace.name,pagelen,page,size"}, "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, @@ -100,9 +101,9 @@ func listBitbucketWorkspaces( for _, r := range resBody.Values { children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: r.Slug, - Name: r.Name, - FullName: r.Name, + Id: r.GroupId(), + Name: r.GroupName(), + FullName: r.GroupName(), }) } return @@ -212,15 +213,15 @@ func searchBitbucketRepos( } // listAllBitbucketWorkspaces returns every workspace slug accessible to the -// authenticated user, following pagination of GET /2.0/workspaces. +// authenticated user, following pagination of GET /2.0/user/workspaces. func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Error) { var slugs []string for page := 1; ; page++ { res, err := apiClient.Get( - "/workspaces", + "/user/workspaces", url.Values{ - "sort": {"slug"}, - "fields": {"values.slug,pagelen,page,size"}, + "sort": {"workspace.slug"}, + "fields": {"values.workspace.slug,pagelen,page,size"}, "page": {fmt.Sprintf("%v", page)}, "pagelen": {"100"}, }, @@ -234,7 +235,7 @@ func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Er return nil, err } for _, w := range resBody.Values { - slugs = append(slugs, w.Slug) + slugs = append(slugs, w.GroupId()) } if len(resBody.Values) == 0 || page*resBody.Pagelen >= resBody.Size { break diff --git a/backend/plugins/bitbucket/models/repo.go b/backend/plugins/bitbucket/models/repo.go index d0785cbba1e..b33df3615b6 100644 --- a/backend/plugins/bitbucket/models/repo.go +++ b/backend/plugins/bitbucket/models/repo.go @@ -117,20 +117,25 @@ type WorkspaceResponse struct { Values []GroupResponse `json:"values"` } -// GroupResponse maps an entry from GET /2.0/workspaces. The cross-workspace -// GET /2.0/user/permissions/workspaces was removed by Bitbucket CHANGE-2770, -// so slug/name now live at the top level instead of under a nested workspace. +// GroupResponse maps an entry from GET /2.0/user/workspaces, the supported +// replacement after Bitbucket CHANGE-2770 removed the cross-workspace +// GET /2.0/user/permissions/workspaces and deprecated GET /2.0/workspaces. +// Each value nests the workspace slug/name under a "workspace" object. type GroupResponse struct { + Workspace WorkspaceItem `json:"workspace"` +} + +type WorkspaceItem struct { Slug string `json:"slug" group:"id"` Name string `json:"name" group:"name"` } func (p GroupResponse) GroupId() string { - return p.Slug + return p.Workspace.Slug } func (p GroupResponse) GroupName() string { - return p.Name + return p.Workspace.Name } type ReposResponse struct { From 31f33081224da91f9d25306da378c3fcfdd4ea4a Mon Sep 17 00:00:00 2001 From: Kenneth Wong Date: Sat, 11 Jul 2026 09:41:07 +0800 Subject: [PATCH 3/4] fix(bitbucket): drop sort/fields on /user/workspaces (400 Invalid field name) GET /2.0/user/workspaces rejects sort=workspace.slug with HTTP 400 'Invalid field name'. The bare paginated call returns the nested workspace objects we parse, so drop the sort/fields query params in both listBitbucketWorkspaces and listAllBitbucketWorkspaces. --- backend/plugins/bitbucket/api/remote_api.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index 4862dafb47c..cf60b1bd51d 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -73,8 +73,9 @@ func listBitbucketWorkspaces( res, err = apiClient.Get( "/user/workspaces", url.Values{ - "sort": {"workspace.slug"}, - "fields": {"values.workspace.slug,values.workspace.name,pagelen,page,size"}, + // No sort/fields: /user/workspaces rejects sort=workspace.slug with + // HTTP 400 "Invalid field name". The bare call returns the nested + // workspace objects we need; WorkspaceResponse ignores extra fields. "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, @@ -220,8 +221,8 @@ func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Er res, err := apiClient.Get( "/user/workspaces", url.Values{ - "sort": {"workspace.slug"}, - "fields": {"values.workspace.slug,pagelen,page,size"}, + // No sort/fields (see listBitbucketWorkspaces): /user/workspaces + // returns 400 on sort=workspace.slug. "page": {fmt.Sprintf("%v", page)}, "pagelen": {"100"}, }, From fbf8e1a85302e7b75a74151736cfd65c92198b8a Mon Sep 17 00:00:00 2001 From: Kenneth Wong Date: Sat, 11 Jul 2026 13:00:16 +0800 Subject: [PATCH 4/4] fix(bitbucket): ignore 410 Gone on sunset issues API Bitbucket has sunset the issue tracker/wiki API; the issues endpoint now returns 410 Gone instead of 404. ignoreHTTPStatus404 only mapped 404 to ErrIgnoreAndContinue, so a 410 fell through to nil, DoAsync retried 3x and failed the whole pipeline ("Collect Issues ended unexpectedly"). Treat 410 like 404 (ignore and continue). Shared helper also covers issue comments and pr commits. Adds a table test for the status-code mapping. --- backend/plugins/bitbucket/tasks/api_common.go | 5 +- .../bitbucket/tasks/api_common_test.go | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/bitbucket/tasks/api_common_test.go diff --git a/backend/plugins/bitbucket/tasks/api_common.go b/backend/plugins/bitbucket/tasks/api_common.go index 352f41cb952..ad21c9b1e65 100644 --- a/backend/plugins/bitbucket/tasks/api_common.go +++ b/backend/plugins/bitbucket/tasks/api_common.go @@ -238,7 +238,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error { if res.StatusCode == http.StatusUnauthorized { return errors.Unauthorized.New("authentication failed, please check your AccessToken") } - if res.StatusCode == http.StatusNotFound { + // 404: repo has no issue tracker. 410 Gone: Bitbucket has sunset the issue + // tracker/wiki API, so the endpoint is permanently removed for the repo. + // Both mean "nothing to collect" — skip gracefully instead of retrying. + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusGone { return api.ErrIgnoreAndContinue } return nil diff --git a/backend/plugins/bitbucket/tasks/api_common_test.go b/backend/plugins/bitbucket/tasks/api_common_test.go new file mode 100644 index 00000000000..9d15e52e9c4 --- /dev/null +++ b/backend/plugins/bitbucket/tasks/api_common_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestIgnoreHTTPStatus404(t *testing.T) { + cases := []struct { + name string + statusCode int + wantIgnore bool // expect ErrIgnoreAndContinue (graceful skip, no retry) + wantErr bool // expect a real error + }{ + {"404 no issue tracker -> ignore", http.StatusNotFound, true, false}, + {"410 issue tracker sunset -> ignore", http.StatusGone, true, false}, + {"401 unauthorized -> error", http.StatusUnauthorized, false, true}, + {"200 ok -> continue", http.StatusOK, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ignoreHTTPStatus404(&http.Response{StatusCode: tc.statusCode}) + switch { + case tc.wantIgnore: + assert.Equal(t, api.ErrIgnoreAndContinue, err) + case tc.wantErr: + assert.Error(t, err) + default: + assert.NoError(t, err) + } + }) + } +}