Skip to content

Commit 44463b0

Browse files
trickyearlobetrickyearlobe
committed
Reduce number of API calls
Signed-off-by: Richard Nixon <[email protected]>
1 parent b263d62 commit 44463b0

File tree

6 files changed

+274
-1644
lines changed

6 files changed

+274
-1644
lines changed

‎go.mod

-11
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,3 @@
11
module github.com/trickyearlobe/bitsync
22

33
go 1.20
4-
5-
require github.com/ktrysmt/go-bitbucket v0.9.63
6-
7-
require (
8-
github.com/golang/protobuf v1.5.3 // indirect
9-
github.com/mitchellh/mapstructure v1.5.0 // indirect
10-
golang.org/x/net v0.12.0 // indirect
11-
golang.org/x/oauth2 v0.10.0 // indirect
12-
google.golang.org/appengine v1.6.7 // indirect
13-
google.golang.org/protobuf v1.31.0 // indirect
14-
)

‎go.sum

-1,573
Large diffs are not rendered by default.

‎main.go

+51-60
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,78 @@
11
package main
22

33
import (
4-
"fmt"
5-
"github.com/ktrysmt/go-bitbucket"
6-
"os"
7-
"os/exec"
8-
"path/filepath"
4+
"fmt"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
98
)
109

11-
var client *bitbucket.Client
12-
1310
func main() {
14-
// Check we have env vars set
15-
if os.Getenv("BBUSER") == "" || os.Getenv("BBAPPPASS") == "" {
16-
fmt.Println("Please set credential in environment vars BBUSER and BBAPPPASS")
17-
fmt.Println(" export BBUSER=ebeneezer")
18-
fmt.Println(" export BBAPPPASS=ijfewIIejdiiowIOEIJEDiojoewfiJIOEF")
19-
return
20-
}
11+
// Check we have env vars set
12+
if os.Getenv("BBUSER") == "" || os.Getenv("BBAPPPASS") == "" {
13+
fmt.Println("Please set credential in environment vars BBUSER and BBAPPPASS")
14+
fmt.Println(" export BBUSER=ebeneezer")
15+
fmt.Println(" export BBAPPPASS=ijfewIIejdiiowIOEIJEDiojoewfiJIOEF")
16+
return
17+
}
2118

22-
// Iterate the workspaces we have access to
23-
client = bitbucket.NewBasicAuth(os.Getenv("BBUSER"), os.Getenv("BBAPPPASS"))
24-
workspaces, err := client.Workspaces.List()
25-
checkErr(err)
26-
for _, workspace := range workspaces.Workspaces {
27-
processWorkspace(workspace)
28-
}
19+
// Iterate the workspaces we have access to
20+
workspaces := fetchOrganisations(os.Getenv("BBUSER"), os.Getenv("BBAPPPASS"))
21+
for _, workspace := range workspaces {
22+
processWorkspace(workspace)
23+
}
2924
}
3025

31-
func processWorkspace(workkspace bitbucket.Workspace) {
32-
fmt.Printf("Processing workspace %v\n", workkspace.Slug)
33-
options := bitbucket.RepositoriesOptions{
34-
Owner: workkspace.Slug,
35-
}
36-
repos, err := client.Repositories.ListForAccount(&options)
37-
checkErr(err)
26+
func processWorkspace(workkspace string) {
27+
fmt.Printf("Processing workspace %v\n", workkspace)
28+
repos := FetchBitbucketRepos(os.Getenv("BBUSER"), os.Getenv("BBAPPPASS"), workkspace)
3829

39-
for _, repo := range repos.Items {
40-
processRepo(workkspace.Slug, repo.Project.Key, repo.Slug, repo.Mainbranch.Name)
41-
}
42-
fmt.Println("DONE")
30+
for _, repo := range repos {
31+
processRepo(workkspace, repo.Project.Key, repo.Slug, repo.Mainbranch.Name)
32+
}
33+
fmt.Println("DONE")
4334
}
4435

4536
func processRepo(workspace, project, repo, mainBranch string) {
46-
//fmt.Printf(" Repo: %v/%v/%v\n", workspace, project, repo)
47-
home, err := os.UserHomeDir()
48-
checkErr(err)
37+
//fmt.Printf(" Repo: %v/%v/%v\n", workspace, project, repo)
38+
home, err := os.UserHomeDir()
39+
checkErr(err)
4940

50-
projectDir := filepath.Join(home, "repos", workspace, project)
51-
repoDir := filepath.Join(home, "repos", workspace, project, repo)
41+
projectDir := filepath.Join(home, "repos", workspace, project)
42+
repoDir := filepath.Join(home, "repos", workspace, project, repo)
5243

53-
// Make the project directory
54-
err = os.MkdirAll(projectDir, 0750)
55-
checkErr(err)
44+
// Make the project directory
45+
err = os.MkdirAll(projectDir, 0750)
46+
checkErr(err)
5647

57-
// Check if the repo is cloned
58-
if _, err = os.Stat(repoDir); err != nil {
59-
cloneRepo(repoDir, workspace, project, repo)
60-
} else {
61-
pullRepo(repoDir, mainBranch)
62-
}
48+
// Check if the repo is cloned
49+
if _, err = os.Stat(repoDir); err != nil {
50+
cloneRepo(repoDir, workspace, project, repo)
51+
} else {
52+
pullRepo(repoDir, mainBranch)
53+
}
6354

6455
}
6556

6657
func pullRepo(repoDir, mainBranch string) {
67-
fmt.Printf(" Pulling %v branch %v\n", repoDir, mainBranch)
68-
out, err := exec.Command("git", "-C", repoDir, "pull", "origin", mainBranch+":"+mainBranch).CombinedOutput()
69-
if err != nil {
70-
fmt.Printf("%v\n", string(out))
71-
}
58+
fmt.Printf(" Pulling %v branch %v\n", repoDir, mainBranch)
59+
out, err := exec.Command("git", "-C", repoDir, "pull", "origin", mainBranch+":"+mainBranch).CombinedOutput()
60+
if err != nil {
61+
fmt.Printf("%v\n", string(out))
62+
}
7263
}
7364

7465
func cloneRepo(repoDir, workspace, project, repo string) {
75-
fmt.Printf(" Cloning %v/%v/%v into %v\n", workspace, project, repo, repoDir)
76-
out, err := exec.Command("git", "clone", "[email protected]:"+workspace+"/"+repo, repoDir).CombinedOutput()
77-
if err != nil {
78-
fmt.Printf("%v\n", string(out))
79-
}
66+
fmt.Printf(" Cloning %v/%v/%v into %v\n", workspace, project, repo, repoDir)
67+
out, err := exec.Command("git", "clone", "[email protected]:"+workspace+"/"+repo, repoDir).CombinedOutput()
68+
if err != nil {
69+
fmt.Printf("%v\n", string(out))
70+
}
8071
}
8172

8273
func checkErr(err error) {
83-
if err != nil {
84-
fmt.Println("An error occured. %v", err)
85-
os.Exit(1)
86-
}
74+
if err != nil {
75+
fmt.Printf("An error occured. %v\n", err)
76+
os.Exit(1)
77+
}
8778
}

‎types_organisations.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"io/ioutil"
6+
"net/http"
7+
"time"
8+
)
9+
10+
func fetchOrganisations(user, password string) []string {
11+
var organisations []string
12+
url := "https://api.bitbucket.org/2.0/workspaces?pagelen=100"
13+
for url != "" {
14+
orgPage := fetchBitbucketPage(user, password, url)
15+
for _, organisation := range orgPage.Values {
16+
organisations = append(organisations, organisation.Slug)
17+
}
18+
url = orgPage.Next
19+
}
20+
return organisations
21+
}
22+
23+
func fetchBitbucketPage(user, password, url string) BitbucketOrganisationsResponse {
24+
var bitbucketOrganisations BitbucketOrganisationsResponse
25+
client := http.Client{}
26+
req, err := http.NewRequest("GET", url, nil)
27+
checkErr(err)
28+
req.SetBasicAuth(user, password)
29+
req.Header.Set("Accept", "application/json")
30+
resp, err := client.Do(req)
31+
checkErr(err)
32+
defer resp.Body.Close()
33+
bodyText, err := ioutil.ReadAll(resp.Body)
34+
checkErr(err)
35+
json.Unmarshal(bodyText, &bitbucketOrganisations)
36+
return bitbucketOrganisations
37+
}
38+
39+
type BitbucketOrganisationsResponse struct {
40+
Values []bitbucketOrganisation `json:"values"`
41+
Pagelen int `json:"pagelen"`
42+
Size int `json:"size"`
43+
Page int `json:"page"`
44+
Next string `json:"next"`
45+
}
46+
47+
type bitbucketOrganisation struct {
48+
Type string `json:"type"`
49+
Uuid string `json:"uuid"`
50+
Name string `json:"name"`
51+
Slug string `json:"slug"`
52+
IsPrivate bool `json:"is_private"`
53+
Links struct {
54+
Avatar bitbucketOrganisationHref `json:"avatar"`
55+
Hooks bitbucketOrganisationHref `json:"hooks"`
56+
Html bitbucketOrganisationHref `json:"html"`
57+
HtmlOverview bitbucketOrganisationHref `json:"html_overview"`
58+
Members bitbucketOrganisationHref `json:"members"`
59+
Owners bitbucketOrganisationHref `json:"owners"`
60+
Projects bitbucketOrganisationHref `json:"projects"`
61+
Repositories bitbucketOrganisationHref `json:"repositories"`
62+
Snippets bitbucketOrganisationHref `json:"snippets"`
63+
Self bitbucketOrganisationHref `json:"self"`
64+
} `json:"links"`
65+
CreatedOn time.Time `json:"created_on"`
66+
}
67+
68+
type bitbucketOrganisationHref struct {
69+
Href string `json:"href"`
70+
}

‎types_repos.go

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"io/ioutil"
6+
"net/http"
7+
"time"
8+
)
9+
10+
// Iterates to fetch all the repos
11+
func FetchBitbucketRepos(user string, password string, workspace string) []BitbucketRepository {
12+
var repositories []BitbucketRepository
13+
url := "https://api.bitbucket.org/2.0/repositories/" + workspace + "?pagelen=100"
14+
for url != "" {
15+
repoPage := fetchBitbucketReposPage(user, password, url)
16+
for _, repository := range repoPage.Values {
17+
repositories = append(repositories, repository)
18+
}
19+
url = repoPage.Next
20+
}
21+
return repositories
22+
}
23+
24+
// Fetches an individual page of repos
25+
func fetchBitbucketReposPage(user, password, url string) BitbucketRepositoriesResponse {
26+
var bitbucketResponse BitbucketRepositoriesResponse
27+
client := http.Client{}
28+
req, err := http.NewRequest("GET", url, nil)
29+
checkErr(err)
30+
req.SetBasicAuth(user, password)
31+
req.Header.Set("Accept", "application/json")
32+
resp, err := client.Do(req)
33+
checkErr(err)
34+
defer resp.Body.Close()
35+
bodyText, err := ioutil.ReadAll(resp.Body)
36+
checkErr(err)
37+
err = json.Unmarshal(bodyText, &bitbucketResponse)
38+
checkErr(err)
39+
return bitbucketResponse
40+
}
41+
42+
type BitbucketRepositoriesResponse struct {
43+
Values []BitbucketRepository `json:"values"`
44+
Pagelen int `json:"pagelen"`
45+
Size int `json:"size"`
46+
Page int `json:"page"`
47+
Next string `json:"next"`
48+
}
49+
50+
type BitbucketRepository struct {
51+
Type string `json:"type"`
52+
FullName string `json:"full_name"`
53+
Links struct {
54+
Self bitbucketRepoHref `json:"self"`
55+
Html bitbucketRepoHref `json:"html"`
56+
Avatar bitbucketRepoHref `json:"avatar"`
57+
Pullrequests bitbucketRepoHref `json:"pullrequests"`
58+
Commits bitbucketRepoHref `json:"commits"`
59+
Forks bitbucketRepoHref `json:"forks"`
60+
Watchers bitbucketRepoHref `json:"watchers"`
61+
Branches bitbucketRepoHref `json:"branches"`
62+
Tags bitbucketRepoHref `json:"tags"`
63+
Downloads bitbucketRepoHref `json:"downloads"`
64+
Source bitbucketRepoHref `json:"source"`
65+
Clone []struct {
66+
Name string `json:"name"`
67+
Href string `json:"href"`
68+
} `json:"clone"`
69+
Hooks bitbucketRepoHref `json:"hooks"`
70+
} `json:"links"`
71+
Name string `json:"name"`
72+
Slug string `json:"slug"`
73+
Description string `json:"description"`
74+
Scm string `json:"scm"`
75+
Website interface{} `json:"website"`
76+
Owner struct {
77+
DisplayName string `json:"display_name"`
78+
Links struct {
79+
Self bitbucketRepoHref `json:"self"`
80+
Avatar bitbucketRepoHref `json:"avatar"`
81+
Html bitbucketRepoHref `json:"html"`
82+
} `json:"links"`
83+
Type string `json:"type"`
84+
Uuid string `json:"uuid"`
85+
Username string `json:"username"`
86+
} `json:"owner"`
87+
Workspace struct {
88+
Type string `json:"type"`
89+
Uuid string `json:"uuid"`
90+
Name string `json:"name"`
91+
Slug string `json:"slug"`
92+
Links struct {
93+
Avatar bitbucketRepoHref `json:"avatar"`
94+
Html bitbucketRepoHref `json:"html"`
95+
Self bitbucketRepoHref `json:"self"`
96+
} `json:"links"`
97+
} `json:"workspace"`
98+
IsPrivate bool `json:"is_private"`
99+
Project struct {
100+
Type string `json:"type"`
101+
Key string `json:"key"`
102+
Uuid string `json:"uuid"`
103+
Name string `json:"name"`
104+
Links struct {
105+
Self bitbucketRepoHref `json:"self"`
106+
Html bitbucketRepoHref `json:"html"`
107+
Avatar bitbucketRepoHref `json:"avatar"`
108+
} `json:"links"`
109+
} `json:"project"`
110+
ForkPolicy string `json:"fork_policy"`
111+
CreatedOn time.Time `json:"created_on"`
112+
UpdatedOn time.Time `json:"updated_on"`
113+
Size int `json:"size"`
114+
Language string `json:"language"`
115+
HasIssues bool `json:"has_issues"`
116+
HasWiki bool `json:"has_wiki"`
117+
Uuid string `json:"uuid"`
118+
Mainbranch struct {
119+
Name string `json:"name"`
120+
Type string `json:"type"`
121+
} `json:"mainbranch"`
122+
OverrideSettings struct {
123+
DefaultMergeStrategy bool `json:"default_merge_strategy"`
124+
BranchingModel bool `json:"branching_model"`
125+
} `json:"override_settings"`
126+
}
127+
128+
type bitbucketRepoHref struct {
129+
Href string `json:"href"`
130+
}

‎types_test.go

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"testing"
6+
)
7+
8+
var organisations []string
9+
10+
func TestFetchOrganisations(t *testing.T) {
11+
organisations := fetchOrganisations(os.Getenv("BBUSER"), os.Getenv("BBAPPPASS"))
12+
if len(organisations) < 1 {
13+
t.Fatalf(`Length of workspaces is %v`, len(organisations))
14+
}
15+
}
16+
17+
func TestFetchRepos(t *testing.T) {
18+
organisations := fetchOrganisations(os.Getenv("BBUSER"), os.Getenv("BBAPPPASS"))
19+
repos := FetchBitbucketRepos(os.Getenv("BBUSER"), os.Getenv("BBAPPPASS"), organisations[0])
20+
if len(repos) < 1 {
21+
t.Fatalf(`Length of repositories is %v`, len(organisations))
22+
}
23+
}

0 commit comments

Comments
 (0)