From 559d9ef06b570418fc313c7168a0d7a948169a80 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 24 Oct 2020 21:24:09 +0200 Subject: [PATCH] Refactor: use NewClient() interface to create a scm client (redo) (#108) * Refactor: save perPage in scm client & use determineClient to load lib clients --- cmd/clone.go | 12 ++++++--- scm/bitbucket.go | 23 ++++++++++------- scm/client.go | 10 +++++--- scm/gitea.go | 61 +++++++++++++++++++++++----------------------- scm/github.go | 50 ++++++++++++++++++------------------- scm/github_test.go | 3 +-- scm/gitlab.go | 47 ++++++++++++++++------------------- 7 files changed, 106 insertions(+), 100 deletions(-) diff --git a/cmd/clone.go b/cmd/clone.go index 191b03f..162533f 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -203,11 +203,17 @@ func getAllUserCloneUrls() ([]scm.Repo, error) { func getCloneUrls(isOrg bool) ([]scm.Repo, error) { asciiTime() PrintConfigs() - client := scm.GetClient(strings.ToLower(os.Getenv("GHORG_SCM_TYPE"))) - if client == nil { - colorlog.PrintError("GHORG_SCM_TYPE not set or unsupported") + scmType := strings.ToLower(os.Getenv("GHORG_SCM_TYPE")) + if len(scmType) == 0 { + colorlog.PrintError("GHORG_SCM_TYPE not set") os.Exit(1) } + client, err := scm.GetClient(scmType) + if err != nil { + colorlog.PrintError(err) + os.Exit(1) + } + if isOrg { return client.GetOrgRepos(targetCloneSource) } diff --git a/scm/bitbucket.go b/scm/bitbucket.go index 6b80da1..753cd38 100644 --- a/scm/bitbucket.go +++ b/scm/bitbucket.go @@ -15,7 +15,10 @@ func init() { registerClient(Bitbucket{}) } -type Bitbucket struct{} +type Bitbucket struct { + // extend the bitbucket client + *bitbucket.Client +} func (_ Bitbucket) GetType() string { return "bitbucket" @@ -23,10 +26,7 @@ func (_ Bitbucket) GetType() string { // GetOrgRepos gets org repos func (c Bitbucket) GetOrgRepos(targetOrg string) ([]Repo, error) { - - client := bitbucket.NewBasicAuth(os.Getenv("GHORG_BITBUCKET_USERNAME"), os.Getenv("GHORG_BITBUCKET_APP_PASSWORD")) - - resp, err := client.Teams.Repositories(targetOrg) + resp, err := c.Teams.Repositories(targetOrg) if err != nil { return []Repo{}, err } @@ -36,10 +36,7 @@ func (c Bitbucket) GetOrgRepos(targetOrg string) ([]Repo, error) { // GetUserRepos gets user repos from bitbucket func (c Bitbucket) GetUserRepos(targetUser string) ([]Repo, error) { - - client := bitbucket.NewBasicAuth(os.Getenv("GHORG_BITBUCKET_USERNAME"), os.Getenv("GHORG_BITBUCKET_APP_PASSWORD")) - - resp, err := client.Users.Repositories(targetUser) + resp, err := c.Users.Repositories(targetUser) if err != nil { return []Repo{}, err } @@ -47,6 +44,14 @@ func (c Bitbucket) GetUserRepos(targetUser string) ([]Repo, error) { return c.filter(resp) } +// NewClient create new bitbucket scm client +func (_ Bitbucket) NewClient() (Client, error) { + user := os.Getenv("GHORG_BITBUCKET_USERNAME") + password := os.Getenv("GHORG_BITBUCKET_APP_PASSWORD") + c := bitbucket.NewBasicAuth(user, password) + return Bitbucket{c}, nil +} + func (_ Bitbucket) filter(resp interface{}) (repoData []Repo, err error) { cloneData := []Repo{} values := resp.(map[string]interface{})["values"].([]interface{}) diff --git a/scm/client.go b/scm/client.go index 475bcb7..a0f2010 100644 --- a/scm/client.go +++ b/scm/client.go @@ -1,7 +1,11 @@ package scm +import "fmt" + // Client define the interface a scm client has to have type Client interface { + NewClient() (Client, error) + GetUserRepos(targetUsername string) ([]Repo, error) GetOrgRepos(targetOrg string) ([]Repo, error) @@ -17,13 +21,13 @@ func registerClient(c Client) { clients = append(clients, c) } -func GetClient(cType string) Client { +func GetClient(cType string) (Client, error) { for i := range clients { if clients[i].GetType() == cType { - return clients[i] + return clients[i].NewClient() } } - return nil + return nil, fmt.Errorf("client type '%s' unsupported", cType) } // SupportedClients return list of all supported clients diff --git a/scm/gitea.go b/scm/gitea.go index 4fe08b0..45a152f 100644 --- a/scm/gitea.go +++ b/scm/gitea.go @@ -17,7 +17,12 @@ func init() { registerClient(Gitea{}) } -type Gitea struct{} +type Gitea struct { + // extend the gitea client + *gitea.Client + // perPage contain the pagination item limit + perPage int +} func (_ Gitea) GetType() string { return "gitea" @@ -26,21 +31,11 @@ func (_ Gitea) GetType() string { // GetOrgRepos fetches repo data from a specific group func (c Gitea) GetOrgRepos(targetOrg string) ([]Repo, error) { repoData := []Repo{} - client, err := c.determineClient() - - if err != nil { - return nil, err - } - - perPage := 10 - if conf, _, err := client.GetGlobalAPISettings(); err == nil && conf != nil { - perPage = conf.MaxResponseItems - } for i := 1; ; i++ { - rps, resp, err := client.ListOrgRepos(targetOrg, gitea.ListOrgReposOptions{ListOptions: gitea.ListOptions{ + rps, resp, err := c.ListOrgRepos(targetOrg, gitea.ListOrgReposOptions{ListOptions: gitea.ListOptions{ Page: i, - PageSize: perPage, + PageSize: c.perPage, }}) if err != nil { @@ -50,7 +45,7 @@ func (c Gitea) GetOrgRepos(targetOrg string) ([]Repo, error) { return nil, err } - repoDataFiltered, err := c.filter(client, rps) + repoDataFiltered, err := c.filter(rps) if err != nil { return nil, err } @@ -68,21 +63,11 @@ func (c Gitea) GetOrgRepos(targetOrg string) ([]Repo, error) { // GetUserRepos gets all of a users gitlab repos func (c Gitea) GetUserRepos(targetUsername string) ([]Repo, error) { repoData := []Repo{} - client, err := c.determineClient() - - if err != nil { - return nil, err - } - - perPage := 10 - if conf, _, err := client.GetGlobalAPISettings(); err == nil && conf != nil { - perPage = conf.MaxResponseItems - } for i := 1; ; i++ { - rps, resp, err := client.ListUserRepos(targetUsername, gitea.ListReposOptions{ListOptions: gitea.ListOptions{ + rps, resp, err := c.ListUserRepos(targetUsername, gitea.ListReposOptions{ListOptions: gitea.ListOptions{ Page: i, - PageSize: perPage, + PageSize: c.perPage, }}) if err != nil { @@ -92,7 +77,7 @@ func (c Gitea) GetUserRepos(targetUsername string) ([]Repo, error) { return nil, err } - repoDataFiltered, err := c.filter(client, rps) + repoDataFiltered, err := c.filter(rps) if err != nil { return nil, err } @@ -107,7 +92,8 @@ func (c Gitea) GetUserRepos(targetUsername string) ([]Repo, error) { return repoData, nil } -func (c Gitea) determineClient() (*gitea.Client, error) { +// NewClient create new gitea scm client +func (_ Gitea) NewClient() (Client, error) { baseURL := os.Getenv("GHORG_SCM_BASE_URL") token := os.Getenv("GHORG_GITEA_TOKEN") @@ -115,10 +101,23 @@ func (c Gitea) determineClient() (*gitea.Client, error) { baseURL = "https://gitea.com" } - return gitea.NewClient(baseURL, gitea.SetToken(token)) + c, err := gitea.NewClient(baseURL, gitea.SetToken(token)) + if err != nil { + return nil, err + } + client := Gitea{Client: c} + + //set small limit so gitea most likely will have a bigger one + client.perPage = 10 + if conf, _, err := client.GetGlobalAPISettings(); err == nil && conf != nil { + // gitea >= 1.13 will tell us the limit it has + client.perPage = conf.MaxResponseItems + } + + return client, nil } -func (c Gitea) filter(client *gitea.Client, rps []*gitea.Repository) (repoData []Repo, err error) { +func (c Gitea) filter(rps []*gitea.Repository) (repoData []Repo, err error) { envTopics := strings.Split(os.Getenv("GHORG_TOPICS"), ",") for _, rp := range rps { @@ -137,7 +136,7 @@ func (c Gitea) filter(client *gitea.Client, rps []*gitea.Repository) (repoData [ // If user defined a list of topics, check if any match with this repo if os.Getenv("GHORG_TOPICS") != "" { - rpTopics, _, err := client.ListRepoTopics(rp.Owner.UserName, rp.Name, gitea.ListRepoTopicsOptions{}) + rpTopics, _, err := c.ListRepoTopics(rp.Owner.UserName, rp.Name, gitea.ListRepoTopicsOptions{}) if err != nil { return []Repo{}, err } diff --git a/scm/github.go b/scm/github.go index 04bfe0a..79b1636 100644 --- a/scm/github.go +++ b/scm/github.go @@ -19,7 +19,10 @@ func init() { } type Github struct { - client *github.Client + // extend the github client + *github.Client + // perPage contain the pagination item limit + perPage int } func (_ Github) GetType() string { @@ -28,17 +31,13 @@ func (_ Github) GetType() string { // GetOrgRepos gets org repos func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) { - if c.client == nil { - c.client = c.newGitHubClient() - } - if os.Getenv("GHORG_SCM_BASE_URL") != "" { - c.client.BaseURL, _ = url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) + c.BaseURL, _ = url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) } opt := &github.RepositoryListByOrgOptions{ Type: "all", - ListOptions: github.ListOptions{PerPage: 100, Page: 0}, + ListOptions: github.ListOptions{PerPage: c.perPage}, } envTopics := strings.Split(os.Getenv("GHORG_TOPICS"), ",") @@ -47,7 +46,7 @@ func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) { var allRepos []*github.Repository for { - repos, resp, err := c.client.Repositories.ListByOrg(context.Background(), targetOrg, opt) + repos, resp, err := c.Repositories.ListByOrg(context.Background(), targetOrg, opt) if err != nil { return nil, err @@ -65,17 +64,13 @@ func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) { // GetUserRepos gets user repos func (c Github) GetUserRepos(targetUser string) ([]Repo, error) { - if c.client == nil { - c.client = c.newGitHubClient() - } - if os.Getenv("GHORG_SCM_BASE_URL") != "" { - c.client.BaseURL, _ = url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) + c.BaseURL, _ = url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) } opt := &github.RepositoryListOptions{ Type: "all", - ListOptions: github.ListOptions{PerPage: 100, Page: 0}, + ListOptions: github.ListOptions{PerPage: c.perPage}, } envTopics := strings.Split(os.Getenv("GHORG_TOPICS"), ",") @@ -84,7 +79,7 @@ func (c Github) GetUserRepos(targetUser string) ([]Repo, error) { var allRepos []*github.Repository for { - repos, resp, err := c.client.Repositories.List(context.Background(), targetUser, opt) + repos, resp, err := c.Repositories.List(context.Background(), targetUser, opt) if err != nil { return nil, err @@ -99,6 +94,20 @@ func (c Github) GetUserRepos(targetUser string) ([]Repo, error) { return c.filter(allRepos, envTopics), nil } +// NewClient create new github scm client +func (_ Github) NewClient() (Client, error) { + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GHORG_GITHUB_TOKEN")}, + ) + tc := oauth2.NewClient(ctx, ts) + c := github.NewClient(tc) + + client := Github{Client: c, perPage: 100} + + return client, nil +} + func (_ Github) addTokenToHTTPSCloneURL(url string, token string) string { splitURL := strings.Split(url, "https://") return "https://" + token + "@" + splitURL[1] @@ -165,14 +174,3 @@ func (c Github) filter(allRepos []*github.Repository, envTopics []string) []Repo return repoData } - -// newGitHubClient creates a github client -func (_ Github) newGitHubClient() *github.Client { - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GHORG_GITHUB_TOKEN")}, - ) - tc := oauth2.NewClient(ctx, ts) - client := github.NewClient(tc) - return client -} diff --git a/scm/github_test.go b/scm/github_test.go index 901a4aa..7429b52 100644 --- a/scm/github_test.go +++ b/scm/github_test.go @@ -52,8 +52,7 @@ func setup() (client *ghpkg.Client, mux *http.ServeMux, serverURL string, teardo func TestGetOrgRepos(t *testing.T) { client, mux, _, teardown := setup() - github := Github{} - github.client = client + github := Github{Client: client} defer teardown() diff --git a/scm/gitlab.go b/scm/gitlab.go index 3439366..7f60e0b 100644 --- a/scm/gitlab.go +++ b/scm/gitlab.go @@ -17,7 +17,10 @@ func init() { registerClient(Gitlab{}) } -type Gitlab struct{} +type Gitlab struct { + // extend the gitlab client + *gitlab.Client +} func (_ Gitlab) GetType() string { return "gitlab" @@ -26,11 +29,6 @@ func (_ Gitlab) GetType() string { // GetOrgRepos fetches repo data from a specific group func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { repoData := []Repo{} - client, err := c.determineClient() - - if err != nil { - return nil, err - } opt := &gitlab.ListGroupProjectsOptions{ ListOptions: gitlab.ListOptions{ @@ -42,7 +40,7 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { for { // Get the first page with projects. - ps, resp, err := client.Groups.ListGroupProjects(targetOrg, opt) + ps, resp, err := c.Groups.ListGroupProjects(targetOrg, opt) if err != nil { if resp != nil && resp.StatusCode == 404 { @@ -66,28 +64,10 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { return repoData, nil } -func (_ Gitlab) determineClient() (*gitlab.Client, error) { - baseURL := os.Getenv("GHORG_SCM_BASE_URL") - token := os.Getenv("GHORG_GITLAB_TOKEN") - - if baseURL != "" { - client, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) - return client, err - } - - return gitlab.NewClient(token) -} - // GetUserRepos gets all of a users gitlab repos func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) { cloneData := []Repo{} - client, err := c.determineClient() - - if err != nil { - return nil, err - } - opt := &gitlab.ListProjectsOptions{ ListOptions: gitlab.ListOptions{ PerPage: perPage, @@ -97,7 +77,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) { for { // Get the first page with projects. - ps, resp, err := client.Projects.ListUserProjects(targetUsername, opt) + ps, resp, err := c.Projects.ListUserProjects(targetUsername, opt) if err != nil { if resp != nil && resp.StatusCode == 404 { return nil, fmt.Errorf("user '%s' does not exist", targetUsername) @@ -120,6 +100,21 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) { return cloneData, nil } +// NewClient create new gitlab scm client +func (_ Gitlab) NewClient() (Client, error) { + baseURL := os.Getenv("GHORG_SCM_BASE_URL") + token := os.Getenv("GHORG_GITLAB_TOKEN") + + var err error + var c *gitlab.Client + if baseURL != "" { + c, err = gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) + } else { + c, err = gitlab.NewClient(token) + } + return Gitlab{c}, err +} + func (_ Gitlab) addTokenToHTTPSCloneURL(url string, token string) string { splitURL := strings.Split(url, "https://") return "https://oauth2:" + token + "@" + splitURL[1]