diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c4d9ff..3a4c169 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) ## [1.7.1] - Unreleased ### Added +- all-groups for cloning all groups on a hosted gitlab instance ### Changed - go version in go.mod to 1.17 and updated all dependencies ### Deprecated ### Removed ### Fixed +Pagination with gitlab cloud ### Security ## [1.7.0] - 9/2/21 diff --git a/README.md b/README.md index 38d3039..b289f36 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,20 @@ $ ghorg ls someorg 1. Update `GHORG_SCM_TYPE` to `gitlab` in your `ghorg/conf.yaml` or via cli flags 1. See [examples/gitlab.md](https://github.com/gabrie30/ghorg/blob/master/examples/gitlab.md) on how to run +#### gitlab specific notes +1. ghorg works slightly differently for hosted gitlab instances and gitlab cloud + 1. To clone all groups within a hosted instance use the keyword "all-groups" when cloning + ```sh + ghorg clone all-groups --base-url=https://${your.hosted.gitlab.com} --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir + ``` + 1. For gitlab cloud you can use the top level group name e.g. for https://gitlab.com/fdroid + ```sh + ghorg clone fdroid --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir + ``` +1. for hosted instances you need to have a `--base-url` set, cloning cloud gitlab should omit this +1. all flags can be permanently set in your $HOME/.config/ghorg/conf.yaml if you have multiple gitlab instances you can create multiple configuration files for each instance and use different config files with the `--config` flag + + ### gitea setup 1. Create [Access Token](https://docs.gitea.io/en-us/api-usage/) (Settings -> Applications -> Generate Token) diff --git a/cmd/clone.go b/cmd/clone.go index 7c6d682..8bc6acd 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -5,6 +5,7 @@ import ( "bufio" "fmt" "log" + "net/url" "os" "path/filepath" "regexp" @@ -518,4 +519,14 @@ func parseParentFolder(argz []string) { } parentFolder = strings.ToLower(argz[0]) + + // If all-group is used set the parent folder to the name of the baseurl + if argz[0] == "all-groups" && os.Getenv("GHORG_SCM_BASE_URL") != "" { + u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) + if err != nil { + return + } + parentFolder = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com") + fmt.Println(parentFolder) + } } diff --git a/examples/gitlab.md b/examples/gitlab.md index 048ed04..7b438de 100644 --- a/examples/gitlab.md +++ b/examples/gitlab.md @@ -1,4 +1,11 @@ -> Note: all flags can be permanently set in your $HOME/.config/ghorg/conf.yaml +## Hosted GitLab Instances + + +clone all groups on a **hosted gitlab** instance **preserving** the directory structure of subgroups + +``` +$ ghorg clone all-groups --base-url=https:// --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir +``` clone a **user** on a **hosted gitlab** instance using a **token** for auth @@ -23,3 +30,19 @@ clone all repos that are **prefixed** with "frontend" **into a folder** called " ``` $ ghorg clone --base-url=https:// --scm=gitlab --match-regex=^frontend --output-dir=design_only ``` + +## Cloud GitLab Orgs + +eg. https://gitlab.com/fdroid + +clone all groups **preserving** the directory structure of subgroups + +``` +$ ghorg clone fdroid --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir +``` + +clone only a **subgroup** + +``` +$ ghorg clone fdroid/metrics-data --scm=gitlab --token=XXXXXXXXXXXXX +``` diff --git a/scm/gitlab.go b/scm/gitlab.go index c29b4fe..7818adc 100644 --- a/scm/gitlab.go +++ b/scm/gitlab.go @@ -5,12 +5,13 @@ import ( "os" "strings" + "github.com/gabrie30/ghorg/colorlog" gitlab "github.com/xanzy/go-gitlab" ) var ( _ Client = Gitlab{} - perPage = 50 + perPage = 100 ) func init() { @@ -28,6 +29,80 @@ func (_ Gitlab) GetType() string { // GetOrgRepos fetches repo data from a specific group func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { + allGroups := []string{} + repoData := []Repo{} + longFetch := false + + if targetOrg == "all-groups" { + longFetch = true + + grps, err := c.GetTopLevelGroups() + if err != nil { + return nil, fmt.Errorf("error getting groups error: %v", err) + } + + allGroups = append(allGroups, grps...) + + } else { + allGroups = append(allGroups, targetOrg) + } + + for _, group := range allGroups { + if longFetch { + msg := fmt.Sprintf("fetching repos for group: %v", group) + colorlog.PrintInfo(msg) + } + repos, err := c.GetGroupRepos(group) + if err != nil { + return nil, fmt.Errorf("error fetching repos for group '%s', error: %v", group, err) + } + + repoData = append(repoData, repos...) + + } + + return repoData, nil +} + +// GetTopLevelGroups all top level org groups +func (c Gitlab) GetTopLevelGroups() ([]string, error) { + allGroups := []string{} + + opt := &gitlab.ListGroupsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: perPage, + Page: 1, + }, + TopLevelOnly: gitlab.Bool(true), + } + + for { + + groups, resp, err := c.Client.Groups.ListGroups(opt) + + if err != nil { + return allGroups, err + } + + for _, g := range groups { + allGroups = append(allGroups, g.Path) + } + + // Exit the loop when we've seen all pages. + if resp.NextPage == 0 { + break + } + + // Update the page number to get the next page. + opt.Page = resp.NextPage + } + + return allGroups, nil +} + +// GetGroupRepos fetches repo data from a specific group +func (c Gitlab) GetGroupRepos(targetGroup string) ([]Repo, error) { + repoData := []Repo{} opt := &gitlab.ListGroupProjectsOptions{ @@ -40,11 +115,11 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { for { // Get the first page with projects. - ps, resp, err := c.Groups.ListGroupProjects(targetOrg, opt) + ps, resp, err := c.Groups.ListGroupProjects(targetGroup, opt) if err != nil { if resp != nil && resp.StatusCode == 404 { - return nil, fmt.Errorf("group '%s' does not exist", targetOrg) + return nil, fmt.Errorf("group '%s' does not exist", targetGroup) } return []Repo{}, err } @@ -53,7 +128,7 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { repoData = append(repoData, c.filter(ps)...) // Exit the loop when we've seen all pages. - if resp.CurrentPage >= resp.TotalPages { + if resp.NextPage == 0 { break } @@ -89,7 +164,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) { cloneData = append(cloneData, c.filter(ps)...) // Exit the loop when we've seen all pages. - if resp.CurrentPage >= resp.TotalPages { + if resp.NextPage == 0 { break } @@ -116,7 +191,8 @@ func (_ Gitlab) NewClient() (Client, error) { } func (_ Gitlab) addTokenToHTTPSCloneURL(url string, token string) string { - splitURL := strings.Split(url, "https://") + // allows for http and https for local testing + splitURL := strings.Split(url, "://") return "https://oauth2:" + token + "@" + splitURL[1] }