Update hosted gitlab to clone all repos and fix gitlab cloud pagination (#152)

CHANGELOG.md

@@ -5,11 +5,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
 ## [1.7.1] - Unreleased
 ### Added
+- all-groups for cloning all groups on a hosted gitlab instance
 ### Changed
 - go version in go.mod to 1.17 and updated all dependencies
 ### Deprecated
 ### Removed
 ### Fixed
+Pagination with gitlab cloud
 ### Security
 
 ## [1.7.0] - 9/2/21

README.md

@@ -106,6 +106,20 @@ $ ghorg ls someorg
 1. Update `GHORG_SCM_TYPE` to `gitlab` in your `ghorg/conf.yaml` or via cli flags
 1. See [examples/gitlab.md](https://github.com/gabrie30/ghorg/blob/master/examples/gitlab.md) on how to run
 
+#### gitlab specific notes
+1. ghorg works slightly differently for hosted gitlab instances and gitlab cloud
+    1. To clone all groups within a hosted instance use the keyword "all-groups" when cloning
+        ```sh
+        ghorg clone all-groups --base-url=https://${your.hosted.gitlab.com} --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir
+        ```
+    1. For gitlab cloud you can use the top level group name e.g. for https://gitlab.com/fdroid
+        ```sh
+        ghorg clone fdroid --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir
+        ```
+1. for hosted instances you need to have a `--base-url` set, cloning cloud gitlab should omit this
+1. all flags can be permanently set in your $HOME/.config/ghorg/conf.yaml if you have multiple gitlab instances you can create multiple configuration files for each instance and use different config files with the `--config` flag
+
+
 ### gitea setup
 
 1. Create [Access Token](https://docs.gitea.io/en-us/api-usage/) (Settings -> Applications -> Generate Token)

cmd/clone.go

@@ -5,6 +5,7 @@ import (
 	"bufio"
 	"fmt"
 	"log"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -518,4 +519,14 @@ func parseParentFolder(argz []string) {
 	}
 
 	parentFolder = strings.ToLower(argz[0])
+
+	// If all-group is used set the parent folder to the name of the baseurl
+	if argz[0] == "all-groups" && os.Getenv("GHORG_SCM_BASE_URL") != "" {
+		u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
+		if err != nil {
+			return
+		}
+		parentFolder = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com")
+		fmt.Println(parentFolder)
+	}
 }

examples/gitlab.md

@@ -1,4 +1,11 @@
-> Note: all flags can be permanently set in your $HOME/.config/ghorg/conf.yaml
+## Hosted GitLab Instances
+
+
+clone all groups on a **hosted gitlab** instance **preserving** the directory structure of subgroups
+
+```
+$ ghorg clone all-groups --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir
+```
 
 clone a **user** on a **hosted gitlab** instance using a **token** for auth
 
@@ -23,3 +30,19 @@ clone all repos that are **prefixed** with "frontend" **into a folder** called "
 ```
 $ ghorg clone <gitlab_group> --base-url=https://<your.instance.gitlab.com> --scm=gitlab --match-regex=^frontend --output-dir=design_only
 ```
+
+## Cloud GitLab Orgs
+
+eg. https://gitlab.com/fdroid
+
+clone all groups **preserving** the directory structure of subgroups
+
+```
+$ ghorg clone fdroid --scm=gitlab --token=XXXXXXXXXXXXX --preserve-dir
+```
+
+clone only a **subgroup**
+
+```
+$ ghorg clone fdroid/metrics-data --scm=gitlab --token=XXXXXXXXXXXXX
+```

scm/gitlab.go

@@ -5,12 +5,13 @@ import (
 	"os"
 	"strings"
 
+	"github.com/gabrie30/ghorg/colorlog"
 	gitlab "github.com/xanzy/go-gitlab"
 )
 
 var (
 	_       Client = Gitlab{}
-	perPage        = 50
+	perPage        = 100
 )
 
 func init() {
@@ -28,6 +29,80 @@ func (_ Gitlab) GetType() string {
 
 // GetOrgRepos fetches repo data from a specific group
 func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
+	allGroups := []string{}
+	repoData := []Repo{}
+	longFetch := false
+
+	if targetOrg == "all-groups" {
+		longFetch = true
+
+		grps, err := c.GetTopLevelGroups()
+		if err != nil {
+			return nil, fmt.Errorf("error getting groups error: %v", err)
+		}
+
+		allGroups = append(allGroups, grps...)
+
+	} else {
+		allGroups = append(allGroups, targetOrg)
+	}
+
+	for _, group := range allGroups {
+		if longFetch {
+			msg := fmt.Sprintf("fetching repos for group: %v", group)
+			colorlog.PrintInfo(msg)
+		}
+		repos, err := c.GetGroupRepos(group)
+		if err != nil {
+			return nil, fmt.Errorf("error fetching repos for group '%s', error: %v", group, err)
+		}
+
+		repoData = append(repoData, repos...)
+
+	}
+
+	return repoData, nil
+}
+
+// GetTopLevelGroups all top level org groups
+func (c Gitlab) GetTopLevelGroups() ([]string, error) {
+	allGroups := []string{}
+
+	opt := &gitlab.ListGroupsOptions{
+		ListOptions: gitlab.ListOptions{
+			PerPage: perPage,
+			Page:    1,
+		},
+		TopLevelOnly: gitlab.Bool(true),
+	}
+
+	for {
+
+		groups, resp, err := c.Client.Groups.ListGroups(opt)
+
+		if err != nil {
+			return allGroups, err
+		}
+
+		for _, g := range groups {
+			allGroups = append(allGroups, g.Path)
+		}
+
+		// Exit the loop when we've seen all pages.
+		if resp.NextPage == 0 {
+			break
+		}
+
+		// Update the page number to get the next page.
+		opt.Page = resp.NextPage
+	}
+
+	return allGroups, nil
+}
+
+// GetGroupRepos fetches repo data from a specific group
+func (c Gitlab) GetGroupRepos(targetGroup string) ([]Repo, error) {
+
 	repoData := []Repo{}
 
 	opt := &gitlab.ListGroupProjectsOptions{
@@ -40,11 +115,11 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
 
 	for {
 		// Get the first page with projects.
-		ps, resp, err := c.Groups.ListGroupProjects(targetOrg, opt)
+		ps, resp, err := c.Groups.ListGroupProjects(targetGroup, opt)
 
 		if err != nil {
 			if resp != nil && resp.StatusCode == 404 {
-				return nil, fmt.Errorf("group '%s' does not exist", targetOrg)
+				return nil, fmt.Errorf("group '%s' does not exist", targetGroup)
 			}
 			return []Repo{}, err
 		}
@@ -53,7 +128,7 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
 		repoData = append(repoData, c.filter(ps)...)
 
 		// Exit the loop when we've seen all pages.
-		if resp.CurrentPage >= resp.TotalPages {
+		if resp.NextPage == 0 {
 			break
 		}
 
@@ -89,7 +164,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
 		cloneData = append(cloneData, c.filter(ps)...)
 
 		// Exit the loop when we've seen all pages.
-		if resp.CurrentPage >= resp.TotalPages {
+		if resp.NextPage == 0 {
 			break
 		}
 
@@ -116,7 +191,8 @@ func (_ Gitlab) NewClient() (Client, error) {
 }
 
 func (_ Gitlab) addTokenToHTTPSCloneURL(url string, token string) string {
-	splitURL := strings.Split(url, "https://")
+	// allows for http and https for local testing
+	splitURL := strings.Split(url, "://")
 	return "https://oauth2:" + token + "@" + splitURL[1]
 }