Adds all-users clone for hosted gitlab instances (#266)

CHANGELOG.md

@@ -2,6 +2,17 @@
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+## [1.9.1] - unreleased
+### Added
+- Ability to clone all users repos on hosted GitLab instances; thanks @mlaily
+### Changed
+### Deprecated
+### Removed
+### Fixed
+- Top level GitLab groups on hosted GitLab instances now fetch all groups; thanks @mlaily
+### Security
+- Bump github.com/xanzy/go-gitlab from 0.74.0 to 0.76.0
+- Bump github.com/spf13/viper from 1.13.0 to 1.14.0
 
 ## [1.9.0] - 11/5/22
 ### Added

cmd/clone.go

@@ -985,6 +985,16 @@ func setOutputDirName(argz []string) {
 		outputDirName = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com")
 	}
 
+	// If all-group is used set the parent folder to the name of the baseurl
+	if argz[0] == "all-users" && os.Getenv("GHORG_SCM_BASE_URL") != "" {
+		u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
+		if err != nil {
+			return
+		}
+		outputDirName = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com")
+		outputDirName = outputDirName + "_users"
+	}
+
 	if os.Getenv("GHORG_BACKUP") == "true" {
 		outputDirName = outputDirName + "_backup"
 	}

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const ghorgVersion = "v1.9.0"
+const ghorgVersion = "v1.9.1"
 
 var versionCmd = &cobra.Command{
 	Use:   "version",

examples/gitlab.md

@@ -49,7 +49,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
     ```
     ghorg clone <gitlab_group> --base-url=https://<your.instance.gitlab.com> --scm=gitlab --match-regex=^frontend --output-dir=design_only
     ```
-#### Cloning User Repos
+#### Cloning a Specific Users Repos
 
 1. Clone a **user** on a **hosted gitlab** instance using a **token** for auth
 
@@ -57,6 +57,23 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
     ghorg clone <gitlab_username> --clone-type=user --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2
     ```
 
+#### Cloning All Users Repos
+
+> Note: "all-users" only works on hosted GitLab instances running 13.0.1 or greater
+
+> Note: You must set `--base-url` which is the url to your instance. If your instance requires an insecure connection you can use the `--insecure-gitlab-client` flag
+
+1. Clone all users repos **into a directory called all-users-repos**
+
+    ```
+    ghorg clone all-users --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --clone-type=user --output-dir=all-users-repos
+    ```
+
+1. Clone all users repos on an **insecure** instance
+
+    ```
+    ghorg clone all-users --base-url=http://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --clone-type=user --insecure-gitlab-client
+
 ## Cloud GitLab Orgs
 
 Examples below use the `gitlab-examples` GitLab cloud organization https://gitlab.com/gitlab-examples

scm/gitlab.go

@@ -153,34 +153,65 @@ func (c Gitlab) GetGroupRepos(targetGroup string) ([]Repo, error) {
 // GetUserRepos gets all of a users gitlab repos
 func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
 	cloneData := []Repo{}
+	targetUsers := []string{}
 
-	opt := &gitlab.ListProjectsOptions{
+	projectOpts := &gitlab.ListProjectsOptions{
 		ListOptions: gitlab.ListOptions{
 			PerPage: perPage,
 			Page:    1,
 		},
 	}
 
-	for {
-		// Get the first page with projects.
-		ps, resp, err := c.Projects.ListUserProjects(targetUsername, opt)
-		if err != nil {
-			if resp != nil && resp.StatusCode == 404 {
-				return nil, fmt.Errorf("user '%s' does not exist", targetUsername)
+	userOpts := &gitlab.ListUsersOptions{
+		ListOptions: gitlab.ListOptions{
+			PerPage: perPage,
+			Page:    1,
+		},
+	}
+
+	if targetUsername == "all-users" {
+		for {
+			allUsers, resp, err := c.Users.ListUsers(userOpts)
+			if err != nil {
+				return nil, fmt.Errorf("error getting all users, err: %v", err)
 			}
-			return []Repo{}, err
+
+			for _, u := range allUsers {
+				targetUsers = append(targetUsers, u.Username)
+			}
+
+			if resp.NextPage == 0 {
+				break
+			}
+
+			// Update the page number to get the next page.
+			userOpts.Page = resp.NextPage
 		}
+	} else {
+		targetUsers = append(targetUsers, targetUsername)
+	}
 
-		// filter from all the projects we've found so far.
-		cloneData = append(cloneData, c.filter(targetUsername, ps)...)
+	for _, targetUser := range targetUsers {
+		for {
+			// Get the first page with projects.
+			ps, resp, err := c.Projects.ListUserProjects(targetUser, projectOpts)
+			if err != nil {
+				fmt.Printf("Error getting repo for user: %v", targetUser)
+				colorlog.PrintError(fmt.Sprintf("Error getting repo for user: %v", targetUser))
+				continue
+			}
 
-		// Exit the loop when we've seen all pages.
-		if resp.NextPage == 0 {
-			break
+			// filter from all the projects we've found so far.
+			cloneData = append(cloneData, c.filter(targetUser, ps)...)
+
+			// Exit the loop when we've seen all pages.
+			if resp.NextPage == 0 {
+				break
+			}
+
+			// Update the page number to get the next page.
+			userOpts.Page = resp.NextPage
 		}
-
-		// Update the page number to get the next page.
-		opt.Page = resp.NextPage
 	}
 
 	return cloneData, nil

scripts/local-gitlab/clone.sh

@@ -25,3 +25,5 @@ ghorg clone group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --
 
 ghorg clone group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-group1
 ghorg clone group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-group1
+
+ghorg clone all-users --scm=gitlab --clone-type=user --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-all-users

scripts/local-gitlab/seed.sh

@@ -16,6 +16,15 @@ curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
     --data '{"path": "group2", "name": "group2" }' \
     "${GITLAB_URL}/api/v4/groups"
 
+# Create 2 users
+curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
+    --header "Content-Type: application/json" \
+    --data '{"email": "testuser1@example.com", "password": "adminadmin1","name": "testuser1","reset_password": "true" }'
+
+curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
+    --header "Content-Type: application/json" \
+    --data '{"email": "testuser2@example.com", "password": "adminadmin1","name": "testuser2","reset_password": "true" }'
+
 sleep 1
 
 # create repos for user

scripts/local-gitlab/start-ee.sh

@@ -9,12 +9,6 @@ GITLAB_HOME=${4:-"$HOME/Desktop/ghorg/local-gitlab-ee-data-${GITLAB_IMAGE_TAG}"}
 GITLAB_HOST=${5:-'gitlab.example.com'}
 GITLAB_URL=${6:-'http://gitlab.example.com'}
 
-function cleanup()
-{
-    docker rm gitlab --force --volumes
-}
-trap cleanup EXIT
-
 if [ "${ENV}" == "ci" ];then
     echo "127.0.0.1 gitlab.example.com" >> /etc/hosts
 fi