diff --git a/CHANGELOG.md b/CHANGELOG.md index 883c4af..76ca4b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) ## [1.9.3] - unreleased ### Added +- Better examples for GitLab +- Better tests for local gitlab enterprise ### Changed ### Deprecated ### Removed ### Fixed +- gitlab hash concurrency issues +- all-users command directory nesting +- ls command to work with output dirs ### Security - Bump github.com/ktrysmt/go-bitbucket from 0.9.54 to 0.9.55 - Bump github.com/xanzy/go-gitlab from 0.76.0 to 0.77.0 diff --git a/cmd/clone.go b/cmd/clone.go index 684fac7..c8ffe19 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -12,6 +12,7 @@ import ( "regexp" "strconv" "strings" + "sync" "github.com/gabrie30/ghorg/colorlog" "github.com/gabrie30/ghorg/configs" @@ -561,6 +562,9 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { var cloneCount, pulledCount, updateRemoteCount int + // maps in go are not safe for concurrent use + var mutex = &sync.RWMutex{} + for i := range cloneTargets { repo := cloneTargets[i] repoSlug := getAppNameFromURL(repo.URL) @@ -568,16 +572,24 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { if repo.Path != "" && os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" { repoSlug = repo.Path } + mutex.Lock() + inHash := repoNameWithCollisions[repo.Name] + mutex.Unlock() // Only GitLab repos can have collisions due to groups and subgroups // If there are collisions and this is a repo with a naming collision change name to avoid collisions - if hasCollisions && repoNameWithCollisions[repo.Name] { + if hasCollisions && inHash { repoSlug = trimCollisionFilename(strings.Replace(repo.Path, "/", "_", -1)) + mutex.Lock() + slugCollision := repoNameWithCollisions[repoSlug] + mutex.Unlock() // If a collision has another collision with trimmed name append a number - if _, ok := repoNameWithCollisions[repoSlug]; ok { + if ok := slugCollision; ok { repoSlug = fmt.Sprintf("_%v_%v", strconv.Itoa(i), repoSlug) } else { + mutex.Lock() repoNameWithCollisions[repoSlug] = true + mutex.Unlock() } } diff --git a/cmd/ls.go b/cmd/ls.go index 521d416..554c5c2 100644 --- a/cmd/ls.go +++ b/cmd/ls.go @@ -46,15 +46,22 @@ func listGhorgHome() { func listGhorgDir(arg string) { - arg = strings.ReplaceAll(arg, "-", "_") - path := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + arg files, err := ioutil.ReadDir(path) if err != nil { - colorlog.PrintError("No clone found with that name. Please check spelling or reclone.") + // ghorg natively uses underscores in folder names, but a user can specify an output dir with underscores + // so first try what the user types if not then try replace + arg = strings.ReplaceAll(arg, "-", "_") + path = os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + arg } + files, err = ioutil.ReadDir(path) + if err != nil { + colorlog.PrintError("No clones found. Please clone some and try again.") + } + + for _, f := range files { if f.IsDir() { str := filepath.Join(path, f.Name()) diff --git a/examples/gitlab.md b/examples/gitlab.md index 7ec31cf..06fc66b 100644 --- a/examples/gitlab.md +++ b/examples/gitlab.md @@ -10,7 +10,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri 1. The `--preserve-dir` flag will mirror the nested directory structure of the groups/subgroups/projects locally to what is on GitLab. This prevents any name collisions with project names. If this flag is ommited all projects will be cloned into a single directory. If there are collisions with project names and `--preserve-dir` is not used the group/subgroup name will be prepended to those projects. An informational message will also be displayed during the clone to let you know if this happens. -1. For all versions of GitLab you can clone groups or sub groups individually +1. For all versions of GitLab you can clone groups or subgroups individually although the behavior is slightly different on hosted vs cloud GitLab ## Hosted GitLab Instances @@ -20,35 +20,93 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri > Note: You must set `--base-url` which is the url to your instance. If your instance requires an insecure connection you can use the `--insecure-gitlab-client` flag -1. Clone all groups **preserving the directory structure** of subgroups +1. Clone **all groups**, **preserving the directory structure** of subgroups ``` ghorg clone all-groups --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir ``` -1. Clone all groups on an **insecure** instance **preserving the directory structure** of subgroups + This would produce a directory structure like ``` - ghorg clone all-groups --base-url=http:// --scm=gitlab --token=XXXXXX --preserve-dir --insecure-gitlab-client + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab + ├── group1 + │   └── project1 + ├── group2 + │   └── project2 + └── group3 + └── subgroup1 + ├── project3 + └── project4 + ``` + +1. Clone **all groups**, **WITHOUT preserving the directory structure** of subgroups + + ``` + ghorg clone all-groups --base-url=https:// --scm=gitlab --token=XXXXXX + ``` + + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab + ├── project1 + ├── project2 + ├── project3 + └── project4 + ``` + #### Cloning Specific Groups -1. Clone a single group, **preserving the directory structure** of any subgroups within that group +1. Clone **a specific group**, **preserving the directory structure** of subgroups ``` - ghorg clone --base-url=https:// --scm=gitlab --preserve-dir + ghorg clone group3 --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir ``` -1. Clone only a **subgroup** + This would produce a directory structure like ``` - ghorg clone / --base-url=https:// --scm=gitlab + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── group3 + └── subgroup1 + ├── project3 + └── project4 ``` -1. clone all repos that are **prefixed** with "frontend" **into a folder** called "design_only" +1. Clone **a specific group**, **WITHOUT preserving the directory structure** of subgroups ``` - ghorg clone --base-url=https:// --scm=gitlab --match-regex=^frontend --output-dir=design_only + ghorg clone group3 --base-url=https:// --scm=gitlab --token=XXXXXX ``` + + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── group3 + ├── project3 + └── project4 + ``` + +1. Clone **a specific subgroup** + + ``` + ghorg clone group3/subgroup1 --base-url=https:// --scm=gitlab --token=XXXXXX + ``` + + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── group3 + └── subgroup1 + ├── project3 + └── project4 + ``` + #### Cloning a Specific Users Repos 1. Clone a **user** on a **hosted gitlab** instance using a **token** for auth @@ -57,22 +115,56 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ghorg clone --clone-type=user --base-url=https:// --scm=gitlab --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2 ``` + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── gitlab_username + ├── project3 + └── project4 + ``` + #### Cloning All Users Repos > Note: "all-users" only works on hosted GitLab instances running 13.0.1 or greater > Note: You must set `--base-url` which is the url to your instance. If your instance requires an insecure connection you can use the `--insecure-gitlab-client` flag -1. Clone all users repos **into a directory called all-users-repos** +1. Clone **all users**, **preserving the directory structure** of users ``` - ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX --clone-type=user --output-dir=all-users-repos + ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir ``` -1. Clone all users repos on an **insecure** instance + This would produce a directory structure like ``` - ghorg clone all-users --base-url=http:// --scm=gitlab --token=XXXXXX --clone-type=user --insecure-gitlab-client + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab_users + ├── user1 + │   └── project1 + ├── user2 + │   └── project2 + └── user3 + ├── project3 + └── project4 + ``` +1. Clone **all users**, **WITHOUT preserving the directory structure** of users + + ``` + ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX + ``` + + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab_users + ├── project1 + ├── project2 + ├── project3 + └── project4 + ``` ## Cloud GitLab Orgs @@ -84,8 +176,68 @@ Examples below use the `gitlab-examples` GitLab cloud organization https://gitla ghorg clone gitlab-examples --scm=gitlab --token=XXXXXX --preserve-dir ``` -1. clone only a **subgroup** + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── gitlab-examples + ├── aws-sam + ├── ci-debug-trace + ├── clojure-web-application + ├── cpp-example + ├── cross-branch-pipelines + ├── docker + ├── docker-cloud + ├── functions + └── ... + ``` + +1. clone only a **subgroup**, **preserving the directory structure** of subgroups + + ``` + ghorg clone gitlab-examples/wayne-enterprises --scm=gitlab --token=XXXXXX --preserve-dir + ``` + + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── gitlab-examples + └── wayne-enterprises + ├── wayne-aerospace + │   └── mission-control + ├── wayne-financial + │   ├── corporate-website + │   ├── customer-upload-tool + │   ├── customer-web-portal + │   ├── customer-web-portal-security-policy-project + │   ├── datagenerator + │   ├── mobile-app + │   └── wayne-financial-security-policy-project + └── wayne-industries + ├── backend-controller + └── microservice + ``` + +1. clone only a **subgroup**, **WITHOUT preserving the directory structure** of subgroups ``` ghorg clone gitlab-examples/wayne-enterprises --scm=gitlab --token=XXXXXX ``` + + This would produce a directory structure like + + ``` + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── wayne-enterprises + ├── backend-controller + ├── corporate-website + ├── customer-upload-tool + ├── customer-web-portal + ├── customer-web-portal-security-policy-project + ├── datagenerator + ├── microservice + ├── mission-control + ├── mobile-app + └── wayne-financial-security-policy-project + ``` diff --git a/scm/gitlab.go b/scm/gitlab.go index 08c8262..fccc6bb 100644 --- a/scm/gitlab.go +++ b/scm/gitlab.go @@ -16,6 +16,7 @@ var ( _ Client = Gitlab{} perPage = 100 gitLabAllGroups = false + gitLabAllUsers = false ) func init() { @@ -176,6 +177,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) { } if targetUsername == "all-users" { + gitLabAllUsers = true for { allUsers, resp, err := c.Users.ListUsers(userOpts) if err != nil { @@ -302,7 +304,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo { // The PathWithNamespace includes the org/group name // https://github.com/gabrie30/ghorg/issues/228 // https://github.com/gabrie30/ghorg/issues/267 - if !gitLabAllGroups { + if !gitLabAllGroups && !gitLabAllUsers { path = strings.TrimPrefix(path, group) } diff --git a/scripts/local-gitlab/clone.sh b/scripts/local-gitlab/clone.sh index ef40f44..384f1ff 100755 --- a/scripts/local-gitlab/clone.sh +++ b/scripts/local-gitlab/clone.sh @@ -2,18 +2,95 @@ set -ex -TOKEN=$1 -GITLAB_URL=$2 +TOKEN=${1:-'password'} +GITLAB_URL=${2:-'http://gitlab.example.com'} export GHORG_INSECURE_GITLAB_CLIENT=true +############ ############ +############ CLONE AND TEST ALL-GROUPS PRESERVING DIRECTORY STRUCTURE ############ +############ ############ + # run twice, once for clone then pull ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos +GOT=$( ghorg ls local-gitlab-v15-repos/group1 | grep -o 'local-gitlab-v15-repos/group1.*') +WANT=$(cat < auth.txt - -BEARER_TOKEN_JSON=$(curl -s --data "@auth.txt" --request POST "${GITLAB_URL}/oauth/token") - -echo "${BEARER_TOKEN_JSON}" - -BEARER_TOKEN=$(echo "${BEARER_TOKEN_JSON}" | jq -r '.access_token' | tr -d '\n') - -rm auth.txt - -TOKEN_NUMS=$(echo "${RANDOM}") - -API_TOKEN=$(curl -s --request POST --header "Authorization: Bearer ${BEARER_TOKEN}" --data "name=admintoken-${TOKEN_NUMS}" --data "expires_at=2050-04-04" --data "scopes[]=api" "${GITLAB_URL}/api/v4/users/1/personal_access_tokens" | jq -r '.token' | tr -d '\n') +API_TOKEN="password" # seed new instance using ./scripts/local-gitlab/seed.sh "${API_TOKEN}" "${GITLAB_URL}" diff --git a/scripts/local-gitlab/seed.sh b/scripts/local-gitlab/seed.sh index 4846b24..84e2bdb 100755 --- a/scripts/local-gitlab/seed.sh +++ b/scripts/local-gitlab/seed.sh @@ -5,35 +5,57 @@ TOKEN=$1 GITLAB_URL=$2 -# Create 2 groups, namespace_id will start at 4 +# Create 3 groups, namespace_id will start at 4 (same thing as Group ID you can find in the UI) curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \ --header "Content-Type: application/json" \ --data '{"path": "group1", "name": "group1" }' \ "${GITLAB_URL}/api/v4/groups" +sleep 5 + curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \ --header "Content-Type: application/json" \ --data '{"path": "group2", "name": "group2" }' \ "${GITLAB_URL}/api/v4/groups" +sleep 5 + +curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \ + --header "Content-Type: application/json" \ + --data '{"path": "group3", "name": "group3" }' \ + "${GITLAB_URL}/api/v4/groups" + +sleep 5 + +curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \ + --header "Content-Type: application/json" \ + --data '{"path": "subgroup-a", "name": "subgroup-a" }' \ + "${GITLAB_URL}/api/v4/groups?parent_id=6" + +sleep 5 + # Create 2 users curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \ --header "Content-Type: application/json" \ - --data '{"email": "testuser1@example.com", "password": "adminadmin1","name": "testuser1","reset_password": "true" }' + --data '{"email": "testuser1@example.com", "password": "adminadmin1","name": "testuser1","username": "testuser1",reset_password": "true" }' \ + "${GITLAB_URL}/api/v4/users" + +sleep 5 curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \ --header "Content-Type: application/json" \ - --data '{"email": "testuser2@example.com", "password": "adminadmin1","name": "testuser2","reset_password": "true" }' + --data '{"email": "testuser2@example.com", "password": "adminadmin1","name": "testuser2","username": "testuser2","reset_password": "true" }' \ + "${GITLAB_URL}/api/v4/users" -sleep 1 +sleep 5 -# create repos for user +# create repos for root user for ((a=0; a <= 3 ; a++)) do - curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=baz${a}&initialize_with_readme=true" + curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=rootrepos${a}&initialize_with_readme=true" done -sleep 1 +sleep 5 # create repos in group1 for ((a=0; a <= 3 ; a++)) @@ -41,7 +63,7 @@ do curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=baz${a}&namespace_id=4&initialize_with_readme=true" done -sleep 1 +sleep 5 # create repos in group2 for ((a=0; a <= 3 ; a++)) @@ -49,4 +71,14 @@ do curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=baz${a}&namespace_id=5&initialize_with_readme=true" done +sleep 5 + +# create repos in group3/subgroup-a +for ((a=0; a <= 3 ; a++)) +do + curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=subgroup_repo_${a}&namespace_id=7&initialize_with_readme=true" +done + +sleep 5 + ./scripts/local-gitlab/clone.sh "${TOKEN}" "${GITLAB_URL}" diff --git a/scripts/local-gitlab/start-ee.sh b/scripts/local-gitlab/start-ee.sh index ed5c8ce..6df5312 100755 --- a/scripts/local-gitlab/start-ee.sh +++ b/scripts/local-gitlab/start-ee.sh @@ -15,6 +15,8 @@ fi docker rm gitlab --force --volumes +rm -rf $HOME/Desktop/ghorg/local-gitlab-v15-* + echo "" echo "To follow gitlab container logs use the following command in a new window" echo "$ docker logs -f gitlab"