github-personal/ghorg
1
0
Fork 0
mirror of https://github.com/outbackdingo/ghorg.git synced 2026-01-27 10:19:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update gitlab docs and local instanace (#273)

Browse Source
This commit is contained in:
gabrie30
2023-01-06 17:34:12 -08:00
committed by GitHub
parent 5a9fbe67eb
commit c3f7aec539
9 changed files with 363 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@@ -5,10 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
## [1.9.3] - unreleased
### Added
- Better examples for GitLab
- Better tests for local gitlab enterprise
### Changed
### Deprecated
### Removed
### Fixed
- gitlab hash concurrency issues
- all-users command directory nesting
- ls command to work with output dirs
### Security
- Bump github.com/ktrysmt/go-bitbucket from 0.9.54 to 0.9.55
- Bump github.com/xanzy/go-gitlab from 0.76.0 to 0.77.0

16
cmd/clone.go
View File

@@ -12,6 +12,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"github.com/gabrie30/ghorg/colorlog"
"github.com/gabrie30/ghorg/configs"
@@ -561,6 +562,9 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
var cloneCount, pulledCount, updateRemoteCount int
// maps in go are not safe for concurrent use
var mutex = &sync.RWMutex{}
for i := range cloneTargets {
repo := cloneTargets[i]
repoSlug := getAppNameFromURL(repo.URL)
@@ -568,16 +572,24 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
if repo.Path != "" && os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" {
repoSlug = repo.Path
}
mutex.Lock()
inHash := repoNameWithCollisions[repo.Name]
mutex.Unlock()
// Only GitLab repos can have collisions due to groups and subgroups
// If there are collisions and this is a repo with a naming collision change name to avoid collisions
if hasCollisions && repoNameWithCollisions[repo.Name] {
if hasCollisions && inHash {
repoSlug = trimCollisionFilename(strings.Replace(repo.Path, "/", "_", -1))
mutex.Lock()
slugCollision := repoNameWithCollisions[repoSlug]
mutex.Unlock()
// If a collision has another collision with trimmed name append a number
if _, ok := repoNameWithCollisions[repoSlug]; ok {
if ok := slugCollision; ok {
repoSlug = fmt.Sprintf("_%v_%v", strconv.Itoa(i), repoSlug)
} else {
mutex.Lock()
repoNameWithCollisions[repoSlug] = true
mutex.Unlock()
}
}

13
cmd/ls.go
View File

@@ -46,15 +46,22 @@ func listGhorgHome() {
func listGhorgDir(arg string) {
arg = strings.ReplaceAll(arg, "-", "_")
path := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + arg
files, err := ioutil.ReadDir(path)
if err != nil {
colorlog.PrintError("No clone found with that name. Please check spelling or reclone.")
// ghorg natively uses underscores in folder names, but a user can specify an output dir with underscores
// so first try what the user types if not then try replace
arg = strings.ReplaceAll(arg, "-", "_")
path = os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + arg
}
files, err = ioutil.ReadDir(path)
if err != nil {
colorlog.PrintError("No clones found. Please clone some and try again.")
}
for _, f := range files {
if f.IsDir() {
str := filepath.Join(path, f.Name())

182
examples/gitlab.md
View File

@@ -10,7 +10,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
1. The `--preserve-dir` flag will mirror the nested directory structure of the groups/subgroups/projects locally to what is on GitLab. This prevents any name collisions with project names. If this flag is ommited all projects will be cloned into a single directory. If there are collisions with project names and `--preserve-dir` is not used the group/subgroup name will be prepended to those projects. An informational message will also be displayed during the clone to let you know if this happens.
1. For all versions of GitLab you can clone groups or sub groups individually
1. For all versions of GitLab you can clone groups or subgroups individually although the behavior is slightly different on hosted vs cloud GitLab
## Hosted GitLab Instances
@@ -20,35 +20,93 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
> Note: You must set `--base-url` which is the url to your instance. If your instance requires an insecure connection you can use the `--insecure-gitlab-client` flag
1. Clone all groups **preserving the directory structure** of subgroups
1. Clone **all groups**, **preserving the directory structure** of subgroups
```
ghorg clone all-groups --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --preserve-dir
```
1. Clone all groups on an **insecure** instance **preserving the directory structure** of subgroups
This would produce a directory structure like
```
ghorg clone all-groups --base-url=http://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --preserve-dir --insecure-gitlab-client
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── your.instance.gitlab
├── group1
│   └── project1
├── group2
│   └── project2
└── group3
└── subgroup1
├── project3
└── project4
```
1. Clone **all groups**, **WITHOUT preserving the directory structure** of subgroups
```
ghorg clone all-groups --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── your.instance.gitlab
├── project1
├── project2
├── project3
└── project4
```
#### Cloning Specific Groups
1. Clone a single group, **preserving the directory structure** of any subgroups within that group
1. Clone **a specific group**, **preserving the directory structure** of subgroups
```
ghorg clone <gitlab_group> --base-url=https://<your.instance.gitlab.com> --scm=gitlab --preserve-dir
ghorg clone group3 --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --preserve-dir
```
1. Clone only a **subgroup**
This would produce a directory structure like
```
ghorg clone <gitlab_group>/<gitlab_sub_group> --base-url=https://<your.instance.gitlab.com> --scm=gitlab
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── group3
└── subgroup1
├── project3
└── project4
```
1. clone all repos that are **prefixed** with "frontend" **into a folder** called "design_only"
1. Clone **a specific group**, **WITHOUT preserving the directory structure** of subgroups
```
ghorg clone <gitlab_group> --base-url=https://<your.instance.gitlab.com> --scm=gitlab --match-regex=^frontend --output-dir=design_only
ghorg clone group3 --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── group3
├── project3
└── project4
```
1. Clone **a specific subgroup**
```
ghorg clone group3/subgroup1 --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── group3
└── subgroup1
├── project3
└── project4
```
#### Cloning a Specific Users Repos
1. Clone a **user** on a **hosted gitlab** instance using a **token** for auth
@@ -57,22 +115,56 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
ghorg clone <gitlab_username> --clone-type=user --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── gitlab_username
├── project3
└── project4
```
#### Cloning All Users Repos
> Note: "all-users" only works on hosted GitLab instances running 13.0.1 or greater
> Note: You must set `--base-url` which is the url to your instance. If your instance requires an insecure connection you can use the `--insecure-gitlab-client` flag
1. Clone all users repos **into a directory called all-users-repos**
1. Clone **all users**, **preserving the directory structure** of users
```
ghorg clone all-users --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --clone-type=user --output-dir=all-users-repos
ghorg clone all-users --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --preserve-dir
```
1. Clone all users repos on an **insecure** instance
This would produce a directory structure like
```
ghorg clone all-users --base-url=http://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --clone-type=user --insecure-gitlab-client
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── your.instance.gitlab_users
├── user1
│   └── project1
├── user2
│   └── project2
└── user3
├── project3
└── project4
```
1. Clone **all users**, **WITHOUT preserving the directory structure** of users
```
ghorg clone all-users --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── your.instance.gitlab_users
├── project1
├── project2
├── project3
└── project4
```
## Cloud GitLab Orgs
@@ -84,8 +176,68 @@ Examples below use the `gitlab-examples` GitLab cloud organization https://gitla
ghorg clone gitlab-examples --scm=gitlab --token=XXXXXX --preserve-dir
```
1. clone only a **subgroup**
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── gitlab-examples
├── aws-sam
├── ci-debug-trace
├── clojure-web-application
├── cpp-example
├── cross-branch-pipelines
├── docker
├── docker-cloud
├── functions
└── ...
```
1. clone only a **subgroup**, **preserving the directory structure** of subgroups
```
ghorg clone gitlab-examples/wayne-enterprises --scm=gitlab --token=XXXXXX --preserve-dir
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── gitlab-examples
└── wayne-enterprises
├── wayne-aerospace
│   └── mission-control
├── wayne-financial
│   ├── corporate-website
│   ├── customer-upload-tool
│   ├── customer-web-portal
│   ├── customer-web-portal-security-policy-project
│   ├── datagenerator
│   ├── mobile-app
│   └── wayne-financial-security-policy-project
└── wayne-industries
├── backend-controller
└── microservice
```
1. clone only a **subgroup**, **WITHOUT preserving the directory structure** of subgroups
```
ghorg clone gitlab-examples/wayne-enterprises --scm=gitlab --token=XXXXXX
```
This would produce a directory structure like
```
/GHORG_ABSOLUTE_PATH_TO_CLONE_TO
└── wayne-enterprises
├── backend-controller
├── corporate-website
├── customer-upload-tool
├── customer-web-portal
├── customer-web-portal-security-policy-project
├── datagenerator
├── microservice
├── mission-control
├── mobile-app
└── wayne-financial-security-policy-project
```

4
scm/gitlab.go
View File

@@ -16,6 +16,7 @@ var (
_ Client = Gitlab{}
perPage = 100
gitLabAllGroups = false
gitLabAllUsers = false
)
func init() {
@@ -176,6 +177,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
}
if targetUsername == "all-users" {
gitLabAllUsers = true
for {
allUsers, resp, err := c.Users.ListUsers(userOpts)
if err != nil {
@@ -302,7 +304,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {
// The PathWithNamespace includes the org/group name
// https://github.com/gabrie30/ghorg/issues/228
// https://github.com/gabrie30/ghorg/issues/267
if !gitLabAllGroups {
if !gitLabAllGroups && !gitLabAllUsers {
path = strings.TrimPrefix(path, group)
}

122
scripts/local-gitlab/clone.sh
View File

@@ -2,18 +2,95 @@
set -ex
TOKEN=$1
GITLAB_URL=$2
TOKEN=${1:-'password'}
GITLAB_URL=${2:-'http://gitlab.example.com'}
export GHORG_INSECURE_GITLAB_CLIENT=true
############ ############
############ CLONE AND TEST ALL-GROUPS PRESERVING DIRECTORY STRUCTURE ############
############ ############
# run twice, once for clone then pull
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos
GOT=$( ghorg ls local-gitlab-v15-repos/group1 | grep -o 'local-gitlab-v15-repos/group1.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos/group1/baz0
local-gitlab-v15-repos/group1/baz1
local-gitlab-v15-repos/group1/baz2
local-gitlab-v15-repos/group1/baz3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "ALL_GROUPS GROUP1 PRESERVE DIR TEST FAILED"
exit 1
fi
GOT=$( ghorg ls local-gitlab-v15-repos/group2 | grep -o 'local-gitlab-v15-repos/group2.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos/group2/baz0
local-gitlab-v15-repos/group2/baz1
local-gitlab-v15-repos/group2/baz2
local-gitlab-v15-repos/group2/baz3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "ALL_GROUPS GROUP2 PRESERVE DIR TEST FAILED"
exit 1
fi
GOT=$( ghorg ls local-gitlab-v15-repos/group3/subgroup-a | grep -o 'local-gitlab-v15-repos/group3/subgroup-a.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos/group3/subgroup-a/subgroup_repo_0
local-gitlab-v15-repos/group3/subgroup-a/subgroup_repo_1
local-gitlab-v15-repos/group3/subgroup-a/subgroup_repo_2
local-gitlab-v15-repos/group3/subgroup-a/subgroup_repo_3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "ALL_GROUPS GROUP3/SUBGROUP-A PRESERVE DIR TEST FAILED"
exit 1
fi
############ ############
############ CLONE AND TEST ALL-GROUPS ############
############ ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-flat
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-flat
GOT=$( ghorg ls local-gitlab-v15-repos-flat | grep -o 'local-gitlab-v15-repos-flat.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-flat/Monitoring
local-gitlab-v15-repos-flat/group1_baz0
local-gitlab-v15-repos-flat/group1_baz1
local-gitlab-v15-repos-flat/group1_baz2
local-gitlab-v15-repos-flat/group1_baz3
local-gitlab-v15-repos-flat/group2_baz0
local-gitlab-v15-repos-flat/group2_baz1
local-gitlab-v15-repos-flat/group2_baz2
local-gitlab-v15-repos-flat/group2_baz3
local-gitlab-v15-repos-flat/subgroup_repo_0
local-gitlab-v15-repos-flat/subgroup_repo_1
local-gitlab-v15-repos-flat/subgroup_repo_2
local-gitlab-v15-repos-flat/subgroup_repo_3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "ALL_GROUPS FLAT TEST FAILED"
exit 1
fi
ghorg clone root --clone-type=user --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-root-user-repos --prune --prune-no-confirm
ghorg clone root --clone-type=user --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-root-user-repos --prune --prune-no-confirm
@@ -26,4 +103,45 @@ ghorg clone group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --
ghorg clone group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-group1
ghorg clone group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-group1
############ ############
############ CLONE AND TEST ALL-USERS PRESERVING DIRECTORY STRUCTURE ############
############ ############
ghorg clone all-users --scm=gitlab --clone-type=user --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-all-users-preserve --preserve-dir
GOT=$(ghorg ls local-gitlab-v15-all-users-preserve/root | grep -o 'local-gitlab-v15-all-users-preserve/root.*')
WANT=$(cat <<EOF
local-gitlab-v15-all-users-preserve/root/rootrepos0
local-gitlab-v15-all-users-preserve/root/rootrepos1
local-gitlab-v15-all-users-preserve/root/rootrepos2
local-gitlab-v15-all-users-preserve/root/rootrepos3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "ALL_USERS PRESERVE DIR TEST FAILED"
exit 1
fi
############ ############
############ CLONE AND TEST ALL-USERS ############
############ ############
ghorg clone all-users --scm=gitlab --clone-type=user --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-all-users
TEST_ALL_USERS_GOT=$(ghorg ls local-gitlab-v15-all-users | grep -o 'local-gitlab-v15-all-users.*')
TEST_ALL_USERS_WANT=$(cat <<EOF
local-gitlab-v15-all-users/rootrepos0
local-gitlab-v15-all-users/rootrepos1
local-gitlab-v15-all-users/rootrepos2
local-gitlab-v15-all-users/rootrepos3
EOF
)
if [ "${TEST_ALL_USERS_WANT}" != "${TEST_ALL_USERS_GOT}" ]
then
echo "ALL_USERS TEST FAILED"
exit 1
fi

17
scripts/local-gitlab/get_credentials.sh
View File

@@ -28,22 +28,9 @@ done
set -x
# Once running get pw with
pw=$(docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password | awk '{print $2}')
docker exec -it gitlab gitlab-rails runner "token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api, :read_api, :sudo], name: 'CI Test Token'); token.set_token('password'); token.save!"
echo "grant_type=password&username=root&password=${pw}" > auth.txt
BEARER_TOKEN_JSON=$(curl -s --data "@auth.txt" --request POST "${GITLAB_URL}/oauth/token")
echo "${BEARER_TOKEN_JSON}"
BEARER_TOKEN=$(echo "${BEARER_TOKEN_JSON}" | jq -r '.access_token' | tr -d '\n')
rm auth.txt
TOKEN_NUMS=$(echo "${RANDOM}")
API_TOKEN=$(curl -s --request POST --header "Authorization: Bearer ${BEARER_TOKEN}" --data "name=admintoken-${TOKEN_NUMS}" --data "expires_at=2050-04-04" --data "scopes[]=api" "${GITLAB_URL}/api/v4/users/1/personal_access_tokens" | jq -r '.token' | tr -d '\n')
API_TOKEN="password"
# seed new instance using
./scripts/local-gitlab/seed.sh "${API_TOKEN}" "${GITLAB_URL}"

48
scripts/local-gitlab/seed.sh
View File

@@ -5,35 +5,57 @@
TOKEN=$1
GITLAB_URL=$2
# Create 2 groups, namespace_id will start at 4
# Create 3 groups, namespace_id will start at 4 (same thing as Group ID you can find in the UI)
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data '{"path": "group1", "name": "group1" }' \
"${GITLAB_URL}/api/v4/groups"
sleep 5
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data '{"path": "group2", "name": "group2" }' \
"${GITLAB_URL}/api/v4/groups"
sleep 5
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data '{"path": "group3", "name": "group3" }' \
"${GITLAB_URL}/api/v4/groups"
sleep 5
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data '{"path": "subgroup-a", "name": "subgroup-a" }' \
"${GITLAB_URL}/api/v4/groups?parent_id=6"
sleep 5
# Create 2 users
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data '{"email": "testuser1@example.com", "password": "adminadmin1","name": "testuser1","reset_password": "true" }'
--data '{"email": "testuser1@example.com", "password": "adminadmin1","name": "testuser1","username": "testuser1",reset_password": "true" }' \
"${GITLAB_URL}/api/v4/users"
sleep 5
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data '{"email": "testuser2@example.com", "password": "adminadmin1","name": "testuser2","reset_password": "true" }'
--data '{"email": "testuser2@example.com", "password": "adminadmin1","name": "testuser2","username": "testuser2","reset_password": "true" }' \
"${GITLAB_URL}/api/v4/users"
sleep 1
sleep 5
# create repos for user
# create repos for root user
for ((a=0; a <= 3 ; a++))
do
curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=baz${a}&initialize_with_readme=true"
curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=rootrepos${a}&initialize_with_readme=true"
done
sleep 1
sleep 5
# create repos in group1
for ((a=0; a <= 3 ; a++))
@@ -41,7 +63,7 @@ do
curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=baz${a}&namespace_id=4&initialize_with_readme=true"
done
sleep 1
sleep 5
# create repos in group2
for ((a=0; a <= 3 ; a++))
@@ -49,4 +71,14 @@ do
curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=baz${a}&namespace_id=5&initialize_with_readme=true"
done
sleep 5
# create repos in group3/subgroup-a
for ((a=0; a <= 3 ; a++))
do
curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects?name=subgroup_repo_${a}&namespace_id=7&initialize_with_readme=true"
done
sleep 5
./scripts/local-gitlab/clone.sh "${TOKEN}" "${GITLAB_URL}"

2
scripts/local-gitlab/start-ee.sh
View File

@@ -15,6 +15,8 @@ fi
docker rm gitlab --force --volumes
rm -rf $HOME/Desktop/ghorg/local-gitlab-v15-*
echo ""
echo "To follow gitlab container logs use the following command in a new window"
echo "$ docker logs -f gitlab"