diff --git a/CHANGELOG.md b/CHANGELOG.md index d749018..da8f56e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) -## [1.10.1] - unreleased +## [1.11.0] - unreleased ### Added - Reclone name and description to reclone output +- GHORG_PRESERVE_SCM_HOSTNAME, note that this feature changes the directory struture that gitlab all-users and all-groups clone into; thanks @rrrix ### Changed ### Deprecated ### Removed diff --git a/cmd/clone.go b/cmd/clone.go index 39d5768..df9a44b 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -163,6 +163,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Setenv("GHORG_GIT_FILTER", filter) } + if cmd.Flags().Changed("preserve-scm-hostname") { + os.Setenv("GHORG_PRESERVE_SCM_HOSTNAME", "true") + } + if cmd.Flags().Changed("skip-archived") { os.Setenv("GHORG_SKIP_ARCHIVED", "true") } @@ -282,6 +286,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Exit(1) } + if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" { + updateAbsolutePathToCloneToWithHostname() + } + setOutputDirName(argz) setOuputDirAbsolutePath() args = argz @@ -974,7 +982,14 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } func writeGhorgStats(date string, allReposToCloneCount, cloneCount, pulledCount, cloneInfosCount, cloneErrorsCount, updateRemoteCount, newCommits, pruneCount int, hasCollisions bool) error { - statsFilePath := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), "_ghorg_stats.csv") + var statsFilePath string + absolutePath := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" { + originalAbsolutePath := os.Getenv("GHORG_ORIGINAL_ABSOLUTE_PATH_TO_CLONE_TO") + statsFilePath = filepath.Join(originalAbsolutePath, "_ghorg_stats.csv") + } else { + statsFilePath = filepath.Join(absolutePath, "_ghorg_stats.csv") + } fileExists := true @@ -1398,22 +1413,23 @@ func setOutputDirName(argz []string) { outputDirName = strings.ToLower(argz[0]) - // If all-group is used set the parent folder to the name of the baseurl - if argz[0] == "all-groups" && os.Getenv("GHORG_SCM_BASE_URL") != "" { - u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) - if err != nil { - return + if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") != "true" { + // If all-group is used set the parent folder to the name of the baseurl + if argz[0] == "all-groups" && os.Getenv("GHORG_SCM_BASE_URL") != "" { + u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) + if err != nil { + colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL, clone may be affected, error: %v", err)) + } + outputDirName = u.Hostname() } - outputDirName = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com") - } - if argz[0] == "all-users" && os.Getenv("GHORG_SCM_BASE_URL") != "" { - u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) - if err != nil { - return + if argz[0] == "all-users" && os.Getenv("GHORG_SCM_BASE_URL") != "" { + u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL")) + if err != nil { + colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL, clone may be affected, error: %v", err)) + } + outputDirName = u.Hostname() } - outputDirName = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com") - outputDirName = outputDirName + "_users" } if os.Getenv("GHORG_BACKUP") == "true" { diff --git a/cmd/examples-copy/bitbucket.md b/cmd/examples-copy/bitbucket.md index 08f4592..be33bf1 100644 --- a/cmd/examples-copy/bitbucket.md +++ b/cmd/examples-copy/bitbucket.md @@ -4,6 +4,8 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help` +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For bitbucket cloud it will be `bitbucket.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL`. + ## Bitbucket Cloud 1. Clone the microsoft workspace using an app-password diff --git a/cmd/examples-copy/gitea.md b/cmd/examples-copy/gitea.md index e1c7b42..810e4d0 100644 --- a/cmd/examples-copy/gitea.md +++ b/cmd/examples-copy/gitea.md @@ -4,6 +4,12 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help` +## Things to know + +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the `GHORG_SCM_BASE_URL` you are cloning from. + +## Examples + 1. Clone an **org** ``` diff --git a/cmd/examples-copy/github.md b/cmd/examples-copy/github.md index 638974d..cabcf6f 100644 --- a/cmd/examples-copy/github.md +++ b/cmd/examples-copy/github.md @@ -4,6 +4,10 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help` +## Things to know + +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For github cloud it will be `github.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL` if set for enterpirse github orgs. + ## GitHub Cloud 1. Clone an **org**, using a token on the commandline @@ -11,6 +15,14 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ``` ghorg clone --token=XXXXXX ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── github_org + ├── repo1 + └── repo2 + ``` 1. Clone all repos from a **github org** that are **prefixed** with "frontend" **into a folder** called "design_only" @@ -24,6 +36,20 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ghorg clone --clone-type=user --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2 ``` +1. Clone an **org**, preserving the cloud scm hostname + ``` + ghorg clone --preserve-cloud-scm-hostname + ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── github.com + └── github_org + ├── repo1 + └── repo2 + ``` + ## GitHub Enterprise 1. Clone an **org** @@ -43,3 +69,31 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ``` ghorg clone --clone-type=user --base-url=https://.com --token=XXXXXX ``` + +1. Clone an **org**, preserving the cloud scm hostname. The github_org may seem redudant in this example but when its needed for the case when you start to set an output directory. + ``` + ghorg clone --base-url=https://.com --preserve-cloud-scm-hostname + ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your-hosted-github.com + └── github_org + ├── repo1 + └── repo2 + ``` + +1. Clone an **org**, preserving the cloud scm hostname **with an output directory** + ``` + ghorg clone --base-url=https://.com --output-dir=my-repos --preserve-cloud-scm-hostname + ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your-hosted-github.com + └── my-repos + ├── myrepo1 + └── myrepo2 + ``` diff --git a/cmd/examples-copy/gitlab.md b/cmd/examples-copy/gitlab.md index 8d42ab5..484977e 100644 --- a/cmd/examples-copy/gitlab.md +++ b/cmd/examples-copy/gitlab.md @@ -14,6 +14,8 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri 1. The `--output-dir` flag overrides the default name given to the folder ghorg creates to clone repos into. The default will be the instance name when cloning `all-groups` or `all-users` or the `group` name when cloning a specific group. The exception is when you are cloning a subgroup and preserving the directory structure, then it will preserve the parent groups of the subgroup. +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For gitlab cloud it will be `gitlab.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL`. + 1. If the group name you are cloning has spaces, substitute the spaces with "-" e.g. ```sh @@ -45,7 +47,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab + └── your.instance.gitlab.com ├── group1 │   └── project1 ├── group2 @@ -66,13 +68,34 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab + └── your.instance.gitlab.com ├── project1 ├── project2 ├── project3 └── project4 ``` +1. Clone **all groups**, **preserving the directory structure** of users, preserving scm hostname + + ```sh + ghorg clone all-groups --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname + ``` + + This would produce a directory structure like + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab.com + └── all-groups + ├── group1 + │   └── project1 + ├── group2 + │   └── project2 + └── group3 + ├── project3 + └── project4 + ``` + #### Cloning Specific Groups 1. Clone **a specific group**, **preserving the directory structure** of subgroups @@ -175,7 +198,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab_users + └── your.instance.gitlab.com ├── user1 │   └── project1 ├── user2 @@ -194,11 +217,30 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab_users - ├── project1 - ├── project2 - ├── project3 - └── project4 + └── your.instance.gitlab.com + ├── user1-repo1 + └── user2-repo1 + ``` + +1. Clone **all users**, **preserving the directory structure** of users, preserving scm hostname + + ```sh + ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname + ``` + + This would produce a directory structure like + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab.com + └── all-users + ├── user1 + │   └── project1 + ├── user2 + │   └── project2 + └── user3 + ├── project3 + └── project4 ``` ## Cloud GitLab Orgs diff --git a/cmd/root.go b/cmd/root.go index 3fe6c07..2590fff 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,6 +3,7 @@ package cmd import ( "errors" "fmt" + "net/url" "os" "path/filepath" @@ -68,6 +69,7 @@ var ( quietMode bool noDirSize bool ghorgStatsEnabled bool + ghorgPreserveScmHostname bool args []string cloneErrors []string cloneInfos []string @@ -83,6 +85,40 @@ var rootCmd = &cobra.Command{ }, } +func getHostname() string { + var hostname string + baseURL := os.Getenv("GHORG_SCM_BASE_URL") + if baseURL != "" { + // Parse the URL to extract the hostname + parsedURL, err := url.Parse(baseURL) + if err != nil { + colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL clone may be affected, error: %v", err)) + } + // Append the hostname to the absolute path + hostname = parsedURL.Hostname() + } else { + // Use the predefined hostname based on the SCM type + hostname = configs.GetCloudScmTypeHostnames() + } + + return hostname +} + +// updateAbsolutePathToCloneToWithHostname modifies the absolute path by appending the hostname if the user has enabled it, +// supporting the GHORG_PRESERVE_SCM_HOSTNAME feature. It checks the GHORG_PRESERVE_SCM_HOSTNAME environment variable, and if set to "true", +// it uses the hostname from GHORG_SCM_BASE_URL if available, otherwise, it defaults to a predefined hostname based on the SCM type. +func updateAbsolutePathToCloneToWithHostname() { + // Verify if GHORG_PRESERVE_SCM_HOSTNAME is set to "true" + if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" { + // Retrieve the hostname from the environment variable + hostname := getHostname() + absolutePath := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + os.Setenv("GHORG_ORIGINAL_ABSOLUTE_PATH_TO_CLONE_TO", absolutePath) + absolutePath = filepath.Join(absolutePath, hostname) + os.Setenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO", configs.EnsureTrailingSlashOnFilePath(absolutePath)) + } +} + // reads in configuration file and updates anything not set to default func getOrSetDefaults(envVar string) { if envVar == "GHORG_COLOR" { @@ -145,6 +181,8 @@ func getOrSetDefaults(envVar string) { os.Setenv(envVar, "owner") case "GHORG_BACKUP": os.Setenv(envVar, "false") + case "GHORG_PRESERVE_SCM_HOSTNAME": + os.Setenv(envVar, "false") case "GHORG_NO_TOKEN": os.Setenv(envVar, "false") case "GHORG_NO_DIR_SIZE": @@ -228,6 +266,7 @@ func InitConfig() { getOrSetDefaults("GHORG_CLONE_PROTOCOL") getOrSetDefaults("GHORG_CLONE_TYPE") getOrSetDefaults("GHORG_SCM_TYPE") + getOrSetDefaults("GHORG_PRESERVE_SCM_HOSTNAME") getOrSetDefaults("GHORG_SKIP_ARCHIVED") getOrSetDefaults("GHORG_SKIP_FORKS") getOrSetDefaults("GHORG_NO_CLEAN") @@ -324,6 +363,7 @@ func init() { cloneCmd.Flags().BoolVar(&quietMode, "quiet", false, "GHORG_QUIET - Emit critical output only") cloneCmd.Flags().BoolVar(&includeSubmodules, "include-submodules", false, "GHORG_INCLUDE_SUBMODULES - Include submodules in all clone and pull operations.") cloneCmd.Flags().BoolVar(&ghorgStatsEnabled, "stats-enabled", false, "GHORG_STATS_ENABLED - Creates a CSV in the GHORG_ABSOLUTE_PATH_TO_CLONE_TO called _ghorg_stats.csv with info about each clone. This allows you to track clone data over time such as number of commits and size in megabytes of the clone directory.") + cloneCmd.Flags().BoolVar(&ghorgPreserveScmHostname, "preserve-scm-hostname", false, "GHORG_PRESERVE_SCM_HOSTNAME - Appends the scm hostname to the GHORG_ABSOLUTE_PATH_TO_CLONE_TO which will organize your clones into specific folders by the scm provider. e.g. /github.com/kuberentes") cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL - Change SCM base url, for on self hosted instances (currently gitlab, gitea and github (use format of https://git.mydomain.com/api/v3))") cloneCmd.Flags().StringVarP(&concurrency, "concurrency", "", "", "GHORG_CONCURRENCY - Max goroutines to spin up while cloning (default 25)") cloneCmd.Flags().StringVarP(&cloneDepth, "clone-depth", "", "", "GHORG_CLONE_DEPTH - Create a shallow clone with a history truncated to the specified number of commits") diff --git a/cmd/version.go b/cmd/version.go index 65cbc71..27cad6e 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" ) -const ghorgVersion = "v1.10.1" +const ghorgVersion = "v1.11.0" var versionCmd = &cobra.Command{ Use: "version", diff --git a/configs/configs.go b/configs/configs.go index 17f20a1..de900bf 100644 --- a/configs/configs.go +++ b/configs/configs.go @@ -11,6 +11,7 @@ import ( "runtime" "strings" + "github.com/gabrie30/ghorg/colorlog" "github.com/gabrie30/ghorg/scm" "github.com/gabrie30/ghorg/utils" @@ -311,6 +312,22 @@ func VerifyTokenSet() error { return nil } +func GetCloudScmTypeHostnames() string { + switch os.Getenv("GHORG_SCM_TYPE") { + case "github": + return "github.com" + case "gitlab": + return "gitlab.com" + case "gitea": + return "gitea.com" + case "bitbucket": + return "bitbucket.com" + default: + colorlog.PrintErrorAndExit("Unsupported GHORG_SCM_TYPE") + return "" + } +} + // VerifyConfigsSetCorrectly makes sure flags are set to appropriate values func VerifyConfigsSetCorrectly() error { scmType := os.Getenv("GHORG_SCM_TYPE") diff --git a/examples/bitbucket.md b/examples/bitbucket.md index 08f4592..be33bf1 100644 --- a/examples/bitbucket.md +++ b/examples/bitbucket.md @@ -4,6 +4,8 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help` +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For bitbucket cloud it will be `bitbucket.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL`. + ## Bitbucket Cloud 1. Clone the microsoft workspace using an app-password diff --git a/examples/gitea.md b/examples/gitea.md index e1c7b42..810e4d0 100644 --- a/examples/gitea.md +++ b/examples/gitea.md @@ -4,6 +4,12 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help` +## Things to know + +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the `GHORG_SCM_BASE_URL` you are cloning from. + +## Examples + 1. Clone an **org** ``` diff --git a/examples/github.md b/examples/github.md index 638974d..cabcf6f 100644 --- a/examples/github.md +++ b/examples/github.md @@ -4,6 +4,10 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help` +## Things to know + +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For github cloud it will be `github.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL` if set for enterpirse github orgs. + ## GitHub Cloud 1. Clone an **org**, using a token on the commandline @@ -11,6 +15,14 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ``` ghorg clone --token=XXXXXX ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── github_org + ├── repo1 + └── repo2 + ``` 1. Clone all repos from a **github org** that are **prefixed** with "frontend" **into a folder** called "design_only" @@ -24,6 +36,20 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ghorg clone --clone-type=user --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2 ``` +1. Clone an **org**, preserving the cloud scm hostname + ``` + ghorg clone --preserve-cloud-scm-hostname + ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── github.com + └── github_org + ├── repo1 + └── repo2 + ``` + ## GitHub Enterprise 1. Clone an **org** @@ -43,3 +69,31 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ``` ghorg clone --clone-type=user --base-url=https://.com --token=XXXXXX ``` + +1. Clone an **org**, preserving the cloud scm hostname. The github_org may seem redudant in this example but when its needed for the case when you start to set an output directory. + ``` + ghorg clone --base-url=https://.com --preserve-cloud-scm-hostname + ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your-hosted-github.com + └── github_org + ├── repo1 + └── repo2 + ``` + +1. Clone an **org**, preserving the cloud scm hostname **with an output directory** + ``` + ghorg clone --base-url=https://.com --output-dir=my-repos --preserve-cloud-scm-hostname + ``` + Will produce the following + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your-hosted-github.com + └── my-repos + ├── myrepo1 + └── myrepo2 + ``` diff --git a/examples/gitlab.md b/examples/gitlab.md index 8d42ab5..484977e 100644 --- a/examples/gitlab.md +++ b/examples/gitlab.md @@ -14,6 +14,8 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri 1. The `--output-dir` flag overrides the default name given to the folder ghorg creates to clone repos into. The default will be the instance name when cloning `all-groups` or `all-users` or the `group` name when cloning a specific group. The exception is when you are cloning a subgroup and preserving the directory structure, then it will preserve the parent groups of the subgroup. +1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For gitlab cloud it will be `gitlab.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL`. + 1. If the group name you are cloning has spaces, substitute the spaces with "-" e.g. ```sh @@ -45,7 +47,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab + └── your.instance.gitlab.com ├── group1 │   └── project1 ├── group2 @@ -66,13 +68,34 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab + └── your.instance.gitlab.com ├── project1 ├── project2 ├── project3 └── project4 ``` +1. Clone **all groups**, **preserving the directory structure** of users, preserving scm hostname + + ```sh + ghorg clone all-groups --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname + ``` + + This would produce a directory structure like + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab.com + └── all-groups + ├── group1 + │   └── project1 + ├── group2 + │   └── project2 + └── group3 + ├── project3 + └── project4 + ``` + #### Cloning Specific Groups 1. Clone **a specific group**, **preserving the directory structure** of subgroups @@ -175,7 +198,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab_users + └── your.instance.gitlab.com ├── user1 │   └── project1 ├── user2 @@ -194,11 +217,30 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO - └── your.instance.gitlab_users - ├── project1 - ├── project2 - ├── project3 - └── project4 + └── your.instance.gitlab.com + ├── user1-repo1 + └── user2-repo1 + ``` + +1. Clone **all users**, **preserving the directory structure** of users, preserving scm hostname + + ```sh + ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname + ``` + + This would produce a directory structure like + + ```sh + /GHORG_ABSOLUTE_PATH_TO_CLONE_TO + └── your.instance.gitlab.com + └── all-users + ├── user1 + │   └── project1 + ├── user2 + │   └── project2 + └── user3 + ├── project3 + └── project4 ``` ## Cloud GitLab Orgs diff --git a/git/git.go b/git/git.go index 496274c..729619b 100644 --- a/git/git.go +++ b/git/git.go @@ -33,6 +33,8 @@ func NewGit() GitClient { func printDebugCmd(cmd *exec.Cmd, repo scm.Repo) error { fmt.Println("------------- GIT DEBUG -------------") + fmt.Printf("GHORG_OUTPUT_DIR=%v\n", os.Getenv("GHORG_OUTPUT_DIR")) + fmt.Printf("GHORG_ABSOLUTE_PATH_TO_CLONE_TO=%v\n", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO")) fmt.Print("Repo Data: ") spew.Dump(repo) fmt.Print("Command Ran: ") diff --git a/sample-conf.yaml b/sample-conf.yaml index 80e68a8..8f5092d 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -40,6 +40,12 @@ GHORG_SCM_BASE_URL: # flag (--include-submodules) GHORG_INCLUDE_SUBMODULES: false +# Appends the scm cloud hostname to the GHORG_ABSOLUTE_PATH_TO_CLONE_TO +# This will organize your clones into specific folders by the scm provider. +# e.g. GHORG_ABSOLUTE_PATH_TO_CLONE_TO/github.com/kuberentes") +# flag (--preserve-scm-hostname) +GHORG_PRESERVE_SCM_HOSTNAME: false + # Allows you to pass arguments to git's filter flag e.g. git clone --filter= this requires git version 2.19 or greater. # Useful for filtering out binary objects from repos # Few caveats, only works on inital clones, however if done on inital clone all subsequent clones will also honor the settings. diff --git a/scripts/bitbucket_cloud_integration_tests.sh b/scripts/bitbucket_cloud_integration_tests.sh index 4596e34..f9ee85d 100755 --- a/scripts/bitbucket_cloud_integration_tests.sh +++ b/scripts/bitbucket_cloud_integration_tests.sh @@ -29,3 +29,14 @@ else echo "Fail: bitbucket org clone, commandline flags take overwrite conf.yaml" exit 1 fi + +# preserve scm hostname +ghorg clone $BITBUCKET_WORKSPACE --token="${BITBUCKET_TOKEN}" --bitbucket-username="${BITBUCKET_USERNAME}" --path=/tmp --output-dir=testing_output_dir --scm=bitbucket --base-url="https://api.bitbucket.org/2.0" --preserve-scm-hostname + +if [ -e /tmp/api.bitbucket.org/testing_output_dir ] +then + echo "Pass: bitbucket org clone, preserve scm hostname" +else + echo "Fail: bitbucket org clone, preserve scm hostname" + exit 1 +fi diff --git a/scripts/github_cloud_integration_tests.sh b/scripts/github_cloud_integration_tests.sh index 95db527..bf7cc58 100755 --- a/scripts/github_cloud_integration_tests.sh +++ b/scripts/github_cloud_integration_tests.sh @@ -25,6 +25,17 @@ else exit 1 fi +# clone an org preserving scm hostname +ghorg clone $GITHUB_ORG --token=$GITHUB_TOKEN --preserve-scm-hostname + +if [ -e $HOME/ghorg/github.com/$GITHUB_ORG/$GHORG_TEST_REPO ] +then + echo "Pass: github org clone preserving scm hostname" +else + echo "Fail: github org clone preserving scm hostname" + exit 1 +fi + # clone an org with no config file to a specific path ghorg clone $GITHUB_ORG --token=$GITHUB_TOKEN --path=/tmp --output-dir=testing_output_dir diff --git a/scripts/gitlab_cloud_integration_tests.sh b/scripts/gitlab_cloud_integration_tests.sh index f71a703..201db79 100755 --- a/scripts/gitlab_cloud_integration_tests.sh +++ b/scripts/gitlab_cloud_integration_tests.sh @@ -22,6 +22,17 @@ else exit 1 fi +# preserve scm hostname +ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples-flat --preserve-scm-hostname + +if [ -e "${HOME}"/ghorg/gitlab.com/examples-flat/microservice ] +then + echo "Pass: gitlab org clone flat file, preserve scm hostname" +else + echo "Fail: gitlab org clone flat file, preserve scm hostname" + exit 1 +fi + # # TOP LEVEL GROUP TESTS # @@ -114,6 +125,24 @@ else exit 1 fi +ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab --clone-snippets --preserve-dir --preserve-scm-hostname + +if [ -e "${HOME}"/ghorg/gitlab.com/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-2-3711655 ] +then + echo "Pass: gitlab group clone snippet 2 with preserve dir, preserve scm hostname" +else + echo "Fail: gitlab group clone snippet 2 with preserve dir, preserve scm hostname" + exit 1 +fi + +if [ -e "${HOME}"/ghorg/gitlab.com/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-1-3711654 ] +then + echo "Pass: gitlab group clone snippet 1 with preserve dir, preserve scm hostname" +else + echo "Fail: gitlab group clone snippet 1 with preserve dir, preserve scm hostname" + exit 1 +fi + # # SUBGROUP TESTS # @@ -153,6 +182,18 @@ else exit 1 fi +# PRESERVE DIR, PRESERVE SCM HOSTNAME +ghorg clone $GITLAB_GROUP/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --preserve-dir --preserve-scm-hostname + +if [ -e "${HOME}"/ghorg/gitlab.com/"${GITLAB_GROUP}"/"${GITLAB_SUB_GROUP}"/wayne-industries/microservice ] +then + echo "Pass: gitlab subgroup clone preserve directories, preserve scm hostname" + rm -rf "${HOME}/ghorg/gitlab.com/${GITLAB_GROUP}" +else + echo "Fail: gitlab subgroup clone preserve directories, preserve scm hostname" + exit 1 +fi + # OUTPUT DIR AND PRESERVE DIR ghorg clone $GITLAB_GROUP/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --preserve-dir --output-dir=examples-subgroup-preserve-output diff --git a/scripts/local-gitlab/integration-tests.sh b/scripts/local-gitlab/integration-tests.sh index 30fa150..39e8b20 100755 --- a/scripts/local-gitlab/integration-tests.sh +++ b/scripts/local-gitlab/integration-tests.sh @@ -154,6 +154,98 @@ echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS TEST FAILED exit 1 fi +############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS, PERSERVE SCM HOSTNAME ############ +ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets --preserve-scm-hostname +ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets --preserve-scm-hostname + +GOT=$( ghorg ls gitlab.example.com/local-gitlab-v15-repos-snippets | grep -o 'gitlab.example.com/local-gitlab-v15-repos-snippets.*') +WANT=$(cat <