diff --git a/CHANGELOG.md b/CHANGELOG.md index 915e36c..b2e8367 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) ## [1.9.13] - Unreleased ### Added +- GHORG_CLONE_SNIPPETS as a way to clone all snippets, gitlab only ### Changed ### Deprecated ### Removed diff --git a/cmd/clone.go b/cmd/clone.go index 3e064ad..6dfe56a 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -185,6 +185,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Setenv("GHORG_CLONE_WIKI", "true") } + if cmd.Flags().Changed("clone-snippets") { + os.Setenv("GHORG_CLONE_SNIPPETS", "true") + } + if cmd.Flags().Changed("insecure-gitlab-client") { os.Setenv("GHORG_INSECURE_GITLAB_CLIENT", "true") } @@ -319,6 +323,7 @@ func getCloneUrls(isOrg bool) ([]scm.Repo, error) { if isOrg { return client.GetOrgRepos(targetCloneSource) } + return client.GetUserRepos(targetCloneSource) } @@ -491,6 +496,16 @@ func hasRepoNameCollisions(repos []scm.Repo) (map[string]bool, bool) { hasCollisions := false for _, repo := range repos { + + // Snippets should never have collions because we append the snippet id to the directory name + if repo.IsGitLabSnippet { + continue + } + + if repo.IsWiki { + continue + } + if _, ok := repoNameWithCollisions[repo.Name]; ok { repoNameWithCollisions[repo.Name] = true hasCollisions = true @@ -544,6 +559,21 @@ func trimCollisionFilename(filename string) string { return filename } +func getCloneableInventory(allRepos []scm.Repo) (int, int, int, int) { + var wikis, snippets, repos, total int + for _, repo := range allRepos { + if repo.IsGitLabSnippet { + snippets++ + } else if repo.IsWiki { + wikis++ + } else { + repos++ + } + } + total = repos + snippets + wikis + return total, repos, snippets, wikis +} + // CloneAllRepos clones all repos func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // Filter repos that have attributes that don't need specific scm api calls @@ -605,6 +635,16 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { targetRepoSeenOnOrg[targetRepo] = true } } + + if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" { + if cloneTarget.IsGitLabSnippet { + targetSnippetOriginalRepo := strings.TrimSuffix(filepath.Base(cloneTarget.GitLabSnippetInfo.URLOfRepo), ".git") + if strings.EqualFold(targetSnippetOriginalRepo, targetRepo) { + flag = true + targetRepoSeenOnOrg[targetRepo] = true + } + } + } } if flag { @@ -654,13 +694,18 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } - repoCount := getRepoCountOnly(cloneTargets) - - if os.Getenv("GHORG_CLONE_WIKI") == "true" { - wikiCount := strconv.Itoa(len(cloneTargets) - repoCount) - colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource + ", including " + wikiCount + " enabled wikis\n") + totalResourcesToClone, reposToCloneCount, snippetToCloneCount, wikisToCloneCount := getCloneableInventory(cloneTargets) + if os.Getenv("GHORG_CLONE_WIKI") == "true" && os.Getenv("GHORG_CLONE_SNIPPETS") == "true" { + m := fmt.Sprintf("%v resources to clone found in %v, %v repos, %v snippets, and %v wikis\n", totalResourcesToClone, targetCloneSource, snippetToCloneCount, reposToCloneCount, wikisToCloneCount) + colorlog.PrintInfo(m) + } else if os.Getenv("GHORG_CLONE_WIKI") == "true" { + m := fmt.Sprintf("%v resources to clone found in %v, %v repos and %v wikis\n", totalResourcesToClone, targetCloneSource, reposToCloneCount, wikisToCloneCount) + colorlog.PrintInfo(m) + } else if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" { + m := fmt.Sprintf("%v resources to clone found in %v, %v repos and %v snippets\n", totalResourcesToClone, targetCloneSource, reposToCloneCount, snippetToCloneCount) + colorlog.PrintInfo(m) } else { - colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource + "\n") + colorlog.PrintInfo(strconv.Itoa(reposToCloneCount) + " repos found in " + targetCloneSource + "\n") } if os.Getenv("GHORG_DRY_RUN") == "true" { @@ -687,19 +732,46 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { for i := range cloneTargets { repo := cloneTargets[i] + + // We use this because we dont want spaces in the final directory, using the web address makes it more file friendly + // In the case of root level snippets we use the title which will have spaces in it, the url uses an ID so its not possible to use name from url + // With snippets that originate on repos, we use that repo name repoSlug := getAppNameFromURL(repo.URL) + + if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet { + repoSlug = getAppNameFromURL(repo.GitLabSnippetInfo.URLOfRepo) + } else if repo.IsGitLabRootLevelSnippet { + repoSlug = repo.Name + } + limit.Execute(func() { if repo.Path != "" && os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" { repoSlug = repo.Path } + mutex.Lock() - inHash := repoNameWithCollisions[repo.Name] + var inHash bool + if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet { + inHash = repoNameWithCollisions[repo.GitLabSnippetInfo.NameOfRepo] + } else { + inHash = repoNameWithCollisions[repo.Name] + } + mutex.Unlock() // Only GitLab repos can have collisions due to groups and subgroups // If there are collisions and this is a repo with a naming collision change name to avoid collisions if hasCollisions && inHash { - repoSlug = trimCollisionFilename(strings.Replace(repo.Path, "/", "_", -1)) - + repoSlug = trimCollisionFilename(strings.Replace(repo.Path, string(os.PathSeparator), "_", -1)) + if repo.IsWiki { + if !strings.HasSuffix(repoSlug, ".wiki") { + repoSlug = repoSlug + ".wiki" + } + } + if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet { + if !strings.HasSuffix(repoSlug, ".snippets") { + repoSlug = repoSlug + ".snippets" + } + } mutex.Lock() slugCollision := repoNameWithCollisions[repoSlug] mutex.Unlock() @@ -713,12 +785,23 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } } + if repo.IsWiki { + if !strings.HasSuffix(repoSlug, ".wiki") { + repoSlug = repoSlug + ".wiki" + } + } + if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet { + if !strings.HasSuffix(repoSlug, ".snippets") { + repoSlug = repoSlug + ".snippets" + } + } + repo.HostPath = filepath.Join(outputDirAbsolutePath, repoSlug) - if repo.IsWiki { - if !strings.HasSuffix(repo.HostPath, ".wiki") { - repo.HostPath = repo.HostPath + ".wiki" - } + if repo.IsGitLabRootLevelSnippet { + repo.HostPath = filepath.Join(outputDirAbsolutePath, "_ghorg_root_level_snippets", repo.GitLabSnippetInfo.Title+"-"+repo.GitLabSnippetInfo.ID) + } else if repo.IsGitLabSnippet { + repo.HostPath = filepath.Join(outputDirAbsolutePath, repoSlug, repo.GitLabSnippetInfo.Title+"-"+repo.GitLabSnippetInfo.ID) } action := "cloning" @@ -726,7 +809,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // prevents git from asking for user for credentials, needs to be unset so creds aren't stored err := git.SetOriginWithCredentials(repo) if err != nil { - e := fmt.Sprintf("Problem setting remote with credentials Repo: %s Error: %v", repo.Name, err) + e := fmt.Sprintf("Problem setting remote with credentials on: %s Error: %v", repo.Name, err) cloneErrors = append(cloneErrors, e) return } @@ -736,13 +819,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { action = "updating remote" // Theres no way to tell if a github repo has a wiki to clone if err != nil && repo.IsWiki { - e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err) cloneInfos = append(cloneInfos, e) return } if err != nil { - e := fmt.Sprintf("Could not update remotes in Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Could not update remotes: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -753,13 +836,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // Theres no way to tell if a github repo has a wiki to clone if err != nil && repo.IsWiki { - e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err) cloneInfos = append(cloneInfos, e) return } if err != nil { - e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -767,7 +850,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } else { err := git.Checkout(repo) if err != nil { - e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes made Repo: %s Error: %v", repo.CloneBranch, repo.URL, err) + e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes made on: %s Error: %v", repo.CloneBranch, repo.URL, err) cloneInfos = append(cloneInfos, e) return } @@ -783,7 +866,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { err = git.Reset(repo) if err != nil { - e := fmt.Sprintf("Problem resetting %s Repo: %s Error: %v", repo.CloneBranch, repo.URL, err) + e := fmt.Sprintf("Problem resetting branch: %s for: %s Error: %v", repo.CloneBranch, repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -791,7 +874,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { err = git.Pull(repo) if err != nil { - e := fmt.Sprintf("Problem trying to pull %v Repo: %s Error: %v", repo.CloneBranch, repo.URL, err) + e := fmt.Sprintf("Problem trying to pull branch: %v for: %s Error: %v", repo.CloneBranch, repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -803,7 +886,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { err = git.FetchAll(repo) if err != nil { - e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -812,7 +895,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { err = git.SetOrigin(repo) if err != nil { - e := fmt.Sprintf("Problem resetting remote Repo: %s Error: %v", repo.Name, err) + e := fmt.Sprintf("Problem resetting remote: %s Error: %v", repo.Name, err) cloneErrors = append(cloneErrors, e) return } @@ -823,13 +906,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // Theres no way to tell if a github repo has a wiki to clone if err != nil && repo.IsWiki { - e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Wiki may be enabled but there was no content to clone: %s Error: %v", repo.URL, err) cloneInfos = append(cloneInfos, e) return } if err != nil { - e := fmt.Sprintf("Problem trying to clone Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Problem trying to clone: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -837,7 +920,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { if os.Getenv("GHORG_BRANCH") != "" { err := git.Checkout(repo) if err != nil { - e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes made Repo: %s Error: %v", repo.CloneBranch, repo.URL, err) + e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes to: %s Error: %v", repo.CloneBranch, repo.URL, err) cloneInfos = append(cloneInfos, e) return } @@ -851,7 +934,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // if repo has wiki, but content does not exist this is going to error if err != nil { - e := fmt.Sprintf("Problem trying to set remote on Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Problem trying to set remote: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) return } @@ -860,14 +943,14 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { err = git.FetchAll(repo) if err != nil { - e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err) + e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) return } } } - colorlog.PrintSuccess("Success " + action + " repo: " + repo.URL + " -> branch: " + repo.CloneBranch) + colorlog.PrintSuccess("Success " + action + " " + repo.URL + " -> branch: " + repo.CloneBranch) }) } @@ -949,11 +1032,11 @@ func pruneRepos(cloneTargets []scm.Repo) { func printCloneStatsMessage(cloneCount, pulledCount, updateRemoteCount int) { if updateRemoteCount > 0 { - colorlog.PrintSuccess(fmt.Sprintf("New repos cloned: %v, existing repos pulled: %v, remotes updated: %v", cloneCount, pulledCount, updateRemoteCount)) + colorlog.PrintSuccess(fmt.Sprintf("New clones: %v, existing resources pulled: %v, remotes updated: %v", cloneCount, pulledCount, updateRemoteCount)) return } - colorlog.PrintSuccess(fmt.Sprintf("New repos cloned: %v, existing repos pulled: %v", cloneCount, pulledCount)) + colorlog.PrintSuccess(fmt.Sprintf("New clones: %v, existing resources pulled: %v", cloneCount, pulledCount)) } func interactiveYesNoPrompt(prompt string) bool { @@ -1033,6 +1116,9 @@ func PrintConfigs() { if os.Getenv("GHORG_CLONE_WIKI") == "true" { colorlog.PrintInfo("* Wikis : " + os.Getenv("GHORG_CLONE_WIKI")) } + if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" { + colorlog.PrintInfo("* Snippets : " + os.Getenv("GHORG_CLONE_SNIPPETS")) + } if configs.GhorgIgnoreDetected() { colorlog.PrintInfo("* Ghorgignore : " + configs.GhorgIgnoreLocation()) } diff --git a/cmd/root.go b/cmd/root.go index 3963cb7..e08ae55 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,6 +55,7 @@ var ( prune bool pruneNoConfirm bool cloneWiki bool + cloneSnippets bool preserveDir bool insecureGitlabClient bool insecureGiteaClient bool @@ -122,6 +123,8 @@ func getOrSetDefaults(envVar string) { os.Setenv(envVar, "false") case "GHORG_CLONE_WIKI": os.Setenv(envVar, "false") + case "GHORG_CLONE_SNIPPETS": + os.Setenv(envVar, "false") case "GHORG_NO_CLEAN": os.Setenv(envVar, "false") case "GHORG_FETCH_ALL": @@ -231,6 +234,7 @@ func InitConfig() { getOrSetDefaults("GHORG_DRY_RUN") getOrSetDefaults("GHORG_GITHUB_USER_OPTION") getOrSetDefaults("GHORG_CLONE_WIKI") + getOrSetDefaults("GHORG_CLONE_SNIPPETS") getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT") getOrSetDefaults("GHORG_INSECURE_GITEA_CLIENT") getOrSetDefaults("GHORG_BACKUP") @@ -305,6 +309,7 @@ func init() { cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - Skip TLS certificate verification for hosted gitlab instances") cloneCmd.Flags().BoolVar(&insecureGiteaClient, "insecure-gitea-client", false, "GHORG_INSECURE_GITEA_CLIENT - Must be set to clone from a Gitea instance using http") cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo") + cloneCmd.Flags().BoolVar(&cloneSnippets, "clone-snippets", false, "GHORG_CLONE_SNIPPETS - Additionally clone all snippets, gitlab only") cloneCmd.Flags().BoolVar(&skipForks, "skip-forks", false, "GHORG_SKIP_FORKS - Skips repo if its a fork, github/gitlab/gitea only") cloneCmd.Flags().BoolVar(&noToken, "no-token", false, "GHORG_NO_TOKEN - Allows you to run ghorg with no token (GHORG__TOKEN), SCM server needs to specify no auth required for api calls") cloneCmd.Flags().BoolVar(&preserveDir, "preserve-dir", false, "GHORG_PRESERVE_DIRECTORY_STRUCTURE - Clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/unit/subunit/app, gitlab only") diff --git a/sample-conf.yaml b/sample-conf.yaml index 1d3d2b6..578e099 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -208,6 +208,10 @@ GHORG_INSECURE_GITLAB_CLIENT: false # flag (--gitlab-group-exclude-match-regex) GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX: +# Additionally clone all snippets +# flag (--clone-snippets) +GHORG_CLONE_SNIPPETS: false + # +-+-+-+-+-+ +-+-+-+-+-+-+-+-+ # |G|I|T|E|A| |S|P|E|C|I|F|I|C| # +-+-+-+-+-+ +-+-+-+-+-+-+-+-+ diff --git a/scm/gitlab.go b/scm/gitlab.go index 56434d1..3dc442d 100644 --- a/scm/gitlab.go +++ b/scm/gitlab.go @@ -14,10 +14,10 @@ import ( ) var ( - _ Client = Gitlab{} - perPage = 100 - gitLabAllGroups = false - gitLabAllUsers = false + _Client = Gitlab{} + perPage = 100 + gitLabAllGroups = false + gitLabAllUsers = false ) func init() { @@ -33,6 +33,25 @@ func (_ Gitlab) GetType() string { return "gitlab" } +func (_ Gitlab) rootLevelSnippet(url string) bool { + baseURL := os.Getenv("GHORG_SCM_BASE_URL") + if baseURL != "" { + customSnippetPattern := regexp.MustCompile(`^` + baseURL + `/-/snippets/\d+$`) + if customSnippetPattern.MatchString(url) { + return true + } + return false + } else { + // cloud instances + // Check if the URL follows the pattern of a root level snippet + rootLevelSnippetPattern := regexp.MustCompile(`^https://gitlab\.com/-/snippets/\d+$`) + if rootLevelSnippetPattern.MatchString(url) { + return true + } + return false + } +} + // GetOrgRepos fetches repo data from a specific group func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { allGroups := []string{} @@ -79,6 +98,12 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) { } + snippets, err := c.GetSnippets(repoData, targetOrg) + if err != nil { + colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err)) + } + repoData = append(repoData, snippets...) + return repoData, nil } @@ -119,6 +144,198 @@ func (c Gitlab) GetTopLevelGroups() ([]string, error) { return allGroups, nil } +// In this case take the cloneURL from the cloneTartet repo and just inject /snippets/:id before the .git +// cloud example +// http clone target url https://gitlab.com/ghorg-test-group/subgroup-2/foobar.git +// http snippet clone url https://gitlab.com/ghorg-test-group/subgroup-2/foobar/snippets/3711587.git +// ssh clone target url git@gitlab.com:ghorg-test-group/subgroup-2/foobar.git +// ssh snippet clone url git@gitlab.com:ghorg-test-group/subgroup-2/foobar/snippets/3711587.git +func (c Gitlab) createRepoSnippetCloneURL(cloneTargetURL string, snippetID string) string { + + // Split the cloneTargetURL into two parts at the ".git" + parts := strings.Split(cloneTargetURL, ".git") + // Insert the "/snippets/:id" before the ".git" + cloneURL := parts[0] + "/snippets/" + snippetID + ".git" + + if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { + return cloneURL + } + + // git@gitlab.example.com:local-gitlab-group3/subgroup-a/subgroup-b/subgroup_b_repo_1/snippets/12.git + + // http://gitlab.example.com/snippets/1.git + if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" { + cloneURL = strings.Replace(cloneURL, "http://", "git@", 1) + } else { + cloneURL = strings.Replace(cloneURL, "https://", "git@", 1) + } + // git@gitlab.example.com/snippets/1.git + cloneURL = strings.Replace(cloneURL, "/", ":", 1) + // git@gitlab.example.com:snippets/1.git + return cloneURL +} + +// hosted example +// root snippet ssh clone url git@gitlab.example.com:snippets/1.git +// root snippet http clone url http://gitlab.example.com/snippets/1.git +func (c Gitlab) createRootLevelSnippetCloneURL(snippetWebURL string) string { + // Web URL example, http://gitlab.example.com/-/snippets/1 + // Both http and ssh clone urls do not have the /-/ in them so just remove it first and add the .git extention + cloneURL := strings.Replace(snippetWebURL, "/-/", "/", -1) + ".git" + if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { + return c.addTokenToCloneURL(cloneURL, os.Getenv("GHORG_GITLAB_TOKEN")) + } + + if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" { + cloneURL = strings.Replace(cloneURL, "http://", "git@", 1) + } else { + cloneURL = strings.Replace(cloneURL, "https://", "git@", 1) + } + // git@gitlab.example.com/snippets/1.git + cloneURL = strings.Replace(cloneURL, "/", ":", 1) + // git@gitlab.example.com:snippets/1.git + return cloneURL +} + +func (c Gitlab) getRepoSnippets(r Repo) []*gitlab.Snippet { + var allSnippets []*gitlab.Snippet + opt := &gitlab.ListProjectSnippetsOptions{ + PerPage: perPage, + Page: 1, + } + + for { + snippets, resp, err := c.ProjectSnippets.ListSnippets(r.ID, opt) + + if resp.StatusCode == 403 { + break + } + + if err != nil { + colorlog.PrintError(fmt.Sprintf("Error fetching snippets for project %s: %v, ignoring error and proceeding to next project", r.Name, err)) + break + } + + allSnippets = append(allSnippets, snippets...) + + // Exit the loop when we've seen all pages. + if resp.NextPage == 0 { + break + } + + // Update the page number to get the next page. + opt.Page = resp.NextPage + } + + return allSnippets +} + +func (c Gitlab) getAllSnippets() []*gitlab.Snippet { + var allSnippets []*gitlab.Snippet + opt := &gitlab.ListAllSnippetsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: perPage, + Page: 1, + }, + } + + for { + snippets, resp, err := c.Snippets.ListAllSnippets(opt) + if err != nil { + colorlog.PrintError(fmt.Sprintf("Issue fetching all snippets, not all snippets will be cloned error: %v", err)) + return allSnippets + } + + allSnippets = append(allSnippets, snippets...) + + // Exit the loop when we've seen all pages. + if resp.NextPage == 0 { + break + } + + // Update the page number to get the next page. + opt.Page = resp.NextPage + } + + return allSnippets +} + +func (c Gitlab) GetSnippets(cloneData []Repo, target string) ([]Repo, error) { + + if os.Getenv("GHORG_CLONE_SNIPPETS") != "true" { + return []Repo{}, nil + } + + var allSnippetsToClone []*gitlab.Snippet + + // Snippets are converted into Repos so we can clone them + snippetsToClone := []Repo{} + + // If it is a cloud group clone iterate over each project and try to get its snippets. We have to do this because if you use the /snippets/all endpoint it will return every public snippet from the cloud. + if os.Getenv("GHORG_CLONE_TYPE") != "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" { + // Iterate over all projects in the group. If it has snippets add them + colorlog.PrintInfo("Note: only snippets you have access to will be cloned. This process may take a while depending on the size of group you are trying to clone, please be patient.") + for _, repo := range cloneData { + snippets := c.getRepoSnippets(repo) + allSnippetsToClone = append(allSnippetsToClone, snippets...) + } + } else { + allSnippets := c.getAllSnippets() + + // if its an all-user or all-group clone, for each repo get its snippets then also include all root level snippets + if target == "all-users" || target == "all-groups" { + for _, repo := range cloneData { + repoSnippets := c.getRepoSnippets(repo) + allSnippetsToClone = append(allSnippetsToClone, repoSnippets...) + } + + for _, snippet := range allSnippets { + if c.rootLevelSnippet(snippet.WebURL) { + allSnippetsToClone = append(allSnippetsToClone, snippet) + } + } + } + + if os.Getenv("GHORG_CLONE_TYPE") == "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" { + + } + + } + + for _, snippet := range allSnippetsToClone { + snippetID := strconv.Itoa(snippet.ID) + snippetTitle := ToSlug(snippet.Title) + s := Repo{} + s.IsGitLabSnippet = true + s.CloneBranch = "main" + s.GitLabSnippetInfo.Title = snippetTitle + s.Name = snippetTitle + s.GitLabSnippetInfo.ID = snippetID + s.URL = snippet.WebURL + // If the snippet is not made on any repo its a root level snippet, this works for cloud + if c.rootLevelSnippet(snippet.WebURL) { + s.IsGitLabRootLevelSnippet = true + s.CloneURL = c.createRootLevelSnippetCloneURL(snippet.WebURL) + cloneData = append(cloneData, s) + } else { + // Since this isn't a root level repo we want to find which repo the snippet is coming from + for _, cloneTarget := range cloneData { + if cloneTarget.ID == strconv.Itoa(snippet.ProjectID) { + s.CloneURL = c.createRepoSnippetCloneURL(cloneTarget.CloneURL, snippetID) + s.Path = cloneTarget.Path + s.GitLabSnippetInfo.URLOfRepo = cloneTarget.URL + s.GitLabSnippetInfo.NameOfRepo = cloneTarget.Name + cloneData = append(cloneData, s) + } + } + } + + snippetsToClone = append(snippetsToClone, s) + } + + return snippetsToClone, nil +} + // GetGroupRepos fetches repo data from a specific group func (c Gitlab) GetGroupRepos(targetGroup string) ([]Repo, error) { @@ -223,6 +440,11 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) { } } + snippets, err := c.GetSnippets(cloneData, targetUsername) + if err != nil { + colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err)) + } + cloneData = append(cloneData, snippets...) return cloneData, nil } @@ -292,6 +514,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo { r := Repo{} r.Name = p.Name + r.ID = strconv.Itoa(p.ID) if os.Getenv("GHORG_BRANCH") == "" { defaultBranch := p.DefaultBranch @@ -320,6 +543,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo { } r.Path = path + r.ID = fmt.Sprint(p.ID) if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { r.CloneURL = c.addTokenToCloneURL(p.HTTPURLToRepo, os.Getenv("GHORG_GITLAB_TOKEN")) r.URL = p.HTTPURLToRepo @@ -332,6 +556,8 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo { if p.WikiEnabled && os.Getenv("GHORG_CLONE_WIKI") == "true" { wiki := Repo{} + // wiki needs name for gitlab name collisions + wiki.Name = p.Name wiki.IsWiki = true wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1) wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1) @@ -348,7 +574,6 @@ func filterGitlabGroupByExcludeMatchRegex(groups []string) []string { regex := fmt.Sprint(os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX")) for i, grp := range groups { - fmt.Println(grp) exclude := false re := regexp.MustCompile(regex) match := re.FindString(grp) @@ -363,3 +588,20 @@ func filterGitlabGroupByExcludeMatchRegex(groups []string) []string { return filteredGroups } + +// ToSlug converts a title into a URL-friendly slug. +func ToSlug(title string) string { + // Convert to lowercase + slug := strings.ToLower(title) + + // Replace spaces and special characters with hyphens + slug = regexp.MustCompile(`[\s\p{P}]+`).ReplaceAllString(slug, "-") + + // Remove any non-alphanumeric characters except for hyphens + slug = regexp.MustCompile(`[^a-z0-9-]+`).ReplaceAllString(slug, "") + + // Trim any leading or trailing hyphens + slug = strings.Trim(slug, "-") + + return slug +} diff --git a/scm/structs.go b/scm/structs.go index dbb133d..8aa32de 100644 --- a/scm/structs.go +++ b/scm/structs.go @@ -1,12 +1,38 @@ package scm -// Repo represents an SCM repo +// Repo represents an SCM repo, should probably be renamed to "cloneable" since we clone wikis and snippets with this type Repo struct { - Name string - HostPath string - Path string - URL string - CloneURL string + // The ID of the repo that is assigned via the SCM provider. This is used for example with gitlab snippets on cloud gropus where we need to know the repo id to look up all he snippets it has. + ID string + // Name is the name of the repo https://www.github.com/gabrie30/ghorg.git the Name would be ghorg + Name string + // HostPath is the path on the users machine that the repo will be cloned to. Its used in all the git commands to locate the directory of the repo. HostPath is updated for wikis and snippets because the folder for the clone is appended with .wiki and .snippet + HostPath string + // Path where the repo is located within the scm provider. Its mostly used with gitlab repos when the directory structure is preserved. In this case the path becomes where to locate the repo in relation to gitlab.com/group/group/group/repo.git => group/group/group/repo + Path string + // URL is the web address of the repo + URL string + // CloneURL is the url for cloning the repo, will be different for ssh vs http clones and will have the .git extention + CloneURL string + // CloneBranch the branch to clone. This will be the default branch if not specified. It will always be main for snippets. CloneBranch string - IsWiki bool + // IsWiki is set to true when the data is for a wiki page + IsWiki bool + // IsGitLabSnippet is set to true when the data is for a gitlab snippet + IsGitLabSnippet bool + // IsGitLabRootLevelSnippet is set to true when a snippet was not created for a repo + IsGitLabRootLevelSnippet bool + // GitLabSnippetInfo provides additional information when the thing we are cloning is a gitlab snippet + GitLabSnippetInfo GitLabSnippet +} + +type GitLabSnippet struct { + // GitLab ID of the snippet + ID string + // Title of the snippet + Title string + // URL of the repo that snippet was made on + URLOfRepo string + // Name of the repo that the snippet was made on + NameOfRepo string } diff --git a/scripts/gitlab_cloud_integration_tests.sh b/scripts/gitlab_cloud_integration_tests.sh index 83122c2..f71a703 100755 --- a/scripts/gitlab_cloud_integration_tests.sh +++ b/scripts/gitlab_cloud_integration_tests.sh @@ -96,23 +96,23 @@ else fi # SNIPPETS -# ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab --clone-snippets --preserve-dir +ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab --clone-snippets --preserve-dir -# if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-2-3711655 ] -# then -# echo "Pass: gitlab group clone snippet 2 with preserve dir" -# else -# echo "Fail: gitlab group clone snippet 2 with preserve dir" -# exit 1 -# fi +if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-2-3711655 ] +then + echo "Pass: gitlab group clone snippet 2 with preserve dir" +else + echo "Fail: gitlab group clone snippet 2 with preserve dir" + exit 1 +fi -# if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-1-3711654 ] -# then -# echo "Pass: gitlab group clone snippet 1 with preserve dir" -# else -# echo "Fail: gitlab group clone snippet 1 with preserve dir" -# exit 1 -# fi +if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-1-3711654 ] +then + echo "Pass: gitlab group clone snippet 1 with preserve dir" +else + echo "Fail: gitlab group clone snippet 1 with preserve dir" + exit 1 +fi # # SUBGROUP TESTS diff --git a/scripts/local-gitlab/integration-tests.sh b/scripts/local-gitlab/integration-tests.sh index 0749f46..30fa150 100755 --- a/scripts/local-gitlab/integration-tests.sh +++ b/scripts/local-gitlab/integration-tests.sh @@ -31,7 +31,7 @@ export GHORG_INSECURE_GITLAB_CLIENT=true -############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR ############ +############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS ############ ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos @@ -81,6 +81,79 @@ echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED local-gitl exit 1 fi +############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS ############ +ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets +ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets + +GOT=$( ghorg ls local-gitlab-v15-repos-snippets | grep -o 'local-gitlab-v15-repos-snippets.*') +WANT=$(cat <