github-personal/ghorg
1
0
Fork 0
mirror of https://github.com/outbackdingo/ghorg.git synced 2026-01-27 10:19:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add snippet support (#416)

Browse Source
This commit is contained in:
gabrie30
2024-07-12 20:29:03 -07:00
committed by GitHub
parent 30ca6dd5d0
commit 3f993196b8
9 changed files with 578 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
## [1.9.13] - Unreleased
### Added
- GHORG_CLONE_SNIPPETS as a way to clone all snippets, gitlab only
### Changed
### Deprecated
### Removed

148
cmd/clone.go
View File

@@ -185,6 +185,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
os.Setenv("GHORG_CLONE_WIKI", "true")
}
if cmd.Flags().Changed("clone-snippets") {
os.Setenv("GHORG_CLONE_SNIPPETS", "true")
}
if cmd.Flags().Changed("insecure-gitlab-client") {
os.Setenv("GHORG_INSECURE_GITLAB_CLIENT", "true")
}
@@ -319,6 +323,7 @@ func getCloneUrls(isOrg bool) ([]scm.Repo, error) {
if isOrg {
return client.GetOrgRepos(targetCloneSource)
}
return client.GetUserRepos(targetCloneSource)
}
@@ -491,6 +496,16 @@ func hasRepoNameCollisions(repos []scm.Repo) (map[string]bool, bool) {
hasCollisions := false
for _, repo := range repos {
// Snippets should never have collions because we append the snippet id to the directory name
if repo.IsGitLabSnippet {
continue
}
if repo.IsWiki {
continue
}
if _, ok := repoNameWithCollisions[repo.Name]; ok {
repoNameWithCollisions[repo.Name] = true
hasCollisions = true
@@ -544,6 +559,21 @@ func trimCollisionFilename(filename string) string {
return filename
}
func getCloneableInventory(allRepos []scm.Repo) (int, int, int, int) {
var wikis, snippets, repos, total int
for _, repo := range allRepos {
if repo.IsGitLabSnippet {
snippets++
} else if repo.IsWiki {
wikis++
} else {
repos++
}
}
total = repos + snippets + wikis
return total, repos, snippets, wikis
}
// CloneAllRepos clones all repos
func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
// Filter repos that have attributes that don't need specific scm api calls
@@ -605,6 +635,16 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
targetRepoSeenOnOrg[targetRepo] = true
}
}
if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" {
if cloneTarget.IsGitLabSnippet {
targetSnippetOriginalRepo := strings.TrimSuffix(filepath.Base(cloneTarget.GitLabSnippetInfo.URLOfRepo), ".git")
if strings.EqualFold(targetSnippetOriginalRepo, targetRepo) {
flag = true
targetRepoSeenOnOrg[targetRepo] = true
}
}
}
}
if flag {
@@ -654,13 +694,18 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}
repoCount := getRepoCountOnly(cloneTargets)
if os.Getenv("GHORG_CLONE_WIKI") == "true" {
wikiCount := strconv.Itoa(len(cloneTargets) - repoCount)
colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource + ", including " + wikiCount + " enabled wikis\n")
totalResourcesToClone, reposToCloneCount, snippetToCloneCount, wikisToCloneCount := getCloneableInventory(cloneTargets)
if os.Getenv("GHORG_CLONE_WIKI") == "true" && os.Getenv("GHORG_CLONE_SNIPPETS") == "true" {
m := fmt.Sprintf("%v resources to clone found in %v, %v repos, %v snippets, and %v wikis\n", totalResourcesToClone, targetCloneSource, snippetToCloneCount, reposToCloneCount, wikisToCloneCount)
colorlog.PrintInfo(m)
} else if os.Getenv("GHORG_CLONE_WIKI") == "true" {
m := fmt.Sprintf("%v resources to clone found in %v, %v repos and %v wikis\n", totalResourcesToClone, targetCloneSource, reposToCloneCount, wikisToCloneCount)
colorlog.PrintInfo(m)
} else if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" {
m := fmt.Sprintf("%v resources to clone found in %v, %v repos and %v snippets\n", totalResourcesToClone, targetCloneSource, reposToCloneCount, snippetToCloneCount)
colorlog.PrintInfo(m)
} else {
colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource + "\n")
colorlog.PrintInfo(strconv.Itoa(reposToCloneCount) + " repos found in " + targetCloneSource + "\n")
}
if os.Getenv("GHORG_DRY_RUN") == "true" {
@@ -687,19 +732,46 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
for i := range cloneTargets {
repo := cloneTargets[i]
// We use this because we dont want spaces in the final directory, using the web address makes it more file friendly
// In the case of root level snippets we use the title which will have spaces in it, the url uses an ID so its not possible to use name from url
// With snippets that originate on repos, we use that repo name
repoSlug := getAppNameFromURL(repo.URL)
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
repoSlug = getAppNameFromURL(repo.GitLabSnippetInfo.URLOfRepo)
} else if repo.IsGitLabRootLevelSnippet {
repoSlug = repo.Name
}
limit.Execute(func() {
if repo.Path != "" && os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" {
repoSlug = repo.Path
}
mutex.Lock()
inHash := repoNameWithCollisions[repo.Name]
var inHash bool
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
inHash = repoNameWithCollisions[repo.GitLabSnippetInfo.NameOfRepo]
} else {
inHash = repoNameWithCollisions[repo.Name]
}
mutex.Unlock()
// Only GitLab repos can have collisions due to groups and subgroups
// If there are collisions and this is a repo with a naming collision change name to avoid collisions
if hasCollisions && inHash {
repoSlug = trimCollisionFilename(strings.Replace(repo.Path, "/", "_", -1))
repoSlug = trimCollisionFilename(strings.Replace(repo.Path, string(os.PathSeparator), "_", -1))
if repo.IsWiki {
if !strings.HasSuffix(repoSlug, ".wiki") {
repoSlug = repoSlug + ".wiki"
}
}
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
if !strings.HasSuffix(repoSlug, ".snippets") {
repoSlug = repoSlug + ".snippets"
}
}
mutex.Lock()
slugCollision := repoNameWithCollisions[repoSlug]
mutex.Unlock()
@@ -713,12 +785,23 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}
}
if repo.IsWiki {
if !strings.HasSuffix(repoSlug, ".wiki") {
repoSlug = repoSlug + ".wiki"
}
}
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
if !strings.HasSuffix(repoSlug, ".snippets") {
repoSlug = repoSlug + ".snippets"
}
}
repo.HostPath = filepath.Join(outputDirAbsolutePath, repoSlug)
if repo.IsWiki {
if !strings.HasSuffix(repo.HostPath, ".wiki") {
repo.HostPath = repo.HostPath + ".wiki"
}
if repo.IsGitLabRootLevelSnippet {
repo.HostPath = filepath.Join(outputDirAbsolutePath, "_ghorg_root_level_snippets", repo.GitLabSnippetInfo.Title+"-"+repo.GitLabSnippetInfo.ID)
} else if repo.IsGitLabSnippet {
repo.HostPath = filepath.Join(outputDirAbsolutePath, repoSlug, repo.GitLabSnippetInfo.Title+"-"+repo.GitLabSnippetInfo.ID)
}
action := "cloning"
@@ -726,7 +809,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
// prevents git from asking for user for credentials, needs to be unset so creds aren't stored
err := git.SetOriginWithCredentials(repo)
if err != nil {
e := fmt.Sprintf("Problem setting remote with credentials Repo: %s Error: %v", repo.Name, err)
e := fmt.Sprintf("Problem setting remote with credentials on: %s Error: %v", repo.Name, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -736,13 +819,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
action = "updating remote"
// Theres no way to tell if a github repo has a wiki to clone
if err != nil && repo.IsWiki {
e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err)
cloneInfos = append(cloneInfos, e)
return
}
if err != nil {
e := fmt.Sprintf("Could not update remotes in Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Could not update remotes: %s Error: %v", repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -753,13 +836,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
// Theres no way to tell if a github repo has a wiki to clone
if err != nil && repo.IsWiki {
e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err)
cloneInfos = append(cloneInfos, e)
return
}
if err != nil {
e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -767,7 +850,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
} else {
err := git.Checkout(repo)
if err != nil {
e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes made Repo: %s Error: %v", repo.CloneBranch, repo.URL, err)
e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes made on: %s Error: %v", repo.CloneBranch, repo.URL, err)
cloneInfos = append(cloneInfos, e)
return
}
@@ -783,7 +866,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
err = git.Reset(repo)
if err != nil {
e := fmt.Sprintf("Problem resetting %s Repo: %s Error: %v", repo.CloneBranch, repo.URL, err)
e := fmt.Sprintf("Problem resetting branch: %s for: %s Error: %v", repo.CloneBranch, repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -791,7 +874,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
err = git.Pull(repo)
if err != nil {
e := fmt.Sprintf("Problem trying to pull %v Repo: %s Error: %v", repo.CloneBranch, repo.URL, err)
e := fmt.Sprintf("Problem trying to pull branch: %v for: %s Error: %v", repo.CloneBranch, repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -803,7 +886,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
err = git.FetchAll(repo)
if err != nil {
e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -812,7 +895,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
err = git.SetOrigin(repo)
if err != nil {
e := fmt.Sprintf("Problem resetting remote Repo: %s Error: %v", repo.Name, err)
e := fmt.Sprintf("Problem resetting remote: %s Error: %v", repo.Name, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -823,13 +906,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
// Theres no way to tell if a github repo has a wiki to clone
if err != nil && repo.IsWiki {
e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Wiki may be enabled but there was no content to clone: %s Error: %v", repo.URL, err)
cloneInfos = append(cloneInfos, e)
return
}
if err != nil {
e := fmt.Sprintf("Problem trying to clone Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Problem trying to clone: %s Error: %v", repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -837,7 +920,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
if os.Getenv("GHORG_BRANCH") != "" {
err := git.Checkout(repo)
if err != nil {
e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes made Repo: %s Error: %v", repo.CloneBranch, repo.URL, err)
e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents, no changes to: %s Error: %v", repo.CloneBranch, repo.URL, err)
cloneInfos = append(cloneInfos, e)
return
}
@@ -851,7 +934,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
// if repo has wiki, but content does not exist this is going to error
if err != nil {
e := fmt.Sprintf("Problem trying to set remote on Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Problem trying to set remote: %s Error: %v", repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
@@ -860,14 +943,14 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
err = git.FetchAll(repo)
if err != nil {
e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err)
e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)
cloneErrors = append(cloneErrors, e)
return
}
}
}
colorlog.PrintSuccess("Success " + action + " repo: " + repo.URL + " -> branch: " + repo.CloneBranch)
colorlog.PrintSuccess("Success " + action + " " + repo.URL + " -> branch: " + repo.CloneBranch)
})
}
@@ -949,11 +1032,11 @@ func pruneRepos(cloneTargets []scm.Repo) {
func printCloneStatsMessage(cloneCount, pulledCount, updateRemoteCount int) {
if updateRemoteCount > 0 {
colorlog.PrintSuccess(fmt.Sprintf("New repos cloned: %v, existing repos pulled: %v, remotes updated: %v", cloneCount, pulledCount, updateRemoteCount))
colorlog.PrintSuccess(fmt.Sprintf("New clones: %v, existing resources pulled: %v, remotes updated: %v", cloneCount, pulledCount, updateRemoteCount))
return
}
colorlog.PrintSuccess(fmt.Sprintf("New repos cloned: %v, existing repos pulled: %v", cloneCount, pulledCount))
colorlog.PrintSuccess(fmt.Sprintf("New clones: %v, existing resources pulled: %v", cloneCount, pulledCount))
}
func interactiveYesNoPrompt(prompt string) bool {
@@ -1033,6 +1116,9 @@ func PrintConfigs() {
if os.Getenv("GHORG_CLONE_WIKI") == "true" {
colorlog.PrintInfo("* Wikis : " + os.Getenv("GHORG_CLONE_WIKI"))
}
if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" {
colorlog.PrintInfo("* Snippets : " + os.Getenv("GHORG_CLONE_SNIPPETS"))
}
if configs.GhorgIgnoreDetected() {
colorlog.PrintInfo("* Ghorgignore : " + configs.GhorgIgnoreLocation())
}

5
cmd/root.go
View File

@@ -55,6 +55,7 @@ var (
prune bool
pruneNoConfirm bool
cloneWiki bool
cloneSnippets bool
preserveDir bool
insecureGitlabClient bool
insecureGiteaClient bool
@@ -122,6 +123,8 @@ func getOrSetDefaults(envVar string) {
os.Setenv(envVar, "false")
case "GHORG_CLONE_WIKI":
os.Setenv(envVar, "false")
case "GHORG_CLONE_SNIPPETS":
os.Setenv(envVar, "false")
case "GHORG_NO_CLEAN":
os.Setenv(envVar, "false")
case "GHORG_FETCH_ALL":
@@ -231,6 +234,7 @@ func InitConfig() {
getOrSetDefaults("GHORG_DRY_RUN")
getOrSetDefaults("GHORG_GITHUB_USER_OPTION")
getOrSetDefaults("GHORG_CLONE_WIKI")
getOrSetDefaults("GHORG_CLONE_SNIPPETS")
getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT")
getOrSetDefaults("GHORG_INSECURE_GITEA_CLIENT")
getOrSetDefaults("GHORG_BACKUP")
@@ -305,6 +309,7 @@ func init() {
cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - Skip TLS certificate verification for hosted gitlab instances")
cloneCmd.Flags().BoolVar(&insecureGiteaClient, "insecure-gitea-client", false, "GHORG_INSECURE_GITEA_CLIENT - Must be set to clone from a Gitea instance using http")
cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo")
cloneCmd.Flags().BoolVar(&cloneSnippets, "clone-snippets", false, "GHORG_CLONE_SNIPPETS - Additionally clone all snippets, gitlab only")
cloneCmd.Flags().BoolVar(&skipForks, "skip-forks", false, "GHORG_SKIP_FORKS - Skips repo if its a fork, github/gitlab/gitea only")
cloneCmd.Flags().BoolVar(&noToken, "no-token", false, "GHORG_NO_TOKEN - Allows you to run ghorg with no token (GHORG_<SCM>_TOKEN), SCM server needs to specify no auth required for api calls")
cloneCmd.Flags().BoolVar(&preserveDir, "preserve-dir", false, "GHORG_PRESERVE_DIRECTORY_STRUCTURE - Clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/unit/subunit/app, gitlab only")

4
sample-conf.yaml
View File

@@ -208,6 +208,10 @@ GHORG_INSECURE_GITLAB_CLIENT: false
# flag (--gitlab-group-exclude-match-regex)
GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX:
# Additionally clone all snippets
# flag (--clone-snippets)
GHORG_CLONE_SNIPPETS: false
# +-+-+-+-+-+ +-+-+-+-+-+-+-+-+
# |G|I|T|E|A| |S|P|E|C|I|F|I|C|
# +-+-+-+-+-+ +-+-+-+-+-+-+-+-+

252
scm/gitlab.go
View File

@@ -14,10 +14,10 @@ import (
)
var (
_ Client = Gitlab{}
perPage = 100
gitLabAllGroups = false
gitLabAllUsers = false
_Client = Gitlab{}
perPage = 100
gitLabAllGroups = false
gitLabAllUsers = false
)
func init() {
@@ -33,6 +33,25 @@ func (_ Gitlab) GetType() string {
return "gitlab"
}
func (_ Gitlab) rootLevelSnippet(url string) bool {
baseURL := os.Getenv("GHORG_SCM_BASE_URL")
if baseURL != "" {
customSnippetPattern := regexp.MustCompile(`^` + baseURL + `/-/snippets/\d+$`)
if customSnippetPattern.MatchString(url) {
return true
}
return false
} else {
// cloud instances
// Check if the URL follows the pattern of a root level snippet
rootLevelSnippetPattern := regexp.MustCompile(`^https://gitlab\.com/-/snippets/\d+$`)
if rootLevelSnippetPattern.MatchString(url) {
return true
}
return false
}
}
// GetOrgRepos fetches repo data from a specific group
func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
allGroups := []string{}
@@ -79,6 +98,12 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
}
snippets, err := c.GetSnippets(repoData, targetOrg)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err))
}
repoData = append(repoData, snippets...)
return repoData, nil
}
@@ -119,6 +144,198 @@ func (c Gitlab) GetTopLevelGroups() ([]string, error) {
return allGroups, nil
}
// In this case take the cloneURL from the cloneTartet repo and just inject /snippets/:id before the .git
// cloud example
// http clone target url https://gitlab.com/ghorg-test-group/subgroup-2/foobar.git
// http snippet clone url https://gitlab.com/ghorg-test-group/subgroup-2/foobar/snippets/3711587.git
// ssh clone target url git@gitlab.com:ghorg-test-group/subgroup-2/foobar.git
// ssh snippet clone url git@gitlab.com:ghorg-test-group/subgroup-2/foobar/snippets/3711587.git
func (c Gitlab) createRepoSnippetCloneURL(cloneTargetURL string, snippetID string) string {
// Split the cloneTargetURL into two parts at the ".git"
parts := strings.Split(cloneTargetURL, ".git")
// Insert the "/snippets/:id" before the ".git"
cloneURL := parts[0] + "/snippets/" + snippetID + ".git"
if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
return cloneURL
}
// git@gitlab.example.com:local-gitlab-group3/subgroup-a/subgroup-b/subgroup_b_repo_1/snippets/12.git
// http://gitlab.example.com/snippets/1.git
if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" {
cloneURL = strings.Replace(cloneURL, "http://", "git@", 1)
} else {
cloneURL = strings.Replace(cloneURL, "https://", "git@", 1)
}
// git@gitlab.example.com/snippets/1.git
cloneURL = strings.Replace(cloneURL, "/", ":", 1)
// git@gitlab.example.com:snippets/1.git
return cloneURL
}
// hosted example
// root snippet ssh clone url git@gitlab.example.com:snippets/1.git
// root snippet http clone url http://gitlab.example.com/snippets/1.git
func (c Gitlab) createRootLevelSnippetCloneURL(snippetWebURL string) string {
// Web URL example, http://gitlab.example.com/-/snippets/1
// Both http and ssh clone urls do not have the /-/ in them so just remove it first and add the .git extention
cloneURL := strings.Replace(snippetWebURL, "/-/", "/", -1) + ".git"
if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
return c.addTokenToCloneURL(cloneURL, os.Getenv("GHORG_GITLAB_TOKEN"))
}
if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" {
cloneURL = strings.Replace(cloneURL, "http://", "git@", 1)
} else {
cloneURL = strings.Replace(cloneURL, "https://", "git@", 1)
}
// git@gitlab.example.com/snippets/1.git
cloneURL = strings.Replace(cloneURL, "/", ":", 1)
// git@gitlab.example.com:snippets/1.git
return cloneURL
}
func (c Gitlab) getRepoSnippets(r Repo) []*gitlab.Snippet {
var allSnippets []*gitlab.Snippet
opt := &gitlab.ListProjectSnippetsOptions{
PerPage: perPage,
Page: 1,
}
for {
snippets, resp, err := c.ProjectSnippets.ListSnippets(r.ID, opt)
if resp.StatusCode == 403 {
break
}
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error fetching snippets for project %s: %v, ignoring error and proceeding to next project", r.Name, err))
break
}
allSnippets = append(allSnippets, snippets...)
// Exit the loop when we've seen all pages.
if resp.NextPage == 0 {
break
}
// Update the page number to get the next page.
opt.Page = resp.NextPage
}
return allSnippets
}
func (c Gitlab) getAllSnippets() []*gitlab.Snippet {
var allSnippets []*gitlab.Snippet
opt := &gitlab.ListAllSnippetsOptions{
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: 1,
},
}
for {
snippets, resp, err := c.Snippets.ListAllSnippets(opt)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Issue fetching all snippets, not all snippets will be cloned error: %v", err))
return allSnippets
}
allSnippets = append(allSnippets, snippets...)
// Exit the loop when we've seen all pages.
if resp.NextPage == 0 {
break
}
// Update the page number to get the next page.
opt.Page = resp.NextPage
}
return allSnippets
}
func (c Gitlab) GetSnippets(cloneData []Repo, target string) ([]Repo, error) {
if os.Getenv("GHORG_CLONE_SNIPPETS") != "true" {
return []Repo{}, nil
}
var allSnippetsToClone []*gitlab.Snippet
// Snippets are converted into Repos so we can clone them
snippetsToClone := []Repo{}
// If it is a cloud group clone iterate over each project and try to get its snippets. We have to do this because if you use the /snippets/all endpoint it will return every public snippet from the cloud.
if os.Getenv("GHORG_CLONE_TYPE") != "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" {
// Iterate over all projects in the group. If it has snippets add them
colorlog.PrintInfo("Note: only snippets you have access to will be cloned. This process may take a while depending on the size of group you are trying to clone, please be patient.")
for _, repo := range cloneData {
snippets := c.getRepoSnippets(repo)
allSnippetsToClone = append(allSnippetsToClone, snippets...)
}
} else {
allSnippets := c.getAllSnippets()
// if its an all-user or all-group clone, for each repo get its snippets then also include all root level snippets
if target == "all-users" || target == "all-groups" {
for _, repo := range cloneData {
repoSnippets := c.getRepoSnippets(repo)
allSnippetsToClone = append(allSnippetsToClone, repoSnippets...)
}
for _, snippet := range allSnippets {
if c.rootLevelSnippet(snippet.WebURL) {
allSnippetsToClone = append(allSnippetsToClone, snippet)
}
}
}
if os.Getenv("GHORG_CLONE_TYPE") == "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" {
}
}
for _, snippet := range allSnippetsToClone {
snippetID := strconv.Itoa(snippet.ID)
snippetTitle := ToSlug(snippet.Title)
s := Repo{}
s.IsGitLabSnippet = true
s.CloneBranch = "main"
s.GitLabSnippetInfo.Title = snippetTitle
s.Name = snippetTitle
s.GitLabSnippetInfo.ID = snippetID
s.URL = snippet.WebURL
// If the snippet is not made on any repo its a root level snippet, this works for cloud
if c.rootLevelSnippet(snippet.WebURL) {
s.IsGitLabRootLevelSnippet = true
s.CloneURL = c.createRootLevelSnippetCloneURL(snippet.WebURL)
cloneData = append(cloneData, s)
} else {
// Since this isn't a root level repo we want to find which repo the snippet is coming from
for _, cloneTarget := range cloneData {
if cloneTarget.ID == strconv.Itoa(snippet.ProjectID) {
s.CloneURL = c.createRepoSnippetCloneURL(cloneTarget.CloneURL, snippetID)
s.Path = cloneTarget.Path
s.GitLabSnippetInfo.URLOfRepo = cloneTarget.URL
s.GitLabSnippetInfo.NameOfRepo = cloneTarget.Name
cloneData = append(cloneData, s)
}
}
}
snippetsToClone = append(snippetsToClone, s)
}
return snippetsToClone, nil
}
// GetGroupRepos fetches repo data from a specific group
func (c Gitlab) GetGroupRepos(targetGroup string) ([]Repo, error) {
@@ -223,6 +440,11 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
}
}
snippets, err := c.GetSnippets(cloneData, targetUsername)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err))
}
cloneData = append(cloneData, snippets...)
return cloneData, nil
}
@@ -292,6 +514,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {
r := Repo{}
r.Name = p.Name
r.ID = strconv.Itoa(p.ID)
if os.Getenv("GHORG_BRANCH") == "" {
defaultBranch := p.DefaultBranch
@@ -320,6 +543,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {
}
r.Path = path
r.ID = fmt.Sprint(p.ID)
if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
r.CloneURL = c.addTokenToCloneURL(p.HTTPURLToRepo, os.Getenv("GHORG_GITLAB_TOKEN"))
r.URL = p.HTTPURLToRepo
@@ -332,6 +556,8 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {
if p.WikiEnabled && os.Getenv("GHORG_CLONE_WIKI") == "true" {
wiki := Repo{}
// wiki needs name for gitlab name collisions
wiki.Name = p.Name
wiki.IsWiki = true
wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
@@ -348,7 +574,6 @@ func filterGitlabGroupByExcludeMatchRegex(groups []string) []string {
regex := fmt.Sprint(os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX"))
for i, grp := range groups {
fmt.Println(grp)
exclude := false
re := regexp.MustCompile(regex)
match := re.FindString(grp)
@@ -363,3 +588,20 @@ func filterGitlabGroupByExcludeMatchRegex(groups []string) []string {
return filteredGroups
}
// ToSlug converts a title into a URL-friendly slug.
func ToSlug(title string) string {
// Convert to lowercase
slug := strings.ToLower(title)
// Replace spaces and special characters with hyphens
slug = regexp.MustCompile(`[\s\p{P}]+`).ReplaceAllString(slug, "-")
// Remove any non-alphanumeric characters except for hyphens
slug = regexp.MustCompile(`[^a-z0-9-]+`).ReplaceAllString(slug, "")
// Trim any leading or trailing hyphens
slug = strings.Trim(slug, "-")
return slug
}

40
scm/structs.go
View File

@@ -1,12 +1,38 @@
package scm
// Repo represents an SCM repo
// Repo represents an SCM repo, should probably be renamed to "cloneable" since we clone wikis and snippets with this
type Repo struct {
Name string
HostPath string
Path string
URL string
CloneURL string
// The ID of the repo that is assigned via the SCM provider. This is used for example with gitlab snippets on cloud gropus where we need to know the repo id to look up all he snippets it has.
ID string
// Name is the name of the repo https://www.github.com/gabrie30/ghorg.git the Name would be ghorg
Name string
// HostPath is the path on the users machine that the repo will be cloned to. Its used in all the git commands to locate the directory of the repo. HostPath is updated for wikis and snippets because the folder for the clone is appended with .wiki and .snippet
HostPath string
// Path where the repo is located within the scm provider. Its mostly used with gitlab repos when the directory structure is preserved. In this case the path becomes where to locate the repo in relation to gitlab.com/group/group/group/repo.git => group/group/group/repo
Path string
// URL is the web address of the repo
URL string
// CloneURL is the url for cloning the repo, will be different for ssh vs http clones and will have the .git extention
CloneURL string
// CloneBranch the branch to clone. This will be the default branch if not specified. It will always be main for snippets.
CloneBranch string
IsWiki bool
// IsWiki is set to true when the data is for a wiki page
IsWiki bool
// IsGitLabSnippet is set to true when the data is for a gitlab snippet
IsGitLabSnippet bool
// IsGitLabRootLevelSnippet is set to true when a snippet was not created for a repo
IsGitLabRootLevelSnippet bool
// GitLabSnippetInfo provides additional information when the thing we are cloning is a gitlab snippet
GitLabSnippetInfo GitLabSnippet
}
type GitLabSnippet struct {
// GitLab ID of the snippet
ID string
// Title of the snippet
Title string
// URL of the repo that snippet was made on
URLOfRepo string
// Name of the repo that the snippet was made on
NameOfRepo string
}

30
scripts/gitlab_cloud_integration_tests.sh
View File

@@ -96,23 +96,23 @@ else
fi
# SNIPPETS
# ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab --clone-snippets --preserve-dir
ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab --clone-snippets --preserve-dir
# if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-2-3711655 ]
# then
# echo "Pass: gitlab group clone snippet 2 with preserve dir"
# else
# echo "Fail: gitlab group clone snippet 2 with preserve dir"
# exit 1
# fi
if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-2-3711655 ]
then
echo "Pass: gitlab group clone snippet 2 with preserve dir"
else
echo "Fail: gitlab group clone snippet 2 with preserve dir"
exit 1
fi
# if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-1-3711654 ]
# then
# echo "Pass: gitlab group clone snippet 1 with preserve dir"
# else
# echo "Fail: gitlab group clone snippet 1 with preserve dir"
# exit 1
# fi
if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-1-3711654 ]
then
echo "Pass: gitlab group clone snippet 1 with preserve dir"
else
echo "Fail: gitlab group clone snippet 1 with preserve dir"
exit 1
fi
#
# SUBGROUP TESTS

141
scripts/local-gitlab/integration-tests.sh
View File

@@ -31,7 +31,7 @@ export GHORG_INSECURE_GITLAB_CLIENT=true
############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR ############
############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos
@@ -81,6 +81,79 @@ echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED local-gitl
exit 1
fi
############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets
GOT=$( ghorg ls local-gitlab-v15-repos-snippets | grep -o 'local-gitlab-v15-repos-snippets.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-snippets/_ghorg_root_level_snippets
local-gitlab-v15-repos-snippets/local-gitlab-group1
local-gitlab-v15-repos-snippets/local-gitlab-group2
local-gitlab-v15-repos-snippets/local-gitlab-group3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS TEST FAILED local-gitlab-group1"
exit 1
fi
GOT=$( ghorg ls local-gitlab-v15-repos-snippets/local-gitlab-group1 | grep -o 'local-gitlab-v15-repos-snippets/local-gitlab-group1.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-snippets/local-gitlab-group1/baz0
local-gitlab-v15-repos-snippets/local-gitlab-group1/baz1
local-gitlab-v15-repos-snippets/local-gitlab-group1/baz2
local-gitlab-v15-repos-snippets/local-gitlab-group1/baz3
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS TEST FAILED local-gitlab-group1"
exit 1
fi
GOT=$( ghorg ls local-gitlab-v15-repos-snippets/local-gitlab-group2 | grep -o 'local-gitlab-v15-repos-snippets/local-gitlab-group2.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz0
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz0.snippets
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz1
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz1.snippets
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz2
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz2.snippets
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz3
local-gitlab-v15-repos-snippets/local-gitlab-group2/baz3.snippets
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS TEST FAILED local-gitlab-group2"
exit 1
fi
GOT=$( ghorg ls local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a | grep -o 'local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup-b
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_0
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_0.snippets
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_1
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_1.snippets
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_2
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_2.snippets
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_3
local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_3.snippets
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS TEST FAILED local-gitlab-group3/subgroup-a"
exit 1
fi
############ CLONE AND TEST ALL-GROUPS, OUTPUT DIR ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-flat
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-flat
@@ -112,6 +185,50 @@ echo "CLONE AND TEST ALL-GROUPS, OUTPUT DIR"
exit 1
fi
############ CLONE AND TEST ALL-GROUPS, OUTPUT DIR, SNIPPETS ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-all-groups-snippets --clone-snippets
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-all-groups-snippets --clone-snippets
GOT=$( ghorg ls local-gitlab-v15-repos-all-groups-snippets | grep -o 'local-gitlab-v15-repos-all-groups-snippets.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-all-groups-snippets/_ghorg_root_level_snippets
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group1_baz0
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group1_baz1
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group1_baz2
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group1_baz3
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz0
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz0.snippets
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz1
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz1.snippets
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz2
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz2.snippets
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz3
local-gitlab-v15-repos-all-groups-snippets/local-gitlab-group2_baz3.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_0
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_0.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_1
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_1.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_2
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_2.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_3
local-gitlab-v15-repos-all-groups-snippets/subgroup_a_repo_3.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_0
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_0.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_1
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_1.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_2
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_2.snippets
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_3
local-gitlab-v15-repos-all-groups-snippets/subgroup_b_repo_3.snippets
EOF
)
if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, OUTPUT DIR"
exit 1
fi
########### CLONE AND TEST ALL-GROUPS, OUTPUT DIR, WIKI ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --clone-wiki --output-dir=local-gitlab-v15-repos-flat-wiki
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --clone-wiki --output-dir=local-gitlab-v15-repos-flat-wiki
@@ -536,6 +653,7 @@ local-gitlab-v15-all-users/rootrepos0
local-gitlab-v15-all-users/rootrepos1
local-gitlab-v15-all-users/rootrepos2
local-gitlab-v15-all-users/rootrepos3
local-gitlab-v15-all-users/testuser1-repo
EOF
)
@@ -545,4 +663,25 @@ echo "CLONE AND TEST ALL-USERS, OUTPUT DIR FAILED"
exit 1
fi
############ CLONE AND TEST ALL-USERS, OUTPUT DIR, SNIPPETS ############
ghorg clone all-users --scm=gitlab --clone-type=user --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-all-users-snippets --clone-snippets
TEST_ALL_USERS_SNIPPETS_GOT=$(ghorg ls local-gitlab-v15-all-users-snippets | grep -o 'local-gitlab-v15-all-users-snippets.*')
TEST_ALL_USERS_SNIPPETS_WANT=$(cat <<EOF
local-gitlab-v15-all-users-snippets/_ghorg_root_level_snippets
local-gitlab-v15-all-users-snippets/rootrepos0
local-gitlab-v15-all-users-snippets/rootrepos1
local-gitlab-v15-all-users-snippets/rootrepos2
local-gitlab-v15-all-users-snippets/rootrepos3
local-gitlab-v15-all-users-snippets/testuser1-repo
local-gitlab-v15-all-users-snippets/testuser1-repo.snippets
EOF
)
if [ "${TEST_ALL_USERS_SNIPPETS_WANT}" != "${TEST_ALL_USERS_SNIPPETS_GOT}" ]
then
echo "CLONE AND TEST ALL-USERS, OUTPUT DIR SNIPPETS FAILED"
exit 1
fi
echo "INTEGRATOIN TESTS FINISHED..."

16
scripts/local-gitlab/seed.sh
View File

@@ -135,6 +135,22 @@ echo ""
echo ""
sleep 1
# create a repo for testuser1, this user has an id of 2
curl --header "PRIVATE-TOKEN: $TOKEN" -X POST "${GITLAB_URL}/api/v4/projects/user/2?name=testuser1-repo&initialize_with_readme=true"
echo -e "\n\n\n"
sleep 1
# create a snippet for testuser1's repo
SNIPPET_DATA='{"title": "my-first-snippet", "file_name": "snippet.txt", "content": "This is my first snippet", "visibility": "public"}'
curl --request POST --header "PRIVATE-TOKEN: $TOKEN" \
--header "Content-Type: application/json" \
--data "${SNIPPET_DATA}" \
"${GITLAB_URL}/api/v4/projects/testuser1%2Ftestuser1-repo/snippets"
echo -e "\n\n\n"
sleep 1
# create repos in group1
for ((a=0; a <= 3 ; a++))
do