diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ea715..997f54b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) ## [1.7.4] - Unreleased ### Added +- GHORG_CLONE_WIKI to clone wiki pages of repos; thanks @ahmadalli ### Changed ### Deprecated ### Removed diff --git a/cmd/clone.go b/cmd/clone.go index b1a7797..ba93be3 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -90,6 +90,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Setenv("GHORG_NO_CLEAN", "true") } + if cmd.Flags().Changed("clone-wiki") { + os.Setenv("GHORG_CLONE_WIKI", "true") + } + if cmd.Flags().Changed("insecure-gitlab-client") { os.Setenv("GHORG_INSECURE_GITLAB_CLIENT", "true") } @@ -216,8 +220,8 @@ func createDirIfNotExist() { } } -func repoExistsLocally(path string) bool { - if _, err := os.Stat(path); os.IsNotExist(err) { +func repoExistsLocally(repo scm.Repo) bool { + if _, err := os.Stat(repo.HostPath); os.IsNotExist(err) { return false } @@ -285,6 +289,18 @@ func filterByRegex(repos []scm.Repo) []scm.Repo { return filteredRepos } +// exclude wikis from repo count +func getRepoCountOnly(targets []scm.Repo) int { + count := 0 + for _, t := range targets { + if !t.IsWiki { + count++ + } + } + + return count +} + // CloneAllRepos clones all repos func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // resc, errc, infoc := make(chan string), make(chan error), make(chan error) @@ -328,7 +344,15 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } - colorlog.PrintInfo(strconv.Itoa(len(cloneTargets)) + " repos found in " + targetCloneSource) + repoCount := getRepoCountOnly(cloneTargets) + + if os.Getenv("GHORG_CLONE_WIKI") == "true" { + wikiCount := strconv.Itoa(len(cloneTargets) - repoCount) + colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource + ", including " + wikiCount + " enabled wikis") + } else { + colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource) + } + fmt.Println() createDirIfNotExist() @@ -342,6 +366,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { limit := limiter.NewConcurrencyLimiter(l) for _, target := range cloneTargets { appName := getAppNameFromURL(target.URL) + branch := target.CloneBranch repo := target @@ -354,16 +379,29 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder, configs.GetCorrectFilePathSeparator(), path) + if repo.IsWiki { + if !strings.HasSuffix(repo.HostPath, ".wiki") { + repo.HostPath = repo.HostPath + ".wiki" + } + } + if os.Getenv("GHORG_BACKUP") == "true" { repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder+"_backup", configs.GetCorrectFilePathSeparator(), path) } action := "cloning" - if repoExistsLocally(repo.HostPath) { + if repoExistsLocally(repo) { if os.Getenv("GHORG_BACKUP") == "true" { err := git.UpdateRemote(repo) + // Theres no way to tell if a github repo has a wiki to clone + if err != nil && repo.IsWiki { + e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err) + cloneInfos = append(cloneInfos, e) + return + } + if err != nil { e := fmt.Sprintf("Could not update remotes in Repo: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) @@ -373,6 +411,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { action = "fetching" err := git.FetchAll(repo) + // Theres no way to tell if a github repo has a wiki to clone + if err != nil && repo.IsWiki { + e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err) + cloneInfos = append(cloneInfos, e) + return + } + if err != nil { e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) @@ -380,7 +425,6 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } } else { - err := git.Checkout(repo) if err != nil { @@ -420,6 +464,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { err = git.Clone(repo) + // Theres no way to tell if a github repo has a wiki to clone + if err != nil && repo.IsWiki { + e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err) + cloneInfos = append(cloneInfos, e) + return + } + if err != nil { e := fmt.Sprintf("Problem trying to clone Repo: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) @@ -439,6 +490,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // we clone with api-key in clone url err = git.SetOrigin(repo) + // if repo has wiki, but content does not exist this is going to error if err != nil { e := fmt.Sprintf("Problem trying to set remote on Repo: %s Error: %v", repo.URL, err) cloneErrors = append(cloneErrors, e) @@ -496,6 +548,9 @@ func PrintConfigs() { if os.Getenv("GHORG_BACKUP") == "true" { colorlog.PrintInfo("* Backup : " + os.Getenv("GHORG_BACKUP")) } + if os.Getenv("GHORG_CLONE_WIKI") == "true" { + colorlog.PrintInfo("* Wikis : " + os.Getenv("GHORG_CLONE_WIKI")) + } if configs.GhorgIgnoreDetected() { colorlog.PrintInfo("* Ghorgignore : true") } diff --git a/cmd/root.go b/cmd/root.go index 4d2e362..6896061 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -29,6 +29,7 @@ var ( skipForks bool backup bool noClean bool + cloneWiki bool preserveDir bool insecureGitlabClient bool args []string @@ -85,6 +86,8 @@ func getOrSetDefaults(envVar string) { os.Setenv(envVar, "false") case "GHORG_SKIP_FORKS": os.Setenv(envVar, "false") + case "GHORG_CLONE_WIKI": + os.Setenv(envVar, "false") case "GHORG_NO_CLEAN": os.Setenv(envVar, "false") case "GHORG_INSECURE_GITLAB_CLIENT": @@ -147,6 +150,7 @@ func InitConfig() { getOrSetDefaults("GHORG_SKIP_ARCHIVED") getOrSetDefaults("GHORG_SKIP_FORKS") getOrSetDefaults("GHORG_NO_CLEAN") + getOrSetDefaults("GHORG_CLONE_WIKI") getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT") getOrSetDefaults("GHORG_BACKUP") getOrSetDefaults("GHORG_CONCURRENCY") @@ -192,6 +196,7 @@ func init() { cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only") cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos") cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances") + cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo") cloneCmd.Flags().BoolVar(&skipForks, "skip-forks", false, "GHORG_SKIP_FORKS - skips repo if its a fork, github/gitlab/gitea only") cloneCmd.Flags().BoolVar(&preserveDir, "preserve-dir", false, "GHORG_PRESERVE_DIRECTORY_STRUCTURE - clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/unit/subunit/app, gitlab only") cloneCmd.Flags().BoolVar(&backup, "backup", false, "GHORG_BACKUP - backup mode, clone as mirror, no working copy (ignores branch parameter)") diff --git a/cmd/version.go b/cmd/version.go index 8006ed2..8a67161 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,6 +11,6 @@ var versionCmd = &cobra.Command{ Short: "Print the version number of Ghorg", Long: `All software has versions. This is Ghorg's`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v1.7.3") + fmt.Println("v1.7.4") }, } diff --git a/sample-conf.yaml b/sample-conf.yaml index a17d491..d06695a 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -135,6 +135,11 @@ GHORG_MATCH_REGEX: # flag (--no-clean) GHORG_NO_CLEAN: +# Additionally clone the wiki page for repo +# default: false +# flag (--clone-wiki) +GHORG_CLONE_WIKI: + # Specifies the location of your ghorg conf.yaml, allowing you to have many configuration files, or none at all # default: ghorg looks in $HOME/.config/ghorg/conf.yaml, if not set in that location nor as a commandline flag, ghorg will use all default values # NOTE: this cannot be set in the configuration file and only available in ghorg v1.7.0+ diff --git a/scm/gitea.go b/scm/gitea.go index aa023f8..e7a7b25 100644 --- a/scm/gitea.go +++ b/scm/gitea.go @@ -197,6 +197,15 @@ func (c Gitea) filter(rps []*gitea.Repository) (repoData []Repo, err error) { r.URL = rp.SSHURL repoData = append(repoData, r) } + + if rp.HasWiki && os.Getenv("GHORG_CLONE_WIKI") == "true" { + wiki := Repo{} + wiki.IsWiki = true + wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1) + wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1) + wiki.CloneBranch = "master" + repoData = append(repoData, wiki) + } } return repoData, nil } diff --git a/scm/github.go b/scm/github.go index d7c8ded..1223d00 100644 --- a/scm/github.go +++ b/scm/github.go @@ -196,6 +196,15 @@ func (c Github) filter(allRepos []*github.Repository, envTopics []string) []Repo r.URL = *ghRepo.SSHURL repoData = append(repoData, r) } + + if ghRepo.GetHasWiki() && os.Getenv("GHORG_CLONE_WIKI") == "true" { + wiki := Repo{} + wiki.IsWiki = true + wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1) + wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1) + wiki.CloneBranch = "master" + repoData = append(repoData, wiki) + } } return repoData diff --git a/scm/gitlab.go b/scm/gitlab.go index fae6a23..4713928 100644 --- a/scm/gitlab.go +++ b/scm/gitlab.go @@ -269,6 +269,15 @@ func (c Gitlab) filter(ps []*gitlab.Project) []Repo { r.URL = p.SSHURLToRepo repoData = append(repoData, r) } + + if p.WikiEnabled && os.Getenv("GHORG_CLONE_WIKI") == "true" { + wiki := Repo{} + wiki.IsWiki = true + wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1) + wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1) + wiki.CloneBranch = "master" + repoData = append(repoData, wiki) + } } return repoData } diff --git a/scm/structs.go b/scm/structs.go index c22646a..dbb133d 100644 --- a/scm/structs.go +++ b/scm/structs.go @@ -8,4 +8,5 @@ type Repo struct { URL string CloneURL string CloneBranch string + IsWiki bool }