Add GHORG_CLONE_WIKI to clone wiki pages (#161)

CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
 ## [1.7.4] - Unreleased
 ### Added
+- GHORG_CLONE_WIKI to clone wiki pages of repos; thanks @ahmadalli
 ### Changed
 ### Deprecated
 ### Removed

cmd/clone.go

@@ -90,6 +90,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Setenv("GHORG_NO_CLEAN", "true")
 	}
 
+	if cmd.Flags().Changed("clone-wiki") {
+		os.Setenv("GHORG_CLONE_WIKI", "true")
+	}
+
 	if cmd.Flags().Changed("insecure-gitlab-client") {
 		os.Setenv("GHORG_INSECURE_GITLAB_CLIENT", "true")
 	}
@@ -216,8 +220,8 @@ func createDirIfNotExist() {
 	}
 }
 
-func repoExistsLocally(path string) bool {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
+func repoExistsLocally(repo scm.Repo) bool {
+	if _, err := os.Stat(repo.HostPath); os.IsNotExist(err) {
 		return false
 	}
 
@@ -285,6 +289,18 @@ func filterByRegex(repos []scm.Repo) []scm.Repo {
 	return filteredRepos
 }
 
+// exclude wikis from repo count
+func getRepoCountOnly(targets []scm.Repo) int {
+	count := 0
+	for _, t := range targets {
+		if !t.IsWiki {
+			count++
+		}
+	}
+
+	return count
+}
+
 // CloneAllRepos clones all repos
 func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 	// resc, errc, infoc := make(chan string), make(chan error), make(chan error)
@@ -328,7 +344,15 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 	}
 
-	colorlog.PrintInfo(strconv.Itoa(len(cloneTargets)) + " repos found in " + targetCloneSource)
+	repoCount := getRepoCountOnly(cloneTargets)
+
+	if os.Getenv("GHORG_CLONE_WIKI") == "true" {
+		wikiCount := strconv.Itoa(len(cloneTargets) - repoCount)
+		colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource + ", including " + wikiCount + " enabled wikis")
+	} else {
+		colorlog.PrintInfo(strconv.Itoa(repoCount) + " repos found in " + targetCloneSource)
+	}
+
 	fmt.Println()
 
 	createDirIfNotExist()
@@ -342,6 +366,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 	limit := limiter.NewConcurrencyLimiter(l)
 	for _, target := range cloneTargets {
 		appName := getAppNameFromURL(target.URL)
+
 		branch := target.CloneBranch
 		repo := target
 
@@ -354,16 +379,29 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 			repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder, configs.GetCorrectFilePathSeparator(), path)
 
+			if repo.IsWiki {
+				if !strings.HasSuffix(repo.HostPath, ".wiki") {
+					repo.HostPath = repo.HostPath + ".wiki"
+				}
+			}
+
 			if os.Getenv("GHORG_BACKUP") == "true" {
 				repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder+"_backup", configs.GetCorrectFilePathSeparator(), path)
 			}
 
 			action := "cloning"
 
-			if repoExistsLocally(repo.HostPath) {
+			if repoExistsLocally(repo) {
 				if os.Getenv("GHORG_BACKUP") == "true" {
 					err := git.UpdateRemote(repo)
 
+					// Theres no way to tell if a github repo has a wiki to clone
+					if err != nil && repo.IsWiki {
+						e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err)
+						cloneInfos = append(cloneInfos, e)
+						return
+					}
+
 					if err != nil {
 						e := fmt.Sprintf("Could not update remotes in Repo: %s Error: %v", repo.URL, err)
 						cloneErrors = append(cloneErrors, e)
@@ -373,6 +411,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 					action = "fetching"
 					err := git.FetchAll(repo)
 
+					// Theres no way to tell if a github repo has a wiki to clone
+					if err != nil && repo.IsWiki {
+						e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err)
+						cloneInfos = append(cloneInfos, e)
+						return
+					}
+
 					if err != nil {
 						e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err)
 						cloneErrors = append(cloneErrors, e)
@@ -380,7 +425,6 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 					}
 
 				} else {
-
 					err := git.Checkout(repo)
 
 					if err != nil {
@@ -420,6 +464,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 				err = git.Clone(repo)
 
+				// Theres no way to tell if a github repo has a wiki to clone
+				if err != nil && repo.IsWiki {
+					e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on Repo: %s Error: %v", repo.URL, err)
+					cloneInfos = append(cloneInfos, e)
+					return
+				}
+
 				if err != nil {
 					e := fmt.Sprintf("Problem trying to clone Repo: %s Error: %v", repo.URL, err)
 					cloneErrors = append(cloneErrors, e)
@@ -439,6 +490,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 				// we clone with api-key in clone url
 				err = git.SetOrigin(repo)
 
+				// if repo has wiki, but content does not exist this is going to error
 				if err != nil {
 					e := fmt.Sprintf("Problem trying to set remote on Repo: %s Error: %v", repo.URL, err)
 					cloneErrors = append(cloneErrors, e)
@@ -496,6 +548,9 @@ func PrintConfigs() {
 	if os.Getenv("GHORG_BACKUP") == "true" {
 		colorlog.PrintInfo("* Backup        : " + os.Getenv("GHORG_BACKUP"))
 	}
+	if os.Getenv("GHORG_CLONE_WIKI") == "true" {
+		colorlog.PrintInfo("* Wikis         : " + os.Getenv("GHORG_CLONE_WIKI"))
+	}
 	if configs.GhorgIgnoreDetected() {
 		colorlog.PrintInfo("* Ghorgignore   : true")
 	}

cmd/root.go

@@ -29,6 +29,7 @@ var (
 	skipForks            bool
 	backup               bool
 	noClean              bool
+	cloneWiki            bool
 	preserveDir          bool
 	insecureGitlabClient bool
 	args                 []string
@@ -85,6 +86,8 @@ func getOrSetDefaults(envVar string) {
 			os.Setenv(envVar, "false")
 		case "GHORG_SKIP_FORKS":
 			os.Setenv(envVar, "false")
+		case "GHORG_CLONE_WIKI":
+			os.Setenv(envVar, "false")
 		case "GHORG_NO_CLEAN":
 			os.Setenv(envVar, "false")
 		case "GHORG_INSECURE_GITLAB_CLIENT":
@@ -147,6 +150,7 @@ func InitConfig() {
 	getOrSetDefaults("GHORG_SKIP_ARCHIVED")
 	getOrSetDefaults("GHORG_SKIP_FORKS")
 	getOrSetDefaults("GHORG_NO_CLEAN")
+	getOrSetDefaults("GHORG_CLONE_WIKI")
 	getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT")
 	getOrSetDefaults("GHORG_BACKUP")
 	getOrSetDefaults("GHORG_CONCURRENCY")
@@ -192,6 +196,7 @@ func init() {
 	cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only")
 	cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos")
 	cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances")
+	cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo")
 	cloneCmd.Flags().BoolVar(&skipForks, "skip-forks", false, "GHORG_SKIP_FORKS - skips repo if its a fork, github/gitlab/gitea only")
 	cloneCmd.Flags().BoolVar(&preserveDir, "preserve-dir", false, "GHORG_PRESERVE_DIRECTORY_STRUCTURE - clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/unit/subunit/app, gitlab only")
 	cloneCmd.Flags().BoolVar(&backup, "backup", false, "GHORG_BACKUP - backup mode, clone as mirror, no working copy (ignores branch parameter)")

cmd/version.go

@@ -11,6 +11,6 @@ var versionCmd = &cobra.Command{
 	Short: "Print the version number of Ghorg",
 	Long:  `All software has versions. This is Ghorg's`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v1.7.3")
+		fmt.Println("v1.7.4")
 	},
 }

sample-conf.yaml

@@ -135,6 +135,11 @@ GHORG_MATCH_REGEX:
 # flag (--no-clean)
 GHORG_NO_CLEAN:
 
+# Additionally clone the wiki page for repo
+# default: false
+# flag (--clone-wiki)
+GHORG_CLONE_WIKI:
+
 # Specifies the location of your ghorg conf.yaml, allowing you to have many configuration files, or none at all
 # default: ghorg looks in $HOME/.config/ghorg/conf.yaml, if not set in that location nor as a commandline flag, ghorg will use all default values
 # NOTE: this cannot be set in the configuration file and only available in ghorg v1.7.0+

scm/gitea.go

@@ -197,6 +197,15 @@ func (c Gitea) filter(rps []*gitea.Repository) (repoData []Repo, err error) {
 			r.URL = rp.SSHURL
 			repoData = append(repoData, r)
 		}
+
+		if rp.HasWiki && os.Getenv("GHORG_CLONE_WIKI") == "true" {
+			wiki := Repo{}
+			wiki.IsWiki = true
+			wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
+			wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
+			wiki.CloneBranch = "master"
+			repoData = append(repoData, wiki)
+		}
 	}
 	return repoData, nil
 }

scm/github.go

@@ -196,6 +196,15 @@ func (c Github) filter(allRepos []*github.Repository, envTopics []string) []Repo
 			r.URL = *ghRepo.SSHURL
 			repoData = append(repoData, r)
 		}
+
+		if ghRepo.GetHasWiki() && os.Getenv("GHORG_CLONE_WIKI") == "true" {
+			wiki := Repo{}
+			wiki.IsWiki = true
+			wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
+			wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
+			wiki.CloneBranch = "master"
+			repoData = append(repoData, wiki)
+		}
 	}
 
 	return repoData

scm/gitlab.go

@@ -269,6 +269,15 @@ func (c Gitlab) filter(ps []*gitlab.Project) []Repo {
 			r.URL = p.SSHURLToRepo
 			repoData = append(repoData, r)
 		}
+
+		if p.WikiEnabled && os.Getenv("GHORG_CLONE_WIKI") == "true" {
+			wiki := Repo{}
+			wiki.IsWiki = true
+			wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
+			wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
+			wiki.CloneBranch = "master"
+			repoData = append(repoData, wiki)
+		}
 	}
 	return repoData
 }

scm/structs.go

@@ -8,4 +8,5 @@ type Repo struct {
 	URL         string
 	CloneURL    string
 	CloneBranch string
+	IsWiki      bool
 }