Add ghorg fetch all (#170)

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 ## [1.7.5] - unreleased
 ### Added
 - GHORG_DRY_RUN to do dry runs on clones
+- GHORG_FETCH_ALL to run fetch all on each repo
 - output for long running repo fetches
 - support for cloning github enterprise repos
 - log repos cloned vs pulled at end of run

cmd/clone.go

@@ -90,6 +90,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Setenv("GHORG_NO_CLEAN", "true")
 	}
 
+	if cmd.Flags().Changed("fetch-all") {
+		os.Setenv("GHORG_FETCH_ALL", "true")
+	}
+
 	if cmd.Flags().Changed("dry-run") {
 		os.Setenv("GHORG_DRY_RUN", "true")
 	}
@@ -480,6 +484,16 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 					action = "pulling"
 					pulledCount++
+
+					if os.Getenv("GHORG_FETCH_ALL") == "true" {
+						err = git.FetchAll(repo)
+
+						if err != nil {
+							e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err)
+							cloneErrors = append(cloneErrors, e)
+							return
+						}
+					}
 				}
 			} else {
 				// if https clone and github/gitlab add personal access token to url
@@ -520,6 +534,16 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 					cloneErrors = append(cloneErrors, e)
 					return
 				}
+
+				if os.Getenv("GHORG_FETCH_ALL") == "true" {
+					err = git.FetchAll(repo)
+
+					if err != nil {
+						e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err)
+						cloneErrors = append(cloneErrors, e)
+						return
+					}
+				}
 			}
 
 			colorlog.PrintSuccess("Success " + action + " repo: " + repo.URL + " -> branch: " + branch)
@@ -591,6 +615,12 @@ func PrintConfigs() {
 	if os.Getenv("GHORG_NO_CLEAN") == "true" {
 		colorlog.PrintInfo("* No Clean      : " + "true")
 	}
+	if os.Getenv("GHORG_FETCH_ALL") == "true" {
+		colorlog.PrintInfo("* Fetch All     : " + "true")
+	}
+	if os.Getenv("GHORG_DRY_RUN") == "true" {
+		colorlog.PrintInfo("* Dry Run       : " + "true")
+	}
 
 	colorlog.PrintInfo("* Config Used   : " + os.Getenv("GHORG_CONF"))
 

cmd/root.go

@@ -19,7 +19,6 @@ var (
 	cloneType            string
 	scmType              string
 	bitbucketUsername    string
-	namespace            string
 	color                string
 	baseURL              string
 	concurrency          string
@@ -33,6 +32,7 @@ var (
 	cloneWiki            bool
 	preserveDir          bool
 	insecureGitlabClient bool
+	fetchAll             bool
 	args                 []string
 	cloneErrors          []string
 	cloneInfos           []string
@@ -91,6 +91,8 @@ func getOrSetDefaults(envVar string) {
 			os.Setenv(envVar, "false")
 		case "GHORG_NO_CLEAN":
 			os.Setenv(envVar, "false")
+		case "GHORG_FETCH_ALL":
+			os.Setenv(envVar, "false")
 		case "GHORG_DRY_RUN":
 			os.Setenv(envVar, "false")
 		case "GHORG_INSECURE_GITLAB_CLIENT":
@@ -153,6 +155,8 @@ func InitConfig() {
 	getOrSetDefaults("GHORG_SKIP_ARCHIVED")
 	getOrSetDefaults("GHORG_SKIP_FORKS")
 	getOrSetDefaults("GHORG_NO_CLEAN")
+	getOrSetDefaults("GHORG_FETCH_ALL")
+	getOrSetDefaults("GHORG_DRY_RUN")
 	getOrSetDefaults("GHORG_CLONE_WIKI")
 	getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT")
 	getOrSetDefaults("GHORG_BACKUP")
@@ -198,6 +202,7 @@ func init() {
 	cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)")
 	cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only")
 	cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos")
+	cloneCmd.Flags().BoolVar(&fetchAll, "fetch-all", false, "GHORG_FETCH_ALL - fetches all remote branches for each repo by running a git fetch --all")
 	cloneCmd.Flags().BoolVar(&dryRun, "dry-run", false, "GHORG_DRY_RUN - perform a dry run of the clone; fetches repos but does not clone them")
 	cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances")
 	cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo")

sample-conf.yaml

@@ -21,7 +21,7 @@ GHORG_GITLAB_TOKEN:
 
 # clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/org/unit/subunit/app
 # flag (--preserve-dir)
-GHORG_PRESERVE_DIRECTORY_STRUCTURE:
+GHORG_PRESERVE_DIRECTORY_STRUCTURE: false
 
 # Skip TLS certificate verification for hosted gitlab instances
 # flag (--insecure-gitlab-client)
@@ -60,12 +60,12 @@ GHORG_SCM_BASE_URL:
 # Which provider to clone from (github, gitlab, gitea or bitbucket)
 # default: github
 # flag (--scm, -s) eg: --scm=gitlab
-GHORG_SCM_TYPE:
+GHORG_SCM_TYPE: github
 
 # How you'd like to clone the repos (https or ssh)
 # default: https
 # flag (--protocol) eg: --protocol=https
-GHORG_CLONE_PROTOCOL:
+GHORG_CLONE_PROTOCOL: https
 
 # This is where your ghorg directory will be created, use absolute pathing, shell expansions will not work
 # default: YOUR_HOME_DIR/ghorg
@@ -79,37 +79,34 @@ GHORG_ABSOLUTE_PATH_TO_CLONE_TO:
 GHORG_BRANCH:
 
 # Type of entity to clone (user or org)
-# default: org
 # flag (--clone-type, -c) eg: --clone-type=user
-GHORG_CLONE_TYPE:
+GHORG_CLONE_TYPE: org
 
 # Color output (enabled, disabled)
-# default: disabled
 # flag (--color) eg: --color=disabled
-GHORG_COLOR:
+GHORG_COLOR: disabled
 
 # A comma separated list of topics to filter by, currently github/gitea only
-# default: ""
 # flag (--topics) eg: --topics=frontend
 # If any topics exist here, ghorg will only clone repos that match at least one of these topics
 GHORG_TOPICS:
 
 # Skip archived repos, currently github/gitlab/gitea only
 # flag (--skip-archived)
-GHORG_SKIP_ARCHIVED:
+GHORG_SKIP_ARCHIVED: false
 
 # Skip repos that are forks, currently github/gitlab/gitea only
 # flag (--skip-forks)
-GHORG_SKIP_FORKS:
+GHORG_SKIP_FORKS: false
 
 # Backup mode, clone as mirror, no working copy (ignores branch parameter)
 # flag (--backup)
-GHORG_BACKUP:
+GHORG_BACKUP: false
 
 # Max goroutines created while cloning
 # default: 25
 # flag (--concurrency)
-GHORG_CONCURRENCY:
+GHORG_CONCURRENCY: 25
 
 # Folder ghorg will clone all repos into. Cloning will result in: GHORG_ABSOLUTE_PATH_TO_CLONE_TO/GHORG_OUTPUT_DIR/*
 # default: {org/user you are cloning}
@@ -128,15 +125,19 @@ GHORG_MATCH_REGEX:
 
 # Only clones new repos and does not perform a git clean on existing repos. Useful if you don't want to lose changes made to repos in the org/user directory.
 # flag (--no-clean)
-GHORG_NO_CLEAN:
+GHORG_NO_CLEAN: false
 
 # Additionally clone the wiki page for repo
 # flag (--clone-wiki)
-GHORG_CLONE_WIKI:
+GHORG_CLONE_WIKI: false
 
 # Perform a dry run of the clone; fetches repos but does not clone them.
 # flag (--dry-run)
-GHORG_DRY_RUN:
+GHORG_DRY_RUN: false
+
+# Perform a dry run of the clone; fetches repos but does not clone them.
+# flag (--fetch-all)
+GHORG_FETCH_ALL: false
 
 # Specifies the location of your ghorg conf.yaml, allowing you to have many configuration files, or none at all
 # default: ghorg looks in $HOME/.config/ghorg/conf.yaml, if not set in that location nor as a commandline flag, ghorg will use all default values