Add GHORG_DRY_RUN

CHANGELOG.md

@@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
+## [1.7.5] - unreleased
+### Added
+- GHORG_DRY_RUN to do dry runs on clones
+- output for long running repo fetches
+### Changed
+### Deprecated
+### Removed
+### Fixed
+- Setting new gitlab token check from config file; thanks @vegas1880
+### Security
 ## [1.7.4] - 11/11/21
 ### Added
 - GHORG_CLONE_WIKI to clone wiki pages of repos; thanks @ahmadalli

cmd/clone.go

@@ -90,6 +90,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Setenv("GHORG_NO_CLEAN", "true")
 	}
 
+	if cmd.Flags().Changed("dry-run") {
+		os.Setenv("GHORG_DRY_RUN", "true")
+	}
+
 	if cmd.Flags().Changed("clone-wiki") {
 		os.Setenv("GHORG_CLONE_WIKI", "true")
 	}
@@ -301,6 +305,15 @@ func getRepoCountOnly(targets []scm.Repo) int {
 	return count
 }
 
+func printDryRun(repos []scm.Repo) {
+	for _, repo := range repos {
+		colorlog.PrintSubtleInfo(repo.URL)
+	}
+	fmt.Println()
+	count := len(repos)
+	colorlog.PrintSuccess(fmt.Sprintf("%v repos to be cloned into: %s%s", count, os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder))
+}
+
 // CloneAllRepos clones all repos
 func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 	// resc, errc, infoc := make(chan string), make(chan error), make(chan error)
@@ -355,6 +368,11 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 	fmt.Println()
 
+	if os.Getenv("GHORG_DRY_RUN") == "true" {
+		printDryRun(cloneTargets)
+		return
+	}
+
 	createDirIfNotExist()
 
 	l, err := strconv.Atoi(os.Getenv("GHORG_CONCURRENCY"))
@@ -509,8 +527,10 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 	// TODO: fix all these if else checks with ghorg_backups
 	if os.Getenv("GHORG_BACKUP") == "true" {
+		fmt.Println()
 		colorlog.PrintSuccess(fmt.Sprintf("Finished! %s%s_backup", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder))
 	} else {
+		fmt.Println()
 		colorlog.PrintSuccess(fmt.Sprintf("Finished! %s%s", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder))
 	}
 }

cmd/root.go

@@ -29,6 +29,7 @@ var (
 	skipForks            bool
 	backup               bool
 	noClean              bool
+	dryRun               bool
 	cloneWiki            bool
 	preserveDir          bool
 	insecureGitlabClient bool
@@ -90,6 +91,8 @@ func getOrSetDefaults(envVar string) {
 			os.Setenv(envVar, "false")
 		case "GHORG_NO_CLEAN":
 			os.Setenv(envVar, "false")
+		case "GHORG_DRY_RUN":
+			os.Setenv(envVar, "false")
 		case "GHORG_INSECURE_GITLAB_CLIENT":
 			os.Setenv(envVar, "false")
 		case "GHORG_BACKUP":
@@ -195,6 +198,7 @@ func init() {
 	cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)")
 	cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only")
 	cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos")
+	cloneCmd.Flags().BoolVar(&dryRun, "dry-run", false, "GHORG_DRY_RUN - perform a dry run of the clone; fetches repos but does not clone them")
 	cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances")
 	cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo")
 	cloneCmd.Flags().BoolVar(&skipForks, "skip-forks", false, "GHORG_SKIP_FORKS - skips repo if its a fork, github/gitlab/gitea only")

cmd/version.go

@@ -11,6 +11,6 @@ var versionCmd = &cobra.Command{
 	Short: "Print the version number of Ghorg",
 	Long:  `All software has versions. This is Ghorg's`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v1.7.4")
+		fmt.Println("v1.7.5")
 	},
 }

sample-conf.yaml

@@ -20,13 +20,11 @@ GHORG_GITHUB_TOKEN:
 GHORG_GITLAB_TOKEN:
 
 # clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/org/unit/subunit/app
-# default: false
 # flag (--preserve-dir)
 GHORG_PRESERVE_DIRECTORY_STRUCTURE:
 
 # Skip TLS certificate verification for hosted gitlab instances
 # flag (--insecure-gitlab-client)
-# default: false
 GHORG_INSECURE_GITLAB_CLIENT:
 
 # +-+-+-+-+-+ +-+-+-+-+-+-+-+-+
@@ -47,7 +45,7 @@ GHORG_GITEA_TOKEN:
 # flag (--token, -t) eg: --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2
 GHORG_BITBUCKET_APP_PASSWORD:
 
-# flag (--bitbucket-username)
+# flag (--bitbucket-username) eg: --bitbucket-username=user123
 GHORG_BITBUCKET_USERNAME:
 
 # +-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -56,59 +54,55 @@ GHORG_BITBUCKET_USERNAME:
 
 # Change SCM base url, for on self hosted instances, gitea require this
 # default: uses github/gitlab public api
-# flag (--base-url)
+# flag (--base-url) eg: --base-url=https://my.instance.com
 GHORG_SCM_BASE_URL:
 
 # Which provider to clone from (github, gitlab, gitea or bitbucket)
 # default: github
-# flag (--scm, -s)
+# flag (--scm, -s) eg: --scm=gitlab
 GHORG_SCM_TYPE:
 
 # How you'd like to clone the repos (https or ssh)
 # default: https
-# flag (--protocol)
+# flag (--protocol) eg: --protocol=https
 GHORG_CLONE_PROTOCOL:
 
 # This is where your ghorg directory will be created, use absolute pathing, shell expansions will not work
 # default: YOUR_HOME_DIR/ghorg
-# flag (--path, -p)
+# flag (--path, -p) eg: --path=/tmp/ghorg
 GHORG_ABSOLUTE_PATH_TO_CLONE_TO:
 
 # Branch ghorg resets and leaves checked out
 # default: default branch
 # NOTE: if no default branch is found on the repo, will fallback to using master
-# flag (--branch, -b)
+# flag (--branch, -b) eg: --branch=develop
 GHORG_BRANCH:
 
 # Type of entity to clone (user or org)
 # default: org
-# to clone all of your own repos from github don't specify any user eg. "ghorg clone --clone-type=user --output-dir=personal"
-# flag (--clone-type, -c)
+# flag (--clone-type, -c) eg: --clone-type=user
 GHORG_CLONE_TYPE:
 
 # Color output (enabled, disabled)
 # default: disabled
-# flag (--color)
+# flag (--color) eg: --color=disabled
 GHORG_COLOR:
 
 # A comma separated list of topics to filter by, currently github/gitea only
 # default: ""
-# flag (--topics)
+# flag (--topics) eg: --topics=frontend
 # If any topics exist here, ghorg will only clone repos that match at least one of these topics
 GHORG_TOPICS:
 
-# Skip archived repos (true/false), currently github/gitlab/gitea only
-# default: false
+# Skip archived repos, currently github/gitlab/gitea only
 # flag (--skip-archived)
 GHORG_SKIP_ARCHIVED:
 
-# Skip repos that are forks (true/false), currently github/gitlab/gitea only
-# default: false
+# Skip repos that are forks, currently github/gitlab/gitea only
 # flag (--skip-forks)
 GHORG_SKIP_FORKS:
 
-# Backup mode, clone as mirror, no working copy (ignores branch parameter) (true/false)
-# default: false
+# Backup mode, clone as mirror, no working copy (ignores branch parameter)
 # flag (--backup)
 GHORG_BACKUP:
 
@@ -119,28 +113,31 @@ GHORG_CONCURRENCY:
 
 # Folder ghorg will clone all repos into. Cloning will result in: GHORG_ABSOLUTE_PATH_TO_CLONE_TO/GHORG_OUTPUT_DIR/*
 # default: {org/user you are cloning}
-# flag (--output-dir)
+# flag (--output-dir) eg: --output-dir=testing
 GHORG_OUTPUT_DIR:
 
 # Only clone repos with matching prefix, can be a comma separated list
 # default: ""
-# flag (--match-prefix)
+# flag (--match-prefix) eg: --match-prefix=backend
 GHORG_MATCH_PREFIX:
 
 # Only clone repos that match name to regex provided
-# flag (--match-regex)
+# default: match all
+# flag (--match-regex) eg: --match-regex=^foo
 GHORG_MATCH_REGEX:
 
 # Only clones new repos and does not perform a git clean on existing repos. Useful if you don't want to lose changes made to repos in the org/user directory.
-# default: false
 # flag (--no-clean)
 GHORG_NO_CLEAN:
 
 # Additionally clone the wiki page for repo
-# default: false
 # flag (--clone-wiki)
 GHORG_CLONE_WIKI:
 
+# Perform a dry run of the clone; fetches repos but does not clone them.
+# flag (--dry-run)
+GHORG_DRY_RUN:
+
 # Specifies the location of your ghorg conf.yaml, allowing you to have many configuration files, or none at all
 # default: ghorg looks in $HOME/.config/ghorg/conf.yaml, if not set in that location nor as a commandline flag, ghorg will use all default values
 # NOTE: this cannot be set in the configuration file and only available in ghorg v1.7.0+

scm/github.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/gabrie30/ghorg/colorlog"
 	"github.com/google/go-github/v32/github"
 	"golang.org/x/oauth2"
 )
@@ -56,6 +57,9 @@ func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) {
 			break
 		}
 
+		if opt.Page%12 == 0 && opt.Page != 0 {
+			colorlog.PrintSubtleInfo("Everything is okay, its just a lot of repos...")
+		}
 		opt.Page = resp.NextPage
 	}