add ghorgignore and skip archived flag

CHANGELOG.md

@@ -7,6 +7,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 ### Added
 - color flag to toggle colorful output
 - scm base url flag for gitlab
+- ghorgignore to ignore specific repos
+- skip archived repos flag
+- how to fix ulmits in readme
 ### Changed
 ### Deprecated
 ### Removed

README.md

@@ -83,6 +83,18 @@ $ security find-internet-password -s gitlab.com  | grep "acct" | awk -F\" '{ pri
 
 To configure with bitbucket you will need to create a new [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html) and update your `$HOME/ghorg/conf.yaml` or use the (--token, -t) and (--bitbucket-username) flags.
 
+## Ignoring Repos
+- To ignore any archived repos while cloning use the `--skip-archived` flag (github/gitlab only)
+- To ignore specific repos create a `ghorgignore` file inside `$HOME/ghorg`. Each line in this file is considered a substring and will be compared against each repos clone url fetched. If the clone url contains the substring it will be excluded from cloning. To prevent accidentally excluding a repo, you should make each line as specific as possible, eg. `https://github.com/gabrie30/ghorg.git` or `git@github.com:gabrie30/ghorg.git` depending on how you clone.
+
+  ```bash
+  # Create ghorgignore
+  touch $HOME/ghorg/ghorgignore
+
+  # update file
+  vi $HOME/ghorg/ghorgignore
+  ```
+
 ## Known issues
 
 - When cloning if you see something like `Username for 'https://gitlab.com': ` the command won't finish. I haven't been able to identify the reason for this occuring. The fix for this is to make sure your token is in the osxkeychain. See the troubleshooting section for how to set this up.
@@ -92,7 +104,15 @@ To configure with bitbucket you will need to create a new [app password](https:/
 - If the `security` command does not return your token, follow this [GitHub Documentation](https://help.github.com/en/articles/caching-your-github-password-in-git). For GitHub tokens you will need to set your token as your username and set nothing as the password when prompted. For GitLab you will need to set your token for both the username and password when prompted. This will correctly store your credentials in the keychain. If you are still having problems see this [StackOverflow Post](https://stackoverflow.com/questions/31305945/git-clone-from-github-over-https-with-two-factor-authentication)
 - If your GitHub account is behind 2fa follow this [Github Documentation](https://github.blog/2013-09-03-two-factor-authentication/#how-does-it-work-for-command-line-git)
 - Make sure your `$ git --version` is >= 2.19.0
-- You may need to increase your ulimits if cloning a large org
+- You see (Error: open /dev/null: too many open files) you need to increase your ulimits, there are lots of docs online for this. For mac the quick and dirty is below
+
+  ```
+  # reset the soft and hard file limit boundaries
+  $ sudo launchctl limit maxfiles 65536 200000
+
+  # actually now set the ulimit boundary
+  $ ulimit -n 20000
+  ```
 
 ### Updating brew tap
 - [See Readme](https://github.com/gabrie30/homebrew-utils/blob/master/README.md)

cmd/clone.go

@@ -2,6 +2,7 @@
 package cmd
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"os/exec"
@@ -24,6 +25,7 @@ var (
 	namespace         string
 	color             string
 	baseURL           string
+	skipArchived      bool
 	args              []string
 )
 
@@ -40,6 +42,8 @@ func init() {
 	cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)")
 	cloneCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "GHORG_GITLAB_DEFAULT_NAMESPACE - gitlab only: limits clone targets to a specific namespace e.g. --namespace=gitlab-org/security-products")
 
+	cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED skips archived repos, github/gitlab only")
+
 	cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL change SCM base url, for on self hosted instances (currently gitlab only, use format of https://git.mydomain.com/api/v3)")
 }
 
@@ -101,6 +105,10 @@ var cloneCmd = &cobra.Command{
 			os.Setenv("GHORG_SCM_BASE_URL", url)
 		}
 
+		if cmd.Flags().Changed("skip-archived") {
+			os.Setenv("GHORG_SKIP_ARCHIVED", "true")
+		}
+
 		configs.GetOrSetToken()
 
 		if cmd.Flags().Changed("token") {
@@ -219,6 +227,21 @@ func printRemainingMessages(infoMessages []error, errors []error) {
 	}
 }
 
+func readGhorgIgnore() ([]string, error) {
+	file, err := os.Open(configs.GhorgIgnoreLocation())
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var lines []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	return lines, scanner.Err()
+}
+
 // CloneAllRepos clones all repos
 func CloneAllRepos() {
 	resc, errc, infoc := make(chan string), make(chan error), make(chan error)
@@ -246,6 +269,38 @@ func CloneAllRepos() {
 		os.Exit(0)
 	}
 
+	// filter repos down based on ghorgignore if one exists
+	_, err = os.Stat(configs.GhorgIgnoreLocation())
+	if !os.IsNotExist(err) {
+		// Open the file parse each line and remove cloneTargets containing
+		toIgnore, err := readGhorgIgnore()
+		if err != nil {
+			colorlog.PrintError("Error parsing your ghorgignore, aborting")
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		colorlog.PrintInfo("Using ghorgignore, filtering repos down...")
+
+		filteredCloneTargets := []string{}
+		var flag bool
+		for _, cloned := range cloneTargets {
+			flag = false
+			for _, ignore := range toIgnore {
+				if strings.Contains(cloned, ignore) {
+					flag = true
+				}
+			}
+
+			if flag == false {
+				filteredCloneTargets = append(filteredCloneTargets, cloned)
+			}
+		}
+
+		cloneTargets = filteredCloneTargets
+
+	}
+
 	colorlog.PrintInfo(strconv.Itoa(len(cloneTargets)) + " repos found in " + args[0])
 	fmt.Println()
 
@@ -341,6 +396,9 @@ func PrintConfigs() {
 	if os.Getenv("GHORG_SCM_BASE_URL") != "" {
 		colorlog.PrintInfo("* Base URL : " + os.Getenv("GHORG_SCM_BASE_URL"))
 	}
+	if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
+		colorlog.PrintInfo("* Skip Archived : " + os.Getenv("GHORG_SKIP_ARCHIVED"))
+	}
 	colorlog.PrintInfo("*************************************")
 	fmt.Println("")
 }

cmd/clone_github.go

@@ -39,6 +39,13 @@ func getGitHubOrgCloneUrls() ([]string, error) {
 	cloneUrls := []string{}
 
 	for _, repo := range allRepos {
+
+		if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
+			if *repo.Archived == true {
+				continue
+			}
+		}
+
 		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
 			cloneUrls = append(cloneUrls, *repo.CloneURL)
 		} else {
@@ -81,6 +88,13 @@ func getGitHubUserCloneUrls() ([]string, error) {
 	cloneUrls := []string{}
 
 	for _, repo := range allRepos {
+
+		if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
+			if *repo.Archived == true {
+				continue
+			}
+		}
+
 		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
 			cloneUrls = append(cloneUrls, *repo.CloneURL)
 		} else {

cmd/clone_gitlab.go

@@ -55,6 +55,12 @@ func getGitLabOrgCloneUrls() ([]string, error) {
 				}
 			}
 
+			if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
+				if p.Archived == true {
+					continue
+				}
+			}
+
 			if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
 				cloneUrls = append(cloneUrls, p.HTTPURLToRepo)
 			} else {
@@ -105,6 +111,13 @@ func getGitLabUserCloneUrls() ([]string, error) {
 
 		// List all the projects we've found so far.
 		for _, p := range ps {
+
+			if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
+				if p.Archived == true {
+					continue
+				}
+			}
+
 			if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
 				cloneUrls = append(cloneUrls, p.HTTPURLToRepo)
 			} else {

configs/configs.go

@@ -45,7 +45,7 @@ func init() {
 func initConfig() {
 
 	viper.SetConfigType("yaml")
-	viper.AddConfigPath(ghorgDir())
+	viper.AddConfigPath(GhorgDir())
 	viper.SetConfigName("conf")
 
 	if err := viper.ReadInConfig(); err != nil {
@@ -67,6 +67,7 @@ func initConfig() {
 	getOrSetDefaults("GHORG_SCM_TYPE")
 	getOrSetDefaults("GHORG_GITLAB_DEFAULT_NAMESPACE")
 	getOrSetDefaults("GHORG_COLOR")
+	getOrSetDefaults("GHORG_SKIP_ARCHIVED")
 	// Optionally set
 	getOrSetDefaults("GHORG_GITHUB_TOKEN")
 	getOrSetDefaults("GHORG_GITLAB_TOKEN")
@@ -112,13 +113,21 @@ func getOrSetDefaults(envVar string) {
 			os.Setenv(envVar, "unset")
 		case "GHORG_COLOR":
 			os.Setenv(envVar, "on")
+		case "GHORG_SKIP_ARCHIVED":
+			os.Setenv(envVar, "false")
 		}
 	} else {
 		os.Setenv(envVar, viper.GetString(envVar))
 	}
 }
 
-func ghorgDir() string {
+// GhorgIgnoreLocation returns the path of users ghorgignore
+func GhorgIgnoreLocation() string {
+	return GhorgDir() + "/ghorgignore"
+}
+
+// GhorgDir returns the ghorg directory path
+func GhorgDir() string {
 	return HomeDir() + "/ghorg"
 }
 

sample-conf.yaml

@@ -34,6 +34,11 @@ GHORG_COLOR:
 # flag (--base-url)
 GHORG_SCM_BASE_URL:
 
+# Skip archived repos (true/false), currently github/gitlab only
+# default: false
+# flag (--skip-archived)
+GHORG_SKIP_ARCHIVED:
+
 # --- GITHUB SPECIFIC ---
 
 # Add your GitHub token