From ac43c9a1fd2579e03ed917990dce2e1cf76ecb2c Mon Sep 17 00:00:00 2001 From: Jay Gabriels Date: Tue, 20 Aug 2019 20:42:30 -0700 Subject: [PATCH] add ghorgignore and skip archived flag --- CHANGELOG.md | 3 +++ README.md | 22 ++++++++++++++++- cmd/clone.go | 58 +++++++++++++++++++++++++++++++++++++++++++++ cmd/clone_github.go | 14 +++++++++++ cmd/clone_gitlab.go | 13 ++++++++++ configs/configs.go | 13 ++++++++-- sample-conf.yaml | 5 ++++ 7 files changed, 125 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2796f18..d2e609e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) ### Added - color flag to toggle colorful output - scm base url flag for gitlab +- ghorgignore to ignore specific repos +- skip archived repos flag +- how to fix ulmits in readme ### Changed ### Deprecated ### Removed diff --git a/README.md b/README.md index d184e5c..4b332fe 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,18 @@ $ security find-internet-password -s gitlab.com | grep "acct" | awk -F\" '{ pri To configure with bitbucket you will need to create a new [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html) and update your `$HOME/ghorg/conf.yaml` or use the (--token, -t) and (--bitbucket-username) flags. +## Ignoring Repos +- To ignore any archived repos while cloning use the `--skip-archived` flag (github/gitlab only) +- To ignore specific repos create a `ghorgignore` file inside `$HOME/ghorg`. Each line in this file is considered a substring and will be compared against each repos clone url fetched. If the clone url contains the substring it will be excluded from cloning. To prevent accidentally excluding a repo, you should make each line as specific as possible, eg. `https://github.com/gabrie30/ghorg.git` or `git@github.com:gabrie30/ghorg.git` depending on how you clone. + + ```bash + # Create ghorgignore + touch $HOME/ghorg/ghorgignore + + # update file + vi $HOME/ghorg/ghorgignore + ``` + ## Known issues - When cloning if you see something like `Username for 'https://gitlab.com': ` the command won't finish. I haven't been able to identify the reason for this occuring. The fix for this is to make sure your token is in the osxkeychain. See the troubleshooting section for how to set this up. @@ -92,7 +104,15 @@ To configure with bitbucket you will need to create a new [app password](https:/ - If the `security` command does not return your token, follow this [GitHub Documentation](https://help.github.com/en/articles/caching-your-github-password-in-git). For GitHub tokens you will need to set your token as your username and set nothing as the password when prompted. For GitLab you will need to set your token for both the username and password when prompted. This will correctly store your credentials in the keychain. If you are still having problems see this [StackOverflow Post](https://stackoverflow.com/questions/31305945/git-clone-from-github-over-https-with-two-factor-authentication) - If your GitHub account is behind 2fa follow this [Github Documentation](https://github.blog/2013-09-03-two-factor-authentication/#how-does-it-work-for-command-line-git) - Make sure your `$ git --version` is >= 2.19.0 -- You may need to increase your ulimits if cloning a large org +- You see (Error: open /dev/null: too many open files) you need to increase your ulimits, there are lots of docs online for this. For mac the quick and dirty is below + + ``` + # reset the soft and hard file limit boundaries + $ sudo launchctl limit maxfiles 65536 200000 + + # actually now set the ulimit boundary + $ ulimit -n 20000 + ``` ### Updating brew tap - [See Readme](https://github.com/gabrie30/homebrew-utils/blob/master/README.md) diff --git a/cmd/clone.go b/cmd/clone.go index 262a3fb..003804a 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -2,6 +2,7 @@ package cmd import ( + "bufio" "fmt" "os" "os/exec" @@ -24,6 +25,7 @@ var ( namespace string color string baseURL string + skipArchived bool args []string ) @@ -40,6 +42,8 @@ func init() { cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)") cloneCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "GHORG_GITLAB_DEFAULT_NAMESPACE - gitlab only: limits clone targets to a specific namespace e.g. --namespace=gitlab-org/security-products") + cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED skips archived repos, github/gitlab only") + cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL change SCM base url, for on self hosted instances (currently gitlab only, use format of https://git.mydomain.com/api/v3)") } @@ -101,6 +105,10 @@ var cloneCmd = &cobra.Command{ os.Setenv("GHORG_SCM_BASE_URL", url) } + if cmd.Flags().Changed("skip-archived") { + os.Setenv("GHORG_SKIP_ARCHIVED", "true") + } + configs.GetOrSetToken() if cmd.Flags().Changed("token") { @@ -219,6 +227,21 @@ func printRemainingMessages(infoMessages []error, errors []error) { } } +func readGhorgIgnore() ([]string, error) { + file, err := os.Open(configs.GhorgIgnoreLocation()) + if err != nil { + return nil, err + } + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, scanner.Err() +} + // CloneAllRepos clones all repos func CloneAllRepos() { resc, errc, infoc := make(chan string), make(chan error), make(chan error) @@ -246,6 +269,38 @@ func CloneAllRepos() { os.Exit(0) } + // filter repos down based on ghorgignore if one exists + _, err = os.Stat(configs.GhorgIgnoreLocation()) + if !os.IsNotExist(err) { + // Open the file parse each line and remove cloneTargets containing + toIgnore, err := readGhorgIgnore() + if err != nil { + colorlog.PrintError("Error parsing your ghorgignore, aborting") + fmt.Println(err) + os.Exit(1) + } + + colorlog.PrintInfo("Using ghorgignore, filtering repos down...") + + filteredCloneTargets := []string{} + var flag bool + for _, cloned := range cloneTargets { + flag = false + for _, ignore := range toIgnore { + if strings.Contains(cloned, ignore) { + flag = true + } + } + + if flag == false { + filteredCloneTargets = append(filteredCloneTargets, cloned) + } + } + + cloneTargets = filteredCloneTargets + + } + colorlog.PrintInfo(strconv.Itoa(len(cloneTargets)) + " repos found in " + args[0]) fmt.Println() @@ -341,6 +396,9 @@ func PrintConfigs() { if os.Getenv("GHORG_SCM_BASE_URL") != "" { colorlog.PrintInfo("* Base URL : " + os.Getenv("GHORG_SCM_BASE_URL")) } + if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" { + colorlog.PrintInfo("* Skip Archived : " + os.Getenv("GHORG_SKIP_ARCHIVED")) + } colorlog.PrintInfo("*************************************") fmt.Println("") } diff --git a/cmd/clone_github.go b/cmd/clone_github.go index 2b22404..625193e 100644 --- a/cmd/clone_github.go +++ b/cmd/clone_github.go @@ -39,6 +39,13 @@ func getGitHubOrgCloneUrls() ([]string, error) { cloneUrls := []string{} for _, repo := range allRepos { + + if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" { + if *repo.Archived == true { + continue + } + } + if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { cloneUrls = append(cloneUrls, *repo.CloneURL) } else { @@ -81,6 +88,13 @@ func getGitHubUserCloneUrls() ([]string, error) { cloneUrls := []string{} for _, repo := range allRepos { + + if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" { + if *repo.Archived == true { + continue + } + } + if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { cloneUrls = append(cloneUrls, *repo.CloneURL) } else { diff --git a/cmd/clone_gitlab.go b/cmd/clone_gitlab.go index 993e335..0842ab9 100644 --- a/cmd/clone_gitlab.go +++ b/cmd/clone_gitlab.go @@ -55,6 +55,12 @@ func getGitLabOrgCloneUrls() ([]string, error) { } } + if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" { + if p.Archived == true { + continue + } + } + if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { cloneUrls = append(cloneUrls, p.HTTPURLToRepo) } else { @@ -105,6 +111,13 @@ func getGitLabUserCloneUrls() ([]string, error) { // List all the projects we've found so far. for _, p := range ps { + + if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" { + if p.Archived == true { + continue + } + } + if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" { cloneUrls = append(cloneUrls, p.HTTPURLToRepo) } else { diff --git a/configs/configs.go b/configs/configs.go index 7d1e0bf..cf2824c 100644 --- a/configs/configs.go +++ b/configs/configs.go @@ -45,7 +45,7 @@ func init() { func initConfig() { viper.SetConfigType("yaml") - viper.AddConfigPath(ghorgDir()) + viper.AddConfigPath(GhorgDir()) viper.SetConfigName("conf") if err := viper.ReadInConfig(); err != nil { @@ -67,6 +67,7 @@ func initConfig() { getOrSetDefaults("GHORG_SCM_TYPE") getOrSetDefaults("GHORG_GITLAB_DEFAULT_NAMESPACE") getOrSetDefaults("GHORG_COLOR") + getOrSetDefaults("GHORG_SKIP_ARCHIVED") // Optionally set getOrSetDefaults("GHORG_GITHUB_TOKEN") getOrSetDefaults("GHORG_GITLAB_TOKEN") @@ -112,13 +113,21 @@ func getOrSetDefaults(envVar string) { os.Setenv(envVar, "unset") case "GHORG_COLOR": os.Setenv(envVar, "on") + case "GHORG_SKIP_ARCHIVED": + os.Setenv(envVar, "false") } } else { os.Setenv(envVar, viper.GetString(envVar)) } } -func ghorgDir() string { +// GhorgIgnoreLocation returns the path of users ghorgignore +func GhorgIgnoreLocation() string { + return GhorgDir() + "/ghorgignore" +} + +// GhorgDir returns the ghorg directory path +func GhorgDir() string { return HomeDir() + "/ghorg" } diff --git a/sample-conf.yaml b/sample-conf.yaml index e7c18c8..ce11c36 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -34,6 +34,11 @@ GHORG_COLOR: # flag (--base-url) GHORG_SCM_BASE_URL: +# Skip archived repos (true/false), currently github/gitlab only +# default: false +# flag (--skip-archived) +GHORG_SKIP_ARCHIVED: + # --- GITHUB SPECIFIC --- # Add your GitHub token