Add/GitHub language filter (#453)

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 - GHORG_NO_DIR_SIZE flag to turn off directory size output which is now enabled by default
 - GHORG_STATS_ENABLED flag to track clone data over time, set to false by default
 - Added two new flags to the `ghorg ls` command: `--long` and `--total`, which provide additional information about the cloned directories.
+- GHORG_GITHUB_FILTER_LANGUAGE to filter clones by repo language
 ### Changed
 ### Deprecated
 ### Removed

cmd/clone.go

@@ -66,6 +66,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Setenv("GHORG_GITHUB_APP_INSTALLATION_ID", cmd.Flag("github-app-installation-id").Value.String())
 	}
 
+	if cmd.Flags().Changed("github-filter-language") {
+		os.Setenv("GHORG_GITHUB_FILTER_LANGUAGE", cmd.Flag("github-filter-language").Value.String())
+	}
+
 	if cmd.Flags().Changed("github-app-id") {
 		os.Setenv("GHORG_GITHUB_APP_ID", cmd.Flag("github-app-id").Value.String())
 	}

cmd/ls.go

@@ -21,6 +21,12 @@ var lsCmd = &cobra.Command{
 	Run:   lsFunc,
 }
 
+var spinningSpinner *spinner.Spinner
+
+func init() {
+	spinningSpinner = spinner.New(spinner.CharSets[14], 100*time.Millisecond)
+}
+
 func lsFunc(cmd *cobra.Command, argz []string) {
 	if len(argz) == 0 {
 		listGhorgHome()
@@ -61,7 +67,6 @@ func listGhorgHome() {
 		return
 	}
 
-	spinningSpinner := spinner.New(spinner.CharSets[14], 100*time.Millisecond)
 	spinningSpinner.Start()
 
 	var totalDirs int
@@ -183,7 +188,6 @@ func listGhorgDir(arg string) {
 		return
 	}
 
-	spinningSpinner := spinner.New(spinner.CharSets[14], 100*time.Millisecond)
 	spinningSpinner.Start()
 
 	var totalDirs int

cmd/root.go

@@ -46,6 +46,7 @@ var (
 	githubAppPemPath             string
 	githubAppInstallationID      string
 	githubUserOption             string
+	githubFilterLanguage         string
 	includeSubmodules            bool
 	skipArchived                 bool
 	skipForks                    bool
@@ -257,6 +258,7 @@ func InitConfig() {
 	getOrSetDefaults("GHORG_TARGET_REPOS_PATH")
 	getOrSetDefaults("GHORG_CLONE_DEPTH")
 	getOrSetDefaults("GHORG_GITHUB_TOKEN")
+	getOrSetDefaults("GHORG_GITHUB_FILTER_LANGUAGE")
 	getOrSetDefaults("GHORG_COLOR")
 	getOrSetDefaults("GHORG_TOPICS")
 	getOrSetDefaults("GHORG_GITLAB_TOKEN")
@@ -342,6 +344,7 @@ func init() {
 	cloneCmd.Flags().StringVarP(&gitFilter, "git-filter", "", "", "GHORG_GIT_FILTER - Allows you to pass arguments to git's filter flag. Useful for filtering out binary objects from repos with --git-filter=blob:none, this requires git version 2.19 or greater.")
 	cloneCmd.Flags().StringVarP(&githubAppPemPath, "github-app-pem-path", "", "", "GHORG_GITHUB_APP_PEM_PATH - Path to your GitHub App PEM file, for authenticating with GitHub App.")
 	cloneCmd.Flags().StringVarP(&githubAppInstallationID, "github-app-installation-id", "", "", "GHORG_GITHUB_APP_INSTALLATION_ID - GitHub App Installation ID, for authenticating with GitHub App.")
+	cloneCmd.Flags().StringVarP(&githubFilterLanguage, "github-filter-language", "", "", "GHORG_GITHUB_FILTER_LANGUAGE - Filter repos by a language. Can be a comma separated value with no spaces.")
 	cloneCmd.Flags().StringVarP(&githubUserOption, "github-user-option", "", "", "GHORG_GITHUB_USER_OPTION - Only available when also using GHORG_CLONE_TYPE: user e.g. --clone-type=user can be one of: all, owner, member (default: owner)")
 	cloneCmd.Flags().StringVarP(&githubAppID, "github-app-id", "", "", "GHORG_GITHUB_APP_ID -  GitHub App ID, for authenticating with GitHub App")
 

sample-conf.yaml

@@ -202,6 +202,11 @@ GHORG_GITHUB_APP_ID:
 # Can be one of: all, owner, member (default: owner)
 GHORG_GITHUB_USER_OPTION:
 
+# Filter repos by a language
+# Can be a comma separated value with no spaces
+# falg (--github-filter-language) e.g.: --github-filter-language=go,ruby,elixir
+GHORG_GITHUB_FILTER_LANGUAGE:
+
 # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 # |G|I|T|L|A|B| |S|P|E|C|I|F|I|C|
 # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

scm/bitbucket.go

@@ -29,6 +29,8 @@ func (_ Bitbucket) GetType() string {
 
 // GetOrgRepos gets org repos
 func (c Bitbucket) GetOrgRepos(targetOrg string) ([]Repo, error) {
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
 	resp, err := c.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: targetOrg})
 	if err != nil {
 		return []Repo{}, err

scm/gitea.go

@@ -34,6 +34,9 @@ func (_ Gitea) GetType() string {
 func (c Gitea) GetOrgRepos(targetOrg string) ([]Repo, error) {
 	repoData := []Repo{}
 
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
+
 	for i := 1; ; i++ {
 		rps, resp, err := c.ListOrgRepos(targetOrg, gitea.ListOrgReposOptions{ListOptions: gitea.ListOptions{
 			Page:     i,
@@ -66,6 +69,9 @@ func (c Gitea) GetOrgRepos(targetOrg string) ([]Repo, error) {
 func (c Gitea) GetUserRepos(targetUsername string) ([]Repo, error) {
 	repoData := []Repo{}
 
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
+
 	for i := 1; ; i++ {
 		rps, resp, err := c.ListUserRepos(targetUsername, gitea.ListReposOptions{ListOptions: gitea.ListOptions{
 			Page:     i,

scm/github.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	"github.com/bradleyfalzon/ghinstallation/v2"
-	"github.com/gabrie30/ghorg/colorlog"
 	"github.com/google/go-github/v62/github"
 	"golang.org/x/oauth2"
 )
@@ -46,10 +45,12 @@ func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) {
 
 	c.SetTokensUsername()
 
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
+
 	// get all pages of results
 	var allRepos []*github.Repository
 	for {
-		pageToPrintMoreInfo := 10
 		repos, resp, err := c.Repositories.ListByOrg(context.Background(), targetOrg, opt)
 
 		if err != nil {
@@ -57,21 +58,9 @@ func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) {
 		}
 		allRepos = append(allRepos, repos...)
 		if resp.NextPage == 0 {
-			// formatting for "Everything is okay, the org just has a lot of repos..."
-			if opt.Page >= pageToPrintMoreInfo {
-				fmt.Println("")
-			}
-
 			break
 		}
 
-		if opt.Page == pageToPrintMoreInfo {
-			fmt.Println("")
-		}
-
-		if opt.Page%pageToPrintMoreInfo == 0 && opt.Page != 0 {
-			colorlog.PrintSubtleInfo("Everything is okay, the org just has a lot of repos...")
-		}
 		opt.Page = resp.NextPage
 	}
 
@@ -84,29 +73,46 @@ func (c Github) GetUserRepos(targetUser string) ([]Repo, error) {
 		c.BaseURL, _ = url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
 	}
 
-	opt := &github.RepositoryListByUserOptions{
-		// https://docs.github.com/en/repositories/creating-and-managing-repositories/about-repositories#about-repository-ownership
-		Type:        os.Getenv("GHORG_GITHUB_USER_OPTION"),
-		ListOptions: github.ListOptions{PerPage: c.perPage},
-	}
+	c.SetTokensUsername()
+
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
 
 	// get all pages of results
 	var allRepos []*github.Repository
+	opt := &github.ListOptions{PerPage: c.perPage, Page: 1}
 
 	for {
+		var repos []*github.Repository
+		var resp *github.Response
+		var err error
 
-		// List the repositories for a user. Passing the empty string will list repositories for the authenticated user.
-		repos, resp, err := c.Repositories.ListByUser(context.Background(), targetUser, opt)
+		if targetUser == tokenUsername {
+
+			authOpt := &github.RepositoryListByAuthenticatedUserOptions{
+				Type:        os.Getenv("GHORG_GITHUB_USER_OPTION"),
+				ListOptions: *opt,
+			}
+			// List repositories for the authenticated user
+			repos, resp, err = c.Repositories.ListByAuthenticatedUser(context.Background(), authOpt)
+		} else {
+			userOpt := &github.RepositoryListByUserOptions{
+				Type:        os.Getenv("GHORG_GITHUB_USER_OPTION"),
+				ListOptions: *opt,
+			}
+			// List repositories for the specified user
+			repos, resp, err = c.Repositories.ListByUser(context.Background(), targetUser, userOpt)
+		}
 
 		if err != nil {
 			return nil, err
 		}
 
-		if targetUser == "" {
+		if targetUser != tokenUsername {
 			userRepos := []*github.Repository{}
 
 			for _, repo := range repos {
-				if *repo.Owner.Type == "User" {
+				if repo.Owner != nil && repo.Owner.Type != nil && *repo.Owner.Type == "User" {
 					userRepos = append(userRepos, repo)
 				}
 			}
@@ -212,6 +218,26 @@ func (c Github) filter(allRepos []*github.Repository) []Repo {
 			continue
 		}
 
+		// NOTE: for some reason forks do not always have a language field set so sometimes they get filtered out
+		if os.Getenv("GHORG_GITHUB_FILTER_LANGUAGE") != "" {
+			if ghRepo.Language != nil {
+				ghLang := strings.ToLower(*ghRepo.Language)
+				userLangs := strings.Split(strings.ToLower(os.Getenv("GHORG_GITHUB_FILTER_LANGUAGE")), ",")
+				matched := false
+				for _, userLang := range userLangs {
+					if ghLang == userLang {
+						matched = true
+						break
+					}
+				}
+				if !matched {
+					continue
+				}
+			} else {
+				continue
+			}
+		}
+
 		r := Repo{}
 
 		r.Name = *ghRepo.Name

scm/gitlab.go

@@ -62,6 +62,9 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
 		colorlog.PrintErrorAndExit("When using the 'all-users' keyword the '--clone-type=user' flag should be set")
 	}
 
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
+
 	if targetOrg == "all-groups" {
 		gitLabAllGroups = true
 		longFetch = true
@@ -83,6 +86,7 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
 
 	for i, group := range allGroups {
 		if longFetch {
+			spinningSpinner.Stop()
 			if i == 0 {
 				fmt.Println("")
 			}
@@ -100,6 +104,7 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
 
 	snippets, err := c.GetSnippets(repoData, targetOrg)
 	if err != nil {
+		spinningSpinner.Stop()
 		colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err))
 	}
 	repoData = append(repoData, snippets...)
@@ -394,6 +399,9 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
 		},
 	}
 
+	spinningSpinner.Start()
+	defer spinningSpinner.Stop()
+
 	if targetUsername == "all-users" {
 		gitLabAllUsers = true
 		for {
@@ -422,7 +430,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
 			// Get the first page with projects.
 			ps, resp, err := c.Projects.ListUserProjects(targetUser, projectOpts)
 			if err != nil {
-				fmt.Printf("Error getting repo for user: %v", targetUser)
+				spinningSpinner.Stop()
 				colorlog.PrintError(fmt.Sprintf("Error getting repo for user: %v", targetUser))
 				continue
 			}
@@ -442,6 +450,7 @@ func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
 
 	snippets, err := c.GetSnippets(cloneData, targetUsername)
 	if err != nil {
+		spinningSpinner.Stop()
 		colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err))
 	}
 	cloneData = append(cloneData, snippets...)

scm/spinner.go

@@ -0,0 +1,15 @@
+package scm
+
+import (
+	"time"
+
+	"github.com/briandowns/spinner"
+)
+
+var (
+	spinningSpinner *spinner.Spinner
+)
+
+func init() {
+	spinningSpinner = spinner.New(spinner.CharSets[14], 100*time.Millisecond)
+}