ghorg/cmd/clone.go

// Package cmd encapsulates the logic for all cli commands
package cmd

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/gabrie30/ghorg/colorlog"
	"github.com/gabrie30/ghorg/configs"
	"github.com/korovkin/limiter"
	"github.com/spf13/cobra"
)

var (
	protocol          string
	path              string
	branch            string
	token             string
	cloneType         string
	scmType           string
	bitbucketUsername string
	namespace         string
	color             string
	baseURL           string
	concurrency       string
	skipArchived      bool
	backup            bool
	args              []string
	cloneErrors       []string
	cloneInfos        []string
)

func init() {
	rootCmd.PersistentFlags().StringVarP(&color, "color", "", "", "GHORG_COLOR - toggles colorful output on/off (default on)")
	rootCmd.AddCommand(cloneCmd)
	cloneCmd.Flags().StringVar(&protocol, "protocol", "", "GHORG_CLONE_PROTOCOL - protocol to clone with, ssh or https, (default https)")
	cloneCmd.Flags().StringVarP(&path, "path", "p", "", "GHORG_ABSOLUTE_PATH_TO_CLONE_TO - absolute path the ghorg_* directory will be created. Must end with / (default $HOME/Desktop/)")
	cloneCmd.Flags().StringVarP(&branch, "branch", "b", "", "GHORG_BRANCH - branch left checked out for each repo cloned (default master)")
	cloneCmd.Flags().StringVarP(&token, "token", "t", "", "GHORG_GITHUB_TOKEN/GHORG_GITLAB_TOKEN/GHORG_BITBUCKET_APP_PASSWORD - scm token to clone with")
	cloneCmd.Flags().StringVarP(&bitbucketUsername, "bitbucket-username", "", "", "GHORG_BITBUCKET_USERNAME - bitbucket only: username associated with the app password")
	cloneCmd.Flags().StringVarP(&scmType, "scm", "s", "", "GHORG_SCM_TYPE - type of scm used, github, gitlab or bitbucket (default github)")
	// TODO: make gitlab terminology make sense https://about.gitlab.com/2016/01/27/comparing-terms-gitlab-github-bitbucket/
	cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)")
	cloneCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "GHORG_GITLAB_DEFAULT_NAMESPACE - gitlab only: limits clone targets to a specific namespace e.g. --namespace=gitlab-org/security-products")
	cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED skips archived repos, github/gitlab only")
	cloneCmd.Flags().BoolVar(&skipArchived, "preserve-dir", false, "GHORG_PRESERVE_DIRECTORY_STRUCTURE clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into *_ghorg/unit/subunit/app, gitlab only")
	cloneCmd.Flags().BoolVar(&backup, "backup", false, "GHORG_BACKUP backup mode, clone as mirror, no working copy (ignores branch parameter)")
	cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL change SCM base url, for on self hosted instances (currently gitlab only, use format of https://git.mydomain.com/api/v3)")
	cloneCmd.Flags().StringVarP(&concurrency, "concurrency", "", "", "GHORG_CONCURRENCY max goroutines to spin up while cloning (default 25)")

}

// Repo represents an SCM repo
type Repo struct {
	Name     string
	Path     string
	URL      string
	CloneURL string
}

var cloneCmd = &cobra.Command{
	Use:   "clone",
	Short: "Clone user or org repos from GitHub, GitLab, or Bitbucket",
	Long:  `Clone user or org repos from GitHub, GitLab, or Bitbucket. See $HOME/ghorg/conf.yaml for defaults, its likely you will need to update some of these values of use the flags to overwrite them. Values are set first by a default value, then based off what is set in $HOME/ghorg/conf.yaml, finally the cli flags, which have the highest level of precedence.`,
	Run: func(cmd *cobra.Command, argz []string) {

		if cmd.Flags().Changed("color") {
			colorToggle := cmd.Flag("color").Value.String()
			if colorToggle == "on" {
				os.Setenv("GHORG_COLOR", colorToggle)
			} else {
				os.Setenv("GHORG_COLOR", "off")
			}

		}

		if len(argz) < 1 {
			colorlog.PrintError("You must provide an org or user to clone")
			os.Exit(1)
		}

		if cmd.Flags().Changed("path") {
			absolutePath := ensureTrailingSlash(cmd.Flag("path").Value.String())
			os.Setenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO", absolutePath)
		}

		if cmd.Flags().Changed("protocol") {
			protocol := cmd.Flag("protocol").Value.String()
			os.Setenv("GHORG_CLONE_PROTOCOL", protocol)
		}

		if cmd.Flags().Changed("branch") {
			os.Setenv("GHORG_BRANCH", cmd.Flag("branch").Value.String())
		}

		if cmd.Flags().Changed("bitbucket-username") {
			os.Setenv("GHORG_BITBUCKET_USERNAME", cmd.Flag("bitbucket-username").Value.String())
		}

		if cmd.Flags().Changed("namespace") {
			os.Setenv("GHORG_GITLAB_DEFAULT_NAMESPACE", cmd.Flag("namespace").Value.String())
		}

		if cmd.Flags().Changed("clone-type") {
			cloneType := strings.ToLower(cmd.Flag("clone-type").Value.String())
			os.Setenv("GHORG_CLONE_TYPE", cloneType)
		}

		if cmd.Flags().Changed("scm") {
			scmType := strings.ToLower(cmd.Flag("scm").Value.String())
			os.Setenv("GHORG_SCM_TYPE", scmType)
		}

		if cmd.Flags().Changed("base-url") {
			url := cmd.Flag("base-url").Value.String()
			os.Setenv("GHORG_SCM_BASE_URL", url)
		}

		if cmd.Flags().Changed("concurrency") {
			g := cmd.Flag("concurrency").Value.String()
			os.Setenv("GHORG_CONCURRENCY", g)
		}

		if cmd.Flags().Changed("skip-archived") {
			os.Setenv("GHORG_SKIP_ARCHIVED", "true")
		}

		if cmd.Flags().Changed("preserve-dir") {
			os.Setenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE", "true")
		}

		if cmd.Flags().Changed("backup") {
			os.Setenv("GHORG_BACKUP", "true")
		}

		configs.GetOrSetToken()

		if cmd.Flags().Changed("token") {
			if os.Getenv("GHORG_SCM_TYPE") == "github" {
				os.Setenv("GHORG_GITHUB_TOKEN", cmd.Flag("token").Value.String())
			} else if os.Getenv("GHORG_SCM_TYPE") == "gitlab" {
				os.Setenv("GHORG_GITLAB_TOKEN", cmd.Flag("token").Value.String())
			} else if os.Getenv("GHORG_SCM_TYPE") == "bitbucket" {
				os.Setenv("GHORG_BITBUCKET_APP_PASSWORD", cmd.Flag("token").Value.String())
			}
		}

		err := configs.VerifyTokenSet()
		if err != nil {
			colorlog.PrintError(err)
			os.Exit(1)
		}

		err = configs.VerifyConfigsSetCorrectly()
		if err != nil {
			colorlog.PrintError(err)
			os.Exit(1)
		}

		args = argz

		CloneAllRepos()
	},
}

// TODO: Figure out how to use go channels for this
func getAllOrgCloneUrls() ([]Repo, error) {
	asciiTime()
	PrintConfigs()
	var repos []Repo
	var err error
	switch os.Getenv("GHORG_SCM_TYPE") {
	case "github":
		repos, err = getGitHubOrgCloneUrls()
	case "gitlab":
		repos, err = getGitLabOrgCloneUrls()
	case "bitbucket":
		repos, err = getBitBucketOrgCloneUrls()
	default:
		colorlog.PrintError("GHORG_SCM_TYPE not set or unsupported, also make sure its all lowercase")
		os.Exit(1)
	}

	return repos, err
}

// TODO: Figure out how to use go channels for this
func getAllUserCloneUrls() ([]Repo, error) {
	asciiTime()
	PrintConfigs()
	var repos []Repo
	var err error
	switch os.Getenv("GHORG_SCM_TYPE") {
	case "github":
		repos, err = getGitHubUserCloneUrls()
	case "gitlab":
		repos, err = getGitLabUserCloneUrls()
	case "bitbucket":
		repos, err = getBitBucketUserCloneUrls()
	default:
		colorlog.PrintError("GHORG_SCM_TYPE not set or unsupported, also make sure its all lowercase")
		os.Exit(1)
	}

	return repos, err
}

func createDirIfNotExist() {
	if _, err := os.Stat(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + args[0] + "_ghorg"); os.IsNotExist(err) {
		err = os.MkdirAll(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), 0700)
		if err != nil {
			panic(err)
		}
	}
}

func repoExistsLocally(path string) bool {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false
	}

	return true
}

func getAppNameFromURL(url string) string {
	withGit := strings.Split(url, "/")
	appName := withGit[len(withGit)-1]
	split := strings.Split(appName, ".")
	return strings.Join(split[0:len(split)-1], ".")
}

func printRemainingMessages() {
	if len(cloneInfos) > 0 {
		fmt.Println()
		colorlog.PrintInfo("============ Info ============")
		fmt.Println()
		for _, i := range cloneInfos {
			colorlog.PrintInfo(i)
		}
		fmt.Println()
	}

	if len(cloneErrors) > 0 {
		fmt.Println()
		colorlog.PrintError("============ Issues ============")
		fmt.Println()
		for _, e := range cloneErrors {
			colorlog.PrintError(e)
		}
		fmt.Println()
	}
}

func readGhorgIgnore() ([]string, error) {
	file, err := os.Open(configs.GhorgIgnoreLocation())
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

// CloneAllRepos clones all repos
func CloneAllRepos() {
	// resc, errc, infoc := make(chan string), make(chan error), make(chan error)

	var cloneTargets []Repo
	var err error

	if os.Getenv("GHORG_CLONE_TYPE") == "org" {
		cloneTargets, err = getAllOrgCloneUrls()
	} else if os.Getenv("GHORG_CLONE_TYPE") == "user" {
		cloneTargets, err = getAllUserCloneUrls()
	} else {
		colorlog.PrintError("GHORG_CLONE_TYPE not set or unsupported")
		os.Exit(1)
	}

	if err != nil {
		colorlog.PrintError("Encountered an error, aborting")
		fmt.Println(err)
		os.Exit(1)
	}

	if len(cloneTargets) == 0 {
		colorlog.PrintInfo("No repos found for " + os.Getenv("GHORG_SCM_TYPE") + " " + os.Getenv("GHORG_CLONE_TYPE") + ": " + args[0] + ", check spelling and verify clone-type (user/org) is set correctly e.g. -c=user")
		os.Exit(0)
	}

	// filter repos down based on ghorgignore if one exists
	_, err = os.Stat(configs.GhorgIgnoreLocation())
	if !os.IsNotExist(err) {
		// Open the file parse each line and remove cloneTargets containing
		toIgnore, err := readGhorgIgnore()
		if err != nil {
			colorlog.PrintError("Error parsing your ghorgignore, aborting")
			fmt.Println(err)
			os.Exit(1)
		}

		colorlog.PrintInfo("Using ghorgignore, filtering repos down...")

		filteredCloneTargets := []Repo{}
		var flag bool
		for _, cloned := range cloneTargets {
			flag = false
			for _, ignore := range toIgnore {
				if strings.Contains(cloned.URL, ignore) {
					flag = true
				}
			}

			if flag == false {
				filteredCloneTargets = append(filteredCloneTargets, cloned)
			}
		}

		cloneTargets = filteredCloneTargets

	}

	colorlog.PrintInfo(strconv.Itoa(len(cloneTargets)) + " repos found in " + args[0])
	fmt.Println()

	createDirIfNotExist()

	l, err := strconv.Atoi(os.Getenv("GHORG_CONCURRENCY"))

	if err != nil {
		log.Fatal("Could not determine GHORG_CONCURRENCY")
	}

	limit := limiter.NewConcurrencyLimiter(l)
	for _, target := range cloneTargets {
		appName := getAppNameFromURL(target.URL)
		branch := os.Getenv("GHORG_BRANCH")
		repo := target

		limit.Execute(func() {

			path := appName
			if repo.Path != "" && os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" {
				path = repo.Path
			}

			repoDir := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + args[0] + "_ghorg" + "/" + path

			if os.Getenv("GHORG_BACKUP") == "true" {
				repoDir = os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO") + args[0] + "_ghorg_backup" + "/" + path
			}

			if repoExistsLocally(repoDir) == true {
				if os.Getenv("GHORG_BACKUP") == "true" {
					cmd := exec.Command("git", "remote", "update")
					cmd.Dir = repoDir
					err := cmd.Run()
					if err != nil {
						e := fmt.Sprintf("Could not update remotes in Repo: %s Error: %v", repo.URL, err)
						cloneErrors = append(cloneErrors, e)
						return
					}
				} else {

					cmd := exec.Command("git", "checkout", branch)
					cmd.Dir = repoDir
					err := cmd.Run()
					if err != nil {
						e := fmt.Sprintf("Could not checkout out %s, branch may not exist, no changes made Repo: %s Error: %v", branch, repo.URL, err)
						cloneInfos = append(cloneInfos, e)
						return
					}

					cmd = exec.Command("git", "clean", "-f", "-d")
					cmd.Dir = repoDir
					err = cmd.Run()
					if err != nil {
						e := fmt.Sprintf("Problem running git clean: %s Error: %v", repo.URL, err)
						cloneErrors = append(cloneErrors, e)
						return
					}

					cmd = exec.Command("git", "reset", "--hard", "origin/"+branch)
					cmd.Dir = repoDir
					err = cmd.Run()
					if err != nil {
						e := fmt.Sprintf("Problem resetting %s Repo: %s Error: %v", branch, repo.URL, err)
						cloneErrors = append(cloneErrors, e)
						return
					}

					cmd = exec.Command("git", "pull", "origin", branch)
					cmd.Dir = repoDir
					err = cmd.Run()
					if err != nil {
						e := fmt.Sprintf("Problem trying to pull %v Repo: %s Error: %v", branch, repo.URL, err)
						cloneErrors = append(cloneErrors, e)
						return
					}
				}
			} else {
				// if https clone and github/gitlab add personal access token to url

				args := []string{"clone", repo.CloneURL, repoDir}
				if os.Getenv("GHORG_BACKUP") == "true" {
					args = append(args, "--mirror")
				}

				cmd := exec.Command("git", args...)
				err := cmd.Run()

				if err != nil {
					e := fmt.Sprintf("Problem trying to clone Repo: %s Error: %v", repo.URL, err)
					cloneErrors = append(cloneErrors, e)
					return
				}
			}

			colorlog.PrintSuccess("Success " + repo.URL)
		})

	}

	limit.Wait()

	printRemainingMessages()

	// TODO: fix all these if else checks with ghorg_backups
	if os.Getenv("GHORG_BACKUP") == "true" {
		colorlog.PrintSuccess(fmt.Sprintf("Finished! %s%s_ghorg_backup", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), args[0]))
	} else {
		colorlog.PrintSuccess(fmt.Sprintf("Finished! %s%s_ghorg", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), args[0]))
	}
}

func asciiTime() {
	colorlog.PrintInfo(
		`
 +-+-+-+-+ +-+-+ +-+-+-+-+-+
 |T|I|M|E| |T|O| |G|H|O|R|G|
 +-+-+-+-+ +-+-+ +-+-+-+-+-+
`)
}

// PrintConfigs shows the user what is set before cloning
func PrintConfigs() {
	colorlog.PrintInfo("*************************************")
	colorlog.PrintInfo("* SCM           : " + os.Getenv("GHORG_SCM_TYPE"))
	colorlog.PrintInfo("* Type          : " + os.Getenv("GHORG_CLONE_TYPE"))
	colorlog.PrintInfo("* Protocol      : " + os.Getenv("GHORG_CLONE_PROTOCOL"))
	colorlog.PrintInfo("* Branch        : " + os.Getenv("GHORG_BRANCH"))
	colorlog.PrintInfo("* Location      : " + os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"))
	colorlog.PrintInfo("* Concurrency   : " + os.Getenv("GHORG_CONCURRENCY"))
	if os.Getenv("GHORG_SCM_BASE_URL") != "" {
		colorlog.PrintInfo("* Base URL      : " + os.Getenv("GHORG_SCM_BASE_URL"))
	}
	if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
		colorlog.PrintInfo("* Skip Archived : " + os.Getenv("GHORG_SKIP_ARCHIVED"))
	}
	if os.Getenv("GHORG_BACKUP") == "true" {
		colorlog.PrintInfo("* Backup        : " + os.Getenv("GHORG_BACKUP"))
	}
	colorlog.PrintInfo("*************************************")
	fmt.Println("")
}

func ensureTrailingSlash(path string) string {
	if string(path[len(path)-1]) == "/" {
		return path
	}

	return path + "/"
}

func addTokenToHTTPSCloneURL(url string, token string) string {
	splitURL := strings.Split(url, "https://")

	if os.Getenv("GHORG_SCM_TYPE") == "gitlab" {
		return "https://oauth2:" + token + "@" + splitURL[1]
	}

	return "https://" + token + "@" + splitURL[1]
}