Add "prune" feature to clone (#206)

CHANGELOG.md

@@ -3,6 +3,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
+## [1.7.16] - 6/2/22
+### Added
+- GHORG_PRUNE setting which allows a user to have Ghorg automatically remove items from their local
+  org clone which have been removed (or archived, if GHORG_SKIP_ARCHIVED is set) upstream.
+- GHORG_PRUNE_NO_CONFIRM which disables the interactive yes/no prompt for every item to be deleted
+  when pruning.
+### Changed
+### Deprecated
+### Removed
+### Fixed
+### Security
+
 ## [1.7.15] - 5/29/22
 ### Added
 - CodeQL security analysis action

cmd/clone.go

@@ -4,6 +4,7 @@ package cmd
 import (
 	"bufio"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/url"
 	"os"
@@ -109,6 +110,14 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Setenv("GHORG_NO_CLEAN", "true")
 	}
 
+	if cmd.Flags().Changed("prune") {
+		os.Setenv("GHORG_PRUNE", "true")
+	}
+
+	if cmd.Flags().Changed("prune-no-confirm") {
+		os.Setenv("GHORG_PRUNE_NO_CONFIRM", "true")
+	}
+
 	if cmd.Flags().Changed("fetch-all") {
 		os.Setenv("GHORG_FETCH_ALL", "true")
 	}
@@ -389,6 +398,31 @@ func printDryRun(repos []scm.Repo) {
 	}
 	count := len(repos)
 	colorlog.PrintSuccess(fmt.Sprintf("%v repos to be cloned into: %s%s", count, os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder))
+
+	if os.Getenv("GHORG_PRUNE") == "true" {
+		cloneLocation := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder)
+		if stat, err := os.Stat(cloneLocation); err == nil && stat.IsDir() {
+			// We check that the clone path exists, otherwise there would definitely be no pruning
+			// to do.
+			colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...")
+
+			files, err := ioutil.ReadDir(cloneLocation)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			eligibleForPrune := 0
+			for _, f := range files {
+				// for each item in the org's clone directory, let's make sure we found a
+				// corresponding repo on the remote.
+				if !sliceContainsNamedRepo(repos, f.Name()) {
+					eligibleForPrune++
+					colorlog.PrintSubtleInfo(fmt.Sprintf("%s not found in remote.", f.Name()))
+				}
+			}
+			colorlog.PrintSuccess(fmt.Sprintf("Local clones eligible for pruning: %d", eligibleForPrune))
+		}
+	}
 }
 
 // CloneAllRepos clones all repos
@@ -476,7 +510,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 				repoSlug = repo.Path
 			}
 
-			repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder, configs.GetCorrectFilePathSeparator(), repoSlug)
+			repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder, repoSlug)
 
 			if repo.IsWiki {
 				if !strings.HasSuffix(repo.HostPath, ".wiki") {
@@ -485,7 +519,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 			}
 
 			if os.Getenv("GHORG_BACKUP") == "true" {
-				repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder+"_backup", configs.GetCorrectFilePathSeparator(), repoSlug)
+				repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder+"_backup", repoSlug)
 			}
 
 			action := "cloning"
@@ -630,6 +664,42 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 	colorlog.PrintSuccess(fmt.Sprintf("New repos cloned: %v, existing repos pulled: %v", cloneCount, pulledCount))
 
+	// Now, clean up local repos that don't exist in remote, if prune flag is set
+	if os.Getenv("GHORG_PRUNE") == "true" {
+		colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...")
+		cloneLocation := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder)
+
+		files, err := ioutil.ReadDir(cloneLocation)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// The first time around, we set userAgreesToDelete to true, otherwise we'd immediately
+		// break out of the loop.
+		userAgreesToDelete := true
+		pruneNoConfirm := os.Getenv("GHORG_PRUNE_NO_CONFIRM") == "true"
+		for _, f := range files {
+			// For each item in the org's clone directory, let's make sure we found a corresponding
+			// repo on the remote.  We check userAgreesToDelete here too, so that if the user says
+			// "No" at any time, we stop trying to prune things altogether.
+			if userAgreesToDelete && !sliceContainsNamedRepo(cloneTargets, f.Name()) {
+				// If the user specified --prune-no-confirm, we needn't prompt interactively.
+				userAgreesToDelete = pruneNoConfirm || interactiveYesNoPrompt(
+					fmt.Sprintf("%s was not found in remote.  Do you want to prune it?", f.Name()))
+				if userAgreesToDelete {
+					colorlog.PrintSubtleInfo(
+						fmt.Sprintf("Deleting %s", filepath.Join(cloneLocation, f.Name())))
+					err = os.RemoveAll(filepath.Join(cloneLocation, f.Name()))
+					if err != nil {
+						log.Fatal(err)
+					}
+				} else {
+					colorlog.PrintError("Pruning cancelled by user.  No more prunes will be considered.")
+				}
+			}
+		}
+	}
+
 	// TODO: fix all these if else checks with ghorg_backups
 	if os.Getenv("GHORG_BACKUP") == "true" {
 		colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s%s_backup", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder))
@@ -638,6 +708,34 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 	}
 }
 
+func interactiveYesNoPrompt(prompt string) bool {
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Print(strings.TrimSpace(prompt) + " (y/N) ")
+	s, err := reader.ReadString('\n')
+	if err != nil {
+		panic(err)
+	}
+
+	s = strings.TrimSpace(s)
+	s = strings.ToLower(s)
+
+	if s == "y" || s == "yes" {
+		return true
+	}
+	return false
+}
+
+// There's probably a nicer way of finding whether any scm.Repo in the slice matches a given name.
+func sliceContainsNamedRepo(haystack []scm.Repo, needle string) bool {
+	for _, repo := range haystack {
+		if repo.Name == needle {
+			return true
+		}
+	}
+
+	return false
+}
+
 func asciiTime() {
 	colorlog.PrintInfo(
 		`
@@ -699,6 +797,13 @@ func PrintConfigs() {
 	if os.Getenv("GHORG_NO_CLEAN") == "true" {
 		colorlog.PrintInfo("* No Clean      : " + "true")
 	}
+	if os.Getenv("GHORG_PRUNE") == "true" {
+		noConfirmText := ""
+		if os.Getenv("GHORG_PRUNE_NO_CONFIRM") == "true" {
+			noConfirmText = " (skipping confirmation)"
+		}
+		colorlog.PrintInfo("* Prune         : " + "true" + noConfirmText)
+	}
 	if os.Getenv("GHORG_FETCH_ALL") == "true" {
 		colorlog.PrintInfo("* Fetch All     : " + "true")
 	}

cmd/ls.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/gabrie30/ghorg/colorlog"
-	"github.com/gabrie30/ghorg/configs"
 	"github.com/spf13/cobra"
 )
 
@@ -58,7 +57,7 @@ func listGhorgDir(arg string) {
 
 	for _, f := range files {
 		if f.IsDir() {
-			str := filepath.Join(path, configs.GetCorrectFilePathSeparator(), f.Name())
+			str := filepath.Join(path, f.Name())
 			colorlog.PrintSubtleInfo(str)
 		}
 	}

cmd/root.go

@@ -31,6 +31,8 @@ var (
 	backup                       bool
 	noClean                      bool
 	dryRun                       bool
+	prune                        bool
+	pruneNoConfirm               bool
 	cloneWiki                    bool
 	preserveDir                  bool
 	insecureGitlabClient         bool
@@ -103,6 +105,10 @@ func getOrSetDefaults(envVar string) {
 			os.Setenv(envVar, "false")
 		case "GHORG_DRY_RUN":
 			os.Setenv(envVar, "false")
+		case "GHORG_PRUNE":
+			os.Setenv(envVar, "false")
+		case "GHORG_PRUNE_NO_CONFIRM":
+			os.Setenv(envVar, "false")
 		case "GHORG_INSECURE_GITLAB_CLIENT":
 			os.Setenv(envVar, "false")
 		case "GHORG_BACKUP":
@@ -176,6 +182,8 @@ func InitConfig() {
 	getOrSetDefaults("GHORG_SKIP_FORKS")
 	getOrSetDefaults("GHORG_NO_CLEAN")
 	getOrSetDefaults("GHORG_FETCH_ALL")
+	getOrSetDefaults("GHORG_PRUNE")
+	getOrSetDefaults("GHORG_PRUNE_NO_CONFIRM")
 	getOrSetDefaults("GHORG_DRY_RUN")
 	getOrSetDefaults("GHORG_CLONE_WIKI")
 	getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT")
@@ -228,6 +236,8 @@ func init() {
 	cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)")
 	cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only")
 	cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos")
+	cloneCmd.Flags().BoolVar(&prune, "prune", false, "GHORG_PRUNE - remove local clones if not found on remote")
+	cloneCmd.Flags().BoolVar(&pruneNoConfirm, "prune-no-confirm", false, "GHORG_PRUNE_NO_CONFIRM - don't prompt on every prune candidate, just delete")
 	cloneCmd.Flags().BoolVar(&fetchAll, "fetch-all", false, "GHORG_FETCH_ALL - fetches all remote branches for each repo by running a git fetch --all")
 	cloneCmd.Flags().BoolVar(&dryRun, "dry-run", false, "GHORG_DRY_RUN - perform a dry run of the clone; fetches repos but does not clone them")
 	cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances")

configs/configs.go

@@ -80,7 +80,7 @@ func GetAbsolutePathToCloneTo() string {
 
 // EnsureTrailingSlashOnFilePath takes a filepath and ensures a single / is appened
 func EnsureTrailingSlashOnFilePath(s string) string {
-	trailing := GetCorrectFilePathSeparator()
+	trailing := string(os.PathSeparator)
 
 	if !strings.HasSuffix(s, trailing) {
 		s = s + trailing
@@ -89,17 +89,6 @@ func EnsureTrailingSlashOnFilePath(s string) string {
 	return s
 }
 
-// GetCorrectFilePathSeparator returns the correct trailing slash based on os
-func GetCorrectFilePathSeparator() string {
-	trailing := "/"
-
-	if runtime.GOOS == "windows" {
-		trailing = "\\"
-	}
-
-	return trailing
-}
-
 // GhorgIgnoreLocation returns the path of users ghorgignore
 func GhorgIgnoreLocation() string {
 	ignoreLocation := os.Getenv("GHORG_IGNORE_PATH")

sample-conf.yaml

@@ -150,6 +150,14 @@ GHORG_CLONE_WIKI: false
 # flag (--dry-run)
 GHORG_DRY_RUN: false
 
+# Remove locally cloned repos that aren't found on remote (e.g., after remote deletion).  With GHORG_SKIP_ARCHIVED set, archived repositories will also be pruned from your local clone.
+# flag (--prune)
+GHORG_PRUNE: false
+
+# Skip interactive y/n prompt when pruning clones (only makes sense with --prune).
+# flag (--prune-no-confirm)
+GHORG_PRUNE_NO_CONFIRM: false
+
 # Fetches all remote branches for each repo by running a git fetch --all
 # flag (--fetch-all)
 GHORG_FETCH_ALL: false