From fa948fcfb1524eb3ee53f3f13bc9937f56d2d5a4 Mon Sep 17 00:00:00 2001 From: paul david <423357+toothbrush@users.noreply.github.com> Date: Thu, 2 Jun 2022 12:53:14 +1000 Subject: [PATCH] Add "prune" feature to `clone` (#206) --- CHANGELOG.md | 12 +++++ cmd/clone.go | 109 ++++++++++++++++++++++++++++++++++++++++++++- cmd/ls.go | 3 +- cmd/root.go | 10 +++++ configs/configs.go | 13 +----- sample-conf.yaml | 8 ++++ 6 files changed, 139 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa57bd5..e2c16e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +## [1.7.16] - 6/2/22 +### Added +- GHORG_PRUNE setting which allows a user to have Ghorg automatically remove items from their local + org clone which have been removed (or archived, if GHORG_SKIP_ARCHIVED is set) upstream. +- GHORG_PRUNE_NO_CONFIRM which disables the interactive yes/no prompt for every item to be deleted + when pruning. +### Changed +### Deprecated +### Removed +### Fixed +### Security + ## [1.7.15] - 5/29/22 ### Added - CodeQL security analysis action diff --git a/cmd/clone.go b/cmd/clone.go index 040c917..9a83d7f 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -4,6 +4,7 @@ package cmd import ( "bufio" "fmt" + "io/ioutil" "log" "net/url" "os" @@ -109,6 +110,14 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Setenv("GHORG_NO_CLEAN", "true") } + if cmd.Flags().Changed("prune") { + os.Setenv("GHORG_PRUNE", "true") + } + + if cmd.Flags().Changed("prune-no-confirm") { + os.Setenv("GHORG_PRUNE_NO_CONFIRM", "true") + } + if cmd.Flags().Changed("fetch-all") { os.Setenv("GHORG_FETCH_ALL", "true") } @@ -389,6 +398,31 @@ func printDryRun(repos []scm.Repo) { } count := len(repos) colorlog.PrintSuccess(fmt.Sprintf("%v repos to be cloned into: %s%s", count, os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder)) + + if os.Getenv("GHORG_PRUNE") == "true" { + cloneLocation := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder) + if stat, err := os.Stat(cloneLocation); err == nil && stat.IsDir() { + // We check that the clone path exists, otherwise there would definitely be no pruning + // to do. + colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...") + + files, err := ioutil.ReadDir(cloneLocation) + if err != nil { + log.Fatal(err) + } + + eligibleForPrune := 0 + for _, f := range files { + // for each item in the org's clone directory, let's make sure we found a + // corresponding repo on the remote. + if !sliceContainsNamedRepo(repos, f.Name()) { + eligibleForPrune++ + colorlog.PrintSubtleInfo(fmt.Sprintf("%s not found in remote.", f.Name())) + } + } + colorlog.PrintSuccess(fmt.Sprintf("Local clones eligible for pruning: %d", eligibleForPrune)) + } + } } // CloneAllRepos clones all repos @@ -476,7 +510,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { repoSlug = repo.Path } - repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder, configs.GetCorrectFilePathSeparator(), repoSlug) + repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder, repoSlug) if repo.IsWiki { if !strings.HasSuffix(repo.HostPath, ".wiki") { @@ -485,7 +519,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } if os.Getenv("GHORG_BACKUP") == "true" { - repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder+"_backup", configs.GetCorrectFilePathSeparator(), repoSlug) + repo.HostPath = filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder+"_backup", repoSlug) } action := "cloning" @@ -630,6 +664,42 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { colorlog.PrintSuccess(fmt.Sprintf("New repos cloned: %v, existing repos pulled: %v", cloneCount, pulledCount)) + // Now, clean up local repos that don't exist in remote, if prune flag is set + if os.Getenv("GHORG_PRUNE") == "true" { + colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...") + cloneLocation := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder) + + files, err := ioutil.ReadDir(cloneLocation) + if err != nil { + log.Fatal(err) + } + + // The first time around, we set userAgreesToDelete to true, otherwise we'd immediately + // break out of the loop. + userAgreesToDelete := true + pruneNoConfirm := os.Getenv("GHORG_PRUNE_NO_CONFIRM") == "true" + for _, f := range files { + // For each item in the org's clone directory, let's make sure we found a corresponding + // repo on the remote. We check userAgreesToDelete here too, so that if the user says + // "No" at any time, we stop trying to prune things altogether. + if userAgreesToDelete && !sliceContainsNamedRepo(cloneTargets, f.Name()) { + // If the user specified --prune-no-confirm, we needn't prompt interactively. + userAgreesToDelete = pruneNoConfirm || interactiveYesNoPrompt( + fmt.Sprintf("%s was not found in remote. Do you want to prune it?", f.Name())) + if userAgreesToDelete { + colorlog.PrintSubtleInfo( + fmt.Sprintf("Deleting %s", filepath.Join(cloneLocation, f.Name()))) + err = os.RemoveAll(filepath.Join(cloneLocation, f.Name())) + if err != nil { + log.Fatal(err) + } + } else { + colorlog.PrintError("Pruning cancelled by user. No more prunes will be considered.") + } + } + } + } + // TODO: fix all these if else checks with ghorg_backups if os.Getenv("GHORG_BACKUP") == "true" { colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s%s_backup", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), parentFolder)) @@ -638,6 +708,34 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { } } +func interactiveYesNoPrompt(prompt string) bool { + reader := bufio.NewReader(os.Stdin) + fmt.Print(strings.TrimSpace(prompt) + " (y/N) ") + s, err := reader.ReadString('\n') + if err != nil { + panic(err) + } + + s = strings.TrimSpace(s) + s = strings.ToLower(s) + + if s == "y" || s == "yes" { + return true + } + return false +} + +// There's probably a nicer way of finding whether any scm.Repo in the slice matches a given name. +func sliceContainsNamedRepo(haystack []scm.Repo, needle string) bool { + for _, repo := range haystack { + if repo.Name == needle { + return true + } + } + + return false +} + func asciiTime() { colorlog.PrintInfo( ` @@ -699,6 +797,13 @@ func PrintConfigs() { if os.Getenv("GHORG_NO_CLEAN") == "true" { colorlog.PrintInfo("* No Clean : " + "true") } + if os.Getenv("GHORG_PRUNE") == "true" { + noConfirmText := "" + if os.Getenv("GHORG_PRUNE_NO_CONFIRM") == "true" { + noConfirmText = " (skipping confirmation)" + } + colorlog.PrintInfo("* Prune : " + "true" + noConfirmText) + } if os.Getenv("GHORG_FETCH_ALL") == "true" { colorlog.PrintInfo("* Fetch All : " + "true") } diff --git a/cmd/ls.go b/cmd/ls.go index 83e16e5..521d416 100644 --- a/cmd/ls.go +++ b/cmd/ls.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/gabrie30/ghorg/colorlog" - "github.com/gabrie30/ghorg/configs" "github.com/spf13/cobra" ) @@ -58,7 +57,7 @@ func listGhorgDir(arg string) { for _, f := range files { if f.IsDir() { - str := filepath.Join(path, configs.GetCorrectFilePathSeparator(), f.Name()) + str := filepath.Join(path, f.Name()) colorlog.PrintSubtleInfo(str) } } diff --git a/cmd/root.go b/cmd/root.go index ffc7d5a..701d994 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -31,6 +31,8 @@ var ( backup bool noClean bool dryRun bool + prune bool + pruneNoConfirm bool cloneWiki bool preserveDir bool insecureGitlabClient bool @@ -103,6 +105,10 @@ func getOrSetDefaults(envVar string) { os.Setenv(envVar, "false") case "GHORG_DRY_RUN": os.Setenv(envVar, "false") + case "GHORG_PRUNE": + os.Setenv(envVar, "false") + case "GHORG_PRUNE_NO_CONFIRM": + os.Setenv(envVar, "false") case "GHORG_INSECURE_GITLAB_CLIENT": os.Setenv(envVar, "false") case "GHORG_BACKUP": @@ -176,6 +182,8 @@ func InitConfig() { getOrSetDefaults("GHORG_SKIP_FORKS") getOrSetDefaults("GHORG_NO_CLEAN") getOrSetDefaults("GHORG_FETCH_ALL") + getOrSetDefaults("GHORG_PRUNE") + getOrSetDefaults("GHORG_PRUNE_NO_CONFIRM") getOrSetDefaults("GHORG_DRY_RUN") getOrSetDefaults("GHORG_CLONE_WIKI") getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT") @@ -228,6 +236,8 @@ func init() { cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)") cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only") cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos") + cloneCmd.Flags().BoolVar(&prune, "prune", false, "GHORG_PRUNE - remove local clones if not found on remote") + cloneCmd.Flags().BoolVar(&pruneNoConfirm, "prune-no-confirm", false, "GHORG_PRUNE_NO_CONFIRM - don't prompt on every prune candidate, just delete") cloneCmd.Flags().BoolVar(&fetchAll, "fetch-all", false, "GHORG_FETCH_ALL - fetches all remote branches for each repo by running a git fetch --all") cloneCmd.Flags().BoolVar(&dryRun, "dry-run", false, "GHORG_DRY_RUN - perform a dry run of the clone; fetches repos but does not clone them") cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances") diff --git a/configs/configs.go b/configs/configs.go index 6c1e085..bc74330 100644 --- a/configs/configs.go +++ b/configs/configs.go @@ -80,7 +80,7 @@ func GetAbsolutePathToCloneTo() string { // EnsureTrailingSlashOnFilePath takes a filepath and ensures a single / is appened func EnsureTrailingSlashOnFilePath(s string) string { - trailing := GetCorrectFilePathSeparator() + trailing := string(os.PathSeparator) if !strings.HasSuffix(s, trailing) { s = s + trailing @@ -89,17 +89,6 @@ func EnsureTrailingSlashOnFilePath(s string) string { return s } -// GetCorrectFilePathSeparator returns the correct trailing slash based on os -func GetCorrectFilePathSeparator() string { - trailing := "/" - - if runtime.GOOS == "windows" { - trailing = "\\" - } - - return trailing -} - // GhorgIgnoreLocation returns the path of users ghorgignore func GhorgIgnoreLocation() string { ignoreLocation := os.Getenv("GHORG_IGNORE_PATH") diff --git a/sample-conf.yaml b/sample-conf.yaml index 5e687b5..45ab4fe 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -150,6 +150,14 @@ GHORG_CLONE_WIKI: false # flag (--dry-run) GHORG_DRY_RUN: false +# Remove locally cloned repos that aren't found on remote (e.g., after remote deletion). With GHORG_SKIP_ARCHIVED set, archived repositories will also be pruned from your local clone. +# flag (--prune) +GHORG_PRUNE: false + +# Skip interactive y/n prompt when pruning clones (only makes sense with --prune). +# flag (--prune-no-confirm) +GHORG_PRUNE_NO_CONFIRM: false + # Fetches all remote branches for each repo by running a git fetch --all # flag (--fetch-all) GHORG_FETCH_ALL: false