From 6f0826452b9cd9127b88dd51fc46dfdfd29d5ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Fraga?= Date: Sat, 10 May 2025 19:27:42 +0100 Subject: [PATCH] feat: allow usage of --prune with --preserve-dir (#521) --- cmd/clone.go | 43 +++++++-- cmd/clone_test.go | 101 ++++++++++++++++++++++ scripts/gitlab_cloud_integration_tests.sh | 15 ++++ 3 files changed, 150 insertions(+), 9 deletions(-) diff --git a/cmd/clone.go b/cmd/clone.go index 75f2c26..3d0bc08 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -5,6 +5,7 @@ import ( "bufio" "crypto/sha256" "fmt" + "io/fs" "log" "net/url" "os" @@ -556,7 +557,13 @@ func printDryRun(repos []scm.Repo) { // to do. colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...") - files, err := os.ReadDir(outputDirAbsolutePath) + var files []os.DirEntry + var err error + if os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" { + files, err = readDirRecursively(outputDirAbsolutePath) + } else { + files, err = os.ReadDir(outputDirAbsolutePath) + } if err != nil { log.Fatal(err) } @@ -599,6 +606,25 @@ func getCloneableInventory(allRepos []scm.Repo) (int, int, int, int) { return total, repos, snippets, wikis } +func isGitRepository(path string) bool { + file, err := os.Stat(filepath.Join(path, ".git")) + return err == nil && file.IsDir() +} + +func readDirRecursively(name string) ([]os.DirEntry, error) { + var files []os.DirEntry + err := filepath.WalkDir(name, func(path string, file fs.DirEntry, err error) error { + if err != nil { + return err + } + if path != outputDirAbsolutePath && file.IsDir() && isGitRepository(path) { + files = append(files, file) + } + return nil + }) + return files, err +} + // CloneAllRepos clones all repos func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { // Filter repos that have attributes that don't need specific scm api calls @@ -1294,7 +1320,13 @@ func pruneRepos(cloneTargets []scm.Repo) int { count := 0 colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...") - files, err := os.ReadDir(outputDirAbsolutePath) + var files []os.DirEntry + var err error + if os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" { + files, err = readDirRecursively(outputDirAbsolutePath) + } else { + files, err = os.ReadDir(outputDirAbsolutePath) + } if err != nil { log.Fatal(err) } @@ -1365,14 +1397,7 @@ func interactiveYesNoPrompt(prompt string) bool { } // There's probably a nicer way of finding whether any scm.Repo in the slice matches a given name. -// TODO, currently this does not work if user sets --preserve-dir see https://github.com/gabrie30/ghorg/issues/210 for more info func sliceContainsNamedRepo(haystack []scm.Repo, needle string) bool { - - if os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" { - colorlog.PrintError("GHORG_PRUNE (--prune) does not currently work in combination with GHORG_PRESERVE_DIRECTORY_STRUCTURE (--preserve-dir), this will come in later versions") - os.Exit(1) - } - for _, repo := range haystack { basepath := filepath.Base(repo.Path) diff --git a/cmd/clone_test.go b/cmd/clone_test.go index c36b3e0..2c1d99b 100644 --- a/cmd/clone_test.go +++ b/cmd/clone_test.go @@ -4,6 +4,7 @@ import ( "errors" "log" "os" + "path/filepath" "reflect" "strings" "testing" @@ -525,3 +526,103 @@ func Test_filterDownReposIfTargetReposPathEnabled(t *testing.T) { }) } } + +func TestReadDirRecursively(t *testing.T) { + dir, err := os.MkdirTemp("", "testing") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(dir) + + outputDirAbsolutePath = dir + + subDir := filepath.Join(dir, "subdir") + if err := os.Mkdir(subDir, 0o755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + gitDir := filepath.Join(subDir, ".git") + if err := os.Mkdir(gitDir, 0o755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + files, err := readDirRecursively(dir) + if err != nil { + t.Fatalf("readDirRecursively returned an error: %v", err) + } + + if len(files) != 1 { + t.Errorf("Expected %d directories, got %d", 1, len(files)) + } + + if len(files) > 0 && files[0].Name() != "subdir" { + t.Errorf("Expected directory name 'subdir', got '%s'", files[0].Name()) + } +} + +func TestReadDirRecursivelyNoGitDir(t *testing.T) { + dir, err := os.MkdirTemp("", "testing") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(dir) + + outputDirAbsolutePath = dir + + subDir := filepath.Join(dir, "subdir") + if err := os.Mkdir(subDir, 0o755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + files, err := readDirRecursively(dir) + if err != nil { + t.Fatalf("readDirRecursively returned an error: %v", err) + } + + if len(files) != 0 { + t.Errorf("Expected %d directories, got %d", 0, len(files)) + } +} + +func TestReadDirRecursivelyWithGitSubmodule(t *testing.T) { + dir, err := os.MkdirTemp("", "testing") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(dir) + + outputDirAbsolutePath = dir + + subDir := filepath.Join(dir, "subdir") + if err := os.Mkdir(subDir, 0o755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + gitDir := filepath.Join(subDir, ".git") + if err := os.Mkdir(gitDir, 0o755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + submoduleDir := filepath.Join(subDir, "submodule") + if err := os.Mkdir(submoduleDir, 0o755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + + submoduleGitFile := filepath.Join(submoduleDir, ".git") + if _, err := os.Create(submoduleGitFile); err != nil { + t.Fatalf("Failed to create file: %v", err) + } + + files, err := readDirRecursively(dir) + if err != nil { + t.Fatalf("readDirRecursively returned an error: %v", err) + } + + if len(files) != 1 { + t.Errorf("Expected %d directories, got %d", 1, len(files)) + } + + if len(files) > 0 && files[0].Name() != "subdir" { + t.Errorf("Expected directory name 'subdir', got '%s'", files[0].Name()) + } +} diff --git a/scripts/gitlab_cloud_integration_tests.sh b/scripts/gitlab_cloud_integration_tests.sh index 201db79..d3b231e 100755 --- a/scripts/gitlab_cloud_integration_tests.sh +++ b/scripts/gitlab_cloud_integration_tests.sh @@ -87,6 +87,21 @@ else exit 1 fi +# PRUNE AND PRESERVE DIR +ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --prune --preserve-dir +git init "${HOME}"/ghorg/"${GITLAB_GROUP}"/wayne-enterprises/wayne-industries/prunable +ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --prune --preserve-dir + +if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP}"/wayne-enterprises/wayne-industries/microservice ] && \ + [ ! -e "${HOME}"/ghorg/"${GITLAB_GROUP}"/wayne-enterprises/wayne-industries/prunable ] +then + echo "Pass: gitlab org clone preserve dir, prune" + rm -rf "${HOME}/ghorg/${GITLAB_GROUP}" +else + echo "Fail: gitlab org clone preserve dir, prune" + exit 1 +fi + # REPO NAME COLLISION ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab