From 37c7d6923823216c6a7013b8fe2dd246bd4ef047 Mon Sep 17 00:00:00 2001 From: Jay Gabriels Date: Sat, 11 Dec 2021 15:41:24 -0800 Subject: [PATCH] Add ghorg fetch all (#170) --- CHANGELOG.md | 1 + cmd/clone.go | 30 ++++++++++++++++++++++++++++++ cmd/root.go | 7 ++++++- sample-conf.yaml | 31 ++++++++++++++++--------------- 4 files changed, 53 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77cfc2f..3a4beb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) ## [1.7.5] - unreleased ### Added - GHORG_DRY_RUN to do dry runs on clones +- GHORG_FETCH_ALL to run fetch all on each repo - output for long running repo fetches - support for cloning github enterprise repos - log repos cloned vs pulled at end of run diff --git a/cmd/clone.go b/cmd/clone.go index 2a84d73..b9b0bcb 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -90,6 +90,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Setenv("GHORG_NO_CLEAN", "true") } + if cmd.Flags().Changed("fetch-all") { + os.Setenv("GHORG_FETCH_ALL", "true") + } + if cmd.Flags().Changed("dry-run") { os.Setenv("GHORG_DRY_RUN", "true") } @@ -480,6 +484,16 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { action = "pulling" pulledCount++ + + if os.Getenv("GHORG_FETCH_ALL") == "true" { + err = git.FetchAll(repo) + + if err != nil { + e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err) + cloneErrors = append(cloneErrors, e) + return + } + } } } else { // if https clone and github/gitlab add personal access token to url @@ -520,6 +534,16 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) { cloneErrors = append(cloneErrors, e) return } + + if os.Getenv("GHORG_FETCH_ALL") == "true" { + err = git.FetchAll(repo) + + if err != nil { + e := fmt.Sprintf("Could not fetch remotes in Repo: %s Error: %v", repo.URL, err) + cloneErrors = append(cloneErrors, e) + return + } + } } colorlog.PrintSuccess("Success " + action + " repo: " + repo.URL + " -> branch: " + branch) @@ -591,6 +615,12 @@ func PrintConfigs() { if os.Getenv("GHORG_NO_CLEAN") == "true" { colorlog.PrintInfo("* No Clean : " + "true") } + if os.Getenv("GHORG_FETCH_ALL") == "true" { + colorlog.PrintInfo("* Fetch All : " + "true") + } + if os.Getenv("GHORG_DRY_RUN") == "true" { + colorlog.PrintInfo("* Dry Run : " + "true") + } colorlog.PrintInfo("* Config Used : " + os.Getenv("GHORG_CONF")) diff --git a/cmd/root.go b/cmd/root.go index 4454499..0b09812 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,7 +19,6 @@ var ( cloneType string scmType string bitbucketUsername string - namespace string color string baseURL string concurrency string @@ -33,6 +32,7 @@ var ( cloneWiki bool preserveDir bool insecureGitlabClient bool + fetchAll bool args []string cloneErrors []string cloneInfos []string @@ -91,6 +91,8 @@ func getOrSetDefaults(envVar string) { os.Setenv(envVar, "false") case "GHORG_NO_CLEAN": os.Setenv(envVar, "false") + case "GHORG_FETCH_ALL": + os.Setenv(envVar, "false") case "GHORG_DRY_RUN": os.Setenv(envVar, "false") case "GHORG_INSECURE_GITLAB_CLIENT": @@ -153,6 +155,8 @@ func InitConfig() { getOrSetDefaults("GHORG_SKIP_ARCHIVED") getOrSetDefaults("GHORG_SKIP_FORKS") getOrSetDefaults("GHORG_NO_CLEAN") + getOrSetDefaults("GHORG_FETCH_ALL") + getOrSetDefaults("GHORG_DRY_RUN") getOrSetDefaults("GHORG_CLONE_WIKI") getOrSetDefaults("GHORG_INSECURE_GITLAB_CLIENT") getOrSetDefaults("GHORG_BACKUP") @@ -198,6 +202,7 @@ func init() { cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "", "GHORG_CLONE_TYPE - clone target type, user or org (default org)") cloneCmd.Flags().BoolVar(&skipArchived, "skip-archived", false, "GHORG_SKIP_ARCHIVED - skips archived repos, github/gitlab/gitea only") cloneCmd.Flags().BoolVar(&noClean, "no-clean", false, "GHORG_NO_CLEAN - only clones new repos and does not perform a git clean on existing repos") + cloneCmd.Flags().BoolVar(&fetchAll, "fetch-all", false, "GHORG_FETCH_ALL - fetches all remote branches for each repo by running a git fetch --all") cloneCmd.Flags().BoolVar(&dryRun, "dry-run", false, "GHORG_DRY_RUN - perform a dry run of the clone; fetches repos but does not clone them") cloneCmd.Flags().BoolVar(&insecureGitlabClient, "insecure-gitlab-client", false, "GHORG_INSECURE_GITLAB_CLIENT - skip TLS certificate verification for hosted gitlab instances") cloneCmd.Flags().BoolVar(&cloneWiki, "clone-wiki", false, "GHORG_CLONE_WIKI - Additionally clone the wiki page for repo") diff --git a/sample-conf.yaml b/sample-conf.yaml index 9b7cc37..155a75e 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -21,7 +21,7 @@ GHORG_GITLAB_TOKEN: # clones repos in a directory structure that matches gitlab namespaces eg company/unit/subunit/app would clone into ghorg/org/unit/subunit/app # flag (--preserve-dir) -GHORG_PRESERVE_DIRECTORY_STRUCTURE: +GHORG_PRESERVE_DIRECTORY_STRUCTURE: false # Skip TLS certificate verification for hosted gitlab instances # flag (--insecure-gitlab-client) @@ -60,12 +60,12 @@ GHORG_SCM_BASE_URL: # Which provider to clone from (github, gitlab, gitea or bitbucket) # default: github # flag (--scm, -s) eg: --scm=gitlab -GHORG_SCM_TYPE: +GHORG_SCM_TYPE: github # How you'd like to clone the repos (https or ssh) # default: https # flag (--protocol) eg: --protocol=https -GHORG_CLONE_PROTOCOL: +GHORG_CLONE_PROTOCOL: https # This is where your ghorg directory will be created, use absolute pathing, shell expansions will not work # default: YOUR_HOME_DIR/ghorg @@ -79,37 +79,34 @@ GHORG_ABSOLUTE_PATH_TO_CLONE_TO: GHORG_BRANCH: # Type of entity to clone (user or org) -# default: org # flag (--clone-type, -c) eg: --clone-type=user -GHORG_CLONE_TYPE: +GHORG_CLONE_TYPE: org # Color output (enabled, disabled) -# default: disabled # flag (--color) eg: --color=disabled -GHORG_COLOR: +GHORG_COLOR: disabled # A comma separated list of topics to filter by, currently github/gitea only -# default: "" # flag (--topics) eg: --topics=frontend # If any topics exist here, ghorg will only clone repos that match at least one of these topics GHORG_TOPICS: # Skip archived repos, currently github/gitlab/gitea only # flag (--skip-archived) -GHORG_SKIP_ARCHIVED: +GHORG_SKIP_ARCHIVED: false # Skip repos that are forks, currently github/gitlab/gitea only # flag (--skip-forks) -GHORG_SKIP_FORKS: +GHORG_SKIP_FORKS: false # Backup mode, clone as mirror, no working copy (ignores branch parameter) # flag (--backup) -GHORG_BACKUP: +GHORG_BACKUP: false # Max goroutines created while cloning # default: 25 # flag (--concurrency) -GHORG_CONCURRENCY: +GHORG_CONCURRENCY: 25 # Folder ghorg will clone all repos into. Cloning will result in: GHORG_ABSOLUTE_PATH_TO_CLONE_TO/GHORG_OUTPUT_DIR/* # default: {org/user you are cloning} @@ -128,15 +125,19 @@ GHORG_MATCH_REGEX: # Only clones new repos and does not perform a git clean on existing repos. Useful if you don't want to lose changes made to repos in the org/user directory. # flag (--no-clean) -GHORG_NO_CLEAN: +GHORG_NO_CLEAN: false # Additionally clone the wiki page for repo # flag (--clone-wiki) -GHORG_CLONE_WIKI: +GHORG_CLONE_WIKI: false # Perform a dry run of the clone; fetches repos but does not clone them. # flag (--dry-run) -GHORG_DRY_RUN: +GHORG_DRY_RUN: false + +# Perform a dry run of the clone; fetches repos but does not clone them. +# flag (--fetch-all) +GHORG_FETCH_ALL: false # Specifies the location of your ghorg conf.yaml, allowing you to have many configuration files, or none at all # default: ghorg looks in $HOME/.config/ghorg/conf.yaml, if not set in that location nor as a commandline flag, ghorg will use all default values