github-personal/ghorg
1
0
Fork 0
mirror of https://github.com/outbackdingo/ghorg.git synced 2026-01-27 10:19:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add/ghorg stats (#449)

Browse Source
This commit is contained in:
gabrie30
2024-09-15 12:28:40 -07:00
committed by GitHub
parent db111bc1c7
commit 0bfd95159c
5 changed files with 203 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
cmd/clone.go
View File

@@ -3,6 +3,7 @@ package cmd
import (
"bufio"
"crypto/sha256"
"fmt"
"log"
"net/url"
@@ -12,6 +13,7 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/gabrie30/ghorg/colorlog"
"github.com/gabrie30/ghorg/configs"
@@ -37,6 +39,9 @@ Or see examples directory at https://github.com/gabrie30/ghorg/tree/master/examp
Run: cloneFunc,
}
var cachedDirSizeMB float64
var isDirSizeCached bool
func cloneFunc(cmd *cobra.Command, argz []string) {
if cmd.Flags().Changed("path") {
absolutePath := configs.EnsureTrailingSlashOnFilePath((cmd.Flag("path").Value.String()))
@@ -157,6 +162,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
os.Setenv("GHORG_SKIP_ARCHIVED", "true")
}
if cmd.Flags().Changed("stats-enabled") {
os.Setenv("GHORG_STATS_ENABLED", "true")
}
if cmd.Flags().Changed("no-clean") {
os.Setenv("GHORG_NO_CLEAN", "true")
}
@@ -914,9 +923,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}
}
var pruneCount int
cloneInfosCount := len(cloneInfos)
cloneErrorsCount := len(cloneErrors)
allReposToCloneCount := len(cloneTargets)
// Now, clean up local repos that don't exist in remote, if prune flag is set
if os.Getenv("GHORG_PRUNE") == "true" {
pruneRepos(cloneTargets)
pruneCount = pruneRepos(cloneTargets)
}
if os.Getenv("GHORG_QUIET") != "true" {
@@ -927,7 +940,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}
}
if os.Getenv("GHORG_EXIT_CODE_ON_CLONE_INFOS") != "0" && len(cloneInfos) > 0 {
// This needs to be called after printFinishedWithDirSize()
if os.Getenv("GHORG_STATS_ENABLED") == "true" {
date := time.Now().Format("2006-01-02 15:04:05")
writeGhorgStats(date, allReposToCloneCount, cloneCount, pulledCount, cloneInfosCount, cloneErrorsCount, updateRemoteCount, newCommits, pruneCount, hasCollisions)
}
if os.Getenv("GHORG_EXIT_CODE_ON_CLONE_INFOS") != "0" && cloneInfosCount > 0 {
exitCode, err := strconv.Atoi(os.Getenv("GHORG_EXIT_CODE_ON_CLONE_INFOS"))
if err != nil {
colorlog.PrintError("Could not convert GHORG_EXIT_CODE_ON_CLONE_INFOS from string to integer")
@@ -937,7 +956,7 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
os.Exit(exitCode)
}
if len(cloneErrors) > 0 {
if cloneErrorsCount > 0 {
exitCode, err := strconv.Atoi(os.Getenv("GHORG_EXIT_CODE_ON_CLONE_ISSUES"))
if err != nil {
colorlog.PrintError("Could not convert GHORG_EXIT_CODE_ON_CLONE_ISSUES from string to integer")
@@ -949,21 +968,136 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}
func writeGhorgStats(date string, allReposToCloneCount, cloneCount, pulledCount, cloneInfosCount, cloneErrorsCount, updateRemoteCount, newCommits, pruneCount int, hasCollisions bool) error {
statsFilePath := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), "_ghorg_stats.csv")
fileExists := true
if _, err := os.Stat(statsFilePath); os.IsNotExist(err) {
fileExists = false
}
header := "datetime,clonePath,scm,cloneType,cloneTarget,totalCount,newClonesCount,existingResourcesPulledCount,dirSizeInMB,newCommits,cloneInfosCount,cloneErrorsCount,updateRemoteCount,pruneCount,hasCollisions,ghorgignore,ghorgVersion\n"
var file *os.File
var err error
if fileExists {
// Read the existing header
existingHeader, err := readFirstLine(statsFilePath)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error reading header from stats file: %v", err))
return err
}
// Check if the existing header is different from the new header, need to add a newline
if existingHeader+"\n" != header {
hashedHeader := fmt.Sprintf("%x", sha256.Sum256([]byte(header)))
newHeaderFilePath := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), fmt.Sprintf("ghorg_stats_new_header_%s.csv", hashedHeader))
// Create a new file with the new header
file, err = os.OpenFile(newHeaderFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error creating new header stats file: %v", err))
return err
}
if _, err := file.WriteString(header); err != nil {
colorlog.PrintError(fmt.Sprintf("Error writing new header to GHORG_STATS file: %v", err))
return err
}
} else {
// Open the existing file in append mode
file, err = os.OpenFile(statsFilePath, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error opening stats file for appending: %v", err))
return err
}
}
} else {
// Create the file and write the header
file, err = os.OpenFile(statsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
colorlog.PrintError(fmt.Sprintf("Error creating stats file: %v", err))
return err
}
if _, err := file.WriteString(header); err != nil {
colorlog.PrintError(fmt.Sprintf("Error writing header to GHORG_STATS file: %v", err))
return err
}
}
defer file.Close()
data := fmt.Sprintf("%v,%v,%v,%v,%v,%v,%v,%v,%.2f,%v,%v,%v,%v,%v,%v,%v,%v\n",
date,
outputDirAbsolutePath,
os.Getenv("GHORG_SCM_TYPE"),
os.Getenv("GHORG_CLONE_TYPE"),
targetCloneSource,
allReposToCloneCount,
cloneCount,
pulledCount,
cachedDirSizeMB,
newCommits,
cloneInfosCount,
cloneErrorsCount,
updateRemoteCount,
pruneCount,
hasCollisions,
configs.GhorgIgnoreDetected(),
GetVersion())
if _, err := file.WriteString(data); err != nil {
colorlog.PrintError(fmt.Sprintf("Error writing data to GHORG_STATS file: %v", err))
return err
}
return nil
}
func readFirstLine(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
scanner := bufio.NewScanner(file)
if scanner.Scan() {
return scanner.Text(), nil
}
if err := scanner.Err(); err != nil {
return "", err
}
return "", nil
}
func printFinishedWithDirSize() {
dirSizeMB, err := calculateDirSizeInMb(outputDirAbsolutePath)
dirSizeMB, err := getCachedOrCalculatedOutputDirSizeInMb()
if err != nil {
if os.Getenv("GHORG_DEBUG") == "true" {
colorlog.PrintError(fmt.Sprintf("Error calculating directory size: %v", err))
}
colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s", outputDirAbsolutePath))
} else {
if dirSizeMB > 1000 {
dirSizeGB := dirSizeMB / 1000
colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s (Size: %.2f GB)", outputDirAbsolutePath, dirSizeGB))
} else {
colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s (Size: %.2f MB)", outputDirAbsolutePath, dirSizeMB))
}
return
}
if dirSizeMB > 1000 {
dirSizeGB := dirSizeMB / 1000
colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s (Size: %.2f GB)", outputDirAbsolutePath, dirSizeGB))
} else {
colorlog.PrintSuccess(fmt.Sprintf("\nFinished! %s (Size: %.2f MB)", outputDirAbsolutePath, dirSizeMB))
}
}
func getCachedOrCalculatedOutputDirSizeInMb() (float64, error) {
if !isDirSizeCached {
dirSizeMB, err := calculateDirSizeInMb(outputDirAbsolutePath)
if err != nil {
return 0, err
}
cachedDirSizeMB = dirSizeMB
isDirSizeCached = true
}
return cachedDirSizeMB, nil
}
func calculateDirSizeInMb(path string) (float64, error) {
@@ -1056,7 +1190,8 @@ func filterByTargetReposPath(cloneTargets []scm.Repo) []scm.Repo {
return cloneTargets
}
func pruneRepos(cloneTargets []scm.Repo) {
func pruneRepos(cloneTargets []scm.Repo) int {
count := 0
colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...")
files, err := os.ReadDir(outputDirAbsolutePath)
@@ -1080,6 +1215,7 @@ func pruneRepos(cloneTargets []scm.Repo) {
colorlog.PrintSubtleInfo(
fmt.Sprintf("Deleting %s", filepath.Join(outputDirAbsolutePath, f.Name())))
err = os.RemoveAll(filepath.Join(outputDirAbsolutePath, f.Name()))
count++
if err != nil {
log.Fatal(err)
}
@@ -1088,6 +1224,8 @@ func pruneRepos(cloneTargets []scm.Repo) {
}
}
}
return count
}
func printCloneStatsMessage(cloneCount, pulledCount, updateRemoteCount, newCommits int) {
@@ -1245,6 +1383,9 @@ func PrintConfigs() {
}
colorlog.PrintInfo("* Config Used : " + os.Getenv("GHORG_CONFIG"))
if os.Getenv("GHORG_STATS_ENABLED") == "true" {
colorlog.PrintInfo("* Stats Enabled : " + os.Getenv("GHORG_STATS_ENABLED"))
}
colorlog.PrintInfo("* Ghorg version : " + GetVersion())
colorlog.PrintInfo("*************************************")

5
cmd/root.go
View File

@@ -67,6 +67,7 @@ var (
noToken bool
quietMode bool
noDirSize bool
ghorgStatsEnabled bool
args []string
cloneErrors []string
cloneInfos []string
@@ -162,6 +163,8 @@ func getOrSetDefaults(envVar string) {
os.Setenv(envVar, "25")
case "GHORG_QUIET":
os.Setenv(envVar, "false")
case "GHORG_STATS_ENABLED":
os.Setenv(envVar, "false")
case "GHORG_EXIT_CODE_ON_CLONE_INFOS":
os.Setenv(envVar, "0")
case "GHORG_EXIT_CODE_ON_CLONE_ISSUES":
@@ -249,6 +252,7 @@ func InitConfig() {
getOrSetDefaults("GHORG_INCLUDE_SUBMODULES")
getOrSetDefaults("GHORG_EXIT_CODE_ON_CLONE_INFOS")
getOrSetDefaults("GHORG_EXIT_CODE_ON_CLONE_ISSUES")
getOrSetDefaults("GHORG_STATS_ENABLED")
// Optionally set
getOrSetDefaults("GHORG_TARGET_REPOS_PATH")
getOrSetDefaults("GHORG_CLONE_DEPTH")
@@ -321,6 +325,7 @@ func init() {
cloneCmd.Flags().BoolVar(&backup, "backup", false, "GHORG_BACKUP - Backup mode, clone as mirror, no working copy (ignores branch parameter)")
cloneCmd.Flags().BoolVar(&quietMode, "quiet", false, "GHORG_QUIET - Emit critical output only")
cloneCmd.Flags().BoolVar(&includeSubmodules, "include-submodules", false, "GHORG_INCLUDE_SUBMODULES - Include submodules in all clone and pull operations.")
cloneCmd.Flags().BoolVar(&ghorgStatsEnabled, "stats-enabled", false, "GHORG_STATS_ENABLED - Creates a CSV in the GHORG_ABSOLUTE_PATH_TO_CLONE_TO called _ghorg_stats.csv with info about each clone. This allows you to track clone data over time such as number of commits and size in megabytes of the clone directory.")
cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL - Change SCM base url, for on self hosted instances (currently gitlab, gitea and github (use format of https://git.mydomain.com/api/v3))")
cloneCmd.Flags().StringVarP(&concurrency, "concurrency", "", "", "GHORG_CONCURRENCY - Max goroutines to spin up while cloning (default 25)")
cloneCmd.Flags().StringVarP(&cloneDepth, "clone-depth", "", "", "GHORG_CLONE_DEPTH - Create a shallow clone with a history truncated to the specified number of commits")