ghorg
ghorg copied to clipboard
gabrie30
Include git error message in log in case of a failed git command
Is your feature request related to a problem? Please describe.
When things go wrong it would be great to get more information about what went wrong.
Currently the log is like this:
Problem trying to pull branch: master for: https://github.com/<org>/<repo> Error: exit status 1
In that case it would be great to see the actual error message from git.
As far as I know the only way to get more information currently is to enable the DEBUG mode (GHORG_DEBUG=true). But — with a huge number of repos — this creates a lot of log, and most of the repos work just fine.
Describe the solution you'd like
In case of a git command not executing successfully (exit status != 0), please include the error message in the logs.
Describe alternatives you've considered
—
Additional context
—
Thanks for ghorg, great tool!
some proof of concept … well … hack (see below)
example output before:
Problem setting remote with credentials on: testRepoOne Error: exit status 128
example output after:
Problem setting remote with credentials on: testRepoOne Error: exit status 128 Output: fatal: not a git repository (or any of the parent directories): .git
Subject: [PATCH] Don’t swallow command output
---
Index: git/git.go
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/git/git.go b/git/git.go
--- a/git/git.go (revision c590809b7694be2a791afaf5beea1bca5010bd9e)
+++ b/git/git.go (date 1742811633651)
@@ -12,19 +12,19 @@
)
type Gitter interface {
- Clone(scm.Repo) error
- Reset(scm.Repo) error
- Pull(scm.Repo) error
- SetOrigin(scm.Repo) error
- SetOriginWithCredentials(scm.Repo) error
- Clean(scm.Repo) error
- Checkout(scm.Repo) error
+ Clone(scm.Repo) (string, error)
+ Reset(scm.Repo) (string, error)
+ Pull(scm.Repo) (string, error)
+ SetOrigin(scm.Repo) (string, error)
+ SetOriginWithCredentials(scm.Repo) (string, error)
+ Clean(scm.Repo) (string, error)
+ Checkout(scm.Repo) (string, error)
RevListCompare(scm.Repo, string, string) (string, error)
ShortStatus(scm.Repo) (string, error)
Branch(scm.Repo) (string, error)
- UpdateRemote(scm.Repo) error
- FetchAll(scm.Repo) error
- FetchCloneBranch(scm.Repo) error
+ UpdateRemote(scm.Repo) (string, error)
+ FetchAll(scm.Repo) (string, error)
+ FetchCloneBranch(scm.Repo) (string, error)
RepoCommitCount(scm.Repo) (int, error)
}
@@ -34,7 +34,7 @@
return GitClient{}
}
-func printDebugCmd(cmd *exec.Cmd, repo scm.Repo) error {
+func printDebugCmd(cmd *exec.Cmd, repo scm.Repo) (string, error) {
fmt.Println("------------- GIT DEBUG -------------")
fmt.Printf("GHORG_OUTPUT_DIR=%v\n", os.Getenv("GHORG_OUTPUT_DIR"))
fmt.Printf("GHORG_ABSOLUTE_PATH_TO_CLONE_TO=%v\n", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"))
@@ -48,10 +48,10 @@
if err != nil {
fmt.Printf("Error: %v\n", err)
}
- return err
+ return string(output), err
}
-func (g GitClient) Clone(repo scm.Repo) error {
+func (g GitClient) Clone(repo scm.Repo) (string, error) {
args := []string{"clone", repo.CloneURL, repo.HostPath}
if os.Getenv("GHORG_INCLUDE_SUBMODULES") == "true" {
@@ -82,31 +82,38 @@
return printDebugCmd(cmd, repo)
}
- err := cmd.Run()
- return err
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) SetOriginWithCredentials(repo scm.Repo) error {
+func (g GitClient) SetOriginWithCredentials(repo scm.Repo) (string, error) {
args := []string{"remote", "set-url", "origin", repo.CloneURL}
cmd := exec.Command("git", args...)
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return string(output), err
+ }
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) SetOrigin(repo scm.Repo) error {
+func withOutput(outputIn []byte, errorIn error) (string, error) {
+ return strings.TrimSpace(string(outputIn)), errorIn
+}
+
+func (g GitClient) SetOrigin(repo scm.Repo) (string, error) {
args := []string{"remote", "set-url", "origin", repo.URL}
cmd := exec.Command("git", args...)
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) Checkout(repo scm.Repo) error {
+func (g GitClient) Checkout(repo scm.Repo) (string, error) {
cmd := exec.Command("git", "checkout", repo.CloneBranch)
cmd.Dir = repo.HostPath
@@ -114,28 +121,28 @@
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) Clean(repo scm.Repo) error {
+func (g GitClient) Clean(repo scm.Repo) (string, error) {
cmd := exec.Command("git", "clean", "-f", "-d")
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) UpdateRemote(repo scm.Repo) error {
+func (g GitClient) UpdateRemote(repo scm.Repo) (string, error) {
cmd := exec.Command("git", "remote", "update")
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) Pull(repo scm.Repo) error {
+func (g GitClient) Pull(repo scm.Repo) (string, error) {
args := []string{"pull", "origin", repo.CloneBranch}
if os.Getenv("GHORG_INCLUDE_SUBMODULES") == "true" {
@@ -157,19 +164,19 @@
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) Reset(repo scm.Repo) error {
+func (g GitClient) Reset(repo scm.Repo) (string, error) {
cmd := exec.Command("git", "reset", "--hard", "origin/"+repo.CloneBranch)
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
-func (g GitClient) FetchAll(repo scm.Repo) error {
+func (g GitClient) FetchAll(repo scm.Repo) (string, error) {
args := []string{"fetch", "--all"}
if os.Getenv("GHORG_CLONE_DEPTH") != "" {
@@ -182,7 +189,7 @@
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
func (g GitClient) Branch(repo scm.Repo) (string, error) {
@@ -191,17 +198,10 @@
cmd := exec.Command("git", args...)
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
- if err := printDebugCmd(cmd, repo); err != nil {
- return "", err
- }
+ return printDebugCmd(cmd, repo)
}
- output, err := cmd.Output()
- if err != nil {
- return "", err
- }
-
- return strings.TrimSpace(string(output)), nil
+ return withOutput(cmd.CombinedOutput())
}
// RevListCompare returns the list of commits in the local branch that are not in the remote branch.
@@ -214,7 +214,7 @@
return strings.TrimSpace(string(output)), nil
}
-func (g GitClient) FetchCloneBranch(repo scm.Repo) error {
+func (g GitClient) FetchCloneBranch(repo scm.Repo) (string, error) {
args := []string{"fetch", "origin", repo.CloneBranch}
if os.Getenv("GHORG_CLONE_DEPTH") != "" {
@@ -227,7 +227,7 @@
if os.Getenv("GHORG_DEBUG") != "" {
return printDebugCmd(cmd, repo)
}
- return cmd.Run()
+ return withOutput(cmd.CombinedOutput())
}
func (g GitClient) ShortStatus(repo scm.Repo) (string, error) {
@@ -236,17 +236,10 @@
cmd := exec.Command("git", args...)
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
- if err := printDebugCmd(cmd, repo); err != nil {
- return "", err
- }
+ return printDebugCmd(cmd, repo)
}
- output, err := cmd.Output()
- if err != nil {
- return "", err
- }
-
- return strings.TrimSpace(string(output)), nil
+ return withOutput(cmd.CombinedOutput())
}
func (g GitClient) RepoCommitCount(repo scm.Repo) (int, error) {
@@ -255,7 +248,7 @@
cmd.Dir = repo.HostPath
if os.Getenv("GHORG_DEBUG") != "" {
- err := printDebugCmd(cmd, repo)
+ _, err := printDebugCmd(cmd, repo)
if err != nil {
return 0, err
}
Index: cmd/clone.go
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/cmd/clone.go b/cmd/clone.go
--- a/cmd/clone.go (revision c590809b7694be2a791afaf5beea1bca5010bd9e)
+++ b/cmd/clone.go (date 1742807809059)
@@ -791,22 +791,24 @@
return
}
+ var msg string
action := "cloning"
+
if repoWillBePulled {
// prevents git from asking for user for credentials, needs to be unset so creds aren't stored
- err := git.SetOriginWithCredentials(repo)
+ msg, err := git.SetOriginWithCredentials(repo)
if err != nil {
- e := fmt.Sprintf("Problem setting remote with credentials on: %s Error: %v", repo.Name, err)
+ e := fmt.Sprintf("Problem setting remote with credentials on: %s Error: %v Output: %v", repo.Name, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
if os.Getenv("GHORG_BACKUP") == "true" {
- err := git.UpdateRemote(repo)
+ msg, err := git.UpdateRemote(repo)
action = "updating remote"
// Theres no way to tell if a github repo has a wiki to clone
if err != nil && repo.IsWiki {
- e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v Output: %v", repo.URL, err, msg)
cloneInfos = append(cloneInfos, e)
return
}
@@ -819,11 +821,11 @@
updateRemoteCount++
} else if os.Getenv("GHORG_NO_CLEAN") == "true" {
action = "fetching"
- err := git.FetchAll(repo)
+ msg, err := git.FetchAll(repo)
// Theres no way to tell if a github repo has a wiki to clone
if err != nil && repo.IsWiki {
- e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v Output: %v", repo.URL, err, msg)
cloneInfos = append(cloneInfos, e)
return
}
@@ -836,23 +838,24 @@
} else {
if os.Getenv("GHORG_FETCH_ALL") == "true" {
- err = git.FetchAll(repo)
+ msg, err = git.FetchAll(repo)
if err != nil {
- e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Could not fetch remotes: %s Error: %v Output: %v", repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
}
- err := git.Checkout(repo)
+ msg, err := git.Checkout(repo)
if err != nil {
git.FetchCloneBranch(repo)
// Retry checkout
- errRetry := git.Checkout(repo)
+ var msg2 string
+ msg2, errRetry := git.Checkout(repo)
if errRetry != nil {
- e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents/commits, no changes made on: %s Error: %v", repo.CloneBranch, repo.URL, errRetry)
+ e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents/commits, no changes made on: %s Error: %v Outputs: %v / %v", repo.CloneBranch, repo.URL, errRetry, msg, msg2)
cloneErrors = append(cloneErrors, e)
return
}
@@ -866,26 +869,26 @@
repo.Commits.CountPrePull = count
- err = git.Clean(repo)
+ msg, err = git.Clean(repo)
if err != nil {
- e := fmt.Sprintf("Problem running git clean: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Problem running git clean: %s Error: %v Output: %v", repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
- err = git.Reset(repo)
+ msg, err = git.Reset(repo)
if err != nil {
- e := fmt.Sprintf("Problem resetting branch: %s for: %s Error: %v", repo.CloneBranch, repo.URL, err)
+ e := fmt.Sprintf("Problem resetting branch: %s for: %s Error: %v Output: %v", repo.CloneBranch, repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
- err = git.Pull(repo)
+ msg, err = git.Pull(repo)
if err != nil {
- e := fmt.Sprintf("Problem trying to pull branch: %v for: %s Error: %v", repo.CloneBranch, repo.URL, err)
+ e := fmt.Sprintf("Problem trying to pull branch: %v for: %s Error: %v Output: %v", repo.CloneBranch, repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
@@ -903,34 +906,35 @@
pulledCount++
}
- err = git.SetOrigin(repo)
+ msg, err = git.SetOrigin(repo)
if err != nil {
- e := fmt.Sprintf("Problem resetting remote: %s Error: %v", repo.Name, err)
+ e := fmt.Sprintf("Problem resetting remote: %s Error: %v Output: %v", repo.Name, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
} else {
// if https clone and github/gitlab add personal access token to url
- err = git.Clone(repo)
+ msg, err = git.Clone(repo)
// Theres no way to tell if a github repo has a wiki to clone
if err != nil && repo.IsWiki {
- e := fmt.Sprintf("Wiki may be enabled but there was no content to clone: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Wiki may be enabled but there was no content to clone: %s Error: %v Output: %v", repo.URL, err, msg)
cloneInfos = append(cloneInfos, e)
return
}
if err != nil {
- e := fmt.Sprintf("Problem trying to clone: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Problem trying to clone: %s Error: %v Output: %v", repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
if os.Getenv("GHORG_BRANCH") != "" {
- err := git.Checkout(repo)
+ var msg2 string
+ msg2, err := git.Checkout(repo)
if err != nil {
- e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents/commits, no changes to: %s Error: %v", repo.CloneBranch, repo.URL, err)
+ e := fmt.Sprintf("Could not checkout out %s, branch may not exist or may not have any contents/commits, no changes to: %s Error: %v Outputs: %v / %v", repo.CloneBranch, repo.URL, err, msg, msg2)
cloneInfos = append(cloneInfos, e)
return
}
@@ -940,20 +944,20 @@
// TODO: make configs around remote name
// we clone with api-key in clone url
- err = git.SetOrigin(repo)
+ msg, err = git.SetOrigin(repo)
// if repo has wiki, but content does not exist this is going to error
if err != nil {
- e := fmt.Sprintf("Problem trying to set remote: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Problem trying to set remote: %s Error: %v Output: %v", repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
if os.Getenv("GHORG_FETCH_ALL") == "true" {
- err = git.FetchAll(repo)
+ msg, err = git.FetchAll(repo)
if err != nil {
- e := fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)
+ e := fmt.Sprintf("Could not fetch remotes: %s Error: %v Output: %v", repo.URL, err, msg)
cloneErrors = append(cloneErrors, e)
return
}
@@ -1042,14 +1046,18 @@
os.Exit(exitCode)
}
- if cloneErrorsCount > 0 {
- exitCode, err := strconv.Atoi(os.Getenv("GHORG_EXIT_CODE_ON_CLONE_ISSUES"))
- if err != nil {
- colorlog.PrintError("Could not convert GHORG_EXIT_CODE_ON_CLONE_ISSUES from string to integer")
- os.Exit(1)
- }
+ if os.Getenv("GHORG_DONT_EXIT_UNDER_TEST") != "true" {
+ if cloneErrorsCount > 0 {
+ exitCode, err := strconv.Atoi(os.Getenv("GHORG_EXIT_CODE_ON_CLONE_ISSUES"))
+ if err != nil {
+ colorlog.PrintError("Could not convert GHORG_EXIT_CODE_ON_CLONE_ISSUES from string to integer")
+ os.Exit(1)
+ }
- os.Exit(exitCode)
+ os.Exit(exitCode)
+ }
+ } else {
+ cloneErrorsCount = 0
}
}
Index: cmd/clone_test.go
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/cmd/clone_test.go b/cmd/clone_test.go
--- a/cmd/clone_test.go (revision c590809b7694be2a791afaf5beea1bca5010bd9e)
+++ b/cmd/clone_test.go (date 1742812169961)
@@ -1,6 +1,9 @@
package cmd
import (
+ "errors"
+ "fmt"
+ "io"
"log"
"os"
"reflect"
@@ -57,48 +60,58 @@
return MockGitClient{}
}
-func (g MockGitClient) Clone(repo scm.Repo) error {
+func (g MockGitClient) Clone(repo scm.Repo) (string, error) {
+ if repo.Name == "cannotPull" {
+ err := os.Mkdir(repo.HostPath, 0o700)
+ if err != nil {
+ log.Fatal(err)
+ }
+ return "", nil
+ }
_, err := os.MkdirTemp(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), repo.Name)
if err != nil {
log.Fatal(err)
}
- return nil
+ return "", nil
}
-func (g MockGitClient) SetOrigin(repo scm.Repo) error {
- return nil
+func (g MockGitClient) SetOrigin(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) SetOriginWithCredentials(repo scm.Repo) error {
- return nil
+func (g MockGitClient) SetOriginWithCredentials(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) Checkout(repo scm.Repo) error {
- return nil
+func (g MockGitClient) Checkout(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) Clean(repo scm.Repo) error {
- return nil
+func (g MockGitClient) Clean(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) UpdateRemote(repo scm.Repo) error {
- return nil
+func (g MockGitClient) UpdateRemote(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) Pull(repo scm.Repo) error {
- return nil
+func (g MockGitClient) Pull(repo scm.Repo) (string, error) {
+ if repo.Name == "cannotPull" {
+ return "fatal: not a git repository (or any of the parent directories): .git", errors.New("exit status 128")
+ }
+ return "", nil
}
-func (g MockGitClient) Reset(repo scm.Repo) error {
- return nil
+func (g MockGitClient) Reset(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) FetchAll(repo scm.Repo) error {
- return nil
+func (g MockGitClient) FetchAll(repo scm.Repo) (string, error) {
+ return "", nil
}
-func (g MockGitClient) FetchCloneBranch(repo scm.Repo) error {
- return nil
+func (g MockGitClient) FetchCloneBranch(repo scm.Repo) (string, error) {
+ return "", nil
}
func (g MockGitClient) RepoCommitCount(repo scm.Repo) (int, error) {
@@ -459,3 +472,50 @@
})
}
}
+
+func TestErrorMessageNotSwallowed(t *testing.T) {
+ defer UnsetEnv("GHORG_")()
+ dir, err := os.MkdirTemp("", "ghorg_test_one")
+ t.Logf("git dir: %v", dir)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer os.RemoveAll(dir)
+
+ setOutputDirName([]string{""})
+
+ os.Setenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO", dir)
+ setOuputDirAbsolutePath()
+
+ os.Setenv("GHORG_CONCURRENCY", "1")
+
+ os.Setenv("GHORG_DONT_EXIT_UNDER_TEST", "true")
+ os.Setenv("GHORG_EXIT_CODE_ON_CLONE_INFOS", "0")
+
+ var testRepos = []scm.Repo{
+ {
+ Name: "cannotPull",
+ URL: "[email protected]:org/cannotPull.git",
+ },
+ }
+
+ mockGit := NewMockGit()
+ CloneAllRepos(mockGit, testRepos)
+
+ rescueStdout := os.Stdout
+ r, w, _ := os.Pipe()
+ os.Stdout = w
+
+ //realGit := git.NewGit()
+ //CloneAllRepos(realGit, testRepos)
+ CloneAllRepos(mockGit, testRepos)
+
+ w.Close()
+ out, _ := io.ReadAll(r)
+ os.Stdout = rescueStdout
+ if !strings.Contains(string(out), "fatal: not a git repository") {
+ t.Errorf("Didn’t capture the git error message, expected a string containing: %s, got: %s", "fatal:not a git repository", string(out))
+ } else {
+ fmt.Printf("We got the output in the error message\n\n%v", string(out))
+ }
+}
Thank you for raising this! I will have to take a look at this, feel free to raise a pull request with your changes. It would help me test your changes out.
Also ghorg uses the git you have installed locally, it generally has bad error output. When things go wrong with git specifically its generally easiest to just run the same git clone command locally that ghorg is trying to run.
When things go wrong with git specifically its generally easiest to just run the same git clone command locally that ghorg is trying to run.
Unless things work locally just fine. ðŸ˜
When things go wrong with git specifically its generally easiest to just run the same git clone command locally that ghorg is trying to run.
Unless things work locally just fine. ðŸ˜
When you use GHORG_DEBUG=true it should show you all the git commands its running, you just need to copy one and run the command. It shouldn't behave differently because its running the same command.
Just FTR: via GHORG_DEBUG=true and GHORG_MATCH_PREFIX={name of repo} I could test it on one failing repo. The issue was due to broken submodules. ðŸ˜
But good that we found this.
Still, the git error message being part of the logs would be helpful (and avoid such manual debugging).
I’ll see if I find the time to create a proper pull request.