Skip to content

Commit 7bfb83e

Browse files
zeripathlunny
authored and
lafriks
committed
Batch hook pre- and post-receive calls (#8602)
* make notifyWatchers work on multiple actions * more efficient multiple notifyWatchers * Make CommitRepoAction take advantage of multiple actions * Batch post and pre-receive results * Set batch to 30 * Auto adjust timeout & add logging * adjust processing message * Add some messages to pre-receive * Make any non-200 status code from pre-receive an error * Add missing hookPrintResults * Remove shortcut for single action * mistaken merge fix * oops * Move master branch to the front * If repo was empty and the master branch is pushed ensure that that is set as the default branch * fixup * fixup * Missed HookOptions in setdefaultbranch * Batch PushUpdateAddTag and PushUpdateDelTag Co-authored-by: Lunny Xiao <[email protected]>
1 parent 114d474 commit 7bfb83e

File tree

10 files changed

+1023
-400
lines changed

10 files changed

+1023
-400
lines changed

‎cmd/hook.go‎

Lines changed: 174 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ import (
2121
"github.com/urfave/cli"
2222
)
2323

24+
const (
25+
hookBatchSize = 30
26+
)
27+
2428
var (
2529
// CmdHook represents the available hooks sub-command.
2630
CmdHook = cli.Command{
@@ -75,12 +79,25 @@ Gitea or set your environment appropriately.`, "")
7579
prID, _ := strconv.ParseInt(os.Getenv(models.ProtectedBranchPRID), 10, 64)
7680
isDeployKey, _ := strconv.ParseBool(os.Getenv(models.EnvIsDeployKey))
7781

78-
buf := bytes.NewBuffer(nil)
82+
hookOptions := private.HookOptions{
83+
UserID: userID,
84+
GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories),
85+
GitObjectDirectory: os.Getenv(private.GitObjectDirectory),
86+
GitQuarantinePath: os.Getenv(private.GitQuarantinePath),
87+
ProtectedBranchID: prID,
88+
IsDeployKey: isDeployKey,
89+
}
90+
7991
scanner := bufio.NewScanner(os.Stdin)
80-
for scanner.Scan() {
81-
buf.Write(scanner.Bytes())
82-
buf.WriteByte('\n')
8392

93+
oldCommitIDs := make([]string, hookBatchSize)
94+
newCommitIDs := make([]string, hookBatchSize)
95+
refFullNames := make([]string, hookBatchSize)
96+
count := 0
97+
total := 0
98+
lastline := 0
99+
100+
for scanner.Scan() {
84101
// TODO: support news feeds for wiki
85102
if isWiki {
86103
continue
@@ -94,29 +111,72 @@ Gitea or set your environment appropriately.`, "")
94111
oldCommitID := string(fields[0])
95112
newCommitID := string(fields[1])
96113
refFullName := string(fields[2])
114+
total++
115+
lastline++
97116

98117
// If the ref is a branch, check if it's protected
99118
if strings.HasPrefix(refFullName, git.BranchPrefix) {
100-
statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{
101-
OldCommitID: oldCommitID,
102-
NewCommitID: newCommitID,
103-
RefFullName: refFullName,
104-
UserID: userID,
105-
GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories),
106-
GitObjectDirectory: os.Getenv(private.GitObjectDirectory),
107-
GitQuarantinePath: os.Getenv(private.GitQuarantinePath),
108-
ProtectedBranchID: prID,
109-
IsDeployKey: isDeployKey,
110-
})
111-
switch statusCode {
112-
case http.StatusInternalServerError:
113-
fail("Internal Server Error", msg)
114-
case http.StatusForbidden:
115-
fail(msg, "")
119+
oldCommitIDs[count] = oldCommitID
120+
newCommitIDs[count] = newCommitID
121+
refFullNames[count] = refFullName
122+
count++
123+
fmt.Fprintf(os.Stdout, "*")
124+
os.Stdout.Sync()
125+
126+
if count >= hookBatchSize {
127+
fmt.Fprintf(os.Stdout, " Checking %d branches\n", count)
128+
os.Stdout.Sync()
129+
130+
hookOptions.OldCommitIDs = oldCommitIDs
131+
hookOptions.NewCommitIDs = newCommitIDs
132+
hookOptions.RefFullNames = refFullNames
133+
statusCode, msg := private.HookPreReceive(username, reponame, hookOptions)
134+
switch statusCode {
135+
case http.StatusOK:
136+
// no-op
137+
case http.StatusInternalServerError:
138+
fail("Internal Server Error", msg)
139+
default:
140+
fail(msg, "")
141+
}
142+
count = 0
143+
lastline = 0
116144
}
145+
} else {
146+
fmt.Fprintf(os.Stdout, ".")
147+
os.Stdout.Sync()
148+
}
149+
if lastline >= hookBatchSize {
150+
fmt.Fprintf(os.Stdout, "\n")
151+
os.Stdout.Sync()
152+
lastline = 0
117153
}
118154
}
119155

156+
if count > 0 {
157+
hookOptions.OldCommitIDs = oldCommitIDs[:count]
158+
hookOptions.NewCommitIDs = newCommitIDs[:count]
159+
hookOptions.RefFullNames = refFullNames[:count]
160+
161+
fmt.Fprintf(os.Stdout, " Checking %d branches\n", count)
162+
os.Stdout.Sync()
163+
164+
statusCode, msg := private.HookPreReceive(username, reponame, hookOptions)
165+
switch statusCode {
166+
case http.StatusInternalServerError:
167+
fail("Internal Server Error", msg)
168+
case http.StatusForbidden:
169+
fail(msg, "")
170+
}
171+
} else if lastline > 0 {
172+
fmt.Fprintf(os.Stdout, "\n")
173+
os.Stdout.Sync()
174+
lastline = 0
175+
}
176+
177+
fmt.Fprintf(os.Stdout, "Checked %d references in total\n", total)
178+
os.Stdout.Sync()
179+
120180
return nil
121181
}
122182

@@ -156,12 +216,24 @@ Gitea or set your environment appropriately.`, "")
156216
pusherID, _ := strconv.ParseInt(os.Getenv(models.EnvPusherID), 10, 64)
157217
pusherName := os.Getenv(models.EnvPusherName)
158218

159-
buf := bytes.NewBuffer(nil)
219+
hookOptions := private.HookOptions{
220+
UserName: pusherName,
221+
UserID: pusherID,
222+
GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories),
223+
GitObjectDirectory: os.Getenv(private.GitObjectDirectory),
224+
GitQuarantinePath: os.Getenv(private.GitQuarantinePath),
225+
}
226+
oldCommitIDs := make([]string, hookBatchSize)
227+
newCommitIDs := make([]string, hookBatchSize)
228+
refFullNames := make([]string, hookBatchSize)
229+
count := 0
230+
total := 0
231+
wasEmpty := false
232+
masterPushed := false
233+
results := make([]private.HookPostReceiveBranchResult, 0)
234+
160235
scanner := bufio.NewScanner(os.Stdin)
161236
for scanner.Scan() {
162-
buf.Write(scanner.Bytes())
163-
buf.WriteByte('\n')
164-
165237
// TODO: support news feeds for wiki
166238
if isWiki {
167239
continue
@@ -172,36 +244,95 @@ Gitea or set your environment appropriately.`, "")
172244
continue
173245
}
174246

175-
oldCommitID := string(fields[0])
176-
newCommitID := string(fields[1])
177-
refFullName := string(fields[2])
247+
fmt.Fprintf(os.Stdout, ".")
248+
oldCommitIDs[count] = string(fields[0])
249+
newCommitIDs[count] = string(fields[1])
250+
refFullNames[count] = string(fields[2])
251+
if refFullNames[count] == git.BranchPrefix+"master" && newCommitIDs[count] != git.EmptySHA && count == total {
252+
masterPushed = true
253+
}
254+
count++
255+
total++
256+
os.Stdout.Sync()
257+
258+
if count >= hookBatchSize {
259+
fmt.Fprintf(os.Stdout, " Processing %d references\n", count)
260+
os.Stdout.Sync()
261+
hookOptions.OldCommitIDs = oldCommitIDs
262+
hookOptions.NewCommitIDs = newCommitIDs
263+
hookOptions.RefFullNames = refFullNames
264+
resp, err := private.HookPostReceive(repoUser, repoName, hookOptions)
265+
if resp == nil {
266+
hookPrintResults(results)
267+
fail("Internal Server Error", err)
268+
}
269+
wasEmpty = wasEmpty || resp.RepoWasEmpty
270+
results = append(results, resp.Results...)
271+
count = 0
272+
}
273+
}
274+
275+
if count == 0 {
276+
if wasEmpty && masterPushed {
277+
// We need to tell the repo to reset the default branch to master
278+
err := private.SetDefaultBranch(repoUser, repoName, "master")
279+
if err != nil {
280+
fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err)
281+
}
282+
}
283+
fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total)
284+
os.Stdout.Sync()
285+
286+
hookPrintResults(results)
287+
return nil
288+
}
178289

179-
res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{
180-
OldCommitID: oldCommitID,
181-
NewCommitID: newCommitID,
182-
RefFullName: refFullName,
183-
UserID: pusherID,
184-
UserName: pusherName,
185-
})
290+
hookOptions.OldCommitIDs = oldCommitIDs[:count]
291+
hookOptions.NewCommitIDs = newCommitIDs[:count]
292+
hookOptions.RefFullNames = refFullNames[:count]
186293

187-
if res == nil {
188-
fail("Internal Server Error", err)
294+
fmt.Fprintf(os.Stdout, " Processing %d references\n", count)
295+
os.Stdout.Sync()
296+
297+
resp, err := private.HookPostReceive(repoUser, repoName, hookOptions)
298+
if resp == nil {
299+
hookPrintResults(results)
300+
fail("Internal Server Error", err)
301+
}
302+
wasEmpty = wasEmpty || resp.RepoWasEmpty
303+
results = append(results, resp.Results...)
304+
305+
fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total)
306+
os.Stdout.Sync()
307+
308+
if wasEmpty && masterPushed {
309+
// We need to tell the repo to reset the default branch to master
310+
err := private.SetDefaultBranch(repoUser, repoName, "master")
311+
if err != nil {
312+
fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err)
189313
}
314+
}
315+
316+
hookPrintResults(results)
190317

191-
if res["message"] == false {
318+
return nil
319+
}
320+
321+
func hookPrintResults(results []private.HookPostReceiveBranchResult) {
322+
for _, res := range results {
323+
if !res.Message {
192324
continue
193325
}
194326

195327
fmt.Fprintln(os.Stderr, "")
196-
if res["create"] == true {
197-
fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res["branch"])
198-
fmt.Fprintf(os.Stderr, " %s\n", res["url"])
328+
if res.Create {
329+
fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res.Branch)
330+
fmt.Fprintf(os.Stderr, " %s\n", res.URL)
199331
} else {
200332
fmt.Fprint(os.Stderr, "Visit the existing pull request:\n")
201-
fmt.Fprintf(os.Stderr, " %s\n", res["url"])
333+
fmt.Fprintf(os.Stderr, " %s\n", res.URL)
202334
}
203335
fmt.Fprintln(os.Stderr, "")
336+
os.Stderr.Sync()
204337
}
205-
206-
return nil
207338
}

0 commit comments

Comments
 (0)