Skip to content

Commit b54c064

Browse files
KN4CK3Rlunny
KN4CK3R
and
lunny
authored
Workaround for container registry push/pull errors (#21862) (#22068)
Backport of #21862 Co-authored-by: Lunny Xiao <[email protected]>
1 parent c0ca9c6 commit b54c064

File tree

5 files changed

+103
-4
lines changed

5 files changed

+103
-4
lines changed

‎modules/packages/content_store.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
3030
return s.store.Open(KeyToRelativePath(key))
3131
}
3232

33+
// FIXME: Workaround to be removed in v1.20
34+
// https://github.com/go-gitea/gitea/issues/19586
35+
func (s *ContentStore) Has(key BlobHash256Key) error {
36+
_, err := s.store.Stat(KeyToRelativePath(key))
37+
return err
38+
}
39+
3340
// Save stores a package blob
3441
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
3542
_, err := s.store.Save(KeyToRelativePath(key), r, size)

‎routers/api/packages/container/blob.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,24 @@ package container
77
import (
88
"context"
99
"encoding/hex"
10+
"errors"
1011
"fmt"
12+
"os"
1113
"strings"
14+
"sync"
1215

1316
"code.gitea.io/gitea/models/db"
1417
packages_model "code.gitea.io/gitea/models/packages"
1518
container_model "code.gitea.io/gitea/models/packages/container"
1619
"code.gitea.io/gitea/modules/log"
1720
packages_module "code.gitea.io/gitea/modules/packages"
1821
container_module "code.gitea.io/gitea/modules/packages/container"
22+
"code.gitea.io/gitea/modules/util"
1923
packages_service "code.gitea.io/gitea/services/packages"
2024
)
2125

26+
var uploadVersionMutex sync.Mutex
27+
2228
// saveAsPackageBlob creates a package blob from an upload
2329
// The uploaded blob gets stored in a special upload version to link them to the package/image
2430
func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) {
@@ -28,6 +34,11 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
2834

2935
contentStore := packages_module.NewContentStore()
3036

37+
var uploadVersion *packages_model.PackageVersion
38+
39+
// FIXME: Replace usage of mutex with database transaction
40+
// https://github.com/go-gitea/gitea/pull/21862
41+
uploadVersionMutex.Lock()
3142
err := db.WithTx(func(ctx context.Context) error {
3243
created := true
3344
p := &packages_model.Package{
@@ -68,11 +79,30 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
6879
}
6980
}
7081

82+
uploadVersion = pv
83+
84+
return nil
85+
})
86+
uploadVersionMutex.Unlock()
87+
if err != nil {
88+
return nil, err
89+
}
90+
91+
err = db.WithTx(func(ctx context.Context) error {
7192
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
7293
if err != nil {
7394
log.Error("Error inserting package blob: %v", err)
7495
return err
7596
}
97+
// FIXME: Workaround to be removed in v1.20
98+
// https://github.com/go-gitea/gitea/issues/19586
99+
if exists {
100+
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
101+
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
102+
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
103+
exists = false
104+
}
105+
}
76106
if !exists {
77107
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
78108
log.Error("Error saving package blob in content store: %v", err)
@@ -83,7 +113,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
83113
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
84114

85115
pf := &packages_model.PackageFile{
86-
VersionID: pv.ID,
116+
VersionID: uploadVersion.ID,
87117
BlobID: pb.ID,
88118
Name: filename,
89119
LowerName: filename,

‎routers/api/packages/container/container.go‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"io"
1111
"net/http"
1212
"net/url"
13+
"os"
1314
"regexp"
1415
"strconv"
1516
"strings"
@@ -24,6 +25,7 @@ import (
2425
container_module "code.gitea.io/gitea/modules/packages/container"
2526
"code.gitea.io/gitea/modules/packages/container/oci"
2627
"code.gitea.io/gitea/modules/setting"
28+
"code.gitea.io/gitea/modules/util"
2729
"code.gitea.io/gitea/routers/api/packages/helper"
2830
packages_service "code.gitea.io/gitea/services/packages"
2931
container_service "code.gitea.io/gitea/services/packages/container"
@@ -193,7 +195,7 @@ func InitiateUploadBlob(ctx *context.Context) {
193195
mount := ctx.FormTrim("mount")
194196
from := ctx.FormTrim("from")
195197
if mount != "" {
196-
blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
198+
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
197199
Image: from,
198200
Digest: mount,
199201
})
@@ -406,7 +408,7 @@ func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescri
406408
return nil, container_model.ErrContainerBlobNotExist
407409
}
408410

409-
return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
411+
return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
410412
OwnerID: ctx.Package.Owner.ID,
411413
Image: ctx.Params("image"),
412414
Digest: digest,
@@ -548,7 +550,7 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe
548550
return nil, container_model.ErrContainerBlobNotExist
549551
}
550552

551-
return container_model.GetContainerBlob(ctx, opts)
553+
return workaroundGetContainerBlob(ctx, opts)
552554
}
553555

554556
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry
@@ -688,3 +690,23 @@ func GetTagList(ctx *context.Context) {
688690
Tags: tags,
689691
})
690692
}
693+
694+
// FIXME: Workaround to be removed in v1.20
695+
// https://github.com/go-gitea/gitea/issues/19586
696+
func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) {
697+
blob, err := container_model.GetContainerBlob(ctx, opts)
698+
if err != nil {
699+
return nil, err
700+
}
701+
702+
err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(blob.Blob.HashSHA256))
703+
if err != nil {
704+
if errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist) {
705+
log.Debug("Package registry inconsistent: blob %s does not exist on file system", blob.Blob.HashSHA256)
706+
return nil, container_model.ErrContainerBlobNotExist
707+
}
708+
return nil, err
709+
}
710+
711+
return blob, nil
712+
}

‎routers/api/packages/container/manifest.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ package container
66

77
import (
88
"context"
9+
"errors"
910
"fmt"
1011
"io"
12+
"os"
1113
"strings"
1214

1315
"code.gitea.io/gitea/models/db"
@@ -19,6 +21,7 @@ import (
1921
packages_module "code.gitea.io/gitea/modules/packages"
2022
container_module "code.gitea.io/gitea/modules/packages/container"
2123
"code.gitea.io/gitea/modules/packages/container/oci"
24+
"code.gitea.io/gitea/modules/util"
2225
packages_service "code.gitea.io/gitea/services/packages"
2326
)
2427

@@ -403,6 +406,15 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
403406
log.Error("Error inserting package blob: %v", err)
404407
return nil, false, "", err
405408
}
409+
// FIXME: Workaround to be removed in v1.20
410+
// https://github.com/go-gitea/gitea/issues/19586
411+
if exists {
412+
err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256))
413+
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
414+
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
415+
exists = false
416+
}
417+
}
406418
if !exists {
407419
contentStore := packages_module.NewContentStore()
408420
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {

‎tests/integration/api_packages_container_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ package integration
66

77
import (
88
"bytes"
9+
"crypto/sha256"
910
"encoding/base64"
1011
"fmt"
1112
"net/http"
1213
"strings"
14+
"sync"
1315
"testing"
1416

1517
"code.gitea.io/gitea/models/db"
@@ -594,6 +596,32 @@ func TestPackageContainer(t *testing.T) {
594596
})
595597
}
596598

599+
// https://github.com/go-gitea/gitea/issues/19586
600+
t.Run("ParallelUpload", func(t *testing.T) {
601+
defer tests.PrintCurrentTest(t)()
602+
603+
url := fmt.Sprintf("%sv2/%s/parallel", setting.AppURL, user.Name)
604+
605+
var wg sync.WaitGroup
606+
for i := 0; i < 10; i++ {
607+
wg.Add(1)
608+
609+
content := []byte{byte(i)}
610+
digest := fmt.Sprintf("sha256:%x", sha256.Sum256(content))
611+
612+
go func() {
613+
defer wg.Done()
614+
615+
req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, digest), bytes.NewReader(content))
616+
addTokenAuthHeader(req, userToken)
617+
resp := MakeRequest(t, req, http.StatusCreated)
618+
619+
assert.Equal(t, digest, resp.Header().Get("Docker-Content-Digest"))
620+
}()
621+
}
622+
wg.Wait()
623+
})
624+
597625
t.Run("OwnerNameChange", func(t *testing.T) {
598626
defer tests.PrintCurrentTest(t)()
599627

0 commit comments

Comments
 (0)