Make mirror feature more configurable (go-gitea#16957)

6543 · 6543 · commit d90b80388ce0 · 2021-09-05T04:27:06.000+02:00
diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini
@@ -793,9 +793,6 @@ PATH =
 ;; Prefix archive files by placing them in a directory named after the repository
 ;PREFIX_ARCHIVE_FILES = true
 ;;
-;; Disable the creation of new mirrors. Pre-existing mirrors remain valid.
-;DISABLE_MIRRORS = false
-;;
 ;; Disable migrating feature.
 ;DISABLE_MIGRATIONS = false
 ;;
@@ -1934,6 +1931,12 @@ PATH =
 ;[mirror]
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Whether gitea's mirror function can be used or not.
+;ENABLED = true
+;; Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
+;DISABLE_NEW_PULL = false
+;; Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid.
+;DISABLE_NEW_PUSH = false
 ;; Default interval as a duration between each check
 ;DEFAULT_INTERVAL = 8h
 ;; Min interval as a duration must be > 1m
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -73,7 +73,6 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `DISABLED_REPO_UNITS`: **_empty_**: Comma separated list of globally disabled repo units. Allowed values: \[repo.issues, repo.ext_issues, repo.pulls, repo.wiki, repo.ext_wiki, repo.projects\]
 - `DEFAULT_REPO_UNITS`: **repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects**: Comma separated list of default repo units. Allowed values: \[repo.code, repo.releases, repo.issues, repo.pulls, repo.wiki, repo.projects\]. Note: Code and Releases can currently not be deactivated. If you specify default repo units you should still list them for future compatibility. External wiki and issue tracker can't be enabled by default as it requires additional settings. Disabled repo units will not be added to new repositories regardless if it is in the default list.
 - `PREFIX_ARCHIVE_FILES`: **true**: Prefix archive files by placing them in a directory named after the repository.
-- `DISABLE_MIRRORS`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
 - `DISABLE_MIGRATIONS`: **false**: Disable migrating feature.
 - `DISABLE_STARS`: **false**: Disable stars feature.
 - `DEFAULT_BRANCH`: **master**: Default branch name of all repositories.
@@ -949,6 +948,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 
 ## Mirror (`mirror`)
 
+- `ENABLED`: **true**: Whether gitea's mirror function can be used or not.
+- `DISABLE_NEW_PULL`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
+- `DISABLE_NEW_PUSH`: **false**: Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid.
 - `DEFAULT_INTERVAL`: **8h**: Default interval between each check
 - `MIN_INTERVAL`: **10m**: Minimum interval for checking. (Must be >1m).
 
diff --git a/integrations/api_settings_test.go b/integrations/api_settings_test.go
@@ -43,7 +43,7 @@ func TestAPIExposedSettings(t *testing.T) {
 
 	DecodeJSON(t, resp, &repo)
 	assert.EqualValues(t, &api.GeneralRepoSettings{
-		MirrorsDisabled:      setting.Repository.DisableMirrors,
+		MirrorsDisabled:      !setting.Mirror.Enabled,
 		HTTPGitDisabled:      setting.Repository.DisableHTTPGit,
 		MigrationsDisabled:   setting.Repository.DisableMigrations,
 		TimeTrackingDisabled: false,
diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go
@@ -6,9 +6,11 @@ package models
 
 import (
 	"errors"
+	"fmt"
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 
 	"xorm.io/xorm"
@@ -57,6 +59,9 @@ func (m *PushMirror) GetRemoteName() string {
 
 // InsertPushMirror inserts a push-mirror to database
 func InsertPushMirror(m *PushMirror) error {
+	if setting.Mirror.DisableNewPush {
+		return fmt.Errorf("creation of new push mirror's are disabled")
+	}
 	_, err := x.Insert(m)
 	return err
 }
diff --git a/modules/setting/mirror.go b/modules/setting/mirror.go
@@ -0,0 +1,53 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+var (
+	// Mirror settings
+	Mirror = struct {
+		Enabled         bool
+		DisableNewPull  bool
+		DisableNewPush  bool
+		DefaultInterval time.Duration
+		MinInterval     time.Duration
+	}{
+		Enabled:         true,
+		DisableNewPull:  false,
+		DisableNewPush:  false,
+		MinInterval:     10 * time.Minute,
+		DefaultInterval: 8 * time.Hour,
+	}
+)
+
+func newMirror() {
+	if err := Cfg.Section("mirror").MapTo(&Mirror); err != nil {
+		log.Fatal("Failed to map Mirror settings: %v", err)
+	}
+
+	// fallback to old config repository.DISABLE_MIRRORS
+	if Cfg.Section("repository").Key("DISABLE_MIRRORS").MustBool(false) {
+		Mirror.DisableNewPull = true
+	}
+
+	if !Mirror.Enabled {
+		Mirror.DisableNewPull = true
+		Mirror.DisableNewPush = true
+	}
+
+	if Mirror.MinInterval.Minutes() < 1 {
+		log.Warn("Mirror.MinInterval is too low, set to 1 minute")
+		Mirror.MinInterval = 1 * time.Minute
+	}
+	if Mirror.DefaultInterval < Mirror.MinInterval {
+		log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to 8 hours")
+		Mirror.DefaultInterval = time.Hour * 8
+	}
+}
diff --git a/modules/setting/repository.go b/modules/setting/repository.go
@@ -41,7 +41,6 @@ var (
 		DisabledRepoUnits                       []string
 		DefaultRepoUnits                        []string
 		PrefixArchiveFiles                      bool
-		DisableMirrors                          bool
 		DisableMigrations                       bool
 		DisableStars                            bool `ini:"DISABLE_STARS"`
 		DefaultBranch                           string
@@ -154,7 +153,6 @@ var (
 		DisabledRepoUnits:                       []string{},
 		DefaultRepoUnits:                        []string{},
 		PrefixArchiveFiles:                      true,
-		DisableMirrors:                          false,
 		DisableMigrations:                       false,
 		DisableStars:                            false,
 		DefaultBranch:                           "master",
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
@@ -347,12 +347,6 @@ var (
 
 	ManifestData string
 
-	// Mirror settings
-	Mirror struct {
-		DefaultInterval time.Duration
-		MinInterval     time.Duration
-	}
-
 	// API settings
 	API = struct {
 		EnableSwagger          bool
@@ -936,17 +930,7 @@ func NewContext() {
 
 	newGit()
 
-	sec = Cfg.Section("mirror")
-	Mirror.MinInterval = sec.Key("MIN_INTERVAL").MustDuration(10 * time.Minute)
-	Mirror.DefaultInterval = sec.Key("DEFAULT_INTERVAL").MustDuration(8 * time.Hour)
-	if Mirror.MinInterval.Minutes() < 1 {
-		log.Warn("Mirror.MinInterval is too low")
-		Mirror.MinInterval = 1 * time.Minute
-	}
-	if Mirror.DefaultInterval < Mirror.MinInterval {
-		log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval")
-		Mirror.DefaultInterval = time.Hour * 8
-	}
+	newMirror()
 
 	Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
 	if len(Langs) == 0 {
diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go
@@ -108,8 +108,8 @@ func Migrate(ctx *context.APIContext) {
 
 	gitServiceType := convert.ToGitServiceType(form.Service)
 
-	if form.Mirror && setting.Repository.DisableMirrors {
-		ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled mirrors"))
+	if form.Mirror && setting.Mirror.DisableNewPull {
+		ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled new mirror creations"))
 		return
 	}
 
diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go
@@ -9,6 +9,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/setting"
 	mirror_service "code.gitea.io/gitea/services/mirror"
 )
 
@@ -42,6 +43,11 @@ func MirrorSync(ctx *context.APIContext) {
 		ctx.Error(http.StatusForbidden, "MirrorSync", "Must have write access")
 	}
 
+	if !setting.Mirror.Enabled {
+		ctx.Status(http.StatusBadRequest)
+		return
+	}
+
 	mirror_service.StartToMirror(repo.ID)
 
 	ctx.Status(http.StatusOK)
diff --git a/routers/api/v1/settings/settings.go b/routers/api/v1/settings/settings.go
@@ -58,7 +58,7 @@ func GetGeneralRepoSettings(ctx *context.APIContext) {
 	//   "200":
 	//     "$ref": "#/responses/GeneralRepoSettings"
 	ctx.JSON(http.StatusOK, api.GeneralRepoSettings{
-		MirrorsDisabled:      setting.Repository.DisableMirrors,
+		MirrorsDisabled:      !setting.Mirror.Enabled,
 		HTTPGitDisabled:      setting.Repository.DisableHTTPGit,
 		MigrationsDisabled:   setting.Repository.DisableMigrations,
 		StarsDisabled:        setting.Repository.DisableStars,
diff --git a/routers/web/org/home.go b/routers/web/org/home.go
@@ -142,7 +142,7 @@ func Home(ctx *context.Context) {
 	ctx.Data["Members"] = members
 	ctx.Data["Teams"] = org.Teams
 
-	ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
+	ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull
 
 	pager := context.NewPagination(int(count), setting.UI.User.RepoPagingNum, page, 5)
 	pager.SetDefaultParams(ctx)
diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go
@@ -152,17 +152,20 @@ func MigratePost(ctx *context.Context) {
 		return
 	}
 
-	serviceType := structs.GitServiceType(form.Service)
+	if form.Mirror && setting.Mirror.DisableNewPull {
+		ctx.Error(http.StatusBadRequest, "MigratePost: the site administrator has disabled creation of new mirrors")
+		return
+	}
 
-	setMigrationContextData(ctx, serviceType)
+	setMigrationContextData(ctx, form.Service)
 
 	ctxUser := checkContextUser(ctx, form.UID)
 	if ctx.Written() {
 		return
 	}
 	ctx.Data["ContextUser"] = ctxUser
 
-	tpl := base.TplName("repo/migrate/" + serviceType.Name())
+	tpl := base.TplName("repo/migrate/" + form.Service.Name())
 
 	if ctx.HasError() {
 		ctx.HTML(http.StatusOK, tpl)
@@ -198,12 +201,12 @@ func MigratePost(ctx *context.Context) {
 
 	var opts = migrations.MigrateOptions{
 		OriginalURL:    form.CloneAddr,
-		GitServiceType: serviceType,
+		GitServiceType: form.Service,
 		CloneAddr:      remoteAddr,
 		RepoName:       form.RepoName,
 		Description:    form.Description,
 		Private:        form.Private || setting.Repository.ForcePrivate,
-		Mirror:         form.Mirror && !setting.Repository.DisableMirrors,
+		Mirror:         form.Mirror,
 		LFS:            form.LFS,
 		LFSEndpoint:    form.LFSEndpoint,
 		AuthUsername:   form.AuthUsername,
@@ -246,7 +249,7 @@ func setMigrationContextData(ctx *context.Context, serviceType structs.GitServic
 
 	ctx.Data["LFSActive"] = setting.LFS.StartServer
 	ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
-	ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
+	ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull
 
 	// Plain git should be first
 	ctx.Data["Services"] = append([]structs.GitServiceType{structs.PlainGitService}, structs.SupportedFullGitService...)
diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go
@@ -52,7 +52,8 @@ func Settings(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("repo.settings")
 	ctx.Data["PageIsSettingsOptions"] = true
 	ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate
-	ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
+	ctx.Data["MirrorsEnabled"] = setting.Mirror.Enabled
+	ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush
 	ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval
 
 	signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath())
@@ -144,7 +145,7 @@ func SettingsPost(ctx *context.Context) {
 		ctx.Redirect(repo.Link() + "/settings")
 
 	case "mirror":
-		if !repo.IsMirror {
+		if !setting.Mirror.Enabled || !repo.IsMirror {
 			ctx.NotFound("", nil)
 			return
 		}
@@ -220,7 +221,7 @@ func SettingsPost(ctx *context.Context) {
 		ctx.Redirect(repo.Link() + "/settings")
 
 	case "mirror-sync":
-		if !repo.IsMirror {
+		if !setting.Mirror.Enabled || !repo.IsMirror {
 			ctx.NotFound("", nil)
 			return
 		}
@@ -231,6 +232,11 @@ func SettingsPost(ctx *context.Context) {
 		ctx.Redirect(repo.Link() + "/settings")
 
 	case "push-mirror-sync":
+		if !setting.Mirror.Enabled {
+			ctx.NotFound("", nil)
+			return
+		}
+
 		m, err := selectPushMirrorByForm(form, repo)
 		if err != nil {
 			ctx.NotFound("", nil)
@@ -243,6 +249,11 @@ func SettingsPost(ctx *context.Context) {
 		ctx.Redirect(repo.Link() + "/settings")
 
 	case "push-mirror-remove":
+		if !setting.Mirror.Enabled {
+			ctx.NotFound("", nil)
+			return
+		}
+
 		// This section doesn't require repo_name/RepoName to be set in the form, don't show it
 		// as an error on the UI for this action
 		ctx.Data["Err_RepoName"] = nil
@@ -267,6 +278,11 @@ func SettingsPost(ctx *context.Context) {
 		ctx.Redirect(repo.Link() + "/settings")
 
 	case "push-mirror-add":
+		if setting.Mirror.DisableNewPush {
+			ctx.NotFound("", nil)
+			return
+		}
+
 		// This section doesn't require repo_name/RepoName to be set in the form, don't show it
 		// as an error on the UI for this action
 		ctx.Data["Err_RepoName"] = nil
@@ -698,6 +714,7 @@ func SettingsPost(ctx *context.Context) {
 
 		log.Trace("Repository was archived: %s/%s", ctx.Repo.Owner.Name, repo.Name)
 		ctx.Redirect(ctx.Repo.RepoLink + "/settings")
+
 	case "unarchive":
 		if !ctx.Repo.IsOwner() {
 			ctx.Error(http.StatusForbidden)
diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go
@@ -22,6 +22,10 @@ var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
 
 // Update checks and updates mirror repositories.
 func Update(ctx context.Context) error {
+	if !setting.Mirror.Enabled {
+		log.Warn("Mirror feature disabled, but cron job enabled: skip update")
+		return nil
+	}
 	log.Trace("Doing: Update")
 
 	handler := func(idx int, bean interface{}) error {
@@ -89,15 +93,24 @@ func syncMirrors(ctx context.Context) {
 
 // InitSyncMirrors initializes a go routine to sync the mirrors
 func InitSyncMirrors() {
+	if !setting.Mirror.Enabled {
+		return
+	}
 	go graceful.GetManager().RunWithShutdownContext(syncMirrors)
 }
 
 // StartToMirror adds repoID to mirror queue
 func StartToMirror(repoID int64) {
+	if !setting.Mirror.Enabled {
+		return
+	}
 	go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID))
 }
 
 // AddPushMirrorToQueue adds the push mirror to the queue
 func AddPushMirrorToQueue(mirrorID int64) {
+	if !setting.Mirror.Enabled {
+		return
+	}
 	go mirrorQueue.Add(fmt.Sprintf("push %d", mirrorID))
 }
diff --git a/templates/org/home.tmpl b/templates/org/home.tmpl
@@ -26,7 +26,7 @@
 			<div class="ui eleven wide column">
 				{{if .CanCreateOrgRepo}}
 					<div class="text right">
-						{{if not .DisabledMirrors}}
+						{{if not .DisableNewPullMirrors}}
 							<a class="ui green button" href="{{AppSubUrl}}/repo/migrate?org={{.Org.ID}}&mirror=1">{{.i18n.Tr "new_migrate"}}</a>
 						{{end}}
 						<a class="ui green button" href="{{AppSubUrl}}/repo/create?org={{.Org.ID}}">{{.i18n.Tr "new_repo"}}</a>
diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl
@@ -1,7 +1,7 @@
 <div class="inline field">
 	<label>{{.i18n.Tr "repo.migrate_options"}}</label>
 	<div class="ui checkbox">
-		{{if .DisableMirrors}}
+		{{if .DisableNewPullMirrors}}
 			<input id="mirror" name="mirror" type="checkbox" readonly>
 			<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
 			<a target=”_blank” href="https://blog.codeberg.org/mirror-repos-easily-created-consuming-resources-forever.html">{{svg "octicon-question"}}</a>
diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl

Original file line number	Diff line number	Diff line change
`@@ -108,8 +108,8 @@ func Migrate(ctx *context.APIContext) {`
`108`	`108`
`109`	`109`	`gitServiceType := convert.ToGitServiceType(form.Service)`
`110`	`110`
`111`		`- if form.Mirror && setting.Repository.DisableMirrors {`
`112`		`- ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled mirrors"))`
	`111`	`+ if form.Mirror && setting.Mirror.DisableNewPull {`
	`112`	`+ ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled new mirror creations"))`
`113`	`113`	`return`
`114`	`114`	`}`
`115`	`115`