Merge branch 'main' into pacman-packages

.devcontainer/devcontainer.json

@@ -6,7 +6,9 @@
     "ghcr.io/devcontainers/features/node:1": {
       "version":"20"
     },
-    "ghcr.io/devcontainers/features/git-lfs:1.1.0": {}
+    "ghcr.io/devcontainers/features/git-lfs:1.1.0": {},
+    "ghcr.io/devcontainers-contrib/features/poetry:2": {},
+    "ghcr.io/devcontainers/features/python:1": {}
   },
   "customizations": {
     "vscode": {

.eslintrc.yaml

@@ -25,10 +25,11 @@ env:
   es2022: true
   node: true
 
-globals:
-  __webpack_public_path__: true
-
 overrides:
+  - files: ["web_src/**/*"]
+    globals:
+      __webpack_public_path__: true
+      process: false # https://github.com/webpack/webpack/issues/15833
   - files: ["web_src/**/*", "docs/**/*"]
     env:
       browser: true

.github/workflows/files-changed.yml

@@ -20,7 +20,6 @@ jobs:
   detect:
     runs-on: ubuntu-latest
     timeout-minutes: 3
-    # Map a step output to a job output
     outputs:
       backend: ${{ steps.changes.outputs.backend }}
       frontend: ${{ steps.changes.outputs.frontend }}
@@ -39,12 +38,14 @@ jobs:
               - "templates/**/*.tmpl"
               - "go.mod"
               - "go.sum"
+              - "Makefile"
 
             frontend:
               - "**/*.js"
               - "web_src/**"
               - "package.json"
               - "package-lock.json"
+              - "Makefile"
 
             docs:
               - "**/*.md"
@@ -56,7 +57,9 @@ jobs:
             templates:
               - "templates/**/*.tmpl"
               - "poetry.lock"
+
             docker:
               - "Dockerfile"
               - "Dockerfile.rootless"
               - "docker/**"
+              - "Makefile"

.github/workflows/pull-compliance.yml

@@ -96,6 +96,7 @@ jobs:
       - run: make deps-frontend
       - run: make lint-frontend
       - run: make checks-frontend
+      - run: make test-frontend
       - run: make frontend
 
   backend:
@@ -110,7 +111,7 @@ jobs:
           check-latest: true
       # no frontend build here as backend should be able to build
       # even without any frontend files
-      - run: make deps-backend deps-tools
+      - run: make deps-backend
       - run: go build -o gitea_no_gcc # test if build succeeds without the sqlite tag
       - name: build-backend-arm64
         run: make backend # test cross compile

.gitignore

@@ -53,8 +53,6 @@ cpu.out
 /bin
 /dist
 /custom/*
-!/custom/conf
-/custom/conf/*
 !/custom/conf/app.example.ini
 /data
 /indexers

CONTRIBUTING.md

@@ -557,7 +557,7 @@ be reviewed by two maintainers and must pass the automatic tests.
 - And then push the tag as `git push origin v$vmaj.$vmin.$`. Drone CI will automatically create a release and upload all the compiled binary. (But currently it doesn't add the release notes automatically. Maybe we should fix that.)
 - If needed send a frontport PR for the changelog to branch `main` and update the version in `docs/config.yaml` to refer to the new version.
 - Send PR to [blog repository](https://gitea.com/gitea/blog) announcing the release.
-- Verify all release assets were correctly published through CI on dl.gitea.io and GitHub releases. Once ACKed:
-  - bump the version of https://dl.gitea.io/gitea/version.json
+- Verify all release assets were correctly published through CI on dl.gitea.com and GitHub releases. Once ACKed:
+  - bump the version of https://dl.gitea.com/gitea/version.json
   - merge the blog post PR
   - announce the release in discord `#announcements`

Makefile

@@ -84,7 +84,7 @@ GITHUB_REF_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 ifneq ($(GITHUB_REF_TYPE),branch)
 	VERSION ?= $(subst v,,$(GITHUB_REF_NAME))
-	GITEA_VERSION ?= $(GITHUB_REF_NAME)
+	GITEA_VERSION ?= $(VERSION)
 else
 	ifneq ($(GITHUB_REF_NAME),)
 		VERSION ?= $(subst release/v,,$(GITHUB_REF_NAME))
@@ -926,11 +926,18 @@ node_modules: package-lock.json
 
 .PHONY: npm-update
 npm-update: node-check | node_modules
-	npx updates -cu
+	npx updates -u -f package.json
 	rm -rf node_modules package-lock.json
 	npm install --package-lock
 	@touch node_modules
 
+.PHONY: poetry-update
+poetry-update: node-check | node_modules
+	npx updates -u -f pyproject.toml
+	rm -rf .venv poetry.lock
+	poetry install
+	@touch .venv
+
 .PHONY: fomantic
 fomantic:
 	rm -rf $(FOMANTIC_WORK_DIR)/build

README.md

@@ -173,8 +173,8 @@ for the full license text.
 
 Looking for an overview of the interface? Check it out!
 
-|![Dashboard](https://dl.gitea.io/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.io/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.io/screenshots/global_issues.png)|
+|![Dashboard](https://dl.gitea.com/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.com/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.com/screenshots/global_issues.png)|
 |:---:|:---:|:---:|
-|![Branches](https://dl.gitea.io/screenshots/branches.png)|![Web Editor](https://dl.gitea.io/screenshots/web_editor.png)|![Activity](https://dl.gitea.io/screenshots/activity.png)|
-|![New Migration](https://dl.gitea.io/screenshots/migration.png)|![Migrating](https://dl.gitea.io/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
-![Pull Request Dark](https://dl.gitea.io/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.io/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.io/screenshots/diff_dark.png)|
+|![Branches](https://dl.gitea.com/screenshots/branches.png)|![Web Editor](https://dl.gitea.com/screenshots/web_editor.png)|![Activity](https://dl.gitea.com/screenshots/activity.png)|
+|![New Migration](https://dl.gitea.com/screenshots/migration.png)|![Migrating](https://dl.gitea.com/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
+![Pull Request Dark](https://dl.gitea.com/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.com/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.com/screenshots/diff_dark.png)|

README_ZH.md

@@ -91,8 +91,8 @@ Fork -> Patch -> Push -> Pull Request
 
 ## 截图
 
-|![Dashboard](https://dl.gitea.io/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.io/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.io/screenshots/global_issues.png)|
+|![Dashboard](https://dl.gitea.com/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.com/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.com/screenshots/global_issues.png)|
 |:---:|:---:|:---:|
-|![Branches](https://dl.gitea.io/screenshots/branches.png)|![Web Editor](https://dl.gitea.io/screenshots/web_editor.png)|![Activity](https://dl.gitea.io/screenshots/activity.png)|
-|![New Migration](https://dl.gitea.io/screenshots/migration.png)|![Migrating](https://dl.gitea.io/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
-![Pull Request Dark](https://dl.gitea.io/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.io/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.io/screenshots/diff_dark.png)|
+|![Branches](https://dl.gitea.com/screenshots/branches.png)|![Web Editor](https://dl.gitea.com/screenshots/web_editor.png)|![Activity](https://dl.gitea.com/screenshots/activity.png)|
+|![New Migration](https://dl.gitea.com/screenshots/migration.png)|![Migrating](https://dl.gitea.com/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
+![Pull Request Dark](https://dl.gitea.com/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.com/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.com/screenshots/diff_dark.png)|

cmd/actions.go

@@ -42,7 +42,7 @@ func runGenerateActionsRunnerToken(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	scope := c.String("scope")
 

cmd/cmd.go

@@ -58,7 +58,7 @@ func confirm() (bool, error) {
 }
 
 func initDB(ctx context.Context) error {
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 	setting.InitSQLLoggersForCli(log.INFO)
 

cmd/doctor.go

@@ -91,7 +91,7 @@ func runRecreateTable(ctx *cli.Context) error {
 	golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info))
 
 	debug := ctx.Bool("debug")
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 
 	if debug {

cmd/dump.go

@@ -182,7 +182,7 @@ func runDump(ctx *cli.Context) error {
 		}
 		fileName += "." + outType
 	}
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	// make sure we are logging to the console no matter what the configuration tells us do to
 	// FIXME: don't use CfgProvider directly

cmd/embedded.go

@@ -99,11 +99,6 @@ type assetFile struct {
 func initEmbeddedExtractor(c *cli.Context) error {
 	setupConsoleLogger(log.ERROR, log.CanColorStderr, os.Stderr)
 
-	// Read configuration file
-	setting.Init(&setting.Options{
-		AllowEmpty: true,
-	})
-
 	patterns, err := compileCollectPatterns(c.Args())
 	if err != nil {
 		return err

cmd/mailer.go

@@ -16,7 +16,7 @@ func runSendMail(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	if err := argsSet(c, "title"); err != nil {
 		return err

cmd/restore_repo.go

@@ -51,7 +51,7 @@ func runRestoreRepository(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	var units []string
 	if s := c.String("units"); s != "" {
 		units = strings.Split(s, ",")

cmd/serv.go

@@ -61,7 +61,7 @@ func setup(ctx context.Context, debug bool) {
 	} else {
 		setupConsoleLogger(log.FATAL, false, os.Stderr)
 	}
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	if debug {
 		setting.RunMode = "dev"
 	}

cmd/web.go

@@ -101,6 +101,110 @@ func createPIDFile(pidPath string) {
 	}
 }
 
+func serveInstall(ctx *cli.Context) error {
+	log.Info("Gitea version: %s%s", setting.AppVer, setting.AppBuiltWith)
+	log.Info("App path: %s", setting.AppPath)
+	log.Info("Work path: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Config file: %s", setting.CustomConf)
+	log.Info("Prepare to run install page")
+
+	routers.InitWebInstallPage(graceful.GetManager().HammerContext())
+
+	// Flag for port number in case first time run conflict
+	if ctx.IsSet("port") {
+		if err := setPort(ctx.String("port")); err != nil {
+			return err
+		}
+	}
+	if ctx.IsSet("install-port") {
+		if err := setPort(ctx.String("install-port")); err != nil {
+			return err
+		}
+	}
+	c := install.Routes()
+	err := listen(c, false)
+	if err != nil {
+		log.Critical("Unable to open listener for installer. Is Gitea already running?")
+		graceful.GetManager().DoGracefulShutdown()
+	}
+	select {
+	case <-graceful.GetManager().IsShutdown():
+		<-graceful.GetManager().Done()
+		log.Info("PID: %d Gitea Web Finished", os.Getpid())
+		log.GetManager().Close()
+		return err
+	default:
+	}
+	return nil
+}
+
+func serveInstalled(ctx *cli.Context) error {
+	setting.InitCfgProvider(setting.CustomConf)
+	setting.LoadCommonSettings()
+	setting.MustInstalled()
+
+	log.Info("Gitea version: %s%s", setting.AppVer, setting.AppBuiltWith)
+	log.Info("App path: %s", setting.AppPath)
+	log.Info("Work path: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Config file: %s", setting.CustomConf)
+	log.Info("Run mode: %s", setting.RunMode)
+	log.Info("Prepare to run web server")
+
+	if setting.AppWorkPathMismatch {
+		log.Error("WORK_PATH from config %q doesn't match other paths from environment variables or command arguments. "+
+			"Only WORK_PATH in config should be set and used. Please remove the other outdated work paths from environment variables and command arguments", setting.CustomConf)
+	}
+
+	rootCfg := setting.CfgProvider
+	if rootCfg.Section("").Key("WORK_PATH").String() == "" {
+		saveCfg, err := rootCfg.PrepareSaving()
+		if err != nil {
+			log.Error("Unable to prepare saving WORK_PATH=%s to config %q: %v\nYou must set it manually, otherwise there might be bugs when accessing the git repositories.", setting.AppWorkPath, setting.CustomConf, err)
+		} else {
+			rootCfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath)
+			saveCfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath)
+			if err = saveCfg.Save(); err != nil {
+				log.Error("Unable to update WORK_PATH=%s to config %q: %v\nYou must set it manually, otherwise there might be bugs when accessing the git repositories.", setting.AppWorkPath, setting.CustomConf, err)
+			}
+		}
+	}
+
+	routers.InitWebInstalled(graceful.GetManager().HammerContext())
+
+	// We check that AppDataPath exists here (it should have been created during installation)
+	// We can't check it in `InitWebInstalled`, because some integration tests
+	// use cmd -> InitWebInstalled, but the AppDataPath doesn't exist during those tests.
+	if _, err := os.Stat(setting.AppDataPath); err != nil {
+		log.Fatal("Can not find APP_DATA_PATH %q", setting.AppDataPath)
+	}
+
+	// Override the provided port number within the configuration
+	if ctx.IsSet("port") {
+		if err := setPort(ctx.String("port")); err != nil {
+			return err
+		}
+	}
+
+	// Set up Chi routes
+	c := routers.NormalRoutes()
+	err := listen(c, true)
+	<-graceful.GetManager().Done()
+	log.Info("PID: %d Gitea Web Finished", os.Getpid())
+	log.GetManager().Close()
+	return err
+}
+
+func servePprof() {
+	http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
+	_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
+	// The pprof server is for debug purpose only, it shouldn't be exposed on public network. At the moment it's not worth to introduce a configurable option for it.
+	log.Info("Starting pprof server on localhost:6060")
+	log.Info("Stopped pprof server: %v", http.ListenAndServe("localhost:6060", nil))
+	finished()
+}
+
 func runWeb(ctx *cli.Context) error {
 	if ctx.Bool("verbose") {
 		setupConsoleLogger(log.TRACE, log.CanColorStdout, os.Stdout)
@@ -128,75 +232,19 @@ func runWeb(ctx *cli.Context) error {
 		createPIDFile(ctx.String("pid"))
 	}
 
-	// Perform pre-initialization
-	needsInstall := install.PreloadSettings(graceful.GetManager().HammerContext())
-	if needsInstall {
-		// Flag for port number in case first time run conflict
-		if ctx.IsSet("port") {
-			if err := setPort(ctx.String("port")); err != nil {
+	if !setting.InstallLock {
+		if err := serveInstall(ctx); err != nil {
 			return err
 		}
-		}
-		if ctx.IsSet("install-port") {
-			if err := setPort(ctx.String("install-port")); err != nil {
-				return err
-			}
-		}
-		c := install.Routes()
-		err := listen(c, false)
-		if err != nil {
-			log.Critical("Unable to open listener for installer. Is Gitea already running?")
-			graceful.GetManager().DoGracefulShutdown()
-		}
-		select {
-		case <-graceful.GetManager().IsShutdown():
-			<-graceful.GetManager().Done()
-			log.Info("PID: %d Gitea Web Finished", os.Getpid())
-			log.GetManager().Close()
-			return err
-		default:
-		}
 	} else {
 		NoInstallListener()
 	}
 
 	if setting.EnablePprof {
-		go func() {
-			http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
-			_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
-			// The pprof server is for debug purpose only, it shouldn't be exposed on public network. At the moment it's not worth to introduce a configurable option for it.
-			log.Info("Starting pprof server on localhost:6060")
-			log.Info("Stopped pprof server: %v", http.ListenAndServe("localhost:6060", nil))
-			finished()
-		}()
+		go servePprof()
 	}
 
-	log.Info("Global init")
-	// Perform global initialization
-	setting.Init(&setting.Options{})
-	routers.GlobalInitInstalled(graceful.GetManager().HammerContext())
-
-	// We check that AppDataPath exists here (it should have been created during installation)
-	// We can't check it in `GlobalInitInstalled`, because some integration tests
-	// use cmd -> GlobalInitInstalled, but the AppDataPath doesn't exist during those tests.
-	if _, err := os.Stat(setting.AppDataPath); err != nil {
-		log.Fatal("Can not find APP_DATA_PATH '%s'", setting.AppDataPath)
-	}
-
-	// Override the provided port number within the configuration
-	if ctx.IsSet("port") {
-		if err := setPort(ctx.String("port")); err != nil {
-			return err
-		}
-	}
-
-	// Set up Chi routes
-	c := routers.NormalRoutes()
-	err := listen(c, true)
-	<-graceful.GetManager().Done()
-	log.Info("PID: %d Gitea Web Finished", os.Getpid())
-	log.GetManager().Close()
-	return err
+	return serveInstalled(ctx)
 }
 
 func setPort(port string) error {
@@ -217,9 +265,15 @@ func setPort(port string) error {
 		defaultLocalURL += ":" + setting.HTTPPort + "/"
 
 		// Save LOCAL_ROOT_URL if port changed
-		setting.CfgProvider.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
-		if err := setting.CfgProvider.Save(); err != nil {
-			return fmt.Errorf("Failed to save config file: %v", err)
+		rootCfg := setting.CfgProvider
+		saveCfg, err := rootCfg.PrepareSaving()
+		if err != nil {
+			return fmt.Errorf("failed to save config file: %v", err)
+		}
+		rootCfg.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
+		saveCfg.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
+		if err = saveCfg.Save(); err != nil {
+			return fmt.Errorf("failed to save config file: %v", err)
 		}
 	}
 	return nil

contrib/environment-to-ini/environment-to-ini.go

@@ -81,8 +81,6 @@ func main() {
 		},
 	}
 	app.Action = runEnvironmentToIni
-	setting.SetCustomPathAndConf("", "", "")
-
 	err := app.Run(os.Args)
 	if err != nil {
 		log.Fatal("Failed to run app with %s: %v", os.Args, err)
@@ -90,12 +88,13 @@ func main() {
 }
 
 func runEnvironmentToIni(c *cli.Context) error {
-	providedCustom := c.String("custom-path")
-	providedConf := c.String("config")
-	providedWorkPath := c.String("work-path")
-	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
+	setting.InitWorkPathAndCfgProvider(os.Getenv, setting.ArgWorkPathAndCustomConf{
+		WorkPath:   c.String("work-path"),
+		CustomPath: c.String("custom-path"),
+		CustomConf: c.String("config"),
+	})
 
-	cfg, err := setting.NewConfigProviderFromFile(&setting.Options{CustomConf: setting.CustomConf, AllowEmpty: true})
+	cfg, err := setting.NewConfigProviderFromFile(setting.CustomConf)
 	if err != nil {
 		log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
 	}

custom/conf/app.example.ini

@@ -119,10 +119,13 @@ RUN_USER = ; git
 ;; Permission for unix socket
 ;UNIX_SOCKET_PERMISSION = 666
 ;;
-;; Local (DMZ) URL for Gitea workers (such as SSH update) accessing web service.
-;; In most cases you do not need to change the default value.
-;; Alter it only if your SSH server node is not the same as HTTP node.
-;; Do not set this variable if PROTOCOL is set to 'unix'.
+;; Local (DMZ) URL for Gitea workers (such as SSH update) accessing web service. In
+;; most cases you do not need to change the default value. Alter it only if
+;; your SSH server node is not the same as HTTP node. For different protocol, the default
+;; values are different. If `PROTOCOL` is `http+unix`, the default value is `http://unix/`.
+;; If `PROTOCOL` is `fcgi` or `fcgi+unix`, the default value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
+;; If listen on `0.0.0.0`, the default value is `%(PROTOCOL)s://localhost:%(HTTP_PORT)s/`, Otherwise the default
+;; value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
 ;LOCAL_ROOT_URL = %(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/
 ;;
 ;; When making local connections pass the PROXY protocol header.
@@ -344,9 +347,6 @@ NAME = gitea
 USER = root
 ;PASSWD = ;Use PASSWD = `your password` for quoting if you use special characters in the password.
 ;SSL_MODE = false ; either "false" (default), "true", or "skip-verify"
-;CHARSET = utf8mb4 ;either "utf8" or "utf8mb4", default is "utf8mb4".
-;;
-;; NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
@@ -1337,10 +1337,10 @@ LEVEL = Info
 ;; Issue indexer storage path, available when ISSUE_INDEXER_TYPE is bleve
 ;ISSUE_INDEXER_PATH = indexers/issues.bleve ; Relative paths will be made absolute against _`AppWorkPath`_.
 ;;
-;; Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch or meilisearch
-;ISSUE_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
+;; Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch (e.g. http://elastic:password@localhost:9200) or meilisearch (e.g. http://:apikey@localhost:7700)
+;ISSUE_INDEXER_CONN_STR =
 ;;
-;; Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
+;; Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch or meilisearch.
 ;ISSUE_INDEXER_NAME = gitea_issues
 ;;
 ;; Timeout the indexer if it takes longer than this to start.
@@ -2159,7 +2159,7 @@ LEVEL = Info
 ;RUN_AT_START = false
 ;ENABLE_SUCCESS_NOTICE = false
 ;SCHEDULE = @every 168h
-;HTTP_ENDPOINT = https://dl.gitea.io/gitea/version.json
+;HTTP_ENDPOINT = https://dl.gitea.com/gitea/version.json
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docker/manifest.rootless.tmpl

@@ -1,12 +1,14 @@
 image: gitea/gitea:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{#if (hasPrefix "refs/heads/release/v" build.ref)}}{{trimPrefix "refs/heads/release/v" build.ref}}-{{/if}}nightly{{/if}}-rootless
 {{#if build.tags}}
 {{#unless (contains "-rc" build.tag)}}
+{{#unless (contains "-dev" build.tag)}}
 tags:
 {{#each build.tags}}
   - {{this}}-rootless
 {{/each}}
   - "latest-rootless"
 {{/unless}}
+{{/unless}}
 {{/if}}
 manifests:
   -

docker/manifest.tmpl

@@ -1,12 +1,14 @@
 image: gitea/gitea:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{#if (hasPrefix "refs/heads/release/v" build.ref)}}{{trimPrefix "refs/heads/release/v" build.ref}}-{{/if}}nightly{{/if}}
 {{#if build.tags}}
 {{#unless (contains "-rc" build.tag)}}
+{{#unless (contains "-dev" build.tag)}}
 tags:
 {{#each build.tags}}
   - {{this}}
 {{/each}}
   - "latest"
 {{/unless}}
+{{/unless}}
 {{/if}}
 manifests:
   -

docs/content/doc/administration/command-line.en-us.md

@@ -108,6 +108,14 @@ Admin operations:
         - `--all`, `-A`: Force a password change for all users
         - `--exclude username`, `-e username`: Exclude the given user. Can be set multiple times.
         - `--unset`: Revoke forced password change for the given users
+    - `generate-access-token`:
+      - Options:
+        - `--username value`, `-u value`: Username. Required.
+        - `--token-name value`, `-t value`: Token name. Required.
+        - `--scopes value`: Comma-separated list of scopes. Scopes follow the format `[read|write]:<block>` or `all` where `<block>` is one of the available visual groups you can see when opening the API page showing the available routes (for example `repo`).
+      - Examples:
+        - `gitea admin user generate-access-token --username myname --token-name mytoken`
+        - `gitea admin user generate-access-token --help`
   - `regenerate`
     - Options:
       - `hooks`: Regenerate Git Hooks for all repositories

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -314,8 +314,11 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `LOCAL_ROOT_URL`: **%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/**: Local
    (DMZ) URL for Gitea workers (such as SSH update) accessing web service. In
    most cases you do not need to change the default value. Alter it only if
-   your SSH server node is not the same as HTTP node. Do not set this variable
-   if `PROTOCOL` is set to `http+unix`.
+   your SSH server node is not the same as HTTP node. For different protocol, the default
+   values are different. If `PROTOCOL` is `http+unix`, the default value is `http://unix/`.
+   If `PROTOCOL` is `fcgi` or `fcgi+unix`, the default value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
+   If listen on `0.0.0.0`, the default value is `%(PROTOCOL)s://localhost:%(HTTP_PORT)s/`, Otherwise the default
+   value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
 - `LOCAL_USE_PROXY_PROTOCOL`: **%(USE_PROXY_PROTOCOL)s**: When making local connections pass the PROXY protocol header.
    This should be set to false if the local connection will go through the proxy.
 - `PER_WRITE_TIMEOUT`: **30s**: Timeout for any write to the connection. (Set to -1 to
@@ -443,7 +446,6 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `SQLITE_TIMEOUT`: **500**: Query timeout for SQLite3 only.
 - `SQLITE_JOURNAL_MODE`: **""**: Change journal mode for SQlite3. Can be used to enable [WAL mode](https://www.sqlite.org/wal.html) when high load causes write congestion. See [SQlite3 docs](https://www.sqlite.org/pragma.html#pragma_journal_mode) for possible values. Defaults to the default for the database file, often DELETE.
 - `ITERATE_BUFFER_SIZE`: **50**: Internal buffer size for iterating.
-- `CHARSET`: **utf8mb4**: For MySQL only, either "utf8" or "utf8mb4". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.
 - `LOG_SQL`: **true**: Log the executed SQL.
 - `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed.
@@ -459,15 +461,15 @@ relation to port exhaustion.
 ## Indexer (`indexer`)
 
 - `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently supported: `bleve`, `db`, `elasticsearch` or `meilisearch`.
-- `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch, or meilisearch. i.e. http://elastic:changeme@localhost:9200
-- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
+- `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch (e.g. http://elastic:password@localhost:9200) or meilisearch (e.g. http://:apikey@localhost:7700)
+- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch or meilisearch.
 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_REPO_TYPES`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`.
 - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
-- `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200
+- `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:password@localhost:9200
 - `REPO_INDEXER_NAME`: **gitea_codes**: Code indexer name, available when `REPO_INDEXER_TYPE` is elasticsearch
 
 - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
@@ -1013,7 +1015,7 @@ Default templates for project boards:
 - `RUN_AT_START`: **false**: Run tasks at start up time (if ENABLED).
 - `ENABLE_SUCCESS_NOTICE`: **true**: Set to false to switch off success notices.
 - `SCHEDULE`: **@every 168h**: Cron syntax for scheduling a work, e.g. `@every 168h`.
-- `HTTP_ENDPOINT`: **https://dl.gitea.io/gitea/version.json**: the endpoint that Gitea will check for newer versions
+- `HTTP_ENDPOINT`: **https://dl.gitea.com/gitea/version.json**: the endpoint that Gitea will check for newer versions
 
 #### Cron -  Delete all old system notices from database (`cron.delete_old_system_notices`)
 

docs/content/doc/help/faq.en-us.md

@@ -396,8 +396,6 @@ Please run `gitea convert`, or run `ALTER DATABASE database_name CHARACTER SET u
 for the database_name and run `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
 for each table in the database.
 
-You will also need to change the app.ini database charset to `CHARSET=utf8mb4`.
-
 ## Why are Emoji displaying only as placeholders or in monochrome
 
 Gitea requires the system or browser to have one of the supported Emoji fonts installed, which are Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji and Twemoji Mozilla. Generally, the operating system should already provide one of these fonts, but especially on Linux, it may be necessary to install them manually.

docs/content/doc/help/faq.zh-cn.md

@@ -31,7 +31,7 @@ menu:
 
 **注意：**此示例也适用于Docker镜像！
 
-在我们的[下载页面](https://dl.gitea.io/gitea/)上，您会看到一个1.7目录，以及1.7.0、1.7.1、1.7.2、1.7.3、1.7.4、1.7.5和1.7.6的目录。
+在我们的[下载页面](https://dl.gitea.com/gitea/)上，您会看到一个1.7目录，以及1.7.0、1.7.1、1.7.2、1.7.3、1.7.4、1.7.5和1.7.6的目录。
 
 1.7目录和1.7.0目录是**不同**的。1.7目录是在每个合并到[`release/v1.7`](https://github.com/go-gitea/gitea/tree/release/v1.7)分支的提交上构建的。
 

docs/content/doc/installation/from-binary.fr-fr.md

@@ -17,10 +17,10 @@ menu:
 
 # Installation avec le binaire pré-compilé
 
-Tous les binaires sont livrés avec le support de SQLite, MySQL et PostgreSQL, et sont construits avec les ressources incorporées. Gardez à l'esprit que cela peut être différent pour les versions antérieures. L'installation basée sur nos binaires est assez simple, il suffit de choisir le fichier correspondant à votre plateforme à partir de la [page de téléchargement](https://dl.gitea.io/gitea). Copiez l'URL et remplacer l'URL dans les commandes suivantes par la nouvelle:
+Tous les binaires sont livrés avec le support de SQLite, MySQL et PostgreSQL, et sont construits avec les ressources incorporées. Gardez à l'esprit que cela peut être différent pour les versions antérieures. L'installation basée sur nos binaires est assez simple, il suffit de choisir le fichier correspondant à votre plateforme à partir de la [page de téléchargement](https://dl.gitea.com/gitea). Copiez l'URL et remplacer l'URL dans les commandes suivantes par la nouvelle:
 
 ```
-wget -O gitea https://dl.gitea.io/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
+wget -O gitea https://dl.gitea.com/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
 chmod +x gitea
 ```
 

docs/content/doc/installation/from-binary.zh-tw.md

@@ -17,10 +17,10 @@ menu:
 
 # 從執行檔安裝
 
-所有的執行檔皆支援 SQLite, MySQL and PostgreSQL，且所有檔案都已經包在執行檔內，這一點跟之前的版本有所不同。關於執行檔的安裝方式非常簡單，只要從[下載頁面](https://dl.gitea.io/gitea)選擇相對應平台，複製下載連結，使用底下指令就可以完成了:
+所有的執行檔皆支援 SQLite, MySQL and PostgreSQL，且所有檔案都已經包在執行檔內，這一點跟之前的版本有所不同。關於執行檔的安裝方式非常簡單，只要從[下載頁面](https://dl.gitea.com/gitea)選擇相對應平台，複製下載連結，使用底下指令就可以完成了:
 
 ```
-wget -O gitea https://dl.gitea.io/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
+wget -O gitea https://dl.gitea.com/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
 chmod +x gitea
 ```
 

docs/content/doc/installation/on-kubernetes.en-us.md

@@ -22,7 +22,7 @@ Gitea provides a Helm Chart to allow for installation on kubernetes.
 A non-customized install can be done with:
 
 ```
-helm repo add gitea-charts https://dl.gitea.io/charts/
+helm repo add gitea-charts https://dl.gitea.com/charts/
 helm install gitea gitea-charts/gitea
 ```
 

docs/content/doc/installation/on-kubernetes.zh-cn.md

@@ -22,7 +22,7 @@ Gitea 已经提供了便于在 Kubernetes 云原生环境中安装所需的 Helm
 默认安装指令为：
 
 ```bash
-helm repo add gitea https://dl.gitea.io/charts
+helm repo add gitea https://dl.gitea.com/charts
 helm repo update
 helm install gitea gitea/gitea
 ```

docs/content/doc/installation/on-kubernetes.zh-tw.md

@@ -22,7 +22,7 @@ Gitea 提供 Helm Chart 用來安裝於 kubernetes。
 非自訂安裝可使用下列指令：
 
 ```
-helm repo add gitea-charts https://dl.gitea.io/charts/
+helm repo add gitea-charts https://dl.gitea.com/charts/
 helm install gitea gitea-charts/gitea
 ```
 

docs/content/doc/installation/upgrade-from-gogs.en-us.md

@@ -27,7 +27,7 @@ There are some basic steps to follow. On a Linux system run as the Gogs user:
 
 * Create a Gogs backup with `gogs backup`. This creates `gogs-backup-[timestamp].zip` file
   containing all important Gogs data. You would need it if you wanted to move to the `gogs` back later.
-* Download the file matching the destination platform from the [downloads page](https://dl.gitea.io/gitea/).
+* Download the file matching the destination platform from the [downloads page](https://dl.gitea.com/gitea/).
  It should be `1.0.x` version. Migrating from `gogs` to any other version is impossible.
 * Put the binary at the desired install location.
 * Copy `gogs/custom/conf/app.ini` to `gitea/custom/conf/app.ini`.
@@ -79,11 +79,11 @@ There are some basic steps to follow. On a Linux system run as the Gogs user:
 After successful migration from `gogs` to `gitea 1.0.x`, it is possible to upgrade `gitea` to a modern version
 in a two steps process.
 
-Upgrade to [`gitea 1.6.4`](https://dl.gitea.io/gitea/1.6.4/) first. Download the file matching
-the destination platform from the [downloads page](https://dl.gitea.io/gitea/1.6.4/) and replace the binary.
+Upgrade to [`gitea 1.6.4`](https://dl.gitea.com/gitea/1.6.4/) first. Download the file matching
+the destination platform from the [downloads page](https://dl.gitea.com/gitea/1.6.4/) and replace the binary.
 Run Gitea at least once and check that everything works as expected.
 
-Then repeat the procedure, but this time using the [latest release](https://dl.gitea.io/gitea/{{< version >}}/).
+Then repeat the procedure, but this time using the [latest release](https://dl.gitea.com/gitea/{{< version >}}/).
 
 ## Upgrading from a more recent version of Gogs
 

docs/content/doc/installation/upgrade-from-gogs.fr-fr.md

@@ -22,7 +22,7 @@ menu:
 Veuillez suivre les étapes ci-dessous. Sur Unix, toute les commandes s'exécutent en tant que l'utilisateur utilisé pour votre installation de Gogs :
 
 * Crééer une sauvegarde de Gogs avec la commande `gogs dump`. Le fichier nouvellement créé `gogs-dump-[timestamp].zip` contient toutes les données de votre instance de Gogs.
-* Téléchargez le fichier correspondant à votre plateforme à partir de la [page de téléchargements](https://dl.gitea.io/gitea).
+* Téléchargez le fichier correspondant à votre plateforme à partir de la [page de téléchargements](https://dl.gitea.com/gitea).
 * Mettez la binaire dans le répertoire d'installation souhaité.
 * Copiez le fichier `gogs/custom/conf/app.ini` vers `gitea/custom/conf/app.ini`.
 * Si vous avez personnalisé les répertoires `templates, public` dans `gogs/custom/`, copiez-les vers `gitea/custom/`.

docs/content/doc/installation/upgrade-from-gogs.zh-tw.md

@@ -27,7 +27,7 @@ menu:
 
 - 使用 `gogs backup` 建立 Gogs 的備份。這會建立檔案 `gogs-backup-[timestamp].zip` 包含所有重要的 Gogs 資料。
   如果稍後您要恢復到 `gogs` 時會用到它。
-- 從[下載頁](https://dl.gitea.io/gitea/)下載對應您平臺的檔案。請下載 `1.0.x` 版，從 `gogs` 遷移到其它版本是不可行的。
+- 從[下載頁](https://dl.gitea.com/gitea/)下載對應您平臺的檔案。請下載 `1.0.x` 版，從 `gogs` 遷移到其它版本是不可行的。
 - 將二進位檔放到適當的安裝位置。
 - 複製 `gogs/custom/conf/app.ini` 到 `gitea/custom/conf/app.ini`。
 - 從 `gogs/custom/` 複製自訂 `templates, public` 到 `gitea/custom/`。
@@ -77,10 +77,10 @@ menu:
 
 成功從 `gogs` 升級到 `gitea 1.0.x` 後再用 2 個步驟即可升級到最新版的 `gitea`。
 
-請先升級到 [`gitea 1.6.4`](https://dl.gitea.io/gitea/1.6.4/)，先從[下載頁](https://dl.gitea.io/gitea/1.6.4/)下載
+請先升級到 [`gitea 1.6.4`](https://dl.gitea.com/gitea/1.6.4/)，先從[下載頁](https://dl.gitea.com/gitea/1.6.4/)下載
 您平臺的二進位檔取代既有的。至少執行一次 Gitea 並確認一切符合預期。
 
-接著重複上述步驟，但這次請使用[最新發行版本](https://dl.gitea.io/gitea/{{< version >}}/)。
+接著重複上述步驟，但這次請使用[最新發行版本](https://dl.gitea.com/gitea/{{< version >}}/)。
 
 ## 從更新版本的 Gogs 升級
 

go.mod

@@ -122,7 +122,7 @@ require (
 	mvdan.cc/xurls/v2 v2.4.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.12
-	xorm.io/xorm v1.3.3-0.20230219231735-056cecc97e9e
+	xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75
 )
 
 require (

go.sum

@@ -1923,5 +1923,5 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1:
 xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
 xorm.io/builder v0.3.12 h1:ASZYX7fQmy+o8UJdhlLHSW57JDOkM8DNhcAF5d0LiJM=
 xorm.io/builder v0.3.12/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
-xorm.io/xorm v1.3.3-0.20230219231735-056cecc97e9e h1:d5PY6mwuQK5/7T6VKfFswaKMzLmGTHkJ/ZS7+cUIAjk=
-xorm.io/xorm v1.3.3-0.20230219231735-056cecc97e9e/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw=
+xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75 h1:ReBAlO50dCIXCWF8Gbi0ZRa62AGAwCJNCPaUNUa7JSg=
+xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw=

main.go

@@ -33,30 +33,60 @@ var (
 	Tags = ""
 	// MakeVersion holds the current Make version if built with make
 	MakeVersion = ""
-
-	originalAppHelpTemplate        = ""
-	originalCommandHelpTemplate    = ""
-	originalSubcommandHelpTemplate = ""
 )
 
 func init() {
 	setting.AppVer = Version
 	setting.AppBuiltWith = formatBuiltWith()
 	setting.AppStartTime = time.Now().UTC()
+}
 
-	// Grab the original help templates
-	originalAppHelpTemplate = cli.AppHelpTemplate
-	originalCommandHelpTemplate = cli.CommandHelpTemplate
-	originalSubcommandHelpTemplate = cli.SubcommandHelpTemplate
+// cmdHelp is our own help subcommand with more information
+// test cases:
+// ./gitea help
+// ./gitea -h
+// ./gitea web help
+// ./gitea web -h (due to cli lib limitation, this won't call our cmdHelp, so no extra info)
+// ./gitea admin
+// ./gitea admin help
+// ./gitea admin auth help
+// ./gitea -c /tmp/app.ini -h
+// ./gitea -c /tmp/app.ini help
+// ./gitea help -c /tmp/app.ini
+// GITEA_WORK_DIR=/tmp ./gitea help
+// GITEA_WORK_DIR=/tmp ./gitea help --work-path /tmp/other
+// GITEA_WORK_DIR=/tmp ./gitea help --config /tmp/app-other.ini
+var cmdHelp = cli.Command{
+	Name:      "help",
+	Aliases:   []string{"h"},
+	Usage:     "Shows a list of commands or help for one command",
+	ArgsUsage: "[command]",
+	Action: func(c *cli.Context) (err error) {
+		args := c.Args()
+		if args.Present() {
+			err = cli.ShowCommandHelp(c, args.First())
+		} else {
+			err = cli.ShowAppHelp(c)
+		}
+		_, _ = fmt.Fprintf(c.App.Writer, `
+DEFAULT CONFIGURATION:
+   AppPath:    %s
+   WorkPath:   %s
+   CustomPath: %s
+   ConfigFile: %s
+
+`, setting.AppPath, setting.AppWorkPath, setting.CustomPath, setting.CustomConf)
+		return err
+	},
 }
 
 func main() {
 	app := cli.NewApp()
 	app.Name = "Gitea"
 	app.Usage = "A painless self-hosted Git service"
-	app.Description = `By default, gitea will start serving using the webserver with no
-arguments - which can alternatively be run by running the subcommand web.`
+	app.Description = `By default, Gitea will start serving using the web-server with no argument, which can alternatively be run by running the subcommand "web".`
 	app.Version = Version + formatBuiltWith()
+	app.EnableBashCompletion = true
 	app.Commands = []cli.Command{
 		cmd.CmdWeb,
 		cmd.CmdServ,
@@ -77,118 +107,80 @@ arguments - which can alternatively be run by running the subcommand web.`
 		cmd.CmdRestoreRepository,
 		cmd.CmdActions,
 	}
-	// Now adjust these commands to add our global configuration options
-
-	// First calculate the default paths and set the AppHelpTemplates in this context
-	setting.SetCustomPathAndConf("", "", "")
-	setAppHelpTemplates()
 
 	// default configuration flags
-	defaultFlags := []cli.Flag{
+	globalFlags := []cli.Flag{
+		cli.HelpFlag,
 		cli.StringFlag{
 			Name:  "custom-path, C",
-			Value: setting.CustomPath,
-			Usage: "Custom path file path",
+			Usage: "Set custom path (defaults to '{WorkPath}/custom')",
 		},
 		cli.StringFlag{
 			Name:  "config, c",
 			Value: setting.CustomConf,
-			Usage: "Custom configuration file path",
+			Usage: "Set custom config file (defaults to '{WorkPath}/custom/conf/app.ini')",
 		},
-		cli.VersionFlag,
 		cli.StringFlag{
 			Name:  "work-path, w",
-			Value: setting.AppWorkPath,
-			Usage: "Set the gitea working path",
+			Usage: "Set Gitea's working path (defaults to the Gitea's binary directory)",
 		},
 	}
 
 	// Set the default to be equivalent to cmdWeb and add the default flags
+	app.Flags = append(app.Flags, globalFlags...)
 	app.Flags = append(app.Flags, cmd.CmdWeb.Flags...)
-	app.Flags = append(app.Flags, defaultFlags...)
-	app.Action = cmd.CmdWeb.Action
-
-	// Add functions to set these paths and these flags to the commands
-	app.Before = establishCustomPath
+	app.Action = prepareWorkPathAndCustomConf(cmd.CmdWeb.Action)
+	app.HideHelp = true // use our own help action to show helps (with more information like default config)
+	app.Commands = append(app.Commands, cmdHelp)
 	for i := range app.Commands {
-		setFlagsAndBeforeOnSubcommands(&app.Commands[i], defaultFlags, establishCustomPath)
+		prepareSubcommands(&app.Commands[i], globalFlags)
 	}
 
-	app.EnableBashCompletion = true
-
 	err := app.Run(os.Args)
 	if err != nil {
-		log.Fatal("Failed to run app with %s: %v", os.Args, err)
+		_, _ = fmt.Fprintf(app.Writer, "\nFailed to run with %s: %v\n", os.Args, err)
 	}
 
 	log.GetManager().Close()
 }
 
-func setFlagsAndBeforeOnSubcommands(command *cli.Command, defaultFlags []cli.Flag, before cli.BeforeFunc) {
+func prepareSubcommands(command *cli.Command, defaultFlags []cli.Flag) {
 	command.Flags = append(command.Flags, defaultFlags...)
-	command.Before = establishCustomPath
+	command.Action = prepareWorkPathAndCustomConf(command.Action)
+	command.HideHelp = true
+	if command.Name != "help" {
+		command.Subcommands = append(command.Subcommands, cmdHelp)
+	}
 	for i := range command.Subcommands {
-		setFlagsAndBeforeOnSubcommands(&command.Subcommands[i], defaultFlags, before)
+		prepareSubcommands(&command.Subcommands[i], defaultFlags)
 	}
 }
 
-func establishCustomPath(ctx *cli.Context) error {
-	var providedCustom string
-	var providedConf string
-	var providedWorkPath string
-
-	currentCtx := ctx
-	for {
-		if len(providedCustom) != 0 && len(providedConf) != 0 && len(providedWorkPath) != 0 {
-			break
+// prepareWorkPathAndCustomConf wraps the Action to prepare the work path and custom config
+// It can't use "Before", because each level's sub-command's Before will be called one by one, so the "init" would be done multiple times
+func prepareWorkPathAndCustomConf(action any) func(ctx *cli.Context) error {
+	return func(ctx *cli.Context) error {
+		var args setting.ArgWorkPathAndCustomConf
+		curCtx := ctx
+		for curCtx != nil {
+			if curCtx.IsSet("work-path") && args.WorkPath == "" {
+				args.WorkPath = curCtx.String("work-path")
 			}
-		if currentCtx == nil {
-			break
+			if curCtx.IsSet("custom-path") && args.CustomPath == "" {
+				args.CustomPath = curCtx.String("custom-path")
 			}
-		if currentCtx.IsSet("custom-path") && len(providedCustom) == 0 {
-			providedCustom = currentCtx.String("custom-path")
+			if curCtx.IsSet("config") && args.CustomConf == "" {
+				args.CustomConf = curCtx.String("config")
 			}
-		if currentCtx.IsSet("config") && len(providedConf) == 0 {
-			providedConf = currentCtx.String("config")
+			curCtx = curCtx.Parent()
 		}
-		if currentCtx.IsSet("work-path") && len(providedWorkPath) == 0 {
-			providedWorkPath = currentCtx.String("work-path")
+		setting.InitWorkPathAndCommonConfig(os.Getenv, args)
+		if ctx.Bool("help") || action == nil {
+			// the default behavior of "urfave/cli": "nil action" means "show help"
+			return cmdHelp.Action.(func(ctx *cli.Context) error)(ctx)
 		}
-		currentCtx = currentCtx.Parent()
-
+		return action.(func(*cli.Context) error)(ctx)
 	}
-	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
-
-	setAppHelpTemplates()
-
-	if ctx.IsSet("version") {
-		cli.ShowVersion(ctx)
-		os.Exit(0)
-	}
-
-	return nil
-}
-
-func setAppHelpTemplates() {
-	cli.AppHelpTemplate = adjustHelpTemplate(originalAppHelpTemplate)
-	cli.CommandHelpTemplate = adjustHelpTemplate(originalCommandHelpTemplate)
-	cli.SubcommandHelpTemplate = adjustHelpTemplate(originalSubcommandHelpTemplate)
-}
-
-func adjustHelpTemplate(originalTemplate string) string {
-	overridden := ""
-	if _, ok := os.LookupEnv("GITEA_CUSTOM"); ok {
-		overridden = "(GITEA_CUSTOM)"
-	}
-
-	return fmt.Sprintf(`%s
-DEFAULT CONFIGURATION:
-     CustomPath:  %s %s
-     CustomConf:  %s
-     AppPath:     %s
-     AppWorkPath: %s
-
-`, originalTemplate, setting.CustomPath, overridden, setting.CustomConf, setting.AppPath, setting.AppWorkPath)
 }
 
 func formatBuiltWith() string {

models/actions/run.go

@@ -39,8 +39,9 @@ type ActionRun struct {
 	IsForkPullRequest bool                         // If this is triggered by a PR from a forked repository or an untrusted user, we need to check if it is approved and limit permissions when running the workflow.
 	NeedApproval      bool                         // may need approval if it's a fork pull request
 	ApprovedBy        int64                        `xorm:"index"` // who approved
-	Event             webhook_module.HookEventType
+	Event             webhook_module.HookEventType // the webhook event that causes the workflow to run
 	EventPayload      string                       `xorm:"LONGTEXT"`
+	TriggerEvent      string                       // the trigger event defined in the `on` configuration of the triggered workflow
 	Status            Status                       `xorm:"index"`
 	Started           timeutil.TimeStamp
 	Stopped           timeutil.TimeStamp

models/actions/run_list.go

@@ -71,6 +71,7 @@ type FindRunOptions struct {
 	WorkflowFileName string
 	TriggerUserID    int64
 	Approved         bool // not util.OptionalBool, it works only when it's true
+	Status           Status
 }
 
 func (opts FindRunOptions) toConds() builder.Cond {
@@ -90,6 +91,9 @@ func (opts FindRunOptions) toConds() builder.Cond {
 	if opts.Approved {
 		cond = cond.And(builder.Gt{"approved_by": 0})
 	}
+	if opts.Status > StatusUnknown {
+		cond = cond.And(builder.Eq{"status": opts.Status})
+	}
 	return cond
 }
 
@@ -106,3 +110,34 @@ func FindRuns(ctx context.Context, opts FindRunOptions) (RunList, int64, error)
 func CountRuns(ctx context.Context, opts FindRunOptions) (int64, error) {
 	return db.GetEngine(ctx).Where(opts.toConds()).Count(new(ActionRun))
 }
+
+type StatusInfo struct {
+	Status          int
+	DisplayedStatus string
+}
+
+// GetStatusInfoList returns a slice of StatusInfo
+func GetStatusInfoList(ctx context.Context) []StatusInfo {
+	// same as those in aggregateJobStatus
+	allStatus := []Status{StatusSuccess, StatusFailure, StatusWaiting, StatusRunning}
+	statusInfoList := make([]StatusInfo, 0, 4)
+	for _, s := range allStatus {
+		statusInfoList = append(statusInfoList, StatusInfo{
+			Status:          int(s),
+			DisplayedStatus: s.String(),
+		})
+	}
+	return statusInfoList
+}
+
+// GetActors returns a slice of Actors
+func GetActors(ctx context.Context, repoID int64) ([]*user_model.User, error) {
+	actors := make([]*user_model.User, 0, 10)
+
+	return actors, db.GetEngine(ctx).Where(builder.In("id", builder.Select("`action_run`.trigger_user_id").From("`action_run`").
+		GroupBy("`action_run`.trigger_user_id").
+		Where(builder.Eq{"`action_run`.repo_id": repoID}))).
+		Cols("id", "name", "full_name", "avatar", "avatar_email", "use_custom_avatar").
+		OrderBy(user_model.GetOrderByName()).
+		Find(&actors)
+}

models/actions/variable.go

@@ -0,0 +1,97 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
+
+	"xorm.io/builder"
+)
+
+type ActionVariable struct {
+	ID          int64              `xorm:"pk autoincr"`
+	OwnerID     int64              `xorm:"UNIQUE(owner_repo_name)"`
+	RepoID      int64              `xorm:"INDEX UNIQUE(owner_repo_name)"`
+	Name        string             `xorm:"UNIQUE(owner_repo_name) NOT NULL"`
+	Data        string             `xorm:"LONGTEXT NOT NULL"`
+	CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"`
+	UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
+}
+
+func init() {
+	db.RegisterModel(new(ActionVariable))
+}
+
+func (v *ActionVariable) Validate() error {
+	if v.OwnerID == 0 && v.RepoID == 0 {
+		return errors.New("the variable is not bound to any scope")
+	}
+	return nil
+}
+
+func InsertVariable(ctx context.Context, ownerID, repoID int64, name, data string) (*ActionVariable, error) {
+	variable := &ActionVariable{
+		OwnerID: ownerID,
+		RepoID:  repoID,
+		Name:    strings.ToUpper(name),
+		Data:    data,
+	}
+	if err := variable.Validate(); err != nil {
+		return variable, err
+	}
+	return variable, db.Insert(ctx, variable)
+}
+
+type FindVariablesOpts struct {
+	db.ListOptions
+	OwnerID int64
+	RepoID  int64
+}
+
+func (opts *FindVariablesOpts) toConds() builder.Cond {
+	cond := builder.NewCond()
+	if opts.OwnerID > 0 {
+		cond = cond.And(builder.Eq{"owner_id": opts.OwnerID})
+	}
+	if opts.RepoID > 0 {
+		cond = cond.And(builder.Eq{"repo_id": opts.RepoID})
+	}
+	return cond
+}
+
+func FindVariables(ctx context.Context, opts FindVariablesOpts) ([]*ActionVariable, error) {
+	var variables []*ActionVariable
+	sess := db.GetEngine(ctx)
+	if opts.PageSize != 0 {
+		sess = db.SetSessionPagination(sess, &opts.ListOptions)
+	}
+	return variables, sess.Where(opts.toConds()).Find(&variables)
+}
+
+func GetVariableByID(ctx context.Context, variableID int64) (*ActionVariable, error) {
+	var variable ActionVariable
+	has, err := db.GetEngine(ctx).Where("id=?", variableID).Get(&variable)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, fmt.Errorf("variable with id %d: %w", variableID, util.ErrNotExist)
+	}
+	return &variable, nil
+}
+
+func UpdateVariable(ctx context.Context, variable *ActionVariable) (bool, error) {
+	count, err := db.GetEngine(ctx).ID(variable.ID).Cols("name", "data").
+		Update(&ActionVariable{
+			Name: variable.Name,
+			Data: variable.Data,
+		})
+	return count != 0, err
+}

models/db/engine.go

@@ -123,7 +123,10 @@ func newXORMEngine() (*xorm.Engine, error) {
 
 // SyncAllTables sync the schemas of all tables, is required by unit test code
 func SyncAllTables() error {
-	return x.StoreEngine("InnoDB").Sync2(tables...)
+	_, err := x.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{
+		WarnIfDatabaseColumnMissed: true,
+	}, tables...)
+	return err
 }
 
 // InitEngine initializes the xorm.Engine and sets it as db.DefaultContext

models/issues/comment.go

@@ -167,7 +167,7 @@ func AsCommentType(typeName string) CommentType {
 
 func (t CommentType) HasContentSupport() bool {
 	switch t {
-	case CommentTypeComment, CommentTypeCode, CommentTypeReview:
+	case CommentTypeComment, CommentTypeCode, CommentTypeReview, CommentTypeDismissReview:
 		return true
 	}
 	return false

models/issues/comment_code.go

@@ -18,11 +18,11 @@ import (
 type CodeComments map[string]map[int64][]*Comment
 
 // FetchCodeComments will return a 2d-map: ["Path"]["Line"] = Comments at line
-func FetchCodeComments(ctx context.Context, issue *Issue, currentUser *user_model.User) (CodeComments, error) {
-	return fetchCodeCommentsByReview(ctx, issue, currentUser, nil)
+func FetchCodeComments(ctx context.Context, issue *Issue, currentUser *user_model.User, showOutdatedComments bool) (CodeComments, error) {
+	return fetchCodeCommentsByReview(ctx, issue, currentUser, nil, showOutdatedComments)
 }
 
-func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *user_model.User, review *Review) (CodeComments, error) {
+func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *user_model.User, review *Review, showOutdatedComments bool) (CodeComments, error) {
 	pathToLineToComment := make(CodeComments)
 	if review == nil {
 		review = &Review{ID: 0}
@@ -33,7 +33,7 @@ func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *u
 		ReviewID: review.ID,
 	}
 
-	comments, err := findCodeComments(ctx, opts, issue, currentUser, review)
+	comments, err := findCodeComments(ctx, opts, issue, currentUser, review, showOutdatedComments)
 	if err != nil {
 		return nil, err
 	}
@@ -47,15 +47,17 @@ func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *u
 	return pathToLineToComment, nil
 }
 
-func findCodeComments(ctx context.Context, opts FindCommentsOptions, issue *Issue, currentUser *user_model.User, review *Review) ([]*Comment, error) {
+func findCodeComments(ctx context.Context, opts FindCommentsOptions, issue *Issue, currentUser *user_model.User, review *Review, showOutdatedComments bool) ([]*Comment, error) {
 	var comments CommentList
 	if review == nil {
 		review = &Review{ID: 0}
 	}
 	conds := opts.ToConds()
-	if review.ID == 0 {
+
+	if !showOutdatedComments && review.ID == 0 {
 		conds = conds.And(builder.Eq{"invalidated": false})
 	}
+
 	e := db.GetEngine(ctx)
 	if err := e.Where(conds).
 		Asc("comment.created_unix").
@@ -118,12 +120,12 @@ func findCodeComments(ctx context.Context, opts FindCommentsOptions, issue *Issu
 }
 
 // FetchCodeCommentsByLine fetches the code comments for a given treePath and line number
-func FetchCodeCommentsByLine(ctx context.Context, issue *Issue, currentUser *user_model.User, treePath string, line int64) ([]*Comment, error) {
+func FetchCodeCommentsByLine(ctx context.Context, issue *Issue, currentUser *user_model.User, treePath string, line int64, showOutdatedComments bool) ([]*Comment, error) {
 	opts := FindCommentsOptions{
 		Type:     CommentTypeCode,
 		IssueID:  issue.ID,
 		TreePath: treePath,
 		Line:     line,
 	}
-	return findCodeComments(ctx, opts, issue, currentUser, nil)
+	return findCodeComments(ctx, opts, issue, currentUser, nil, showOutdatedComments)
 }

models/issues/comment_test.go

@@ -50,7 +50,7 @@ func TestFetchCodeComments(t *testing.T) {
 
 	issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2})
 	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
-	res, err := issues_model.FetchCodeComments(db.DefaultContext, issue, user)
+	res, err := issues_model.FetchCodeComments(db.DefaultContext, issue, user, false)
 	assert.NoError(t, err)
 	assert.Contains(t, res, "README.md")
 	assert.Contains(t, res["README.md"], int64(4))
@@ -58,7 +58,7 @@ func TestFetchCodeComments(t *testing.T) {
 	assert.Equal(t, int64(4), res["README.md"][4][0].ID)
 
 	user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
-	res, err = issues_model.FetchCodeComments(db.DefaultContext, issue, user2)
+	res, err = issues_model.FetchCodeComments(db.DefaultContext, issue, user2, false)
 	assert.NoError(t, err)
 	assert.Len(t, res, 1)
 }

models/issues/issue_list.go

@@ -229,39 +229,41 @@ func (issues IssueList) loadMilestones(ctx context.Context) error {
 	return nil
 }
 
-func (issues IssueList) getProjectIDs() []int64 {
-	ids := make(container.Set[int64], len(issues))
-	for _, issue := range issues {
-		ids.Add(issue.ProjectID())
-	}
-	return ids.Values()
-}
+func (issues IssueList) LoadProjects(ctx context.Context) error {
+	issueIDs := issues.getIssueIDs()
+	projectMaps := make(map[int64]*project_model.Project, len(issues))
+	left := len(issueIDs)
 
-func (issues IssueList) loadProjects(ctx context.Context) error {
-	projectIDs := issues.getProjectIDs()
-	if len(projectIDs) == 0 {
-		return nil
+	type projectWithIssueID struct {
+		*project_model.Project `xorm:"extends"`
+		IssueID                int64
 	}
 
-	projectMaps := make(map[int64]*project_model.Project, len(projectIDs))
-	left := len(projectIDs)
 	for left > 0 {
 		limit := db.DefaultMaxInSize
 		if left < limit {
 			limit = left
 		}
+
+		projects := make([]*projectWithIssueID, 0, limit)
 		err := db.GetEngine(ctx).
-			In("id", projectIDs[:limit]).
-			Find(&projectMaps)
+			Table("project").
+			Select("project.*, project_issue.issue_id").
+			Join("INNER", "project_issue", "project.id = project_issue.project_id").
+			In("project_issue.issue_id", issueIDs[:limit]).
+			Find(&projects)
 		if err != nil {
 			return err
 		}
+		for _, project := range projects {
+			projectMaps[project.IssueID] = project.Project
+		}
 		left -= limit
-		projectIDs = projectIDs[limit:]
+		issueIDs = issueIDs[limit:]
 	}
 
 	for _, issue := range issues {
-		issue.Project = projectMaps[issue.ProjectID()]
+		issue.Project = projectMaps[issue.ID]
 	}
 	return nil
 }
@@ -541,7 +543,7 @@ func (issues IssueList) loadAttributes(ctx context.Context) error {
 		return fmt.Errorf("issue.loadAttributes: loadMilestones: %w", err)
 	}
 
-	if err := issues.loadProjects(ctx); err != nil {
+	if err := issues.LoadProjects(ctx); err != nil {
 		return fmt.Errorf("issue.loadAttributes: loadProjects: %w", err)
 	}
 

models/issues/issue_list_test.go

@@ -66,8 +66,12 @@ func TestIssueList_LoadAttributes(t *testing.T) {
 		}
 		if issue.ID == int64(1) {
 			assert.Equal(t, int64(400), issue.TotalTrackedTime)
+			assert.NotNil(t, issue.Project)
 		} else if issue.ID == int64(2) {
 			assert.Equal(t, int64(3682), issue.TotalTrackedTime)
+			assert.Nil(t, issue.Project)
+		} else {
+			assert.Nil(t, issue.Project)
 		}
 	}
 }

models/issues/issue_project.go

@@ -27,11 +27,6 @@ func (issue *Issue) LoadProject(ctx context.Context) (err error) {
 	return err
 }
 
-// ProjectID return project id if issue was assigned to one
-func (issue *Issue) ProjectID() int64 {
-	return issue.projectID(db.DefaultContext)
-}
-
 func (issue *Issue) projectID(ctx context.Context) int64 {
 	var ip project_model.ProjectIssue
 	has, err := db.GetEngine(ctx).Where("issue_id=?", issue.ID).Get(&ip)

models/issues/review.go

@@ -141,7 +141,7 @@ func (r *Review) LoadCodeComments(ctx context.Context) (err error) {
 	if err = r.loadIssue(ctx); err != nil {
 		return
 	}
-	r.CodeComments, err = fetchCodeCommentsByReview(ctx, r.Issue, nil, r)
+	r.CodeComments, err = fetchCodeCommentsByReview(ctx, r.Issue, nil, r, false)
 	return err
 }
 

models/issues/tracked_time.go

@@ -6,6 +6,7 @@ package issues
 import (
 	"context"
 	"errors"
+	"fmt"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
@@ -176,7 +177,9 @@ func AddTime(user *user_model.User, issue *Issue, amount int64, created time.Tim
 		Issue: issue,
 		Repo:  issue.Repo,
 		Doer:  user,
-		Content: util.SecToTime(amount),
+		// Content before v1.21 did store the formated string instead of seconds,
+		// so use "|" as delimeter to mark the new format
+		Content: fmt.Sprintf("|%d", amount),
 		Type:    CommentTypeAddTimeManual,
 		TimeID:  t.ID,
 	}); err != nil {
@@ -254,7 +257,9 @@ func DeleteIssueUserTimes(issue *Issue, user *user_model.User) error {
 		Issue: issue,
 		Repo:  issue.Repo,
 		Doer:  user,
-		Content: "- " + util.SecToTime(removedTime),
+		// Content before v1.21 did store the formated string instead of seconds,
+		// so use "|" as delimeter to mark the new format
+		Content: fmt.Sprintf("|%d", removedTime),
 		Type:    CommentTypeDeleteTimeManual,
 	}); err != nil {
 		return err
@@ -283,7 +288,9 @@ func DeleteTime(t *TrackedTime) error {
 		Issue: t.Issue,
 		Repo:  t.Issue.Repo,
 		Doer:  t.User,
-		Content: "- " + util.SecToTime(t.Time),
+		// Content before v1.21 did store the formated string instead of seconds,
+		// so use "|" as delimeter to mark the new format
+		Content: fmt.Sprintf("|%d", t.Time),
 		Type:    CommentTypeDeleteTimeManual,
 	}); err != nil {
 		return err

models/issues/tracked_time_test.go

@@ -35,7 +35,7 @@ func TestAddTime(t *testing.T) {
 	assert.Equal(t, int64(3661), tt.Time)
 
 	comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{Type: issues_model.CommentTypeAddTimeManual, PosterID: 3, IssueID: 1})
-	assert.Equal(t, "1 hour 1 minute", comment.Content)
+	assert.Equal(t, "|3661", comment.Content)
 }
 
 func TestGetTrackedTimes(t *testing.T) {

models/migrations/base/tests.go

@@ -147,9 +147,9 @@ func MainTest(m *testing.M) {
 		os.Exit(1)
 	}
 
+	setting.CustomPath = filepath.Join(setting.AppWorkPath, "custom")
 	setting.AppDataPath = tmpDataPath
 
-	setting.SetCustomPathAndConf("", "", "")
 	unittest.InitSettings()
 	if err = git.InitFull(context.Background()); err != nil {
 		fmt.Printf("Unable to InitFull: %v\n", err)

models/migrations/migrations.go

@@ -503,6 +503,10 @@ var migrations = []Migration{
 
 	// v260 -> v261
 	NewMigration("Drop custom_labels column of action_runner table", v1_21.DropCustomLabelsColumnOfActionRunner),
+	// v261 -> v262
+	NewMigration("Add variable table", v1_21.CreateVariableTable),
+	// v262 -> v263
+	NewMigration("Add TriggerEvent to action_run table", v1_21.AddTriggerEventToActionRun),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_21/v261.go

@@ -0,0 +1,24 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_21 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreateVariableTable(x *xorm.Engine) error {
+	type ActionVariable struct {
+		ID          int64              `xorm:"pk autoincr"`
+		OwnerID     int64              `xorm:"UNIQUE(owner_repo_name)"`
+		RepoID      int64              `xorm:"INDEX UNIQUE(owner_repo_name)"`
+		Name        string             `xorm:"UNIQUE(owner_repo_name) NOT NULL"`
+		Data        string             `xorm:"LONGTEXT NOT NULL"`
+		CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"`
+		UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
+	}
+
+	return x.Sync(new(ActionVariable))
+}

models/migrations/v1_21/v262.go

@@ -0,0 +1,16 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_21 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddTriggerEventToActionRun(x *xorm.Engine) error {
+	type ActionRun struct {
+		TriggerEvent string
+	}
+
+	return x.Sync(new(ActionRun))
+}

models/secret/secret.go

@@ -5,38 +5,17 @@ package secret
 
 import (
 	"context"
-	"fmt"
-	"regexp"
+	"errors"
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
 	secret_module "code.gitea.io/gitea/modules/secret"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
-	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/builder"
 )
 
-type ErrSecretInvalidValue struct {
-	Name *string
-	Data *string
-}
-
-func (err ErrSecretInvalidValue) Error() string {
-	if err.Name != nil {
-		return fmt.Sprintf("secret name %q is invalid", *err.Name)
-	}
-	if err.Data != nil {
-		return fmt.Sprintf("secret data %q is invalid", *err.Data)
-	}
-	return util.ErrInvalidArgument.Error()
-}
-
-func (err ErrSecretInvalidValue) Unwrap() error {
-	return util.ErrInvalidArgument
-}
-
 // Secret represents a secret
 type Secret struct {
 	ID          int64
@@ -74,24 +53,11 @@ func init() {
 	db.RegisterModel(new(Secret))
 }
 
-var (
-	secretNameReg            = regexp.MustCompile("^[A-Z_][A-Z0-9_]*$")
-	forbiddenSecretPrefixReg = regexp.MustCompile("^GIT(EA|HUB)_")
-)
-
-// Validate validates the required fields and formats.
 func (s *Secret) Validate() error {
-	switch {
-	case len(s.Name) == 0 || len(s.Name) > 50:
-		return ErrSecretInvalidValue{Name: &s.Name}
-	case len(s.Data) == 0:
-		return ErrSecretInvalidValue{Data: &s.Data}
-	case !secretNameReg.MatchString(s.Name) ||
-		forbiddenSecretPrefixReg.MatchString(s.Name):
-		return ErrSecretInvalidValue{Name: &s.Name}
-	default:
-		return nil
+	if s.OwnerID == 0 && s.RepoID == 0 {
+		return errors.New("the secret is not bound to any scope")
 	}
+	return nil
 }
 
 type FindSecretsOptions struct {

models/unittest/testdb.go

@@ -42,12 +42,14 @@ func fatalTestError(fmtStr string, args ...interface{}) {
 	os.Exit(1)
 }
 
-// InitSettings initializes config provider and load common setttings for tests
+// InitSettings initializes config provider and load common settings for tests
 func InitSettings(extraConfigs ...string) {
-	setting.Init(&setting.Options{
-		AllowEmpty:  true,
-		ExtraConfig: strings.Join(extraConfigs, "\n"),
-	})
+	if setting.CustomConf == "" {
+		setting.CustomConf = filepath.Join(setting.CustomPath, "conf/app-unittest-tmp.ini")
+		_ = os.Remove(setting.CustomConf)
+	}
+	setting.InitCfgProvider(setting.CustomConf, strings.Join(extraConfigs, "\n"))
+	setting.LoadCommonSettings()
 
 	if err := setting.PrepareAppDataPath(); err != nil {
 		log.Fatalf("Can not prepare APP_DATA_PATH: %v", err)
@@ -69,7 +71,7 @@ type TestOptions struct {
 // MainTest a reusable TestMain(..) function for unit tests that need to use a
 // test database. Creates the test database, and sets necessary settings.
 func MainTest(m *testing.M, testOpts *TestOptions) {
-	setting.SetCustomPathAndConf("", "", "")
+	setting.CustomPath = filepath.Join(testOpts.GiteaRootPath, "custom")
 	InitSettings()
 
 	var err error

models/user/setting_keys.go

@@ -8,6 +8,8 @@ const (
 	SettingsKeyHiddenCommentTypes = "issue.hidden_comment_types"
 	// SettingsKeyDiffWhitespaceBehavior is the setting key for whitespace behavior of diff
 	SettingsKeyDiffWhitespaceBehavior = "diff.whitespace_behaviour"
+	// SettingsKeyShowOutdatedComments is the setting key wether or not to show outdated comments in PRs
+	SettingsKeyShowOutdatedComments = "comment_code.show_outdated"
 	// UserActivityPubPrivPem is user's private key
 	UserActivityPubPrivPem = "activitypub.priv_pem"
 	// UserActivityPubPubPem is user's public key

modules/actions/github.go

@@ -8,33 +8,33 @@ import (
 )
 
 const (
-	githubEventPullRequest              = "pull_request"
-	githubEventPullRequestTarget        = "pull_request_target"
-	githubEventPullRequestReviewComment = "pull_request_review_comment"
-	githubEventPullRequestReview        = "pull_request_review"
-	githubEventRegistryPackage          = "registry_package"
-	githubEventCreate                   = "create"
-	githubEventDelete                   = "delete"
-	githubEventFork                     = "fork"
-	githubEventPush                     = "push"
-	githubEventIssues                   = "issues"
-	githubEventIssueComment             = "issue_comment"
-	githubEventRelease                  = "release"
-	githubEventPullRequestComment       = "pull_request_comment"
-	githubEventGollum                   = "gollum"
+	GithubEventPullRequest              = "pull_request"
+	GithubEventPullRequestTarget        = "pull_request_target"
+	GithubEventPullRequestReviewComment = "pull_request_review_comment"
+	GithubEventPullRequestReview        = "pull_request_review"
+	GithubEventRegistryPackage          = "registry_package"
+	GithubEventCreate                   = "create"
+	GithubEventDelete                   = "delete"
+	GithubEventFork                     = "fork"
+	GithubEventPush                     = "push"
+	GithubEventIssues                   = "issues"
+	GithubEventIssueComment             = "issue_comment"
+	GithubEventRelease                  = "release"
+	GithubEventPullRequestComment       = "pull_request_comment"
+	GithubEventGollum                   = "gollum"
 )
 
 // canGithubEventMatch check if the input Github event can match any Gitea event.
 func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEventType) bool {
 	switch eventName {
-	case githubEventRegistryPackage:
+	case GithubEventRegistryPackage:
 		return triggedEvent == webhook_module.HookEventPackage
 
 	// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#gollum
-	case githubEventGollum:
+	case GithubEventGollum:
 		return triggedEvent == webhook_module.HookEventWiki
 
-	case githubEventIssues:
+	case GithubEventIssues:
 		switch triggedEvent {
 		case webhook_module.HookEventIssues,
 			webhook_module.HookEventIssueAssign,
@@ -46,7 +46,7 @@ func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEvent
 			return false
 		}
 
-	case githubEventPullRequest, githubEventPullRequestTarget:
+	case GithubEventPullRequest, GithubEventPullRequestTarget:
 		switch triggedEvent {
 		case webhook_module.HookEventPullRequest,
 			webhook_module.HookEventPullRequestSync,
@@ -58,7 +58,7 @@ func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEvent
 			return false
 		}
 
-	case githubEventPullRequestReview:
+	case GithubEventPullRequestReview:
 		switch triggedEvent {
 		case webhook_module.HookEventPullRequestReviewApproved,
 			webhook_module.HookEventPullRequestReviewComment,

modules/actions/github_test.go

@@ -21,85 +21,85 @@ func TestCanGithubEventMatch(t *testing.T) {
 		// registry_package event
 		{
 			"registry_package matches",
-			githubEventRegistryPackage,
+			GithubEventRegistryPackage,
 			webhook_module.HookEventPackage,
 			true,
 		},
 		{
 			"registry_package cannot match",
-			githubEventRegistryPackage,
+			GithubEventRegistryPackage,
 			webhook_module.HookEventPush,
 			false,
 		},
 		// issues event
 		{
 			"issue matches",
-			githubEventIssues,
+			GithubEventIssues,
 			webhook_module.HookEventIssueLabel,
 			true,
 		},
 		{
 			"issue cannot match",
-			githubEventIssues,
+			GithubEventIssues,
 			webhook_module.HookEventIssueComment,
 			false,
 		},
 		// issue_comment event
 		{
 			"issue_comment matches",
-			githubEventIssueComment,
+			GithubEventIssueComment,
 			webhook_module.HookEventIssueComment,
 			true,
 		},
 		{
 			"issue_comment cannot match",
-			githubEventIssueComment,
+			GithubEventIssueComment,
 			webhook_module.HookEventIssues,
 			false,
 		},
 		// pull_request event
 		{
 			"pull_request matches",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequestSync,
 			true,
 		},
 		{
 			"pull_request cannot match",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequestComment,
 			false,
 		},
 		// pull_request_target event
 		{
 			"pull_request_target matches",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequest,
 			true,
 		},
 		{
 			"pull_request_target cannot match",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequestComment,
 			false,
 		},
 		// pull_request_review event
 		{
 			"pull_request_review matches",
-			githubEventPullRequestReview,
+			GithubEventPullRequestReview,
 			webhook_module.HookEventPullRequestReviewComment,
 			true,
 		},
 		{
 			"pull_request_review cannot match",
-			githubEventPullRequestReview,
+			GithubEventPullRequestReview,
 			webhook_module.HookEventPullRequestComment,
 			false,
 		},
 		// other events
 		{
 			"create event",
-			githubEventCreate,
+			GithubEventCreate,
 			webhook_module.HookEventCreate,
 			true,
 		},

modules/actions/workflows.go

@@ -20,6 +20,14 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+type DetectedWorkflow struct {
+	EntryName    string
+	TriggerEvent string
+	Commit       *git.Commit
+	Ref          string
+	Content      []byte
+}
+
 func init() {
 	model.OnDecodeNodeError = func(node yaml.Node, out interface{}, err error) {
 		// Log the error instead of panic or fatal.
@@ -89,13 +97,13 @@ func GetEventsFromContent(content []byte) ([]*jobparser.Event, error) {
 	return events, nil
 }
 
-func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) (map[string][]byte, error) {
+func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) ([]*DetectedWorkflow, error) {
 	entries, err := ListWorkflows(commit)
 	if err != nil {
 		return nil, err
 	}
 
-	workflows := make(map[string][]byte, len(entries))
+	workflows := make([]*DetectedWorkflow, 0, len(entries))
 	for _, entry := range entries {
 		content, err := GetContentFromEntry(entry)
 		if err != nil {
@@ -109,7 +117,13 @@ func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventTy
 		for _, evt := range events {
 			log.Trace("detect workflow %q for event %#v matching %q", entry.Name(), evt, triggedEvent)
 			if detectMatched(commit, triggedEvent, payload, evt) {
-				workflows[entry.Name()] = content
+				dwf := &DetectedWorkflow{
+					EntryName:    entry.Name(),
+					TriggerEvent: evt.Name,
+					Commit:       commit,
+					Content:      content,
+				}
+				workflows = append(workflows, dwf)
 			}
 		}
 	}

modules/actions/workflows_test.go

@@ -23,77 +23,77 @@ func TestDetectMatched(t *testing.T) {
 		expected     bool
 	}{
 		{
-			desc:         "HookEventCreate(create) matches githubEventCreate(create)",
+			desc:         "HookEventCreate(create) matches GithubEventCreate(create)",
 			triggedEvent: webhook_module.HookEventCreate,
 			payload:      nil,
 			yamlOn:       "on: create",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventIssues(issues) `opened` action matches githubEventIssues(issues)",
+			desc:         "HookEventIssues(issues) `opened` action matches GithubEventIssues(issues)",
 			triggedEvent: webhook_module.HookEventIssues,
 			payload:      &api.IssuePayload{Action: api.HookIssueOpened},
 			yamlOn:       "on: issues",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventIssues(issues) `milestoned` action matches githubEventIssues(issues)",
+			desc:         "HookEventIssues(issues) `milestoned` action matches GithubEventIssues(issues)",
 			triggedEvent: webhook_module.HookEventIssues,
 			payload:      &api.IssuePayload{Action: api.HookIssueMilestoned},
 			yamlOn:       "on: issues",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequestSync(pull_request_sync) matches githubEventPullRequest(pull_request)",
+			desc:         "HookEventPullRequestSync(pull_request_sync) matches GithubEventPullRequest(pull_request)",
 			triggedEvent: webhook_module.HookEventPullRequestSync,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueSynchronized},
 			yamlOn:       "on: pull_request",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequest(pull_request) `label_updated` action doesn't match githubEventPullRequest(pull_request) with no activity type",
+			desc:         "HookEventPullRequest(pull_request) `label_updated` action doesn't match GithubEventPullRequest(pull_request) with no activity type",
 			triggedEvent: webhook_module.HookEventPullRequest,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
 			yamlOn:       "on: pull_request",
 			expected:     false,
 		},
 		{
-			desc:         "HookEventPullRequest(pull_request) `label_updated` action matches githubEventPullRequest(pull_request) with `label` activity type",
+			desc:         "HookEventPullRequest(pull_request) `label_updated` action matches GithubEventPullRequest(pull_request) with `label` activity type",
 			triggedEvent: webhook_module.HookEventPullRequest,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
 			yamlOn:       "on:\n  pull_request:\n    types: [labeled]",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequestReviewComment(pull_request_review_comment) matches githubEventPullRequestReviewComment(pull_request_review_comment)",
+			desc:         "HookEventPullRequestReviewComment(pull_request_review_comment) matches GithubEventPullRequestReviewComment(pull_request_review_comment)",
 			triggedEvent: webhook_module.HookEventPullRequestReviewComment,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueReviewed},
 			yamlOn:       "on:\n  pull_request_review_comment:\n    types: [created]",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequestReviewRejected(pull_request_review_rejected) doesn't match githubEventPullRequestReview(pull_request_review) with `dismissed` activity type (we don't support `dismissed` at present)",
+			desc:         "HookEventPullRequestReviewRejected(pull_request_review_rejected) doesn't match GithubEventPullRequestReview(pull_request_review) with `dismissed` activity type (we don't support `dismissed` at present)",
 			triggedEvent: webhook_module.HookEventPullRequestReviewRejected,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueReviewed},
 			yamlOn:       "on:\n  pull_request_review:\n    types: [dismissed]",
 			expected:     false,
 		},
 		{
-			desc:         "HookEventRelease(release) `published` action matches githubEventRelease(release) with `published` activity type",
+			desc:         "HookEventRelease(release) `published` action matches GithubEventRelease(release) with `published` activity type",
 			triggedEvent: webhook_module.HookEventRelease,
 			payload:      &api.ReleasePayload{Action: api.HookReleasePublished},
 			yamlOn:       "on:\n  release:\n    types: [published]",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPackage(package) `created` action doesn't match githubEventRegistryPackage(registry_package) with `updated` activity type",
+			desc:         "HookEventPackage(package) `created` action doesn't match GithubEventRegistryPackage(registry_package) with `updated` activity type",
 			triggedEvent: webhook_module.HookEventPackage,
 			payload:      &api.PackagePayload{Action: api.HookPackageCreated},
 			yamlOn:       "on:\n  registry_package:\n    types: [updated]",
 			expected:     false,
 		},
 		{
-			desc:         "HookEventWiki(wiki) matches githubEventGollum(gollum)",
+			desc:         "HookEventWiki(wiki) matches GithubEventGollum(gollum)",
 			triggedEvent: webhook_module.HookEventWiki,
 			payload:      nil,
 			yamlOn:       "on: gollum",

modules/base/tool.go

@@ -210,7 +210,7 @@ func EntryIcon(entry *git.TreeEntry) string {
 			return "file-symlink-file"
 		}
 		if te.IsDir() {
-			return "file-submodule"
+			return "file-directory-symlink"
 		}
 		return "file-symlink-file"
 	case entry.IsDir():

modules/context/repo.go

@@ -593,7 +593,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 
 	ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
 	if setting.Indexer.RepoIndexerEnabled {
-		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
+		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
 	}
 
 	if ctx.IsSigned {

modules/doctor/doctor.go

@@ -28,7 +28,7 @@ type Check struct {
 }
 
 func initDBSkipLogger(ctx context.Context) error {
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 	if err := db.InitEngine(ctx); err != nil {
 		return fmt.Errorf("db.InitEngine: %w", err)

modules/doctor/paths.go

@@ -66,7 +66,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo
 		return err
 	}
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	configurationFiles := []configurationFile{
 		{"Configuration File Path", setting.CustomConf, false, true, false},

modules/indexer/code/bleve/bleve.go

@@ -1,14 +1,13 @@
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-package code
+package bleve
 
 import (
 	"bufio"
 	"context"
 	"fmt"
 	"io"
-	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -17,12 +16,13 @@ import (
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
-	gitea_bleve "code.gitea.io/gitea/modules/indexer/bleve"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/typesniffer"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/blevesearch/bleve/v2"
 	analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -31,10 +31,8 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
-	"github.com/blevesearch/bleve/v2/index/upsidedown"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search/query"
-	"github.com/ethantkoenig/rupture"
 	"github.com/go-enry/go-enry/v2"
 )
 
@@ -59,38 +57,6 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
 	})
 }
 
-// openBleveIndexer open the index at the specified path, checking for metadata
-// updates and bleve version updates.  If index needs to be created (or
-// re-created), returns (nil, nil)
-func openBleveIndexer(path string, latestVersion int) (bleve.Index, error) {
-	_, err := os.Stat(path)
-	if err != nil && os.IsNotExist(err) {
-		return nil, nil
-	} else if err != nil {
-		return nil, err
-	}
-
-	metadata, err := rupture.ReadIndexMetadata(path)
-	if err != nil {
-		return nil, err
-	}
-	if metadata.Version < latestVersion {
-		// the indexer is using a previous version, so we should delete it and
-		// re-populate
-		return nil, util.RemoveAll(path)
-	}
-
-	index, err := bleve.Open(path)
-	if err != nil && err == upsidedown.IncompatibleVersion {
-		// the indexer was built with a previous version of bleve, so we should
-		// delete it and re-populate
-		return nil, util.RemoveAll(path)
-	} else if err != nil {
-		return nil, err
-	}
-	return index, nil
-}
-
 // RepoIndexerData data stored in the repo indexer
 type RepoIndexerData struct {
 	RepoID    int64
@@ -111,8 +77,8 @@ const (
 	repoIndexerLatestVersion = 6
 )
 
-// createBleveIndexer create a bleve repo indexer if one does not already exist
-func createBleveIndexer(path string, latestVersion int) (bleve.Index, error) {
+// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
+func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 	docMapping := bleve.NewDocumentMapping()
 	numericFieldMapping := bleve.NewNumericFieldMapping()
 	numericFieldMapping.IncludeInAll = false
@@ -147,42 +113,28 @@ func createBleveIndexer(path string, latestVersion int) (bleve.Index, error) {
 	mapping.AddDocumentMapping(repoIndexerDocType, docMapping)
 	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
 
-	indexer, err := bleve.New(path, mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
-		Version: latestVersion,
-	}); err != nil {
-		return nil, err
-	}
-	return indexer, nil
+	return mapping, nil
 }
 
-var _ Indexer = &BleveIndexer{}
+var _ internal.Indexer = &Indexer{}
 
-// BleveIndexer represents a bleve indexer implementation
-type BleveIndexer struct {
-	indexDir string
-	indexer  bleve.Index
+// Indexer represents a bleve indexer implementation
+type Indexer struct {
+	inner                    *inner_bleve.Indexer
+	indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much
 }
 
-// NewBleveIndexer creates a new bleve local indexer
-func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) {
-	indexer := &BleveIndexer{
-		indexDir: indexDir,
+// NewIndexer creates a new bleve local indexer
+func NewIndexer(indexDir string) *Indexer {
+	inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping)
+	return &Indexer{
+		Indexer: inner,
+		inner:   inner,
 	}
-	created, err := indexer.init()
-	if err != nil {
-		indexer.Close()
-		return nil, false, err
-	}
-	return indexer, created, err
 }
 
-func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string,
-	update fileUpdate, repo *repo_model.Repository, batch *gitea_bleve.FlushingBatch,
+func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string,
+	update internal.FileUpdate, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch,
 ) error {
 	// Ignore vendored files in code search
 	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
@@ -227,7 +179,7 @@ func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteClose
 	if _, err = batchReader.Discard(1); err != nil {
 		return err
 	}
-	id := filenameIndexerID(repo.ID, update.Filename)
+	id := internal.FilenameIndexerID(repo.ID, update.Filename)
 	return batch.Index(id, &RepoIndexerData{
 		RepoID:    repo.ID,
 		CommitID:  commitSha,
@@ -237,50 +189,14 @@ func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteClose
 	})
 }
 
-func (b *BleveIndexer) addDelete(filename string, repo *repo_model.Repository, batch *gitea_bleve.FlushingBatch) error {
-	id := filenameIndexerID(repo.ID, filename)
+func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch) error {
+	id := internal.FilenameIndexerID(repo.ID, filename)
 	return batch.Delete(id)
 }
 
-// init init the indexer
-func (b *BleveIndexer) init() (bool, error) {
-	var err error
-	b.indexer, err = openBleveIndexer(b.indexDir, repoIndexerLatestVersion)
-	if err != nil {
-		return false, err
-	}
-	if b.indexer != nil {
-		return false, nil
-	}
-
-	b.indexer, err = createBleveIndexer(b.indexDir, repoIndexerLatestVersion)
-	if err != nil {
-		return false, err
-	}
-
-	return true, nil
-}
-
-// Close close the indexer
-func (b *BleveIndexer) Close() {
-	log.Debug("Closing repo indexer")
-	if b.indexer != nil {
-		err := b.indexer.Close()
-		if err != nil {
-			log.Error("Error whilst closing the repository indexer: %v", err)
-		}
-	}
-	log.Info("PID: %d Repository Indexer closed", os.Getpid())
-}
-
-// Ping does nothing
-func (b *BleveIndexer) Ping() bool {
-	return true
-}
-
 // Index indexes the data
-func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
-	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
+func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
 	if len(changes.Updates) > 0 {
 
 		// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
@@ -308,14 +224,14 @@ func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, s
 }
 
 // Delete deletes indexes by ids
-func (b *BleveIndexer) Delete(repoID int64) error {
+func (b *Indexer) Delete(_ context.Context, repoID int64) error {
 	query := numericEqualityQuery(repoID, "RepoID")
 	searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
-	result, err := b.indexer.Search(searchRequest)
+	result, err := b.inner.Indexer.Search(searchRequest)
 	if err != nil {
 		return err
 	}
-	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
 	for _, hit := range result.Hits {
 		if err = batch.Delete(hit.ID); err != nil {
 			return err
@@ -326,7 +242,7 @@ func (b *BleveIndexer) Delete(repoID int64) error {
 
 // Search searches for files in the specified repo.
 // Returns the matching file-paths
-func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	var (
 		indexerQuery query.Query
 		keywordQuery query.Query
@@ -379,14 +295,14 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 	}
 
-	result, err := b.indexer.SearchInContext(ctx, searchRequest)
+	result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
 	if err != nil {
 		return 0, nil, nil, err
 	}
 
 	total := int64(result.Total)
 
-	searchResults := make([]*SearchResult, len(result.Hits))
+	searchResults := make([]*internal.SearchResult, len(result.Hits))
 	for i, hit := range result.Hits {
 		startIndex, endIndex := -1, -1
 		for _, locations := range hit.Locations["Content"] {
@@ -405,11 +321,11 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		if t, err := time.Parse(time.RFC3339, hit.Fields["UpdatedAt"].(string)); err == nil {
 			updatedUnix = timeutil.TimeStamp(t.Unix())
 		}
-		searchResults[i] = &SearchResult{
+		searchResults[i] = &internal.SearchResult{
 			RepoID:      int64(hit.Fields["RepoID"].(float64)),
 			StartIndex:  startIndex,
 			EndIndex:    endIndex,
-			Filename:    filenameOfIndexerID(hit.ID),
+			Filename:    internal.FilenameOfIndexerID(hit.ID),
 			Content:     hit.Fields["Content"].(string),
 			CommitID:    hit.Fields["CommitID"].(string),
 			UpdatedUnix: updatedUnix,
@@ -418,7 +334,7 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		}
 	}
 
-	searchResultLanguages := make([]*SearchResultLanguages, 0, 10)
+	searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
 	if len(language) > 0 {
 		// Use separate query to go get all language counts
 		facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
@@ -426,7 +342,7 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		facetRequest.IncludeLocations = true
 		facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 
-		if result, err = b.indexer.Search(facetRequest); err != nil {
+		if result, err = b.inner.Indexer.Search(facetRequest); err != nil {
 			return 0, nil, nil, err
 		}
 
@@ -436,7 +352,7 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		if len(term.Term) == 0 {
 			continue
 		}
-		searchResultLanguages = append(searchResultLanguages, &SearchResultLanguages{
+		searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
 			Language: term.Term,
 			Color:    enry.GetColor(term.Term),
 			Count:    term.Count,

modules/indexer/code/bleve_test.go

@@ -1,30 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package code
-
-import (
-	"testing"
-
-	"code.gitea.io/gitea/models/unittest"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBleveIndexAndSearch(t *testing.T) {
-	unittest.PrepareTestEnv(t)
-
-	dir := t.TempDir()
-
-	idx, _, err := NewBleveIndexer(dir)
-	if err != nil {
-		assert.Fail(t, "Unable to create bleve indexer Error: %v", err)
-		if idx != nil {
-			idx.Close()
-		}
-		return
-	}
-	defer idx.Close()
-
-	testIndexer("beleve", t, idx)
-}

modules/indexer/code/elastic_search_test.go

@@ -1,41 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package code
-
-import (
-	"os"
-	"testing"
-
-	"code.gitea.io/gitea/models/unittest"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestESIndexAndSearch(t *testing.T) {
-	unittest.PrepareTestEnv(t)
-
-	u := os.Getenv("TEST_INDEXER_CODE_ES_URL")
-	if u == "" {
-		t.SkipNow()
-		return
-	}
-
-	indexer, _, err := NewElasticSearchIndexer(u, "gitea_codes")
-	if err != nil {
-		assert.Fail(t, "Unable to create ES indexer Error: %v", err)
-		if indexer != nil {
-			indexer.Close()
-		}
-		return
-	}
-	defer indexer.Close()
-
-	testIndexer("elastic_search", t, indexer)
-}
-
-func TestIndexPos(t *testing.T) {
-	startIdx, endIdx := indexPos("test index start and end", "start", "end")
-	assert.EqualValues(t, 11, startIdx)
-	assert.EqualValues(t, 24, endIdx)
-}

modules/indexer/code/elasticsearch/elasticsearch.go

@@ -1,25 +1,23 @@
 // Copyright 2020 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-package code
+package elasticsearch
 
 import (
 	"bufio"
 	"context"
-	"errors"
 	"fmt"
 	"io"
-	"net"
 	"strconv"
 	"strings"
-	"sync"
-	"time"
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
-	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -38,63 +36,22 @@ const (
 	esMultiMatchTypePhrasePrefix = "phrase_prefix"
 )
 
-var _ Indexer = &ElasticSearchIndexer{}
+var _ internal.Indexer = &Indexer{}
 
-// ElasticSearchIndexer implements Indexer interface
-type ElasticSearchIndexer struct {
-	client           *elastic.Client
-	indexerAliasName string
-	available        bool
-	stopTimer        chan struct{}
-	lock             sync.RWMutex
+// Indexer implements Indexer interface
+type Indexer struct {
+	inner                    *inner_elasticsearch.Indexer
+	indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
 }
 
-// NewElasticSearchIndexer creates a new elasticsearch indexer
-func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bool, error) {
-	opts := []elastic.ClientOptionFunc{
-		elastic.SetURL(url),
-		elastic.SetSniff(false),
-		elastic.SetHealthcheckInterval(10 * time.Second),
-		elastic.SetGzip(false),
+// NewIndexer creates a new elasticsearch indexer
+func NewIndexer(url, indexerName string) *Indexer {
+	inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping)
+	indexer := &Indexer{
+		inner:   inner,
+		Indexer: inner,
 	}
-
-	logger := log.GetLogger(log.DEFAULT)
-
-	opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
-	opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
-	opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
-
-	client, err := elastic.NewClient(opts...)
-	if err != nil {
-		return nil, false, err
-	}
-
-	indexer := &ElasticSearchIndexer{
-		client:           client,
-		indexerAliasName: indexerName,
-		available:        true,
-		stopTimer:        make(chan struct{}),
-	}
-
-	ticker := time.NewTicker(10 * time.Second)
-	go func() {
-		for {
-			select {
-			case <-ticker.C:
-				indexer.checkAvailability()
-			case <-indexer.stopTimer:
-				ticker.Stop()
-				return
-			}
-		}
-	}()
-
-	exists, err := indexer.init()
-	if err != nil {
-		indexer.Close()
-		return nil, false, err
-	}
-	return indexer, !exists, err
+	return indexer
 }
 
 const (
@@ -127,72 +84,7 @@ const (
 	}`
 )
 
-func (b *ElasticSearchIndexer) realIndexerName() string {
-	return fmt.Sprintf("%s.v%d", b.indexerAliasName, esRepoIndexerLatestVersion)
-}
-
-// Init will initialize the indexer
-func (b *ElasticSearchIndexer) init() (bool, error) {
-	ctx := graceful.GetManager().HammerContext()
-	exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx)
-	if err != nil {
-		return false, b.checkError(err)
-	}
-	if !exists {
-		mapping := defaultMapping
-
-		createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !createIndex.Acknowledged {
-			return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping)
-		}
-	}
-
-	// check version
-	r, err := b.client.Aliases().Do(ctx)
-	if err != nil {
-		return false, b.checkError(err)
-	}
-
-	realIndexerNames := r.IndicesByAlias(b.indexerAliasName)
-	if len(realIndexerNames) < 1 {
-		res, err := b.client.Alias().
-			Add(b.realIndexerName(), b.indexerAliasName).
-			Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !res.Acknowledged {
-			return false, fmt.Errorf("create alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
-		}
-	} else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() {
-		log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.",
-			realIndexerNames[0], b.realIndexerName())
-		res, err := b.client.Alias().
-			Remove(realIndexerNames[0], b.indexerAliasName).
-			Add(b.realIndexerName(), b.indexerAliasName).
-			Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !res.Acknowledged {
-			return false, fmt.Errorf("change alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
-		}
-	}
-
-	return exists, nil
-}
-
-// Ping checks if elastic is available
-func (b *ElasticSearchIndexer) Ping() bool {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	return b.available
-}
-
-func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
+func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
 	// Ignore vendored files in code search
 	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
 		return nil, nil
@@ -235,11 +127,11 @@ func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.Wr
 	if _, err = batchReader.Discard(1); err != nil {
 		return nil, err
 	}
-	id := filenameIndexerID(repo.ID, update.Filename)
+	id := internal.FilenameIndexerID(repo.ID, update.Filename)
 
 	return []elastic.BulkableRequest{
 		elastic.NewBulkIndexRequest().
-			Index(b.indexerAliasName).
+			Index(b.inner.VersionedIndexName()).
 			Id(id).
 			Doc(map[string]interface{}{
 				"repo_id":    repo.ID,
@@ -251,15 +143,15 @@ func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.Wr
 	}, nil
 }
 
-func (b *ElasticSearchIndexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest {
-	id := filenameIndexerID(repo.ID, filename)
+func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest {
+	id := internal.FilenameIndexerID(repo.ID, filename)
 	return elastic.NewBulkDeleteRequest().
-		Index(b.indexerAliasName).
+		Index(b.inner.VersionedIndexName()).
 		Id(id)
 }
 
 // Index will save the index data
-func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
+func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
 	reqs := make([]elastic.BulkableRequest, 0)
 	if len(changes.Updates) > 0 {
 		// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
@@ -288,21 +180,21 @@ func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repos
 	}
 
 	if len(reqs) > 0 {
-		_, err := b.client.Bulk().
-			Index(b.indexerAliasName).
+		_, err := b.inner.Client.Bulk().
+			Index(b.inner.VersionedIndexName()).
 			Add(reqs...).
 			Do(ctx)
-		return b.checkError(err)
+		return err
 	}
 	return nil
 }
 
 // Delete deletes indexes by ids
-func (b *ElasticSearchIndexer) Delete(repoID int64) error {
-	_, err := b.client.DeleteByQuery(b.indexerAliasName).
+func (b *Indexer) Delete(ctx context.Context, repoID int64) error {
+	_, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()).
 		Query(elastic.NewTermsQuery("repo_id", repoID)).
-		Do(graceful.GetManager().HammerContext())
-	return b.checkError(err)
+		Do(ctx)
+	return err
 }
 
 // indexPos find words positions for start and the following end on content. It will
@@ -321,8 +213,8 @@ func indexPos(content, start, end string) (int, int) {
 	return startIdx, startIdx + len(start) + endIdx + len(end)
 }
 
-func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
-	hits := make([]*SearchResult, 0, pageSize)
+func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+	hits := make([]*internal.SearchResult, 0, pageSize)
 	for _, hit := range searchResult.Hits.Hits {
 		// FIXME: There is no way to get the position the keyword on the content currently on the same request.
 		// So we get it from content, this may made the query slower. See
@@ -341,7 +233,7 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
 			panic(fmt.Sprintf("2===%#v", hit.Highlight))
 		}
 
-		repoID, fileName := parseIndexerID(hit.Id)
+		repoID, fileName := internal.ParseIndexerID(hit.Id)
 		res := make(map[string]interface{})
 		if err := json.Unmarshal(hit.Source, &res); err != nil {
 			return 0, nil, nil, err
@@ -349,7 +241,7 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
 
 		language := res["language"].(string)
 
-		hits = append(hits, &SearchResult{
+		hits = append(hits, &internal.SearchResult{
 			RepoID:      repoID,
 			Filename:    fileName,
 			CommitID:    res["commit_id"].(string),
@@ -365,14 +257,14 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
 	return searchResult.TotalHits(), hits, extractAggs(searchResult), nil
 }
 
-func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
-	var searchResultLanguages []*SearchResultLanguages
+func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLanguages {
+	var searchResultLanguages []*internal.SearchResultLanguages
 	agg, found := searchResult.Aggregations.Terms("language")
 	if found {
-		searchResultLanguages = make([]*SearchResultLanguages, 0, 10)
+		searchResultLanguages = make([]*internal.SearchResultLanguages, 0, 10)
 
 		for _, bucket := range agg.Buckets {
-			searchResultLanguages = append(searchResultLanguages, &SearchResultLanguages{
+			searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
 				Language: bucket.Key.(string),
 				Color:    enry.GetColor(bucket.Key.(string)),
 				Count:    int(bucket.DocCount),
@@ -383,7 +275,7 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
 }
 
 // Search searches for codes and language stats by given conditions.
-func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	searchType := esMultiMatchTypeBestFields
 	if isMatch {
 		searchType = esMultiMatchTypePhrasePrefix
@@ -412,8 +304,8 @@ func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, lang
 	}
 
 	if len(language) == 0 {
-		searchResult, err := b.client.Search().
-			Index(b.indexerAliasName).
+		searchResult, err := b.inner.Client.Search().
+			Index(b.inner.VersionedIndexName()).
 			Aggregation("language", aggregation).
 			Query(query).
 			Highlight(
@@ -426,26 +318,26 @@ func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, lang
 			From(start).Size(pageSize).
 			Do(ctx)
 		if err != nil {
-			return 0, nil, nil, b.checkError(err)
+			return 0, nil, nil, err
 		}
 
 		return convertResult(searchResult, kw, pageSize)
 	}
 
 	langQuery := elastic.NewMatchQuery("language", language)
-	countResult, err := b.client.Search().
-		Index(b.indexerAliasName).
+	countResult, err := b.inner.Client.Search().
+		Index(b.inner.VersionedIndexName()).
 		Aggregation("language", aggregation).
 		Query(query).
-		Size(0). // We only needs stats information
+		Size(0). // We only need stats information
 		Do(ctx)
 	if err != nil {
-		return 0, nil, nil, b.checkError(err)
+		return 0, nil, nil, err
 	}
 
 	query = query.Must(langQuery)
-	searchResult, err := b.client.Search().
-		Index(b.indexerAliasName).
+	searchResult, err := b.inner.Client.Search().
+		Index(b.inner.VersionedIndexName()).
 		Query(query).
 		Highlight(
 			elastic.NewHighlight().
@@ -457,56 +349,10 @@ func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, lang
 		From(start).Size(pageSize).
 		Do(ctx)
 	if err != nil {
-		return 0, nil, nil, b.checkError(err)
+		return 0, nil, nil, err
 	}
 
 	total, hits, _, err := convertResult(searchResult, kw, pageSize)
 
 	return total, hits, extractAggs(countResult), err
 }
-
-// Close implements indexer
-func (b *ElasticSearchIndexer) Close() {
-	select {
-	case <-b.stopTimer:
-	default:
-		close(b.stopTimer)
-	}
-}
-
-func (b *ElasticSearchIndexer) checkError(err error) error {
-	var opErr *net.OpError
-	if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
-		return err
-	}
-
-	b.setAvailability(false)
-
-	return err
-}
-
-func (b *ElasticSearchIndexer) checkAvailability() {
-	if b.Ping() {
-		return
-	}
-
-	// Request cluster state to check if elastic is available again
-	_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
-	if err != nil {
-		b.setAvailability(false)
-		return
-	}
-
-	b.setAvailability(true)
-}
-
-func (b *ElasticSearchIndexer) setAvailability(available bool) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.available == available {
-		return
-	}
-
-	b.available = available
-}

modules/indexer/code/elasticsearch/elasticsearch_test.go

@@ -0,0 +1,16 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIndexPos(t *testing.T) {
+	startIdx, endIdx := indexPos("test index start and end", "start", "end")
+	assert.EqualValues(t, 11, startIdx)
+	assert.EqualValues(t, 24, endIdx)
+}

modules/indexer/code/git.go

@@ -10,23 +10,11 @@ import (
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 )
 
-type fileUpdate struct {
-	Filename string
-	BlobSha  string
-	Size     int64
-	Sized    bool
-}
-
-// repoChanges changes (file additions/updates/removals) to a repo
-type repoChanges struct {
-	Updates          []fileUpdate
-	RemovedFilenames []string
-}
-
 func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) {
 	stdout, _, err := git.NewCommand(ctx, "show-ref", "-s").AddDynamicArguments(git.BranchPrefix + repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
 	if err != nil {
@@ -36,7 +24,7 @@ func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (stri
 }
 
 // getRepoChanges returns changes to repo since last indexer update
-func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
+func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) {
 	status, err := repo_model.GetIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode)
 	if err != nil {
 		return nil, err
@@ -67,16 +55,16 @@ func isIndexable(entry *git.TreeEntry) bool {
 }
 
 // parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command
-func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
+func parseGitLsTreeOutput(stdout []byte) ([]internal.FileUpdate, error) {
 	entries, err := git.ParseTreeEntries(stdout)
 	if err != nil {
 		return nil, err
 	}
 	idxCount := 0
-	updates := make([]fileUpdate, len(entries))
+	updates := make([]internal.FileUpdate, len(entries))
 	for _, entry := range entries {
 		if isIndexable(entry) {
-			updates[idxCount] = fileUpdate{
+			updates[idxCount] = internal.FileUpdate{
 				Filename: entry.Name(),
 				BlobSha:  entry.ID.String(),
 				Size:     entry.Size(),
@@ -89,8 +77,8 @@ func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
 }
 
 // genesisChanges get changes to add repo to the indexer for the first time
-func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
-	var changes repoChanges
+func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) {
+	var changes internal.RepoChanges
 	stdout, _, runErr := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()})
 	if runErr != nil {
 		return nil, runErr
@@ -102,20 +90,20 @@ func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision s
 }
 
 // nonGenesisChanges get changes since the previous indexer update
-func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
+func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) {
 	diffCmd := git.NewCommand(ctx, "diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision)
 	stdout, _, runErr := diffCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
 	if runErr != nil {
 		// previous commit sha may have been removed by a force push, so
 		// try rebuilding from scratch
 		log.Warn("git diff: %v", runErr)
-		if err := indexer.Delete(repo.ID); err != nil {
+		if err := (*globalIndexer.Load()).Delete(ctx, repo.ID); err != nil {
 			return nil, err
 		}
 		return genesisChanges(ctx, repo, revision)
 	}
 
-	var changes repoChanges
+	var changes internal.RepoChanges
 	var err error
 	updatedFilenames := make([]string, 0, 10)
 	for _, line := range strings.Split(stdout, "\n") {

modules/indexer/code/indexer.go

@@ -7,86 +7,41 @@ import (
 	"context"
 	"os"
 	"runtime/pprof"
-	"strconv"
-	"strings"
+	"sync/atomic"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/indexer/code/bleve"
+	"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 )
 
-// SearchResult result of performing a search in a repo
-type SearchResult struct {
-	RepoID      int64
-	StartIndex  int
-	EndIndex    int
-	Filename    string
-	Content     string
-	CommitID    string
-	UpdatedUnix timeutil.TimeStamp
-	Language    string
-	Color       string
+var (
+	indexerQueue *queue.WorkerPoolQueue[*internal.IndexerData]
+	// globalIndexer is the global indexer, it cannot be nil.
+	// When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready.
+	// So it's always safe use it as *globalIndexer.Load() and call its methods.
+	globalIndexer atomic.Pointer[internal.Indexer]
+	dummyIndexer  *internal.Indexer
+)
+
+func init() {
+	i := internal.NewDummyIndexer()
+	dummyIndexer = &i
+	globalIndexer.Store(dummyIndexer)
 }
 
-// SearchResultLanguages result of top languages count in search results
-type SearchResultLanguages struct {
-	Language string
-	Color    string
-	Count    int
-}
-
-// Indexer defines an interface to index and search code contents
-type Indexer interface {
-	Ping() bool
-	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
-	Delete(repoID int64) error
-	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
-	Close()
-}
-
-func filenameIndexerID(repoID int64, filename string) string {
-	return indexerID(repoID) + "_" + filename
-}
-
-func indexerID(id int64) string {
-	return strconv.FormatInt(id, 36)
-}
-
-func parseIndexerID(indexerID string) (int64, string) {
-	index := strings.IndexByte(indexerID, '_')
-	if index == -1 {
-		log.Error("Unexpected ID in repo indexer: %s", indexerID)
-	}
-	repoID, _ := strconv.ParseInt(indexerID[:index], 36, 64)
-	return repoID, indexerID[index+1:]
-}
-
-func filenameOfIndexerID(indexerID string) string {
-	index := strings.IndexByte(indexerID, '_')
-	if index == -1 {
-		log.Error("Unexpected ID in repo indexer: %s", indexerID)
-	}
-	return indexerID[index+1:]
-}
-
-// IndexerData represents data stored in the code indexer
-type IndexerData struct {
-	RepoID int64
-}
-
-var indexerQueue *queue.WorkerPoolQueue[*IndexerData]
-
-func index(ctx context.Context, indexer Indexer, repoID int64) error {
+func index(ctx context.Context, indexer internal.Indexer, repoID int64) error {
 	repo, err := repo_model.GetRepositoryByID(ctx, repoID)
 	if repo_model.IsErrRepoNotExist(err) {
-		return indexer.Delete(repoID)
+		return indexer.Delete(ctx, repoID)
 	}
 	if err != nil {
 		return err
@@ -139,7 +94,7 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error {
 // Init initialize the repo indexer
 func Init() {
 	if !setting.Indexer.RepoIndexerEnabled {
-		indexer.Close()
+		(*globalIndexer.Load()).Close()
 		return
 	}
 
@@ -153,7 +108,7 @@ func Init() {
 		}
 		cancel()
 		log.Debug("Closing repository indexer")
-		indexer.Close()
+		(*globalIndexer.Load()).Close()
 		log.Info("PID: %d Repository Indexer closed", os.Getpid())
 		finished()
 	})
@@ -163,13 +118,8 @@ func Init() {
 	// Create the Queue
 	switch setting.Indexer.RepoType {
 	case "bleve", "elasticsearch":
-		handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
-			idx, err := indexer.get()
-			if idx == nil || err != nil {
-				log.Warn("Codes indexer handler: indexer is not ready, retry later.")
-				return items
-			}
-
+		handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) {
+			indexer := *globalIndexer.Load()
 			for _, indexerData := range items {
 				log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
 
@@ -188,11 +138,7 @@ func Init() {
 					code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105)
 				*/
 				if err := index(ctx, indexer, indexerData.RepoID); err != nil {
-					if !idx.Ping() {
-						log.Error("Code indexer handler: indexer is unavailable.")
 					unhandled = append(unhandled, indexerData)
-						continue
-					}
 					if !setting.IsInTesting {
 						log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err)
 					}
@@ -213,8 +159,8 @@ func Init() {
 		pprof.SetGoroutineLabels(ctx)
 		start := time.Now()
 		var (
-			rIndexer Indexer
-			populate bool
+			rIndexer internal.Indexer
+			existed  bool
 			err      error
 		)
 		switch setting.Indexer.RepoType {
@@ -228,10 +174,11 @@ func Init() {
 				}
 			}()
 
-			rIndexer, populate, err = NewBleveIndexer(setting.Indexer.RepoPath)
+			rIndexer = bleve.NewIndexer(setting.Indexer.RepoPath)
+			existed, err = rIndexer.Init(ctx)
 			if err != nil {
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
 				close(waitChannel)
 				log.Fatal("PID: %d Unable to initialize the bleve Repository Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.RepoPath, err)
 			}
@@ -245,23 +192,31 @@ func Init() {
 				}
 			}()
 
-			rIndexer, populate, err = NewElasticSearchIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName)
+			rIndexer = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName)
 			if err != nil {
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
+				close(waitChannel)
+				log.Fatal("PID: %d Unable to create the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
+			}
+			existed, err = rIndexer.Init(ctx)
+			if err != nil {
+				cancel()
+				(*globalIndexer.Load()).Close()
 				close(waitChannel)
 				log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
 			}
+
 		default:
 			log.Fatal("PID: %d Unknown Indexer type: %s", os.Getpid(), setting.Indexer.RepoType)
 		}
 
-		indexer.set(rIndexer)
+		globalIndexer.Store(&rIndexer)
 
 		// Start processing the queue
 		go graceful.GetManager().RunWithCancel(indexerQueue)
 
-		if populate {
+		if !existed { // populate the index because it's created for the first time
 			go graceful.GetManager().RunWithShutdownContext(populateRepoIndexer)
 		}
 		select {
@@ -283,18 +238,18 @@ func Init() {
 			case <-graceful.GetManager().IsShutdown():
 				log.Warn("Shutdown before Repository Indexer completed initialization")
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
 			case duration, ok := <-waitChannel:
 				if !ok {
 					log.Warn("Repository Indexer Initialization failed")
 					cancel()
-					indexer.Close()
+					(*globalIndexer.Load()).Close()
 					return
 				}
 				log.Info("Repository Indexer Initialization took %v", duration)
 			case <-time.After(timeout):
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
 				log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
 			}
 		}()
@@ -303,21 +258,15 @@ func Init() {
 
 // UpdateRepoIndexer update a repository's entries in the indexer
 func UpdateRepoIndexer(repo *repo_model.Repository) {
-	indexData := &IndexerData{RepoID: repo.ID}
+	indexData := &internal.IndexerData{RepoID: repo.ID}
 	if err := indexerQueue.Push(indexData); err != nil {
 		log.Error("Update repo index data %v failed: %v", indexData, err)
 	}
 }
 
 // IsAvailable checks if issue indexer is available
-func IsAvailable() bool {
-	idx, err := indexer.get()
-	if err != nil {
-		log.Error("IsAvailable(): unable to get indexer: %v", err)
-		return false
-	}
-
-	return idx.Ping()
+func IsAvailable(ctx context.Context) bool {
+	return (*globalIndexer.Load()).Ping(ctx) == nil
 }
 
 // populateRepoIndexer populate the repo indexer with pre-existing data. This
@@ -368,7 +317,7 @@ func populateRepoIndexer(ctx context.Context) {
 				return
 			default:
 			}
-			if err := indexerQueue.Push(&IndexerData{RepoID: id}); err != nil {
+			if err := indexerQueue.Push(&internal.IndexerData{RepoID: id}); err != nil {
 				log.Error("indexerQueue.Push: %v", err)
 				return
 			}

modules/indexer/code/indexer_test.go

@@ -5,11 +5,15 @@ package code
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/indexer/code/bleve"
+	"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 
 	_ "code.gitea.io/gitea/models"
 
@@ -22,7 +26,7 @@ func TestMain(m *testing.M) {
 	})
 }
 
-func testIndexer(name string, t *testing.T, indexer Indexer) {
+func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 	t.Run(name, func(t *testing.T) {
 		var repoID int64 = 1
 		err := index(git.DefaultContext, indexer, repoID)
@@ -81,6 +85,48 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
 			})
 		}
 
-		assert.NoError(t, indexer.Delete(repoID))
+		assert.NoError(t, indexer.Delete(context.Background(), repoID))
 	})
 }
+
+func TestBleveIndexAndSearch(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+
+	dir := t.TempDir()
+
+	idx := bleve.NewIndexer(dir)
+	_, err := idx.Init(context.Background())
+	if err != nil {
+		assert.Fail(t, "Unable to create bleve indexer Error: %v", err)
+		if idx != nil {
+			idx.Close()
+		}
+		return
+	}
+	defer idx.Close()
+
+	testIndexer("beleve", t, idx)
+}
+
+func TestESIndexAndSearch(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+
+	u := os.Getenv("TEST_INDEXER_CODE_ES_URL")
+	if u == "" {
+		t.SkipNow()
+		return
+	}
+
+	indexer := elasticsearch.NewIndexer(u, "gitea_codes")
+	if _, err := indexer.Init(context.Background()); err != nil {
+		assert.Fail(t, "Unable to init ES indexer Error: %v", err)
+		if indexer != nil {
+			indexer.Close()
+		}
+		return
+	}
+
+	defer indexer.Close()
+
+	testIndexer("elastic_search", t, indexer)
+}

modules/indexer/code/internal/indexer.go

@@ -0,0 +1,43 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/indexer/internal"
+)
+
+// Indexer defines an interface to index and search code contents
+type Indexer interface {
+	internal.Indexer
+	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error
+	Delete(ctx context.Context, repoID int64) error
+	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
+}
+
+// NewDummyIndexer returns a dummy indexer
+func NewDummyIndexer() Indexer {
+	return &dummyIndexer{
+		Indexer: internal.NewDummyIndexer(),
+	}
+}
+
+type dummyIndexer struct {
+	internal.Indexer
+}
+
+func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+	return 0, nil, nil, fmt.Errorf("indexer is not ready")
+}

modules/indexer/code/internal/model.go

@@ -0,0 +1,44 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import "code.gitea.io/gitea/modules/timeutil"
+
+type FileUpdate struct {
+	Filename string
+	BlobSha  string
+	Size     int64
+	Sized    bool
+}
+
+// RepoChanges changes (file additions/updates/removals) to a repo
+type RepoChanges struct {
+	Updates          []FileUpdate
+	RemovedFilenames []string
+}
+
+// IndexerData represents data stored in the code indexer
+type IndexerData struct {
+	RepoID int64
+}
+
+// SearchResult result of performing a search in a repo
+type SearchResult struct {
+	RepoID      int64
+	StartIndex  int
+	EndIndex    int
+	Filename    string
+	Content     string
+	CommitID    string
+	UpdatedUnix timeutil.TimeStamp
+	Language    string
+	Color       string
+}
+
+// SearchResultLanguages result of top languages count in search results
+type SearchResultLanguages struct {
+	Language string
+	Color    string
+	Count    int
+}

modules/indexer/code/internal/util.go

@@ -0,0 +1,32 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"strings"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func FilenameIndexerID(repoID int64, filename string) string {
+	return internal.Base36(repoID) + "_" + filename
+}
+
+func ParseIndexerID(indexerID string) (int64, string) {
+	index := strings.IndexByte(indexerID, '_')
+	if index == -1 {
+		log.Error("Unexpected ID in repo indexer: %s", indexerID)
+	}
+	repoID, _ := internal.ParseBase36(indexerID[:index])
+	return repoID, indexerID[index+1:]
+}
+
+func FilenameOfIndexerID(indexerID string) string {
+	index := strings.IndexByte(indexerID, '_')
+	if index == -1 {
+		log.Error("Unexpected ID in repo indexer: %s", indexerID)
+	}
+	return indexerID[index+1:]
+}

modules/indexer/code/search.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/highlight"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -25,6 +26,8 @@ type Result struct {
 	FormattedLines string
 }
 
+type SearchResultLanguages = internal.SearchResultLanguages
+
 func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
 	startIndex := selectionStartIndex
 	numLinesBefore := 0
@@ -61,7 +64,7 @@ func writeStrings(buf *bytes.Buffer, strs ...string) error {
 	return nil
 }
 
-func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, error) {
+func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Result, error) {
 	startLineNum := 1 + strings.Count(result.Content[:startIndex], "\n")
 
 	var formattedLinesBuffer bytes.Buffer
@@ -109,12 +112,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
 }
 
 // PerformSearch perform a search on a repository
-func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
+func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*internal.SearchResultLanguages, error) {
 	if len(keyword) == 0 {
 		return 0, nil, nil, nil
 	}
 
-	total, results, resultLanguages, err := indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
+	total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
 	if err != nil {
 		return 0, nil, nil, err
 	}

modules/indexer/code/wrapped.go

@@ -1,104 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package code
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	repo_model "code.gitea.io/gitea/models/repo"
-	"code.gitea.io/gitea/modules/log"
-)
-
-var indexer = newWrappedIndexer()
-
-// ErrWrappedIndexerClosed is the error returned if the indexer was closed before it was ready
-var ErrWrappedIndexerClosed = fmt.Errorf("Indexer closed before ready")
-
-type wrappedIndexer struct {
-	internal Indexer
-	lock     sync.RWMutex
-	cond     *sync.Cond
-	closed   bool
-}
-
-func newWrappedIndexer() *wrappedIndexer {
-	w := &wrappedIndexer{}
-	w.cond = sync.NewCond(w.lock.RLocker())
-	return w
-}
-
-func (w *wrappedIndexer) set(indexer Indexer) {
-	w.lock.Lock()
-	defer w.lock.Unlock()
-	if w.closed {
-		// Too late!
-		indexer.Close()
-	}
-	w.internal = indexer
-	w.cond.Broadcast()
-}
-
-func (w *wrappedIndexer) get() (Indexer, error) {
-	w.lock.RLock()
-	defer w.lock.RUnlock()
-	if w.internal == nil {
-		if w.closed {
-			return nil, ErrWrappedIndexerClosed
-		}
-		w.cond.Wait()
-		if w.closed {
-			return nil, ErrWrappedIndexerClosed
-		}
-	}
-	return w.internal, nil
-}
-
-// Ping checks if elastic is available
-func (w *wrappedIndexer) Ping() bool {
-	indexer, err := w.get()
-	if err != nil {
-		log.Warn("Failed to get indexer: %v", err)
-		return false
-	}
-	return indexer.Ping()
-}
-
-func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
-	indexer, err := w.get()
-	if err != nil {
-		return err
-	}
-	return indexer.Index(ctx, repo, sha, changes)
-}
-
-func (w *wrappedIndexer) Delete(repoID int64) error {
-	indexer, err := w.get()
-	if err != nil {
-		return err
-	}
-	return indexer.Delete(repoID)
-}
-
-func (w *wrappedIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
-	indexer, err := w.get()
-	if err != nil {
-		return 0, nil, nil, err
-	}
-	return indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
-}
-
-func (w *wrappedIndexer) Close() {
-	w.lock.Lock()
-	defer w.lock.Unlock()
-	if w.closed {
-		return
-	}
-	w.closed = true
-	w.cond.Broadcast()
-	if w.internal != nil {
-		w.internal.Close()
-	}
-}

modules/indexer/internal/base32.go

@@ -0,0 +1,21 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"fmt"
+	"strconv"
+)
+
+func Base36(i int64) string {
+	return strconv.FormatInt(i, 36)
+}
+
+func ParseBase36(s string) (int64, error) {
+	i, err := strconv.ParseInt(s, 36, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid base36 integer %q: %w", s, err)
+	}
+	return i, nil
+}

modules/indexer/internal/bleve/indexer.go

@@ -0,0 +1,103 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/ethantkoenig/rupture"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic bleve indexer implementation
+type Indexer struct {
+	Indexer bleve.Index
+
+	indexDir      string
+	version       int
+	mappingGetter MappingGetter
+}
+
+type MappingGetter func() (mapping.IndexMapping, error)
+
+func NewIndexer(indexDir string, version int, mappingGetter func() (mapping.IndexMapping, error)) *Indexer {
+	return &Indexer{
+		indexDir:      indexDir,
+		version:       version,
+		mappingGetter: mappingGetter,
+	}
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+	if i == nil {
+		return false, fmt.Errorf("cannot init nil indexer")
+	}
+
+	if i.Indexer != nil {
+		return false, fmt.Errorf("indexer is already initialized")
+	}
+
+	indexer, version, err := openIndexer(i.indexDir, i.version)
+	if err != nil {
+		return false, err
+	}
+	if indexer != nil {
+		i.Indexer = indexer
+		return true, nil
+	}
+
+	if version != 0 {
+		log.Warn("Found older bleve index with version %d, Gitea will remove it and rebuild", version)
+	}
+
+	indexMapping, err := i.mappingGetter()
+	if err != nil {
+		return false, err
+	}
+
+	indexer, err = bleve.New(i.indexDir, indexMapping)
+	if err != nil {
+		return false, err
+	}
+
+	if err = rupture.WriteIndexMetadata(i.indexDir, &rupture.IndexMetadata{
+		Version: i.version,
+	}); err != nil {
+		return false, err
+	}
+
+	i.Indexer = indexer
+
+	return false, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(_ context.Context) error {
+	if i == nil {
+		return fmt.Errorf("cannot ping nil indexer")
+	}
+	if i.Indexer == nil {
+		return fmt.Errorf("indexer is not initialized")
+	}
+	return nil
+}
+
+func (i *Indexer) Close() {
+	if i == nil {
+		return
+	}
+
+	if err := i.Indexer.Close(); err != nil {
+		log.Error("Failed to close bleve indexer in %q: %v", i.indexDir, err)
+	}
+	i.Indexer = nil
+}

modules/indexer/internal/bleve/util.go

@@ -0,0 +1,49 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"errors"
+	"os"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/index/upsidedown"
+	"github.com/ethantkoenig/rupture"
+)
+
+// openIndexer open the index at the specified path, checking for metadata
+// updates and bleve version updates.  If index needs to be created (or
+// re-created), returns (nil, nil)
+func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
+	_, err := os.Stat(path)
+	if err != nil && os.IsNotExist(err) {
+		return nil, 0, nil
+	} else if err != nil {
+		return nil, 0, err
+	}
+
+	metadata, err := rupture.ReadIndexMetadata(path)
+	if err != nil {
+		return nil, 0, err
+	}
+	if metadata.Version < latestVersion {
+		// the indexer is using a previous version, so we should delete it and
+		// re-populate
+		return nil, metadata.Version, util.RemoveAll(path)
+	}
+
+	index, err := bleve.Open(path)
+	if err != nil {
+		if errors.Is(err, upsidedown.IncompatibleVersion) {
+			log.Warn("Indexer was built with a previous version of bleve, deleting and rebuilding")
+			return nil, 0, util.RemoveAll(path)
+		}
+		return nil, 0, err
+	}
+
+	return index, 0, nil
+}

modules/indexer/internal/db/indexer.go

@@ -0,0 +1,34 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic db indexer implementation
+type Indexer struct{}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+	// Return true to indicate that the index was opened/existed.
+	// So that the indexer will not try to populate the index, the data is already there.
+	return true, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(_ context.Context) error {
+	// No need to ping database to check if it is available.
+	// If the database goes down, Gitea will go down, so nobody will care if the indexer is available.
+	return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+	// nothing to do
+}

modules/indexer/internal/elasticsearch/indexer.go

@@ -0,0 +1,92 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+
+	"github.com/olivere/elastic/v7"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic elasticsearch indexer implementation
+type Indexer struct {
+	Client *elastic.Client
+
+	url       string
+	indexName string
+	version   int
+	mapping   string
+}
+
+func NewIndexer(url, indexName string, version int, mapping string) *Indexer {
+	return &Indexer{
+		url:       url,
+		indexName: indexName,
+		version:   version,
+		mapping:   mapping,
+	}
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(ctx context.Context) (bool, error) {
+	if i == nil {
+		return false, fmt.Errorf("cannot init nil indexer")
+	}
+	if i.Client != nil {
+		return false, fmt.Errorf("indexer is already initialized")
+	}
+
+	client, err := i.initClient()
+	if err != nil {
+		return false, err
+	}
+	i.Client = client
+
+	exists, err := i.Client.IndexExists(i.VersionedIndexName()).Do(ctx)
+	if err != nil {
+		return false, err
+	}
+	if exists {
+		return true, nil
+	}
+
+	if err := i.createIndex(ctx); err != nil {
+		return false, err
+	}
+
+	return exists, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(ctx context.Context) error {
+	if i == nil {
+		return fmt.Errorf("cannot ping nil indexer")
+	}
+	if i.Client == nil {
+		return fmt.Errorf("indexer is not initialized")
+	}
+
+	resp, err := i.Client.ClusterHealth().Do(ctx)
+	if err != nil {
+		return err
+	}
+	if resp.Status != "green" {
+		// see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
+		return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status)
+	}
+	return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+	if i == nil {
+		return
+	}
+	i.Client = nil
+}

modules/indexer/internal/elasticsearch/util.go

@@ -0,0 +1,68 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/olivere/elastic/v7"
+)
+
+// VersionedIndexName returns the full index name with version
+func (i *Indexer) VersionedIndexName() string {
+	return versionedIndexName(i.indexName, i.version)
+}
+
+func versionedIndexName(indexName string, version int) string {
+	if version == 0 {
+		// Old index name without version
+		return indexName
+	}
+	return fmt.Sprintf("%s.v%d", indexName, version)
+}
+
+func (i *Indexer) createIndex(ctx context.Context) error {
+	createIndex, err := i.Client.CreateIndex(i.VersionedIndexName()).BodyString(i.mapping).Do(ctx)
+	if err != nil {
+		return err
+	}
+	if !createIndex.Acknowledged {
+		return fmt.Errorf("create index %s with %s failed", i.VersionedIndexName(), i.mapping)
+	}
+
+	i.checkOldIndexes(ctx)
+
+	return nil
+}
+
+func (i *Indexer) initClient() (*elastic.Client, error) {
+	opts := []elastic.ClientOptionFunc{
+		elastic.SetURL(i.url),
+		elastic.SetSniff(false),
+		elastic.SetHealthcheckInterval(10 * time.Second),
+		elastic.SetGzip(false),
+	}
+
+	logger := log.GetLogger(log.DEFAULT)
+
+	opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
+	opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
+	opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
+
+	return elastic.NewClient(opts...)
+}
+
+func (i *Indexer) checkOldIndexes(ctx context.Context) {
+	for v := 0; v < i.version; v++ {
+		indexName := versionedIndexName(i.indexName, v)
+		exists, err := i.Client.IndexExists(indexName).Do(ctx)
+		if err == nil && exists {
+			log.Warn("Found older elasticsearch index named %q, Gitea will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName)
+		}
+	}
+}

modules/indexer/internal/indexer.go

@@ -0,0 +1,37 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"context"
+	"fmt"
+)
+
+// Indexer defines an basic indexer interface
+type Indexer interface {
+	// Init initializes the indexer
+	// returns true if the index was opened/existed (with data populated), false if it was created/not-existed (with no data)
+	Init(ctx context.Context) (bool, error)
+	// Ping checks if the indexer is available
+	Ping(ctx context.Context) error
+	// Close closes the indexer
+	Close()
+}
+
+// NewDummyIndexer returns a dummy indexer
+func NewDummyIndexer() Indexer {
+	return &dummyIndexer{}
+}
+
+type dummyIndexer struct{}
+
+func (d *dummyIndexer) Init(ctx context.Context) (bool, error) {
+	return false, fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Ping(ctx context.Context) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Close() {}

modules/indexer/internal/meilisearch/indexer.go

@@ -0,0 +1,92 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package meilisearch
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/meilisearch/meilisearch-go"
+)
+
+// Indexer represents a basic meilisearch indexer implementation
+type Indexer struct {
+	Client *meilisearch.Client
+
+	url, apiKey string
+	indexName   string
+	version     int
+}
+
+func NewIndexer(url, apiKey, indexName string, version int) *Indexer {
+	return &Indexer{
+		url:       url,
+		apiKey:    apiKey,
+		indexName: indexName,
+		version:   version,
+	}
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+	if i == nil {
+		return false, fmt.Errorf("cannot init nil indexer")
+	}
+
+	if i.Client != nil {
+		return false, fmt.Errorf("indexer is already initialized")
+	}
+
+	i.Client = meilisearch.NewClient(meilisearch.ClientConfig{
+		Host:   i.url,
+		APIKey: i.apiKey,
+	})
+
+	_, err := i.Client.GetIndex(i.VersionedIndexName())
+	if err == nil {
+		return true, nil
+	}
+	_, err = i.Client.CreateIndex(&meilisearch.IndexConfig{
+		Uid:        i.VersionedIndexName(),
+		PrimaryKey: "id",
+	})
+	if err != nil {
+		return false, err
+	}
+
+	i.checkOldIndexes()
+
+	_, err = i.Client.Index(i.VersionedIndexName()).UpdateFilterableAttributes(&[]string{"repo_id"})
+	return false, err
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(ctx context.Context) error {
+	if i == nil {
+		return fmt.Errorf("cannot ping nil indexer")
+	}
+	if i.Client == nil {
+		return fmt.Errorf("indexer is not initialized")
+	}
+	resp, err := i.Client.Health()
+	if err != nil {
+		return err
+	}
+	if resp.Status != "available" {
+		// See https://docs.meilisearch.com/reference/api/health.html#status
+		return fmt.Errorf("status of meilisearch is not available: %s", resp.Status)
+	}
+	return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+	if i == nil {
+		return
+	}
+	if i.Client == nil {
+		return
+	}
+	i.Client = nil
+}

modules/indexer/internal/meilisearch/util.go

@@ -0,0 +1,38 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package meilisearch
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+// VersionedIndexName returns the full index name with version
+func (i *Indexer) VersionedIndexName() string {
+	return versionedIndexName(i.indexName, i.version)
+}
+
+func versionedIndexName(indexName string, version int) string {
+	if version == 0 {
+		// Old index name without version
+		return indexName
+	}
+
+	// The format of the index name is <index_name>_v<version>, not <index_name>.v<version> like elasticsearch.
+	// Because meilisearch does not support "." in index name, it should contain only alphanumeric characters, hyphens (-) and underscores (_).
+	// See https://www.meilisearch.com/docs/learn/core_concepts/indexes#index-uid
+
+	return fmt.Sprintf("%s_v%d", indexName, version)
+}
+
+func (i *Indexer) checkOldIndexes() {
+	for v := 0; v < i.version; v++ {
+		indexName := versionedIndexName(i.indexName, v)
+		_, err := i.Client.GetIndex(indexName)
+		if err == nil {
+			log.Warn("Found older meilisearch index named %q, Gitea will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName)
+		}
+	}
+}

modules/indexer/issues/bleve/bleve.go

@@ -1,17 +1,14 @@
 // Copyright 2018 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-package issues
+package bleve
 
 import (
 	"context"
-	"fmt"
-	"os"
-	"strconv"
 
-	gitea_bleve "code.gitea.io/gitea/modules/indexer/bleve"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/util"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -19,10 +16,8 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
-	"github.com/blevesearch/bleve/v2/index/upsidedown"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search/query"
-	"github.com/ethantkoenig/rupture"
 )
 
 const (
@@ -31,20 +26,6 @@ const (
 	issueIndexerLatestVersion = 2
 )
 
-// indexerID a bleve-compatible unique identifier for an integer id
-func indexerID(id int64) string {
-	return strconv.FormatInt(id, 36)
-}
-
-// idOfIndexerID the integer id associated with an indexer id
-func idOfIndexerID(indexerID string) (int64, error) {
-	id, err := strconv.ParseInt(indexerID, 36, 64)
-	if err != nil {
-		return 0, fmt.Errorf("Unexpected indexer ID %s: %w", indexerID, err)
-	}
-	return id, nil
-}
-
 // numericEqualityQuery a numeric equality query for the given value and field
 func numericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
 	f := float64(value)
@@ -72,49 +53,16 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
 
 const maxBatchSize = 16
 
-// openIndexer open the index at the specified path, checking for metadata
-// updates and bleve version updates.  If index needs to be created (or
-// re-created), returns (nil, nil)
-func openIndexer(path string, latestVersion int) (bleve.Index, error) {
-	_, err := os.Stat(path)
-	if err != nil && os.IsNotExist(err) {
-		return nil, nil
-	} else if err != nil {
-		return nil, err
-	}
-
-	metadata, err := rupture.ReadIndexMetadata(path)
-	if err != nil {
-		return nil, err
-	}
-	if metadata.Version < latestVersion {
-		// the indexer is using a previous version, so we should delete it and
-		// re-populate
-		return nil, util.RemoveAll(path)
-	}
-
-	index, err := bleve.Open(path)
-	if err != nil && err == upsidedown.IncompatibleVersion {
-		// the indexer was built with a previous version of bleve, so we should
-		// delete it and re-populate
-		return nil, util.RemoveAll(path)
-	} else if err != nil {
-		return nil, err
-	}
-
-	return index, nil
-}
-
-// BleveIndexerData an update to the issue indexer
-type BleveIndexerData IndexerData
+// IndexerData an update to the issue indexer
+type IndexerData internal.IndexerData
 
 // Type returns the document type, for bleve's mapping.Classifier interface.
-func (i *BleveIndexerData) Type() string {
+func (i *IndexerData) Type() string {
 	return issueIndexerDocType
 }
 
-// createIssueIndexer create an issue indexer if one does not already exist
-func createIssueIndexer(path string, latestVersion int) (bleve.Index, error) {
+// generateIssueIndexMapping generates the bleve index mapping for issues
+func generateIssueIndexMapping() (mapping.IndexMapping, error) {
 	mapping := bleve.NewIndexMapping()
 	docMapping := bleve.NewDocumentMapping()
 
@@ -144,68 +92,31 @@ func createIssueIndexer(path string, latestVersion int) (bleve.Index, error) {
 	mapping.AddDocumentMapping(issueIndexerDocType, docMapping)
 	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
 
-	index, err := bleve.New(path, mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
-		Version: latestVersion,
-	}); err != nil {
-		return nil, err
-	}
-	return index, nil
+	return mapping, nil
 }
 
-var _ Indexer = &BleveIndexer{}
+var _ internal.Indexer = &Indexer{}
 
-// BleveIndexer implements Indexer interface
-type BleveIndexer struct {
-	indexDir string
-	indexer  bleve.Index
+// Indexer implements Indexer interface
+type Indexer struct {
+	inner                    *inner_bleve.Indexer
+	indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much
 }
 
-// NewBleveIndexer creates a new bleve local indexer
-func NewBleveIndexer(indexDir string) *BleveIndexer {
-	return &BleveIndexer{
-		indexDir: indexDir,
-	}
-}
-
-// Init will initialize the indexer
-func (b *BleveIndexer) Init() (bool, error) {
-	var err error
-	b.indexer, err = openIndexer(b.indexDir, issueIndexerLatestVersion)
-	if err != nil {
-		return false, err
-	}
-	if b.indexer != nil {
-		return true, nil
-	}
-
-	b.indexer, err = createIssueIndexer(b.indexDir, issueIndexerLatestVersion)
-	return false, err
-}
-
-// Ping does nothing
-func (b *BleveIndexer) Ping() bool {
-	return true
-}
-
-// Close will close the bleve indexer
-func (b *BleveIndexer) Close() {
-	if b.indexer != nil {
-		if err := b.indexer.Close(); err != nil {
-			log.Error("Error whilst closing indexer: %v", err)
-		}
+// NewIndexer creates a new bleve local indexer
+func NewIndexer(indexDir string) *Indexer {
+	inner := inner_bleve.NewIndexer(indexDir, issueIndexerLatestVersion, generateIssueIndexMapping)
+	return &Indexer{
+		Indexer: inner,
+		inner:   inner,
 	}
 }
 
 // Index will save the index data
-func (b *BleveIndexer) Index(issues []*IndexerData) error {
-	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
+func (b *Indexer) Index(_ context.Context, issues []*internal.IndexerData) error {
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
 	for _, issue := range issues {
-		if err := batch.Index(indexerID(issue.ID), struct {
+		if err := batch.Index(indexer_internal.Base36(issue.ID), struct {
 			RepoID   int64
 			Title    string
 			Content  string
@@ -223,10 +134,10 @@ func (b *BleveIndexer) Index(issues []*IndexerData) error {
 }
 
 // Delete deletes indexes by ids
-func (b *BleveIndexer) Delete(ids ...int64) error {
-	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
+func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
 	for _, id := range ids {
-		if err := batch.Delete(indexerID(id)); err != nil {
+		if err := batch.Delete(indexer_internal.Base36(id)); err != nil {
 			return err
 		}
 	}
@@ -235,7 +146,7 @@ func (b *BleveIndexer) Delete(ids ...int64) error {
 
 // Search searches for issues by given conditions.
 // Returns the matching issue IDs
-func (b *BleveIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
+func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
 	var repoQueriesP []*query.NumericRangeQuery
 	for _, repoID := range repoIDs {
 		repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "RepoID"))
@@ -255,20 +166,20 @@ func (b *BleveIndexer) Search(ctx context.Context, keyword string, repoIDs []int
 	search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false)
 	search.SortBy([]string{"-_score"})
 
-	result, err := b.indexer.SearchInContext(ctx, search)
+	result, err := b.inner.Indexer.SearchInContext(ctx, search)
 	if err != nil {
 		return nil, err
 	}
 
-	ret := SearchResult{
-		Hits: make([]Match, 0, len(result.Hits)),
+	ret := internal.SearchResult{
+		Hits: make([]internal.Match, 0, len(result.Hits)),
 	}
 	for _, hit := range result.Hits {
-		id, err := idOfIndexerID(hit.ID)
+		id, err := indexer_internal.ParseBase36(hit.ID)
 		if err != nil {
 			return nil, err
 		}
-		ret.Hits = append(ret.Hits, Match{
+		ret.Hits = append(ret.Hits, internal.Match{
 			ID: id,
 		})
 	}

modules/indexer/issues/bleve/bleve_test.go

@@ -1,26 +1,28 @@
 // Copyright 2018 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-package issues
+package bleve
 
 import (
 	"context"
 	"testing"
 
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
+
 	"github.com/stretchr/testify/assert"
 )
 
 func TestBleveIndexAndSearch(t *testing.T) {
 	dir := t.TempDir()
-	indexer := NewBleveIndexer(dir)
+	indexer := NewIndexer(dir)
 	defer indexer.Close()
 
-	if _, err := indexer.Init(); err != nil {
+	if _, err := indexer.Init(context.Background()); err != nil {
 		assert.Fail(t, "Unable to initialize bleve indexer: %v", err)
 		return
 	}
 
-	err := indexer.Index([]*IndexerData{
+	err := indexer.Index(context.Background(), []*internal.IndexerData{
 		{
 			ID:      1,
 			RepoID:  2,

modules/indexer/issues/db.go

@@ -1,56 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package issues
-
-import (
-	"context"
-
-	"code.gitea.io/gitea/models/db"
-	issues_model "code.gitea.io/gitea/models/issues"
-)
-
-// DBIndexer implements Indexer interface to use database's like search
-type DBIndexer struct{}
-
-// Init dummy function
-func (i *DBIndexer) Init() (bool, error) {
-	return false, nil
-}
-
-// Ping checks if database is available
-func (i *DBIndexer) Ping() bool {
-	return db.GetEngine(db.DefaultContext).Ping() != nil
-}
-
-// Index dummy function
-func (i *DBIndexer) Index(issue []*IndexerData) error {
-	return nil
-}
-
-// Delete dummy function
-func (i *DBIndexer) Delete(ids ...int64) error {
-	return nil
-}
-
-// Close dummy function
-func (i *DBIndexer) Close() {
-}
-
-// Search dummy function
-func (i *DBIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
-	total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
-	if err != nil {
-		return nil, err
-	}
-	result := SearchResult{
-		Total: total,
-		Hits:  make([]Match, 0, limit),
-	}
-	for _, id := range ids {
-		result.Hits = append(result.Hits, Match{
-			ID: id,
-		})
-	}
-	return &result, nil
-}

modules/indexer/issues/db/db.go

@@ -0,0 +1,54 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db
+
+import (
+	"context"
+
+	issues_model "code.gitea.io/gitea/models/issues"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_db "code.gitea.io/gitea/modules/indexer/internal/db"
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer implements Indexer interface to use database's like search
+type Indexer struct {
+	indexer_internal.Indexer
+}
+
+func NewIndexer() *Indexer {
+	return &Indexer{
+		Indexer: &inner_db.Indexer{},
+	}
+}
+
+// Index dummy function
+func (i *Indexer) Index(_ context.Context, _ []*internal.IndexerData) error {
+	return nil
+}
+
+// Delete dummy function
+func (i *Indexer) Delete(_ context.Context, _ ...int64) error {
+	return nil
+}
+
+// Search searches for issues
+func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
+	total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
+	if err != nil {
+		return nil, err
+	}
+	result := internal.SearchResult{
+		Total: total,
+		Hits:  make([]internal.Match, 0, limit),
+	}
+	for _, id := range ids {
+		result.Hits = append(result.Hits, internal.Match{
+			ID: id,
+		})
+	}
+	return &result, nil
+}

modules/indexer/issues/elastic_search.go

@@ -1,287 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package issues
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net"
-	"strconv"
-	"sync"
-	"time"
-
-	"code.gitea.io/gitea/modules/graceful"
-	"code.gitea.io/gitea/modules/log"
-
-	"github.com/olivere/elastic/v7"
-)
-
-var _ Indexer = &ElasticSearchIndexer{}
-
-// ElasticSearchIndexer implements Indexer interface
-type ElasticSearchIndexer struct {
-	client      *elastic.Client
-	indexerName string
-	available   bool
-	stopTimer   chan struct{}
-	lock        sync.RWMutex
-}
-
-// NewElasticSearchIndexer creates a new elasticsearch indexer
-func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, error) {
-	opts := []elastic.ClientOptionFunc{
-		elastic.SetURL(url),
-		elastic.SetSniff(false),
-		elastic.SetHealthcheckInterval(10 * time.Second),
-		elastic.SetGzip(false),
-	}
-
-	logger := log.GetLogger(log.DEFAULT)
-	opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
-	opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
-	opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
-
-	client, err := elastic.NewClient(opts...)
-	if err != nil {
-		return nil, err
-	}
-
-	indexer := &ElasticSearchIndexer{
-		client:      client,
-		indexerName: indexerName,
-		available:   true,
-		stopTimer:   make(chan struct{}),
-	}
-
-	ticker := time.NewTicker(10 * time.Second)
-	go func() {
-		for {
-			select {
-			case <-ticker.C:
-				indexer.checkAvailability()
-			case <-indexer.stopTimer:
-				ticker.Stop()
-				return
-			}
-		}
-	}()
-
-	return indexer, nil
-}
-
-const (
-	defaultMapping = `{
-		"mappings": {
-			"properties": {
-				"id": {
-					"type": "integer",
-					"index": true
-				},
-				"repo_id": {
-					"type": "integer",
-					"index": true
-				},
-				"title": {
-					"type": "text",
-					"index": true
-				},
-				"content": {
-					"type": "text",
-					"index": true
-				},
-				"comments": {
-					"type" : "text",
-					"index": true
-				}
-			}
-		}
-	}`
-)
-
-// Init will initialize the indexer
-func (b *ElasticSearchIndexer) Init() (bool, error) {
-	ctx := graceful.GetManager().HammerContext()
-	exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
-	if err != nil {
-		return false, b.checkError(err)
-	}
-
-	if !exists {
-		mapping := defaultMapping
-
-		createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !createIndex.Acknowledged {
-			return false, errors.New("init failed")
-		}
-
-		return false, nil
-	}
-	return true, nil
-}
-
-// Ping checks if elastic is available
-func (b *ElasticSearchIndexer) Ping() bool {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	return b.available
-}
-
-// Index will save the index data
-func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
-	if len(issues) == 0 {
-		return nil
-	} else if len(issues) == 1 {
-		issue := issues[0]
-		_, err := b.client.Index().
-			Index(b.indexerName).
-			Id(fmt.Sprintf("%d", issue.ID)).
-			BodyJson(map[string]interface{}{
-				"id":       issue.ID,
-				"repo_id":  issue.RepoID,
-				"title":    issue.Title,
-				"content":  issue.Content,
-				"comments": issue.Comments,
-			}).
-			Do(graceful.GetManager().HammerContext())
-		return b.checkError(err)
-	}
-
-	reqs := make([]elastic.BulkableRequest, 0)
-	for _, issue := range issues {
-		reqs = append(reqs,
-			elastic.NewBulkIndexRequest().
-				Index(b.indexerName).
-				Id(fmt.Sprintf("%d", issue.ID)).
-				Doc(map[string]interface{}{
-					"id":       issue.ID,
-					"repo_id":  issue.RepoID,
-					"title":    issue.Title,
-					"content":  issue.Content,
-					"comments": issue.Comments,
-				}),
-		)
-	}
-
-	_, err := b.client.Bulk().
-		Index(b.indexerName).
-		Add(reqs...).
-		Do(graceful.GetManager().HammerContext())
-	return b.checkError(err)
-}
-
-// Delete deletes indexes by ids
-func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
-	if len(ids) == 0 {
-		return nil
-	} else if len(ids) == 1 {
-		_, err := b.client.Delete().
-			Index(b.indexerName).
-			Id(fmt.Sprintf("%d", ids[0])).
-			Do(graceful.GetManager().HammerContext())
-		return b.checkError(err)
-	}
-
-	reqs := make([]elastic.BulkableRequest, 0)
-	for _, id := range ids {
-		reqs = append(reqs,
-			elastic.NewBulkDeleteRequest().
-				Index(b.indexerName).
-				Id(fmt.Sprintf("%d", id)),
-		)
-	}
-
-	_, err := b.client.Bulk().
-		Index(b.indexerName).
-		Add(reqs...).
-		Do(graceful.GetManager().HammerContext())
-	return b.checkError(err)
-}
-
-// Search searches for issues by given conditions.
-// Returns the matching issue IDs
-func (b *ElasticSearchIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
-	kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
-	query := elastic.NewBoolQuery()
-	query = query.Must(kwQuery)
-	if len(repoIDs) > 0 {
-		repoStrs := make([]interface{}, 0, len(repoIDs))
-		for _, repoID := range repoIDs {
-			repoStrs = append(repoStrs, repoID)
-		}
-		repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
-		query = query.Must(repoQuery)
-	}
-	searchResult, err := b.client.Search().
-		Index(b.indexerName).
-		Query(query).
-		Sort("_score", false).
-		From(start).Size(limit).
-		Do(ctx)
-	if err != nil {
-		return nil, b.checkError(err)
-	}
-
-	hits := make([]Match, 0, limit)
-	for _, hit := range searchResult.Hits.Hits {
-		id, _ := strconv.ParseInt(hit.Id, 10, 64)
-		hits = append(hits, Match{
-			ID: id,
-		})
-	}
-
-	return &SearchResult{
-		Total: searchResult.TotalHits(),
-		Hits:  hits,
-	}, nil
-}
-
-// Close implements indexer
-func (b *ElasticSearchIndexer) Close() {
-	select {
-	case <-b.stopTimer:
-	default:
-		close(b.stopTimer)
-	}
-}
-
-func (b *ElasticSearchIndexer) checkError(err error) error {
-	var opErr *net.OpError
-	if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
-		return err
-	}
-
-	b.setAvailability(false)
-
-	return err
-}
-
-func (b *ElasticSearchIndexer) checkAvailability() {
-	if b.Ping() {
-		return
-	}
-
-	// Request cluster state to check if elastic is available again
-	_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
-	if err != nil {
-		b.setAvailability(false)
-		return
-	}
-
-	b.setAvailability(true)
-}
-
-func (b *ElasticSearchIndexer) setAvailability(available bool) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.available == available {
-		return
-	}
-
-	b.available = available
-}

modules/indexer/issues/elasticsearch/elasticsearch.go

@@ -0,0 +1,177 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"code.gitea.io/gitea/modules/graceful"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
+
+	"github.com/olivere/elastic/v7"
+)
+
+const (
+	issueIndexerLatestVersion = 0
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer implements Indexer interface
+type Indexer struct {
+	inner                    *inner_elasticsearch.Indexer
+	indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
+}
+
+// NewIndexer creates a new elasticsearch indexer
+func NewIndexer(url, indexerName string) *Indexer {
+	inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping)
+	indexer := &Indexer{
+		inner:   inner,
+		Indexer: inner,
+	}
+	return indexer
+}
+
+const (
+	defaultMapping = `{
+		"mappings": {
+			"properties": {
+				"id": {
+					"type": "integer",
+					"index": true
+				},
+				"repo_id": {
+					"type": "integer",
+					"index": true
+				},
+				"title": {
+					"type": "text",
+					"index": true
+				},
+				"content": {
+					"type": "text",
+					"index": true
+				},
+				"comments": {
+					"type" : "text",
+					"index": true
+				}
+			}
+		}
+	}`
+)
+
+// Index will save the index data
+func (b *Indexer) Index(ctx context.Context, issues []*internal.IndexerData) error {
+	if len(issues) == 0 {
+		return nil
+	} else if len(issues) == 1 {
+		issue := issues[0]
+		_, err := b.inner.Client.Index().
+			Index(b.inner.VersionedIndexName()).
+			Id(fmt.Sprintf("%d", issue.ID)).
+			BodyJson(map[string]interface{}{
+				"id":       issue.ID,
+				"repo_id":  issue.RepoID,
+				"title":    issue.Title,
+				"content":  issue.Content,
+				"comments": issue.Comments,
+			}).
+			Do(ctx)
+		return err
+	}
+
+	reqs := make([]elastic.BulkableRequest, 0)
+	for _, issue := range issues {
+		reqs = append(reqs,
+			elastic.NewBulkIndexRequest().
+				Index(b.inner.VersionedIndexName()).
+				Id(fmt.Sprintf("%d", issue.ID)).
+				Doc(map[string]interface{}{
+					"id":       issue.ID,
+					"repo_id":  issue.RepoID,
+					"title":    issue.Title,
+					"content":  issue.Content,
+					"comments": issue.Comments,
+				}),
+		)
+	}
+
+	_, err := b.inner.Client.Bulk().
+		Index(b.inner.VersionedIndexName()).
+		Add(reqs...).
+		Do(graceful.GetManager().HammerContext())
+	return err
+}
+
+// Delete deletes indexes by ids
+func (b *Indexer) Delete(ctx context.Context, ids ...int64) error {
+	if len(ids) == 0 {
+		return nil
+	} else if len(ids) == 1 {
+		_, err := b.inner.Client.Delete().
+			Index(b.inner.VersionedIndexName()).
+			Id(fmt.Sprintf("%d", ids[0])).
+			Do(ctx)
+		return err
+	}
+
+	reqs := make([]elastic.BulkableRequest, 0)
+	for _, id := range ids {
+		reqs = append(reqs,
+			elastic.NewBulkDeleteRequest().
+				Index(b.inner.VersionedIndexName()).
+				Id(fmt.Sprintf("%d", id)),
+		)
+	}
+
+	_, err := b.inner.Client.Bulk().
+		Index(b.inner.VersionedIndexName()).
+		Add(reqs...).
+		Do(graceful.GetManager().HammerContext())
+	return err
+}
+
+// Search searches for issues by given conditions.
+// Returns the matching issue IDs
+func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
+	kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
+	query := elastic.NewBoolQuery()
+	query = query.Must(kwQuery)
+	if len(repoIDs) > 0 {
+		repoStrs := make([]interface{}, 0, len(repoIDs))
+		for _, repoID := range repoIDs {
+			repoStrs = append(repoStrs, repoID)
+		}
+		repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
+		query = query.Must(repoQuery)
+	}
+	searchResult, err := b.inner.Client.Search().
+		Index(b.inner.VersionedIndexName()).
+		Query(query).
+		Sort("_score", false).
+		From(start).Size(limit).
+		Do(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	hits := make([]internal.Match, 0, limit)
+	for _, hit := range searchResult.Hits.Hits {
+		id, _ := strconv.ParseInt(hit.Id, 10, 64)
+		hits = append(hits, internal.Match{
+			ID: id,
+		})
+	}
+
+	return &internal.SearchResult{
+		Total: searchResult.TotalHits(),
+		Hits:  hits,
+	}, nil
+}

modules/indexer/issues/indexer.go

@@ -5,16 +5,20 @@ package issues
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"runtime/pprof"
-	"sync"
+	"sync/atomic"
 	"time"
 
-	"code.gitea.io/gitea/models/db"
+	db_model "code.gitea.io/gitea/models/db"
 	issues_model "code.gitea.io/gitea/models/issues"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/indexer/issues/bleve"
+	"code.gitea.io/gitea/modules/indexer/issues/db"
+	"code.gitea.io/gitea/modules/indexer/issues/elasticsearch"
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
+	"code.gitea.io/gitea/modules/indexer/issues/meilisearch"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/queue"
@@ -22,81 +26,22 @@ import (
 	"code.gitea.io/gitea/modules/util"
 )
 
-// IndexerData data stored in the issue indexer
-type IndexerData struct {
-	ID       int64    `json:"id"`
-	RepoID   int64    `json:"repo_id"`
-	Title    string   `json:"title"`
-	Content  string   `json:"content"`
-	Comments []string `json:"comments"`
-	IsDelete bool     `json:"is_delete"`
-	IDs      []int64  `json:"ids"`
-}
-
-// Match represents on search result
-type Match struct {
-	ID    int64   `json:"id"`
-	Score float64 `json:"score"`
-}
-
-// SearchResult represents search results
-type SearchResult struct {
-	Total int64
-	Hits  []Match
-}
-
-// Indexer defines an interface to indexer issues contents
-type Indexer interface {
-	Init() (bool, error)
-	Ping() bool
-	Index(issue []*IndexerData) error
-	Delete(ids ...int64) error
-	Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
-	Close()
-}
-
-type indexerHolder struct {
-	indexer   Indexer
-	mutex     sync.RWMutex
-	cond      *sync.Cond
-	cancelled bool
-}
-
-func newIndexerHolder() *indexerHolder {
-	h := &indexerHolder{}
-	h.cond = sync.NewCond(h.mutex.RLocker())
-	return h
-}
-
-func (h *indexerHolder) cancel() {
-	h.mutex.Lock()
-	defer h.mutex.Unlock()
-	h.cancelled = true
-	h.cond.Broadcast()
-}
-
-func (h *indexerHolder) set(indexer Indexer) {
-	h.mutex.Lock()
-	defer h.mutex.Unlock()
-	h.indexer = indexer
-	h.cond.Broadcast()
-}
-
-func (h *indexerHolder) get() Indexer {
-	h.mutex.RLock()
-	defer h.mutex.RUnlock()
-	if h.indexer == nil && !h.cancelled {
-		h.cond.Wait()
-	}
-	return h.indexer
-}
-
 var (
 	// issueIndexerQueue queue of issue ids to be updated
-	issueIndexerQueue *queue.WorkerPoolQueue[*IndexerData]
-	holder            = newIndexerHolder()
+	issueIndexerQueue *queue.WorkerPoolQueue[*internal.IndexerData]
+	// globalIndexer is the global indexer, it cannot be nil.
+	// When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready.
+	// So it's always safe use it as *globalIndexer.Load() and call its methods.
+	globalIndexer atomic.Pointer[internal.Indexer]
+	dummyIndexer  *internal.Indexer
 )
 
+func init() {
+	i := internal.NewDummyIndexer()
+	dummyIndexer = &i
+	globalIndexer.Store(dummyIndexer)
+}
+
 // InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
 // all issue index done.
 func InitIssueIndexer(syncReindex bool) {
@@ -107,34 +52,24 @@ func InitIssueIndexer(syncReindex bool) {
 	// Create the Queue
 	switch setting.Indexer.IssueType {
 	case "bleve", "elasticsearch", "meilisearch":
-		handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
-			indexer := holder.get()
-			if indexer == nil {
-				log.Warn("Issue indexer handler: indexer is not ready, retry later.")
-				return items
-			}
-			toIndex := make([]*IndexerData, 0, len(items))
+		handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) {
+			indexer := *globalIndexer.Load()
+			toIndex := make([]*internal.IndexerData, 0, len(items))
 			for _, indexerData := range items {
 				log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
 				if indexerData.IsDelete {
-					if err := indexer.Delete(indexerData.IDs...); err != nil {
+					if err := indexer.Delete(ctx, indexerData.IDs...); err != nil {
 						log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err)
-						if !indexer.Ping() {
-							log.Error("Issue indexer handler: indexer is unavailable when deleting")
 						unhandled = append(unhandled, indexerData)
 					}
-					}
 					continue
 				}
 				toIndex = append(toIndex, indexerData)
 			}
-			if err := indexer.Index(toIndex); err != nil {
+			if err := indexer.Index(ctx, toIndex); err != nil {
 				log.Error("Error whilst indexing: %v Error: %v", toIndex, err)
-				if !indexer.Ping() {
-					log.Error("Issue indexer handler: indexer is unavailable when indexing")
 				unhandled = append(unhandled, toIndex...)
 			}
-			}
 			return unhandled
 		}
 
@@ -144,7 +79,7 @@ func InitIssueIndexer(syncReindex bool) {
 			log.Fatal("Unable to create issue indexer queue")
 		}
 	default:
-		issueIndexerQueue = queue.CreateSimpleQueue[*IndexerData](ctx, "issue_indexer", nil)
+		issueIndexerQueue = queue.CreateSimpleQueue[*internal.IndexerData](ctx, "issue_indexer", nil)
 	}
 
 	graceful.GetManager().RunAtTerminate(finished)
@@ -154,7 +89,11 @@ func InitIssueIndexer(syncReindex bool) {
 		pprof.SetGoroutineLabels(ctx)
 		start := time.Now()
 		log.Info("PID %d: Initializing Issue Indexer: %s", os.Getpid(), setting.Indexer.IssueType)
-		var populate bool
+		var (
+			issueIndexer internal.Indexer
+			existed      bool
+			err          error
+		)
 		switch setting.Indexer.IssueType {
 		case "bleve":
 			defer func() {
@@ -162,62 +101,45 @@ func InitIssueIndexer(syncReindex bool) {
 					log.Error("PANIC whilst initializing issue indexer: %v\nStacktrace: %s", err, log.Stack(2))
 					log.Error("The indexer files are likely corrupted and may need to be deleted")
 					log.Error("You can completely remove the %q directory to make Gitea recreate the indexes", setting.Indexer.IssuePath)
-					holder.cancel()
+					globalIndexer.Store(dummyIndexer)
 					log.Fatal("PID: %d Unable to initialize the Bleve Issue Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.IssuePath, err)
 				}
 			}()
-			issueIndexer := NewBleveIndexer(setting.Indexer.IssuePath)
-			exist, err := issueIndexer.Init()
+			issueIndexer = bleve.NewIndexer(setting.Indexer.IssuePath)
+			existed, err = issueIndexer.Init(ctx)
 			if err != nil {
-				holder.cancel()
 				log.Fatal("Unable to initialize Bleve Issue Indexer at path: %s Error: %v", setting.Indexer.IssuePath, err)
 			}
-			populate = !exist
-			holder.set(issueIndexer)
-			graceful.GetManager().RunAtTerminate(func() {
-				log.Debug("Closing issue indexer")
-				issueIndexer := holder.get()
-				if issueIndexer != nil {
-					issueIndexer.Close()
-				}
-				log.Info("PID: %d Issue Indexer closed", os.Getpid())
-			})
-			log.Debug("Created Bleve Indexer")
 		case "elasticsearch":
-			issueIndexer, err := NewElasticSearchIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName)
-			if err != nil {
-				log.Fatal("Unable to initialize Elastic Search Issue Indexer at connection: %s Error: %v", setting.Indexer.IssueConnStr, err)
-			}
-			exist, err := issueIndexer.Init()
+			issueIndexer = elasticsearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName)
+			existed, err = issueIndexer.Init(ctx)
 			if err != nil {
 				log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err)
 			}
-			populate = !exist
-			holder.set(issueIndexer)
 		case "db":
-			issueIndexer := &DBIndexer{}
-			holder.set(issueIndexer)
+			issueIndexer = db.NewIndexer()
 		case "meilisearch":
-			issueIndexer, err := NewMeilisearchIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName)
-			if err != nil {
-				log.Fatal("Unable to initialize Meilisearch Issue Indexer at connection: %s Error: %v", setting.Indexer.IssueConnStr, err)
-			}
-			exist, err := issueIndexer.Init()
+			issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName)
+			existed, err = issueIndexer.Init(ctx)
 			if err != nil {
 				log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err)
 			}
-			populate = !exist
-			holder.set(issueIndexer)
 		default:
-			holder.cancel()
 			log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
 		}
+		globalIndexer.Store(&issueIndexer)
+
+		graceful.GetManager().RunAtTerminate(func() {
+			log.Debug("Closing issue indexer")
+			(*globalIndexer.Load()).Close()
+			log.Info("PID: %d Issue Indexer closed", os.Getpid())
+		})
 
 		// Start processing the queue
 		go graceful.GetManager().RunWithCancel(issueIndexerQueue)
 
 		// Populate the index
-		if populate {
+		if !existed {
 			if syncReindex {
 				graceful.GetManager().RunWithShutdownContext(populateIssueIndexer)
 			} else {
@@ -266,8 +188,8 @@ func populateIssueIndexer(ctx context.Context) {
 		default:
 		}
 		repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{
-			ListOptions: db.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize},
-			OrderBy:     db.SearchOrderByID,
+			ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize},
+			OrderBy:     db_model.SearchOrderByID,
 			Private:     true,
 			Collaborate: util.OptionalBoolFalse,
 		})
@@ -320,7 +242,7 @@ func UpdateIssueIndexer(issue *issues_model.Issue) {
 			comments = append(comments, comment.Content)
 		}
 	}
-	indexerData := &IndexerData{
+	indexerData := &internal.IndexerData{
 		ID:       issue.ID,
 		RepoID:   issue.RepoID,
 		Title:    issue.Title,
@@ -345,7 +267,7 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) {
 	if len(ids) == 0 {
 		return
 	}
-	indexerData := &IndexerData{
+	indexerData := &internal.IndexerData{
 		IDs:      ids,
 		IsDelete: true,
 	}
@@ -358,12 +280,7 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) {
 // WARNNING: You have to ensure user have permission to visit repoIDs' issues
 func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) {
 	var issueIDs []int64
-	indexer := holder.get()
-
-	if indexer == nil {
-		log.Error("SearchIssuesByKeyword(): unable to get indexer!")
-		return nil, fmt.Errorf("unable to get issue indexer")
-	}
+	indexer := *globalIndexer.Load()
 	res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0)
 	if err != nil {
 		return nil, err
@@ -375,12 +292,6 @@ func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string)
 }
 
 // IsAvailable checks if issue indexer is available
-func IsAvailable() bool {
-	indexer := holder.get()
-	if indexer == nil {
-		log.Error("IsAvailable(): unable to get indexer!")
-		return false
-	}
-
-	return indexer.Ping()
+func IsAvailable(ctx context.Context) bool {
+	return (*globalIndexer.Load()).Ping(ctx) == nil
 }

modules/indexer/issues/indexer_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/indexer/issues/bleve"
 	"code.gitea.io/gitea/modules/setting"
 
 	_ "code.gitea.io/gitea/models"
@@ -42,8 +43,7 @@ func TestBleveSearchIssues(t *testing.T) {
 	setting.LoadQueueSettings()
 	InitIssueIndexer(true)
 	defer func() {
-		indexer := holder.get()
-		if bleveIndexer, ok := indexer.(*BleveIndexer); ok {
+		if bleveIndexer, ok := (*globalIndexer.Load()).(*bleve.Indexer); ok {
 			bleveIndexer.Close()
 		}
 	}()

modules/indexer/issues/internal/indexer.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+)
+
+// Indexer defines an interface to indexer issues contents
+type Indexer interface {
+	internal.Indexer
+	Index(ctx context.Context, issue []*IndexerData) error
+	Delete(ctx context.Context, ids ...int64) error
+	Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
+}
+
+// NewDummyIndexer returns a dummy indexer
+func NewDummyIndexer() Indexer {
+	return &dummyIndexer{
+		Indexer: internal.NewDummyIndexer(),
+	}
+}
+
+type dummyIndexer struct {
+	internal.Indexer
+}
+
+func (d *dummyIndexer) Index(ctx context.Context, issue []*IndexerData) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Delete(ctx context.Context, ids ...int64) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
+	return nil, fmt.Errorf("indexer is not ready")
+}

modules/indexer/issues/internal/model.go

@@ -0,0 +1,27 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+// IndexerData data stored in the issue indexer
+type IndexerData struct {
+	ID       int64    `json:"id"`
+	RepoID   int64    `json:"repo_id"`
+	Title    string   `json:"title"`
+	Content  string   `json:"content"`
+	Comments []string `json:"comments"`
+	IsDelete bool     `json:"is_delete"`
+	IDs      []int64  `json:"ids"`
+}
+
+// Match represents on search result
+type Match struct {
+	ID    int64   `json:"id"`
+	Score float64 `json:"score"`
+}
+
+// SearchResult represents search results
+type SearchResult struct {
+	Total int64
+	Hits  []Match
+}

modules/indexer/issues/meilisearch.go

@@ -1,173 +0,0 @@
-// Copyright 2023 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package issues
-
-import (
-	"context"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/meilisearch/meilisearch-go"
-)
-
-var _ Indexer = &MeilisearchIndexer{}
-
-// MeilisearchIndexer implements Indexer interface
-type MeilisearchIndexer struct {
-	client      *meilisearch.Client
-	indexerName string
-	available   bool
-	stopTimer   chan struct{}
-	lock        sync.RWMutex
-}
-
-// MeilisearchIndexer creates a new meilisearch indexer
-func NewMeilisearchIndexer(url, apiKey, indexerName string) (*MeilisearchIndexer, error) {
-	client := meilisearch.NewClient(meilisearch.ClientConfig{
-		Host:   url,
-		APIKey: apiKey,
-	})
-
-	indexer := &MeilisearchIndexer{
-		client:      client,
-		indexerName: indexerName,
-		available:   true,
-		stopTimer:   make(chan struct{}),
-	}
-
-	ticker := time.NewTicker(10 * time.Second)
-	go func() {
-		for {
-			select {
-			case <-ticker.C:
-				indexer.checkAvailability()
-			case <-indexer.stopTimer:
-				ticker.Stop()
-				return
-			}
-		}
-	}()
-
-	return indexer, nil
-}
-
-// Init will initialize the indexer
-func (b *MeilisearchIndexer) Init() (bool, error) {
-	_, err := b.client.GetIndex(b.indexerName)
-	if err == nil {
-		return true, nil
-	}
-	_, err = b.client.CreateIndex(&meilisearch.IndexConfig{
-		Uid:        b.indexerName,
-		PrimaryKey: "id",
-	})
-	if err != nil {
-		return false, b.checkError(err)
-	}
-
-	_, err = b.client.Index(b.indexerName).UpdateFilterableAttributes(&[]string{"repo_id"})
-	return false, b.checkError(err)
-}
-
-// Ping checks if meilisearch is available
-func (b *MeilisearchIndexer) Ping() bool {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	return b.available
-}
-
-// Index will save the index data
-func (b *MeilisearchIndexer) Index(issues []*IndexerData) error {
-	if len(issues) == 0 {
-		return nil
-	}
-	for _, issue := range issues {
-		_, err := b.client.Index(b.indexerName).AddDocuments(issue)
-		if err != nil {
-			return b.checkError(err)
-		}
-	}
-	// TODO: bulk send index data
-	return nil
-}
-
-// Delete deletes indexes by ids
-func (b *MeilisearchIndexer) Delete(ids ...int64) error {
-	if len(ids) == 0 {
-		return nil
-	}
-
-	for _, id := range ids {
-		_, err := b.client.Index(b.indexerName).DeleteDocument(strconv.FormatInt(id, 10))
-		if err != nil {
-			return b.checkError(err)
-		}
-	}
-	// TODO: bulk send deletes
-	return nil
-}
-
-// Search searches for issues by given conditions.
-// Returns the matching issue IDs
-func (b *MeilisearchIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
-	repoFilters := make([]string, 0, len(repoIDs))
-	for _, repoID := range repoIDs {
-		repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10))
-	}
-	filter := strings.Join(repoFilters, " OR ")
-	searchRes, err := b.client.Index(b.indexerName).Search(keyword, &meilisearch.SearchRequest{
-		Filter: filter,
-		Limit:  int64(limit),
-		Offset: int64(start),
-	})
-	if err != nil {
-		return nil, b.checkError(err)
-	}
-
-	hits := make([]Match, 0, len(searchRes.Hits))
-	for _, hit := range searchRes.Hits {
-		hits = append(hits, Match{
-			ID: int64(hit.(map[string]interface{})["id"].(float64)),
-		})
-	}
-	return &SearchResult{
-		Total: searchRes.TotalHits,
-		Hits:  hits,
-	}, nil
-}
-
-// Close implements indexer
-func (b *MeilisearchIndexer) Close() {
-	select {
-	case <-b.stopTimer:
-	default:
-		close(b.stopTimer)
-	}
-}
-
-func (b *MeilisearchIndexer) checkError(err error) error {
-	return err
-}
-
-func (b *MeilisearchIndexer) checkAvailability() {
-	_, err := b.client.Health()
-	if err != nil {
-		b.setAvailability(false)
-		return
-	}
-	b.setAvailability(true)
-}
-
-func (b *MeilisearchIndexer) setAvailability(available bool) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.available == available {
-		return
-	}
-
-	b.available = available
-}

modules/indexer/issues/meilisearch/meilisearch.go

@@ -0,0 +1,98 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package meilisearch
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_meilisearch "code.gitea.io/gitea/modules/indexer/internal/meilisearch"
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
+
+	"github.com/meilisearch/meilisearch-go"
+)
+
+const (
+	issueIndexerLatestVersion = 0
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer implements Indexer interface
+type Indexer struct {
+	inner                    *inner_meilisearch.Indexer
+	indexer_internal.Indexer // do not composite inner_meilisearch.Indexer directly to avoid exposing too much
+}
+
+// NewIndexer creates a new meilisearch indexer
+func NewIndexer(url, apiKey, indexerName string) *Indexer {
+	inner := inner_meilisearch.NewIndexer(url, apiKey, indexerName, issueIndexerLatestVersion)
+	indexer := &Indexer{
+		inner:   inner,
+		Indexer: inner,
+	}
+	return indexer
+}
+
+// Index will save the index data
+func (b *Indexer) Index(_ context.Context, issues []*internal.IndexerData) error {
+	if len(issues) == 0 {
+		return nil
+	}
+	for _, issue := range issues {
+		_, err := b.inner.Client.Index(b.inner.VersionedIndexName()).AddDocuments(issue)
+		if err != nil {
+			return err
+		}
+	}
+	// TODO: bulk send index data
+	return nil
+}
+
+// Delete deletes indexes by ids
+func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
+	if len(ids) == 0 {
+		return nil
+	}
+
+	for _, id := range ids {
+		_, err := b.inner.Client.Index(b.inner.VersionedIndexName()).DeleteDocument(strconv.FormatInt(id, 10))
+		if err != nil {
+			return err
+		}
+	}
+	// TODO: bulk send deletes
+	return nil
+}
+
+// Search searches for issues by given conditions.
+// Returns the matching issue IDs
+func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
+	repoFilters := make([]string, 0, len(repoIDs))
+	for _, repoID := range repoIDs {
+		repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10))
+	}
+	filter := strings.Join(repoFilters, " OR ")
+	searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{
+		Filter: filter,
+		Limit:  int64(limit),
+		Offset: int64(start),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	hits := make([]internal.Match, 0, len(searchRes.Hits))
+	for _, hit := range searchRes.Hits {
+		hits = append(hits, internal.Match{
+			ID: int64(hit.(map[string]interface{})["id"].(float64)),
+		})
+	}
+	return &internal.SearchResult{
+		Total: searchRes.TotalHits,
+		Hits:  hits,
+	}, nil
+}