corrected package blob structure, pacman database is created on get request automatically from package metadata in database

docs/content/doc/usage/packages/arch.en-us.md

@@ -141,11 +141,9 @@ curl -X DELETE \
   http://localhost:3000/api/packages/{user}/arch/remove \
   --header "username: {user}" \
   --header "email: user@email.com" \
-  --header "distro: archlinux" \
   --header "target: package" \
   --header "time: {rmtime}" \
   --header "version: {version-release}" \
-  --header "arch: x86_64" \
   --header 'Content-Type: application/octet-stream' \
   --data-binary @md.sig
 ```

modules/packages/arch/metadata.go

@@ -21,31 +21,32 @@ import (
 
 // Metadata for arch package.
 type Metadata struct {
-	Filename       string
+	Filename       string   `json:"filename"`
-	Name           string
+	Name           string   `json:"name"`
-	Base           string
+	Base           string   `json:"base"`
-	Version        string
+	Version        string   `json:"version"`
-	Description    string
+	Description    string   `json:"description"`
-	CompressedSize int64
+	CompressedSize int64    `json:"compressed-size"`
-	InstalledSize  int64
+	InstalledSize  int64    `json:"installed-size"`
-	MD5            string
+	MD5            string   `json:"md5"`
-	SHA256         string
+	SHA256         string   `json:"sha256"`
-	URL            string
+	URL            string   `json:"url"`
-	BuildDate      int64
+	BuildDate      int64    `json:"build-date"`
-	BaseDomain     string
+	BaseDomain     string   `json:"base-domain"`
-	Packager       string
+	Packager       string   `json:"packager"`
-	Provides       []string
+	Distribution   string   `json:"distribution"`
-	License        []string
+	Provides       []string `json:"provides"`
-	Arch           []string
+	License        []string `json:"license"`
-	Depends        []string
+	Arch           []string `json:"arch"`
-	OptDepends     []string
+	Depends        []string `json:"depends"`
-	MakeDepends    []string
+	OptDepends     []string `json:"opt-depends"`
-	CheckDepends   []string
+	MakeDepends    []string `json:"make-depends"`
-	Backup         []string
+	CheckDepends   []string `json:"check-depends"`
+	Backup         []string `json:"backup"`
 }
 
 // Function that recieves arch package archive data and returns it's metadata.
-func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) {
+func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata, error) {
 	pkginfo, err := getPkginfo(pkg)
 	if err != nil {
 		return nil, err
@@ -56,6 +57,7 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) {
 		CompressedSize: int64(len(pkg)),
 		MD5:            md5sum(pkg),
 		SHA256:         sha256sum(pkg),
+		Distribution:   distribution,
 	}
 	for _, line := range strings.Split(pkginfo, "\n") {
 		splt := strings.Split(line, " = ")
@@ -211,24 +213,18 @@ func Join(s ...string) string {
 	return rez
 }
 
-// Add or update existing package entry in database archived data.
+// Create pacman database archive based on provided package metadata structs.
-func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) {
+func CreatePacmanDb(mds []*Metadata) ([]byte, error) {
-	// Read existing entries in archive.
+	entries := make(map[string][]byte)
-	entries, err := readEntries(db)
+
-	if err != nil {
+	for _, md := range mds {
-		return nil, err
+		entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc())
 	}
 
-	// Remove entries related old package versions.
-	entries = CleanOldEntries(entries, md.Name)
-
-	// Add new package entry to list.
-	entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc())
-
 	var out bytes.Buffer
 
 	// Write entries to new buffer and return it.
-	err = writeToArchive(entries, &out)
+	err := writeToArchive(entries, &out)
 	if err != nil {
 		return nil, err
 	}
@@ -236,73 +232,7 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) {
 	return out.Bytes(), nil
 }
 
-// Clean entries for old package versions from pacman database.
+// Write pacman package entries to tarball.
-func CleanOldEntries(entries map[string][]byte, pkg string) map[string][]byte {
-	out := map[string][]byte{}
-	for entry, value := range entries {
-		splt := strings.Split(entry, "-")
-		basename := strings.Join(splt[0:len(splt)-2], "-")
-		if pkg != basename {
-			out[entry] = value
-		}
-	}
-	return out
-}
-
-// Add or update existing package entry in database archived data.
-func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) {
-	// Read existing entries in archive.
-	entries, err := readEntries(db)
-	if err != nil {
-		return nil, err
-	}
-
-	// Add new package entry to list.
-	delete(entries, pkg+"-"+ver+"/desc")
-
-	var out bytes.Buffer
-
-	// Write entries to new buffer and return it.
-	err = writeToArchive(entries, &out)
-	if err != nil {
-		return nil, err
-	}
-
-	return out.Bytes(), nil
-}
-
-// Read database entries containing in pacman archive.
-func readEntries(dbarchive []byte) (map[string][]byte, error) {
-	gzf, err := gzip.NewReader(bytes.NewReader(dbarchive))
-	if err != nil {
-		return map[string][]byte{}, nil
-	}
-
-	var entries = map[string][]byte{}
-
-	tarReader := tar.NewReader(gzf)
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		}
-
-		if err != nil {
-			return nil, err
-		}
-
-		if header.Typeflag == tar.TypeReg {
-			content, err := io.ReadAll(tarReader)
-			if err != nil {
-				return nil, err
-			}
-			entries[header.Name] = content
-		}
-	}
-	return entries, nil
-}
-
-// Write pacman package entries to empty buffer.
 func writeToArchive(files map[string][]byte, buf io.Writer) error {
 	gw := gzip.NewWriter(buf)
 	defer gw.Close()

routers/api/packages/arch/arch.go

@@ -4,6 +4,7 @@
 package arch
 
 import (
+	"bytes"
 	"encoding/hex"
 	"io"
 	"net/http"
@@ -87,7 +88,7 @@ func Push(ctx *context.Context) {
 	}
 
 	// Parse metadata contained in arch package archive.
-	md, err := arch_module.EjectMetadata(filename, setting.Domain, pkgdata)
+	md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata)
 	if err != nil {
 		apiError(ctx, http.StatusBadRequest, err)
 		return
@@ -121,20 +122,6 @@ func Push(ctx *context.Context) {
 		return
 	}
 
-	// Save file related to arch package description.
-	_, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{
-		Organization: org,
-		User:         user,
-		Metadata:     md,
-		Data:         []byte(md.GetDbDesc()),
-		Filename:     filename + ".desc",
-		Distro:       distro,
-	})
-	if err != nil {
-		apiError(ctx, http.StatusInternalServerError, err)
-		return
-	}
-
 	// Automatically connect repository for provided package if name matched.
 	err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid)
 	if err != nil {
@@ -157,38 +144,32 @@ func Get(ctx *context.Context) {
 	// Packages are stored in different way from pacman databases, and loaded
 	// with LoadPackageFile function.
 	if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") {
-		pkgdata, err := arch_service.LoadPackageFile(ctx, distro, file)
+		pkgdata, err := arch_service.LoadFile(ctx, distro, file)
 		if err != nil {
 			apiError(ctx, http.StatusNotFound, err)
 			return
 		}
 
-		_, err = ctx.Resp.Write(pkgdata)
+		ctx.ServeContent(bytes.NewReader(pkgdata), &context.ServeHeaderOptions{
-		if err != nil {
+			Filename:      file,
-			apiError(ctx, http.StatusInternalServerError, err)
+			CacheDuration: time.Minute * 5,
-			return
+		})
-		}
-
-		ctx.Resp.WriteHeader(http.StatusOK)
 		return
 	}
 
-	// Pacman databases are stored directly in gitea file storage and could be
+	// Pacman databases is not stored in gitea's storage, it is created for
-	// loaded with name as a key.
+	// incoming request and cached.
 	if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") {
-		data, err := arch_service.LoadPacmanDatabase(ctx, owner, distro, arch, file)
+		db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro)
-		if err != nil {
-			apiError(ctx, http.StatusNotFound, err)
-			return
-		}
-
-		_, err = ctx.Resp.Write(data)
 		if err != nil {
 			apiError(ctx, http.StatusInternalServerError, err)
 			return
 		}
 
-		ctx.Resp.WriteHeader(http.StatusOK)
+		ctx.ServeContent(bytes.NewReader(db), &context.ServeHeaderOptions{
+			Filename:      file,
+			CacheDuration: time.Minute * 5,
+		})
 		return
 	}
 
@@ -200,11 +181,9 @@ func Remove(ctx *context.Context) {
 	var (
 		owner   = ctx.Params("username")
 		email   = ctx.Req.Header.Get("email")
-		distro  = ctx.Req.Header.Get("distro")
 		target  = ctx.Req.Header.Get("target")
 		stime   = ctx.Req.Header.Get("time")
 		version = ctx.Req.Header.Get("version")
-		arch    = strings.Split(ctx.Req.Header.Get("arch"), " ")
 	)
 
 	// Parse sent time and check if it is within last minute.
@@ -242,21 +221,8 @@ func Remove(ctx *context.Context) {
 		return
 	}
 
-	// Remove pacman database entries related to package.
-	err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version)
-	if err != nil {
-		apiError(ctx, http.StatusInternalServerError, err)
-		return
-	}
-
 	// Remove package files and pacman database entry.
-	err = arch_service.RemovePackage(ctx, &arch_service.RemoveParameters{
+	err = arch_service.RemovePackage(ctx, org.AsUser(), target, version)
-		User:         user,
-		Organization: org,
-		Owner:        owner,
-		Name:         target,
-		Version:      version,
-	})
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
 		return

services/packages/arch/db_manager.go

@@ -5,25 +5,31 @@ package arch
 
 import (
 	"bytes"
+	"fmt"
+	"io"
+	"strings"
 
-	org "code.gitea.io/gitea/models/organization"
+	"code.gitea.io/gitea/models/db"
-	pkg "code.gitea.io/gitea/models/packages"
+	org_model "code.gitea.io/gitea/models/organization"
+	pkg_model "code.gitea.io/gitea/models/packages"
 	"code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/packages"
 	"code.gitea.io/gitea/modules/packages/arch"
-	svc "code.gitea.io/gitea/services/packages"
+	pkg_service "code.gitea.io/gitea/services/packages"
 )
 
 // Parameters required to save new arch package.
 type SaveFileParams struct {
-	*org.Organization
+	*org_model.Organization
 	*user.User
 	*arch.Metadata
 	Data     []byte
 	Filename string
 	Distro   string
+	IsLead   bool
 }
 
 // This function create new package, version and package file properties in
@@ -36,25 +42,26 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
 	}
 	defer buf.Close()
 
-	pv, _, err := svc.CreatePackageOrAddFileToExisting(
+	pv, _, err := pkg_service.CreatePackageOrAddFileToExisting(
-		&svc.PackageCreationInfo{
+		&pkg_service.PackageCreationInfo{
-			PackageInfo: svc.PackageInfo{
+			PackageInfo: pkg_service.PackageInfo{
 				Owner:       p.Organization.AsUser(),
-				PackageType: pkg.TypeArch,
+				PackageType: pkg_model.TypeArch,
 				Name:        p.Metadata.Name,
 				Version:     p.Metadata.Version,
 			},
 			Creator:  p.User,
 			Metadata: p.Metadata,
 		},
-		&svc.PackageFileCreationInfo{
+		&pkg_service.PackageFileCreationInfo{
-			PackageFileInfo: svc.PackageFileInfo{
+			PackageFileInfo: pkg_service.PackageFileInfo{
 				Filename:     p.Filename,
 				CompositeKey: p.Distro + "-" + p.Filename,
 			},
 			Creator:           p.User,
 			Data:              buf,
 			OverwriteExisting: true,
+			IsLead:            p.IsLead,
 		},
 	)
 	if err != nil {
@@ -63,12 +70,46 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
 	return pv.PackageID, nil
 }
 
+// Get data related to provided file name and distribution, and update download
+// counter if actual package file is retrieved from database.
+func LoadFile(ctx *context.Context, distro, file string) ([]byte, error) {
+	db := db.GetEngine(ctx)
+
+	pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file}
+
+	ok, err := db.Get(pkgfile)
+	if err != nil || !ok {
+		return nil, fmt.Errorf("%+v %t", err, ok)
+	}
+
+	blob, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID)
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.HasSuffix(file, ".pkg.tar.zst") {
+		err = pkg_model.IncrementDownloadCounter(ctx, pkgfile.VersionID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	cs := packages.NewContentStore()
+
+	obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256))
+	if err != nil {
+		return nil, err
+	}
+
+	return io.ReadAll(obj)
+}
+
 // Automatically connect repository to pushed package, if package with provided
 // with provided name exists in namespace scope.
 func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error {
 	repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository)
 	if err == nil {
-		err = pkg.SetRepositoryLink(ctx, pkgid, repo.ID)
+		err = pkg_model.SetRepositoryLink(ctx, pkgid, repo.ID)
 		if err != nil {
 			return err
 		}
@@ -76,19 +117,61 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid
 	return nil
 }
 
-type RemoveParameters struct {
+// This function is collecting information about packages in some organization/
-	*user.User
+// user space, and created pacman database archive based on package metadata.
-	*org.Organization
+func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([]byte, error) {
-	Owner   string
+	u, err := user.GetUserByName(ctx, owner)
-	Name    string
+	if err != nil {
-	Version string
+		return nil, err
+	}
+
+	pkgs, err := pkg_model.GetPackagesByType(ctx, u.ID, pkg_model.TypeArch)
+	if err != nil {
+		return nil, err
+	}
+
+	var mds []*arch.Metadata
+
+	for _, pkg := range pkgs {
+		vers, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name)
+		if err != nil {
+			return nil, err
+		}
+		for i := len(vers) - 1; i >= 0; i-- {
+			var md arch.Metadata
+			err = json.Unmarshal([]byte(vers[i].MetadataJSON), &md)
+			if err != nil {
+				return nil, err
+			}
+			if checkArchitecture(architecture, md.Arch) && md.Distribution == distro {
+				mds = append(mds, &md)
+				break
+			}
+		}
+	}
+
+	return arch.CreatePacmanDb(mds)
 }
 
-// Remove package and it's blobs from gitea.
+// Remove specific package version related to provided user or organization.
-func RemovePackage(ctx *context.Context, p *RemoveParameters) error {
+func RemovePackage(ctx *context.Context, u *user.User, name, version string) error {
-	ver, err := pkg.GetVersionByNameAndVersion(ctx, p.Organization.ID, pkg.TypeArch, p.Name, p.Version)
+	ver, err := pkg_model.GetVersionByNameAndVersion(ctx, u.ID, pkg_model.TypeArch, name, version)
 	if err != nil {
 		return err
 	}
-	return svc.RemovePackageVersion(p.User, ver)
+
+	return pkg_service.RemovePackageVersion(u, ver)
+}
+
+// Check wether package architecture is relevant for requestsed database.
+func checkArchitecture(architecture string, list []string) bool {
+	for _, v := range list {
+		if v == "any" {
+			return true
+		}
+		if v == architecture {
+			return true
+		}
+	}
+	return false
 }

services/packages/arch/file_manager.go

@@ -1,153 +0,0 @@
-// Copyright 2023 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package arch
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"strings"
-
-	"code.gitea.io/gitea/models/db"
-	pkg_mdl "code.gitea.io/gitea/models/packages"
-	"code.gitea.io/gitea/modules/context"
-	"code.gitea.io/gitea/modules/packages"
-	"code.gitea.io/gitea/modules/packages/arch"
-	"code.gitea.io/gitea/modules/setting"
-)
-
-// Get data related to provided file name and distribution, and update download
-// counter if actual package file is retrieved from database.
-func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) {
-	db := db.GetEngine(ctx)
-
-	pkgfile := &pkg_mdl.PackageFile{CompositeKey: distro + "-" + file}
-
-	ok, err := db.Get(pkgfile)
-	if err != nil || !ok {
-		return nil, fmt.Errorf("%+v %t", err, ok)
-	}
-
-	blob, err := pkg_mdl.GetBlobByID(ctx, pkgfile.BlobID)
-	if err != nil {
-		return nil, err
-	}
-
-	if strings.HasSuffix(file, ".pkg.tar.zst") {
-		err = pkg_mdl.IncrementDownloadCounter(ctx, pkgfile.VersionID)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	cs := packages.NewContentStore()
-
-	obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256))
-	if err != nil {
-		return nil, err
-	}
-
-	return io.ReadAll(obj)
-}
-
-// Get data related to pacman database file or symlink.
-func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file string) ([]byte, error) {
-	cs := packages.NewContentStore()
-
-	file = strings.TrimPrefix(file, owner+".")
-
-	dbname := strings.TrimSuffix(arch.Join(owner, distro, architecture, file), ".tar.gz")
-
-	obj, err := cs.Get(packages.BlobHash256Key(dbname))
-	if err != nil {
-		return nil, err
-	}
-
-	return io.ReadAll(obj)
-}
-
-// This function will update information about package in related pacman databases
-// or create them if they do not exist.
-func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owner string) error {
-	// If architecure is not specified or any, package will be automatically
-	// saved to pacman databases with most popular architectures.
-	if len(md.Arch) == 0 || md.Arch[0] == "any" {
-		md.Arch = popularArchitectures()
-	}
-
-	cs := packages.NewContentStore()
-
-	// Update pacman database files for each architecture.
-	for _, architecture := range md.Arch {
-		db := arch.Join(owner, distro, architecture, setting.Domain, "db")
-		dbkey := packages.BlobHash256Key(db)
-
-		var dbdata []byte
-
-		dbobj, err := cs.Get(dbkey)
-		if err == nil {
-			dbdata, err = io.ReadAll(dbobj)
-			if err != nil {
-				return err
-			}
-		}
-
-		newdata, err := arch.UpdatePacmanDbEntry(dbdata, md)
-		if err != nil {
-			return err
-		}
-
-		err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata)))
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func RemoveDbEntry(ctx *context.Context, architectures []string, owner, distro, pkg, ver string) error {
-	cs := packages.NewContentStore()
-
-	// If architecures are not specified or any, package will be automatically
-	// removed from pacman databases with most popular architectures.
-	if len(architectures) == 0 || architectures[0] == "any" {
-		architectures = popularArchitectures()
-	}
-
-	for _, architecture := range architectures {
-		db := arch.Join(owner, distro, architecture, setting.Domain, "db")
-		dbkey := packages.BlobHash256Key(db)
-
-		var dbdata []byte
-
-		dbobj, err := cs.Get(dbkey)
-		if err != nil {
-			return err
-		}
-
-		dbdata, err = io.ReadAll(dbobj)
-		if err != nil {
-			return err
-		}
-
-		newdata, err := arch.RemoveDbEntry(dbdata, pkg, ver)
-		if err != nil {
-			return err
-		}
-
-		err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata)))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func popularArchitectures() []string {
-	return []string{
-		"x86_64", "arm", "i686", "pentium4",
-		"armv7h", "armv6h", "aarch64", "riscv64",
-	}
-}