changed pacman database creation mechanics, storing package desc as file in object storage, metadata structure update

modules/packages/arch/metadata.go

@@ -5,10 +5,9 @@ package arch
 
 import (
 	"archive/tar"
+	"bufio"
 	"bytes"
 	"compress/gzip"
-	"crypto/md5"
-	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -17,11 +16,30 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/container"
+	pkg_module "code.gitea.io/gitea/modules/packages"
 	"github.com/mholt/archiver/v3"
 )
 
-// Metadata for arch package.
+// JSON with pacakage parameters that are not related to specific
+// architecture/distribution that will be stored in sql database.
 type Metadata struct {
+	Name         string
+	Version      string
+	URL          string   `json:"url"`
+	Description  string   `json:"description"`
+	Provides     []string `json:"provides,omitempty"`
+	License      []string `json:"license,omitempty"`
+	Depends      []string `json:"depends,omitempty"`
+	OptDepends   []string `json:"opt-depends,omitempty"`
+	MakeDepends  []string `json:"make-depends,omitempty"`
+	CheckDepends []string `json:"check-depends,omitempty"`
+	Backup       []string `json:"backup,omitempty"`
+	DistroArch   []string `json:"distro-arch,omitempty"`
+}
+
+// Package description file that will be saved as .desc file in object storage.
+// Resulting file will be used to create pacman database.
+type DbDesc struct {
 	Filename       string   `json:"filename"`
 	Name           string   `json:"name"`
 	Base           string   `json:"base"`
@@ -34,33 +52,31 @@ type Metadata struct {
 	URL            string   `json:"url"`
 	BuildDate      int64    `json:"build-date"`
 	Packager       string   `json:"packager"`
-	Provides       []string `json:"provides"`
-	License        []string `json:"license"`
-	Arch           []string `json:"arch"`
-	Depends        []string `json:"depends"`
-	OptDepends     []string `json:"opt-depends"`
-	MakeDepends    []string `json:"make-depends"`
-	CheckDepends   []string `json:"check-depends"`
-	Backup         []string `json:"backup"`
-	// This list is created to ensure the consistency of pacman database file
-	// for specific combination of distribution and architecture. This value
-	// is used when creating pacman database, to ensure that only packages
-	// with specific combination of distribution and architecture are present
-	// in pacman database file.
-	DistroArch []string `json:"distro-arch"`
+	Provides       []string `json:"provides,omitempty"`
+	License        []string `json:"license,omitempty"`
+	Arch           []string `json:"arch,omitempty"`
+	Depends        []string `json:"depends,omitempty"`
+	OptDepends     []string `json:"opt-depends,omitempty"`
+	MakeDepends    []string `json:"make-depends,omitempty"`
+	CheckDepends   []string `json:"check-depends,omitempty"`
+	Backup         []string `json:"backup,omitempty"`
 }
 
 // Function that receives arch package archive data and returns it's metadata.
-func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) {
-	pkginfo, err := getPkginfo(pkg)
+func EjectMetadata(filename, distribution string, buf *pkg_module.HashedBuffer) (*DbDesc, error) {
+	pkginfo, err := getPkginfo(buf)
 	if err != nil {
 		return nil, err
 	}
-	md := Metadata{
+
+	// Add package blob parameters to arch related desc.
+	hashMD5, _, hashSHA256, _ := buf.Sums()
+	md := DbDesc{
 		Filename:       filename,
-		CompressedSize: int64(len(pkg)),
-		MD5:            md5sum(pkg),
-		SHA256:         sha256sum(pkg),
+		Name:           filename,
+		CompressedSize: buf.Size(),
+		MD5:            hex.EncodeToString(hashMD5),
+		SHA256:         hex.EncodeToString(hashSHA256),
 	}
 	for _, line := range strings.Split(pkginfo, "\n") {
 		splt := strings.Split(line, " = ")
@@ -98,7 +114,6 @@ func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error)
 			md.License = append(md.License, splt[1])
 		case "arch":
 			md.Arch = append(md.Arch, splt[1])
-			md.DistroArch = append(md.DistroArch, distribution+"-"+splt[1])
 		case "depend":
 			md.Depends = append(md.Depends, splt[1])
 		case "optdepend":
@@ -114,10 +129,10 @@ func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error)
 	return &md, nil
 }
 
-func getPkginfo(data []byte) (string, error) {
-	pkgreader := bytes.NewReader(data)
+func getPkginfo(data io.Reader) (string, error) {
+	br := bufio.NewReader(data)
 	zstd := archiver.NewTarZstd()
-	err := zstd.Open(pkgreader, int64(250000))
+	err := zstd.Open(br, int64(250000))
 	if err != nil {
 		return ``, err
 	}
@@ -137,19 +152,9 @@ func getPkginfo(data []byte) (string, error) {
 	}
 }
 
-func md5sum(data []byte) string {
-	sum := md5.Sum(data)
-	return hex.EncodeToString(sum[:])
-}
-
-func sha256sum(data []byte) string {
-	sum := sha256.Sum256(data)
-	return hex.EncodeToString(sum[:])
-}
-
 // This function returns pacman package description in unarchived raw database
 // format.
-func (m *Metadata) GetDbDesc() string {
+func (m *DbDesc) GetDbDesc() string {
 	return strings.Join(rmEmptyStrings([]string{
 		formatField("FILENAME", m.Filename),
 		formatField("NAME", m.Name),
@@ -200,13 +205,7 @@ func rmEmptyStrings(s []string) []string {
 }
 
 // Create pacman database archive based on provided package metadata structs.
-func CreatePacmanDb(mds []*Metadata) ([]byte, error) {
-	entries := make(map[string][]byte)
-
-	for _, md := range mds {
-		entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc())
-	}
-
+func CreatePacmanDb(entries map[string][]byte) ([]byte, error) {
 	var out bytes.Buffer
 
 	// Write entries to new buffer and return it.

routers/api/packages/arch/arch.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/context"
+	pkg_module "code.gitea.io/gitea/modules/packages"
 	arch_module "code.gitea.io/gitea/modules/packages/arch"
 	"code.gitea.io/gitea/routers/api/packages/helper"
 	arch_service "code.gitea.io/gitea/services/packages/arch"
@@ -25,28 +26,79 @@ func Push(ctx *context.Context) {
 		sign     = ctx.Params("sign")
 	)
 
-	// Read package to memory for package validation.
-	pkgdata, err := io.ReadAll(ctx.Req.Body)
+	upload, close, err := ctx.UploadStream()
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
-	defer ctx.Req.Body.Close()
+	if close {
+		defer upload.Close()
+	}
 
-	// Parse metadata contained in arch package archive.
-	md, err := arch_module.EjectMetadata(filename, distro, pkgdata)
+	buf, err := pkg_module.CreateHashedBufferFromReader(upload)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	// Parse metadata related to package contained in arch package archive.
+	pkgmd, err := arch_module.EjectMetadata(filename, distro, buf)
 	if err != nil {
 		apiError(ctx, http.StatusBadRequest, err)
 		return
 	}
 
+	if _, err := buf.Seek(0, io.SeekStart); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	// Metadata related to SQL database.
+	dbmd := &arch_module.Metadata{
+		Name:         pkgmd.Name,
+		Version:      pkgmd.Version,
+		URL:          pkgmd.URL,
+		Description:  pkgmd.Description,
+		Provides:     pkgmd.Provides,
+		License:      pkgmd.License,
+		Depends:      pkgmd.Depends,
+		OptDepends:   pkgmd.OptDepends,
+		MakeDepends:  pkgmd.MakeDepends,
+		CheckDepends: pkgmd.CheckDepends,
+		Backup:       pkgmd.Backup,
+		DistroArch:   []string{distro + "-" + pkgmd.Arch[0]},
+	}
+
 	// Save file related to arch package.
 	pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{
 		Creator:  ctx.ContextUser,
 		Owner:    ctx.Package.Owner,
-		Metadata: md,
 		Filename: filename,
-		Data:     pkgdata,
+		Buf:      buf,
+		Metadata: dbmd,
+		Distro:   distro,
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	r := io.NopCloser(bytes.NewReader([]byte(pkgmd.GetDbDesc())))
+	buf, err = pkg_module.CreateHashedBufferFromReader(r)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	// Save file related to arch package description for pacman database.
+	_, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{
+		Creator:  ctx.ContextUser,
+		Owner:    ctx.Package.Owner,
+		Filename: pkgmd.Name + "-" + pkgmd.Version + "-" + pkgmd.Arch[0] + ".desc",
+		Buf:      buf,
+		Metadata: dbmd,
 		Distro:   distro,
 	})
 	if err != nil {
@@ -57,12 +109,20 @@ func Push(ctx *context.Context) {
 	// Decoding package signature, if present saving with package as file.
 	sigdata, err := hex.DecodeString(sign)
 	if err == nil {
+		r := io.NopCloser(bytes.NewReader(sigdata))
+		buf, err := pkg_module.CreateHashedBufferFromReader(r)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+		defer buf.Close()
+
 		_, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{
 			Creator:  ctx.ContextUser,
 			Owner:    ctx.Package.Owner,
-			Metadata: md,
-			Data:     sigdata,
+			Buf:      buf,
 			Filename: filename + ".sig",
+			Metadata: dbmd,
 			Distro:   distro,
 		})
 		if err != nil {
@@ -74,7 +134,7 @@ func Push(ctx *context.Context) {
 	// Add new architectures and distribution info to package version metadata.
 	err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{
 		User: ctx.Package.Owner,
-		Md:   md,
+		Md:   dbmd,
 	})
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
@@ -82,7 +142,7 @@ func Push(ctx *context.Context) {
 	}
 
 	// Automatically connect repository for provided package if name matched.
-	err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid)
+	err = arch_service.RepositoryAutoconnect(ctx, owner, dbmd.Name, pkgid)
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
 		return

services/packages/arch/db_manager.go

@@ -4,8 +4,8 @@
 package arch
 
 import (
-	"bytes"
 	"fmt"
+	"io"
 	"sort"
 	"strings"
 
@@ -41,7 +41,6 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error {
 		return err
 	}
 
-	currmd.Arch = arch.UnifiedList(currmd.Arch, p.Md.Arch)
 	currmd.DistroArch = arch.UnifiedList(currmd.DistroArch, p.Md.DistroArch)
 
 	b, err := json.Marshal(&currmd)
@@ -59,7 +58,7 @@ type SaveFileParams struct {
 	Creator  *user.User
 	Owner    *user.User
 	Metadata *arch.Metadata
-	Data     []byte
+	Buf      packages.HashedSizeReader
 	Filename string
 	Distro   string
 	IsLead   bool
@@ -69,13 +68,7 @@ type SaveFileParams struct {
 // database, and writes blob to file storage. If package/version/blob exists it
 // will overwrite existing data. Package id and error will be returned.
 func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
-	buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(p.Data))
-	if err != nil {
-		return 0, err
-	}
-	defer buf.Close()
-
-	pv, _, err := pkg_service.CreatePackageOrAddFileToExisting(
+	ver, _, err := pkg_service.CreatePackageOrAddFileToExisting(
 		&pkg_service.PackageCreationInfo{
 			PackageInfo: pkg_service.PackageInfo{
 				Owner:       p.Owner,
@@ -92,7 +85,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
 				CompositeKey: p.Distro + "-" + p.Filename,
 			},
 			Creator:           p.Creator,
-			Data:              buf,
+			Data:              p.Buf,
 			OverwriteExisting: true,
 			IsLead:            p.IsLead,
 		},
@@ -100,7 +93,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return pv.PackageID, nil
+	return ver.ID, nil
 }
 
 // Get data related to provided file name and distribution, and update download
@@ -158,7 +151,7 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([
 		return nil, err
 	}
 
-	var mds []*arch.Metadata
+	var entries = make(map[string][]byte)
 
 	for _, pkg := range pkgs {
 		versions, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name)
@@ -176,14 +169,30 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([
 			if err != nil {
 				return nil, err
 			}
-			if checkPackageCompatability(distro, architecture, md.DistroArch) {
-				mds = append(mds, &md)
+			var found bool
+			for _, da := range md.DistroArch {
+				if da == distro+"-"+architecture {
+					desckey := pkg.Name + "-" + version.Version + "-" + architecture + ".desc"
+					descfile, err := GetFileObject(ctx, distro, desckey)
+					if err != nil {
+						return nil, err
+					}
+					descbytes, err := io.ReadAll(descfile)
+					if err != nil {
+						return nil, err
+					}
+					entries[pkg.Name+"-"+version.Version+"/desc"] = descbytes
+					found = true
+					break
+				}
+			}
+			if found {
 				break
 			}
 		}
 	}
 
-	return arch.CreatePacmanDb(mds)
+	return arch.CreatePacmanDb(entries)
 }
 
 // Remove specific package version related to provided user or organization.
@@ -195,16 +204,3 @@ func RemovePackage(ctx *context.Context, u *user.User, name, version string) err
 
 	return pkg_service.RemovePackageVersion(u, ver)
 }
-
-// This function will check, wether package should be added to resulting database.
-func checkPackageCompatability(distro, arch string, distroarchs []string) bool {
-	for _, distroarch := range distroarchs {
-		if distroarch == distro+"-any" {
-			return true
-		}
-		if distroarch == distro+"-"+arch {
-			return true
-		}
-	}
-	return false
-}

templates/package/content/arch.tmpl

@@ -30,26 +30,6 @@ Server = <gitea-origin-url data-url="{{AppSubUrl}}/api/packages/{{.PackageDescri
 				<td>{{.PackageDescriptor.Metadata.Description}}</td>
 			</tr>
 
-			<tr>
-				<td class="collapsing"><h5>Packager</h5></td>
-				<td>{{.PackageDescriptor.Metadata.Packager}}</td>
-			</tr>
-
-			<tr>
-				<td class="collapsing"><h5>Compressed size</h5></td>
-				<td>{{FileSize .PackageDescriptor.Metadata.CompressedSize}}</td>
-			</tr>
-
-			<tr>
-				<td class="collapsing"><h5>Installed size</h5></td>
-				<td>{{FileSize .PackageDescriptor.Metadata.InstalledSize}}</td>
-			</tr>
-
-			<tr>
-				<td class="collapsing"><h5>Build date</h5></td>
-				<td>{{DateTime "short" .PackageDescriptor.Metadata.BuildDate}}</td>
-			</tr>
-
 			{{if .PackageDescriptor.Metadata.Provides}}
 			<tr>
 				<td class="collapsing"><h5>Provides</h5></td>
@@ -57,13 +37,6 @@ Server = <gitea-origin-url data-url="{{AppSubUrl}}/api/packages/{{.PackageDescri
 			</tr>
 			{{end}}
 
-			{{if .PackageDescriptor.Metadata.Arch}}
-			<tr>
-				<td class="collapsing"><h5>Architecture</h5></td>
-				<td>{{StringUtils.Join $.PackageDescriptor.Metadata.Arch " "}}</td>
-			</tr>
-			{{end}}
-
 			{{if .PackageDescriptor.Metadata.Depends}}
 			<tr>
 				<td class="collapsing"><h5>Depends</h5></td>
@@ -98,13 +71,6 @@ Server = <gitea-origin-url data-url="{{AppSubUrl}}/api/packages/{{.PackageDescri
 				<td>{{StringUtils.Join $.PackageDescriptor.Metadata.Backup " "}}</td>
 			</tr>
 			{{end}}
-
-			{{if .PackageDescriptor.Metadata.DistroArch}}
-			<tr>
-				<td class="collapsing"><h5>Available for</h5></td>
-				<td>{{StringUtils.Join $.PackageDescriptor.Metadata.DistroArch " "}}</td>
-			</tr>
-			{{end}}
 		</tbody>
 	</table>
 	</div>