From 3df8d52fb8fcccb051f1e8bae2ec7cbfe1127490 Mon Sep 17 00:00:00 2001 From: Danila Fominykh Date: Fri, 28 Jul 2023 16:32:01 +0300 Subject: [PATCH] changed pacman database creation mechanics, storing package desc as file in object storage, metadata structure update --- modules/packages/arch/metadata.go | 89 ++++++++++++++-------------- routers/api/packages/arch/arch.go | 82 +++++++++++++++++++++---- services/packages/arch/db_manager.go | 54 ++++++++--------- templates/package/content/arch.tmpl | 34 ----------- 4 files changed, 140 insertions(+), 119 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index c2608003a1..2806e69ce8 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -5,10 +5,9 @@ package arch import ( "archive/tar" + "bufio" "bytes" "compress/gzip" - "crypto/md5" - "crypto/sha256" "encoding/hex" "fmt" "io" @@ -17,11 +16,30 @@ import ( "strings" "code.gitea.io/gitea/modules/container" + pkg_module "code.gitea.io/gitea/modules/packages" "github.com/mholt/archiver/v3" ) -// Metadata for arch package. +// JSON with pacakage parameters that are not related to specific +// architecture/distribution that will be stored in sql database. type Metadata struct { + Name string + Version string + URL string `json:"url"` + Description string `json:"description"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt-depends,omitempty"` + MakeDepends []string `json:"make-depends,omitempty"` + CheckDepends []string `json:"check-depends,omitempty"` + Backup []string `json:"backup,omitempty"` + DistroArch []string `json:"distro-arch,omitempty"` +} + +// Package description file that will be saved as .desc file in object storage. +// Resulting file will be used to create pacman database. +type DbDesc struct { Filename string `json:"filename"` Name string `json:"name"` Base string `json:"base"` @@ -34,33 +52,31 @@ type Metadata struct { URL string `json:"url"` BuildDate int64 `json:"build-date"` Packager string `json:"packager"` - Provides []string `json:"provides"` - License []string `json:"license"` - Arch []string `json:"arch"` - Depends []string `json:"depends"` - OptDepends []string `json:"opt-depends"` - MakeDepends []string `json:"make-depends"` - CheckDepends []string `json:"check-depends"` - Backup []string `json:"backup"` - // This list is created to ensure the consistency of pacman database file - // for specific combination of distribution and architecture. This value - // is used when creating pacman database, to ensure that only packages - // with specific combination of distribution and architecture are present - // in pacman database file. - DistroArch []string `json:"distro-arch"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Arch []string `json:"arch,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt-depends,omitempty"` + MakeDepends []string `json:"make-depends,omitempty"` + CheckDepends []string `json:"check-depends,omitempty"` + Backup []string `json:"backup,omitempty"` } // Function that receives arch package archive data and returns it's metadata. -func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) { - pkginfo, err := getPkginfo(pkg) +func EjectMetadata(filename, distribution string, buf *pkg_module.HashedBuffer) (*DbDesc, error) { + pkginfo, err := getPkginfo(buf) if err != nil { return nil, err } - md := Metadata{ + + // Add package blob parameters to arch related desc. + hashMD5, _, hashSHA256, _ := buf.Sums() + md := DbDesc{ Filename: filename, - CompressedSize: int64(len(pkg)), - MD5: md5sum(pkg), - SHA256: sha256sum(pkg), + Name: filename, + CompressedSize: buf.Size(), + MD5: hex.EncodeToString(hashMD5), + SHA256: hex.EncodeToString(hashSHA256), } for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") @@ -98,7 +114,6 @@ func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) md.License = append(md.License, splt[1]) case "arch": md.Arch = append(md.Arch, splt[1]) - md.DistroArch = append(md.DistroArch, distribution+"-"+splt[1]) case "depend": md.Depends = append(md.Depends, splt[1]) case "optdepend": @@ -114,10 +129,10 @@ func EjectMetadata(filename, distribution string, pkg []byte) (*Metadata, error) return &md, nil } -func getPkginfo(data []byte) (string, error) { - pkgreader := bytes.NewReader(data) +func getPkginfo(data io.Reader) (string, error) { + br := bufio.NewReader(data) zstd := archiver.NewTarZstd() - err := zstd.Open(pkgreader, int64(250000)) + err := zstd.Open(br, int64(250000)) if err != nil { return ``, err } @@ -137,19 +152,9 @@ func getPkginfo(data []byte) (string, error) { } } -func md5sum(data []byte) string { - sum := md5.Sum(data) - return hex.EncodeToString(sum[:]) -} - -func sha256sum(data []byte) string { - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]) -} - // This function returns pacman package description in unarchived raw database // format. -func (m *Metadata) GetDbDesc() string { +func (m *DbDesc) GetDbDesc() string { return strings.Join(rmEmptyStrings([]string{ formatField("FILENAME", m.Filename), formatField("NAME", m.Name), @@ -200,13 +205,7 @@ func rmEmptyStrings(s []string) []string { } // Create pacman database archive based on provided package metadata structs. -func CreatePacmanDb(mds []*Metadata) ([]byte, error) { - entries := make(map[string][]byte) - - for _, md := range mds { - entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) - } - +func CreatePacmanDb(entries map[string][]byte) ([]byte, error) { var out bytes.Buffer // Write entries to new buffer and return it. diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index b36e4da448..21ac898582 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/context" + pkg_module "code.gitea.io/gitea/modules/packages" arch_module "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/routers/api/packages/helper" arch_service "code.gitea.io/gitea/services/packages/arch" @@ -25,28 +26,79 @@ func Push(ctx *context.Context) { sign = ctx.Params("sign") ) - // Read package to memory for package validation. - pkgdata, err := io.ReadAll(ctx.Req.Body) + upload, close, err := ctx.UploadStream() if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - defer ctx.Req.Body.Close() + if close { + defer upload.Close() + } - // Parse metadata contained in arch package archive. - md, err := arch_module.EjectMetadata(filename, distro, pkgdata) + buf, err := pkg_module.CreateHashedBufferFromReader(upload) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + // Parse metadata related to package contained in arch package archive. + pkgmd, err := arch_module.EjectMetadata(filename, distro, buf) if err != nil { apiError(ctx, http.StatusBadRequest, err) return } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // Metadata related to SQL database. + dbmd := &arch_module.Metadata{ + Name: pkgmd.Name, + Version: pkgmd.Version, + URL: pkgmd.URL, + Description: pkgmd.Description, + Provides: pkgmd.Provides, + License: pkgmd.License, + Depends: pkgmd.Depends, + OptDepends: pkgmd.OptDepends, + MakeDepends: pkgmd.MakeDepends, + CheckDepends: pkgmd.CheckDepends, + Backup: pkgmd.Backup, + DistroArch: []string{distro + "-" + pkgmd.Arch[0]}, + } + // Save file related to arch package. pkgid, err := arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Metadata: md, Filename: filename, - Data: pkgdata, + Buf: buf, + Metadata: dbmd, + Distro: distro, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + r := io.NopCloser(bytes.NewReader([]byte(pkgmd.GetDbDesc()))) + buf, err = pkg_module.CreateHashedBufferFromReader(r) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + // Save file related to arch package description for pacman database. + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ + Creator: ctx.ContextUser, + Owner: ctx.Package.Owner, + Filename: pkgmd.Name + "-" + pkgmd.Version + "-" + pkgmd.Arch[0] + ".desc", + Buf: buf, + Metadata: dbmd, Distro: distro, }) if err != nil { @@ -57,12 +109,20 @@ func Push(ctx *context.Context) { // Decoding package signature, if present saving with package as file. sigdata, err := hex.DecodeString(sign) if err == nil { + r := io.NopCloser(bytes.NewReader(sigdata)) + buf, err := pkg_module.CreateHashedBufferFromReader(r) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ Creator: ctx.ContextUser, Owner: ctx.Package.Owner, - Metadata: md, - Data: sigdata, + Buf: buf, Filename: filename + ".sig", + Metadata: dbmd, Distro: distro, }) if err != nil { @@ -74,7 +134,7 @@ func Push(ctx *context.Context) { // Add new architectures and distribution info to package version metadata. err = arch_service.UpdateMetadata(ctx, &arch_service.UpdateMetadataParameters{ User: ctx.Package.Owner, - Md: md, + Md: dbmd, }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -82,7 +142,7 @@ func Push(ctx *context.Context) { } // Automatically connect repository for provided package if name matched. - err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) + err = arch_service.RepositoryAutoconnect(ctx, owner, dbmd.Name, pkgid) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index 7ac72874da..d01927bd31 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -4,8 +4,8 @@ package arch import ( - "bytes" "fmt" + "io" "sort" "strings" @@ -41,7 +41,6 @@ func UpdateMetadata(ctx *context.Context, p *UpdateMetadataParameters) error { return err } - currmd.Arch = arch.UnifiedList(currmd.Arch, p.Md.Arch) currmd.DistroArch = arch.UnifiedList(currmd.DistroArch, p.Md.DistroArch) b, err := json.Marshal(&currmd) @@ -59,7 +58,7 @@ type SaveFileParams struct { Creator *user.User Owner *user.User Metadata *arch.Metadata - Data []byte + Buf packages.HashedSizeReader Filename string Distro string IsLead bool @@ -69,13 +68,7 @@ type SaveFileParams struct { // database, and writes blob to file storage. If package/version/blob exists it // will overwrite existing data. Package id and error will be returned. func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { - buf, err := packages.CreateHashedBufferFromReader(bytes.NewReader(p.Data)) - if err != nil { - return 0, err - } - defer buf.Close() - - pv, _, err := pkg_service.CreatePackageOrAddFileToExisting( + ver, _, err := pkg_service.CreatePackageOrAddFileToExisting( &pkg_service.PackageCreationInfo{ PackageInfo: pkg_service.PackageInfo{ Owner: p.Owner, @@ -92,7 +85,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { CompositeKey: p.Distro + "-" + p.Filename, }, Creator: p.Creator, - Data: buf, + Data: p.Buf, OverwriteExisting: true, IsLead: p.IsLead, }, @@ -100,7 +93,7 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { if err != nil { return 0, err } - return pv.PackageID, nil + return ver.ID, nil } // Get data related to provided file name and distribution, and update download @@ -158,7 +151,7 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ return nil, err } - var mds []*arch.Metadata + var entries = make(map[string][]byte) for _, pkg := range pkgs { versions, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) @@ -176,14 +169,30 @@ func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([ if err != nil { return nil, err } - if checkPackageCompatability(distro, architecture, md.DistroArch) { - mds = append(mds, &md) + var found bool + for _, da := range md.DistroArch { + if da == distro+"-"+architecture { + desckey := pkg.Name + "-" + version.Version + "-" + architecture + ".desc" + descfile, err := GetFileObject(ctx, distro, desckey) + if err != nil { + return nil, err + } + descbytes, err := io.ReadAll(descfile) + if err != nil { + return nil, err + } + entries[pkg.Name+"-"+version.Version+"/desc"] = descbytes + found = true + break + } + } + if found { break } } } - return arch.CreatePacmanDb(mds) + return arch.CreatePacmanDb(entries) } // Remove specific package version related to provided user or organization. @@ -195,16 +204,3 @@ func RemovePackage(ctx *context.Context, u *user.User, name, version string) err return pkg_service.RemovePackageVersion(u, ver) } - -// This function will check, wether package should be added to resulting database. -func checkPackageCompatability(distro, arch string, distroarchs []string) bool { - for _, distroarch := range distroarchs { - if distroarch == distro+"-any" { - return true - } - if distroarch == distro+"-"+arch { - return true - } - } - return false -} diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 0831ce8fce..e58ddd669a 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -30,26 +30,6 @@ Server =
Packager
- {{.PackageDescriptor.Metadata.Packager}} - - - -
Compressed size
- {{FileSize .PackageDescriptor.Metadata.CompressedSize}} - - - -
Installed size
- {{FileSize .PackageDescriptor.Metadata.InstalledSize}} - - - -
Build date
- {{DateTime "short" .PackageDescriptor.Metadata.BuildDate}} - - {{if .PackageDescriptor.Metadata.Provides}}
Provides
@@ -57,13 +37,6 @@ Server =
Architecture
- {{StringUtils.Join $.PackageDescriptor.Metadata.Arch " "}} - - {{end}} - {{if .PackageDescriptor.Metadata.Depends}}
Depends
@@ -98,13 +71,6 @@ Server = {{end}} - - {{if .PackageDescriptor.Metadata.DistroArch}} - -
Available for
- {{StringUtils.Join $.PackageDescriptor.Metadata.DistroArch " "}} - - {{end}}