From 56772de7306bc97ad694f9a178e77b20d11f65c4 Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Sun, 2 Jul 2023 23:03:34 +0300 Subject: [PATCH] corrected package blob structure, pacman database is created on get request automatically from package metadata in database --- docs/content/doc/usage/packages/arch.en-us.md | 2 - modules/packages/arch/metadata.go | 134 ++++----------- routers/api/packages/arch/arch.go | 64 ++------ services/packages/arch/db_manager.go | 125 +++++++++++--- services/packages/arch/file_manager.go | 153 ------------------ 5 files changed, 151 insertions(+), 327 deletions(-) delete mode 100644 services/packages/arch/file_manager.go diff --git a/docs/content/doc/usage/packages/arch.en-us.md b/docs/content/doc/usage/packages/arch.en-us.md index 36366ea006..d2d02ec1ad 100644 --- a/docs/content/doc/usage/packages/arch.en-us.md +++ b/docs/content/doc/usage/packages/arch.en-us.md @@ -141,11 +141,9 @@ curl -X DELETE \ http://localhost:3000/api/packages/{user}/arch/remove \ --header "username: {user}" \ --header "email: user@email.com" \ - --header "distro: archlinux" \ --header "target: package" \ --header "time: {rmtime}" \ --header "version: {version-release}" \ - --header "arch: x86_64" \ --header 'Content-Type: application/octet-stream' \ --data-binary @md.sig ``` diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index a55df76f25..629d94b22b 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -21,31 +21,32 @@ import ( // Metadata for arch package. type Metadata struct { - Filename string - Name string - Base string - Version string - Description string - CompressedSize int64 - InstalledSize int64 - MD5 string - SHA256 string - URL string - BuildDate int64 - BaseDomain string - Packager string - Provides []string - License []string - Arch []string - Depends []string - OptDepends []string - MakeDepends []string - CheckDepends []string - Backup []string + Filename string `json:"filename"` + Name string `json:"name"` + Base string `json:"base"` + Version string `json:"version"` + Description string `json:"description"` + CompressedSize int64 `json:"compressed-size"` + InstalledSize int64 `json:"installed-size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + URL string `json:"url"` + BuildDate int64 `json:"build-date"` + BaseDomain string `json:"base-domain"` + Packager string `json:"packager"` + Distribution string `json:"distribution"` + Provides []string `json:"provides"` + License []string `json:"license"` + Arch []string `json:"arch"` + Depends []string `json:"depends"` + OptDepends []string `json:"opt-depends"` + MakeDepends []string `json:"make-depends"` + CheckDepends []string `json:"check-depends"` + Backup []string `json:"backup"` } // Function that recieves arch package archive data and returns it's metadata. -func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { +func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata, error) { pkginfo, err := getPkginfo(pkg) if err != nil { return nil, err @@ -56,6 +57,7 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { CompressedSize: int64(len(pkg)), MD5: md5sum(pkg), SHA256: sha256sum(pkg), + Distribution: distribution, } for _, line := range strings.Split(pkginfo, "\n") { splt := strings.Split(line, " = ") @@ -211,24 +213,18 @@ func Join(s ...string) string { return rez } -// Add or update existing package entry in database archived data. -func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { - // Read existing entries in archive. - entries, err := readEntries(db) - if err != nil { - return nil, err +// Create pacman database archive based on provided package metadata structs. +func CreatePacmanDb(mds []*Metadata) ([]byte, error) { + entries := make(map[string][]byte) + + for _, md := range mds { + entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) } - // Remove entries related old package versions. - entries = CleanOldEntries(entries, md.Name) - - // Add new package entry to list. - entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc()) - var out bytes.Buffer // Write entries to new buffer and return it. - err = writeToArchive(entries, &out) + err := writeToArchive(entries, &out) if err != nil { return nil, err } @@ -236,73 +232,7 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) { return out.Bytes(), nil } -// Clean entries for old package versions from pacman database. -func CleanOldEntries(entries map[string][]byte, pkg string) map[string][]byte { - out := map[string][]byte{} - for entry, value := range entries { - splt := strings.Split(entry, "-") - basename := strings.Join(splt[0:len(splt)-2], "-") - if pkg != basename { - out[entry] = value - } - } - return out -} - -// Add or update existing package entry in database archived data. -func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) { - // Read existing entries in archive. - entries, err := readEntries(db) - if err != nil { - return nil, err - } - - // Add new package entry to list. - delete(entries, pkg+"-"+ver+"/desc") - - var out bytes.Buffer - - // Write entries to new buffer and return it. - err = writeToArchive(entries, &out) - if err != nil { - return nil, err - } - - return out.Bytes(), nil -} - -// Read database entries containing in pacman archive. -func readEntries(dbarchive []byte) (map[string][]byte, error) { - gzf, err := gzip.NewReader(bytes.NewReader(dbarchive)) - if err != nil { - return map[string][]byte{}, nil - } - - var entries = map[string][]byte{} - - tarReader := tar.NewReader(gzf) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - if header.Typeflag == tar.TypeReg { - content, err := io.ReadAll(tarReader) - if err != nil { - return nil, err - } - entries[header.Name] = content - } - } - return entries, nil -} - -// Write pacman package entries to empty buffer. +// Write pacman package entries to tarball. func writeToArchive(files map[string][]byte, buf io.Writer) error { gw := gzip.NewWriter(buf) defer gw.Close() diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 52dfad0641..5ed794fedf 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -4,6 +4,7 @@ package arch import ( + "bytes" "encoding/hex" "io" "net/http" @@ -87,7 +88,7 @@ func Push(ctx *context.Context) { } // Parse metadata contained in arch package archive. - md, err := arch_module.EjectMetadata(filename, setting.Domain, pkgdata) + md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata) if err != nil { apiError(ctx, http.StatusBadRequest, err) return @@ -121,20 +122,6 @@ func Push(ctx *context.Context) { return } - // Save file related to arch package description. - _, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{ - Organization: org, - User: user, - Metadata: md, - Data: []byte(md.GetDbDesc()), - Filename: filename + ".desc", - Distro: distro, - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - // Automatically connect repository for provided package if name matched. err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid) if err != nil { @@ -157,38 +144,32 @@ func Get(ctx *context.Context) { // Packages are stored in different way from pacman databases, and loaded // with LoadPackageFile function. if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") { - pkgdata, err := arch_service.LoadPackageFile(ctx, distro, file) + pkgdata, err := arch_service.LoadFile(ctx, distro, file) if err != nil { apiError(ctx, http.StatusNotFound, err) return } - _, err = ctx.Resp.Write(pkgdata) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - ctx.Resp.WriteHeader(http.StatusOK) + ctx.ServeContent(bytes.NewReader(pkgdata), &context.ServeHeaderOptions{ + Filename: file, + CacheDuration: time.Minute * 5, + }) return } - // Pacman databases are stored directly in gitea file storage and could be - // loaded with name as a key. + // Pacman databases is not stored in gitea's storage, it is created for + // incoming request and cached. if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") { - data, err := arch_service.LoadPacmanDatabase(ctx, owner, distro, arch, file) - if err != nil { - apiError(ctx, http.StatusNotFound, err) - return - } - - _, err = ctx.Resp.Write(data) + db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - ctx.Resp.WriteHeader(http.StatusOK) + ctx.ServeContent(bytes.NewReader(db), &context.ServeHeaderOptions{ + Filename: file, + CacheDuration: time.Minute * 5, + }) return } @@ -200,11 +181,9 @@ func Remove(ctx *context.Context) { var ( owner = ctx.Params("username") email = ctx.Req.Header.Get("email") - distro = ctx.Req.Header.Get("distro") target = ctx.Req.Header.Get("target") stime = ctx.Req.Header.Get("time") version = ctx.Req.Header.Get("version") - arch = strings.Split(ctx.Req.Header.Get("arch"), " ") ) // Parse sent time and check if it is within last minute. @@ -242,21 +221,8 @@ func Remove(ctx *context.Context) { return } - // Remove pacman database entries related to package. - err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - // Remove package files and pacman database entry. - err = arch_service.RemovePackage(ctx, &arch_service.RemoveParameters{ - User: user, - Organization: org, - Owner: owner, - Name: target, - Version: version, - }) + err = arch_service.RemovePackage(ctx, org.AsUser(), target, version) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/arch/db_manager.go b/services/packages/arch/db_manager.go index b893d70c9c..361f5e7477 100644 --- a/services/packages/arch/db_manager.go +++ b/services/packages/arch/db_manager.go @@ -5,25 +5,31 @@ package arch import ( "bytes" + "fmt" + "io" + "strings" - org "code.gitea.io/gitea/models/organization" - pkg "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/db" + org_model "code.gitea.io/gitea/models/organization" + pkg_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/packages/arch" - svc "code.gitea.io/gitea/services/packages" + pkg_service "code.gitea.io/gitea/services/packages" ) // Parameters required to save new arch package. type SaveFileParams struct { - *org.Organization + *org_model.Organization *user.User *arch.Metadata Data []byte Filename string Distro string + IsLead bool } // This function create new package, version and package file properties in @@ -36,25 +42,26 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { } defer buf.Close() - pv, _, err := svc.CreatePackageOrAddFileToExisting( - &svc.PackageCreationInfo{ - PackageInfo: svc.PackageInfo{ + pv, _, err := pkg_service.CreatePackageOrAddFileToExisting( + &pkg_service.PackageCreationInfo{ + PackageInfo: pkg_service.PackageInfo{ Owner: p.Organization.AsUser(), - PackageType: pkg.TypeArch, + PackageType: pkg_model.TypeArch, Name: p.Metadata.Name, Version: p.Metadata.Version, }, Creator: p.User, Metadata: p.Metadata, }, - &svc.PackageFileCreationInfo{ - PackageFileInfo: svc.PackageFileInfo{ + &pkg_service.PackageFileCreationInfo{ + PackageFileInfo: pkg_service.PackageFileInfo{ Filename: p.Filename, CompositeKey: p.Distro + "-" + p.Filename, }, Creator: p.User, Data: buf, OverwriteExisting: true, + IsLead: p.IsLead, }, ) if err != nil { @@ -63,12 +70,46 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) { return pv.PackageID, nil } +// Get data related to provided file name and distribution, and update download +// counter if actual package file is retrieved from database. +func LoadFile(ctx *context.Context, distro, file string) ([]byte, error) { + db := db.GetEngine(ctx) + + pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file} + + ok, err := db.Get(pkgfile) + if err != nil || !ok { + return nil, fmt.Errorf("%+v %t", err, ok) + } + + blob, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID) + if err != nil { + return nil, err + } + + if strings.HasSuffix(file, ".pkg.tar.zst") { + err = pkg_model.IncrementDownloadCounter(ctx, pkgfile.VersionID) + if err != nil { + return nil, err + } + } + + cs := packages.NewContentStore() + + obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) + if err != nil { + return nil, err + } + + return io.ReadAll(obj) +} + // Automatically connect repository to pushed package, if package with provided // with provided name exists in namespace scope. func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error { repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository) if err == nil { - err = pkg.SetRepositoryLink(ctx, pkgid, repo.ID) + err = pkg_model.SetRepositoryLink(ctx, pkgid, repo.ID) if err != nil { return err } @@ -76,19 +117,61 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid return nil } -type RemoveParameters struct { - *user.User - *org.Organization - Owner string - Name string - Version string +// This function is collecting information about packages in some organization/ +// user space, and created pacman database archive based on package metadata. +func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([]byte, error) { + u, err := user.GetUserByName(ctx, owner) + if err != nil { + return nil, err + } + + pkgs, err := pkg_model.GetPackagesByType(ctx, u.ID, pkg_model.TypeArch) + if err != nil { + return nil, err + } + + var mds []*arch.Metadata + + for _, pkg := range pkgs { + vers, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name) + if err != nil { + return nil, err + } + for i := len(vers) - 1; i >= 0; i-- { + var md arch.Metadata + err = json.Unmarshal([]byte(vers[i].MetadataJSON), &md) + if err != nil { + return nil, err + } + if checkArchitecture(architecture, md.Arch) && md.Distribution == distro { + mds = append(mds, &md) + break + } + } + } + + return arch.CreatePacmanDb(mds) } -// Remove package and it's blobs from gitea. -func RemovePackage(ctx *context.Context, p *RemoveParameters) error { - ver, err := pkg.GetVersionByNameAndVersion(ctx, p.Organization.ID, pkg.TypeArch, p.Name, p.Version) +// Remove specific package version related to provided user or organization. +func RemovePackage(ctx *context.Context, u *user.User, name, version string) error { + ver, err := pkg_model.GetVersionByNameAndVersion(ctx, u.ID, pkg_model.TypeArch, name, version) if err != nil { return err } - return svc.RemovePackageVersion(p.User, ver) + + return pkg_service.RemovePackageVersion(u, ver) +} + +// Check wether package architecture is relevant for requestsed database. +func checkArchitecture(architecture string, list []string) bool { + for _, v := range list { + if v == "any" { + return true + } + if v == architecture { + return true + } + } + return false } diff --git a/services/packages/arch/file_manager.go b/services/packages/arch/file_manager.go deleted file mode 100644 index 4f146baa4d..0000000000 --- a/services/packages/arch/file_manager.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package arch - -import ( - "bytes" - "fmt" - "io" - "strings" - - "code.gitea.io/gitea/models/db" - pkg_mdl "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/packages" - "code.gitea.io/gitea/modules/packages/arch" - "code.gitea.io/gitea/modules/setting" -) - -// Get data related to provided file name and distribution, and update download -// counter if actual package file is retrieved from database. -func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) { - db := db.GetEngine(ctx) - - pkgfile := &pkg_mdl.PackageFile{CompositeKey: distro + "-" + file} - - ok, err := db.Get(pkgfile) - if err != nil || !ok { - return nil, fmt.Errorf("%+v %t", err, ok) - } - - blob, err := pkg_mdl.GetBlobByID(ctx, pkgfile.BlobID) - if err != nil { - return nil, err - } - - if strings.HasSuffix(file, ".pkg.tar.zst") { - err = pkg_mdl.IncrementDownloadCounter(ctx, pkgfile.VersionID) - if err != nil { - return nil, err - } - } - - cs := packages.NewContentStore() - - obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256)) - if err != nil { - return nil, err - } - - return io.ReadAll(obj) -} - -// Get data related to pacman database file or symlink. -func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file string) ([]byte, error) { - cs := packages.NewContentStore() - - file = strings.TrimPrefix(file, owner+".") - - dbname := strings.TrimSuffix(arch.Join(owner, distro, architecture, file), ".tar.gz") - - obj, err := cs.Get(packages.BlobHash256Key(dbname)) - if err != nil { - return nil, err - } - - return io.ReadAll(obj) -} - -// This function will update information about package in related pacman databases -// or create them if they do not exist. -func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owner string) error { - // If architecure is not specified or any, package will be automatically - // saved to pacman databases with most popular architectures. - if len(md.Arch) == 0 || md.Arch[0] == "any" { - md.Arch = popularArchitectures() - } - - cs := packages.NewContentStore() - - // Update pacman database files for each architecture. - for _, architecture := range md.Arch { - db := arch.Join(owner, distro, architecture, setting.Domain, "db") - dbkey := packages.BlobHash256Key(db) - - var dbdata []byte - - dbobj, err := cs.Get(dbkey) - if err == nil { - dbdata, err = io.ReadAll(dbobj) - if err != nil { - return err - } - } - - newdata, err := arch.UpdatePacmanDbEntry(dbdata, md) - if err != nil { - return err - } - - err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata))) - if err != nil { - return err - } - } - - return nil -} - -func RemoveDbEntry(ctx *context.Context, architectures []string, owner, distro, pkg, ver string) error { - cs := packages.NewContentStore() - - // If architecures are not specified or any, package will be automatically - // removed from pacman databases with most popular architectures. - if len(architectures) == 0 || architectures[0] == "any" { - architectures = popularArchitectures() - } - - for _, architecture := range architectures { - db := arch.Join(owner, distro, architecture, setting.Domain, "db") - dbkey := packages.BlobHash256Key(db) - - var dbdata []byte - - dbobj, err := cs.Get(dbkey) - if err != nil { - return err - } - - dbdata, err = io.ReadAll(dbobj) - if err != nil { - return err - } - - newdata, err := arch.RemoveDbEntry(dbdata, pkg, ver) - if err != nil { - return err - } - - err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata))) - if err != nil { - return err - } - } - return nil -} - -func popularArchitectures() []string { - return []string{ - "x86_64", "arm", "i686", "pentium4", - "armv7h", "armv6h", "aarch64", "riscv64", - } -}