Mirrors/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2024-09-01 14:56:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

corrected package blob structure, pacman database is created on get request automatically from package metadata in database

Browse Source
This commit is contained in:
dancheg97 2023-07-02 23:03:34 +03:00
parent 6d1037d506
commit 56772de730
5 changed files with 151 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/content/doc/usage/packages/arch.en-us.md
View File

@ -141,11 +141,9 @@ curl -X DELETE \
http://localhost:3000/api/packages/{user}/arch/remove \
--header "username: {user}" \
--header "email: user@email.com" \
--header "distro: archlinux" \
--header "target: package" \
--header "time: {rmtime}" \
--header "version: {version-release}" \
--header "arch: x86_64" \
--header 'Content-Type: application/octet-stream' \
--data-binary @md.sig
```

134
modules/packages/arch/metadata.go
View File

@ -21,31 +21,32 @@ import (
// Metadata for arch package.
type Metadata struct {
Filename string
Name string
Base string
Version string
Description string
CompressedSize int64
InstalledSize int64
MD5 string
SHA256 string
URL string
BuildDate int64
BaseDomain string
Packager string
Provides []string
License []string
Arch []string
Depends []string
OptDepends []string
MakeDepends []string
CheckDepends []string
Backup []string
Filename string `json:"filename"`
Name string `json:"name"`
Base string `json:"base"`
Version string `json:"version"`
Description string `json:"description"`
CompressedSize int64 `json:"compressed-size"`
InstalledSize int64 `json:"installed-size"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
URL string `json:"url"`
BuildDate int64 `json:"build-date"`
BaseDomain string `json:"base-domain"`
Packager string `json:"packager"`
Distribution string `json:"distribution"`
Provides []string `json:"provides"`
License []string `json:"license"`
Arch []string `json:"arch"`
Depends []string `json:"depends"`
OptDepends []string `json:"opt-depends"`
MakeDepends []string `json:"make-depends"`
CheckDepends []string `json:"check-depends"`
Backup []string `json:"backup"`
}
// Function that recieves arch package archive data and returns it's metadata.
func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) {
func EjectMetadata(filename, distribution, domain string, pkg []byte) (*Metadata, error) {
pkginfo, err := getPkginfo(pkg)
if err != nil {
return nil, err
@ -56,6 +57,7 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) {
CompressedSize: int64(len(pkg)),
MD5: md5sum(pkg),
SHA256: sha256sum(pkg),
Distribution: distribution,
}
for _, line := range strings.Split(pkginfo, "\n") {
splt := strings.Split(line, " = ")
@ -211,24 +213,18 @@ func Join(s ...string) string {
return rez
}
// Add or update existing package entry in database archived data.
func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) {
// Read existing entries in archive.
entries, err := readEntries(db)
if err != nil {
return nil, err
// Create pacman database archive based on provided package metadata structs.
func CreatePacmanDb(mds []*Metadata) ([]byte, error) {
entries := make(map[string][]byte)
for _, md := range mds {
entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc())
}
// Remove entries related old package versions.
entries = CleanOldEntries(entries, md.Name)
// Add new package entry to list.
entries[md.Name+"-"+md.Version+"/desc"] = []byte(md.GetDbDesc())
var out bytes.Buffer
// Write entries to new buffer and return it.
err = writeToArchive(entries, &out)
err := writeToArchive(entries, &out)
if err != nil {
return nil, err
}
@ -236,73 +232,7 @@ func UpdatePacmanDbEntry(db []byte, md *Metadata) ([]byte, error) {
return out.Bytes(), nil
}
// Clean entries for old package versions from pacman database.
func CleanOldEntries(entries map[string][]byte, pkg string) map[string][]byte {
out := map[string][]byte{}
for entry, value := range entries {
splt := strings.Split(entry, "-")
basename := strings.Join(splt[0:len(splt)-2], "-")
if pkg != basename {
out[entry] = value
}
}
return out
}
// Add or update existing package entry in database archived data.
func RemoveDbEntry(db []byte, pkg, ver string) ([]byte, error) {
// Read existing entries in archive.
entries, err := readEntries(db)
if err != nil {
return nil, err
}
// Add new package entry to list.
delete(entries, pkg+"-"+ver+"/desc")
var out bytes.Buffer
// Write entries to new buffer and return it.
err = writeToArchive(entries, &out)
if err != nil {
return nil, err
}
return out.Bytes(), nil
}
// Read database entries containing in pacman archive.
func readEntries(dbarchive []byte) (map[string][]byte, error) {
gzf, err := gzip.NewReader(bytes.NewReader(dbarchive))
if err != nil {
return map[string][]byte{}, nil
}
var entries = map[string][]byte{}
tarReader := tar.NewReader(gzf)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if header.Typeflag == tar.TypeReg {
content, err := io.ReadAll(tarReader)
if err != nil {
return nil, err
}
entries[header.Name] = content
}
}
return entries, nil
}
// Write pacman package entries to empty buffer.
// Write pacman package entries to tarball.
func writeToArchive(files map[string][]byte, buf io.Writer) error {
gw := gzip.NewWriter(buf)
defer gw.Close()

64
routers/api/packages/arch/arch.go
View File

@ -4,6 +4,7 @@
package arch
import (
"bytes"
"encoding/hex"
"io"
"net/http"
@ -87,7 +88,7 @@ func Push(ctx *context.Context) {
}
// Parse metadata contained in arch package archive.
md, err := arch_module.EjectMetadata(filename, setting.Domain, pkgdata)
md, err := arch_module.EjectMetadata(filename, distro, setting.Domain, pkgdata)
if err != nil {
apiError(ctx, http.StatusBadRequest, err)
return
@ -121,20 +122,6 @@ func Push(ctx *context.Context) {
return
}
// Save file related to arch package description.
_, err = arch_service.SaveFile(ctx, &arch_service.SaveFileParams{
Organization: org,
User: user,
Metadata: md,
Data: []byte(md.GetDbDesc()),
Filename: filename + ".desc",
Distro: distro,
})
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
// Automatically connect repository for provided package if name matched.
err = arch_service.RepositoryAutoconnect(ctx, owner, md.Name, pkgid)
if err != nil {
@ -157,38 +144,32 @@ func Get(ctx *context.Context) {
// Packages are stored in different way from pacman databases, and loaded
// with LoadPackageFile function.
if strings.HasSuffix(file, "tar.zst") || strings.HasSuffix(file, "zst.sig") {
pkgdata, err := arch_service.LoadPackageFile(ctx, distro, file)
pkgdata, err := arch_service.LoadFile(ctx, distro, file)
if err != nil {
apiError(ctx, http.StatusNotFound, err)
return
}
_, err = ctx.Resp.Write(pkgdata)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
ctx.Resp.WriteHeader(http.StatusOK)
ctx.ServeContent(bytes.NewReader(pkgdata), &context.ServeHeaderOptions{
Filename: file,
CacheDuration: time.Minute * 5,
})
return
}
// Pacman databases are stored directly in gitea file storage and could be
// loaded with name as a key.
// Pacman databases is not stored in gitea's storage, it is created for
// incoming request and cached.
if strings.HasSuffix(file, ".db.tar.gz") || strings.HasSuffix(file, ".db") {
data, err := arch_service.LoadPacmanDatabase(ctx, owner, distro, arch, file)
if err != nil {
apiError(ctx, http.StatusNotFound, err)
return
}
_, err = ctx.Resp.Write(data)
db, err := arch_service.CreatePacmanDb(ctx, owner, arch, distro)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
ctx.Resp.WriteHeader(http.StatusOK)
ctx.ServeContent(bytes.NewReader(db), &context.ServeHeaderOptions{
Filename: file,
CacheDuration: time.Minute * 5,
})
return
}
@ -200,11 +181,9 @@ func Remove(ctx *context.Context) {
var (
owner = ctx.Params("username")
email = ctx.Req.Header.Get("email")
distro = ctx.Req.Header.Get("distro")
target = ctx.Req.Header.Get("target")
stime = ctx.Req.Header.Get("time")
version = ctx.Req.Header.Get("version")
arch = strings.Split(ctx.Req.Header.Get("arch"), " ")
)
// Parse sent time and check if it is within last minute.
@ -242,21 +221,8 @@ func Remove(ctx *context.Context) {
return
}
// Remove pacman database entries related to package.
err = arch_service.RemoveDbEntry(ctx, arch, owner, distro, target, version)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
// Remove package files and pacman database entry.
err = arch_service.RemovePackage(ctx, &arch_service.RemoveParameters{
User: user,
Organization: org,
Owner: owner,
Name: target,
Version: version,
})
err = arch_service.RemovePackage(ctx, org.AsUser(), target, version)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return

125
services/packages/arch/db_manager.go
View File

@ -5,25 +5,31 @@ package arch
import (
"bytes"
"fmt"
"io"
"strings"
org "code.gitea.io/gitea/models/organization"
pkg "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/db"
org_model "code.gitea.io/gitea/models/organization"
pkg_model "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/packages/arch"
svc "code.gitea.io/gitea/services/packages"
pkg_service "code.gitea.io/gitea/services/packages"
)
// Parameters required to save new arch package.
type SaveFileParams struct {
*org.Organization
*org_model.Organization
*user.User
*arch.Metadata
Data []byte
Filename string
Distro string
IsLead bool
}
// This function create new package, version and package file properties in
@ -36,25 +42,26 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
}
defer buf.Close()
pv, _, err := svc.CreatePackageOrAddFileToExisting(
&svc.PackageCreationInfo{
PackageInfo: svc.PackageInfo{
pv, _, err := pkg_service.CreatePackageOrAddFileToExisting(
&pkg_service.PackageCreationInfo{
PackageInfo: pkg_service.PackageInfo{
Owner: p.Organization.AsUser(),
PackageType: pkg.TypeArch,
PackageType: pkg_model.TypeArch,
Name: p.Metadata.Name,
Version: p.Metadata.Version,
},
Creator: p.User,
Metadata: p.Metadata,
},
&svc.PackageFileCreationInfo{
PackageFileInfo: svc.PackageFileInfo{
&pkg_service.PackageFileCreationInfo{
PackageFileInfo: pkg_service.PackageFileInfo{
Filename: p.Filename,
CompositeKey: p.Distro + "-" + p.Filename,
},
Creator: p.User,
Data: buf,
OverwriteExisting: true,
IsLead: p.IsLead,
},
)
if err != nil {
@ -63,12 +70,46 @@ func SaveFile(ctx *context.Context, p *SaveFileParams) (int64, error) {
return pv.PackageID, nil
}
// Get data related to provided file name and distribution, and update download
// counter if actual package file is retrieved from database.
func LoadFile(ctx *context.Context, distro, file string) ([]byte, error) {
db := db.GetEngine(ctx)
pkgfile := &pkg_model.PackageFile{CompositeKey: distro + "-" + file}
ok, err := db.Get(pkgfile)
if err != nil || !ok {
return nil, fmt.Errorf("%+v %t", err, ok)
}
blob, err := pkg_model.GetBlobByID(ctx, pkgfile.BlobID)
if err != nil {
return nil, err
}
if strings.HasSuffix(file, ".pkg.tar.zst") {
err = pkg_model.IncrementDownloadCounter(ctx, pkgfile.VersionID)
if err != nil {
return nil, err
}
}
cs := packages.NewContentStore()
obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256))
if err != nil {
return nil, err
}
return io.ReadAll(obj)
}
// Automatically connect repository to pushed package, if package with provided
// with provided name exists in namespace scope.
func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid int64) error {
repo, err := repo.GetRepositoryByOwnerAndName(ctx, owner, repository)
if err == nil {
err = pkg.SetRepositoryLink(ctx, pkgid, repo.ID)
err = pkg_model.SetRepositoryLink(ctx, pkgid, repo.ID)
if err != nil {
return err
}
@ -76,19 +117,61 @@ func RepositoryAutoconnect(ctx *context.Context, owner, repository string, pkgid
return nil
}
type RemoveParameters struct {
*user.User
*org.Organization
Owner string
Name string
Version string
// This function is collecting information about packages in some organization/
// user space, and created pacman database archive based on package metadata.
func CreatePacmanDb(ctx *context.Context, owner, architecture, distro string) ([]byte, error) {
u, err := user.GetUserByName(ctx, owner)
if err != nil {
return nil, err
}
pkgs, err := pkg_model.GetPackagesByType(ctx, u.ID, pkg_model.TypeArch)
if err != nil {
return nil, err
}
var mds []*arch.Metadata
for _, pkg := range pkgs {
vers, err := pkg_model.GetVersionsByPackageName(ctx, u.ID, pkg_model.TypeArch, pkg.Name)
if err != nil {
return nil, err
}
for i := len(vers) - 1; i >= 0; i-- {
var md arch.Metadata
err = json.Unmarshal([]byte(vers[i].MetadataJSON), &md)
if err != nil {
return nil, err
}
if checkArchitecture(architecture, md.Arch) && md.Distribution == distro {
mds = append(mds, &md)
break
}
}
}
return arch.CreatePacmanDb(mds)
}
// Remove package and it's blobs from gitea.
func RemovePackage(ctx *context.Context, p *RemoveParameters) error {
ver, err := pkg.GetVersionByNameAndVersion(ctx, p.Organization.ID, pkg.TypeArch, p.Name, p.Version)
// Remove specific package version related to provided user or organization.
func RemovePackage(ctx *context.Context, u *user.User, name, version string) error {
ver, err := pkg_model.GetVersionByNameAndVersion(ctx, u.ID, pkg_model.TypeArch, name, version)
if err != nil {
return err
}
return svc.RemovePackageVersion(p.User, ver)
return pkg_service.RemovePackageVersion(u, ver)
}
// Check wether package architecture is relevant for requestsed database.
func checkArchitecture(architecture string, list []string) bool {
for _, v := range list {
if v == "any" {
return true
}
if v == architecture {
return true
}
}
return false
}

153
services/packages/arch/file_manager.go
View File

@ -1,153 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package arch
import (
"bytes"
"fmt"
"io"
"strings"
"code.gitea.io/gitea/models/db"
pkg_mdl "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/packages/arch"
"code.gitea.io/gitea/modules/setting"
)
// Get data related to provided file name and distribution, and update download
// counter if actual package file is retrieved from database.
func LoadPackageFile(ctx *context.Context, distro, file string) ([]byte, error) {
db := db.GetEngine(ctx)
pkgfile := &pkg_mdl.PackageFile{CompositeKey: distro + "-" + file}
ok, err := db.Get(pkgfile)
if err != nil || !ok {
return nil, fmt.Errorf("%+v %t", err, ok)
}
blob, err := pkg_mdl.GetBlobByID(ctx, pkgfile.BlobID)
if err != nil {
return nil, err
}
if strings.HasSuffix(file, ".pkg.tar.zst") {
err = pkg_mdl.IncrementDownloadCounter(ctx, pkgfile.VersionID)
if err != nil {
return nil, err
}
}
cs := packages.NewContentStore()
obj, err := cs.Get(packages.BlobHash256Key(blob.HashSHA256))
if err != nil {
return nil, err
}
return io.ReadAll(obj)
}
// Get data related to pacman database file or symlink.
func LoadPacmanDatabase(ctx *context.Context, owner, distro, architecture, file string) ([]byte, error) {
cs := packages.NewContentStore()
file = strings.TrimPrefix(file, owner+".")
dbname := strings.TrimSuffix(arch.Join(owner, distro, architecture, file), ".tar.gz")
obj, err := cs.Get(packages.BlobHash256Key(dbname))
if err != nil {
return nil, err
}
return io.ReadAll(obj)
}
// This function will update information about package in related pacman databases
// or create them if they do not exist.
func UpdatePacmanDatabases(ctx *context.Context, md *arch.Metadata, distro, owner string) error {
// If architecure is not specified or any, package will be automatically
// saved to pacman databases with most popular architectures.
if len(md.Arch) == 0 || md.Arch[0] == "any" {
md.Arch = popularArchitectures()
}
cs := packages.NewContentStore()
// Update pacman database files for each architecture.
for _, architecture := range md.Arch {
db := arch.Join(owner, distro, architecture, setting.Domain, "db")
dbkey := packages.BlobHash256Key(db)
var dbdata []byte
dbobj, err := cs.Get(dbkey)
if err == nil {
dbdata, err = io.ReadAll(dbobj)
if err != nil {
return err
}
}
newdata, err := arch.UpdatePacmanDbEntry(dbdata, md)
if err != nil {
return err
}
err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata)))
if err != nil {
return err
}
}
return nil
}
func RemoveDbEntry(ctx *context.Context, architectures []string, owner, distro, pkg, ver string) error {
cs := packages.NewContentStore()
// If architecures are not specified or any, package will be automatically
// removed from pacman databases with most popular architectures.
if len(architectures) == 0 || architectures[0] == "any" {
architectures = popularArchitectures()
}
for _, architecture := range architectures {
db := arch.Join(owner, distro, architecture, setting.Domain, "db")
dbkey := packages.BlobHash256Key(db)
var dbdata []byte
dbobj, err := cs.Get(dbkey)
if err != nil {
return err
}
dbdata, err = io.ReadAll(dbobj)
if err != nil {
return err
}
newdata, err := arch.RemoveDbEntry(dbdata, pkg, ver)
if err != nil {
return err
}
err = cs.Save(dbkey, bytes.NewReader(newdata), int64(len(newdata)))
if err != nil {
return err
}
}
return nil
}
func popularArchitectures() []string {
return []string{
"x86_64", "arm", "i686", "pentium4",
"armv7h", "armv6h", "aarch64", "riscv64",
}
}