additional validations for package metadata fields, split package metadata into file and version related structures, better test coverage and refactoring in metadata module

This commit is contained in:
Danila Fominykh 2023-10-30 21:53:14 -03:00
parent f8b78cba23
commit 8aeada8c30
No known key found for this signature in database
GPG Key ID: FC04D07F75B663FC
6 changed files with 587 additions and 264 deletions

View File

@ -142,7 +142,7 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
case TypeAlpine: case TypeAlpine:
metadata = &alpine.VersionMetadata{} metadata = &alpine.VersionMetadata{}
case TypeArch: case TypeArch:
metadata = &arch.Metadata{} metadata = &arch.VersionMetadata{}
case TypeCargo: case TypeCargo:
metadata = &cargo.Metadata{} metadata = &cargo.Metadata{}
case TypeChef: case TypeChef:

View File

@ -6,50 +6,38 @@ package arch
import ( import (
"archive/tar" "archive/tar"
"bufio" "bufio"
"bytes"
"compress/gzip" "compress/gzip"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
"regexp"
"strconv" "strconv"
"strings" "strings"
packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"github.com/mholt/archiver/v3" "github.com/mholt/archiver/v3"
) )
// JSON with pacakage parameters that are not related to specific type Package struct {
// architecture/distribution that will be stored in sql database. Name string `json:"name"`
type Metadata struct { Version string `json:"version"`
URL string `json:"url"` VersionMetadata VersionMetadata
Description string `json:"description"` FileMetadata FileMetadata
Provides []string `json:"provides,omitempty"`
License []string `json:"license,omitempty"`
Depends []string `json:"depends,omitempty"`
OptDepends []string `json:"opt_depends,omitempty"`
MakeDepends []string `json:"make_depends,omitempty"`
CheckDepends []string `json:"check_depends,omitempty"`
} }
// Package description file that will be saved as .desc file in object storage. // Arch package metadata related to specific version.
// This file will be used to create pacman database. // Version metadata the same across different architectures and distributions.
type DbDesc struct { type VersionMetadata struct {
Filename string `json:"filename"`
Name string `json:"name"`
Base string `json:"base"` Base string `json:"base"`
Version string `json:"version"`
Description string `json:"description"` Description string `json:"description"`
CompressedSize int64 `json:"compressed_size"`
InstalledSize int64 `json:"installed_size"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
ProjectURL string `json:"project_url"` ProjectURL string `json:"project_url"`
BuildDate int64 `json:"build_date"` Groups []string `json:"groups,omitempty"`
Packager string `json:"packager"`
Provides []string `json:"provides,omitempty"` Provides []string `json:"provides,omitempty"`
License []string `json:"license,omitempty"` License []string `json:"license,omitempty"`
Arch []string `json:"arch,omitempty"`
Depends []string `json:"depends,omitempty"` Depends []string `json:"depends,omitempty"`
OptDepends []string `json:"opt_depends,omitempty"` OptDepends []string `json:"opt_depends,omitempty"`
MakeDepends []string `json:"make_depends,omitempty"` MakeDepends []string `json:"make_depends,omitempty"`
@ -57,143 +45,235 @@ type DbDesc struct {
Backup []string `json:"backup,omitempty"` Backup []string `json:"backup,omitempty"`
} }
// Metadata related to specific pakcage file.
// This metadata might vary for different architecture and distribution.
type FileMetadata struct {
CompressedSize int64 `json:"compressed_size"`
InstalledSize int64 `json:"installed_size"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
BuildDate int64 `json:"build_date"`
Packager string `json:"packager"`
Arch string `json:"arch"`
}
// Function that receives arch package archive data and returns it's metadata. // Function that receives arch package archive data and returns it's metadata.
func ParseMetadata(file, distro string, b *packages_module.HashedBuffer) (*DbDesc, error) { func ParsePackage(r io.Reader, md5, sha256 []byte, size int64) (*Package, error) {
pkginfo, err := getPkginfo(b)
if err != nil {
return nil, err
}
// Add package blob parameters to arch related desc.
hashMD5, _, hashSHA256, _ := b.Sums()
md := DbDesc{
Filename: file,
Name: file,
CompressedSize: b.Size(),
MD5: hex.EncodeToString(hashMD5),
SHA256: hex.EncodeToString(hashSHA256),
}
for _, line := range strings.Split(pkginfo, "\n") {
splt := strings.Split(line, " = ")
if len(splt) != 2 {
continue
}
var (
parameter = splt[0]
value = splt[1]
)
switch parameter {
case "pkgname":
md.Name = value
case "pkgbase":
md.Base = value
case "pkgver":
md.Version = value
case "pkgdesc":
md.Description = value
case "url":
md.ProjectURL = value
case "packager":
md.Packager = value
case "provides":
md.Provides = append(md.Provides, value)
case "license":
md.License = append(md.License, value)
case "arch":
md.Arch = append(md.Arch, value)
case "depend":
md.Depends = append(md.Depends, value)
case "optdepend":
md.OptDepends = append(md.OptDepends, value)
case "makedepend":
md.MakeDepends = append(md.MakeDepends, value)
case "checkdepend":
md.CheckDepends = append(md.CheckDepends, value)
case "backup":
md.Backup = append(md.Backup, value)
case "builddate":
md.BuildDate, err = strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, err
}
case "size":
md.InstalledSize, err = strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, err
}
}
}
return &md, nil
}
// Eject .PKGINFO file as string from package archive.
func getPkginfo(data io.Reader) (string, error) {
br := bufio.NewReader(data)
zstd := archiver.NewTarZstd() zstd := archiver.NewTarZstd()
err := zstd.Open(br, int64(250000)) err := zstd.Open(r, 0)
if err != nil { if err != nil {
return ``, err return nil, err
} }
defer zstd.Close()
var pkg *Package
var mtree bool
for { for {
f, err := zstd.Read() f, err := zstd.Read()
if err != nil { if err == io.EOF {
return ``, err break
} }
if f.Name() != ".PKGINFO" { if err != nil {
return nil, err
}
defer f.Close()
switch f.Name() {
case ".PKGINFO":
pkg, err = ParsePackageInfo(f)
if err != nil {
return nil, err
}
case ".MTREE":
mtree = true
}
}
if pkg == nil {
return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found")
}
if !mtree {
return nil, util.NewInvalidArgumentErrorf(".MTREE file not found")
}
pkg.FileMetadata.CompressedSize = size
pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256)
pkg.FileMetadata.MD5 = hex.EncodeToString(md5)
return pkg, nil
}
// Function that accepts reader for .PKGINFO file from package archive,
// validates all field according to PKGBUILD spec and returns package.
func ParsePackageInfo(r io.Reader) (*Package, error) {
p := &Package{}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "#") {
continue continue
} }
b, err := io.ReadAll(f)
i := strings.IndexRune(line, '=')
if i == -1 {
continue
}
key := strings.TrimSpace(line[:i])
value := strings.TrimSpace(line[i+1:])
switch key {
case "pkgname":
p.Name = value
case "pkgbase":
p.VersionMetadata.Base = value
case "pkgver":
p.Version = value
case "pkgdesc":
p.VersionMetadata.Description = value
case "url":
p.VersionMetadata.ProjectURL = value
case "packager":
p.FileMetadata.Packager = value
case "arch":
p.FileMetadata.Arch = value
case "provides":
p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value)
case "license":
p.VersionMetadata.License = append(p.VersionMetadata.License, value)
case "depend":
p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value)
case "optdepend":
p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value)
case "makedepend":
p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value)
case "checkdepend":
p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value)
case "backup":
p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value)
case "group":
p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value)
case "builddate":
bd, err := strconv.ParseInt(value, 10, 64)
if err != nil { if err != nil {
return ``, err return nil, err
} }
return string(b), nil p.FileMetadata.BuildDate = bd
case "size":
is, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, err
} }
p.FileMetadata.InstalledSize = is
}
}
return p, errors.Join(scanner.Err(), ValidatePackageSpec(p))
}
// Arch package validation according to PKGBUILD specification:
// https://man.archlinux.org/man/PKGBUILD.5
func ValidatePackageSpec(p *Package) error {
var (
reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`)
reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`)
reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`)
rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`)
)
if !reName.MatchString(p.Name) {
return util.NewInvalidArgumentErrorf("invalid package name")
}
if !reName.MatchString(p.VersionMetadata.Base) {
return util.NewInvalidArgumentErrorf("invalid package base")
}
if !reVer.MatchString(p.Version) {
return util.NewInvalidArgumentErrorf("invalid package version")
}
if p.FileMetadata.Arch == "" {
return util.NewInvalidArgumentErrorf("architecture should be specified")
}
if p.VersionMetadata.ProjectURL != "" {
if !validation.IsValidURL(p.VersionMetadata.ProjectURL) {
return util.NewInvalidArgumentErrorf("invalid project URL")
}
}
for _, cd := range p.VersionMetadata.CheckDepends {
if !rePkgVer.MatchString(cd) {
return util.NewInvalidArgumentErrorf("invalid check dependency: " + cd)
}
}
for _, d := range p.VersionMetadata.Depends {
if !rePkgVer.MatchString(d) {
return util.NewInvalidArgumentErrorf("invalid dependency: " + d)
}
}
for _, md := range p.VersionMetadata.MakeDepends {
if !rePkgVer.MatchString(md) {
return util.NewInvalidArgumentErrorf("invalid make dependency: " + md)
}
}
for _, p := range p.VersionMetadata.Provides {
if !rePkgVer.MatchString(p) {
return util.NewInvalidArgumentErrorf("invalid provides: " + p)
}
}
for _, od := range p.VersionMetadata.OptDepends {
if !reOptDep.MatchString(od) {
return util.NewInvalidArgumentErrorf("invalid optional dependency: " + od)
}
}
for _, bf := range p.VersionMetadata.Backup {
if strings.HasPrefix(bf, "/") {
return util.NewInvalidArgumentErrorf("backup file contains leading forward slash")
}
}
return nil
} }
// Create pacman package description file. // Create pacman package description file.
func (m *DbDesc) String() string { func (p *Package) Desc() string {
entries := []struct{ key, value string }{ entries := [40]string{
{"FILENAME", m.Filename}, "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch),
{"NAME", m.Name}, "NAME", p.Name,
{"BASE", m.Base}, "BASE", p.VersionMetadata.Base,
{"VERSION", m.Version}, "VERSION", p.Version,
{"DESC", m.Description}, "DESC", p.VersionMetadata.Description,
{"CSIZE", fmt.Sprintf("%d", m.CompressedSize)}, "GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"),
{"ISIZE", fmt.Sprintf("%d", m.InstalledSize)}, "CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize),
{"MD5SUM", m.MD5}, "ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize),
{"SHA256SUM", m.SHA256}, "MD5SUM", p.FileMetadata.MD5,
{"URL", m.ProjectURL}, "SHA256SUM", p.FileMetadata.SHA256,
{"LICENSE", strings.Join(m.License, "\n")}, "URL", p.VersionMetadata.ProjectURL,
{"ARCH", strings.Join(m.Arch, "\n")}, "LICENSE", strings.Join(p.VersionMetadata.License, "\n"),
{"BUILDDATE", fmt.Sprintf("%d", m.BuildDate)}, "ARCH", p.FileMetadata.Arch,
{"PACKAGER", m.Packager}, "BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate),
{"PROVIDES", strings.Join(m.Provides, "\n")}, "PACKAGER", p.FileMetadata.Packager,
{"DEPENDS", strings.Join(m.Depends, "\n")}, "PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"),
{"OPTDEPENDS", strings.Join(m.OptDepends, "\n")}, "DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"),
{"MAKEDEPENDS", strings.Join(m.MakeDepends, "\n")}, "OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"),
{"CHECKDEPENDS", strings.Join(m.CheckDepends, "\n")}, "MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"),
"CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"),
} }
var result string var result string
for _, e := range entries { for i := 0; i < 40; i += 2 {
if e.value != "" { if entries[i+1] != "" {
result += fmt.Sprintf("%%%s%%\n%s\n\n", e.key, e.value) result += fmt.Sprintf("%%%s%%\n%s\n\n", entries[i], entries[i+1])
} }
} }
return result return result
} }
// Create pacman database archive based on provided package metadata structs. // Create pacman database archive based on provided package metadata structs.
func CreatePacmanDb(entries map[string][]byte) (io.ReadSeeker, error) { func CreatePacmanDb(entries map[string][]byte) (*bytes.Buffer, error) {
out, err := packages_module.NewHashedBuffer() var b bytes.Buffer
if err != nil {
return nil, err
}
gw := gzip.NewWriter(out) gw := gzip.NewWriter(&b)
tw := tar.NewWriter(gw) tw := tar.NewWriter(gw)
for name, content := range entries { for name, content := range entries {
@ -204,20 +284,13 @@ func CreatePacmanDb(entries map[string][]byte) (io.ReadSeeker, error) {
} }
if err := tw.WriteHeader(header); err != nil { if err := tw.WriteHeader(header); err != nil {
tw.Close() return nil, errors.Join(err, tw.Close(), gw.Close())
gw.Close()
return nil, err
} }
if _, err := tw.Write(content); err != nil { if _, err := tw.Write(content); err != nil {
tw.Close() return nil, errors.Join(err, tw.Close(), gw.Close())
gw.Close()
return nil, err
} }
} }
tw.Close() return &b, errors.Join(tw.Close(), gw.Close())
gw.Close()
return out, nil
} }

View File

@ -4,43 +4,336 @@
package arch package arch
import ( import (
"bytes"
"encoding/base64" "encoding/base64"
"errors"
"io" "io"
"os" "os"
"strings"
"testing" "testing"
"testing/fstest" "testing/fstest"
"time" "time"
packages_module "code.gitea.io/gitea/modules/packages"
"github.com/mholt/archiver/v3" "github.com/mholt/archiver/v3"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
const pkginfo = `# Generated by makepkg 6.0.2 func TestParsePackage(t *testing.T) {
// Minimal PKGINFO contents and test FS
const PKGINFO = `pkgname = a
pkgbase = b
pkgver = 1-2
arch = x86_64
`
fs := fstest.MapFS{
"pkginfo": &fstest.MapFile{
Data: []byte(PKGINFO),
Mode: os.ModePerm,
ModTime: time.Now(),
},
"mtree": &fstest.MapFile{
Data: []byte("data"),
Mode: os.ModePerm,
ModTime: time.Now(),
},
}
// Test .PKGINFO file
pinf, err := fs.Stat("pkginfo")
assert.NoError(t, err)
pfile, err := fs.Open("pkginfo")
assert.NoError(t, err)
parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO")
assert.NoError(t, err)
// Test .MTREE file
minf, err := fs.Stat("mtree")
assert.NoError(t, err)
mfile, err := fs.Open("mtree")
assert.NoError(t, err)
marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE")
assert.NoError(t, err)
t.Run("normal archive", func(t *testing.T) {
var buf bytes.Buffer
archive := archiver.NewTarZstd()
archive.Create(&buf)
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: pinf,
CustomName: parcname,
},
ReadCloser: pfile,
})
assert.NoError(t, errors.Join(pfile.Close(), err))
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: minf,
CustomName: marcname,
},
ReadCloser: mfile,
})
assert.NoError(t, errors.Join(mfile.Close(), archive.Close(), err))
_, err = ParsePackage(&buf, []byte{}, []byte{}, 0)
assert.NoError(t, err)
})
t.Run("missing .PKGINFO", func(t *testing.T) {
var buf bytes.Buffer
archive := archiver.NewTarZstd()
archive.Create(&buf)
assert.NoError(t, archive.Close())
_, err = ParsePackage(&buf, []byte{}, []byte{}, 0)
assert.Error(t, err)
assert.Contains(t, err.Error(), ".PKGINFO file not found")
})
t.Run("missing .MTREE", func(t *testing.T) {
var buf bytes.Buffer
pfile, err := fs.Open("pkginfo")
assert.NoError(t, err)
archive := archiver.NewTarZstd()
archive.Create(&buf)
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: pinf,
CustomName: parcname,
},
ReadCloser: pfile,
})
assert.NoError(t, errors.Join(pfile.Close(), archive.Close(), err))
_, err = ParsePackage(&buf, []byte{}, []byte{}, 0)
assert.Error(t, err)
assert.Contains(t, err.Error(), ".MTREE file not found")
})
}
func TestParsePackageInfo(t *testing.T) {
const PKGINFO = `# Generated by makepkg 6.0.2
# using fakeroot version 1.31 # using fakeroot version 1.31
pkgname = zstd pkgname = a
pkgbase = zstd pkgbase = b
pkgver = 1.5.5-1 pkgver = 1-2
pkgdesc = Zstandard - Fast real-time compression algorithm pkgdesc = comment
url = https://facebook.github.io/zstd/ url = https://example.com/
builddate = 1681646714 group = group
packager = Jelle van der Waa <jelle@archlinux.org> builddate = 3
size = 1500453 packager = Name Surname <login@example.com>
size = 5
arch = x86_64 arch = x86_64
license = BSD license = BSD
license = GPL2 provides = pvd
provides = libzstd.so=1-64 depend = smth
depend = glibc optdepend = hex
depend = gcc-libs checkdepend = ola
depend = zlib
depend = xz
depend = lz4
makedepend = cmake makedepend = cmake
makedepend = gtest backup = usr/bin/paket1
makedepend = ninja
` `
p, err := ParsePackageInfo(strings.NewReader(PKGINFO))
assert.NoError(t, err)
assert.Equal(t, Package{
Name: "a",
Version: "1-2",
VersionMetadata: VersionMetadata{
Base: "b",
Description: "comment",
ProjectURL: "https://example.com/",
Groups: []string{"group"},
Provides: []string{"pvd"},
License: []string{"BSD"},
Depends: []string{"smth"},
OptDepends: []string{"hex"},
MakeDepends: []string{"cmake"},
CheckDepends: []string{"ola"},
Backup: []string{"usr/bin/paket1"},
},
FileMetadata: FileMetadata{
InstalledSize: 5,
BuildDate: 3,
Packager: "Name Surname <login@example.com>",
Arch: "x86_64",
},
}, *p)
}
func TestValidatePackageSpec(t *testing.T) {
var newpkg = func() Package {
return Package{
Name: "abc",
Version: "1-1",
VersionMetadata: VersionMetadata{
Base: "ghx",
Description: "whoami",
ProjectURL: "https://example.com/",
Groups: []string{"gnome"},
Provides: []string{"abc", "def"},
License: []string{"GPL"},
Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"},
OptDepends: []string{"git: something", "make"},
MakeDepends: []string{"chrom"},
CheckDepends: []string{"bariy"},
Backup: []string{"etc/pacman.d/filo"},
},
FileMetadata: FileMetadata{
CompressedSize: 1,
InstalledSize: 2,
MD5: "abc",
SHA256: "def",
BuildDate: 3,
Packager: "smon",
Arch: "x86_64",
},
}
}
t.Run("valid package", func(t *testing.T) {
p := newpkg()
err := ValidatePackageSpec(&p)
assert.NoError(t, err)
})
t.Run("invalid package name", func(t *testing.T) {
p := newpkg()
p.Name = "!$%@^!*&()"
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid package name")
})
t.Run("invalid package base", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Base = "!$%@^!*&()"
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid package base")
})
t.Run("invalid package version", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Base = "una-luna?"
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid package base")
})
t.Run("invalid package version", func(t *testing.T) {
p := newpkg()
p.Version = "una-luna"
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid package version")
})
t.Run("missing architecture", func(t *testing.T) {
p := newpkg()
p.FileMetadata.Arch = ""
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "architecture should be specified")
})
t.Run("invalid URL", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.ProjectURL = "http%%$#"
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid project URL")
})
t.Run("invalid check dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.CheckDepends = []string{"Err^_^"}
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid check dependency")
})
t.Run("invalid dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Depends = []string{"^^abc"}
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid dependency")
})
t.Run("invalid make dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.MakeDepends = []string{"^m^"}
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid make dependency")
})
t.Run("invalid provides", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Provides = []string{"^m^"}
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid provides")
})
t.Run("invalid optional dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.OptDepends = []string{"^m^:MM"}
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid optional dependency")
})
t.Run("invalid optional dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Backup = []string{"/ola/cola"}
err := ValidatePackageSpec(&p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "backup file contains leading forward slash")
})
}
func TestDescString(t *testing.T) {
const pkgdesc = `%FILENAME% const pkgdesc = `%FILENAME%
zstd-1.5.5-1-x86_64.pkg.tar.zst zstd-1.5.5-1-x86_64.pkg.tar.zst
@ -56,6 +349,10 @@ zstd
%DESC% %DESC%
Zstandard - Fast real-time compression algorithm Zstandard - Fast real-time compression algorithm
%GROUPS%
dummy1
dummy2
%CSIZE% %CSIZE%
401 401
@ -94,93 +391,52 @@ zlib
xz xz
lz4 lz4
%OPTDEPENDS%
dummy3
dummy4
%MAKEDEPENDS% %MAKEDEPENDS%
cmake cmake
gtest gtest
ninja ninja
%CHECKDEPENDS%
dummy5
dummy6
` `
const dbarchive = "H4sIAAAAAAAA/0rLzEnVS60oYaAhMDAwMDA3NwfTBgYG6LSBgYEpEtuAwcDQwMzUgEHBgJaOgoHS4pLEIgYDiu1C99wQASmlubmVA+2IUTAKRsEoGAV0B4AAAAD//2VF3KIACAAA" md := &Package{
func TestMetadata(t *testing.T) {
fs := fstest.MapFS{
"pkginfo": &fstest.MapFile{
Data: []byte(pkginfo),
Mode: os.ModePerm,
ModTime: time.Now(),
},
}
info, err := fs.Stat("pkginfo")
assert.NoError(t, err)
file, err := fs.Open("pkginfo")
assert.NoError(t, err)
buf, err := packages_module.NewHashedBuffer()
assert.NoError(t, err)
archive := archiver.NewTarZstd()
archive.Create(buf)
n, err := archiver.NameInArchive(info, ".PKGINFO", ".PKGINFO")
assert.NoError(t, err)
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: info,
CustomName: n,
},
ReadCloser: file,
})
file.Close()
archive.Close()
assert.NoError(t, err)
md, err := ParseMetadata("zstd-1.5.5-1-x86_64.pkg.tar.zst", "archlinux", buf)
assert.NoError(t, err)
assert.Equal(t, md.Name, "zstd")
assert.Equal(t, md.Base, "zstd")
assert.Equal(t, md.Version, "1.5.5-1")
assert.Equal(t, md.Description, "Zstandard - Fast real-time compression algorithm")
assert.Equal(t, md.ProjectURL, "https://facebook.github.io/zstd/")
assert.Equal(t, md.BuildDate, int64(1681646714))
assert.Equal(t, md.Packager, "Jelle van der Waa <jelle@archlinux.org>")
assert.Equal(t, md.InstalledSize, int64(1500453))
assert.Equal(t, md.Arch, []string{"x86_64"})
assert.Equal(t, md.License, []string{"BSD", "GPL2"})
assert.Equal(t, md.Provides, []string{"libzstd.so=1-64"})
assert.Equal(t, md.Depends, []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"})
assert.Equal(t, md.MakeDepends, []string{"cmake", "gtest", "ninja"})
}
func TestDescString(t *testing.T) {
md := &DbDesc{
Filename: "zstd-1.5.5-1-x86_64.pkg.tar.zst",
Name: "zstd", Name: "zstd",
Base: "zstd",
Version: "1.5.5-1", Version: "1.5.5-1",
VersionMetadata: VersionMetadata{
Base: "zstd",
Description: "Zstandard - Fast real-time compression algorithm", Description: "Zstandard - Fast real-time compression algorithm",
ProjectURL: "https://facebook.github.io/zstd/",
Groups: []string{"dummy1", "dummy2"},
Provides: []string{"libzstd.so=1-64"},
License: []string{"BSD", "GPL2"},
Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"},
OptDepends: []string{"dummy3", "dummy4"},
MakeDepends: []string{"cmake", "gtest", "ninja"},
CheckDepends: []string{"dummy5", "dummy6"},
},
FileMetadata: FileMetadata{
CompressedSize: 401, CompressedSize: 401,
InstalledSize: 1500453, InstalledSize: 1500453,
MD5: "5016660ef3d9aa148a7b72a08d3df1b2", MD5: "5016660ef3d9aa148a7b72a08d3df1b2",
SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd",
ProjectURL: "https://facebook.github.io/zstd/",
BuildDate: 1681646714, BuildDate: 1681646714,
Packager: "Jelle van der Waa <jelle@archlinux.org>", Packager: "Jelle van der Waa <jelle@archlinux.org>",
Provides: []string{"libzstd.so=1-64"}, Arch: "x86_64",
License: []string{"BSD", "GPL2"}, },
Arch: []string{"x86_64"},
Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"},
MakeDepends: []string{"cmake", "gtest", "ninja"},
} }
desc := md.String() assert.Equal(t, pkgdesc, md.Desc())
assert.Equal(t, pkgdesc, desc)
} }
func TestDatabase(t *testing.T) { func TestCreatePacmanDb(t *testing.T) {
const dbarchive = "H4sIAAAAAAAA/0rLzEnVS60oYaAhMDAwMDA3NwfTBgYG6LSBgYEpEtuAwcDQwMzUgEHBgJaOgoHS4pLEIgYDiu1C99wQASmlubmVA+2IUTAKRsEoGAV0B4AAAAD//2VF3KIACAAA"
db, err := CreatePacmanDb(map[string][]byte{ db, err := CreatePacmanDb(map[string][]byte{
"file.ext": []byte("dummy"), "file.ext": []byte("dummy"),
}) })

View File

@ -4,6 +4,7 @@
package arch package arch
import ( import (
"bytes"
"net/http" "net/http"
"strings" "strings"
@ -95,7 +96,7 @@ func Get(ctx *context.Context) {
return return
} }
ctx.ServeContent(db, &context.ServeHeaderOptions{ ctx.ServeContent(bytes.NewReader(db.Bytes()), &context.ServeHeaderOptions{
Filename: file, Filename: file,
}) })
return return

View File

@ -83,7 +83,7 @@ func getPackageFile(ctx *context.Context, distro, file string) (*packages_model.
// requested combination of architecture and distribution. When/If the first // requested combination of architecture and distribution. When/If the first
// compatible version is found, related desc file will be loaded from package // compatible version is found, related desc file will be loaded from package
// properties and added to resulting .db.tar.gz archive. // properties and added to resulting .db.tar.gz archive.
func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (io.ReadSeeker, error) { func CreatePacmanDb(ctx *context.Context, owner, arch, distro string) (*bytes.Buffer, error) {
pkgs, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeArch) pkgs, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeArch)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -24,7 +24,9 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro,
} }
defer buf.Close() defer buf.Close()
desc, err := arch_module.ParseMetadata(filename, distro, buf) md5, _, sha256, _ := buf.Sums()
p, err := arch_module.ParsePackage(buf, md5, sha256, buf.Size())
if err != nil { if err != nil {
return false, nil, err return false, nil, err
} }
@ -35,7 +37,7 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro,
} }
properties := map[string]string{ properties := map[string]string{
"desc": desc.String(), "desc": p.Desc(),
} }
if sign != "" { if sign != "" {
_, err := hex.DecodeString(sign) _, err := hex.DecodeString(sign)
@ -50,20 +52,11 @@ func UploadArchPackage(ctx *context.Context, upload io.Reader, filename, distro,
PackageInfo: packages_service.PackageInfo{ PackageInfo: packages_service.PackageInfo{
Owner: ctx.Package.Owner, Owner: ctx.Package.Owner,
PackageType: packages_model.TypeArch, PackageType: packages_model.TypeArch,
Name: desc.Name, Name: p.Name,
Version: desc.Version, Version: p.Version,
}, },
Creator: ctx.Doer, Creator: ctx.Doer,
Metadata: &arch_module.Metadata{ Metadata: p.VersionMetadata,
URL: desc.ProjectURL,
Description: desc.Description,
Provides: desc.Provides,
License: desc.License,
Depends: desc.Depends,
OptDepends: desc.OptDepends,
MakeDepends: desc.MakeDepends,
CheckDepends: desc.CheckDepends,
},
}, },
&packages_service.PackageFileCreationInfo{ &packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{ PackageFileInfo: packages_service.PackageFileInfo{