From 808a7aed6725661fb43e59f2e319a4dd2d61b44e Mon Sep 17 00:00:00 2001 From: dancheg97 Date: Wed, 21 Jun 2023 14:05:25 +0300 Subject: [PATCH] replaced zstd with archiver and limit reader to speed up metadata ejection and remove windows incompatible dependency --- go.mod | 2 +- modules/packages/arch/metadata.go | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7d783a7524..8e297a6a16 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( gitea.com/lunny/levelqueue v0.4.2-0.20220729054728-f020868cc2f7 github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 - github.com/DataDog/zstd v1.4.5 github.com/NYTimes/gziphandler v1.1.1 github.com/ProtonMail/gopenpgp/v2 v2.7.1 github.com/PuerkitoBio/goquery v1.8.1 @@ -133,6 +132,7 @@ require ( git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect github.com/ClickHouse/ch-go v0.55.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.9.1 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 5433138a67..e0fba239e2 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -4,11 +4,13 @@ package arch import ( + "bytes" "crypto/md5" "crypto/sha256" "encoding/hex" "errors" "fmt" + "io" "io/fs" "os" "path" @@ -16,7 +18,6 @@ import ( "strings" "time" - "github.com/DataDog/zstd" "github.com/mholt/archiver/v3" ) @@ -50,11 +51,15 @@ type Metadata struct { // Function that recieves arch package archive data and returns it's metadata. func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { - d, err := zstd.Decompress(nil, pkg) + pkgreader := io.LimitReader(bytes.NewReader(pkg), 250000) + var buf bytes.Buffer + err := archiver.DefaultZstd.Decompress(pkgreader, &buf) if err != nil { - return nil, err + if !errors.Is(err, io.ErrUnexpectedEOF) { + return nil, err + } } - splt := strings.Split(string(d), "PKGINFO") + splt := strings.Split(buf.String(), "PKGINFO") if len(splt) < 2 { return nil, errors.New("unable to eject .PKGINFO from archive") } @@ -87,7 +92,7 @@ func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) { MakeDepends: ejectStrings(raw, "makedepend"), CheckDepends: ejectStrings(raw, "checkdepend"), Backup: ejectStrings(raw, "backup"), - }, err + }, nil } func ejectString(raw, field string) string {