mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-11-14 22:59:29 +01:00
feat: add architecture-specific removal support for arch package (#5351)
Some checks are pending
/ release (push) Waiting to run
testing / backend-checks (push) Waiting to run
testing / frontend-checks (push) Waiting to run
testing / test-unit (push) Blocked by required conditions
testing / test-remote-cacher (map[image:docker.io/bitnami/redis:7.2 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:docker.io/bitnami/valkey:7.2 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:ghcr.io/microsoft/garnet-alpine:1.0.14 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:registry.redict.io/redict:7.3.0-scratch port:6379]) (push) Blocked by required conditions
testing / test-mysql (push) Blocked by required conditions
testing / test-pgsql (push) Blocked by required conditions
testing / test-sqlite (push) Blocked by required conditions
testing / security-check (push) Blocked by required conditions
Some checks are pending
/ release (push) Waiting to run
testing / backend-checks (push) Waiting to run
testing / frontend-checks (push) Waiting to run
testing / test-unit (push) Blocked by required conditions
testing / test-remote-cacher (map[image:docker.io/bitnami/redis:7.2 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:docker.io/bitnami/valkey:7.2 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:ghcr.io/microsoft/garnet-alpine:1.0.14 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:registry.redict.io/redict:7.3.0-scratch port:6379]) (push) Blocked by required conditions
testing / test-mysql (push) Blocked by required conditions
testing / test-pgsql (push) Blocked by required conditions
testing / test-sqlite (push) Blocked by required conditions
testing / security-check (push) Blocked by required conditions
- [x] add architecture-specific removal support - [x] Fix upload competition - [x] Fix not checking input when downloading docs: https://codeberg.org/forgejo/docs/pulls/874 ### Release notes - [ ] I do not want this change to show in the release notes. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5351 Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org> Co-authored-by: Exploding Dragon <explodingfkl@gmail.com> Co-committed-by: Exploding Dragon <explodingfkl@gmail.com>
This commit is contained in:
parent
89d9307d56
commit
89742c4913
7 changed files with 151 additions and 100 deletions
|
@ -39,8 +39,8 @@ const (
|
||||||
var (
|
var (
|
||||||
reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`)
|
reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`)
|
||||||
reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`)
|
reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`)
|
||||||
reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?(:.*)?$`)
|
reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?(:.*)?$`)
|
||||||
rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?$`)
|
rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?$`)
|
||||||
|
|
||||||
magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD}
|
magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD}
|
||||||
magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A}
|
magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A}
|
||||||
|
@ -71,7 +71,7 @@ type VersionMetadata struct {
|
||||||
Conflicts []string `json:"conflicts,omitempty"`
|
Conflicts []string `json:"conflicts,omitempty"`
|
||||||
Replaces []string `json:"replaces,omitempty"`
|
Replaces []string `json:"replaces,omitempty"`
|
||||||
Backup []string `json:"backup,omitempty"`
|
Backup []string `json:"backup,omitempty"`
|
||||||
Xdata []string `json:"xdata,omitempty"`
|
XData []string `json:"xdata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileMetadata Metadata related to specific package file.
|
// FileMetadata Metadata related to specific package file.
|
||||||
|
@ -125,7 +125,7 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) {
|
||||||
defer tarball.Close()
|
defer tarball.Close()
|
||||||
|
|
||||||
var pkg *Package
|
var pkg *Package
|
||||||
var mtree bool
|
var mTree bool
|
||||||
|
|
||||||
for {
|
for {
|
||||||
f, err := tarball.Read()
|
f, err := tarball.Read()
|
||||||
|
@ -135,24 +135,24 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
switch f.Name() {
|
switch f.Name() {
|
||||||
case ".PKGINFO":
|
case ".PKGINFO":
|
||||||
pkg, err = ParsePackageInfo(tarballType, f)
|
pkg, err = ParsePackageInfo(tarballType, f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
_ = f.Close()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
case ".MTREE":
|
case ".MTREE":
|
||||||
mtree = true
|
mTree = true
|
||||||
}
|
}
|
||||||
|
_ = f.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
if pkg == nil {
|
if pkg == nil {
|
||||||
return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found")
|
return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !mtree {
|
if !mTree {
|
||||||
return nil, util.NewInvalidArgumentErrorf(".MTREE file not found")
|
return nil, util.NewInvalidArgumentErrorf(".MTREE file not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -220,7 +220,7 @@ func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) {
|
||||||
case "replaces":
|
case "replaces":
|
||||||
p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value)
|
p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value)
|
||||||
case "xdata":
|
case "xdata":
|
||||||
p.VersionMetadata.Xdata = append(p.VersionMetadata.Xdata, value)
|
p.VersionMetadata.XData = append(p.VersionMetadata.XData, value)
|
||||||
case "builddate":
|
case "builddate":
|
||||||
bd, err := strconv.ParseInt(value, 10, 64)
|
bd, err := strconv.ParseInt(value, 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -260,48 +260,43 @@ func ValidatePackageSpec(p *Package) error {
|
||||||
return util.NewInvalidArgumentErrorf("invalid project URL")
|
return util.NewInvalidArgumentErrorf("invalid project URL")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, cd := range p.VersionMetadata.CheckDepends {
|
for _, checkDepend := range p.VersionMetadata.CheckDepends {
|
||||||
if !rePkgVer.MatchString(cd) {
|
if !rePkgVer.MatchString(checkDepend) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid check dependency: %s", cd)
|
return util.NewInvalidArgumentErrorf("invalid check dependency: %s", checkDepend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, d := range p.VersionMetadata.Depends {
|
for _, depend := range p.VersionMetadata.Depends {
|
||||||
if !rePkgVer.MatchString(d) {
|
if !rePkgVer.MatchString(depend) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid dependency: %s", d)
|
return util.NewInvalidArgumentErrorf("invalid dependency: %s", depend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, md := range p.VersionMetadata.MakeDepends {
|
for _, makeDepend := range p.VersionMetadata.MakeDepends {
|
||||||
if !rePkgVer.MatchString(md) {
|
if !rePkgVer.MatchString(makeDepend) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid make dependency: %s", md)
|
return util.NewInvalidArgumentErrorf("invalid make dependency: %s", makeDepend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range p.VersionMetadata.Provides {
|
for _, provide := range p.VersionMetadata.Provides {
|
||||||
if !rePkgVer.MatchString(p) {
|
if !rePkgVer.MatchString(provide) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid provides: %s", p)
|
return util.NewInvalidArgumentErrorf("invalid provides: %s", provide)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range p.VersionMetadata.Conflicts {
|
for _, conflict := range p.VersionMetadata.Conflicts {
|
||||||
if !rePkgVer.MatchString(p) {
|
if !rePkgVer.MatchString(conflict) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid conflicts: %s", p)
|
return util.NewInvalidArgumentErrorf("invalid conflicts: %s", conflict)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range p.VersionMetadata.Replaces {
|
for _, replace := range p.VersionMetadata.Replaces {
|
||||||
if !rePkgVer.MatchString(p) {
|
if !rePkgVer.MatchString(replace) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid replaces: %s", p)
|
return util.NewInvalidArgumentErrorf("invalid replaces: %s", replace)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range p.VersionMetadata.Replaces {
|
for _, optDepend := range p.VersionMetadata.OptDepends {
|
||||||
if !rePkgVer.MatchString(p) {
|
if !reOptDep.MatchString(optDepend) {
|
||||||
return util.NewInvalidArgumentErrorf("invalid xdata: %s", p)
|
return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", optDepend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, od := range p.VersionMetadata.OptDepends {
|
for _, b := range p.VersionMetadata.Backup {
|
||||||
if !reOptDep.MatchString(od) {
|
if strings.HasPrefix(b, "/") {
|
||||||
return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", od)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, bf := range p.VersionMetadata.Backup {
|
|
||||||
if strings.HasPrefix(bf, "/") {
|
|
||||||
return util.NewInvalidArgumentErrorf("backup file contains leading forward slash")
|
return util.NewInvalidArgumentErrorf("backup file contains leading forward slash")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -175,18 +175,20 @@ func CommonRoutes() *web.Route {
|
||||||
arch.PushPackage(ctx)
|
arch.PushPackage(ctx)
|
||||||
return
|
return
|
||||||
} else if isDelete {
|
} else if isDelete {
|
||||||
if groupLen < 2 {
|
if groupLen < 3 {
|
||||||
ctx.Status(http.StatusBadRequest)
|
ctx.Status(http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if groupLen == 2 {
|
if groupLen == 3 {
|
||||||
ctx.SetParams("group", "")
|
ctx.SetParams("group", "")
|
||||||
ctx.SetParams("package", pathGroups[0])
|
ctx.SetParams("package", pathGroups[0])
|
||||||
ctx.SetParams("version", pathGroups[1])
|
ctx.SetParams("version", pathGroups[1])
|
||||||
|
ctx.SetParams("arch", pathGroups[2])
|
||||||
} else {
|
} else {
|
||||||
ctx.SetParams("group", strings.Join(pathGroups[:groupLen-2], "/"))
|
ctx.SetParams("group", strings.Join(pathGroups[:groupLen-3], "/"))
|
||||||
ctx.SetParams("package", pathGroups[groupLen-2])
|
ctx.SetParams("package", pathGroups[groupLen-3])
|
||||||
ctx.SetParams("version", pathGroups[groupLen-1])
|
ctx.SetParams("version", pathGroups[groupLen-2])
|
||||||
|
ctx.SetParams("arch", pathGroups[groupLen-1])
|
||||||
}
|
}
|
||||||
reqPackageAccess(perm.AccessModeWrite)(ctx)
|
reqPackageAccess(perm.AccessModeWrite)(ctx)
|
||||||
if ctx.Written() {
|
if ctx.Written() {
|
||||||
|
|
|
@ -9,12 +9,14 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
packages_model "code.gitea.io/gitea/models/packages"
|
packages_model "code.gitea.io/gitea/models/packages"
|
||||||
packages_module "code.gitea.io/gitea/modules/packages"
|
packages_module "code.gitea.io/gitea/modules/packages"
|
||||||
arch_module "code.gitea.io/gitea/modules/packages/arch"
|
arch_module "code.gitea.io/gitea/modules/packages/arch"
|
||||||
|
"code.gitea.io/gitea/modules/sync"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
"code.gitea.io/gitea/routers/api/packages/helper"
|
"code.gitea.io/gitea/routers/api/packages/helper"
|
||||||
"code.gitea.io/gitea/services/context"
|
"code.gitea.io/gitea/services/context"
|
||||||
|
@ -25,6 +27,8 @@ import (
|
||||||
var (
|
var (
|
||||||
archPkgOrSig = regexp.MustCompile(`^.*\.pkg\.tar\.\w+(\.sig)*$`)
|
archPkgOrSig = regexp.MustCompile(`^.*\.pkg\.tar\.\w+(\.sig)*$`)
|
||||||
archDBOrSig = regexp.MustCompile(`^.*.db(\.tar\.gz)*(\.sig)*$`)
|
archDBOrSig = regexp.MustCompile(`^.*.db(\.tar\.gz)*(\.sig)*$`)
|
||||||
|
|
||||||
|
locker = sync.NewExclusivePool()
|
||||||
)
|
)
|
||||||
|
|
||||||
func apiError(ctx *context.Context, status int, obj any) {
|
func apiError(ctx *context.Context, status int, obj any) {
|
||||||
|
@ -33,6 +37,14 @@ func apiError(ctx *context.Context, status int, obj any) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func refreshLocker(ctx *context.Context, group string) func() {
|
||||||
|
key := fmt.Sprintf("pkg_%d_arch_pkg_%s", ctx.Package.Owner.ID, group)
|
||||||
|
locker.CheckIn(key)
|
||||||
|
return func() {
|
||||||
|
locker.CheckOut(key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func GetRepositoryKey(ctx *context.Context) {
|
func GetRepositoryKey(ctx *context.Context) {
|
||||||
_, pub, err := arch_service.GetOrCreateKeyPair(ctx, ctx.Package.Owner.ID)
|
_, pub, err := arch_service.GetOrCreateKeyPair(ctx, ctx.Package.Owner.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -48,7 +60,8 @@ func GetRepositoryKey(ctx *context.Context) {
|
||||||
|
|
||||||
func PushPackage(ctx *context.Context) {
|
func PushPackage(ctx *context.Context) {
|
||||||
group := ctx.Params("group")
|
group := ctx.Params("group")
|
||||||
|
releaser := refreshLocker(ctx, group)
|
||||||
|
defer releaser()
|
||||||
upload, needToClose, err := ctx.UploadStream()
|
upload, needToClose, err := ctx.UploadStream()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
apiError(ctx, http.StatusInternalServerError, err)
|
apiError(ctx, http.StatusInternalServerError, err)
|
||||||
|
@ -154,6 +167,7 @@ func PushPackage(ctx *context.Context) {
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
apiError(ctx, http.StatusInternalServerError, err)
|
apiError(ctx, http.StatusInternalServerError, err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, group, p.FileMetadata.Arch); err != nil {
|
if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, group, p.FileMetadata.Arch); err != nil {
|
||||||
apiError(ctx, http.StatusInternalServerError, err)
|
apiError(ctx, http.StatusInternalServerError, err)
|
||||||
|
@ -169,7 +183,7 @@ func GetPackageOrDB(ctx *context.Context) {
|
||||||
arch = ctx.Params("arch")
|
arch = ctx.Params("arch")
|
||||||
)
|
)
|
||||||
if archPkgOrSig.MatchString(file) {
|
if archPkgOrSig.MatchString(file) {
|
||||||
pkg, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID)
|
pkg, u, pf, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, util.ErrNotExist) {
|
if errors.Is(err, util.ErrNotExist) {
|
||||||
apiError(ctx, http.StatusNotFound, err)
|
apiError(ctx, http.StatusNotFound, err)
|
||||||
|
@ -178,15 +192,12 @@ func GetPackageOrDB(ctx *context.Context) {
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
helper.ServePackageFile(ctx, pkg, u, pf)
|
||||||
ctx.ServeContent(pkg, &context.ServeHeaderOptions{
|
|
||||||
Filename: file,
|
|
||||||
})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if archDBOrSig.MatchString(file) {
|
if archDBOrSig.MatchString(file) {
|
||||||
pkg, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID,
|
pkg, u, pf, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID,
|
||||||
strings.HasSuffix(file, ".sig"))
|
strings.HasSuffix(file, ".sig"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, util.ErrNotExist) {
|
if errors.Is(err, util.ErrNotExist) {
|
||||||
|
@ -196,9 +207,7 @@ func GetPackageOrDB(ctx *context.Context) {
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
ctx.ServeContent(pkg, &context.ServeHeaderOptions{
|
helper.ServePackageFile(ctx, pkg, u, pf)
|
||||||
Filename: file,
|
|
||||||
})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -207,10 +216,13 @@ func GetPackageOrDB(ctx *context.Context) {
|
||||||
|
|
||||||
func RemovePackage(ctx *context.Context) {
|
func RemovePackage(ctx *context.Context) {
|
||||||
var (
|
var (
|
||||||
group = ctx.Params("group")
|
group = ctx.Params("group")
|
||||||
pkg = ctx.Params("package")
|
pkg = ctx.Params("package")
|
||||||
ver = ctx.Params("version")
|
ver = ctx.Params("version")
|
||||||
|
pkgArch = ctx.Params("arch")
|
||||||
)
|
)
|
||||||
|
releaser := refreshLocker(ctx, group)
|
||||||
|
defer releaser()
|
||||||
pv, err := packages_model.GetVersionByNameAndVersion(
|
pv, err := packages_model.GetVersionByNameAndVersion(
|
||||||
ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver,
|
ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver,
|
||||||
)
|
)
|
||||||
|
@ -229,7 +241,13 @@ func RemovePackage(ctx *context.Context) {
|
||||||
}
|
}
|
||||||
deleted := false
|
deleted := false
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
if file.CompositeKey == group {
|
extName := fmt.Sprintf("-%s.pkg.tar%s", pkgArch, filepath.Ext(file.LowerName))
|
||||||
|
if strings.HasSuffix(file.LowerName, ".sig") {
|
||||||
|
extName = fmt.Sprintf("-%s.pkg.tar%s.sig", pkgArch,
|
||||||
|
filepath.Ext(strings.TrimSuffix(file.LowerName, filepath.Ext(file.LowerName))))
|
||||||
|
}
|
||||||
|
if file.CompositeKey == group &&
|
||||||
|
strings.HasSuffix(file.LowerName, extName) {
|
||||||
deleted = true
|
deleted = true
|
||||||
err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file)
|
err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -242,6 +260,7 @@ func RemovePackage(ctx *context.Context) {
|
||||||
err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, group)
|
err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, group)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
apiError(ctx, http.StatusInternalServerError, err)
|
apiError(ctx, http.StatusInternalServerError, err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
ctx.Status(http.StatusNoContent)
|
ctx.Status(http.StatusNoContent)
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
|
@ -20,6 +21,7 @@ import (
|
||||||
packages_module "code.gitea.io/gitea/modules/packages"
|
packages_module "code.gitea.io/gitea/modules/packages"
|
||||||
arch_module "code.gitea.io/gitea/modules/packages/arch"
|
arch_module "code.gitea.io/gitea/modules/packages/arch"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/sync"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
packages_service "code.gitea.io/gitea/services/packages"
|
packages_service "code.gitea.io/gitea/services/packages"
|
||||||
|
|
||||||
|
@ -28,6 +30,8 @@ import (
|
||||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var locker = sync.NewExclusivePool()
|
||||||
|
|
||||||
func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) {
|
func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) {
|
||||||
return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeArch, arch_module.RepositoryPackage, arch_module.RepositoryVersion)
|
return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeArch, arch_module.RepositoryPackage, arch_module.RepositoryVersion)
|
||||||
}
|
}
|
||||||
|
@ -101,6 +105,9 @@ func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages
|
||||||
|
|
||||||
// BuildPacmanDB Create db signature cache
|
// BuildPacmanDB Create db signature cache
|
||||||
func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error {
|
func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error {
|
||||||
|
key := fmt.Sprintf("pkg_%d_arch_db_%s", ownerID, group)
|
||||||
|
locker.CheckIn(key)
|
||||||
|
defer locker.CheckOut(key)
|
||||||
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -173,15 +180,18 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
defer db.Close()
|
||||||
gw := gzip.NewWriter(db)
|
gw := gzip.NewWriter(db)
|
||||||
|
defer gw.Close()
|
||||||
tw := tar.NewWriter(gw)
|
tw := tar.NewWriter(gw)
|
||||||
|
defer tw.Close()
|
||||||
count := 0
|
count := 0
|
||||||
for _, pkg := range pkgs {
|
for _, pkg := range pkgs {
|
||||||
versions, err := packages_model.GetVersionsByPackageName(
|
versions, err := packages_model.GetVersionsByPackageName(
|
||||||
ctx, ownerID, packages_model.TypeArch, pkg.Name,
|
ctx, ownerID, packages_model.TypeArch, pkg.Name,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
return nil, err
|
||||||
}
|
}
|
||||||
sort.Slice(versions, func(i, j int) bool {
|
sort.Slice(versions, func(i, j int) bool {
|
||||||
return versions[i].CreatedUnix > versions[j].CreatedUnix
|
return versions[i].CreatedUnix > versions[j].CreatedUnix
|
||||||
|
@ -190,7 +200,7 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages
|
||||||
for _, ver := range versions {
|
for _, ver := range versions {
|
||||||
files, err := packages_model.GetFilesByVersionID(ctx, ver.ID)
|
files, err := packages_model.GetFilesByVersionID(ctx, ver.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
return nil, err
|
||||||
}
|
}
|
||||||
var pf *packages_model.PackageFile
|
var pf *packages_model.PackageFile
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
|
@ -213,7 +223,7 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages
|
||||||
ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription,
|
ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
return nil, err
|
||||||
}
|
}
|
||||||
if len(pps) >= 1 {
|
if len(pps) >= 1 {
|
||||||
meta := []byte(pps[0].Value)
|
meta := []byte(pps[0].Value)
|
||||||
|
@ -223,60 +233,50 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages
|
||||||
Mode: int64(os.ModePerm),
|
Mode: int64(os.ModePerm),
|
||||||
}
|
}
|
||||||
if err = tw.WriteHeader(header); err != nil {
|
if err = tw.WriteHeader(header); err != nil {
|
||||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
return nil, err
|
||||||
}
|
}
|
||||||
if _, err := tw.Write(meta); err != nil {
|
if _, err := tw.Write(meta); err != nil {
|
||||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
return nil, err
|
||||||
}
|
}
|
||||||
count++
|
count++
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
defer gw.Close()
|
|
||||||
defer tw.Close()
|
|
||||||
if count == 0 {
|
if count == 0 {
|
||||||
return nil, errors.Join(db.Close(), io.EOF)
|
return nil, io.EOF
|
||||||
}
|
}
|
||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPackageFile Get data related to provided filename and distribution, for package files
|
// GetPackageFile Get data related to provided filename and distribution, for package files
|
||||||
// update download counter.
|
// update download counter.
|
||||||
func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, error) {
|
func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
|
||||||
pf, err := getPackageFile(ctx, group, file, ownerID)
|
fileSplit := strings.Split(file, "-")
|
||||||
if err != nil {
|
if len(fileSplit) <= 3 {
|
||||||
return nil, err
|
return nil, nil, nil, errors.New("invalid file format, need <name>-<version>-<release>-<arch>.pkg.<archive>")
|
||||||
}
|
}
|
||||||
|
|
||||||
filestream, _, _, err := packages_service.GetPackageFileStream(ctx, pf)
|
|
||||||
return filestream, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ejects parameters required to get package file property from file name.
|
|
||||||
func getPackageFile(ctx context.Context, group, file string, ownerID int64) (*packages_model.PackageFile, error) {
|
|
||||||
var (
|
var (
|
||||||
splt = strings.Split(file, "-")
|
pkgName = strings.Join(fileSplit[0:len(fileSplit)-3], "-")
|
||||||
pkgname = strings.Join(splt[0:len(splt)-3], "-")
|
pkgVer = fileSplit[len(fileSplit)-3] + "-" + fileSplit[len(fileSplit)-2]
|
||||||
vername = splt[len(splt)-3] + "-" + splt[len(splt)-2]
|
|
||||||
)
|
)
|
||||||
|
version, err := packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeArch, pkgName, pkgVer)
|
||||||
version, err := packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeArch, pkgname, vername)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group)
|
pkgFile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
return pkgfile, nil
|
|
||||||
|
return packages_service.GetPackageFileStream(ctx, pkgFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) {
|
func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
|
||||||
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
fileName := fmt.Sprintf("%s.db", arch)
|
fileName := fmt.Sprintf("%s.db", arch)
|
||||||
if signFile {
|
if signFile {
|
||||||
|
@ -284,10 +284,9 @@ func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, si
|
||||||
}
|
}
|
||||||
file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, group)
|
file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, group)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
filestream, _, _, err := packages_service.GetPackageFileStream(ctx, file)
|
return packages_service.GetPackageFileStream(ctx, file)
|
||||||
return filestream, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files
|
// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files
|
||||||
|
|
|
@ -16,11 +16,11 @@ pacman-key --lsign-key '{{$.SignMail}}'</code></pre>
|
||||||
<pre
|
<pre
|
||||||
class="code-block"><code>
|
class="code-block"><code>
|
||||||
{{- if gt (len $.Groups) 1 -}}
|
{{- if gt (len $.Groups) 1 -}}
|
||||||
# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi" $.PackageDescriptor.Package.LowerName}}
|
# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi" $.PackageDescriptor.Package.LowerName}}
|
||||||
|
|
||||||
{{end -}}
|
{{end -}}
|
||||||
{{- $GroupSize := (len .Groups) -}}
|
{{- $GroupSize := (len .Groups) -}}
|
||||||
{{- range $i,$v := .Groups -}}
|
{{- range $i,$v := .Groups -}}
|
||||||
{{- if gt $i 0}}
|
{{- if gt $i 0}}
|
||||||
{{end -}}{{- if gt $GroupSize 1 -}}
|
{{end -}}{{- if gt $GroupSize 1 -}}
|
||||||
# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi.item" .}}
|
# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi.item" .}}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
{{if eq .PackageDescriptor.Package.Type "arch"}}
|
{{if eq .PackageDescriptor.Package.Type "arch"}}
|
||||||
{{range .PackageDescriptor.Metadata.License}}<div class="item" title="{{$.locale.Tr "packages.details.license"}}">{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}</div>{{end}}
|
{{if .PackageDescriptor.Metadata.ProjectURL}}<div class="item">{{svg "octicon-link-external" 16 "tw-mr-2"}} <a href="{{.PackageDescriptor.Metadata.ProjectURL}}" target="_blank" rel="noopener noreferrer me">{{ctx.Locale.Tr "packages.details.project_site"}}</a></div>{{end}}
|
||||||
{{if .PackageDescriptor.Metadata.ProjectURL}}<div class="item">{{svg "octicon-link-external" 16 "mr-3"}} <a href="{{.PackageDescriptor.Metadata.ProjectURL}}" target="_blank" rel="noopener noreferrer me">{{ctx.Locale.Tr "packages.details.project_site"}}</a></div>{{end}}
|
{{range .PackageDescriptor.Metadata.License}}<div class="item" title="{{$.locale.Tr "packages.details.license"}}">{{svg "octicon-law" 16 "tw-mr-2"}} {{.}}</div>{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
|
@ -258,11 +259,15 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA
|
||||||
AddBasicAuth(user.Name)
|
AddBasicAuth(user.Name)
|
||||||
MakeRequest(t, req, http.StatusCreated)
|
MakeRequest(t, req, http.StatusCreated)
|
||||||
|
|
||||||
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil).
|
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1/any", nil).
|
||||||
AddBasicAuth(user.Name)
|
AddBasicAuth(user.Name)
|
||||||
MakeRequest(t, req, http.StatusNotFound)
|
MakeRequest(t, req, http.StatusNotFound)
|
||||||
|
|
||||||
req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1", nil).
|
req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1/x86_64", nil).
|
||||||
|
AddBasicAuth(user.Name)
|
||||||
|
MakeRequest(t, req, http.StatusNoContent)
|
||||||
|
|
||||||
|
req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1/any", nil).
|
||||||
AddBasicAuth(user.Name)
|
AddBasicAuth(user.Name)
|
||||||
MakeRequest(t, req, http.StatusNoContent)
|
MakeRequest(t, req, http.StatusNoContent)
|
||||||
|
|
||||||
|
@ -270,12 +275,22 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA
|
||||||
respPkg := MakeRequest(t, req, http.StatusOK)
|
respPkg := MakeRequest(t, req, http.StatusOK)
|
||||||
files, err := listTarGzFiles(respPkg.Body.Bytes())
|
files, err := listTarGzFiles(respPkg.Body.Bytes())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, files, 1) // other pkg in L225
|
require.Len(t, files, 1)
|
||||||
|
|
||||||
req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil).
|
req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1/any", nil).
|
||||||
AddBasicAuth(user.Name)
|
AddBasicAuth(user.Name)
|
||||||
MakeRequest(t, req, http.StatusNoContent)
|
MakeRequest(t, req, http.StatusNoContent)
|
||||||
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
|
|
||||||
|
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db").
|
||||||
|
AddBasicAuth(user.Name)
|
||||||
|
MakeRequest(t, req, http.StatusNotFound)
|
||||||
|
|
||||||
|
req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1/aarch64", nil).
|
||||||
|
AddBasicAuth(user.Name)
|
||||||
|
MakeRequest(t, req, http.StatusNoContent)
|
||||||
|
|
||||||
|
req = NewRequest(t, "GET", groupURL+"/aarch64/base.db").
|
||||||
|
AddBasicAuth(user.Name)
|
||||||
MakeRequest(t, req, http.StatusNotFound)
|
MakeRequest(t, req, http.StatusNotFound)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -294,12 +309,33 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA
|
||||||
resp := MakeRequest(t, req, http.StatusOK)
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
require.Equal(t, pkgs[key], resp.Body.Bytes())
|
require.Equal(t, pkgs[key], resp.Body.Bytes())
|
||||||
|
|
||||||
req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil).
|
req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1/any", nil).
|
||||||
AddBasicAuth(user.Name)
|
AddBasicAuth(user.Name)
|
||||||
MakeRequest(t, req, http.StatusNoContent)
|
MakeRequest(t, req, http.StatusNoContent)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
t.Run("Concurrent Upload", func(t *testing.T) {
|
||||||
|
defer tests.PrintCurrentTest(t)()
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
targets := []string{"any", "aarch64", "x86_64"}
|
||||||
|
for _, tag := range targets {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i string) {
|
||||||
|
defer wg.Done()
|
||||||
|
req := NewRequestWithBody(t, "PUT", rootURL, bytes.NewReader(pkgs[i])).
|
||||||
|
AddBasicAuth(user.Name)
|
||||||
|
MakeRequest(t, req, http.StatusCreated)
|
||||||
|
}(tag)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
for _, target := range targets {
|
||||||
|
req := NewRequestWithBody(t, "DELETE", rootURL+"/test/1.0.0-1/"+target, nil).
|
||||||
|
AddBasicAuth(user.Name)
|
||||||
|
MakeRequest(t, req, http.StatusNoContent)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func getProperty(data, key string) string {
|
func getProperty(data, key string) string {
|
||||||
|
@ -318,10 +354,10 @@ func getProperty(data, key string) string {
|
||||||
|
|
||||||
func listTarGzFiles(data []byte) (fstest.MapFS, error) {
|
func listTarGzFiles(data []byte) (fstest.MapFS, error) {
|
||||||
reader, err := gzip.NewReader(bytes.NewBuffer(data))
|
reader, err := gzip.NewReader(bytes.NewBuffer(data))
|
||||||
defer reader.Close()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
defer reader.Close()
|
||||||
tarRead := tar.NewReader(reader)
|
tarRead := tar.NewReader(reader)
|
||||||
files := make(fstest.MapFS)
|
files := make(fstest.MapFS)
|
||||||
for {
|
for {
|
||||||
|
|
Loading…
Reference in a new issue