Add Debian package registry (#24426)

Co-authored-by: @awkwardbunny

This PR adds a Debian package registry.
You can follow [this
tutorial](https://www.baeldung.com/linux/create-debian-package) to build
a *.deb package for testing.
Source packages are not supported at the moment and I did not find
documentation of the architecture "all" and how these packages should be
treated.


![grafik](https://user-images.githubusercontent.com/1666336/218126879-eb80a866-775c-4c8e-8529-5797203a64e6.png)

Part of #20751.

Revised copy of #22854.

---------

Co-authored-by: Brian Hong <[email protected]>
Co-authored-by: techknowlogick <[email protected]>
Co-authored-by: Giteabot <[email protected]>

Author: 4 people

assets/go-licenses.json

@@ -25,7 +25,7 @@ func TestMigratePackages(t *testing.T) {
 	creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
 
 	content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
-	buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
+	buf, err := packages_module.CreateHashedBufferFromReaderWithSize(strings.NewReader(content), 1024)
 	assert.NoError(t, err)
 	defer buf.Close()
 

cmd/migrate_storage_test.go

@@ -2496,6 +2496,8 @@ ROUTER = console
 ;LIMIT_SIZE_CONDA = -1
 ;; Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 ;LIMIT_SIZE_CONTAINER = -1
+;; Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_DEBIAN = -1
 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 ;LIMIT_SIZE_GENERIC = -1
 ;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

custom/conf/app.example.ini

@@ -1251,6 +1251,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -0,0 +1,134 @@
+---
+date: "2023-01-07T00:00:00+00:00"
+title: "Debian Packages Repository"
+slug: "packages/debian"
+draft: false
+toc: false
+menu:
+  sidebar:
+    parent: "packages"
+    name: "Debian"
+    weight: 35
+    identifier: "debian"
+---
+
+# Debian Packages Repository
+
+Publish [Debian](https://www.debian.org/distrib/packages) packages for your user or organization.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Requirements
+
+To work with the Debian registry, you need to use a HTTP client like `curl` to upload and a package manager like `apt` to consume packages.
+
+The following examples use `apt`.
+
+## Configuring the package registry
+
+To register the Debian registry add the url to the list of known apt sources:
+
+```shell
+echo "deb https://gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+```
+
+| Placeholder    | Description |
+| -------------- | ----------- |
+| `owner`        | The owner of the package. |
+| `distribution` | The distribution to use. |
+| `component`    | The component to use. |
+
+If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}):
+
+```shell
+echo "deb https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+```
+
+The Debian registry files are signed with a PGP key which must be known to apt:
+
+```shell
+sudo curl https://gitea.example.com/api/packages/{owner}/debian/repository.key -o /etc/apt/trusted.gpg.d/gitea-{owner}.asc
+```
+
+Afterwards update the local package index:
+
+```shell
+apt update
+```
+
+## Publish a package
+
+To publish a Debian package (`*.deb`), perform a HTTP `PUT` operation with the package content in the request body.
+
+```
+PUT https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/upload
+```
+
+| Parameter      | Description |
+| -------------- | ----------- |
+| `owner`        | The owner of the package. |
+| `distribution` | The distribution may match the release name of the OS, ex: `bionic`. |
+| `component`    | The component can be used to group packages or just `main` or similar. |
+
+Example request using HTTP Basic authentication:
+
+```shell
+curl --user your_username:your_password_or_token \
+     --upload-file path/to/file.deb \
+     https://gitea.example.com/api/packages/testuser/debian/pool/bionic/main/upload
+```
+
+If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
+You cannot publish a file with the same name twice to a package. You must delete the existing package version first.
+
+The server reponds with the following HTTP Status codes.
+
+| HTTP Status Code  | Meaning |
+| ----------------- | ------- |
+| `201 Created`     | The package has been published. |
+| `400 Bad Request` | The package name, version, distribution, component or architecture are invalid. |
+| `409 Conflict`    | A package file with the same combination of parameters exists already. |
+
+## Delete a package
+
+To delete a Debian package perform a HTTP `DELETE` operation. This will delete the package version too if there is no file left.
+
+```
+DELETE https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/{package_name}/{package_version}/{architecture}
+```
+
+| Parameter         | Description |
+| ----------------- | ----------- |
+| `owner`           | The owner of the package. |
+| `package_name`    | The package name. |
+| `package_version` | The package version. |
+| `distribution`    | The package distribution. |
+| `component`       | The package component. |
+| `architecture`    | The package architecture. |
+
+Example request using HTTP Basic authentication:
+
+```shell
+curl --user your_username:your_token_or_password -X DELETE \
+     https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64
+```
+
+The server reponds with the following HTTP Status codes.
+
+| HTTP Status Code  | Meaning |
+| ----------------- | ------- |
+| `204 No Content`  | Success |
+| `404 Not Found`   | The package or file was not found. |
+
+## Install a package
+
+To install a package from the Debian registry, execute the following commands:
+
+```shell
+# use latest version
+apt install {package_name}
+# use specific version
+apt install {package_name}={package_version}
+```

docs/content/doc/usage/packages/debian.en-us.md

@@ -33,6 +33,7 @@ The following package managers are currently supported:
 | [Conan]({{< relref "doc/usage/packages/conan.en-us.md" >}}) | C++ | `conan` |
 | [Conda]({{< relref "doc/usage/packages/conda.en-us.md" >}}) | - | `conda` |
 | [Container]({{< relref "doc/usage/packages/container.en-us.md" >}}) | - | any OCI compliant client |
+| [Debian]({{< relref "doc/usage/packages/debian.en-us.md" >}}) | - | `apt` |
 | [Generic]({{< relref "doc/usage/packages/generic.en-us.md" >}}) | - | any HTTP client |
 | [Helm]({{< relref "doc/usage/packages/helm.en-us.md" >}}) | - | any HTTP client, `cm-push` |
 | [Maven]({{< relref "doc/usage/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` |

docs/content/doc/usage/packages/overview.en-us.md

@@ -17,6 +17,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/alecthomas/chroma/v2 v2.5.0
+	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
 	github.com/blevesearch/bleve/v2 v2.3.6
 	github.com/bufbuild/connect-go v1.3.1
 	github.com/buildkite/terminal-to-html/v3 v3.7.0
@@ -96,6 +97,7 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tstranex/u2f v1.0.0
+	github.com/ulikunitz/xz v0.5.11
 	github.com/urfave/cli v1.22.12
 	github.com/xanzy/go-gitlab v0.80.2
 	github.com/xeipuuv/gojsonschema v1.2.0
@@ -260,7 +262,6 @@ require (
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/toqueteos/webbrowser v1.2.0 // indirect
-	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/unknwon/com v1.0.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.44.0 // indirect

go.mod

@@ -162,6 +162,8 @@ github.com/bits-and-blooms/bitset v1.1.10/go.mod h1:w0XsmFg8qg6cmpTtJ0z3pKgjTDBM
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
+github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blevesearch/bleve/v2 v2.0.5/go.mod h1:ZjWibgnbRX33c+vBRgla9QhPb4QOjD6fdVJ+R1Bk8LM=
 github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4=
 github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4=

go.sum

@@ -489,6 +489,8 @@ var migrations = []Migration{
 	NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable),
 	// v255 -> v256
 	NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
+	// v256 -> v257
+	NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/migrations.go

@@ -0,0 +1,23 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_20 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddIsInternalColumnToPackage(x *xorm.Engine) error {
+	type Package struct {
+		ID               int64  `xorm:"pk autoincr"`
+		OwnerID          int64  `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		RepoID           int64  `xorm:"INDEX"`
+		Type             string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		Name             string `xorm:"NOT NULL"`
+		LowerName        string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		SemverCompatible bool   `xorm:"NOT NULL DEFAULT false"`
+		IsInternal       bool   `xorm:"NOT NULL DEFAULT false"`
+	}
+
+	return x.Sync(new(Package))
+}

models/migrations/v1_20/v256.go

@@ -101,16 +101,7 @@ func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit
 		return nil, err
 	}
 
-	pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs))
-	for _, pf := range pfs {
-		pfd, err := packages.GetPackageFileDescriptor(ctx, pf)
-		if err != nil {
-			return nil, err
-		}
-		pfds = append(pfds, pfd)
-	}
-
-	return pfds, nil
+	return packages.GetPackageFileDescriptors(ctx, pfs)
 }
 
 // GetManifestVersions gets all package versions representing the matching manifest

models/packages/container/search.go

@@ -0,0 +1,131 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+	"context"
+	"strconv"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/packages"
+	debian_module "code.gitea.io/gitea/modules/packages/debian"
+
+	"xorm.io/builder"
+)
+
+type PackageSearchOptions struct {
+	OwnerID      int64
+	Distribution string
+	Component    string
+	Architecture string
+}
+
+// SearchLatestPackages gets the latest packages matching the search options
+func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*packages.PackageFileDescriptor, error) {
+	var cond builder.Cond = builder.Eq{
+		"package_file.is_lead":        true,
+		"package.type":                packages.TypeDebian,
+		"package.owner_id":            opts.OwnerID,
+		"package.is_internal":         false,
+		"package_version.is_internal": false,
+	}
+
+	props := make(map[string]string)
+	if opts.Distribution != "" {
+		props[debian_module.PropertyDistribution] = opts.Distribution
+	}
+	if opts.Component != "" {
+		props[debian_module.PropertyComponent] = opts.Component
+	}
+	if opts.Architecture != "" {
+		props[debian_module.PropertyArchitecture] = opts.Architecture
+	}
+
+	if len(props) > 0 {
+		var propsCond builder.Cond = builder.Eq{
+			"package_property.ref_type": packages.PropertyTypeFile,
+		}
+		propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id"))
+
+		propsCondBlock := builder.NewCond()
+		for name, value := range props {
+			propsCondBlock = propsCondBlock.Or(builder.Eq{
+				"package_property.name":  name,
+				"package_property.value": value,
+			})
+		}
+		propsCond = propsCond.And(propsCondBlock)
+
+		cond = cond.And(builder.Eq{
+			strconv.Itoa(len(props)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"),
+		})
+	}
+
+	cond = cond.
+		And(builder.Expr("pv2.id IS NULL"))
+
+	joinCond := builder.
+		Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))").
+		And(builder.Eq{"pv2.is_internal": false})
+
+	pfs := make([]*packages.PackageFile, 0, 10)
+	err := db.GetEngine(ctx).
+		Table("package_file").
+		Select("package_file.*").
+		Join("INNER", "package_version", "package_version.id = package_file.version_id").
+		Join("LEFT", "package_version pv2", joinCond).
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Where(cond).
+		Desc("package_version.created_unix").
+		Find(&pfs)
+	if err != nil {
+		return nil, err
+	}
+
+	return packages.GetPackageFileDescriptors(ctx, pfs)
+}
+
+// GetDistributions gets all available distributions
+func GetDistributions(ctx context.Context, ownerID int64) ([]string, error) {
+	return getDistinctPropertyValues(ctx, ownerID, "", debian_module.PropertyDistribution)
+}
+
+// GetComponents gets all available components for the given distribution
+func GetComponents(ctx context.Context, ownerID int64, distribution string) ([]string, error) {
+	return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyComponent)
+}
+
+// GetArchitectures gets all available architectures for the given distribution
+func GetArchitectures(ctx context.Context, ownerID int64, distribution string) ([]string, error) {
+	return getDistinctPropertyValues(ctx, ownerID, distribution, debian_module.PropertyArchitecture)
+}
+
+func getDistinctPropertyValues(ctx context.Context, ownerID int64, distribution, propName string) ([]string, error) {
+	var cond builder.Cond = builder.Eq{
+		"package_property.ref_type": packages.PropertyTypeFile,
+		"package_property.name":     propName,
+		"package.type":              packages.TypeDebian,
+		"package.owner_id":          ownerID,
+	}
+	if distribution != "" {
+		innerCond := builder.
+			Expr("pp.ref_id = package_property.ref_id").
+			And(builder.Eq{
+				"pp.ref_type": packages.PropertyTypeFile,
+				"pp.name":     debian_module.PropertyDistribution,
+				"pp.value":    distribution,
+			})
+		cond = cond.And(builder.Exists(builder.Select("pp.ref_id").From("package_property pp").Where(innerCond)))
+	}
+
+	values := make([]string, 0, 5)
+	return values, db.GetEngine(ctx).
+		Table("package_property").
+		Distinct("package_property.value").
+		Join("INNER", "package_file", "package_file.id = package_property.ref_id").
+		Join("INNER", "package_version", "package_version.id = package_file.version_id").
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Where(cond).
+		Find(&values)
+}