Skip to content

Commit 1f14609

Browse files
lunny6543zeripath
authored
Add migrate repo archiver and packages storage support on command line (#20757)
* Add migrate repo archiver and packages storage support on command line * Fix typo * Use stdCtx * Use packageblob and fix command description * Add migrate packages unit tests * Fix comment year * Fix the migrate storage command line description * Update cmd/migrate_storage.go Co-authored-by: zeripath <[email protected]> * Update cmd/migrate_storage.go Co-authored-by: zeripath <[email protected]> * Update cmd/migrate_storage.go Co-authored-by: zeripath <[email protected]> * Fix test Co-authored-by: 6543 <[email protected]> Co-authored-by: zeripath <[email protected]>
1 parent 86c85c1 commit 1f14609

File tree

11 files changed

+187
-134
lines changed

11 files changed

+187
-134
lines changed

‎cmd/main_test.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"testing"
9+
10+
"code.gitea.io/gitea/models/unittest"
11+
"code.gitea.io/gitea/modules/setting"
12+
)
13+
14+
func init() {
15+
setting.SetCustomPathAndConf("", "", "")
16+
setting.LoadForTest()
17+
}
18+
19+
func TestMain(m *testing.M) {
20+
unittest.MainTest(m, &unittest.TestOptions{
21+
GiteaRootPath: "..",
22+
})
23+
}

‎cmd/migrate_storage.go

Lines changed: 48 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ import (
1212
"code.gitea.io/gitea/models/db"
1313
git_model "code.gitea.io/gitea/models/git"
1414
"code.gitea.io/gitea/models/migrations"
15+
packages_model "code.gitea.io/gitea/models/packages"
1516
repo_model "code.gitea.io/gitea/models/repo"
1617
user_model "code.gitea.io/gitea/models/user"
1718
"code.gitea.io/gitea/modules/log"
19+
packages_module "code.gitea.io/gitea/modules/packages"
1820
"code.gitea.io/gitea/modules/setting"
1921
"code.gitea.io/gitea/modules/storage"
2022

@@ -25,13 +27,13 @@ import (
2527
var CmdMigrateStorage = cli.Command{
2628
Name: "migrate-storage",
2729
Usage: "Migrate the storage",
28-
Description: "This is a command for migrating storage.",
30+
Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
2931
Action: runMigrateStorage,
3032
Flags: []cli.Flag{
3133
cli.StringFlag{
3234
Name: "type, t",
3335
Value: "",
34-
Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
36+
Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'",
3537
},
3638
cli.StringFlag{
3739
Name: "storage, s",
@@ -80,34 +82,53 @@ var CmdMigrateStorage = cli.Command{
8082
},
8183
}
8284

83-
func migrateAttachments(dstStorage storage.ObjectStorage) error {
84-
return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
85+
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
86+
return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
8587
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
8688
return err
8789
})
8890
}
8991

90-
func migrateLFS(dstStorage storage.ObjectStorage) error {
91-
return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
92+
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
93+
return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
9294
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
9395
return err
9496
})
9597
}
9698

97-
func migrateAvatars(dstStorage storage.ObjectStorage) error {
98-
return user_model.IterateUser(func(user *user_model.User) error {
99+
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
100+
return db.IterateObjects(ctx, func(user *user_model.User) error {
99101
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
100102
return err
101103
})
102104
}
103105

104-
func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
105-
return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
106+
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
107+
return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
106108
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
107109
return err
108110
})
109111
}
110112

113+
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
114+
return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
115+
p, err := archiver.RelativePath()
116+
if err != nil {
117+
return err
118+
}
119+
_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
120+
return err
121+
})
122+
}
123+
124+
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
125+
return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
126+
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
127+
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
128+
return err
129+
})
130+
}
131+
111132
func runMigrateStorage(ctx *cli.Context) error {
112133
stdCtx, cancel := installSignals()
113134
defer cancel()
@@ -127,8 +148,6 @@ func runMigrateStorage(ctx *cli.Context) error {
127148
return err
128149
}
129150

130-
goCtx := context.Background()
131-
132151
if err := storage.Init(); err != nil {
133152
return err
134153
}
@@ -145,13 +164,13 @@ func runMigrateStorage(ctx *cli.Context) error {
145164
return nil
146165
}
147166
dstStorage, err = storage.NewLocalStorage(
148-
goCtx,
167+
stdCtx,
149168
storage.LocalStorageConfig{
150169
Path: p,
151170
})
152171
case string(storage.MinioStorageType):
153172
dstStorage, err = storage.NewMinioStorage(
154-
goCtx,
173+
stdCtx,
155174
storage.MinioStorageConfig{
156175
Endpoint: ctx.String("minio-endpoint"),
157176
AccessKeyID: ctx.String("minio-access-key-id"),
@@ -162,35 +181,29 @@ func runMigrateStorage(ctx *cli.Context) error {
162181
UseSSL: ctx.Bool("minio-use-ssl"),
163182
})
164183
default:
165-
return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
184+
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
166185
}
167186
if err != nil {
168187
return err
169188
}
170189

190+
migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
191+
"attachments": migrateAttachments,
192+
"lfs": migrateLFS,
193+
"avatars": migrateAvatars,
194+
"repo-avatars": migrateRepoAvatars,
195+
"repo-archivers": migrateRepoArchivers,
196+
"packages": migratePackages,
197+
}
198+
171199
tp := strings.ToLower(ctx.String("type"))
172-
switch tp {
173-
case "attachments":
174-
if err := migrateAttachments(dstStorage); err != nil {
175-
return err
176-
}
177-
case "lfs":
178-
if err := migrateLFS(dstStorage); err != nil {
179-
return err
180-
}
181-
case "avatars":
182-
if err := migrateAvatars(dstStorage); err != nil {
183-
return err
184-
}
185-
case "repo-avatars":
186-
if err := migrateRepoAvatars(dstStorage); err != nil {
200+
if m, ok := migratedMethods[tp]; ok {
201+
if err := m(stdCtx, dstStorage); err != nil {
187202
return err
188203
}
189-
default:
190-
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
204+
log.Info("%s files have successfully been copied to the new storage.", tp)
205+
return nil
191206
}
192207

193-
log.Warn("All files have been copied to the new placement but old files are still on the original placement.")
194-
195-
return nil
208+
return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
196209
}

‎cmd/migrate_storage_test.go

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"os"
10+
"strings"
11+
"testing"
12+
13+
"code.gitea.io/gitea/models/packages"
14+
"code.gitea.io/gitea/models/unittest"
15+
user_model "code.gitea.io/gitea/models/user"
16+
packages_module "code.gitea.io/gitea/modules/packages"
17+
"code.gitea.io/gitea/modules/storage"
18+
packages_service "code.gitea.io/gitea/services/packages"
19+
20+
"github.com/stretchr/testify/assert"
21+
)
22+
23+
func TestMigratePackages(t *testing.T) {
24+
assert.NoError(t, unittest.PrepareTestDatabase())
25+
26+
creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
27+
28+
content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
29+
buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
30+
assert.NoError(t, err)
31+
defer buf.Close()
32+
33+
v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{
34+
PackageInfo: packages_service.PackageInfo{
35+
Owner: creator,
36+
PackageType: packages.TypeGeneric,
37+
Name: "test",
38+
Version: "1.0.0",
39+
},
40+
Creator: creator,
41+
SemverCompatible: true,
42+
VersionProperties: map[string]string{},
43+
}, &packages_service.PackageFileCreationInfo{
44+
PackageFileInfo: packages_service.PackageFileInfo{
45+
Filename: "a.go",
46+
},
47+
Data: buf,
48+
IsLead: true,
49+
})
50+
assert.NoError(t, err)
51+
assert.NotNil(t, v)
52+
assert.NotNil(t, f)
53+
54+
ctx := context.Background()
55+
56+
p, err := os.MkdirTemp(os.TempDir(), "migrated_packages")
57+
assert.NoError(t, err)
58+
59+
dstStorage, err := storage.NewLocalStorage(
60+
ctx,
61+
storage.LocalStorageConfig{
62+
Path: p,
63+
})
64+
assert.NoError(t, err)
65+
66+
err = migratePackages(ctx, dstStorage)
67+
assert.NoError(t, err)
68+
69+
entries, err := os.ReadDir(p)
70+
assert.NoError(t, err)
71+
assert.EqualValues(t, 2, len(entries))
72+
assert.EqualValues(t, "01", entries[0].Name())
73+
assert.EqualValues(t, "tmp", entries[1].Name())
74+
}

‎models/db/iterate.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package db
6+
7+
import (
8+
"context"
9+
10+
"code.gitea.io/gitea/modules/setting"
11+
)
12+
13+
// IterateObjects iterate all the Bean object
14+
func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
15+
var start int
16+
batchSize := setting.Database.IterateBufferSize
17+
sess := GetEngine(ctx)
18+
for {
19+
repos := make([]*Object, 0, batchSize)
20+
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
21+
return err
22+
}
23+
if len(repos) == 0 {
24+
return nil
25+
}
26+
start += len(repos)
27+
28+
for _, repo := range repos {
29+
if err := f(repo); err != nil {
30+
return err
31+
}
32+
}
33+
}
34+
}

‎models/git/lfs.go

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
278278
return committer.Commit()
279279
}
280280

281-
// IterateLFS iterates lfs object
282-
func IterateLFS(f func(mo *LFSMetaObject) error) error {
283-
var start int
284-
const batchSize = 100
285-
e := db.GetEngine(db.DefaultContext)
286-
for {
287-
mos := make([]*LFSMetaObject, 0, batchSize)
288-
if err := e.Limit(batchSize, start).Find(&mos); err != nil {
289-
return err
290-
}
291-
if len(mos) == 0 {
292-
return nil
293-
}
294-
start += len(mos)
295-
296-
for _, mo := range mos {
297-
if err := f(mo); err != nil {
298-
return err
299-
}
300-
}
301-
}
302-
}
303-
304281
// CopyLFS copies LFS data from one repo to another
305282
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
306283
var lfsObjects []*LFSMetaObject

‎models/repo/attachment.go

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
226226
return err
227227
}
228228

229-
// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
230-
func IterateAttachment(f func(attach *Attachment) error) error {
231-
var start int
232-
const batchSize = 100
233-
for {
234-
attachments := make([]*Attachment, 0, batchSize)
235-
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil {
236-
return err
237-
}
238-
if len(attachments) == 0 {
239-
return nil
240-
}
241-
start += len(attachments)
242-
243-
for _, attach := range attachments {
244-
if err := f(attach); err != nil {
245-
return err
246-
}
247-
}
248-
}
249-
}
250-
251229
// CountOrphanedAttachments returns the number of bad attachments
252230
func CountOrphanedAttachments() (int64, error) {
253231
return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").

‎models/repo/repo_list.go

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -15,36 +15,12 @@ import (
1515
"code.gitea.io/gitea/models/unit"
1616
user_model "code.gitea.io/gitea/models/user"
1717
"code.gitea.io/gitea/modules/container"
18-
"code.gitea.io/gitea/modules/setting"
1918
"code.gitea.io/gitea/modules/structs"
2019
"code.gitea.io/gitea/modules/util"
2120

2221
"xorm.io/builder"
2322
)
2423

25-
// IterateRepository iterate repositories
26-
func IterateRepository(f func(repo *Repository) error) error {
27-
var start int
28-
batchSize := setting.Database.IterateBufferSize
29-
sess := db.GetEngine(db.DefaultContext)
30-
for {
31-
repos := make([]*Repository, 0, batchSize)
32-
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
33-
return err
34-
}
35-
if len(repos) == 0 {
36-
return nil
37-
}
38-
start += len(repos)
39-
40-
for _, repo := range repos {
41-
if err := f(repo); err != nil {
42-
return err
43-
}
44-
}
45-
}
46-
}
47-
4824
// FindReposMapByIDs find repos as map
4925
func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
5026
return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)

0 commit comments

Comments
 (0)