mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-11-10 12:49:52 +01:00
Distinguish LFS object errors to ignore missing objects during migration (#31702)
Fix #31137. Replace #31623 #31697. When migrating LFS objects, if there's any object that failed (like some objects are losted, which is not really critical), Gitea will stop migrating LFS immediately but treat the migration as successful. This PR checks the error according to the [LFS api doc](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses). > LFS object error codes should match HTTP status codes where possible: > > - 404 - The object does not exist on the server. > - 409 - The specified hash algorithm disagrees with the server's acceptable options. > - 410 - The object was removed by the owner. > - 422 - Validation error. If the error is `404`, it's safe to ignore it and continue migration. Otherwise, stop the migration and mark it as failed to ensure data integrity of LFS objects. And maybe we should also ignore others errors (maybe `410`? I'm not sure what's the difference between "does not exist" and "removed by the owner".), we can add it later when some users report that they have failed to migrate LFS because of an error which should be ignored. (cherry picked from commit 09b56fc0690317891829906d45c1d645794c63d5)
This commit is contained in:
parent
51f414d466
commit
2302cf63c8
4 changed files with 46 additions and 4 deletions
|
@ -136,14 +136,13 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc
|
|||
|
||||
for _, object := range result.Objects {
|
||||
if object.Error != nil {
|
||||
objectError := errors.New(object.Error.Message)
|
||||
log.Trace("Error on object %v: %v", object.Pointer, objectError)
|
||||
log.Trace("Error on object %v: %v", object.Pointer, object.Error)
|
||||
if uc != nil {
|
||||
if _, err := uc(object.Pointer, objectError); err != nil {
|
||||
if _, err := uc(object.Pointer, object.Error); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := dc(object.Pointer, nil, objectError); err != nil {
|
||||
if err := dc(object.Pointer, nil, object.Error); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,11 @@
|
|||
package lfs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -64,6 +68,39 @@ type ObjectError struct {
|
|||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
var (
|
||||
// See https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses
|
||||
// LFS object error codes should match HTTP status codes where possible:
|
||||
// 404 - The object does not exist on the server.
|
||||
// 409 - The specified hash algorithm disagrees with the server's acceptable options.
|
||||
// 410 - The object was removed by the owner.
|
||||
// 422 - Validation error.
|
||||
|
||||
ErrObjectNotExist = util.ErrNotExist // the object does not exist on the server
|
||||
ErrObjectHashMismatch = errors.New("the specified hash algorithm disagrees with the server's acceptable options")
|
||||
ErrObjectRemoved = errors.New("the object was removed by the owner")
|
||||
ErrObjectValidation = errors.New("validation error")
|
||||
)
|
||||
|
||||
func (e *ObjectError) Error() string {
|
||||
return fmt.Sprintf("[%d] %s", e.Code, e.Message)
|
||||
}
|
||||
|
||||
func (e *ObjectError) Unwrap() error {
|
||||
switch e.Code {
|
||||
case 404:
|
||||
return ErrObjectNotExist
|
||||
case 409:
|
||||
return ErrObjectHashMismatch
|
||||
case 410:
|
||||
return ErrObjectRemoved
|
||||
case 422:
|
||||
return ErrObjectValidation
|
||||
default:
|
||||
return errors.New(e.Message)
|
||||
}
|
||||
}
|
||||
|
||||
// PointerBlob associates a Git blob with a Pointer.
|
||||
type PointerBlob struct {
|
||||
Hash string
|
||||
|
|
|
@ -6,6 +6,7 @@ package repository
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
@ -182,6 +183,10 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re
|
|||
downloadObjects := func(pointers []lfs.Pointer) error {
|
||||
err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error {
|
||||
if objectError != nil {
|
||||
if errors.Is(objectError, lfs.ErrObjectNotExist) {
|
||||
log.Warn("Repo[%-v]: Ignore missing LFS object %-v: %v", repo, p, objectError)
|
||||
return nil
|
||||
}
|
||||
return objectError
|
||||
}
|
||||
|
||||
|
|
|
@ -172,6 +172,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
|||
lfsClient := lfs.NewClient(endpoint, httpTransport)
|
||||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil {
|
||||
log.Error("Failed to store missing LFS objects for repository: %v", err)
|
||||
return repo, fmt.Errorf("StoreMissingLfsObjectsInRepository: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue