···5353}
54545555func main() {
5656- var err error
5757-5856 flag.Parse()
59576058 // generate data
···8381var emojiRE = regexp.MustCompile(`\{Emoji:"([^"]*)"`)
84828583func generate() ([]byte, error) {
8686- var err error
8787-8884 // load gemoji data
8985 res, err := http.Get(gemojiURL)
9086 if err != nil {
+1-1
cmd/migrate_storage_test.go
···91919292 srcStorage, _ := createLocalStorage(t)
9393 defer test.MockVariableValue(&storage.ActionsArtifacts, srcStorage)()
9494- id := int64(0)
9494+ id := int64(42)
95959696 addArtifact := func(storagePath string, status actions.ArtifactStatus) {
9797 id++
+8
custom/conf/app.example.ini
···328328;; Maximum number of locks returned per page
329329;LFS_LOCKS_PAGING_NUM = 50
330330;;
331331+;; When clients make lfs batch requests, reject them if there are more pointers than this number
332332+;; zero means 'unlimited'
333333+;LFS_MAX_BATCH_SIZE = 0
334334+;;
331335;; Allow graceful restarts using SIGHUP to fork
332336;ALLOW_GRACEFUL_RESTARTS = true
333337;;
···26712675;;
26722676;; override the minio base path if storage type is minio
26732677;MINIO_BASE_PATH = lfs/
26782678+26792679+;[lfs_client]
26802680+;; When mirroring an upstream lfs endpoint, limit the number of pointers in each batch request to this number
26812681+;BATCH_SIZE = 20
2674268226752683;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
26762684;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
···136136// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
137137// if it is not already present.
138138func NewLFSMetaObject(ctx context.Context, repoID int64, p lfs.Pointer) (*LFSMetaObject, error) {
139139- var err error
140140-141139 ctx, committer, err := db.TxContext(ctx)
142140 if err != nil {
143141 return nil, err
···41414242func TestToUTF8(t *testing.T) {
4343 resetDefaultCharsetsOrder()
4444- var res string
4545- var err error
46444745 // Note: golang compiler seems so behave differently depending on the current
4846 // locale, so some conversions might behave differently. For that reason, we don't
4947 // depend on particular conversions but in expected behaviors.
50485151- res, err = ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
4949+ res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
5250 require.NoError(t, err)
5351 assert.Equal(t, "ABC", res)
5452
+23-13
modules/git/repo_index.go
···5050}
51515252// ReadTreeToTemporaryIndex reads a treeish to a temporary index file
5353-func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename, tmpDir string, cancel context.CancelFunc, err error) {
5353+func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilename, tmpDir string, cancel context.CancelFunc, err error) {
5454+ defer func() {
5555+ // if error happens and there is a cancel function, do clean up
5656+ if err != nil && cancel != nil {
5757+ cancel()
5858+ cancel = nil
5959+ }
6060+ }()
6161+6262+ removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs
6363+ return func() {
6464+ if err := util.RemoveAll(dir); err != nil {
6565+ log.Error("failed to remove tmp index dir: %v", err)
6666+ }
6767+ }
6868+ }
6969+5470 tmpDir, err = os.MkdirTemp("", "index")
5571 if err != nil {
5656- return filename, tmpDir, cancel, err
7272+ return "", "", nil, err
5773 }
58745959- filename = filepath.Join(tmpDir, ".tmp-index")
6060- cancel = func() {
6161- err := util.RemoveAll(tmpDir)
6262- if err != nil {
6363- log.Error("failed to remove tmp index file: %v", err)
6464- }
6565- }
6666- err = repo.ReadTreeToIndex(treeish, filename)
7575+ tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index")
7676+ cancel = removeDirFn(tmpDir)
7777+ err = repo.ReadTreeToIndex(treeish, tmpIndexFilename)
6778 if err != nil {
6868- defer cancel()
6969- return "", "", func() {}, err
7979+ return "", "", cancel, err
7080 }
7171- return filename, tmpDir, cancel, err
8181+ return tmpIndexFilename, tmpDir, cancel, err
7282}
73837484// EmptyIndex empties the index
+2-3
modules/lfs/http_client.go
···1616 "code.gitea.io/gitea/modules/json"
1717 "code.gitea.io/gitea/modules/log"
1818 "code.gitea.io/gitea/modules/proxy"
1919+ "code.gitea.io/gitea/modules/setting"
1920)
2020-2121-const httpBatchSize = 20
22212322// HTTPClient is used to communicate with the LFS server
2423// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
···30293130// BatchSize returns the preferred size of batchs to process
3231func (c *HTTPClient) BatchSize() int {
3333- return httpBatchSize
3232+ return setting.LFSClient.BatchSize
3433}
35343635func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient {
···114114}
115115116116// URL gets the redirect URL to a file
117117-func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
117117+func (l *LocalStorage) URL(path, name string, reqParams url.Values) (*url.URL, error) {
118118 return nil, ErrURLNotSupported
119119}
120120
+6-2
modules/storage/minio.go
···276276}
277277278278// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
279279-func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
280280- reqParams := make(url.Values)
279279+func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) {
280280+ // copy serveDirectReqParams
281281+ reqParams, err := url.ParseQuery(serveDirectReqParams.Encode())
282282+ if err != nil {
283283+ return nil, err
284284+ }
281285 // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
282286 reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
283287 u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
+2-2
modules/storage/storage.go
···6363 Save(path string, r io.Reader, size int64) (int64, error)
6464 Stat(path string) (os.FileInfo, error)
6565 Delete(path string) error
6666- URL(path, name string) (*url.URL, error)
6666+ URL(path, name string, reqParams url.Values) (*url.URL, error)
6767 IterateObjects(path string, iterator func(path string, obj Object) error) error
6868}
6969···131131 ActionsArtifacts ObjectStorage = UninitializedStorage
132132)
133133134134-// Init init the stoarge
134134+// Init init the storage
135135func Init() error {
136136 for _, f := range []func() error{
137137 initAttachments,
+6
release-notes/5789.md
···11+fix: [commit](https://codeberg.org/forgejo/forgejo/commit/362ad0ba39bdbc87202e349678e21fc2a75ff7cb) Update force-pushed tags too when syncing mirrors
22+chore: [commit](https://codeberg.org/forgejo/forgejo/commit/b308bcca7c950b7f0d127ee4282019c2a9923299) Improved diff view performance
33+fix: [commit](https://codeberg.org/forgejo/forgejo/commit/4c5bdddf7751a35985c08ba6506f1f30103749d6) Fix `missing signature key` error when pulling Docker images with `SERVE_DIRECT` enabled
44+fix: [commit](https://codeberg.org/forgejo/forgejo/commit/2c5fdb108ff9e23e8f907fb6afe59177c6bb202e) Fix the missing menu in organization project view page
55+feat: [commit](https://codeberg.org/forgejo/forgejo/commit/1e595979625e54d375a0eaa440b84ef5e17af160) Add new [lfs_client].BATCH_SIZE and [server].LFS_MAX_BATCH_SIZE config settings.
66+fix: [commit](https://codeberg.org/forgejo/forgejo/commit/2358c0d899faec8311e46dcb0550041496bcd532) Properly clean temporary index files
+1-1
routers/api/actions/artifacts.go
···437437 for _, artifact := range artifacts {
438438 var downloadURL string
439439 if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
440440- u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName)
440440+ u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName, nil)
441441 if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
442442 log.Error("Error getting serve direct url: %v", err)
443443 }
+1-1
routers/api/actions/artifactsv4.go
···530530 respData := GetSignedArtifactURLResponse{}
531531532532 if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
533533- u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath)
533533+ u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath, nil)
534534 if u != nil && err == nil {
535535 respData.SignedUrl = u.String()
536536 }
···214214215215 if setting.LFS.Storage.MinioConfig.ServeDirect {
216216 // If we have a signed url (S3, object storage), redirect to this directly.
217217- u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
217217+ u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name(), nil)
218218 if u != nil && err == nil {
219219 ctx.Redirect(u.String())
220220 return
···341341 rPath := archiver.RelativePath()
342342 if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
343343 // If we have a signed url (S3, object storage), redirect to this directly.
344344- u, err := storage.RepoArchives.URL(rPath, downloadName)
344344+ u, err := storage.RepoArchives.URL(rPath, downloadName, nil)
345345 if u != nil && err == nil {
346346 ctx.Redirect(u.String())
347347 return
···134134135135 if setting.Attachment.Storage.MinioConfig.ServeDirect {
136136 // If we have a signed url (S3, object storage), redirect to this directly.
137137- u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name)
137137+ u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name, nil)
138138139139 if u != nil && err == nil {
140140 ctx.Redirect(u.String())
···5454 }
55555656 if setting.LFS.Storage.MinioConfig.ServeDirect {
5757- // If we have a signed url (S3, object storage), redirect to this directly.
5858- u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
5757+ // If we have a signed url (S3, object storage, blob storage), redirect to this directly.
5858+ u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name(), nil)
5959 if u != nil && err == nil {
6060 ctx.Redirect(u.String())
6161 return nil
···505505 rPath := archiver.RelativePath()
506506 if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
507507 // If we have a signed url (S3, object storage), redirect to this directly.
508508- u, err := storage.RepoArchives.URL(rPath, downloadName)
508508+ u, err := storage.RepoArchives.URL(rPath, downloadName, nil)
509509 if u != nil && err == nil {
510510 if archiver.ReleaseID != 0 {
511511 err = repo_model.CountArchiveDownload(ctx, ctx.Repo.Repository.ID, archiver.ReleaseID, archiver.Type)
-1
routers/web/repo/view.go
···147147 // this should be impossible; if subTreeEntry exists so should this.
148148 continue
149149 }
150150- var err error
151150 childEntries, err := subTree.ListEntries()
152151 if err != nil {
153152 return "", nil, err
+26-34
services/gitdiff/gitdiff.go
···379379}
380380381381// GetTailSection creates a fake DiffLineSection if the last section is not the end of the file
382382-func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommitID, rightCommitID string) *DiffSection {
382382+func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommit, rightCommit *git.Commit) *DiffSection {
383383 if len(diffFile.Sections) == 0 || diffFile.Type != DiffFileChange || diffFile.IsBin || diffFile.IsLFSFile {
384384 return nil
385385 }
386386- leftCommit, err := gitRepo.GetCommit(leftCommitID)
387387- if err != nil {
388388- return nil
389389- }
390390- rightCommit, err := gitRepo.GetCommit(rightCommitID)
391391- if err != nil {
392392- return nil
393393- }
386386+394387 lastSection := diffFile.Sections[len(diffFile.Sections)-1]
395388 lastLine := lastSection.Lines[len(lastSection.Lines)-1]
396389 leftLineCount := getCommitFileLineCount(leftCommit, diffFile.Name)
···532525 lastFile := createDiffFile(diff, line)
533526 diff.End = lastFile.Name
534527 diff.IsIncomplete = true
535535- _, err := io.Copy(io.Discard, reader)
536536- if err != nil {
537537- // By the definition of io.Copy this never returns io.EOF
538538- return diff, fmt.Errorf("error during io.Copy: %w", err)
539539- }
540528 break parsingLoop
541529 }
542530···10971085 MaxFiles int
10981086 WhitespaceBehavior git.TrustedCmdArgs
10991087 DirectComparison bool
10881088+ FileOnly bool
11001089}
1101109011021091// GetDiff builds a Diff between two commits of a repository.
···11051094func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) {
11061095 repoPath := gitRepo.Path
1107109610971097+ var beforeCommit *git.Commit
11081098 commit, err := gitRepo.GetCommit(opts.AfterCommitID)
11091099 if err != nil {
11101100 return nil, err
11111101 }
1112110211131113- cmdDiff := git.NewCommand(gitRepo.Ctx)
11031103+ cmdCtx, cmdCancel := context.WithCancel(ctx)
11041104+ defer cmdCancel()
11051105+11061106+ cmdDiff := git.NewCommand(cmdCtx)
11141107 objectFormat, err := gitRepo.GetObjectFormat()
11151108 if err != nil {
11161109 return nil, err
···11321125 AddArguments(opts.WhitespaceBehavior...).
11331126 AddDynamicArguments(actualBeforeCommitID, opts.AfterCommitID)
11341127 opts.BeforeCommitID = actualBeforeCommitID
11281128+11291129+ var err error
11301130+ beforeCommit, err = gitRepo.GetCommit(opts.BeforeCommitID)
11311131+ if err != nil {
11321132+ return nil, err
11331133+ }
11351134 }
1136113511371136 // In git 2.31, git diff learned --skip-to which we can use to shortcut skip to file
···11661165 _ = writer.Close()
11671166 }()
1168116711691169- diff, err := ParsePatch(ctx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile)
11681168+ diff, err := ParsePatch(cmdCtx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile)
11691169+ // Ensure the git process is killed if it didn't exit already
11701170+ cmdCancel()
11701171 if err != nil {
11711172 return nil, fmt.Errorf("unable to ParsePatch: %w", err)
11721173 }
···12071208 diffFile.IsGenerated = analyze.IsGenerated(diffFile.Name)
12081209 }
1209121012101210- tailSection := diffFile.GetTailSection(gitRepo, opts.BeforeCommitID, opts.AfterCommitID)
12111211+ tailSection := diffFile.GetTailSection(gitRepo, beforeCommit, commit)
12111212 if tailSection != nil {
12121213 diffFile.Sections = append(diffFile.Sections, tailSection)
12131214 }
12141215 }
1215121612161216- separator := "..."
12171217- if opts.DirectComparison {
12181218- separator = ".."
12171217+ if opts.FileOnly {
12181218+ return diff, nil
12191219 }
1220122012211221- diffPaths := []string{opts.BeforeCommitID + separator + opts.AfterCommitID}
12221222- if len(opts.BeforeCommitID) == 0 || opts.BeforeCommitID == objectFormat.EmptyObjectID().String() {
12231223- diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID}
12241224- }
12251225- diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...)
12261226- if err != nil && strings.Contains(err.Error(), "no merge base") {
12271227- // git >= 2.28 now returns an error if base and head have become unrelated.
12281228- // previously it would return the results of git diff --shortstat base head so let's try that...
12291229- diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID}
12301230- diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...)
12311231- }
12211221+ stats, err := GetPullDiffStats(gitRepo, opts)
12321222 if err != nil {
12331223 return nil, err
12341224 }
1235122512261226+ diff.NumFiles, diff.TotalAddition, diff.TotalDeletion = stats.NumFiles, stats.TotalAddition, stats.TotalDeletion
12271227+12361228 return diff, nil
12371229}
1238123012391231type PullDiffStats struct {
12401240- TotalAddition, TotalDeletion int
12321232+ NumFiles, TotalAddition, TotalDeletion int
12411233}
1242123412431235// GetPullDiffStats
···12611253 diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID}
12621254 }
1263125512641264- _, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...)
12561256+ diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...)
12651257 if err != nil && strings.Contains(err.Error(), "no merge base") {
12661258 // git >= 2.28 now returns an error if base and head have become unrelated.
12671259 // previously it would return the results of git diff --shortstat base head so let's try that...
12681260 diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID}
12691269- _, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...)
12611261+ diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...)
12701262 }
12711263 if err != nil {
12721264 return nil, err
···192192 }
193193 }
194194195195+ if setting.LFS.MaxBatchSize != 0 && len(br.Objects) > setting.LFS.MaxBatchSize {
196196+ writeStatus(ctx, http.StatusRequestEntityTooLarge)
197197+ return
198198+ }
199199+195200 contentStore := lfs_module.NewContentStore()
196201197202 var responseObjects []*lfs_module.ObjectResponse
···480485 var link *lfs_module.Link
481486 if setting.LFS.Storage.MinioConfig.ServeDirect {
482487 // If we have a signed url (S3, object storage), redirect to this directly.
483483- u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid)
488488+ u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid, nil)
484489 if u != nil && err == nil {
485490 // Presigned url does not need the Authorization header
486491 // https://github.com/go-gitea/gitea/issues/21525
+3-3
services/packages/packages.go
···602602 return nil, nil, nil, err
603603 }
604604605605- return GetPackageBlobStream(ctx, pf, pb)
605605+ return GetPackageBlobStream(ctx, pf, pb, nil)
606606}
607607608608// GetPackageBlobStream returns the content of the specific package blob
609609// If the storage supports direct serving and it's enabled, only the direct serving url is returned.
610610-func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
610610+func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
611611 key := packages_module.BlobHash256Key(pb.HashSHA256)
612612613613 cs := packages_module.NewContentStore()
···617617 var err error
618618619619 if cs.ShouldServeDirect() {
620620- u, err = cs.GetServeDirectURL(key, pf.Name)
620620+ u, err = cs.GetServeDirectURL(key, pf.Name, serveDirectReqParams)
621621 if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
622622 log.Error("Error getting serve direct url: %v", err)
623623 }