Use a migration test instead of a wrong test which populated the meta test repositories and fix a migration bug (#36160)

The test `TestGiteaUploadUpdateGitForPullRequest` modified the shared
meta test repositories directly, so this PR removes that test and
replaces it with an integration test that migrates a real repository
from gitea.com into a local test instance.

This PR also fixes a bug where pull-request migrations were not
correctly syncing head branches to the database.
This commit is contained in:
Lunny Xiao
2025-12-17 12:00:07 -08:00
committed by GitHub
parent ad49b7bf31
commit ebf9b4dc6b
6 changed files with 200 additions and 302 deletions

View File

@@ -14,6 +14,7 @@ type Uploader interface {
CreateMilestones(ctx context.Context, milestones ...*Milestone) error
CreateReleases(ctx context.Context, releases ...*Release) error
SyncTags(ctx context.Context) error
SyncBranches(ctx context.Context) error
CreateLabels(ctx context.Context, labels ...*Label) error
CreateIssues(ctx context.Context, issues ...*Issue) error
CreateComments(ctx context.Context, comments ...*Comment) error

View File

@@ -358,6 +358,11 @@ func (g *RepositoryDumper) SyncTags(ctx context.Context) error {
return nil
}
// SyncBranches syncs branches in the database
func (g *RepositoryDumper) SyncBranches(ctx context.Context) error {
return nil
}
// CreateIssues creates issues
func (g *RepositoryDumper) CreateIssues(_ context.Context, issues ...*base.Issue) error {
var err error

View File

@@ -368,6 +368,11 @@ func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error {
return repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo)
}
func (g *GiteaLocalUploader) SyncBranches(ctx context.Context) error {
_, err := repo_module.SyncRepoBranchesWithRepo(ctx, g.repo, g.gitRepo, g.doer.ID)
return err
}
// CreateIssues creates issues
func (g *GiteaLocalUploader) CreateIssues(ctx context.Context, issues ...*base.Issue) error {
iss := make([]*issues_model.Issue, 0, len(issues))

View File

@@ -5,9 +5,6 @@
package migrations
import (
"fmt"
"os"
"path/filepath"
"strconv"
"testing"
"time"
@@ -17,15 +14,10 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/stretchr/testify/assert"
@@ -228,297 +220,3 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, linkedUser.ID, target.GetUserID())
}
func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
unittest.PrepareTestEnv(t)
//
// fromRepo master
//
fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
baseRef := "master"
// this is very different from the real situation. It should be a bare repository for all the Gitea managed repositories
assert.NoError(t, git.InitRepository(t.Context(), fromRepo.RepoPath(), false, fromRepo.ObjectFormatName))
err := gitrepo.RunCmd(t.Context(), fromRepo, gitcmd.NewCommand("symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef))
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("# Testing Repository\n\nOriginally created in: "+fromRepo.RepoPath()), 0o644))
assert.NoError(t, git.AddChanges(t.Context(), fromRepo.RepoPath(), true))
signature := git.Signature{
Email: "test@example.com",
Name: "test",
When: time.Now(),
}
assert.NoError(t, git.CommitChanges(t.Context(), fromRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "Initial Commit",
}))
fromGitRepo, err := gitrepo.OpenRepository(t.Context(), fromRepo)
assert.NoError(t, err)
defer fromGitRepo.Close()
baseSHA, err := fromGitRepo.GetBranchCommitID(baseRef)
assert.NoError(t, err)
//
// fromRepo branch1
//
headRef := "branch1"
_, err = gitrepo.RunCmdString(t.Context(), fromRepo, gitcmd.NewCommand("checkout", "-b").AddDynamicArguments(headRef))
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("SOMETHING"), 0o644))
assert.NoError(t, git.AddChanges(t.Context(), fromRepo.RepoPath(), true))
signature.When = time.Now()
assert.NoError(t, git.CommitChanges(t.Context(), fromRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "Pull request",
}))
assert.NoError(t, err)
headSHA, err := fromGitRepo.GetBranchCommitID(headRef)
assert.NoError(t, err)
fromRepoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: fromRepo.OwnerID})
//
// forkRepo branch2
//
forkHeadRef := "branch2"
forkRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 8})
assert.NoError(t, git.Clone(t.Context(), fromRepo.RepoPath(), forkRepo.RepoPath(), git.CloneRepoOptions{
Branch: headRef,
}))
_, err = gitrepo.RunCmdString(t.Context(), forkRepo, gitcmd.NewCommand("checkout", "-b").AddDynamicArguments(forkHeadRef))
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte("# branch2 "+forkRepo.RepoPath()), 0o644))
assert.NoError(t, git.AddChanges(t.Context(), forkRepo.RepoPath(), true))
assert.NoError(t, git.CommitChanges(t.Context(), forkRepo.RepoPath(), git.CommitChangesOptions{
Committer: &signature,
Author: &signature,
Message: "branch2 commit",
}))
forkGitRepo, err := gitrepo.OpenRepository(t.Context(), forkRepo)
assert.NoError(t, err)
defer forkGitRepo.Close()
forkHeadSHA, err := forkGitRepo.GetBranchCommitID(forkHeadRef)
assert.NoError(t, err)
toRepoName := "migrated"
ctx := t.Context()
uploader := NewGiteaLocalUploader(ctx, fromRepoOwner, fromRepoOwner.Name, toRepoName)
uploader.gitServiceType = structs.GiteaService
assert.NoError(t, repo_service.Init(t.Context()))
assert.NoError(t, uploader.CreateRepo(ctx, &base.Repository{
Description: "description",
OriginalURL: fromRepo.RepoPath(),
CloneURL: fromRepo.RepoPath(),
IsPrivate: false,
IsMirror: true,
}, base.MigrateOptions{
GitServiceType: structs.GiteaService,
Private: false,
Mirror: true,
}))
for _, testCase := range []struct {
name string
head string
logFilter []string
logFiltered []bool
pr base.PullRequest
}{
{
name: "fork, good Head.SHA",
head: fmt.Sprintf("%s/%s", forkRepo.OwnerName, forkHeadRef),
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: forkRepo.RepoPath(),
Ref: forkHeadRef,
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: forkRepo.OwnerName,
},
},
},
{
name: "fork, invalid Head.Ref",
head: "unknown repository",
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: forkRepo.RepoPath(),
Ref: "INVALID",
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: forkRepo.OwnerName,
},
},
logFilter: []string{"Fetch branch from"},
logFiltered: []bool{true},
},
{
name: "invalid fork CloneURL",
head: "unknown repository",
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: "UNLIKELY",
Ref: forkHeadRef,
SHA: forkHeadSHA,
RepoName: forkRepo.Name,
OwnerName: "WRONG",
},
},
logFilter: []string{"AddRemote"},
logFiltered: []bool{true},
},
{
name: "no fork, good Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: headSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
},
{
name: "no fork, empty Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
logFilter: []string{"Empty reference", "Cannot remove local head"},
logFiltered: []bool{true, false},
},
{
name: "no fork, invalid Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "brokenSHA",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
logFilter: []string{"Deprecated local head"},
logFiltered: []bool{true},
},
{
name: "no fork, not found Head.SHA",
head: headRef,
pr: base.PullRequest{
PatchURL: "",
Number: 1,
State: "open",
Base: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: baseRef,
SHA: baseSHA,
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
Head: base.PullRequestBranch{
CloneURL: fromRepo.RepoPath(),
Ref: headRef,
SHA: "2697b352310fcd01cbd1f3dbd43b894080027f68",
RepoName: fromRepo.Name,
OwnerName: fromRepo.OwnerName,
},
},
logFilter: []string{"Deprecated local head", "Cannot remove local head"},
logFiltered: []bool{true, false},
},
} {
t.Run(testCase.name, func(t *testing.T) {
stopMark := fmt.Sprintf(">>>>>>>>>>>>>STOP: %s<<<<<<<<<<<<<<<", testCase.name)
logChecker, cleanup := test.NewLogChecker(log.DEFAULT)
logChecker.Filter(testCase.logFilter...).StopMark(stopMark)
defer cleanup()
testCase.pr.EnsuredSafe = true
head, err := uploader.updateGitForPullRequest(ctx, &testCase.pr)
assert.NoError(t, err)
assert.Equal(t, testCase.head, head)
log.Info(stopMark)
logFiltered, logStopped := logChecker.Check(5 * time.Second)
assert.True(t, logStopped)
if len(testCase.logFilter) > 0 {
assert.Equal(t, testCase.logFiltered, logFiltered, "for log message filters: %v", testCase.logFilter)
}
})
}
}

View File

@@ -478,6 +478,15 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
break
}
}
if len(mapInsertedPRIndexes) > 0 {
// The pull requests migrating process may created head branches in the base repository
// because head repository maybe a fork one which will not be migrated. So that we need
// to sync branches again.
log.Trace("syncing branches after migrating pull requests")
if err = uploader.SyncBranches(ctx); err != nil {
return err
}
}
}
if opts.Comments && supportAllComments {

View File

@@ -10,18 +10,25 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
issues_model "code.gitea.io/gitea/models/issues"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/services/migrations"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMigrateLocalPath(t *testing.T) {
@@ -112,3 +119,176 @@ func Test_UpdateCommentsMigrationsByType(t *testing.T) {
err := issues_model.UpdateCommentsMigrationsByType(t.Context(), structs.GithubService, "1", 1)
assert.NoError(t, err)
}
func Test_MigrateFromGiteaToGitea(t *testing.T) {
defer tests.PrepareTestEnv(t)()
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"})
session := loginUser(t, owner.Name)
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeAll)
resp, err := http.Get("https://gitea.com/gitea")
if err != nil || resp.StatusCode != http.StatusOK {
if resp != nil {
resp.Body.Close()
}
t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name())
}
resp.Body.Close()
repoName := fmt.Sprintf("gitea-to-gitea-%d", time.Now().UnixNano())
cloneAddr := "https://gitea.com/gitea/test_repo.git"
req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate", &structs.MigrateRepoOptions{
CloneAddr: cloneAddr,
RepoOwnerID: owner.ID,
RepoName: repoName,
Service: structs.GiteaService.Name(),
Wiki: true,
Milestones: true,
Labels: true,
Issues: true,
PullRequests: true,
Releases: true,
}).AddTokenAuth(token)
MakeRequest(t, req, http.StatusCreated)
migratedRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: repoName})
assert.Equal(t, owner.ID, migratedRepo.OwnerID)
assert.Equal(t, structs.GiteaService, migratedRepo.OriginalServiceType)
assert.Equal(t, cloneAddr, migratedRepo.OriginalURL)
issueCount := unittest.GetCount(t,
&issues_model.Issue{RepoID: migratedRepo.ID},
unittest.Cond("is_pull = ?", false),
)
assert.Equal(t, 7, issueCount)
pullCount := unittest.GetCount(t,
&issues_model.Issue{RepoID: migratedRepo.ID},
unittest.Cond("is_pull = ?", true),
)
assert.Equal(t, 6, pullCount)
issue4, err := issues_model.GetIssueWithAttrsByIndex(t.Context(), migratedRepo.ID, 4)
require.NoError(t, err)
assert.Equal(t, owner.ID, issue4.PosterID)
assert.Equal(t, "Ghost", issue4.OriginalAuthor)
assert.Equal(t, int64(-1), issue4.OriginalAuthorID)
assert.Equal(t, "what is this repo about?", issue4.Title)
assert.True(t, issue4.IsClosed)
assert.True(t, issue4.IsLocked)
if assert.NotNil(t, issue4.Milestone) {
assert.Equal(t, "V1", issue4.Milestone.Name)
}
labelNames := make([]string, 0, len(issue4.Labels))
for _, label := range issue4.Labels {
labelNames = append(labelNames, label.Name)
}
assert.Contains(t, labelNames, "Question")
reactionTypes := make([]string, 0, len(issue4.Reactions))
for _, reaction := range issue4.Reactions {
reactionTypes = append(reactionTypes, reaction.Type)
}
assert.ElementsMatch(t, []string{"laugh"}, reactionTypes) // gitea's author is ghost which will be ignored when migrating reactions
comments, err := issues_model.FindComments(t.Context(), &issues_model.FindCommentsOptions{
IssueID: issue4.ID,
Type: issues_model.CommentTypeComment,
})
require.NoError(t, err)
require.Len(t, comments, 2)
assert.Equal(t, owner.ID, comments[0].PosterID)
assert.Equal(t, int64(689), comments[0].OriginalAuthorID)
assert.Equal(t, "6543", comments[0].OriginalAuthor)
assert.Contains(t, comments[0].Content, "TESTSET for gitea2gitea")
assert.Equal(t, owner.ID, comments[1].PosterID)
assert.Equal(t, "Ghost", comments[1].OriginalAuthor)
assert.Equal(t, int64(-1), comments[1].OriginalAuthorID)
assert.Equal(t, "Oh!", strings.TrimSpace(comments[1].Content))
pr12, err := issues_model.GetPullRequestByIndex(t.Context(), migratedRepo.ID, 12)
require.NoError(t, err)
assert.Equal(t, owner.ID, pr12.Issue.PosterID)
assert.Equal(t, "6543", pr12.Issue.OriginalAuthor)
assert.Equal(t, int64(689), pr12.Issue.OriginalAuthorID)
assert.Equal(t, "Dont Touch", pr12.Issue.Title)
assert.True(t, pr12.Issue.IsClosed)
assert.True(t, pr12.HasMerged)
assert.Equal(t, "827aa28a907853e5ddfa40c8f9bc52471a2685fd", pr12.MergedCommitID)
assert.NoError(t, pr12.Issue.LoadMilestone(t.Context()))
if assert.NotNil(t, pr12.Issue.Milestone) {
assert.Equal(t, "V2 Finalize", pr12.Issue.Milestone.Name)
}
assert.Contains(t, pr12.Issue.Content, "dont touch")
pr8, err := issues_model.GetPullRequestByIndex(t.Context(), migratedRepo.ID, 8)
require.NoError(t, err)
assert.Equal(t, owner.ID, pr8.Issue.PosterID)
assert.Equal(t, "6543", pr8.Issue.OriginalAuthor)
assert.Equal(t, int64(689), pr8.Issue.OriginalAuthorID)
assert.Equal(t, "add garbage for close pull", pr8.Issue.Title)
assert.True(t, pr8.Issue.IsClosed)
assert.False(t, pr8.HasMerged)
assert.Contains(t, pr8.Issue.Content, "well you'll see")
pr13, err := issues_model.GetPullRequestByIndex(t.Context(), migratedRepo.ID, 13)
require.NoError(t, err)
assert.Equal(t, owner.ID, pr13.Issue.PosterID)
assert.Equal(t, "6543", pr13.Issue.OriginalAuthor)
assert.Equal(t, int64(689), pr13.Issue.OriginalAuthorID)
assert.Equal(t, "extend", pr13.Issue.Title)
assert.False(t, pr13.Issue.IsClosed)
assert.False(t, pr13.HasMerged)
assert.True(t, pr13.Issue.IsLocked)
gitRepo, err := gitrepo.OpenRepository(t.Context(), migratedRepo)
require.NoError(t, err)
defer gitRepo.Close()
branches, _, err := gitRepo.GetBranchNames(0, 0)
require.NoError(t, err)
assert.ElementsMatch(t, []string{"6543-patch-1", "master", "6543-forks/add-xkcd-2199"}, branches) // last branch comes from the pull request
branchNames, err := git_model.FindBranchNames(t.Context(), git_model.FindBranchOptions{
RepoID: migratedRepo.ID,
})
require.NoError(t, err)
assert.ElementsMatch(t, []string{"6543-patch-1", "master", "6543-forks/add-xkcd-2199"}, branchNames)
tags, _, err := gitRepo.GetTagInfos(0, 0)
require.NoError(t, err)
tagNames := make([]string, 0, len(tags))
for _, tag := range tags {
tagNames = append(tagNames, tag.Name)
}
assert.ElementsMatch(t, []string{"V1", "v2-rc1"}, tagNames)
releases, err := db.Find[repo_model.Release](t.Context(), repo_model.FindReleasesOptions{
RepoID: migratedRepo.ID,
IncludeDrafts: true,
IncludeTags: false,
})
require.NoError(t, err)
require.Len(t, releases, 2)
releaseMap := make(map[string]*repo_model.Release, len(releases))
for _, rel := range releases {
releaseMap[rel.TagName] = rel
assert.Equal(t, owner.ID, rel.PublisherID)
assert.Equal(t, "6543", rel.OriginalAuthor)
assert.Equal(t, int64(689), rel.OriginalAuthorID)
assert.False(t, rel.IsDraft)
}
require.Contains(t, releaseMap, "v2-rc1")
v2Release := releaseMap["v2-rc1"]
assert.Equal(t, "Second Release", v2Release.Title)
assert.True(t, v2Release.IsPrerelease)
assert.Contains(t, v2Release.Note, "this repo has:")
require.Contains(t, releaseMap, "V1")
v1Release := releaseMap["V1"]
assert.Equal(t, "First Release", v1Release.Title)
assert.False(t, v1Release.IsPrerelease)
assert.Equal(t, "as title", strings.TrimSpace(v1Release.Note))
}