[FEAT] add fallback repo search using git grep

2024-11-27 12:53:57 +01:00 · 2024-02-20 11:05:42 +00:00 · 2024-02-20 11:05:42 +00:00 · 51fb6f3983
commit 51fb6f3983
parent 0533022d63
6 changed files with 246 additions and 63 deletions
--- a/routers/web/repo/search.go
+++ b/routers/web/repo/search.go
@ -10,17 +10,13 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	code_indexer "code.gitea.io/gitea/modules/indexer/code"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/repository/files"
 )

 const tplSearch base.TplName = "repo/search"

 // Search render repository search page
 func Search(ctx *context.Context) {
-	if !setting.Indexer.RepoIndexerEnabled {
-		ctx.Redirect(ctx.Repo.RepoLink)
-		return
-	}
-
 	language := ctx.FormTrim("l")
 	keyword := ctx.FormTrim("q")

@ -37,31 +33,49 @@ func Search(ctx *context.Context) {
 		return
 	}

+	ctx.Data["SourcePath"] = ctx.Repo.Repository.Link()
+
 	page := ctx.FormInt("page")
 	if page <= 0 {
 		page = 1
 	}

-	total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
-		language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
-	if err != nil {
-		if code_indexer.IsAvailable(ctx) {
-			ctx.ServerError("SearchResults", err)
+	if setting.Indexer.RepoIndexerEnabled {
+		ctx.Data["CodeIndexerEnabled"] = true
+
+		total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
+			language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
+		if err != nil {
+			if code_indexer.IsAvailable(ctx) {
+				ctx.ServerError("SearchResults", err)
+				return
+			}
+			ctx.Data["CodeIndexerUnavailable"] = true
+		} else {
+			ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
+		}
+
+		ctx.Data["SearchResults"] = searchResults
+		ctx.Data["SearchResultLanguages"] = searchResultLanguages
+
+		pager := context.NewPagination(total, setting.UI.RepoSearchPagingNum, page, 5)
+		pager.SetDefaultParams(ctx)
+		pager.AddParam(ctx, "l", "Language")
+		ctx.Data["Page"] = pager
+	} else {
+		data, err := files.NewRepoGrep(ctx, ctx.Repo.Repository, keyword)
+		if err != nil {
+			ctx.ServerError("NewRepoGrep", err)
 			return
 		}
-		ctx.Data["CodeIndexerUnavailable"] = true
-	} else {
-		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
+
+		ctx.Data["CodeIndexerEnabled"] = false
+		ctx.Data["SearchResults"] = data
+
+		pager := context.NewPagination(len(data), setting.UI.RepoSearchPagingNum, page, 5)
+		pager.SetDefaultParams(ctx)
+		ctx.Data["Page"] = pager
 	}

-	ctx.Data["SourcePath"] = ctx.Repo.Repository.Link()
-	ctx.Data["SearchResults"] = searchResults
-	ctx.Data["SearchResultLanguages"] = searchResultLanguages
-
-	pager := context.NewPagination(total, setting.UI.RepoSearchPagingNum, page, 5)
-	pager.SetDefaultParams(ctx)
-	pager.AddParam(ctx, "l", "Language")
-	ctx.Data["Page"] = pager
-
 	ctx.HTML(http.StatusOK, tplSearch)
 }
--- a/services/repository/files/search.go
+++ b/services/repository/files/search.go
@ -0,0 +1,90 @@
+package files
+
+import (
+	"context"
+	"html/template"
+	"strconv"
+	"strings"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/highlight"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"github.com/go-enry/go-enry/v2"
+)
+
+type Result struct {
+	RepoID         int64 // ignored
+	Filename       string
+	CommitID       string             // branch
+	UpdatedUnix    timeutil.TimeStamp // ignored
+	Language       string
+	Color          string
+	LineNumbers    []int64
+	FormattedLines template.HTML
+}
+
+const pHEAD = "HEAD:"
+
+func NewRepoGrep(ctx context.Context, repo *repo_model.Repository, keyword string) ([]*Result, error) {
+	t, _, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	data := []*Result{}
+
+	stdout, _, err := git.NewCommand(ctx,
+		"grep",
+		"-1", // n before and after lines
+		"-z",
+		"--heading",
+		"--break",         // easier parsing
+		"--fixed-strings", // disallow regex for now
+		"-n",              // line nums
+		"-i",              // ignore case
+		"--full-name",     // full file path, rel to repo
+		//"--column",        // for adding better highlighting support
+	).
+		AddDynamicArguments(keyword).
+		AddArguments("HEAD").
+		RunStdString(&git.RunOpts{Dir: t.Path})
+	if err != nil {
+		return data, nil // non zero exit code when there are no results
+	}
+
+	for _, block := range strings.Split(stdout, "\n\n") {
+		res := Result{CommitID: repo.DefaultBranch}
+		code := []string{}
+
+		for _, line := range strings.Split(block, "\n") {
+			if strings.HasPrefix(line, pHEAD) {
+				res.Filename = strings.TrimPrefix(line, pHEAD)
+				continue
+			}
+
+			if ln, after, ok := strings.Cut(line, "\x00"); ok {
+				i, err := strconv.ParseInt(ln, 10, 64)
+				if err != nil {
+					continue
+				}
+
+				res.LineNumbers = append(res.LineNumbers, i)
+				code = append(code, after)
+			}
+		}
+
+		if res.Filename == "" || len(code) == 0 || len(res.LineNumbers) == 0 {
+			continue
+		}
+
+		res.FormattedLines, res.Language = highlight.Code(res.Filename, "", strings.Join(code, "\n"))
+		res.Color = enry.GetColor(res.Language)
+
+		data = append(data, &res)
+	}
+
+	return data, nil
+}
--- a/services/repository/files/search_test.go
+++ b/services/repository/files/search_test.go
@ -0,0 +1,48 @@
+package files
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/contexttest"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewRepoGrep(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+	ctx, _ := contexttest.MockContext(t, "user2/repo1")
+	ctx.SetParams(":id", "1")
+	contexttest.LoadRepo(t, ctx, 1)
+	contexttest.LoadRepoCommit(t, ctx)
+	contexttest.LoadUser(t, ctx, 2)
+	contexttest.LoadGitRepo(t, ctx)
+	defer ctx.Repo.GitRepo.Close()
+
+	t.Run("with result", func(t *testing.T) {
+		res, err := NewRepoGrep(ctx, ctx.Repo.Repository, "Description")
+		assert.NoError(t, err)
+
+		expected := []*Result{
+			{
+				RepoID:         0,
+				Filename:       "README.md",
+				CommitID:       "master",
+				UpdatedUnix:    0,
+				Language:       "Markdown",
+				Color:          "#083fa1",
+				LineNumbers:    []int64{2, 3},
+				FormattedLines: "\nDescription for repo1",
+			},
+		}
+
+		assert.EqualValues(t, res, expected)
+	})
+
+	t.Run("empty result", func(t *testing.T) {
+		res, err := NewRepoGrep(ctx, ctx.Repo.Repository, "keyword that does not match in the repo")
+		assert.NoError(t, err)
+
+		assert.EqualValues(t, res, []*Result{})
+	})
+}
--- a/templates/repo/home.tmpl
+++ b/templates/repo/home.tmpl
@ -11,23 +11,21 @@
 				{{if $description}}<span class="description">{{$description | RenderCodeBlock}}</span>{{else if .IsRepositoryAdmin}}<span class="no-description text-italic">{{ctx.Locale.Tr "repo.no_desc"}}</span>{{end}}
 				<a class="link" href="{{.Repository.Website}}">{{.Repository.Website}}</a>
 			</div>
-			{{if .RepoSearchEnabled}}
-				<div class="ui repo-search">
-					<form class="ui form ignore-dirty" action="{{.RepoLink}}/search" method="get">
-						<div class="field">
-							<div class="ui small action input{{if .CodeIndexerUnavailable}} disabled left icon{{end}}"{{if .CodeIndexerUnavailable}} data-tooltip-content="{{ctx.Locale.Tr "repo.search.code_search_unavailable"}}"{{end}}>
-								<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable}} disabled{{end}} placeholder="{{ctx.Locale.Tr "repo.search.search_repo"}}">
-								{{if .CodeIndexerUnavailable}}
-									<i class="icon">{{svg "octicon-alert"}}</i>
-								{{end}}
-								<button class="ui small icon button"{{if .CodeIndexerUnavailable}} disabled{{end}} type="submit">
-									{{svg "octicon-search"}}
-								</button>
-							</div>
+			<div class="ui repo-search">
+				<form class="ui form ignore-dirty" action="{{.RepoLink}}/search" method="get">
+					<div class="field">
+						<div class="ui small action input{{if .CodeIndexerUnavailable}} disabled left icon{{end}}"{{if .CodeIndexerUnavailable}} data-tooltip-content="{{ctx.Locale.Tr "repo.search.code_search_unavailable"}}"{{end}}>
+							<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable}} disabled{{end}} placeholder="{{ctx.Locale.Tr "repo.search.search_repo"}}">
+							{{if .CodeIndexerUnavailable}}
+								<i class="icon">{{svg "octicon-alert"}}</i>
+							{{end}}
+							<button class="ui small icon button"{{if .CodeIndexerUnavailable}} disabled{{end}} type="submit">
+								{{svg "octicon-search"}}
+							</button>
 						</div>
-					</form>
-				</div>
-			{{end}}
+					</div>
+				</form>
+			</div>
 		</div>
 		<div class="gt-df gt-ac gt-fw gt-gap-2" id="repo-topics">
 			{{range .Topics}}<a class="ui repo-topic large label topic gt-m-0" href="{{AppSubUrl}}/explore/repos?q={{.Name}}&topic=1">{{.Name}}</a>{{end}}
--- a/templates/repo/search.tmpl
+++ b/templates/repo/search.tmpl
@ -6,14 +6,16 @@
 			<form class="ui form ignore-dirty" method="get">
 				<div class="ui fluid action input">
 					<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable}} disabled{{end}} placeholder="{{ctx.Locale.Tr "repo.search.search_repo"}}">
-					<div class="ui dropdown selection {{if .CodeIndexerUnavailable}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "repo.search.type.tooltip"}}">
-						<input name="t" type="hidden"{{if .CodeIndexerUnavailable}} disabled{{end}} value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
-						<div class="text">{{ctx.Locale.Tr (printf "repo.search.%s" (or .queryType "fuzzy"))}}</div>
-						<div class="menu">
-							<div class="item" data-value="" data-tooltip-content="{{ctx.Locale.Tr "repo.search.fuzzy.tooltip"}}">{{ctx.Locale.Tr "repo.search.fuzzy"}}</div>
-							<div class="item" data-value="match" data-tooltip-content="{{ctx.Locale.Tr "repo.search.match.tooltip"}}">{{ctx.Locale.Tr "repo.search.match"}}</div>
+					{{if .CodeIndexerEnabled}}
+						<div class="ui dropdown selection {{if .CodeIndexerUnavailable}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "repo.search.type.tooltip"}}">
+							<input name="t" type="hidden"{{if .CodeIndexerUnavailable}} disabled{{end}} value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
+							<div class="text">{{ctx.Locale.Tr (printf "repo.search.%s" (or .queryType "fuzzy"))}}</div>
+							<div class="menu">
+								<div class="item" data-value="" data-tooltip-content="{{ctx.Locale.Tr "repo.search.fuzzy.tooltip"}}">{{ctx.Locale.Tr "repo.search.fuzzy"}}</div>
+								<div class="item" data-value="match" data-tooltip-content="{{ctx.Locale.Tr "repo.search.match.tooltip"}}">{{ctx.Locale.Tr "repo.search.match"}}</div>
+							</div>
 						</div>
-					</div>
+					{{end}}
 					<button class="ui icon button"{{if .CodeIndexerUnavailable}} disabled{{end}} type="submit">{{svg "octicon-search" 16}}</button>
 				</div>
 			</form>
@ -41,7 +43,7 @@
 						<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
 							<h4 class="ui top attached normal header gt-df gt-fw">
 								<span class="file gt-f1">{{.Filename}}</span>
-								<a role="button" class="ui basic tiny button" rel="nofollow" href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments .Filename}}">{{ctx.Locale.Tr "repo.diff.view_file"}}</a>
+								<a role="button" class="ui basic tiny button" rel="nofollow" href="{{$.SourcePath}}/src/{{if $.CodeIndexerEnabled}}commit{{else}}branch{{end}}/{{PathEscape $result.CommitID}}/{{PathEscapeSegments .Filename}}">{{ctx.Locale.Tr "repo.diff.view_file"}}</a>
 							</h4>
 							<div class="ui attached table segment">
 								<div class="file-body file-code code-view">
@ -50,7 +52,7 @@
 											<tr>
 												<td class="lines-num">
 													{{range .LineNumbers}}
-														<a href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
+														<a href="{{$.SourcePath}}/src/{{if $.CodeIndexerEnabled}}commit{{else}}branch{{end}}/{{PathEscape $result.CommitID}}/{{PathEscapeSegments $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
 													{{end}}
 												</td>
 												<td class="lines-code chroma"><code class="code-inner">{{.FormattedLines}}</code></td>
--- a/tests/integration/repo_search_test.go
+++ b/tests/integration/repo_search_test.go
@ -11,14 +11,15 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	code_indexer "code.gitea.io/gitea/modules/indexer/code"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
 	"code.gitea.io/gitea/tests"

 	"github.com/PuerkitoBio/goquery"
 	"github.com/stretchr/testify/assert"
 )

-func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
-	filenameSelections := doc.doc.Find(".repository.search").Find(".repo-search-result").Find(".header").Find("span.file")
+func resultFilenames(t testing.TB, doc *goquery.Selection) []string {
+	filenameSelections := doc.Find(".header").Find("span.file")
 	result := make([]string, filenameSelections.Length())
 	filenameSelections.Each(func(i int, selection *goquery.Selection) {
 		result[i] = selection.Text()
@ -26,36 +27,66 @@ func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
 	return result
 }

-func TestSearchRepo(t *testing.T) {
+func checkResultLinks(t *testing.T, substr string, doc *goquery.Selection) {
+	t.Helper()
+	linkSelections := doc.Find("a[href]")
+	linkSelections.Each(func(i int, selection *goquery.Selection) {
+		assert.Contains(t, selection.AttrOr("href", ""), substr)
+	})
+}
+
+func testSearchRepo(t *testing.T, useExternalIndexer bool) {
 	defer tests.PrepareTestEnv(t)()
+	defer test.MockVariableValue(&setting.Indexer.RepoIndexerEnabled, useExternalIndexer)()

 	repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
 	assert.NoError(t, err)

-	executeIndexer(t, repo, code_indexer.UpdateRepoIndexer)
+	gitReference := "/branch/" + repo.DefaultBranch

-	testSearch(t, "/user2/repo1/search?q=Description&page=1", []string{"README.md"})
+	if useExternalIndexer {
+		gitReference = "/commit/"
+		executeIndexer(t, repo, code_indexer.UpdateRepoIndexer)
+	}

-	setting.Indexer.IncludePatterns = setting.IndexerGlobFromString("**.txt")
-	setting.Indexer.ExcludePatterns = setting.IndexerGlobFromString("**/y/**")
+	testSearch(t, "/user2/repo1/search?q=Description&page=1", gitReference, []string{"README.md"})

-	repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "glob")
-	assert.NoError(t, err)
+	if useExternalIndexer {
+		setting.Indexer.IncludePatterns = setting.IndexerGlobFromString("**.txt")
+		setting.Indexer.ExcludePatterns = setting.IndexerGlobFromString("**/y/**")

-	executeIndexer(t, repo, code_indexer.UpdateRepoIndexer)
+		repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "glob")
+		assert.NoError(t, err)

-	testSearch(t, "/user2/glob/search?q=loren&page=1", []string{"a.txt"})
-	testSearch(t, "/user2/glob/search?q=file3&page=1", []string{"x/b.txt"})
-	testSearch(t, "/user2/glob/search?q=file4&page=1", []string{})
-	testSearch(t, "/user2/glob/search?q=file5&page=1", []string{})
+		executeIndexer(t, repo, code_indexer.UpdateRepoIndexer)
+
+		testSearch(t, "/user2/glob/search?q=loren&page=1", gitReference, []string{"a.txt"})
+		testSearch(t, "/user2/glob/search?q=file3&page=1", gitReference, []string{"x/b.txt"})
+		testSearch(t, "/user2/glob/search?q=file4&page=1", gitReference, []string{})
+		testSearch(t, "/user2/glob/search?q=file5&page=1", gitReference, []string{})
+	}
 }

-func testSearch(t *testing.T, url string, expected []string) {
+func TestIndexerSearchRepo(t *testing.T) {
+	testSearchRepo(t, true)
+}
+
+func TestNoIndexerSearchRepo(t *testing.T) {
+	testSearchRepo(t, false)
+}
+
+func testSearch(t *testing.T, url, gitRef string, expected []string) {
 	req := NewRequest(t, "GET", url)
 	resp := MakeRequest(t, req, http.StatusOK)

-	filenames := resultFilenames(t, NewHTMLParser(t, resp.Body))
+	doc := NewHTMLParser(t, resp.Body).doc.
+		Find(".repository.search").
+		Find(".repo-search-result")
+
+	filenames := resultFilenames(t, doc)
 	assert.EqualValues(t, expected, filenames)
+
+	checkResultLinks(t, gitRef, doc)
 }

 func executeIndexer(t *testing.T, repo *repo_model.Repository, op func(*repo_model.Repository)) {