Merge pull request 'feat: support regexp in git-grep search' (#4968) from yoctozepto/git-grep-regexp into forgejo
Some checks failed
/ release (push) Waiting to run
testing / security-check (push) Blocked by required conditions
testing / backend-checks (push) Waiting to run
testing / frontend-checks (push) Waiting to run
testing / test-unit (push) Blocked by required conditions
testing / test-e2e (push) Blocked by required conditions
testing / test-remote-cacher (map[image:docker.io/bitnami/redis:7.2 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:docker.io/bitnami/valkey:7.2 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:ghcr.io/microsoft/garnet-alpine:1.0.14 port:6379]) (push) Blocked by required conditions
testing / test-remote-cacher (map[image:registry.redict.io/redict:7.3.0-scratch port:6379]) (push) Blocked by required conditions
testing / test-mysql (push) Blocked by required conditions
testing / test-pgsql (push) Blocked by required conditions
testing / test-sqlite (push) Blocked by required conditions
Integration tests for the release process / release-simulation (push) Has been cancelled

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4968
Reviewed-by: Shiny Nematoda <snematoda@noreply.codeberg.org>
This commit is contained in:
Otto 2024-10-04 17:10:14 +00:00
commit f7f7800460
14 changed files with 182 additions and 56 deletions

View file

@ -27,12 +27,20 @@ type GrepResult struct {
HighlightedRanges [][3]int
}
type grepMode int
const (
FixedGrepMode grepMode = iota
FixedAnyGrepMode
RegExpGrepMode
)
type GrepOptions struct {
RefName string
MaxResultLimit int
MatchesPerFile int
ContextLineNumber int
IsFuzzy bool
Mode grepMode
PathSpec []setting.Glob
}
@ -74,12 +82,20 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
var results []*GrepResult
// -I skips binary files
cmd := NewCommand(ctx, "grep",
"-I", "--null", "--break", "--heading", "--column",
"--fixed-strings", "--line-number", "--ignore-case", "--full-name")
"-I", "--null", "--break", "--heading",
"--line-number", "--ignore-case", "--full-name")
if opts.Mode == RegExpGrepMode {
// No `--column` -- regexp mode does not support highlighting in the
// current implementation as the length of the match is unknown from
// `grep` but required for highlighting.
cmd.AddArguments("--perl-regexp")
} else {
cmd.AddArguments("--fixed-strings", "--column")
}
cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber))
cmd.AddOptionValues("--max-count", fmt.Sprint(opts.MatchesPerFile))
words := []string{search}
if opts.IsFuzzy {
if opts.Mode == FixedAnyGrepMode {
words = strings.Fields(search)
}
for _, word := range words {
@ -148,6 +164,7 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
if lineNum, lineCode, ok := strings.Cut(line, "\x00"); ok {
lineNumInt, _ := strconv.Atoi(lineNum)
res.LineNumbers = append(res.LineNumbers, lineNumInt)
// We support highlighting only when `--column` parameter is used.
if lineCol, lineCode2, ok := strings.Cut(lineCode, "\x00"); ok {
lineColInt, _ := strconv.Atoi(lineCol)
start := lineColInt - 1

View file

@ -201,3 +201,34 @@ func TestGrepRefs(t *testing.T) {
assert.Len(t, res, 1)
assert.Equal(t, "A", res[0].LineCodes[0])
}
func TestGrepCanHazRegexOnDemand(t *testing.T) {
tmpDir := t.TempDir()
err := InitRepository(DefaultContext, tmpDir, false, Sha1ObjectFormat.Name())
require.NoError(t, err)
gitRepo, err := openRepositoryWithDefaultContext(tmpDir)
require.NoError(t, err)
defer gitRepo.Close()
require.NoError(t, os.WriteFile(path.Join(tmpDir, "matching"), []byte("It's a match!"), 0o666))
require.NoError(t, os.WriteFile(path.Join(tmpDir, "not-matching"), []byte("Orisitamatch?"), 0o666))
err = AddChanges(tmpDir, true)
require.NoError(t, err)
err = CommitChanges(tmpDir, CommitChangesOptions{Message: "Add fixtures for regexp test"})
require.NoError(t, err)
// should find nothing by default...
res, err := GrepSearch(context.Background(), gitRepo, "\\bmatch\\b", GrepOptions{})
require.NoError(t, err)
assert.Empty(t, res)
// ... unless configured explicitly
res, err = GrepSearch(context.Background(), gitRepo, "\\bmatch\\b", GrepOptions{Mode: RegExpGrepMode})
require.NoError(t, err)
assert.Len(t, res, 1)
assert.Equal(t, "matching", res[0].Filename)
}

View file

@ -173,6 +173,8 @@ union = Union
union_tooltip = Include results that match any of the whitespace seperated keywords
exact = Exact
exact_tooltip = Include only results that match the exact search term
regexp = RegExp
regexp_tooltip = Interpret the search term as a regular expression
repo_kind = Search repos...
user_kind = Search users...
org_kind = Search orgs...

View file

@ -36,10 +36,18 @@ func Code(ctx *context.Context) {
keyword := ctx.FormTrim("q")
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
if mode := ctx.FormTrim("mode"); len(mode) > 0 {
isFuzzy = mode == "fuzzy"
}
ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language
ctx.Data["IsFuzzy"] = isFuzzy
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
if isFuzzy {
ctx.Data["CodeSearchMode"] = "fuzzy"
} else {
ctx.Data["CodeSearchMode"] = "exact"
}
ctx.Data["PageIsViewCode"] = true
if keyword == "" {

View file

@ -17,16 +17,53 @@ import (
const tplSearch base.TplName = "repo/search"
type searchMode int
const (
ExactSearchMode searchMode = iota
FuzzySearchMode
RegExpSearchMode
)
func searchModeFromString(s string) searchMode {
switch s {
case "fuzzy", "union":
return FuzzySearchMode
case "regexp":
return RegExpSearchMode
default:
return ExactSearchMode
}
}
func (m searchMode) String() string {
switch m {
case ExactSearchMode:
return "exact"
case FuzzySearchMode:
return "fuzzy"
case RegExpSearchMode:
return "regexp"
default:
panic("cannot happen")
}
}
// Search render repository search page
func Search(ctx *context.Context) {
language := ctx.FormTrim("l")
keyword := ctx.FormTrim("q")
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
mode := ExactSearchMode
if modeStr := ctx.FormString("mode"); len(modeStr) > 0 {
mode = searchModeFromString(modeStr)
} else if ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) { // for backward compatibility in links
mode = FuzzySearchMode
}
ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language
ctx.Data["IsFuzzy"] = isFuzzy
ctx.Data["CodeSearchMode"] = mode.String()
ctx.Data["PageIsViewCode"] = true
if keyword == "" {
@ -47,7 +84,7 @@ func Search(ctx *context.Context) {
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
RepoIDs: []int64{ctx.Repo.Repository.ID},
Keyword: keyword,
IsKeywordFuzzy: isFuzzy,
IsKeywordFuzzy: mode == FuzzySearchMode,
Language: language,
Paginator: &db.ListOptions{
Page: page,
@ -63,12 +100,20 @@ func Search(ctx *context.Context) {
} else {
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
}
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
} else {
res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, git.GrepOptions{
grepOpt := git.GrepOptions{
ContextLineNumber: 1,
IsFuzzy: isFuzzy,
RefName: ctx.Repo.RefName,
})
}
switch mode {
case FuzzySearchMode:
grepOpt.Mode = git.FixedAnyGrepMode
ctx.Data["CodeSearchMode"] = "union"
case RegExpSearchMode:
grepOpt.Mode = git.RegExpGrepMode
}
res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, grepOpt)
if err != nil {
ctx.ServerError("GrepSearch", err)
return
@ -88,6 +133,7 @@ func Search(ctx *context.Context) {
Lines: code_indexer.HighlightSearchResultCode(r.Filename, r.LineNumbers, r.HighlightedRanges, strings.Join(r.LineCodes, "\n")),
})
}
ctx.Data["CodeSearchOptions"] = []string{"exact", "union", "regexp"}
}
ctx.Data["CodeIndexerDisabled"] = !setting.Indexer.RepoIndexerEnabled

View file

@ -41,10 +41,18 @@ func CodeSearch(ctx *context.Context) {
keyword := ctx.FormTrim("q")
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
if mode := ctx.FormTrim("mode"); len(mode) > 0 {
isFuzzy = mode == "fuzzy"
}
ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language
ctx.Data["IsFuzzy"] = isFuzzy
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
if isFuzzy {
ctx.Data["CodeSearchMode"] = "fuzzy"
} else {
ctx.Data["CodeSearchMode"] = "exact"
}
ctx.Data["IsCodePage"] = true
if keyword == "" {

View file

@ -417,7 +417,7 @@ func SearchWikiContents(ctx context.Context, repo *repo_model.Repository, keywor
return git.GrepSearch(ctx, gitRepo, keyword, git.GrepOptions{
ContextLineNumber: 0,
IsFuzzy: true,
Mode: git.FixedAnyGrepMode,
RefName: repo.GetWikiBranchName(),
MaxResultLimit: 10,
MatchesPerFile: 3,

View file

@ -5,7 +5,7 @@
{{if $.CodeIndexerDisabled}}
{{$branchURLPrefix := printf "%s/search/branch/" $.RepoLink}}
{{$tagURLPrefix := printf "%s/search/tag/" $.RepoLink}}
{{$suffix := printf "?q=%s&fuzzy=%t" (.Keyword|QueryEscape) .IsFuzzy}}
{{$suffix := printf "?q=%s&mode=%s" (.Keyword|QueryEscape) .CodeSearchMode}}
{{template "repo/branch_dropdown" dict "root" . "ContainerClasses" "tw-mb-3" "branchURLPrefix" $branchURLPrefix "branchURLSuffix" $suffix "tagURLPrefix" $tagURLPrefix "tagURLSuffix" $suffix}}
{{end}}
{{template "shared/search/code/search" .}}

View file

@ -1,7 +1,7 @@
<div class="flex-text-block tw-flex-wrap">
{{range $term := .SearchResultLanguages}}
<a class="ui {{if eq $.Language $term.Language}}primary{{end}} basic label tw-m-0"
href="?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}&fuzzy={{$.IsFuzzy}}">
href="?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}&mode={{$.CodeSearchMode}}">
<i class="color-icon tw-mr-2" style="background-color: {{$term.Color}}"></i>
{{$term.Language}}
<div class="detail">{{$term.Count}}</div>

View file

@ -1,11 +1,11 @@
<form class="ui form ignore-dirty">
{{template "shared/search/combo_fuzzy"
{{template "shared/search/combo_multi"
dict
"Value" .Keyword
"Disabled" .CodeIndexerUnavailable
"IsFuzzy" .IsFuzzy
"Placeholder" (ctx.Locale.Tr "search.code_kind")
"CodeIndexerDisabled" $.CodeIndexerDisabled}}
"Selected" $.CodeSearchMode
"Options" $.CodeSearchOptions}}
</form>
<div class="divider"></div>
<div class="ui user list">

View file

@ -2,14 +2,12 @@
{{/* Disabled (optional) - if search field/button has to be disabled */}}
{{/* Placeholder (optional) - placeholder text to be used */}}
{{/* IsFuzzy - state of the fuzzy/union search toggle */}}
{{/* CodeIndexerDisabled (optional) - if the performed search is done using git-grep */}}
{{/* Tooltip (optional) - a tooltip to be displayed on button hover */}}
<div class="ui small fluid action input">
{{template "shared/search/input" dict "Value" .Value "Disabled" .Disabled "Placeholder" .Placeholder}}
{{template "shared/search/fuzzy"
dict
"Disabled" .Disabled
"IsFuzzy" .IsFuzzy
"CodeIndexerDisabled" .CodeIndexerDisabled}}
"IsFuzzy" .IsFuzzy}}
{{template "shared/search/button" dict "Disabled" .Disabled "Tooltip" .Tooltip}}
</div>

View file

@ -0,0 +1,24 @@
{{/* Value - value of the search field (for search results page) */}}
{{/* Disabled (optional) - if search field/button has to be disabled */}}
{{/* Placeholder (optional) - placeholder text to be used */}}
{{/* Selected - the currently selected option */}}
{{/* Options - options available to choose from */}}
{{/* Tooltip (optional) - a tooltip to be displayed on button hover */}}
<div class="ui small fluid action input">
{{template "shared/search/input" dict "Value" .Value "Disabled" .Disabled "Placeholder" .Placeholder}}
<div class="ui small dropdown selection {{if .Disabled}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "search.type_tooltip"}}">
<div class="text">
{{ctx.Locale.Tr (printf "search.%s" .Selected)}}
</div>
<div class="menu" data-test-tag="fuzzy-dropdown">
{{range $opt := .Options}}
{{$isActive := eq $.Selected $opt}}
<label class="{{if $isActive}}active {{end}}item" data-value="{{$opt}}" data-tooltip-content="{{ctx.Locale.Tr (printf "search.%s_tooltip" $opt)}}">
<input hidden type="radio" name="mode" value="{{$opt}}"{{if $isActive}} checked{{end}}/>
{{ctx.Locale.Tr (printf "search.%s" $opt)}}
</label>
{{end}}
</div>
</div>
{{template "shared/search/button" dict "Disabled" .Disabled "Tooltip" .Tooltip}}
</div>

View file

@ -1,21 +1,15 @@
{{/* Disabled (optional) - if dropdown has to be disabled */}}
{{/* IsFuzzy - state of the fuzzy search toggle */}}
<div class="ui small dropdown selection {{if .Disabled}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "search.type_tooltip"}}" data-test-tag="fuzzy-dropdown">
{{$fuzzyType := "fuzzy"}}
{{if .CodeIndexerDisabled}}
{{$fuzzyType = "union"}}
{{end}}
<div class="ui small dropdown selection {{if .Disabled}} disabled{{end}}" data-tooltip-content="{{ctx.Locale.Tr "search.type_tooltip"}}">
<input name="fuzzy" type="hidden"{{if .Disabled}} disabled{{end}} value="{{.IsFuzzy}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
<div class="text">{{/*
if code indexer is disabled display fuzzy as union
*/}}{{if .IsFuzzy}}{{/*
*/}}{{ctx.Locale.Tr (printf "search.%s" $fuzzyType)}}{{/*
<div class="text">{{if .IsFuzzy}}{{/*
*/}}{{ctx.Locale.Tr "search.fuzzy"}}{{/*
*/}}{{else}}{{/*
*/}}{{ctx.Locale.Tr "search.exact"}}{{/*
*/}}{{end}}</div>
<div class="menu">
<div class="item" data-value="true" data-tooltip-content="{{ctx.Locale.Tr (printf "search.%s_tooltip" $fuzzyType)}}">{{/*
*/}}{{ctx.Locale.Tr (printf "search.%s" $fuzzyType)}}</div>
<div class="item" data-value="true" data-tooltip-content="{{ctx.Locale.Tr "search.fuzzy_tooltip"}}">{{/*
*/}}{{ctx.Locale.Tr "search.fuzzy"}}</div>
<div class="item" data-value="false" data-tooltip-content="{{ctx.Locale.Tr "search.exact_tooltip"}}">{{ctx.Locale.Tr "search.exact"}}</div>
</div>
</div>

View file

@ -79,25 +79,25 @@ func testSearchRepo(t *testing.T, indexer bool) {
}
testSearch(t, "/user2/glob/search?q=loren&page=1", []string{"a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=loren&page=1&fuzzy=false", []string{"a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=loren&page=1&mode=exact", []string{"a.txt"}, indexer)
if indexer {
// fuzzy search: matches both file3 (x/b.txt) and file1 (a.txt)
// when indexer is enabled
testSearch(t, "/user2/glob/search?q=file3&page=1", []string{"x/b.txt", "a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&page=1", []string{"x/b.txt", "a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file5&page=1", []string{"x/b.txt", "a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file3&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file5&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
} else {
// fuzzy search: Union/OR of all the keywords
// when indexer is disabled
testSearch(t, "/user2/glob/search?q=file3+file1&page=1", []string{"a.txt", "x/b.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&page=1", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file5&page=1", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file3+file1&mode=union&page=1", []string{"a.txt", "x/b.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&mode=union&page=1", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file5&mode=union&page=1", []string{}, indexer)
}
testSearch(t, "/user2/glob/search?q=file3&page=1&fuzzy=false", []string{"x/b.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&page=1&fuzzy=false", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file5&page=1&fuzzy=false", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file3&page=1&mode=exact", []string{"x/b.txt"}, indexer)
testSearch(t, "/user2/glob/search?q=file4&page=1&mode=exact", []string{}, indexer)
testSearch(t, "/user2/glob/search?q=file5&page=1&mode=exact", []string{}, indexer)
}
func testSearch(t *testing.T, url string, expected []string, indexer bool) {
@ -113,21 +113,19 @@ func testSearch(t *testing.T, url string, expected []string, indexer bool) {
branchDropdown := container.Find(".js-branch-tag-selector")
assert.EqualValues(t, indexer, len(branchDropdown.Nodes) == 0)
// if indexer is disabled "fuzzy" should be displayed as "union"
expectedFuzzy := "Fuzzy"
if !indexer {
expectedFuzzy = "Union"
}
dropdownOptions := container.
Find(".menu[data-test-tag=fuzzy-dropdown]").
Find("input[type=radio][name=mode]").
Map(func(_ int, sel *goquery.Selection) string {
attr, exists := sel.Attr("value")
assert.True(t, exists)
return attr
})
fuzzyDropdown := container.Find(".ui.dropdown[data-test-tag=fuzzy-dropdown]")
actualFuzzyText := fuzzyDropdown.Find(".menu .item[data-value=true]").First().Text()
assert.EqualValues(t, expectedFuzzy, actualFuzzyText)
if fuzzyDropdown.
Find("input[name=fuzzy][value=true]").
Length() != 0 {
actualFuzzyText = fuzzyDropdown.Find("div.text").First().Text()
assert.EqualValues(t, expectedFuzzy, actualFuzzyText)
if indexer {
assert.EqualValues(t, []string{"exact", "fuzzy"}, dropdownOptions)
} else {
assert.EqualValues(t, []string{"exact", "union", "regexp"}, dropdownOptions)
}
filenames := resultFilenames(t, doc)