Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions changelog/unreleased/enhancement-search-index-force-flag.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
Enhancement: Add a --force flag to the search index command

`ocis search index` skipped every file that was already indexed and had not
changed since. There was no way to index such files again, for example after
enabling OCR in Tika, other than deleting the whole search index, which makes
search unavailable until it has been rebuilt.

The command now has a `--force` flag. With it, every file of the given space,
or of all spaces, is extracted and indexed again, while the existing index
entries stay searchable. Without the flag the behaviour is unchanged.

https://github.com/owncloud/ocis/issues/9106
115 changes: 63 additions & 52 deletions protogen/gen/ocis/services/search/v0/search.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions protogen/gen/ocis/services/search/v0/search.swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,10 @@
},
"userId": {
"type": "string"
},
"force": {
"type": "boolean",
"description": "force indexes all resources again, including those that are already\nindexed and unchanged."
}
}
},
Expand Down
3 changes: 3 additions & 0 deletions protogen/proto/ocis/services/search/v0/search.proto
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ message SearchIndexResponse {
message IndexSpaceRequest {
string space_id = 1;
string user_id = 2;
// force indexes all resources again, including those that are already
// indexed and unchanged.
bool force = 3;
}

message IndexSpaceResponse {
Expand Down
8 changes: 8 additions & 0 deletions services/search/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,14 @@ ocis search index --all-spaces

Note that either `--space $SPACE_ID` or `--all-spaces` must be set.

By default, files that are already indexed and have not changed since are skipped. To index them again anyway, for example after enabling OCR in Tika so that existing files get their text extracted, add `--force`:

```shell
ocis search index --all-spaces --force
```

The existing index entries stay searchable while they are being replaced, so there is no search downtime for the space.

The command does not open the index itself: it asks the running search service to do the indexing over gRPC. The search service must therefore be running, and the command works with the default `SEARCH_ENGINE_BLEVE_SCALE=false` (see [Scaling](#scaling)); the service's exclusive lock on the index does not block it.

The command waits for the service to walk the space and gives up after 10 minutes by default. The walk is cancelled when that happens, so a large space, or one where the content extractor is slow on a few files, may need a longer limit: `--timeout 2h` (a Go duration; `0` waits indefinitely).
Expand Down
5 changes: 5 additions & 0 deletions services/search/pkg/command/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ func Index(cfg *config.Config) *cli.Command {
Name: "all-spaces",
Usage: "index all spaces instead. This or --space is required.",
},
&cli.BoolFlag{
Name: "force",
Usage: "index all files again, even those that are already indexed and have not changed, e.g. after enabling OCR in Tika. Without it, unchanged files are skipped.",
},
&cli.DurationFlag{
Name: "timeout",
Usage: "how long to wait for the indexing to finish before giving up, e.g. 2h. The walk is cancelled when the timeout expires. 0 waits indefinitely.",
Expand Down Expand Up @@ -78,6 +82,7 @@ func Index(cfg *config.Config) *cli.Command {
c := searchsvc.NewSearchProviderService("com.owncloud.api.search", grpcClient)
_, err = c.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
SpaceId: ctx.String("space"),
Force: ctx.Bool("force"),
}, func(opts *client.CallOptions) { opts.RequestTimeout = timeout })
if err != nil {
fmt.Println("failed to index space: " + err.Error())
Expand Down
2 changes: 1 addition & 1 deletion services/search/pkg/search/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func HandleEvents(s Searcher, bus events.Consumer, logger log.Logger, cfg *confi
}

indexSpaceDebouncer := NewSpaceDebouncer(time.Duration(cfg.Events.DebounceDuration)*time.Millisecond, func(ctx context.Context, id *provider.StorageSpaceId) {
if err := s.IndexSpace(ctx, id); err != nil {
if err := s.IndexSpace(ctx, id, false); err != nil {
logger.Error().Err(err).Interface("spaceID", id).Msg("error while indexing a space")
}
})
Expand Down
7 changes: 6 additions & 1 deletion services/search/pkg/search/events_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,12 @@ var _ = DescribeTable("events",
})

for _, mck := range mcks {
s.On(mck, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) {
args := []any{mock.Anything, mock.Anything}
if mck == "IndexSpace" {
// events never force a re-index of unchanged files
args = append(args, false)
}
s.On(mck, args...).Return(nil).Run(func(_ mock.Arguments) {
calls.Add(1)
})
}
Expand Down
21 changes: 11 additions & 10 deletions services/search/pkg/search/mocks/searcher.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 7 additions & 4 deletions services/search/pkg/search/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ var (
// Searcher is the interface to the SearchService
type Searcher interface {
Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
IndexSpace(ctx context.Context, rID *provider.StorageSpaceId) error
IndexSpace(ctx context.Context, rID *provider.StorageSpaceId, force bool) error
TrashItem(ctx context.Context, rID *provider.ResourceId)
UpsertItem(ctx context.Context, ref *provider.Reference)
UpdateTags(ctx context.Context, ref *provider.Reference)
Expand Down Expand Up @@ -477,8 +477,11 @@ const (
_missingMinCount = 5
)

// IndexSpace (re)indexes all resources of a given space.
func (s *Service) IndexSpace(ctx context.Context, spaceID *provider.StorageSpaceId) error {
// IndexSpace (re)indexes all resources of a given space. Resources that are
// already indexed and unchanged are skipped, unless force is set: then every
// resource is extracted and indexed again, e.g. after the extractor
// configuration changed.
func (s *Service) IndexSpace(ctx context.Context, spaceID *provider.StorageSpaceId, force bool) error {
ownerCtx, err := getAuthContext(ctx, s.serviceAccountID, s.gatewaySelector, s.serviceAccountSecret, s.logger)
if err != nil {
return err
Expand Down Expand Up @@ -539,7 +542,7 @@ func (s *Service) IndexSpace(ctx context.Context, spaceID *provider.StorageSpace

resourceID := storagespace.FormatResourceID(info.Id)
r, err := s.engine.Lookup(resourceID)
if err == nil && r.Extracted {
if !force && err == nil && r.Extracted {
fileMtime := utils.TSToTime(info.Mtime)
docMtime, parseErr := time.Parse(time.RFC3339Nano, r.Mtime)
if parseErr == nil && !docMtime.Before(fileMtime) {
Expand Down
Loading
Loading