From ab10172e2c1da8ae320603a602536b3e5d21075a Mon Sep 17 00:00:00 2001 From: Aditi Ahuja <48997495+metonymic-smokey@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:44:08 +0530 Subject: [PATCH] MB-57871, MB-62230: Upgrade to zapx/v16@v16.1.7 + go-faiss@v1.0.23 (#2088) MB-62230: * Add elements to bitmap in a batch. * 5f8f0f0 Aditi Ahuja | MB-62230 - Pre-filtering performance optimisations MB-57871: * 7bab2f9 Abhi Dangeti | MB-57871: One more nil check before dereferencing postings * bf971e6 Abhi Dangeti | Add nil pointer checks for postings before dereference * 9a80f5f Abhi Dangeti | One more case for zero chunk size / mode * 0c6f397 Mohd Shaad Khan | Handle zero chunk size --------- Co-authored-by: Abhinav Dangeti --- go.mod | 4 ++-- go.sum | 8 ++++---- index/scorch/optimize_knn.go | 2 +- index/scorch/snapshot_index_vr.go | 15 +++++++++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index c15a1089f..2beb43aac 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/bits-and-blooms/bitset v1.12.0 github.com/blevesearch/bleve_index_api v1.1.12 github.com/blevesearch/geo v0.1.20 - github.com/blevesearch/go-faiss v1.0.22 + github.com/blevesearch/go-faiss v1.0.23 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/goleveldb v1.0.1 @@ -24,7 +24,7 @@ require ( github.com/blevesearch/zapx/v13 v13.3.10 github.com/blevesearch/zapx/v14 v14.3.10 github.com/blevesearch/zapx/v15 v15.3.15 - github.com/blevesearch/zapx/v16 v16.1.7-0.20241016180148-7bab2f98bc3d + github.com/blevesearch/zapx/v16 v16.1.7 github.com/couchbase/moss v0.2.0 github.com/golang/protobuf v1.3.2 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index f51d12ef3..a8a83fac2 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/blevesearch/bleve_index_api v1.1.12 h1:P4bw9/G/5rulOF7SJ9l4FsDoo7UFJ+ github.com/blevesearch/bleve_index_api v1.1.12/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= -github.com/blevesearch/go-faiss v1.0.22 h1:j6jwgCOy2a2EQUTOYxjBA59rMn5KPA0jbfYyHNgc2Ls= -github.com/blevesearch/go-faiss v1.0.22/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/blevesearch/go-faiss v1.0.23 h1:Wmc5AFwDLKGl2L6mjLX1Da3vCL0EKa2uHHSorcIS1Uc= +github.com/blevesearch/go-faiss v1.0.23/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= @@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= github.com/blevesearch/zapx/v15 v15.3.15 h1:JydcGIq279tmTrfBBSPDF/VOiCMBLQ+rJulTXrGFlGA= github.com/blevesearch/zapx/v15 v15.3.15/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/blevesearch/zapx/v16 v16.1.7-0.20241016180148-7bab2f98bc3d h1:XpLXi4MVxygh7yQaMfSt/ta7TgfaCYD6pBUfk+FfaHo= -github.com/blevesearch/zapx/v16 v16.1.7-0.20241016180148-7bab2f98bc3d/go.mod h1:I1CnZUekZFlJZ+cUCnmoJZTD49suuunjuH6BdOTtT9g= +github.com/blevesearch/zapx/v16 v16.1.7 h1:I07qV6l1rPda19zyof9Q2J9E8cjZ57pQhNY0+ePI5vM= +github.com/blevesearch/zapx/v16 v16.1.7/go.mod h1:JqQlOqlRVaYDkpLIl3JnKql8u4zKTNlVEa3nLsi0Gn8= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/optimize_knn.go b/index/scorch/optimize_knn.go index 7f0f0907b..ca179574c 100644 --- a/index/scorch/optimize_knn.go +++ b/index/scorch/optimize_knn.go @@ -115,7 +115,7 @@ func (o *OptimizeVR) Finish() error { } eligibleLocalDocNums := make([]uint64, - eligibleVectorInternalIDs.Stats().Cardinality) + eligibleVectorInternalIDs.GetCardinality()) // get the (segment-)local document numbers for i, docNum := range eligibleVectorInternalIDs.ToArray() { localDocNum := o.snapshot.localDocNumFromGlobal(index, diff --git a/index/scorch/snapshot_index_vr.go b/index/scorch/snapshot_index_vr.go index 9d32fd6aa..320364bc7 100644 --- a/index/scorch/snapshot_index_vr.go +++ b/index/scorch/snapshot_index_vr.go @@ -64,10 +64,17 @@ type IndexSnapshotVectorReader struct { // (AND) operations. Eg. finding the eligible doc IDs present in a segment. func (i *IndexSnapshotVectorReader) getEligibleDocIDs() *roaring.Bitmap { res := roaring.NewBitmap() - // converts the doc IDs to uint32 and returns - for _, eligibleDocInternalID := range i.eligibleDocIDs { - internalDocID, _ := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID)) - res.Add(uint32(internalDocID)) + if len(i.eligibleDocIDs) > 0 { + internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs)) + // converts the doc IDs to uint32 and returns + for _, eligibleDocInternalID := range i.eligibleDocIDs { + internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID)) + if err != nil { + continue + } + internalDocIDs = append(internalDocIDs, uint32(internalDocID)) + } + res.AddMany(internalDocIDs) } return res }