Merge remote-tracking branch 'origin/master' into pre-filter-perf

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/bits-and-blooms/bitset v1.12.0
 	github.com/blevesearch/bleve_index_api v1.1.12
 	github.com/blevesearch/geo v0.1.20
+	github.com/blevesearch/go-faiss v1.0.22-0.20240919162919-05a9ee21155a
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/blevesearch/go-porterstemmer v1.0.3
 	github.com/blevesearch/goleveldb v1.0.1
@@ -32,7 +33,6 @@ require (
 )
 
 require (
-	github.com/blevesearch/go-faiss v1.0.22-0.20240919162919-05a9ee21155a // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
 	github.com/couchbase/ghistogram v0.1.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect

mapping/mapping_vectors.go

@@ -19,9 +19,9 @@ package mapping
 
 import (
 	"fmt"
-	"math"
 	"reflect"
 
+	faiss "github.com/blevesearch/go-faiss"
 	"github.com/blevesearch/bleve/v2/document"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
@@ -262,20 +262,6 @@ func validateVectorFieldAlias(field *FieldMapping, parentName string,
 	return nil
 }
 
-func NormalizeVector(vector []float32) []float32 {
-	// first calculate the magnitude of the vector
-	var mag float64
-	for _, v := range vector {
-		mag += float64(v) * float64(v)
-	}
-	// cannot normalize a zero vector
-	// if the magnitude is 1, then the vector is already normalized
-	if mag != 0 && mag != 1 {
-		mag = math.Sqrt(mag)
-		// normalize the vector
-		for i, v := range vector {
-			vector[i] = float32(float64(v) / mag)
-		}
-	}
-	return vector
+func NormalizeVector(vec []float32) []float32 {
+	return faiss.NormalizeVector(vec)
 }

mapping/mapping_vectors_test.go

@@ -18,6 +18,7 @@
 package mapping
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -306,3 +307,28 @@ func TestProcessVector(t *testing.T) {
 		}
 	}
 }
+
+func TestNormalizeVector(t *testing.T) {
+	vectors := [][]float32{
+		[]float32{1,2,3,4,5},
+		[]float32{1,0,0,0,0},
+		[]float32{0.182574183,0.365148365,0.547722578,0.730296731},
+		[]float32{1,1,1,1,1,1,1,1},
+		[]float32{0},
+	}
+
+	expectedNormalizedVectors := [][]float32{
+		[]float32{0.13483998,0.26967996,0.40451995,0.5393599,0.67419994},
+		[]float32{1,0,0,0,0},
+		[]float32{0.18257418,0.36514837,0.5477226,0.73029673},
+		[]float32{0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338},
+		[]float32{0},
+	}
+
+	for i := 0; i < len(vectors); i++ {
+		normalizedVector := NormalizeVector(vectors[i])
+		if !reflect.DeepEqual(normalizedVector, expectedNormalizedVectors[i]) {
+			t.Errorf("[vector-%d] Expected: %v, Got: %v", i+1, expectedNormalizedVectors[i], normalizedVector)
+		}
+	}
+}