From e89330492a4f8d823eff2e921cd5efb46ee01751 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 22 Apr 2025 12:03:46 +0200 Subject: [PATCH 01/45] wrap ES knn queries with patience query --- .../mapper/vectors/DenseVectorFieldMapper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index b9ea4b78ec499..4b8a534d3f5c4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -31,6 +31,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.PatienceKnnVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.util.BitUtil; @@ -2198,9 +2199,9 @@ private Query createKnnBitQuery( BitSetProducer parentFilter ) { elementType.checkDimensions(dims, queryVector.length); - Query knnQuery = parentFilter != null + Query knnQuery = PatienceKnnVectorQuery.fromByteQuery(parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter); + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter)); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2225,9 +2226,9 @@ private Query createKnnByteQuery( float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorByteElementsAppender(queryVector), squaredMagnitude); } - Query knnQuery = parentFilter != null + Query knnQuery = PatienceKnnVectorQuery.fromByteQuery(parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter); + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter)); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2278,9 +2279,9 @@ && isNotUnitVector(squaredMagnitude)) { adjustedK = Math.min((int) Math.ceil(k * oversample), OVERSAMPLE_LIMIT); numCands = Math.max(adjustedK, numCands); } - Query knnQuery = parentFilter != null + Query knnQuery = PatienceKnnVectorQuery.fromFloatQuery(parentFilter != null ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, numCands, parentFilter) - : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter); + : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter)); if (rescore) { knnQuery = new RescoreKnnVectorQuery( name(), From a0634d4a688736a12097971d5102ab6c180362d0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 23 Apr 2025 09:44:51 +0000 Subject: [PATCH 02/45] [CI] Auto commit changes from spotless --- .../vectors/DenseVectorFieldMapper.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 4b8a534d3f5c4..b2c350d8ac057 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2199,9 +2199,11 @@ private Query createKnnBitQuery( BitSetProducer parentFilter ) { elementType.checkDimensions(dims, queryVector.length); - Query knnQuery = PatienceKnnVectorQuery.fromByteQuery(parentFilter != null - ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter)); + Query knnQuery = PatienceKnnVectorQuery.fromByteQuery( + parentFilter != null + ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter) + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter) + ); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2226,9 +2228,11 @@ private Query createKnnByteQuery( float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorByteElementsAppender(queryVector), squaredMagnitude); } - Query knnQuery = PatienceKnnVectorQuery.fromByteQuery(parentFilter != null - ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter)); + Query knnQuery = PatienceKnnVectorQuery.fromByteQuery( + parentFilter != null + ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter) + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter) + ); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2279,9 +2283,11 @@ && isNotUnitVector(squaredMagnitude)) { adjustedK = Math.min((int) Math.ceil(k * oversample), OVERSAMPLE_LIMIT); numCands = Math.max(adjustedK, numCands); } - Query knnQuery = PatienceKnnVectorQuery.fromFloatQuery(parentFilter != null - ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, numCands, parentFilter) - : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter)); + Query knnQuery = PatienceKnnVectorQuery.fromFloatQuery( + parentFilter != null + ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, numCands, parentFilter) + : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter) + ); if (rescore) { knnQuery = new RescoreKnnVectorQuery( name(), From e62dac9eea00a9021cbc0190d1795ce634609826 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 8 May 2025 13:42:29 +0000 Subject: [PATCH 03/45] [CI] Auto commit changes from spotless --- .../vectors/DenseVectorFieldMapper.java | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 33c0cdc118542..5393e96c8ed19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2318,9 +2318,11 @@ private Query createKnnBitQuery( KnnSearchStrategy searchStrategy ) { elementType.checkDimensions(dims, queryVector.length); - Query knnQuery = PatienceKnnVectorQuery.fromByteQuery(parentFilter != null - ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy)); + Query knnQuery = PatienceKnnVectorQuery.fromByteQuery( + parentFilter != null + ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy) + ); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2346,9 +2348,11 @@ private Query createKnnByteQuery( float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorByteElementsAppender(queryVector), squaredMagnitude); } - Query knnQuery = PatienceKnnVectorQuery.fromByteQuery(parentFilter != null - ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy)); + Query knnQuery = PatienceKnnVectorQuery.fromByteQuery( + parentFilter != null + ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy) + ); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( @@ -2401,17 +2405,19 @@ && isNotUnitVector(squaredMagnitude)) { adjustedK = Math.min((int) Math.ceil(k * oversample), OVERSAMPLE_LIMIT); numCands = Math.max(adjustedK, numCands); } - Query knnQuery = PatienceKnnVectorQuery.fromFloatQuery(parentFilter != null - ? new ESDiversifyingChildrenFloatKnnVectorQuery( - name(), - queryVector, - filter, - adjustedK, - numCands, - parentFilter, - knnSearchStrategy - ) - : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy)); + Query knnQuery = PatienceKnnVectorQuery.fromFloatQuery( + parentFilter != null + ? new ESDiversifyingChildrenFloatKnnVectorQuery( + name(), + queryVector, + filter, + adjustedK, + numCands, + parentFilter, + knnSearchStrategy + ) + : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy) + ); if (rescore) { knnQuery = new RescoreKnnVectorQuery( From 23494d1850e0633a5d5365fe789c941599546218 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 27 May 2025 17:09:25 +0200 Subject: [PATCH 04/45] refactoring, test fixes --- .../vectors/DenseVectorFieldMapper.java | 143 ++++++++++++++---- .../vectors/DenseVectorFieldTypeTests.java | 47 +++--- 2 files changed, 146 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 5393e96c8ed19..592e8863209e1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -31,6 +31,8 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.KnnByteVectorQuery; +import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.PatienceKnnVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; @@ -98,6 +100,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_VERSION_CREATED; import static org.elasticsearch.common.Strings.format; +import static org.elasticsearch.common.Strings.isNullOrBlank; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; /** @@ -107,6 +110,7 @@ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; private static final float EPS = 1e-3f; public static final int BBQ_MIN_DIMS = 64; + private static final String EARLY_EXIT_PARAM_NAME = "early_exit"; public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; @@ -1334,6 +1338,8 @@ public boolean validateDimension(int dim, boolean throwOnError) { return supportsDimension; } + abstract boolean earlyExit(); + abstract boolean doEquals(IndexOptions other); abstract int doHashCode(); @@ -1371,16 +1377,21 @@ public enum VectorIndexType { public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); + Object earlyExitNode = indexOptionsMap.remove(EARLY_EXIT_PARAM_NAME); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } + if (earlyExitNode == null) { + earlyExitNode = true; + } + boolean earlyExit = XContentMapValues.nodeBooleanValue(earlyExitNode); int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new HnswIndexOptions(m, efConstruction); + return new HnswIndexOptions(m, efConstruction, earlyExit); } @Override @@ -1399,12 +1410,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); + Object earlyExitNode = indexOptionsMap.remove(EARLY_EXIT_PARAM_NAME); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } + if (earlyExitNode == null) { + earlyExitNode = true; + } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); Float confidenceInterval = null; @@ -1415,8 +1430,9 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti if (hasRescoreIndexVersion(indexVersion)) { rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion); } + boolean earlyExit = XContentMapValues.nodeBooleanValue(earlyExitNode); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval, rescoreVector); + return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval, rescoreVector, earlyExit); } @Override @@ -1434,12 +1450,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); + Object earlyExitNode = indexOptionsMap.remove(EARLY_EXIT_PARAM_NAME); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } + if (earlyExitNode == null) { + earlyExitNode = true; + } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); Float confidenceInterval = null; @@ -1450,8 +1470,9 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti if (hasRescoreIndexVersion(indexVersion)) { rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion); } + boolean earlyExit = XContentMapValues.nodeBooleanValue(earlyExitNode); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new Int4HnswIndexOptions(m, efConstruction, confidenceInterval, rescoreVector); + return new Int4HnswIndexOptions(m, efConstruction, confidenceInterval, rescoreVector, earlyExit); } @Override @@ -1538,12 +1559,16 @@ public boolean supportsDimension(int dims) { public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); + Object earlyExitNode = indexOptionsMap.remove(EARLY_EXIT_PARAM_NAME); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } + if (earlyExitNode == null) { + earlyExitNode = true; + } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); RescoreVector rescoreVector = null; @@ -1553,8 +1578,9 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti rescoreVector = new RescoreVector(DEFAULT_OVERSAMPLE); } } + boolean earlyExit = XContentMapValues.nodeBooleanValue(earlyExitNode); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new BBQHnswIndexOptions(m, efConstruction, rescoreVector); + return new BBQHnswIndexOptions(m, efConstruction, rescoreVector, earlyExit); } @Override @@ -1667,6 +1693,11 @@ boolean updatableTo(IndexOptions update) { || update.type.equals(VectorIndexType.INT4_HNSW) || update.type.equals(VectorIndexType.INT4_FLAT); } + + @Override + boolean earlyExit() { + return false; + } } static class FlatIndexOptions extends IndexOptions { @@ -1696,6 +1727,11 @@ boolean updatableTo(IndexOptions update) { return true; } + @Override + boolean earlyExit() { + return false; + } + @Override public boolean doEquals(IndexOptions o) { return o instanceof FlatIndexOptions; @@ -1711,14 +1747,20 @@ static class Int4HnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; private final float confidenceInterval; + private final boolean earlyExit; Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { + this(m, efConstruction, confidenceInterval, rescoreVector, true); + } + + Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector, boolean earlyExit) { super(VectorIndexType.INT4_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; // The default confidence interval for int4 is dynamic quantiles, this provides the best relevancy and is // effectively required for int4 to behave well across a wide range of data. this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval; + this.earlyExit = earlyExit; } @Override @@ -1737,6 +1779,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (rescoreVector != null) { rescoreVector.toXContent(builder, params); } + builder.field(EARLY_EXIT_PARAM_NAME, earlyExit); builder.endObject(); return builder; } @@ -1781,6 +1824,11 @@ boolean updatableTo(IndexOptions update) { } return updatable; } + + @Override + boolean earlyExit() { + return earlyExit; + } } static class Int4FlatIndexOptions extends QuantizedIndexOptions { @@ -1838,18 +1886,29 @@ boolean updatableTo(IndexOptions update) { || update.type.equals(VectorIndexType.INT4_HNSW); } + @Override + boolean earlyExit() { + return false; + } + } public static class Int8HnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; private final Float confidenceInterval; + private final boolean earlyExit; public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { + this(m, efConstruction, confidenceInterval, rescoreVector, true); + } + + public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector, boolean earlyExit) { super(VectorIndexType.INT8_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; this.confidenceInterval = confidenceInterval; + this.earlyExit = earlyExit; } @Override @@ -1870,6 +1929,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (rescoreVector != null) { rescoreVector.toXContent(builder, params); } + builder.field(EARLY_EXIT_PARAM_NAME, earlyExit); builder.endObject(); return builder; } @@ -1921,16 +1981,27 @@ boolean updatableTo(IndexOptions update) { } return updatable; } + + @Override + boolean earlyExit() { + return earlyExit; + } } static class HnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; + private final boolean earlyExit; HnswIndexOptions(int m, int efConstruction) { + this(m, efConstruction, true); + } + + HnswIndexOptions(int m, int efConstruction, boolean earlyExit) { super(VectorIndexType.HNSW); this.m = m; this.efConstruction = efConstruction; + this.earlyExit = earlyExit; } @Override @@ -1954,12 +2025,18 @@ boolean updatableTo(IndexOptions update) { || (update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= m); } + @Override + boolean earlyExit() { + return earlyExit; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("type", type); builder.field("m", m); builder.field("ef_construction", efConstruction); + builder.field(EARLY_EXIT_PARAM_NAME, earlyExit); builder.endObject(); return builder; } @@ -1986,11 +2063,17 @@ public String toString() { public static class BBQHnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; + private final boolean earlyExit; public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) { + this(m, efConstruction, rescoreVector, true); + } + + public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector, boolean earlyExit) { super(VectorIndexType.BBQ_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; + this.earlyExit = earlyExit; } @Override @@ -2024,6 +2107,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (rescoreVector != null) { rescoreVector.toXContent(builder, params); } + builder.field(EARLY_EXIT_PARAM_NAME, earlyExit); builder.endObject(); return builder; } @@ -2038,6 +2122,11 @@ public boolean validateDimension(int dim, boolean throwOnError) { } return supportsDimension; } + + @Override + boolean earlyExit() { + return earlyExit; + } } static class BBQFlatIndexOptions extends QuantizedIndexOptions { @@ -2090,6 +2179,11 @@ public boolean validateDimension(int dim, boolean throwOnError) { return supportsDimension; } + @Override + boolean earlyExit() { + return false; + } + } public record RescoreVector(float oversample) implements ToXContentObject { @@ -2318,11 +2412,12 @@ private Query createKnnBitQuery( KnnSearchStrategy searchStrategy ) { elementType.checkDimensions(dims, queryVector.length); - Query knnQuery = PatienceKnnVectorQuery.fromByteQuery( - parentFilter != null - ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy) - ); + KnnByteVectorQuery knnByteVectorQuery = parentFilter != null + ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); + // TODO: add saturation threshold and patience params ? + Query knnQuery = indexOptions.earlyExit() ? + PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2348,11 +2443,12 @@ private Query createKnnByteQuery( float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorByteElementsAppender(queryVector), squaredMagnitude); } - Query knnQuery = PatienceKnnVectorQuery.fromByteQuery( - parentFilter != null - ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) - : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy) - ); + KnnByteVectorQuery knnByteVectorQuery = parentFilter != null + ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) + : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); + // TODO: add saturation threshold and patience params ? + Query knnQuery = indexOptions.earlyExit() ? + PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( @@ -2405,19 +2501,12 @@ && isNotUnitVector(squaredMagnitude)) { adjustedK = Math.min((int) Math.ceil(k * oversample), OVERSAMPLE_LIMIT); numCands = Math.max(adjustedK, numCands); } - Query knnQuery = PatienceKnnVectorQuery.fromFloatQuery( - parentFilter != null - ? new ESDiversifyingChildrenFloatKnnVectorQuery( - name(), - queryVector, - filter, - adjustedK, - numCands, - parentFilter, - knnSearchStrategy - ) - : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy) - ); + KnnFloatVectorQuery knnFloatVectorQuery = parentFilter != null + ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, numCands, parentFilter, + knnSearchStrategy) : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); + // TODO: add saturation threshold and patience params ? + Query knnQuery = indexOptions.earlyExit() ? + PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery) : knnFloatVectorQuery; if (rescore) { knnQuery = new RescoreKnnVectorQuery( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 5877ce9003ff5..7b15e38e5d3b4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -71,13 +71,15 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() { randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()), + randomBoolean() ), new DenseVectorFieldMapper.Int4HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()), + randomBoolean() ), new DenseVectorFieldMapper.FlatIndexOptions(), new DenseVectorFieldMapper.Int8FlatIndexOptions( @@ -91,7 +93,8 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() { new DenseVectorFieldMapper.BBQHnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()), + randomBoolean() ), new DenseVectorFieldMapper.BBQFlatIndexOptions(randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())) ); @@ -107,13 +110,15 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsHnswQuantized(Dens randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - rescoreVector + rescoreVector, + randomBoolean() ), new DenseVectorFieldMapper.Int4HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - rescoreVector + rescoreVector, + randomBoolean() ), new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector) ); @@ -558,13 +563,17 @@ public void testRescoreOversampleUsedWithoutQuantization() { ); if (elementType == BYTE) { - ESKnnByteVectorQuery esKnnQuery = (ESKnnByteVectorQuery) knnQuery; - assertThat(esKnnQuery.getK(), is(100)); - assertThat(esKnnQuery.kParam(), is(10)); + KnnByteVectorQuery knnByteVectorQuery = (KnnByteVectorQuery) knnQuery; + assertThat(knnByteVectorQuery.getK(), is(100)); + if (knnByteVectorQuery instanceof ESKnnByteVectorQuery esKnnByteVectorQuery) { + assertThat(esKnnByteVectorQuery.kParam(), is(10)); + } } else { - ESKnnFloatVectorQuery esKnnQuery = (ESKnnFloatVectorQuery) knnQuery; - assertThat(esKnnQuery.getK(), is(100)); - assertThat(esKnnQuery.kParam(), is(10)); + KnnFloatVectorQuery knnFloatVectorQuery = (KnnFloatVectorQuery) knnQuery; + assertThat(knnFloatVectorQuery.getK(), is(100)); + if (knnFloatVectorQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { + assertThat(esKnnFloatVectorQuery.kParam(), is(10)); + } } } @@ -610,7 +619,7 @@ public void testRescoreOversampleQueryOverrides() { null, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertTrue(query instanceof ESKnnFloatVectorQuery); + assertTrue(query instanceof KnnFloatVectorQuery); // verify we can override a `0` to a positive number fieldType = new DenseVectorFieldType( @@ -635,8 +644,10 @@ public void testRescoreOversampleQueryOverrides() { ); assertTrue(query instanceof RescoreKnnVectorQuery); assertThat(((RescoreKnnVectorQuery) query).k(), equalTo(10)); - ESKnnFloatVectorQuery esKnnQuery = (ESKnnFloatVectorQuery) ((RescoreKnnVectorQuery) query).innerQuery(); - assertThat(esKnnQuery.kParam(), equalTo(20)); + KnnFloatVectorQuery esKnnQuery = (KnnFloatVectorQuery) ((RescoreKnnVectorQuery) query).innerQuery(); + if (esKnnQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { + assertThat(esKnnFloatVectorQuery.kParam(), equalTo(20)); + } } @@ -709,9 +720,11 @@ private static void checkRescoreQueryParameters( randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); RescoreKnnVectorQuery rescoreQuery = (RescoreKnnVectorQuery) query; - ESKnnFloatVectorQuery esKnnQuery = (ESKnnFloatVectorQuery) rescoreQuery.innerQuery(); + KnnFloatVectorQuery knnQuery = (KnnFloatVectorQuery) rescoreQuery.innerQuery(); assertThat("Unexpected total results", rescoreQuery.k(), equalTo(expectedResults)); - assertThat("Unexpected k parameter", esKnnQuery.kParam(), equalTo(expectedK)); - assertThat("Unexpected candidates", esKnnQuery.getK(), equalTo(expectedCandidates)); + assertThat("Unexpected candidates", knnQuery.getK(), equalTo(expectedCandidates)); + if (knnQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { + assertThat("Unexpected k parameter", esKnnFloatVectorQuery.kParam(), equalTo(expectedK)); + } } } From acd122f00a7ab3d8a5b449371bd41f9caf665e94 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 27 May 2025 15:21:12 +0000 Subject: [PATCH 05/45] [CI] Auto commit changes from spotless --- .../vectors/DenseVectorFieldMapper.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 9c2695ad940e1..f1ecc787bf3b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -108,7 +108,6 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_VERSION_CREATED; import static org.elasticsearch.common.Strings.format; -import static org.elasticsearch.common.Strings.isNullOrBlank; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; /** @@ -2428,8 +2427,7 @@ private Query createKnnBitQuery( ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); // TODO: add saturation threshold and patience params ? - Query knnQuery = indexOptions.earlyExit() ? - PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; + Query knnQuery = indexOptions.earlyExit() ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2459,8 +2457,7 @@ private Query createKnnByteQuery( ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); // TODO: add saturation threshold and patience params ? - Query knnQuery = indexOptions.earlyExit() ? - PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; + Query knnQuery = indexOptions.earlyExit() ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( @@ -2514,11 +2511,18 @@ && isNotUnitVector(squaredMagnitude)) { numCands = Math.max(adjustedK, numCands); } KnnFloatVectorQuery knnFloatVectorQuery = parentFilter != null - ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, numCands, parentFilter, - knnSearchStrategy) : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); + ? new ESDiversifyingChildrenFloatKnnVectorQuery( + name(), + queryVector, + filter, + adjustedK, + numCands, + parentFilter, + knnSearchStrategy + ) + : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); // TODO: add saturation threshold and patience params ? - Query knnQuery = indexOptions.earlyExit() ? - PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery) : knnFloatVectorQuery; + Query knnQuery = indexOptions.earlyExit() ? PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery) : knnFloatVectorQuery; if (rescore) { knnQuery = new RescoreKnnVectorQuery( From 98c42d2307f76ad332016aaa392f010d48746c30 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 28 May 2025 15:11:16 +0200 Subject: [PATCH 06/45] make early termination disabled by default --- .../vectors/DenseVectorFieldMapper.java | 17 +++++---- .../vectors/DenseVectorFieldTypeTests.java | 38 +++++++++++-------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index f1ecc787bf3b8..fb66e02739979 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -117,7 +117,10 @@ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; private static final float EPS = 1e-3f; public static final int BBQ_MIN_DIMS = 64; + + // early termination configuration private static final String EARLY_EXIT_PARAM_NAME = "early_exit"; + private static final boolean DEFAULT_EARLY_EXIT = false; public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; @@ -1426,7 +1429,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } if (earlyExitNode == null) { - earlyExitNode = true; + earlyExitNode = DEFAULT_EARLY_EXIT; } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); @@ -1704,7 +1707,7 @@ boolean updatableTo(IndexOptions update) { @Override boolean earlyExit() { - return false; + return DEFAULT_EARLY_EXIT; } } @@ -1737,7 +1740,7 @@ boolean updatableTo(IndexOptions update) { @Override boolean earlyExit() { - return false; + return DEFAULT_EARLY_EXIT; } @Override @@ -1758,7 +1761,7 @@ static class Int4HnswIndexOptions extends QuantizedIndexOptions { private final boolean earlyExit; Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { - this(m, efConstruction, confidenceInterval, rescoreVector, true); + this(m, efConstruction, confidenceInterval, rescoreVector, DEFAULT_EARLY_EXIT); } Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector, boolean earlyExit) { @@ -1896,7 +1899,7 @@ boolean updatableTo(IndexOptions update) { @Override boolean earlyExit() { - return false; + return DEFAULT_EARLY_EXIT; } } @@ -1908,7 +1911,7 @@ public static class Int8HnswIndexOptions extends QuantizedIndexOptions { private final boolean earlyExit; public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { - this(m, efConstruction, confidenceInterval, rescoreVector, true); + this(m, efConstruction, confidenceInterval, rescoreVector, DEFAULT_EARLY_EXIT); } public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector, boolean earlyExit) { @@ -2002,7 +2005,7 @@ static class HnswIndexOptions extends IndexOptions { private final boolean earlyExit; HnswIndexOptions(int m, int efConstruction) { - this(m, efConstruction, true); + this(m, efConstruction, DEFAULT_EARLY_EXIT); } HnswIndexOptions(int m, int efConstruction, boolean earlyExit) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index dc1e3c4b3d951..b333817397925 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.PatienceKnnVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; @@ -478,7 +479,7 @@ public void testCreateKnnQueryMaxDims() { null, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertThat(query, instanceOf(KnnByteVectorQuery.class)); + assertThat(query, instanceOf(ESKnnByteVectorQuery.class)); } } @@ -577,16 +578,20 @@ public void testRescoreOversampleUsedWithoutQuantization() { ); if (elementType == BYTE) { - KnnByteVectorQuery knnByteVectorQuery = (KnnByteVectorQuery) knnQuery; - assertThat(knnByteVectorQuery.getK(), is(100)); - if (knnByteVectorQuery instanceof ESKnnByteVectorQuery esKnnByteVectorQuery) { - assertThat(esKnnByteVectorQuery.kParam(), is(10)); + if (knnQuery instanceof PatienceKnnVectorQuery patienceKnnVectorQuery) { + assertThat(patienceKnnVectorQuery.getK(), is(100)); + } else { + ESKnnByteVectorQuery knnByteVectorQuery = (ESKnnByteVectorQuery) knnQuery; + assertThat(knnByteVectorQuery.getK(), is(100)); + assertThat(knnByteVectorQuery.kParam(), is(10)); } } else { - KnnFloatVectorQuery knnFloatVectorQuery = (KnnFloatVectorQuery) knnQuery; - assertThat(knnFloatVectorQuery.getK(), is(100)); - if (knnFloatVectorQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { - assertThat(esKnnFloatVectorQuery.kParam(), is(10)); + if (knnQuery instanceof PatienceKnnVectorQuery patienceKnnVectorQuery) { + assertThat(patienceKnnVectorQuery.getK(), is(100)); + } else { + ESKnnFloatVectorQuery knnFloatVectorQuery = (ESKnnFloatVectorQuery) knnQuery; + assertThat(knnFloatVectorQuery.getK(), is(100)); + assertThat(knnFloatVectorQuery.kParam(), is(10)); } } } @@ -635,7 +640,7 @@ public void testRescoreOversampleQueryOverrides() { null, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertTrue(query instanceof KnnFloatVectorQuery); + assertTrue(query instanceof ESKnnFloatVectorQuery); // verify we can override a `0` to a positive number fieldType = new DenseVectorFieldType( @@ -738,11 +743,14 @@ private static void checkRescoreQueryParameters( randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); RescoreKnnVectorQuery rescoreQuery = (RescoreKnnVectorQuery) query; - KnnFloatVectorQuery knnQuery = (KnnFloatVectorQuery) rescoreQuery.innerQuery(); - assertThat("Unexpected total results", rescoreQuery.k(), equalTo(expectedResults)); - assertThat("Unexpected candidates", knnQuery.getK(), equalTo(expectedCandidates)); - if (knnQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { - assertThat("Unexpected k parameter", esKnnFloatVectorQuery.kParam(), equalTo(expectedK)); + Query innerQuery = rescoreQuery.innerQuery(); + if (innerQuery instanceof PatienceKnnVectorQuery patienceKnnVectorQuery) { + assertThat("Unexpected candidates", patienceKnnVectorQuery.getK(), equalTo(expectedCandidates)); + } else { + ESKnnFloatVectorQuery knnQuery = (ESKnnFloatVectorQuery) innerQuery; + assertThat("Unexpected total results", rescoreQuery.k(), equalTo(expectedResults)); + assertThat("Unexpected candidates", knnQuery.getK(), equalTo(expectedCandidates)); + assertThat("Unexpected k parameter", knnQuery.kParam(), equalTo(expectedK)); } } } From cbae622bc9dfa204026b712f30e96e89a7771069 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 29 May 2025 15:29:34 +0200 Subject: [PATCH 07/45] add patience and saturation threshold params --- .../vectors/DenseVectorFieldMapper.java | 28 +++++++++++-------- .../vectors/DenseVectorFieldTypeTests.java | 1 - 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index fb66e02739979..f9521f974c22b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1396,7 +1396,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } if (earlyExitNode == null) { - earlyExitNode = true; + earlyExitNode = DEFAULT_EARLY_EXIT; } boolean earlyExit = XContentMapValues.nodeBooleanValue(earlyExitNode); int m = XContentMapValues.nodeIntegerValue(mNode); @@ -1707,7 +1707,7 @@ boolean updatableTo(IndexOptions update) { @Override boolean earlyExit() { - return DEFAULT_EARLY_EXIT; + return false; } } @@ -1740,7 +1740,7 @@ boolean updatableTo(IndexOptions update) { @Override boolean earlyExit() { - return DEFAULT_EARLY_EXIT; + return false; } @Override @@ -1899,7 +1899,7 @@ boolean updatableTo(IndexOptions update) { @Override boolean earlyExit() { - return DEFAULT_EARLY_EXIT; + return false; } } @@ -2077,7 +2077,7 @@ public static class BBQHnswIndexOptions extends QuantizedIndexOptions { private final boolean earlyExit; public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) { - this(m, efConstruction, rescoreVector, true); + this(m, efConstruction, rescoreVector, DEFAULT_EARLY_EXIT); } public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector, boolean earlyExit) { @@ -2429,8 +2429,10 @@ private Query createKnnBitQuery( KnnByteVectorQuery knnByteVectorQuery = parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); - // TODO: add saturation threshold and patience params ? - Query knnQuery = indexOptions.earlyExit() ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; + // TODO : fix reading of saturation threshold and patience params ? + Query knnQuery = indexOptions != null && indexOptions.earlyExit() + ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k*0.3)) + : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2459,8 +2461,10 @@ private Query createKnnByteQuery( KnnByteVectorQuery knnByteVectorQuery = parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); - // TODO: add saturation threshold and patience params ? - Query knnQuery = indexOptions.earlyExit() ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery) : knnByteVectorQuery; + // TODO: fix reading of saturation threshold and patience params ? + Query knnQuery = indexOptions != null && indexOptions.earlyExit() + ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k*0.3)) + : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( @@ -2524,8 +2528,10 @@ && isNotUnitVector(squaredMagnitude)) { knnSearchStrategy ) : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); - // TODO: add saturation threshold and patience params ? - Query knnQuery = indexOptions.earlyExit() ? PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery) : knnFloatVectorQuery; + // TODO: fix reading of saturation threshold and patience params ? + Query knnQuery = indexOptions != null && indexOptions.earlyExit() + ? PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery, 0.95, (int) (k*0.3)) + : knnFloatVectorQuery; if (rescore) { knnQuery = new RescoreKnnVectorQuery( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index b333817397925..4760487f3c8fb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper.vectors; -import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.PatienceKnnVectorQuery; import org.apache.lucene.search.Query; From a186b9d11d839e59689ba257214819504abb4daa Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 29 May 2025 15:30:10 +0200 Subject: [PATCH 08/45] spotless --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index f9521f974c22b..cfe35f754f36b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2431,7 +2431,7 @@ private Query createKnnBitQuery( : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); // TODO : fix reading of saturation threshold and patience params ? Query knnQuery = indexOptions != null && indexOptions.earlyExit() - ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k*0.3)) + ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k * 0.3)) : knnByteVectorQuery; if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( @@ -2463,7 +2463,7 @@ private Query createKnnByteQuery( : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); // TODO: fix reading of saturation threshold and patience params ? Query knnQuery = indexOptions != null && indexOptions.earlyExit() - ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k*0.3)) + ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k * 0.3)) : knnByteVectorQuery; if (similarityThreshold != null) { @@ -2530,7 +2530,7 @@ && isNotUnitVector(squaredMagnitude)) { : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); // TODO: fix reading of saturation threshold and patience params ? Query knnQuery = indexOptions != null && indexOptions.earlyExit() - ? PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery, 0.95, (int) (k*0.3)) + ? PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery, 0.95, (int) (k * 0.3)) : knnFloatVectorQuery; if (rescore) { From 56b8ff8cf3a19319a3217527f1feeab1946023bb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 11 Jun 2025 10:22:54 +0000 Subject: [PATCH 09/45] [CI] Auto commit changes from spotless --- .../mapper/vectors/DenseVectorFieldMapper.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index cdf2239c5f05e..2be800c4f6387 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2648,14 +2648,14 @@ && isNotUnitVector(squaredMagnitude)) { } else { KnnFloatVectorQuery knnFloatVectorQuery = parentFilter != null ? new ESDiversifyingChildrenFloatKnnVectorQuery( - name(), - queryVector, - filter, - adjustedK, - numCands, - parentFilter, - knnSearchStrategy - ) + name(), + queryVector, + filter, + adjustedK, + numCands, + parentFilter, + knnSearchStrategy + ) : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); // TODO: fix reading of saturation threshold and patience params ? knnQuery = indexOptions != null && indexOptions.earlyExit() From 54afb00cbe6f328936ba0f31073334106ce389dc Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 16 Jun 2025 10:43:18 +0200 Subject: [PATCH 10/45] minor fix --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index cdf2239c5f05e..05fba8228ad1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1477,7 +1477,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } if (earlyExitNode == null) { - earlyExitNode = true; + earlyExitNode = DEFAULT_EARLY_EXIT; } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); @@ -1586,7 +1586,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } if (earlyExitNode == null) { - earlyExitNode = true; + earlyExitNode = DEFAULT_EARLY_EXIT; } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); From 571e68f671005241c62e9814324bd355a1c48aad Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 18 Jun 2025 10:12:43 +0000 Subject: [PATCH 11/45] [CI] Auto commit changes from spotless --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 537ce2010ccf9..363a2f7bffbe4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1829,7 +1829,6 @@ public static class Int4HnswIndexOptions extends QuantizedIndexOptions { private final float confidenceInterval; private final boolean earlyExit; - Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { this(m, efConstruction, confidenceInterval, rescoreVector, DEFAULT_EARLY_EXIT); } From e40d17194acb5377f22a9b684f21d299a8b9da28 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 18 Jun 2025 12:17:37 +0200 Subject: [PATCH 12/45] minor fix --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 537ce2010ccf9..8b0a4040a3aec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1830,7 +1830,7 @@ public static class Int4HnswIndexOptions extends QuantizedIndexOptions { private final boolean earlyExit; - Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { + public Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { this(m, efConstruction, confidenceInterval, rescoreVector, DEFAULT_EARLY_EXIT); } From bf4e6607d63a29d8a4b4d89ffe372db1adb10ef9 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 19 Jun 2025 15:38:48 +0200 Subject: [PATCH 13/45] refactored and simplified, added option to knn tester --- .../esql/kibana/definition/functions/knn.json | 4 +- .../esql/kibana/docs/functions/knn.md | 2 +- .../elasticsearch/test/knn/CmdLineArgs.java | 14 +- .../test/knn/KnnIndexTester.java | 2 +- .../elasticsearch/test/knn/KnnSearcher.java | 23 +- .../vectors/DenseVectorFieldMapper.java | 213 ++++++++---------- .../vectors/DenseVectorFieldMapperTests.java | 1 + .../vectors/DenseVectorFieldTypeTests.java | 17 +- 8 files changed, 144 insertions(+), 132 deletions(-) diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json index 48d3e582eec58..3e6f86dd6f868 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json @@ -5,8 +5,8 @@ "description" : "Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors.", "signatures" : [ ], "examples" : [ - "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc", - "from colors metadata _score\n| where knn(rgb_vector, [0,255,255], {\"k\": 4})\n| sort _score desc" + "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc, color asc", + "from colors metadata _score\n| where knn(rgb_vector, [0,255,255], {\"k\": 4})\n| sort _score desc, color asc" ], "preview" : true, "snapshot_only" : true diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/knn.md b/docs/reference/query-languages/esql/kibana/docs/functions/knn.md index 45d1f294ea0a8..22892483700b4 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/knn.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/knn.md @@ -6,5 +6,5 @@ Finds the k nearest vectors to a query vector, as measured by a similarity metri ```esql from colors metadata _score | where knn(rgb_vector, [0, 120, 0]) -| sort _score desc +| sort _score desc, color asc ``` diff --git a/qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java b/qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java index a5f66ce5ea837..f13031f1d10c2 100644 --- a/qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java +++ b/qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java @@ -47,7 +47,8 @@ record CmdLineArgs( VectorSimilarityFunction vectorSpace, int quantizeBits, VectorEncoding vectorEncoding, - int dimensions + int dimensions, + boolean earlyTermination ) implements ToXContentObject { static final ParseField DOC_VECTORS_FIELD = new ParseField("doc_vectors"); @@ -70,6 +71,7 @@ record CmdLineArgs( static final ParseField QUANTIZE_BITS_FIELD = new ParseField("quantize_bits"); static final ParseField VECTOR_ENCODING_FIELD = new ParseField("vector_encoding"); static final ParseField DIMENSIONS_FIELD = new ParseField("dimensions"); + static final ParseField EARLY_TERMINATION_FIELD = new ParseField("early_termination"); static CmdLineArgs fromXContent(XContentParser parser) throws IOException { Builder builder = PARSER.apply(parser, null); @@ -99,6 +101,7 @@ static CmdLineArgs fromXContent(XContentParser parser) throws IOException { PARSER.declareInt(Builder::setQuantizeBits, QUANTIZE_BITS_FIELD); PARSER.declareString(Builder::setVectorEncoding, VECTOR_ENCODING_FIELD); PARSER.declareInt(Builder::setDimensions, DIMENSIONS_FIELD); + PARSER.declareBoolean(Builder::setEarlyTermination, EARLY_TERMINATION_FIELD); } @Override @@ -157,6 +160,7 @@ static class Builder { private int quantizeBits = 8; private VectorEncoding vectorEncoding = VectorEncoding.FLOAT32; private int dimensions; + private boolean earlyTermination; public Builder setDocVectors(String docVectors) { this.docVectors = PathUtils.get(docVectors); @@ -258,6 +262,11 @@ public Builder setDimensions(int dimensions) { return this; } + public Builder setEarlyTermination(Boolean patience) { + this.earlyTermination = patience; + return this; + } + public CmdLineArgs build() { if (docVectors == null) { throw new IllegalArgumentException("Document vectors path must be provided"); @@ -285,7 +294,8 @@ public CmdLineArgs build() { vectorSpace, quantizeBits, vectorEncoding, - dimensions + dimensions, + earlyTermination ); } } diff --git a/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java b/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java index dcce1fd304b06..2aa4bcbea4e19 100644 --- a/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java +++ b/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java @@ -202,7 +202,7 @@ public static void main(String[] args) throws Exception { } if (cmdLineArgs.queryVectors() != null) { KnnSearcher knnSearcher = new KnnSearcher(indexPath, cmdLineArgs); - knnSearcher.runSearch(result); + knnSearcher.runSearch(result, cmdLineArgs.earlyTermination()); } formattedResults.results.add(result); } diff --git a/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java b/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java index ec90dd46ef5b5..5776b1615a5a2 100644 --- a/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java +++ b/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java @@ -33,6 +33,9 @@ import org.apache.lucene.queries.function.valuesource.FloatKnnVectorFieldSource; import org.apache.lucene.queries.function.valuesource.FloatVectorSimilarityFunction; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnByteVectorQuery; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.PatienceKnnVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; @@ -113,7 +116,7 @@ class KnnSearcher { this.searchThreads = cmdLineArgs.searchThreads(); } - void runSearch(KnnIndexTester.Results finalResults) throws IOException { + void runSearch(KnnIndexTester.Results finalResults, boolean earlyTermination) throws IOException { TopDocs[] results = new TopDocs[numQueryVectors]; int[][] resultIds = new int[numQueryVectors][]; long elapsed, totalCpuTimeMS, totalVisited = 0; @@ -139,10 +142,10 @@ void runSearch(KnnIndexTester.Results finalResults) throws IOException { for (int i = 0; i < numQueryVectors; i++) { if (vectorEncoding.equals(VectorEncoding.BYTE)) { targetReader.next(targetBytes); - doVectorQuery(targetBytes, searcher); + doVectorQuery(targetBytes, searcher, earlyTermination); } else { targetReader.next(target); - doVectorQuery(target, searcher); + doVectorQuery(target, searcher, earlyTermination); } } targetReader.reset(); @@ -151,10 +154,10 @@ void runSearch(KnnIndexTester.Results finalResults) throws IOException { for (int i = 0; i < numQueryVectors; i++) { if (vectorEncoding.equals(VectorEncoding.BYTE)) { targetReader.next(targetBytes); - results[i] = doVectorQuery(targetBytes, searcher); + results[i] = doVectorQuery(targetBytes, searcher, earlyTermination); } else { targetReader.next(target); - results[i] = doVectorQuery(target, searcher); + results[i] = doVectorQuery(target, searcher, earlyTermination); } } KnnIndexTester.ThreadDetails endThreadDetails = new KnnIndexTester.ThreadDetails(); @@ -249,7 +252,7 @@ private boolean isNewer(Path path, Path... others) throws IOException { return true; } - TopDocs doVectorQuery(byte[] vector, IndexSearcher searcher) throws IOException { + TopDocs doVectorQuery(byte[] vector, IndexSearcher searcher, boolean earlyTermination) throws IOException { Query knnQuery; if (overSamplingFactor > 1f) { throw new IllegalArgumentException("oversampling factor > 1 is not supported for byte vectors"); @@ -265,6 +268,9 @@ TopDocs doVectorQuery(byte[] vector, IndexSearcher searcher) throws IOException null, DenseVectorFieldMapper.FilterHeuristic.ACORN.getKnnSearchStrategy() ); + if (indexType == KnnIndexTester.IndexType.HNSW && earlyTermination) { + knnQuery = PatienceKnnVectorQuery.fromByteQuery((KnnByteVectorQuery) knnQuery); + } } QueryProfiler profiler = new QueryProfiler(); TopDocs docs = searcher.search(knnQuery, this.topK); @@ -273,7 +279,7 @@ TopDocs doVectorQuery(byte[] vector, IndexSearcher searcher) throws IOException return new TopDocs(new TotalHits(profiler.getVectorOpsCount(), docs.totalHits.relation()), docs.scoreDocs); } - TopDocs doVectorQuery(float[] vector, IndexSearcher searcher) throws IOException { + TopDocs doVectorQuery(float[] vector, IndexSearcher searcher, boolean earlyTermination) throws IOException { Query knnQuery; int topK = this.topK; if (overSamplingFactor > 1f) { @@ -292,6 +298,9 @@ TopDocs doVectorQuery(float[] vector, IndexSearcher searcher) throws IOException null, DenseVectorFieldMapper.FilterHeuristic.ACORN.getKnnSearchStrategy() ); + if (indexType == KnnIndexTester.IndexType.HNSW && earlyTermination) { + knnQuery = PatienceKnnVectorQuery.fromFloatQuery((KnnFloatVectorQuery) knnQuery); + } } if (overSamplingFactor > 1f) { // oversample the topK results to get more candidates for the final result diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index bcb3957e63c4f..357ceb27ad899 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -123,10 +123,7 @@ public class DenseVectorFieldMapper extends FieldMapper { private static final float EPS = 1e-3f; public static final int BBQ_MIN_DIMS = 64; - // early termination configuration - private static final String EARLY_EXIT_PARAM_NAME = "early_exit"; - private static final boolean DEFAULT_EARLY_EXIT = false; - + private static final boolean DEFAULT_HNSW_EARLY_TERMINATION = false; public static final FeatureFlag IVF_FORMAT = new FeatureFlag("ivf_format"); public static boolean isNotUnitVector(float magnitude) { @@ -215,7 +212,7 @@ private static boolean defaultOversampleForBBQ(IndexVersion version) { public static final int MAX_DIMS_COUNT_BIT = 4096 * Byte.SIZE; // maximum allowed number of dimensions public static final short MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING = 128; // minimum number of dims for floats to be dynamically mapped to - // vector + // vector public static final int MAGNITUDE_BYTES = 4; public static final int OVERSAMPLE_LIMIT = 10_000; // Max oversample allowed public static final float DEFAULT_OVERSAMPLE = 3.0F; // Default oversample value @@ -1356,8 +1353,6 @@ public boolean validateDimension(int dim, boolean throwOnError) { return supportsDimension; } - abstract boolean earlyExit(); - abstract boolean doEquals(DenseVectorIndexOptions other); abstract int doHashCode(); @@ -1399,21 +1394,21 @@ public enum VectorIndexType { public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - Object earlyExitNode = indexOptionsMap.remove(EARLY_EXIT_PARAM_NAME); + Object earlyTerminationObject = indexOptionsMap.remove("early_termination"); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } - if (earlyExitNode == null) { - earlyExitNode = DEFAULT_EARLY_EXIT; + if (earlyTerminationObject == null) { + earlyTerminationObject = DEFAULT_HNSW_EARLY_TERMINATION; } - boolean earlyExit = XContentMapValues.nodeBooleanValue(earlyExitNode); + boolean earlyTermination = XContentMapValues.nodeBooleanValue(earlyTerminationObject); int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new HnswIndexOptions(m, efConstruction, earlyExit); + return new HnswIndexOptions(m, efConstruction, earlyTermination); } @Override @@ -1432,15 +1427,15 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - Object earlyExitNode = indexOptionsMap.remove(EARLY_EXIT_PARAM_NAME); + Object earlyTerminationNode = indexOptionsMap.remove("early_termination"); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } - if (earlyExitNode == null) { - earlyExitNode = DEFAULT_EARLY_EXIT; + if (earlyTerminationNode == null) { + earlyTerminationNode = DEFAULT_HNSW_EARLY_TERMINATION; } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); @@ -1600,9 +1595,9 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map= m); } - @Override - boolean earlyExit() { - return earlyExit; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("type", type); builder.field("m", m); builder.field("ef_construction", efConstruction); - builder.field(EARLY_EXIT_PARAM_NAME, earlyExit); + builder.field("early_termination", earlyTermination); builder.endObject(); return builder; } @@ -2132,34 +2104,35 @@ public boolean doEquals(DenseVectorIndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; HnswIndexOptions that = (HnswIndexOptions) o; - return m == that.m && efConstruction == that.efConstruction; + return m == that.m && efConstruction == that.efConstruction && earlyTermination == that.earlyTermination; } @Override public int doHashCode() { - return Objects.hash(m, efConstruction); + return Objects.hash(m, efConstruction, earlyTermination); } @Override public String toString() { - return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + "}"; + return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + ", early_termination=" + + earlyTermination + "}"; } } public static class BBQHnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; - private final boolean earlyExit; + private final boolean earlyTermination; public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) { - this(m, efConstruction, rescoreVector, DEFAULT_EARLY_EXIT); + this(m, efConstruction, rescoreVector, DEFAULT_HNSW_EARLY_TERMINATION); } - public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector, boolean earlyExit) { + public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector, boolean earlyTermination) { super(VectorIndexType.BBQ_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; - this.earlyExit = earlyExit; + this.earlyTermination = earlyTermination; } @Override @@ -2176,12 +2149,12 @@ public boolean updatableTo(DenseVectorIndexOptions update) { @Override boolean doEquals(DenseVectorIndexOptions other) { BBQHnswIndexOptions that = (BBQHnswIndexOptions) other; - return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector); + return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector) && earlyTermination == that.earlyTermination; } @Override int doHashCode() { - return Objects.hash(m, efConstruction, rescoreVector); + return Objects.hash(m, efConstruction, rescoreVector, earlyTermination); } @Override @@ -2193,7 +2166,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (rescoreVector != null) { rescoreVector.toXContent(builder, params); } - builder.field(EARLY_EXIT_PARAM_NAME, earlyExit); + builder.field("early_termination", earlyTermination); builder.endObject(); return builder; } @@ -2208,11 +2181,6 @@ public boolean validateDimension(int dim, boolean throwOnError) { } return supportsDimension; } - - @Override - boolean earlyExit() { - return earlyExit; - } } static class BBQFlatIndexOptions extends QuantizedIndexOptions { @@ -2265,11 +2233,6 @@ public boolean validateDimension(int dim, boolean throwOnError) { return supportsDimension; } - @Override - boolean earlyExit() { - return false; - } - } static class BBQIVFIndexOptions extends QuantizedIndexOptions { @@ -2294,10 +2257,6 @@ public boolean updatableTo(DenseVectorIndexOptions update) { } @Override - boolean earlyExit() { - return false; - } - boolean doEquals(DenseVectorIndexOptions other) { BBQIVFIndexOptions that = (BBQIVFIndexOptions) other; return clusterSize == that.clusterSize @@ -2558,13 +2517,10 @@ private Query createKnnBitQuery( KnnSearchStrategy searchStrategy ) { elementType.checkDimensions(dims, queryVector.length); - KnnByteVectorQuery knnByteVectorQuery = parentFilter != null + Query knnQuery = parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); - // TODO : fix reading of saturation threshold and patience params ? - Query knnQuery = indexOptions != null && indexOptions.earlyExit() - ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k * 0.3)) - : knnByteVectorQuery; + knnQuery = maybeWrapPatience(knnQuery); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2590,14 +2546,10 @@ private Query createKnnByteQuery( float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorByteElementsAppender(queryVector), squaredMagnitude); } - KnnByteVectorQuery knnByteVectorQuery = parentFilter != null + Query knnQuery = parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); - // TODO: fix reading of saturation threshold and patience params ? - Query knnQuery = indexOptions != null && indexOptions.earlyExit() - ? PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery, 0.95, (int) (k * 0.3)) - : knnByteVectorQuery; - + knnQuery = maybeWrapPatience(knnQuery); if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2608,6 +2560,44 @@ private Query createKnnByteQuery( return knnQuery; } + private Query maybeWrapPatience(Query knnQuery) { + Query finalQuery = knnQuery; + if (knnQuery instanceof KnnByteVectorQuery knnByteVectorQuery) { + finalQuery = maybeWrapPatienceByte(knnByteVectorQuery, Math.max(7, (int) (knnByteVectorQuery.getK() * 0.3))); + } else if (knnQuery instanceof KnnFloatVectorQuery knnFloatVectorQuery) { + finalQuery = maybeWrapPatienceFloat(knnFloatVectorQuery, Math.max(7, (int) (knnFloatVectorQuery.getK() * 0.3))); + } + return finalQuery; + } + + private Query maybeWrapPatienceByte(KnnByteVectorQuery knnQuery, int patience) { + Query returnedQuery = knnQuery; + if (indexOptions instanceof HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + } else if (indexOptions instanceof Int8HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + } else if (indexOptions instanceof Int4HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + } else if (indexOptions instanceof BBQHnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + } + return returnedQuery; + } + + private Query maybeWrapPatienceFloat(KnnFloatVectorQuery knnQuery, int patience) { + Query returnedQuery = knnQuery; + if (indexOptions instanceof HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); + } else if (indexOptions instanceof Int8HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); + } else if (indexOptions instanceof Int4HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); + } else if (indexOptions instanceof BBQHnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); + } + return returnedQuery; + } + private Query createKnnFloatQuery( float[] queryVector, int k, @@ -2663,7 +2653,7 @@ && isNotUnitVector(squaredMagnitude)) { ) : new IVFKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, bbqIndexOptions.defaultNProbe); } else { - KnnFloatVectorQuery knnFloatVectorQuery = parentFilter != null + knnQuery = parentFilter != null ? new ESDiversifyingChildrenFloatKnnVectorQuery( name(), queryVector, @@ -2674,10 +2664,7 @@ && isNotUnitVector(squaredMagnitude)) { knnSearchStrategy ) : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); - // TODO: fix reading of saturation threshold and patience params ? - knnQuery = indexOptions != null && indexOptions.earlyExit() - ? PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery, 0.95, (int) (k * 0.3)) - : knnFloatVectorQuery; + knnQuery = maybeWrapPatience(knnQuery); } if (rescore) { knnQuery = new RescoreKnnVectorQuery( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 149c908fd380b..6c69218324d96 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -1647,6 +1647,7 @@ public void testMergeDims() throws IOException { .field("type", "int8_hnsw") .field("m", 16) .field("ef_construction", 100) + .field("early_termination", false) .endObject(); b.endObject(); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 76468319ccb2a..510969252998c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -122,7 +122,12 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQua rescoreVector, randomBoolean() ), - new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector) + new DenseVectorFieldMapper.BBQHnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + rescoreVector, + randomBoolean() + ) ); } @@ -641,7 +646,7 @@ public void testRescoreOversampleQueryOverrides() { null, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertTrue(query instanceof ESKnnFloatVectorQuery); + assertTrue(query instanceof ESKnnFloatVectorQuery || query instanceof PatienceKnnVectorQuery); // verify we can override a `0` to a positive number fieldType = new DenseVectorFieldType( @@ -666,12 +671,12 @@ public void testRescoreOversampleQueryOverrides() { randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); assertTrue(query instanceof RescoreKnnVectorQuery); - assertThat(((RescoreKnnVectorQuery) query).k(), equalTo(10)); - KnnFloatVectorQuery esKnnQuery = (KnnFloatVectorQuery) ((RescoreKnnVectorQuery) query).innerQuery(); - if (esKnnQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { + RescoreKnnVectorQuery rescoreKnnVectorQuery = (RescoreKnnVectorQuery) query; + assertThat(rescoreKnnVectorQuery.k(), equalTo(10)); + Query innerQuery = rescoreKnnVectorQuery.innerQuery(); + if (innerQuery instanceof ESKnnFloatVectorQuery esKnnFloatVectorQuery) { assertThat(esKnnFloatVectorQuery.kParam(), equalTo(20)); } - } public void testFilterSearchThreshold() { From c7bcb82c134be2a983f6dd878c723a6c87391767 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 19 Jun 2025 15:48:14 +0200 Subject: [PATCH 14/45] reverted docs changes --- .../query-languages/esql/kibana/definition/functions/knn.json | 4 ++-- .../query-languages/esql/kibana/docs/functions/knn.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json index 3e6f86dd6f868..48d3e582eec58 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json @@ -5,8 +5,8 @@ "description" : "Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors.", "signatures" : [ ], "examples" : [ - "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc, color asc", - "from colors metadata _score\n| where knn(rgb_vector, [0,255,255], {\"k\": 4})\n| sort _score desc, color asc" + "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc", + "from colors metadata _score\n| where knn(rgb_vector, [0,255,255], {\"k\": 4})\n| sort _score desc" ], "preview" : true, "snapshot_only" : true diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/knn.md b/docs/reference/query-languages/esql/kibana/docs/functions/knn.md index 22892483700b4..45d1f294ea0a8 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/knn.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/knn.md @@ -6,5 +6,5 @@ Finds the k nearest vectors to a query vector, as measured by a similarity metri ```esql from colors metadata _score | where knn(rgb_vector, [0, 120, 0]) -| sort _score desc, color asc +| sort _score desc ``` From f5fbbb45f131bfbc452499cef11fb88123edae83 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 19 Jun 2025 13:57:26 +0000 Subject: [PATCH 15/45] [CI] Auto commit changes from spotless --- .../vectors/DenseVectorFieldMapper.java | 30 ++++++++++++++----- .../vectors/DenseVectorFieldTypeTests.java | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 357ceb27ad899..0fde5cbfc725e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1819,7 +1819,13 @@ public Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, this(m, efConstruction, confidenceInterval, rescoreVector, DEFAULT_HNSW_EARLY_TERMINATION); } - public Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector, boolean earlyTermination) { + public Int4HnswIndexOptions( + int m, + int efConstruction, + Float confidenceInterval, + RescoreVector rescoreVector, + boolean earlyTermination + ) { super(VectorIndexType.INT4_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; @@ -1877,7 +1883,7 @@ public String toString() { + confidenceInterval + ", rescore_vector=" + (rescoreVector == null ? "none" : rescoreVector) - + ", early_termination=" + + ", early_termination=" + earlyTermination + "}"; } @@ -1966,7 +1972,13 @@ public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, this(m, efConstruction, confidenceInterval, rescoreVector, DEFAULT_HNSW_EARLY_TERMINATION); } - public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector, boolean earlyTermination) { + public Int8HnswIndexOptions( + int m, + int efConstruction, + Float confidenceInterval, + RescoreVector rescoreVector, + boolean earlyTermination + ) { super(VectorIndexType.INT8_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; @@ -2026,7 +2038,7 @@ public String toString() { + confidenceInterval + ", rescore_vector=" + (rescoreVector == null ? "none" : rescoreVector) - + ", early_termination=" + + ", early_termination=" + earlyTermination + "}"; } @@ -2104,7 +2116,7 @@ public boolean doEquals(DenseVectorIndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; HnswIndexOptions that = (HnswIndexOptions) o; - return m == that.m && efConstruction == that.efConstruction && earlyTermination == that.earlyTermination; + return m == that.m && efConstruction == that.efConstruction && earlyTermination == that.earlyTermination; } @Override @@ -2114,8 +2126,7 @@ public int doHashCode() { @Override public String toString() { - return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + ", early_termination=" - + earlyTermination + "}"; + return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + ", early_termination=" + earlyTermination + "}"; } } @@ -2149,7 +2160,10 @@ public boolean updatableTo(DenseVectorIndexOptions update) { @Override boolean doEquals(DenseVectorIndexOptions other) { BBQHnswIndexOptions that = (BBQHnswIndexOptions) other; - return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector) && earlyTermination == that.earlyTermination; + return m == that.m + && efConstruction == that.efConstruction + && Objects.equals(rescoreVector, that.rescoreVector) + && earlyTermination == that.earlyTermination; } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 510969252998c..888549e028d99 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -646,7 +646,7 @@ public void testRescoreOversampleQueryOverrides() { null, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertTrue(query instanceof ESKnnFloatVectorQuery || query instanceof PatienceKnnVectorQuery); + assertTrue(query instanceof ESKnnFloatVectorQuery || query instanceof PatienceKnnVectorQuery); // verify we can override a `0` to a positive number fieldType = new DenseVectorFieldType( From ee047a8d4e0880b30af20418a61744e86de52752 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 19 Jun 2025 17:13:31 +0200 Subject: [PATCH 16/45] adjusted knnsearcher --- .../java/org/elasticsearch/test/knn/KnnSearcher.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java b/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java index 5776b1615a5a2..0f5ab9a8a2ca5 100644 --- a/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java +++ b/qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java @@ -308,9 +308,12 @@ TopDocs doVectorQuery(float[] vector, IndexSearcher searcher, boolean earlyTermi } QueryProfiler profiler = new QueryProfiler(); TopDocs docs = searcher.search(knnQuery, this.topK); - QueryProfilerProvider queryProfilerProvider = (QueryProfilerProvider) knnQuery; - queryProfilerProvider.profile(profiler); - return new TopDocs(new TotalHits(profiler.getVectorOpsCount(), docs.totalHits.relation()), docs.scoreDocs); + if (knnQuery instanceof QueryProfilerProvider queryProfilerProvider) { + queryProfilerProvider.profile(profiler); + return new TopDocs(new TotalHits(profiler.getVectorOpsCount(), docs.totalHits.relation()), docs.scoreDocs); + } else { + return docs; + } } private static float checkResults(int[][] results, int[][] nn, int topK) { From 19df568587c4148e1f0825aab9b1b4dcf3dcd9e5 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 20 Jun 2025 10:11:53 +0200 Subject: [PATCH 17/45] default index settings test --- .../80_dense_vector_indexed_by_default.yml | 4 ++++ .../index/mapper/vectors/DenseVectorFieldTypeTests.java | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 0238a1781d278..60dc335450711 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -61,6 +61,7 @@ setup: type: hnsw m: 32 ef_construction: 200 + early_termination: false --- "Not indexed vector": @@ -89,6 +90,7 @@ setup: type: dense_vector dims: 5 index: false + early_termination: false --- "Not indexed vector - vector similarity": - do: @@ -147,6 +149,7 @@ setup: - match: { test_default_index_options.mappings.properties.vector.index: true } - match: { test_default_index_options.mappings.properties.vector.similarity: cosine } - match: { test_default_index_options.mappings.properties.vector.index_options.type: int8_hnsw } + - match: { test_default_index_options.mappings.properties.vector.index_options.early_termination: false } --- "Default index options for dense_vector element type byte": - requires: @@ -174,3 +177,4 @@ setup: - match: { test_default_index_options.mappings.properties.vector.index: true } - match: { test_default_index_options.mappings.properties.vector.similarity: cosine } - is_false: test_default_index_options.mappings.properties.vector.index_options.type + - match: { test_default_index_options.mappings.properties.vector.index_options.early_termination: false } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 888549e028d99..043678ea9d63d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -681,9 +681,12 @@ public void testRescoreOversampleQueryOverrides() { public void testFilterSearchThreshold() { List>> cases = List.of( - Tuple.tuple(FLOAT, q -> ((ESKnnFloatVectorQuery) q).getStrategy()), - Tuple.tuple(BYTE, q -> ((ESKnnByteVectorQuery) q).getStrategy()), - Tuple.tuple(BIT, q -> ((ESKnnByteVectorQuery) q).getStrategy()) + Tuple.tuple(FLOAT, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? + patienceKnnVectorQuery.getSearchStrategy() : ((ESKnnFloatVectorQuery) q).getStrategy()), + Tuple.tuple(BYTE, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? + patienceKnnVectorQuery.getSearchStrategy() : ((ESKnnByteVectorQuery) q).getStrategy()), + Tuple.tuple(BIT, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? + patienceKnnVectorQuery.getSearchStrategy() : ((ESKnnByteVectorQuery) q).getStrategy()) ); for (var tuple : cases) { DenseVectorFieldType fieldType = new DenseVectorFieldType( From dbdbc6d591fccf0b48a6ad128dc65937ae6be2e4 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 20 Jun 2025 12:21:46 +0200 Subject: [PATCH 18/45] test fixes --- .../80_dense_vector_indexed_by_default.yml | 2 - .../vectors/DenseVectorFieldTypeTests.java | 40 ++++++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 60dc335450711..8c3682683b0cc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -90,7 +90,6 @@ setup: type: dense_vector dims: 5 index: false - early_termination: false --- "Not indexed vector - vector similarity": - do: @@ -177,4 +176,3 @@ setup: - match: { test_default_index_options.mappings.properties.vector.index: true } - match: { test_default_index_options.mappings.properties.vector.similarity: cosine } - is_false: test_default_index_options.mappings.properties.vector.index_options.type - - match: { test_default_index_options.mappings.properties.vector.index_options.early_termination: false } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 043678ea9d63d..8ec5e667f72c6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -682,11 +682,11 @@ public void testRescoreOversampleQueryOverrides() { public void testFilterSearchThreshold() { List>> cases = List.of( Tuple.tuple(FLOAT, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? - patienceKnnVectorQuery.getSearchStrategy() : ((ESKnnFloatVectorQuery) q).getStrategy()), + null : ((ESKnnFloatVectorQuery) q).getStrategy()), Tuple.tuple(BYTE, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? - patienceKnnVectorQuery.getSearchStrategy() : ((ESKnnByteVectorQuery) q).getStrategy()), + null : ((ESKnnByteVectorQuery) q).getStrategy()), Tuple.tuple(BIT, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? - patienceKnnVectorQuery.getSearchStrategy() : ((ESKnnByteVectorQuery) q).getStrategy()) + null : ((ESKnnByteVectorQuery) q).getStrategy()) ); for (var tuple : cases) { DenseVectorFieldType fieldType = new DenseVectorFieldType( @@ -713,22 +713,24 @@ public void testFilterSearchThreshold() { DenseVectorFieldMapper.FilterHeuristic.FANOUT ); KnnSearchStrategy strategy = tuple.v2().apply(query); - assertTrue(strategy instanceof KnnSearchStrategy.Hnsw); - assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(0)); - - query = fieldType.createKnnQuery( - VectorData.fromFloats(new float[] { 1, 4, 10 }), - 10, - 100, - 0f, - null, - null, - null, - DenseVectorFieldMapper.FilterHeuristic.ACORN - ); - strategy = tuple.v2().apply(query); - assertTrue(strategy instanceof KnnSearchStrategy.Hnsw); - assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(60)); + if (strategy != null) { + assertTrue(strategy instanceof KnnSearchStrategy.Hnsw); + assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(0)); + + query = fieldType.createKnnQuery( + VectorData.fromFloats(new float[]{1, 4, 10}), + 10, + 100, + 0f, + null, + null, + null, + DenseVectorFieldMapper.FilterHeuristic.ACORN + ); + strategy = tuple.v2().apply(query); + assertTrue(strategy instanceof KnnSearchStrategy.Hnsw); + assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(60)); + } } } From 4df4be2b2d270529cfdf0e15284e784fa98ac94c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 20 Jun 2025 10:30:54 +0000 Subject: [PATCH 19/45] [CI] Auto commit changes from spotless --- .../vectors/DenseVectorFieldTypeTests.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 8ec5e667f72c6..f9aaa9ba17529 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -681,12 +681,18 @@ public void testRescoreOversampleQueryOverrides() { public void testFilterSearchThreshold() { List>> cases = List.of( - Tuple.tuple(FLOAT, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? - null : ((ESKnnFloatVectorQuery) q).getStrategy()), - Tuple.tuple(BYTE, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? - null : ((ESKnnByteVectorQuery) q).getStrategy()), - Tuple.tuple(BIT, q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? - null : ((ESKnnByteVectorQuery) q).getStrategy()) + Tuple.tuple( + FLOAT, + q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? null : ((ESKnnFloatVectorQuery) q).getStrategy() + ), + Tuple.tuple( + BYTE, + q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? null : ((ESKnnByteVectorQuery) q).getStrategy() + ), + Tuple.tuple( + BIT, + q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? null : ((ESKnnByteVectorQuery) q).getStrategy() + ) ); for (var tuple : cases) { DenseVectorFieldType fieldType = new DenseVectorFieldType( @@ -718,7 +724,7 @@ public void testFilterSearchThreshold() { assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(0)); query = fieldType.createKnnQuery( - VectorData.fromFloats(new float[]{1, 4, 10}), + VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, 0f, From b5c52daf9b06c72ea17e4288552358fd039e75cb Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 20 Jun 2025 12:38:28 +0200 Subject: [PATCH 20/45] Merge --- .../vectors/DenseVectorFieldTypeTests.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index f9aaa9ba17529..0f5701a736e57 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -681,18 +681,12 @@ public void testRescoreOversampleQueryOverrides() { public void testFilterSearchThreshold() { List>> cases = List.of( - Tuple.tuple( - FLOAT, - q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? null : ((ESKnnFloatVectorQuery) q).getStrategy() - ), - Tuple.tuple( - BYTE, - q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? null : ((ESKnnByteVectorQuery) q).getStrategy() - ), - Tuple.tuple( - BIT, - q -> q instanceof PatienceKnnVectorQuery patienceKnnVectorQuery ? null : ((ESKnnByteVectorQuery) q).getStrategy() - ) + Tuple.tuple(FLOAT, q -> q instanceof PatienceKnnVectorQuery ? + null : ((ESKnnFloatVectorQuery) q).getStrategy()), + Tuple.tuple(BYTE, q -> q instanceof PatienceKnnVectorQuery ? + null : ((ESKnnByteVectorQuery) q).getStrategy()), + Tuple.tuple(BIT, q -> q instanceof PatienceKnnVectorQuery ? + null : ((ESKnnByteVectorQuery) q).getStrategy()) ); for (var tuple : cases) { DenseVectorFieldType fieldType = new DenseVectorFieldType( From ac4c879c31d3d17739b811e6b9f78c0bf2520928 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 20 Jun 2025 10:46:09 +0000 Subject: [PATCH 21/45] [CI] Auto commit changes from spotless --- .../index/mapper/vectors/DenseVectorFieldTypeTests.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 0f5701a736e57..1f24535240849 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -681,12 +681,9 @@ public void testRescoreOversampleQueryOverrides() { public void testFilterSearchThreshold() { List>> cases = List.of( - Tuple.tuple(FLOAT, q -> q instanceof PatienceKnnVectorQuery ? - null : ((ESKnnFloatVectorQuery) q).getStrategy()), - Tuple.tuple(BYTE, q -> q instanceof PatienceKnnVectorQuery ? - null : ((ESKnnByteVectorQuery) q).getStrategy()), - Tuple.tuple(BIT, q -> q instanceof PatienceKnnVectorQuery ? - null : ((ESKnnByteVectorQuery) q).getStrategy()) + Tuple.tuple(FLOAT, q -> q instanceof PatienceKnnVectorQuery ? null : ((ESKnnFloatVectorQuery) q).getStrategy()), + Tuple.tuple(BYTE, q -> q instanceof PatienceKnnVectorQuery ? null : ((ESKnnByteVectorQuery) q).getStrategy()), + Tuple.tuple(BIT, q -> q instanceof PatienceKnnVectorQuery ? null : ((ESKnnByteVectorQuery) q).getStrategy()) ); for (var tuple : cases) { DenseVectorFieldType fieldType = new DenseVectorFieldType( From 8967a5c9db0798b3c4f8550072ccd5e1c2bd79bf Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 24 Jun 2025 15:34:17 +0200 Subject: [PATCH 22/45] test fix --- .../index/mapper/vectors/DenseVectorFieldTypeTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 1f24535240849..5994a99461f92 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -248,7 +248,7 @@ public void testCreateNestedKnnQuery() { if (query instanceof RescoreKnnVectorQuery rescoreKnnVectorQuery) { query = rescoreKnnVectorQuery.innerQuery(); } - assertThat(query, instanceOf(DiversifyingChildrenFloatKnnVectorQuery.class)); + assertTrue(query instanceof DiversifyingChildrenFloatKnnVectorQuery || query instanceof PatienceKnnVectorQuery); } { DenseVectorFieldType field = new DenseVectorFieldType( @@ -279,7 +279,7 @@ public void testCreateNestedKnnQuery() { producer, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertThat(query, instanceOf(DiversifyingChildrenByteKnnVectorQuery.class)); + assertTrue(query instanceof DiversifyingChildrenByteKnnVectorQuery || query instanceof PatienceKnnVectorQuery); vectorData = new VectorData(floatQueryVector, null); query = field.createKnnQuery( From fa06fbd52382042f329b44d95e1cf88669214033 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 24 Jun 2025 15:59:39 +0200 Subject: [PATCH 23/45] test fix --- .../test/search.vectors/80_dense_vector_indexed_by_default.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 8c3682683b0cc..f46bd9214b0ad 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -61,7 +61,6 @@ setup: type: hnsw m: 32 ef_construction: 200 - early_termination: false --- "Not indexed vector": From 6b7060581bdd211c0abb8256594da44815d4ef88 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 24 Jun 2025 16:24:26 +0200 Subject: [PATCH 24/45] test fix --- .../80_dense_vector_indexed_by_default.yml | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index f46bd9214b0ad..d82d67554e6ea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -48,19 +48,13 @@ setup: indices.get_mapping: index: test - - match: - test: - mappings: - properties: - vector: - type: dense_vector - dims: 5 - index: true - similarity: dot_product - index_options: - type: hnsw - m: 32 - ef_construction: 200 + - match: { test.mappings.properties.vector.type: dense_vector } + - match: { test.mappings.properties.vector.dims: 5 } + - match: { test.mappings.properties.vector.index: true } + - match: { test.mappings.properties.vector.similarity: dot_product } + - match: { test.mappings.properties.vector.index_options.type: hnsw } + - match: { test.mappings.properties.vector.index_options.m: 32 } + - match: { test.mappings.properties.vector.index_options.ef_construction: 200 } --- "Not indexed vector": From 483960fa777ee3c5cac41a9e77fb1e605361d6a9 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 25 Jun 2025 16:24:33 +0200 Subject: [PATCH 25/45] rest compatibility check with search features --- .../80_dense_vector_indexed_by_default.yml | 36 +++++++++++++++++++ .../elasticsearch/search/SearchFeatures.java | 4 ++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index d82d67554e6ea..09ec55aeef84a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -25,8 +25,43 @@ setup: - match: { test.mappings.properties.vector.dims: 5 } - match: { test.mappings.properties.vector.index: true } - match: { test.mappings.properties.vector.similarity: cosine } + +--- +"Indexed by default with specified similarity and index options": + - do: + indices.create: + index: test + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + similarity: dot_product + index_options: + type: hnsw + m: 32 + ef_construction: 200 + + - match: { acknowledged: true } + + - do: + indices.get_mapping: + index: test + + - match: { test.mappings.properties.vector.type: dense_vector } + - match: { test.mappings.properties.vector.dims: 5 } + - match: { test.mappings.properties.vector.index: true } + - match: { test.mappings.properties.vector.similarity: dot_product } + - match: { test.mappings.properties.vector.index_options.type: hnsw } + - match: { test.mappings.properties.vector.index_options.m: 32 } + - match: { test.mappings.properties.vector.index_options.ef_construction: 200 } + - match: { test.mappings.properties.vector.index_options.early_termination: null } + --- "Indexed by default with specified similarity and index options": + - cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] + - reason: "requires early termination for hnsw to be exposed" - do: indices.create: index: test @@ -55,6 +90,7 @@ setup: - match: { test.mappings.properties.vector.index_options.type: hnsw } - match: { test.mappings.properties.vector.index_options.m: 32 } - match: { test.mappings.properties.vector.index_options.ef_construction: 200 } + - match: { test.mappings.properties.vector.index_options.early_termination: false } --- "Not indexed vector": diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java index 0c2f7c2aa625b..2bee1e3b2d67f 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java +++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java @@ -32,6 +32,7 @@ public Set getFeatures() { public static final NodeFeature INT_SORT_FOR_INT_SHORT_BYTE_FIELDS = new NodeFeature("search.sort.int_sort_for_int_short_byte_fields"); static final NodeFeature MULTI_MATCH_CHECKS_POSITIONS = new NodeFeature("search.multi.match.checks.positions"); public static final NodeFeature BBQ_HNSW_DEFAULT_INDEXING = new NodeFeature("search.vectors.mappers.default_bbq_hnsw"); + public static final NodeFeature EXPOSE_HNSW_EARLY_TERMINATION = new NodeFeature("search.vectors.mappers.expose_hnsw_early_termination"); @Override public Set getTestFeatures() { @@ -41,7 +42,8 @@ public Set getTestFeatures() { RESCORER_MISSING_FIELD_BAD_REQUEST, INT_SORT_FOR_INT_SHORT_BYTE_FIELDS, MULTI_MATCH_CHECKS_POSITIONS, - BBQ_HNSW_DEFAULT_INDEXING + BBQ_HNSW_DEFAULT_INDEXING, + EXPOSE_HNSW_EARLY_TERMINATION ); } } From 4446bfa41302e6d40485e7984f0b427b1863fe95 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 25 Jun 2025 16:49:22 +0200 Subject: [PATCH 26/45] rest compatibility check with search features --- .../test/search.vectors/80_dense_vector_indexed_by_default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 09ec55aeef84a..1d1444f656dd6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -59,7 +59,7 @@ setup: - match: { test.mappings.properties.vector.index_options.early_termination: null } --- -"Indexed by default with specified similarity and index options": +"Indexed by default with specified similarity and index options with early_termination": - cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] - reason: "requires early termination for hnsw to be exposed" - do: From e8fcd520fae29a76c77e425442f1813b7ba306a2 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 25 Jun 2025 17:00:18 +0200 Subject: [PATCH 27/45] rest compatibility check with search features --- .../search.vectors/80_dense_vector_indexed_by_default.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 1d1444f656dd6..e4f4b69445930 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -60,8 +60,9 @@ setup: --- "Indexed by default with specified similarity and index options with early_termination": - - cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] - - reason: "requires early termination for hnsw to be exposed" + - requires: + cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] + reason: "requires early termination for hnsw to be exposed" - do: indices.create: index: test From 89b5273dc23a0bdcefb12ad226f32eb4555f8971 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 25 Jun 2025 17:22:05 +0200 Subject: [PATCH 28/45] rest compatibility check with search features --- .../test/search.vectors/80_dense_vector_indexed_by_default.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index e4f4b69445930..7433ae6458119 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -56,7 +56,6 @@ setup: - match: { test.mappings.properties.vector.index_options.type: hnsw } - match: { test.mappings.properties.vector.index_options.m: 32 } - match: { test.mappings.properties.vector.index_options.ef_construction: 200 } - - match: { test.mappings.properties.vector.index_options.early_termination: null } --- "Indexed by default with specified similarity and index options with early_termination": From 631b73884b54661bb2c11ce9b7f62e9c0594f321 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 26 Jun 2025 14:15:47 +0200 Subject: [PATCH 29/45] index version check --- .../elasticsearch/index/IndexVersions.java | 1 + .../vectors/DenseVectorFieldMapper.java | 168 ++++++++++++------ .../vectors/DenseVectorFieldTypeTests.java | 22 ++- .../mapper/SemanticTextFieldMapper.java | 2 +- .../mapper/SemanticTextFieldMapperTests.java | 10 +- 5 files changed, 139 insertions(+), 64 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 6ff33cf05d51f..e2f17a06ae640 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -178,6 +178,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_10_2_2 = def(9_030_0_00, Version.LUCENE_10_2_2); public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2); public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2); + public static final IndexVersion EXPOSE_EARLY_TERMINATION = def(9_033_0_00, Version.LUCENE_10_2_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 6780a90e4dd21..f55f09699972d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -199,6 +199,7 @@ private static boolean defaultOversampleForBBQ(IndexVersion version) { public static final IndexVersion DEFAULT_TO_INT8 = IndexVersions.DEFAULT_DENSE_VECTOR_TO_INT8_HNSW; public static final IndexVersion DEFAULT_TO_BBQ = IndexVersions.DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW; public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersions.V_8_9_0; + public static final IndexVersion EXPOSE_EARLY_TERMINATION = IndexVersions.EXPOSE_EARLY_TERMINATION; public static final NodeFeature RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature("mapper.dense_vector.rescore_vector"); public static final NodeFeature RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature( @@ -306,7 +307,7 @@ public Builder(String name, IndexVersion indexVersionCreated) { this.indexOptions = new Parameter<>( "index_options", true, - () -> defaultIndexOptions(defaultInt8Hnsw, defaultBBQ8Hnsw), + () -> defaultIndexOptions(defaultInt8Hnsw, defaultBBQ8Hnsw, indexVersionCreated), (n, c, o) -> o == null ? null : parseIndexOptions(n, o, indexVersionCreated), m -> toType(m).indexOptions, (b, n, v) -> { @@ -353,20 +354,26 @@ public Builder(String name, IndexVersion indexVersionCreated) { }); } - private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boolean defaultBBQHnsw) { + private DenseVectorIndexOptions defaultIndexOptions( + boolean defaultInt8Hnsw, + boolean defaultBBQHnsw, + IndexVersion indexVersionCreated + ) { if (this.dims != null && this.dims.isConfigured() && elementType.getValue() == ElementType.FLOAT && this.indexed.getValue()) { if (defaultBBQHnsw && this.dims.getValue() >= BBQ_DIMS_DEFAULT_THRESHOLD) { return new BBQHnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, - new RescoreVector(DEFAULT_OVERSAMPLE) + new RescoreVector(DEFAULT_OVERSAMPLE), + indexVersionCreated ); } else if (defaultInt8Hnsw) { return new Int8HnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, null, - null + null, + indexVersionCreated ); } } @@ -1332,9 +1339,15 @@ public final String toString() { public abstract static class DenseVectorIndexOptions implements IndexOptions { final VectorIndexType type; + final IndexVersion indexVersion; DenseVectorIndexOptions(VectorIndexType type) { + this(type, null); + } + + DenseVectorIndexOptions(VectorIndexType type, IndexVersion indexVersion) { this.type = type; + this.indexVersion = indexVersion; } abstract KnnVectorsFormat getVectorsFormat(ElementType elementType); @@ -1404,6 +1417,11 @@ abstract static class QuantizedIndexOptions extends DenseVectorIndexOptions { super(type); this.rescoreVector = rescoreVector; } + + QuantizedIndexOptions(VectorIndexType type, RescoreVector rescoreVector, IndexVersion indexVersion) { + super(type, indexVersion); + this.rescoreVector = rescoreVector; + } } public enum VectorIndexType { @@ -1412,21 +1430,27 @@ public enum VectorIndexType { public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - Object earlyTerminationObject = indexOptionsMap.remove("early_termination"); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } - if (earlyTerminationObject == null) { - earlyTerminationObject = DEFAULT_HNSW_EARLY_TERMINATION; + + boolean earlyTermination; + if (indexVersion != null && indexVersion.onOrAfter(EXPOSE_EARLY_TERMINATION)) { + Object earlyTerminationObject = indexOptionsMap.remove("early_termination"); + if (earlyTerminationObject == null) { + earlyTerminationObject = DEFAULT_HNSW_EARLY_TERMINATION; + } + earlyTermination = XContentMapValues.nodeBooleanValue(earlyTerminationObject); + } else { + earlyTermination = false; } - boolean earlyTermination = XContentMapValues.nodeBooleanValue(earlyTerminationObject); int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new HnswIndexOptions(m, efConstruction, earlyTermination); + return new HnswIndexOptions(m, efConstruction, earlyTermination, indexVersion); } @Override @@ -1445,16 +1469,12 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - Object earlyTerminationNode = indexOptionsMap.remove("early_termination"); if (mNode == null) { mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; } if (efConstructionNode == null) { efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } - if (earlyTerminationNode == null) { - earlyTerminationNode = DEFAULT_HNSW_EARLY_TERMINATION; - } int m = XContentMapValues.nodeIntegerValue(mNode); int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); RescoreVector rescoreVector = null; @@ -1613,9 +1643,18 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map Date: Thu, 26 Jun 2025 14:50:08 +0200 Subject: [PATCH 30/45] fixed testMeta --- .../elasticsearch/mapping-reference/dense-vector.md | 3 +++ .../index/mapper/vectors/DenseVectorFieldMapperTests.java | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index 7f3a701bde3f8..420b37596b2c7 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -281,6 +281,9 @@ $$$dense-vector-index-options$$$ : In case a knn query specifies a `rescore_vector` parameter, the query `rescore_vector` parameter will be used instead. : See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details. ::::: + +`early_termination` +: (Optional, boolean) Apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is expected to produce Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. :::: diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index de65ecf33318b..3ef69199d0fd2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -125,12 +125,18 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I b.startObject("rescore_vector"); b.field("oversample", DEFAULT_OVERSAMPLE); b.endObject(); + if (indexVersion.onOrAfter(DenseVectorFieldMapper.EXPOSE_EARLY_TERMINATION)) { + b.field("early_termination", false); + } b.endObject(); } else { b.startObject("index_options"); b.field("type", "int8_hnsw"); b.field("m", 16); b.field("ef_construction", 100); + if (indexVersion.onOrAfter(DenseVectorFieldMapper.EXPOSE_EARLY_TERMINATION)) { + b.field("early_termination", false); + } b.endObject(); } } From 783f8118adde38856cc8a419035a1eb44cdb1b38 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 26 Jun 2025 15:12:09 +0200 Subject: [PATCH 31/45] test fix --- .../test/search.vectors/80_dense_vector_indexed_by_default.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 7433ae6458119..3e604ed595b70 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -156,6 +156,9 @@ setup: - requires: cluster_features: "gte_v8.14.0" reason: 'dense_vector indexed as int8_hnsw by default was added in 8.14' + - requires: + cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] + reason: "requires early termination for hnsw to be exposed" - do: indices.create: index: test_default_index_options From c89b98c55d01d357713286f6537557b2d974fcf7 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 26 Jun 2025 17:24:05 +0200 Subject: [PATCH 32/45] use indexVersion in minimalMapping --- .../index/mapper/vectors/DenseVectorFieldMapperTests.java | 4 ++++ .../java/org/elasticsearch/index/mapper/MapperTestCase.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 3ef69199d0fd2..ec61ae1b8d0cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.CodecService; @@ -147,6 +148,9 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I b.field("type", "hnsw"); b.field("m", 5); b.field("ef_construction", 50); + if (indexVersion.onOrAfter(DenseVectorFieldMapper.EXPOSE_EARLY_TERMINATION)) { + b.field("early_termination", false); + } b.endObject(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 7e127ba307942..8b0ada490a16b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -519,7 +519,7 @@ public void testBoostNotAllowed() throws IOException { MapperParsingException e = expectThrows( MapperParsingException.class, () -> createMapperService(boostNotAllowedIndexVersion(), fieldMapping(b -> { - minimalMapping(b); + minimalMapping(b, boostNotAllowedIndexVersion()); b.field("boost", 2.0); })) ); From f03029db748a6d1d33eac1c02a205dc4908fdbc7 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 26 Jun 2025 17:24:29 +0200 Subject: [PATCH 33/45] spotless --- .../index/mapper/vectors/DenseVectorFieldMapperTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index ec61ae1b8d0cb..9f23a6023a13a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.CodecService; From d9bd37480fb9257b1dcfe235b5664212f5e68263 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 27 Jun 2025 11:01:35 +0200 Subject: [PATCH 34/45] test fix --- .../index/mapper/vectors/DenseVectorFieldTypeTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 3d966486f55e9..5fdb475e63701 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -461,7 +461,7 @@ public void testCreateKnnQueryMaxDims() { if (query instanceof RescoreKnnVectorQuery rescoreKnnVectorQuery) { query = rescoreKnnVectorQuery.innerQuery(); } - assertThat(query, instanceOf(KnnFloatVectorQuery.class)); + assertTrue(query instanceof KnnFloatVectorQuery || query instanceof PatienceKnnVectorQuery); } { // byte type with 4096 dims @@ -491,7 +491,7 @@ public void testCreateKnnQueryMaxDims() { null, randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) ); - assertThat(query, instanceOf(ESKnnByteVectorQuery.class)); + assertTrue(query instanceof ESKnnByteVectorQuery || query instanceof PatienceKnnVectorQuery); } } From 95b061c271e12c5eb33be41bf855ea3a360c6429 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 27 Jun 2025 11:58:22 +0200 Subject: [PATCH 35/45] skip rest compat test --- rest-api-spec/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 7b73575f76ef3..ffaff7b019e1e 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -90,4 +90,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively") task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed") task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist") + task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Default index options for dense_vector", "Introduced early_termination option") }) From 0efd8d971a082713acc6b4c90554a1fbf5e9791c Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 27 Jun 2025 12:26:42 +0200 Subject: [PATCH 36/45] skip rest compat test --- rest-api-spec/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ffaff7b019e1e..667d94e92755e 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -91,4 +91,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed") task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist") task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Default index options for dense_vector", "Introduced early_termination option") + task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Indexed by default with specified similarity and index options", "Introduced early_termination option") }) From 5b5c997db4c38ad02bf5180d5fe8bf03db273a3f Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 27 Jun 2025 13:40:12 +0200 Subject: [PATCH 37/45] add yaml test --- .../240_knn_search_early_termination.yml | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml new file mode 100644 index 0000000000000..7a9e23b3ef9b6 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml @@ -0,0 +1,166 @@ +setup: + - requires: + cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] + reason: "requires early termination for hnsw to be exposed" + - do: + indices.create: + index: bbq_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: dot_product + index_options: + type: bbq_hnsw + early_termination: true + + - do: + index: + index: bbq_hnsw + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + index: + index: bbq_hnsw + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + + - do: + index: + index: bbq_hnsw + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] + - do: + index: + index: bbq_hnsw + id: "4" + body: + vector: [ 0.123, 0.32 , -0.205, -0.63 , 0.132, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.904, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.161, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.032, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.294, -0.157, -0.1 , -0.279, -0.098, -0.176 ] + + - do: + index: + index: bbq_hnsw + id: "5" + body: + vector: [ 0.456, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.103, 0.078, -0.488, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.084 , -0.093, + -0.285, 0.336, -0.272, 0.369, -0.743, 0.086, -0.132, 0.475, + -0.224, 0.203, -0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.789 ] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + +--- +"Run knn search with early termination": + + - do: + search: + index: bbq_hnsw + body: + profile: true + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total.value: 5 } # collector sees k docs + - length: { hits.hits: 3 } # size docs retrieved + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.2._id: "3" } + +--- +"Profile knn search with early termination": + + - do: + search: + index: bbq_hnsw + body: + profile: true + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 } + + # Search with similarity to check number of operations are propagated correctly + - do: + search: + index: bbq_hnsw + body: + profile: true + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + similarity: 100000 + + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 } From 197bb585b543031a91396111efe8af70c9cf0b67 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 27 Jun 2025 14:28:34 +0200 Subject: [PATCH 38/45] yaml test fix --- .../240_knn_search_early_termination.yml | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml index 7a9e23b3ef9b6..2742f7bd6dcf4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml @@ -15,7 +15,7 @@ setup: type: dense_vector dims: 64 index: true - similarity: dot_product + similarity: max_inner_product index_options: type: bbq_hnsw early_termination: true @@ -88,11 +88,13 @@ setup: -0.224, 0.203, -0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.789 ] - # Flush in order to provoke a merge later - do: indices.flush: index: bbq_hnsw + - do: + indices.refresh: {} + --- "Run knn search with early termination": @@ -100,24 +102,23 @@ setup: search: index: bbq_hnsw body: - profile: true - knn: - field: vector - query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, - 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, - 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, - -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , - -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, - -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, - -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, - -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] - k: 3 - num_candidates: 3 + query: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + num_candidates: 5 + k: 3 - - match: { hits.total.value: 5 } # collector sees k docs - - length: { hits.hits: 3 } # size docs retrieved - - match: { hits.hits.0._id: "1" } - - match: { hits.hits.1._id: "2" } + - match: { hits.total.value: 3 } + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "1" } - match: { hits.hits.2._id: "3" } --- @@ -128,20 +129,21 @@ setup: index: bbq_hnsw body: profile: true - knn: - field: vector - query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, - 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, - 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, - -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , - -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, - -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, - -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, - -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] - k: 3 - num_candidates: 3 + query: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 - - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 } + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: null } # Search with similarity to check number of operations are propagated correctly - do: @@ -163,4 +165,4 @@ setup: num_candidates: 3 similarity: 100000 - - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 } + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 5 } From 840d3fd581e15095e4ee246f61141e6624b3240f Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 27 Jun 2025 15:30:12 +0200 Subject: [PATCH 39/45] Update docs/changelog/127223.yaml --- docs/changelog/127223.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/127223.yaml diff --git a/docs/changelog/127223.yaml b/docs/changelog/127223.yaml new file mode 100644 index 0000000000000..cc405c9f906d6 --- /dev/null +++ b/docs/changelog/127223.yaml @@ -0,0 +1,5 @@ +pr: 127223 +summary: Wrap ES KNN queries with PatienceKNN query +area: Vector Search +type: feature +issues: [] From dae3d72c49aac26b8f92c0661a8f8bf0b0db99eb Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 30 Jun 2025 14:21:20 +0200 Subject: [PATCH 40/45] switch to dynamic index setting --- .../index-settings/index-modules.md | 3 + .../mapping-reference/dense-vector.md | 3 - rest-api-spec/build.gradle | 2 - .../240_knn_search_early_termination.yml | 168 ------------- .../80_dense_vector_indexed_by_default.yml | 60 +---- .../elasticsearch/search/query/VectorIT.java | 51 ++++ .../common/settings/IndexScopedSettings.java | 1 + .../elasticsearch/index/IndexSettings.java | 11 + .../elasticsearch/index/IndexVersions.java | 1 - .../vectors/DenseVectorFieldMapper.java | 234 +++++------------- .../elasticsearch/search/SearchFeatures.java | 4 +- .../search/vectors/KnnVectorQueryBuilder.java | 4 +- .../vectors/DenseVectorFieldMapperTests.java | 40 +-- .../vectors/DenseVectorFieldTypeTests.java | 91 +++---- .../action/TransportResumeFollowAction.java | 3 +- .../mapper/SemanticTextFieldMapper.java | 2 +- .../mapper/SemanticTextFieldMapperTests.java | 10 +- 17 files changed, 222 insertions(+), 466 deletions(-) delete mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml diff --git a/docs/reference/elasticsearch/index-settings/index-modules.md b/docs/reference/elasticsearch/index-settings/index-modules.md index 4ab35b9d80a88..a6dfe13a3a0e6 100644 --- a/docs/reference/elasticsearch/index-settings/index-modules.md +++ b/docs/reference/elasticsearch/index-settings/index-modules.md @@ -259,3 +259,6 @@ $$$index-esql-stored-fields-sequential-proportion$$$ `index.esql.stored_fields_sequential_proportion` : Tuning parameter for deciding when {{esql}} will load [Stored fields](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#stored-fields) using a strategy tuned for loading dense sequence of documents. Allows values between 0.0 and 1.0 and defaults to 0.2. Indices with documents smaller than 10kb may see speed improvements loading `text` fields by setting this lower. + +$$$index-dense-vector-hnsw-filter-heuristic$$$ `index.dense_vector.hnsw_filter_heuristic` +: Whether to apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is only applicable to `dense_vector` fields with `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. Defaults to `false`. diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index 420b37596b2c7..7f3a701bde3f8 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -281,9 +281,6 @@ $$$dense-vector-index-options$$$ : In case a knn query specifies a `rescore_vector` parameter, the query `rescore_vector` parameter will be used instead. : See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details. ::::: - -`early_termination` -: (Optional, boolean) Apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is expected to produce Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. :::: diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 667d94e92755e..7b73575f76ef3 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -90,6 +90,4 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively") task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed") task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist") - task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Default index options for dense_vector", "Introduced early_termination option") - task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Indexed by default with specified similarity and index options", "Introduced early_termination option") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml deleted file mode 100644 index 2742f7bd6dcf4..0000000000000 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml +++ /dev/null @@ -1,168 +0,0 @@ -setup: - - requires: - cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] - reason: "requires early termination for hnsw to be exposed" - - do: - indices.create: - index: bbq_hnsw - body: - settings: - index: - number_of_shards: 1 - mappings: - properties: - vector: - type: dense_vector - dims: 64 - index: true - similarity: max_inner_product - index_options: - type: bbq_hnsw - early_termination: true - - - do: - index: - index: bbq_hnsw - id: "1" - body: - vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, - 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, - 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, - -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, - -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, - -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, - -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, - -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] - - do: - index: - index: bbq_hnsw - id: "2" - body: - vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, - -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, - 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, - -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, - -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, - -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, - 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, - -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] - - - do: - index: - index: bbq_hnsw - id: "3" - body: - name: rabbit.jpg - vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , - 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, - 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, - -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, - -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, - -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, - 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, - -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] - - do: - index: - index: bbq_hnsw - id: "4" - body: - vector: [ 0.123, 0.32 , -0.205, -0.63 , 0.132, 0.201, 0.167, -0.313, - 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, - 0.307, -0.083, 0.504, 0.255, -0.904, 0.289, -0.226, -0.132, - -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, - -0.285, 0.336, -0.272, 0.161, -0.282, 0.086, -0.132, 0.475, - -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, - -0.028, 0.321, -0.032, -0.009, -0.001 , 0.031, -0.533, 0.45, - -0.683, 1.331, 0.294, -0.157, -0.1 , -0.279, -0.098, -0.176 ] - - - do: - index: - index: bbq_hnsw - id: "5" - body: - vector: [ 0.456, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, - 0.176, 0.531, -0.375, 0.334, -0.103, 0.078, -0.488, 0.272, - 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, - -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.084 , -0.093, - -0.285, 0.336, -0.272, 0.369, -0.743, 0.086, -0.132, 0.475, - -0.224, 0.203, -0.439, 0.064, 0.246, -0.396, 0.297, 0.242, - -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, - -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.789 ] - - do: - indices.flush: - index: bbq_hnsw - - - do: - indices.refresh: {} - ---- -"Run knn search with early termination": - - - do: - search: - index: bbq_hnsw - body: - query: - knn: - field: vector - query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, - 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, - 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, - -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , - -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, - -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, - -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, - -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] - num_candidates: 5 - k: 3 - - - match: { hits.total.value: 3 } - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } - - match: { hits.hits.2._id: "3" } - ---- -"Profile knn search with early termination": - - - do: - search: - index: bbq_hnsw - body: - profile: true - query: - knn: - field: vector - query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, - 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, - 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, - -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , - -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, - -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, - -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, - -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] - k: 3 - num_candidates: 3 - - - match: { profile.shards.0.dfs.knn.0.vector_operations_count: null } - - # Search with similarity to check number of operations are propagated correctly - - do: - search: - index: bbq_hnsw - body: - profile: true - knn: - field: vector - query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, - 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, - 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, - -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , - -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, - -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, - -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, - -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] - k: 3 - num_candidates: 3 - similarity: 100000 - - - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 5 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml index 3e604ed595b70..0238a1781d278 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml @@ -25,7 +25,6 @@ setup: - match: { test.mappings.properties.vector.dims: 5 } - match: { test.mappings.properties.vector.index: true } - match: { test.mappings.properties.vector.similarity: cosine } - --- "Indexed by default with specified similarity and index options": - do: @@ -49,48 +48,19 @@ setup: indices.get_mapping: index: test - - match: { test.mappings.properties.vector.type: dense_vector } - - match: { test.mappings.properties.vector.dims: 5 } - - match: { test.mappings.properties.vector.index: true } - - match: { test.mappings.properties.vector.similarity: dot_product } - - match: { test.mappings.properties.vector.index_options.type: hnsw } - - match: { test.mappings.properties.vector.index_options.m: 32 } - - match: { test.mappings.properties.vector.index_options.ef_construction: 200 } - ---- -"Indexed by default with specified similarity and index options with early_termination": - - requires: - cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] - reason: "requires early termination for hnsw to be exposed" - - do: - indices.create: - index: test - body: - mappings: - properties: - vector: - type: dense_vector - dims: 5 - similarity: dot_product - index_options: - type: hnsw - m: 32 - ef_construction: 200 - - - match: { acknowledged: true } - - - do: - indices.get_mapping: - index: test - - - match: { test.mappings.properties.vector.type: dense_vector } - - match: { test.mappings.properties.vector.dims: 5 } - - match: { test.mappings.properties.vector.index: true } - - match: { test.mappings.properties.vector.similarity: dot_product } - - match: { test.mappings.properties.vector.index_options.type: hnsw } - - match: { test.mappings.properties.vector.index_options.m: 32 } - - match: { test.mappings.properties.vector.index_options.ef_construction: 200 } - - match: { test.mappings.properties.vector.index_options.early_termination: false } + - match: + test: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: true + similarity: dot_product + index_options: + type: hnsw + m: 32 + ef_construction: 200 --- "Not indexed vector": @@ -156,9 +126,6 @@ setup: - requires: cluster_features: "gte_v8.14.0" reason: 'dense_vector indexed as int8_hnsw by default was added in 8.14' - - requires: - cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ] - reason: "requires early termination for hnsw to be exposed" - do: indices.create: index: test_default_index_options @@ -180,7 +147,6 @@ setup: - match: { test_default_index_options.mappings.properties.vector.index: true } - match: { test_default_index_options.mappings.properties.vector.similarity: cosine } - match: { test_default_index_options.mappings.properties.vector.index_options.type: int8_hnsw } - - match: { test_default_index_options.mappings.properties.vector.index_options.early_termination: false } --- "Default index options for dense_vector element type byte": - requires: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java index 82f63ebbbee12..3dabe1b37b43e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java @@ -127,4 +127,55 @@ public void testFilteredQueryStrategy() { }); } + public void testHnswEarlyTerminationQuery() { + float[] vector = new float[16]; + randomVector(vector, 25); + int upperLimit = 35; + var query = new KnnSearchBuilder(VECTOR_FIELD, vector, 1, 1, null, null); + assertResponse(client().prepareSearch(INDEX_NAME).setKnnSearch(List.of(query)).setSize(1).setProfile(true), response -> { + assertNotEquals(0, response.getHits().getHits().length); + var profileResults = response.getProfileResults(); + long vectorOpsSum = profileResults.values() + .stream() + .mapToLong( + pr -> pr.getQueryPhase() + .getSearchProfileDfsPhaseResult() + .getQueryProfileShardResult() + .stream() + .mapToLong(qpr -> qpr.getVectorOperationsCount().longValue()) + .sum() + ) + .sum(); + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(DenseVectorFieldMapper.HNSW_EARLY_TERMINATION.getKey(), true)) + .get(); + assertResponse( + client().prepareSearch(INDEX_NAME).setKnnSearch(List.of(query)).setSize(1).setProfile(true), + earlyTerminationResponse -> { + assertNotEquals(0, earlyTerminationResponse.getHits().getHits().length); + var earlyTerminationResults = earlyTerminationResponse.getProfileResults(); + long earlyTerminationVectorOpsSum = earlyTerminationResults.values() + .stream() + .mapToLong( + pr -> pr.getQueryPhase() + .getSearchProfileDfsPhaseResult() + .getQueryProfileShardResult() + .stream() + .mapToLong(qpr -> qpr.getVectorOperationsCount().longValue()) + .sum() + ) + .sum(); + assertTrue( + "earlyTerminationVectorOps [" + earlyTerminationVectorOpsSum + "] is not lt vectorOps [" + vectorOpsSum + "]", + earlyTerminationVectorOpsSum < vectorOpsSum + // if both switch to brute-force due to excessive exploration, they will both equal to upperLimit + || (earlyTerminationVectorOpsSum == vectorOpsSum && vectorOpsSum == upperLimit + 1) + ); + } + ); + }); + } + } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 0ff64f14dc17c..796b03211432b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -159,6 +159,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING, IndexSettings.INDEX_SEARCH_IDLE_AFTER, DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC, + DenseVectorFieldMapper.HNSW_EARLY_TERMINATION, IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY, IndexSettings.IGNORE_ABOVE_SETTING, FieldMapper.IGNORE_MALFORMED_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 4b89ab2e60021..eac2ef3d42b61 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -916,6 +916,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile int maxNgramDiff; private volatile int maxShingleDiff; private volatile DenseVectorFieldMapper.FilterHeuristic hnswFilterHeuristic; + private volatile boolean earlyTermination; private volatile TimeValue searchIdleAfter; private volatile int maxAnalyzedOffset; private volatile boolean weightMatchesEnabled; @@ -1113,6 +1114,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); hnswFilterHeuristic = scopedSettings.get(DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC); + earlyTermination = scopedSettings.get(DenseVectorFieldMapper.HNSW_EARLY_TERMINATION); indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings); recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false @@ -1227,6 +1229,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti ); scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); scopedSettings.addSettingsUpdateConsumer(DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC, this::setHnswFilterHeuristic); + scopedSettings.addSettingsUpdateConsumer(DenseVectorFieldMapper.HNSW_EARLY_TERMINATION, this::setHnswEarlyTermination); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1858,6 +1861,14 @@ private void setHnswFilterHeuristic(DenseVectorFieldMapper.FilterHeuristic heuri this.hnswFilterHeuristic = heuristic; } + public boolean getHnswEarlyTermination() { + return this.earlyTermination; + } + + private void setHnswEarlyTermination(boolean earlyTermination) { + this.earlyTermination = earlyTermination; + } + public SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions() { return seqNoIndexOptions; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index e2f17a06ae640..6ff33cf05d51f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -178,7 +178,6 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_10_2_2 = def(9_030_0_00, Version.LUCENE_10_2_2); public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2); public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2); - public static final IndexVersion EXPOSE_EARLY_TERMINATION = def(9_033_0_00, Version.LUCENE_10_2_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index f55f09699972d..37084e61a360b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -175,6 +175,14 @@ public KnnSearchStrategy getKnnSearchStrategy() { Setting.Property.Dynamic ); + public static final Setting HNSW_EARLY_TERMINATION = Setting.boolSetting( + "index.dense_vector.hnsw_early_termination", + DEFAULT_HNSW_EARLY_TERMINATION, + Setting.Property.IndexScope, + Setting.Property.ServerlessPublic, + Setting.Property.Dynamic + ); + private static boolean hasRescoreIndexVersion(IndexVersion version) { return version.onOrAfter(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS) || version.between(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); @@ -199,7 +207,6 @@ private static boolean defaultOversampleForBBQ(IndexVersion version) { public static final IndexVersion DEFAULT_TO_INT8 = IndexVersions.DEFAULT_DENSE_VECTOR_TO_INT8_HNSW; public static final IndexVersion DEFAULT_TO_BBQ = IndexVersions.DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW; public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersions.V_8_9_0; - public static final IndexVersion EXPOSE_EARLY_TERMINATION = IndexVersions.EXPOSE_EARLY_TERMINATION; public static final NodeFeature RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature("mapper.dense_vector.rescore_vector"); public static final NodeFeature RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature( @@ -307,7 +314,7 @@ public Builder(String name, IndexVersion indexVersionCreated) { this.indexOptions = new Parameter<>( "index_options", true, - () -> defaultIndexOptions(defaultInt8Hnsw, defaultBBQ8Hnsw, indexVersionCreated), + () -> defaultIndexOptions(defaultInt8Hnsw, defaultBBQ8Hnsw), (n, c, o) -> o == null ? null : parseIndexOptions(n, o, indexVersionCreated), m -> toType(m).indexOptions, (b, n, v) -> { @@ -354,26 +361,20 @@ public Builder(String name, IndexVersion indexVersionCreated) { }); } - private DenseVectorIndexOptions defaultIndexOptions( - boolean defaultInt8Hnsw, - boolean defaultBBQHnsw, - IndexVersion indexVersionCreated - ) { + private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boolean defaultBBQHnsw) { if (this.dims != null && this.dims.isConfigured() && elementType.getValue() == ElementType.FLOAT && this.indexed.getValue()) { if (defaultBBQHnsw && this.dims.getValue() >= BBQ_DIMS_DEFAULT_THRESHOLD) { return new BBQHnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, - new RescoreVector(DEFAULT_OVERSAMPLE), - indexVersionCreated + new RescoreVector(DEFAULT_OVERSAMPLE) ); } else if (defaultInt8Hnsw) { return new Int8HnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, null, - null, - indexVersionCreated + null ); } } @@ -1437,20 +1438,10 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map createKnnFloatQuery( queryVector.asFloatVector(), @@ -2593,7 +2478,8 @@ public Query createKnnQuery( filter, similarityThreshold, parentFilter, - knnSearchStrategy + knnSearchStrategy, + hnswEarlyTermination ); case BIT -> createKnnBitQuery( queryVector.asByteVector(), @@ -2602,7 +2488,8 @@ public Query createKnnQuery( filter, similarityThreshold, parentFilter, - knnSearchStrategy + knnSearchStrategy, + hnswEarlyTermination ); }; } @@ -2622,13 +2509,16 @@ private Query createKnnBitQuery( Query filter, Float similarityThreshold, BitSetProducer parentFilter, - KnnSearchStrategy searchStrategy + KnnSearchStrategy searchStrategy, + boolean hnswEarlyTermination ) { elementType.checkDimensions(dims, queryVector.length); Query knnQuery = parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); - knnQuery = maybeWrapPatience(knnQuery); + if (hnswEarlyTermination) { + knnQuery = maybeWrapPatience(knnQuery); + } if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2646,7 +2536,8 @@ private Query createKnnByteQuery( Query filter, Float similarityThreshold, BitSetProducer parentFilter, - KnnSearchStrategy searchStrategy + KnnSearchStrategy searchStrategy, + boolean hnswEarlyTermination ) { elementType.checkDimensions(dims, queryVector.length); @@ -2657,7 +2548,9 @@ private Query createKnnByteQuery( Query knnQuery = parentFilter != null ? new ESDiversifyingChildrenByteKnnVectorQuery(name(), queryVector, filter, k, numCands, parentFilter, searchStrategy) : new ESKnnByteVectorQuery(name(), queryVector, k, numCands, filter, searchStrategy); - knnQuery = maybeWrapPatience(knnQuery); + if (hnswEarlyTermination) { + knnQuery = maybeWrapPatience(knnQuery); + } if (similarityThreshold != null) { knnQuery = new VectorSimilarityQuery( knnQuery, @@ -2680,13 +2573,13 @@ private Query maybeWrapPatience(Query knnQuery) { private Query maybeWrapPatienceByte(KnnByteVectorQuery knnQuery, int patience) { Query returnedQuery = knnQuery; - if (indexOptions instanceof HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + if (indexOptions instanceof HnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof Int8HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + } else if (indexOptions instanceof Int8HnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof Int4HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + } else if (indexOptions instanceof Int4HnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof BBQHnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + } else if (indexOptions instanceof BBQHnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); } return returnedQuery; @@ -2694,13 +2587,13 @@ private Query maybeWrapPatienceByte(KnnByteVectorQuery knnQuery, int patience) { private Query maybeWrapPatienceFloat(KnnFloatVectorQuery knnQuery, int patience) { Query returnedQuery = knnQuery; - if (indexOptions instanceof HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + if (indexOptions instanceof HnswIndexOptions hnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof Int8HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + } else if (indexOptions instanceof Int8HnswIndexOptions hnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof Int4HnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + } else if (indexOptions instanceof Int4HnswIndexOptions hnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof BBQHnswIndexOptions hnswIndexOptions && hnswIndexOptions.earlyTermination) { + } else if (indexOptions instanceof BBQHnswIndexOptions hnswIndexOptions) { returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); } return returnedQuery; @@ -2714,7 +2607,8 @@ private Query createKnnFloatQuery( Query filter, Float similarityThreshold, BitSetProducer parentFilter, - KnnSearchStrategy knnSearchStrategy + KnnSearchStrategy knnSearchStrategy, + boolean hnswEarlyTermination ) { elementType.checkDimensions(dims, queryVector.length); elementType.checkVectorBounds(queryVector); @@ -2772,7 +2666,9 @@ && isNotUnitVector(squaredMagnitude)) { knnSearchStrategy ) : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, knnSearchStrategy); - knnQuery = maybeWrapPatience(knnQuery); + if (hnswEarlyTermination) { + knnQuery = maybeWrapPatience(knnQuery); + } } if (rescore) { knnQuery = new RescoreKnnVectorQuery( @@ -2902,16 +2798,14 @@ public void parse(DocumentParserContext context) throws IOException { denseVectorIndexOptions = new BBQHnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, - new RescoreVector(DEFAULT_OVERSAMPLE), - indexCreatedVersion + new RescoreVector(DEFAULT_OVERSAMPLE) ); } else if (defaultInt8Hnsw) { denseVectorIndexOptions = new Int8HnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, null, - null, - indexCreatedVersion + null ); } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java index 2bee1e3b2d67f..0c2f7c2aa625b 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java +++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java @@ -32,7 +32,6 @@ public Set getFeatures() { public static final NodeFeature INT_SORT_FOR_INT_SHORT_BYTE_FIELDS = new NodeFeature("search.sort.int_sort_for_int_short_byte_fields"); static final NodeFeature MULTI_MATCH_CHECKS_POSITIONS = new NodeFeature("search.multi.match.checks.positions"); public static final NodeFeature BBQ_HNSW_DEFAULT_INDEXING = new NodeFeature("search.vectors.mappers.default_bbq_hnsw"); - public static final NodeFeature EXPOSE_HNSW_EARLY_TERMINATION = new NodeFeature("search.vectors.mappers.expose_hnsw_early_termination"); @Override public Set getTestFeatures() { @@ -42,8 +41,7 @@ public Set getTestFeatures() { RESCORER_MISSING_FIELD_BAD_REQUEST, INT_SORT_FOR_INT_SHORT_BYTE_FIELDS, MULTI_MATCH_CHECKS_POSITIONS, - BBQ_HNSW_DEFAULT_INDEXING, - EXPOSE_HNSW_EARLY_TERMINATION + BBQ_HNSW_DEFAULT_INDEXING ); } } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 87f9a50c64c17..ea0c15642eb74 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -553,6 +553,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { } } DenseVectorFieldMapper.FilterHeuristic heuristic = context.getIndexSettings().getHnswFilterHeuristic(); + boolean hnswEarlyTermination = context.getIndexSettings().getHnswEarlyTermination(); return vectorFieldType.createKnnQuery( queryVector, k, @@ -561,7 +562,8 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { filterQuery, vectorSimilarity, parentBitSet, - heuristic + heuristic, + hnswEarlyTermination ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 9f23a6023a13a..0f161d4a1e44f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -125,18 +125,12 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I b.startObject("rescore_vector"); b.field("oversample", DEFAULT_OVERSAMPLE); b.endObject(); - if (indexVersion.onOrAfter(DenseVectorFieldMapper.EXPOSE_EARLY_TERMINATION)) { - b.field("early_termination", false); - } b.endObject(); } else { b.startObject("index_options"); b.field("type", "int8_hnsw"); b.field("m", 16); b.field("ef_construction", 100); - if (indexVersion.onOrAfter(DenseVectorFieldMapper.EXPOSE_EARLY_TERMINATION)) { - b.field("early_termination", false); - } b.endObject(); } } @@ -147,9 +141,6 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I b.field("type", "hnsw"); b.field("m", 5); b.field("ef_construction", 50); - if (indexVersion.onOrAfter(DenseVectorFieldMapper.EXPOSE_EARLY_TERMINATION)) { - b.field("early_termination", false); - } b.endObject(); } } @@ -1683,7 +1674,6 @@ public void testMergeDims() throws IOException { .field("type", "int8_hnsw") .field("m", 16) .field("ef_construction", 100) - .field("early_termination", false) .endObject(); b.endObject(); }); @@ -2439,7 +2429,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2457,7 +2448,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2475,7 +2467,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2493,7 +2486,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2511,7 +2505,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("element_type [byte] vectors do not support NaN values but found [NaN] at dim [0];")); @@ -2526,7 +2521,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2544,7 +2540,8 @@ public void testByteVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2579,7 +2576,8 @@ public void testFloatVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("element_type [float] vectors do not support NaN values but found [NaN] at dim [0];")); @@ -2594,7 +2592,8 @@ public void testFloatVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( @@ -2612,7 +2611,8 @@ public void testFloatVectorQueryBoundaries() throws IOException { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 5fdb475e63701..f3c348c68ce7a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -59,29 +59,25 @@ private static DenseVectorFieldMapper.RescoreVector randomRescoreVector() { private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsNonQuantized() { return randomFrom( - new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), IndexVersion.current()), + new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), new DenseVectorFieldMapper.FlatIndexOptions() ); } public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsAll() { return randomFrom( - new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), IndexVersion.current()), + new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), new DenseVectorFieldMapper.Int8HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()), - randomBoolean(), - IndexVersion.current() + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.Int4HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()), - randomBoolean(), - IndexVersion.current() + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.FlatIndexOptions(), new DenseVectorFieldMapper.Int8FlatIndexOptions( @@ -95,9 +91,7 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsA new DenseVectorFieldMapper.BBQHnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()), - randomBoolean(), - IndexVersion.current() + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.BBQFlatIndexOptions(randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())) ); @@ -115,25 +109,15 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQua randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - rescoreVector, - randomBoolean(), - IndexVersion.current() + rescoreVector ), new DenseVectorFieldMapper.Int4HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - rescoreVector, - randomBoolean(), - IndexVersion.current() + rescoreVector ), - new DenseVectorFieldMapper.BBQHnswIndexOptions( - randomIntBetween(1, 100), - randomIntBetween(1, 10_000), - rescoreVector, - randomBoolean(), - IndexVersion.current() - ) + new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector) ); } @@ -249,7 +233,8 @@ public void testCreateNestedKnnQuery() { null, null, producer, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); if (query instanceof RescoreKnnVectorQuery rescoreKnnVectorQuery) { query = rescoreKnnVectorQuery.innerQuery(); @@ -283,7 +268,8 @@ public void testCreateNestedKnnQuery() { null, null, producer, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); assertTrue(query instanceof DiversifyingChildrenByteKnnVectorQuery || query instanceof PatienceKnnVectorQuery); @@ -296,9 +282,10 @@ public void testCreateNestedKnnQuery() { null, null, producer, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); - assertThat(query, instanceOf(DiversifyingChildrenByteKnnVectorQuery.class)); + assertTrue(query instanceof DiversifyingChildrenByteKnnVectorQuery || query instanceof PatienceKnnVectorQuery); } } @@ -369,7 +356,8 @@ public void testFloatCreateKnnQuery() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("to perform knn search on field [f], its mapping must have [index] set to [true]")); @@ -399,7 +387,8 @@ public void testFloatCreateKnnQuery() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("The [dot_product] similarity can only be used with unit-length vectors.")); @@ -425,7 +414,8 @@ public void testFloatCreateKnnQuery() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); @@ -456,7 +446,8 @@ public void testCreateKnnQueryMaxDims() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); if (query instanceof RescoreKnnVectorQuery rescoreKnnVectorQuery) { query = rescoreKnnVectorQuery.innerQuery(); @@ -489,7 +480,8 @@ public void testCreateKnnQueryMaxDims() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); assertTrue(query instanceof ESKnnByteVectorQuery || query instanceof PatienceKnnVectorQuery); } @@ -517,7 +509,8 @@ public void testByteCreateKnnQuery() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("to perform knn search on field [f], its mapping must have [index] set to [true]")); @@ -543,7 +536,8 @@ public void testByteCreateKnnQuery() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); @@ -558,7 +552,8 @@ public void testByteCreateKnnQuery() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); @@ -586,7 +581,8 @@ public void testRescoreOversampleUsedWithoutQuantization() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); if (elementType == BYTE) { @@ -650,7 +646,8 @@ public void testRescoreOversampleQueryOverrides() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); assertTrue(query instanceof ESKnnFloatVectorQuery || query instanceof PatienceKnnVectorQuery); @@ -674,7 +671,8 @@ public void testRescoreOversampleQueryOverrides() { null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); assertTrue(query instanceof RescoreKnnVectorQuery); RescoreKnnVectorQuery rescoreKnnVectorQuery = (RescoreKnnVectorQuery) query; @@ -713,7 +711,8 @@ public void testFilterSearchThreshold() { null, null, null, - DenseVectorFieldMapper.FilterHeuristic.FANOUT + DenseVectorFieldMapper.FilterHeuristic.FANOUT, + randomBoolean() ); KnnSearchStrategy strategy = tuple.v2().apply(query); if (strategy != null) { @@ -728,11 +727,14 @@ public void testFilterSearchThreshold() { null, null, null, - DenseVectorFieldMapper.FilterHeuristic.ACORN + DenseVectorFieldMapper.FilterHeuristic.ACORN, + randomBoolean() ); strategy = tuple.v2().apply(query); - assertTrue(strategy instanceof KnnSearchStrategy.Hnsw); - assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(60)); + if (strategy != null) { + assertThat(strategy, instanceOf(KnnSearchStrategy.Hnsw.class)); + assertThat(((KnnSearchStrategy.Hnsw) strategy).filteredSearchThreshold(), equalTo(60)); + } } } } @@ -754,7 +756,8 @@ private static void checkRescoreQueryParameters( null, null, null, - randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()) + randomFrom(DenseVectorFieldMapper.FilterHeuristic.values()), + randomBoolean() ); RescoreKnnVectorQuery rescoreQuery = (RescoreKnnVectorQuery) query; Query innerQuery = rescoreQuery.innerQuery(); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index 18e125a7ae1ce..b0be3e21bbc7c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -531,7 +531,8 @@ static String[] extractLeaderShardHistoryUUIDs(Map ccrIndexMetad DataTier.TIER_PREFERENCE_SETTING, IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING, MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING, - DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC + DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC, + DenseVectorFieldMapper.HNSW_EARLY_TERMINATION ); public static Settings filter(Settings originalSettings) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index c1746aa1cc161..fd5f1ce2735a9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1254,7 +1254,7 @@ static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorI int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); - return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector, IndexVersion.current()); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); } static SemanticTextIndexOptions defaultIndexOptions(IndexVersion indexVersionCreated, MinimalServiceSettings modelSettings) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 92ec3579cdb46..0fab22d45d08c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -1175,7 +1175,7 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultDenseVector // These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ. int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null, IndexVersion.current()); + return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null); } private static SemanticTextIndexOptions defaultDenseVectorSemanticIndexOptions() { @@ -1186,7 +1186,7 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDens int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); - return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector, IndexVersion.current()); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); } private static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() { @@ -1268,7 +1268,7 @@ public void testDefaultIndexOptions() throws IOException { null, new SemanticTextIndexOptions( SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, null, null, null) + new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, null, null) ) ); @@ -1348,7 +1348,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { null, new SemanticTextIndexOptions( SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, 0.4f, null, null) + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, 0.4f, null) ) ); @@ -1375,7 +1375,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { null, new SemanticTextIndexOptions( SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, 0f, null, null) + new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, 0f, null) ) ); From 8de4fdaa4ad13a7d3fd8c0f463a0df075e8ef6dd Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 30 Jun 2025 15:48:06 +0200 Subject: [PATCH 41/45] Update server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java Co-authored-by: Benjamin Trent --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 37084e61a360b..99b553a0f6357 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -176,7 +176,7 @@ public KnnSearchStrategy getKnnSearchStrategy() { ); public static final Setting HNSW_EARLY_TERMINATION = Setting.boolSetting( - "index.dense_vector.hnsw_early_termination", + "index.dense_vector.hnsw_enable_early_termination", DEFAULT_HNSW_EARLY_TERMINATION, Setting.Property.IndexScope, Setting.Property.ServerlessPublic, From c5a51e9672407cff4de6fe5c2ad1609425345aa4 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 30 Jun 2025 15:48:38 +0200 Subject: [PATCH 42/45] simplify and typo fix --- .../index-settings/index-modules.md | 2 +- .../vectors/DenseVectorFieldMapper.java | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/reference/elasticsearch/index-settings/index-modules.md b/docs/reference/elasticsearch/index-settings/index-modules.md index a6dfe13a3a0e6..542c5bc0f811c 100644 --- a/docs/reference/elasticsearch/index-settings/index-modules.md +++ b/docs/reference/elasticsearch/index-settings/index-modules.md @@ -260,5 +260,5 @@ $$$index-esql-stored-fields-sequential-proportion$$$ `index.esql.stored_fields_sequential_proportion` : Tuning parameter for deciding when {{esql}} will load [Stored fields](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#stored-fields) using a strategy tuned for loading dense sequence of documents. Allows values between 0.0 and 1.0 and defaults to 0.2. Indices with documents smaller than 10kb may see speed improvements loading `text` fields by setting this lower. -$$$index-dense-vector-hnsw-filter-heuristic$$$ `index.dense_vector.hnsw_filter_heuristic` +$$$index-dense-vector-hnsw-early-termination$$$ `index.dense_vector.hnsw_early_termination` : Whether to apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is only applicable to `dense_vector` fields with `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. Defaults to `false`. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 37084e61a360b..deef5aec801f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2564,37 +2564,37 @@ private Query createKnnByteQuery( private Query maybeWrapPatience(Query knnQuery) { Query finalQuery = knnQuery; if (knnQuery instanceof KnnByteVectorQuery knnByteVectorQuery) { - finalQuery = maybeWrapPatienceByte(knnByteVectorQuery, Math.max(7, (int) (knnByteVectorQuery.getK() * 0.3))); + finalQuery = maybeWrapPatienceByte(knnByteVectorQuery); } else if (knnQuery instanceof KnnFloatVectorQuery knnFloatVectorQuery) { - finalQuery = maybeWrapPatienceFloat(knnFloatVectorQuery, Math.max(7, (int) (knnFloatVectorQuery.getK() * 0.3))); + finalQuery = maybeWrapPatienceFloat(knnFloatVectorQuery); } return finalQuery; } - private Query maybeWrapPatienceByte(KnnByteVectorQuery knnQuery, int patience) { + private Query maybeWrapPatienceByte(KnnByteVectorQuery knnQuery) { Query returnedQuery = knnQuery; if (indexOptions instanceof HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); } else if (indexOptions instanceof Int8HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); } else if (indexOptions instanceof Int4HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); } else if (indexOptions instanceof BBQHnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery, 0.995, patience); + returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); } return returnedQuery; } - private Query maybeWrapPatienceFloat(KnnFloatVectorQuery knnQuery, int patience) { + private Query maybeWrapPatienceFloat(KnnFloatVectorQuery knnQuery) { Query returnedQuery = knnQuery; - if (indexOptions instanceof HnswIndexOptions hnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof Int8HnswIndexOptions hnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof Int4HnswIndexOptions hnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); - } else if (indexOptions instanceof BBQHnswIndexOptions hnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery, 0.995, patience); + if (indexOptions instanceof HnswIndexOptions) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); + } else if (indexOptions instanceof Int8HnswIndexOptions) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); + } else if (indexOptions instanceof Int4HnswIndexOptions) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); + } else if (indexOptions instanceof BBQHnswIndexOptions) { + returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); } return returnedQuery; } From c2ef4d2d5becd3299feb3e79c9430204958ec6d7 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 30 Jun 2025 16:01:47 +0200 Subject: [PATCH 43/45] remove unneeded changes --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index b1c144f598480..4a2b7fe18aa6d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1340,15 +1340,9 @@ public final String toString() { public abstract static class DenseVectorIndexOptions implements IndexOptions { final VectorIndexType type; - final IndexVersion indexVersion; DenseVectorIndexOptions(VectorIndexType type) { - this(type, null); - } - - DenseVectorIndexOptions(VectorIndexType type, IndexVersion indexVersion) { this.type = type; - this.indexVersion = indexVersion; } abstract KnnVectorsFormat getVectorsFormat(ElementType elementType); @@ -1418,11 +1412,6 @@ abstract static class QuantizedIndexOptions extends DenseVectorIndexOptions { super(type); this.rescoreVector = rescoreVector; } - - QuantizedIndexOptions(VectorIndexType type, RescoreVector rescoreVector, IndexVersion indexVersion) { - super(type, indexVersion); - this.rescoreVector = rescoreVector; - } } public enum VectorIndexType { @@ -1476,6 +1465,7 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map Date: Mon, 30 Jun 2025 16:47:55 +0200 Subject: [PATCH 44/45] simplify --- .../vectors/DenseVectorFieldMapper.java | 39 +++++-------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 4a2b7fe18aa6d..06da336c539bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2553,40 +2553,19 @@ private Query createKnnByteQuery( private Query maybeWrapPatience(Query knnQuery) { Query finalQuery = knnQuery; - if (knnQuery instanceof KnnByteVectorQuery knnByteVectorQuery) { - finalQuery = maybeWrapPatienceByte(knnByteVectorQuery); - } else if (knnQuery instanceof KnnFloatVectorQuery knnFloatVectorQuery) { - finalQuery = maybeWrapPatienceFloat(knnFloatVectorQuery); + if (knnQuery instanceof KnnByteVectorQuery knnByteVectorQuery && canApplyPatienceQuery()) { + finalQuery = PatienceKnnVectorQuery.fromByteQuery(knnByteVectorQuery); + } else if (knnQuery instanceof KnnFloatVectorQuery knnFloatVectorQuery && canApplyPatienceQuery()) { + finalQuery = PatienceKnnVectorQuery.fromFloatQuery(knnFloatVectorQuery); } return finalQuery; } - private Query maybeWrapPatienceByte(KnnByteVectorQuery knnQuery) { - Query returnedQuery = knnQuery; - if (indexOptions instanceof HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); - } else if (indexOptions instanceof Int8HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); - } else if (indexOptions instanceof Int4HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); - } else if (indexOptions instanceof BBQHnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromByteQuery(knnQuery); - } - return returnedQuery; - } - - private Query maybeWrapPatienceFloat(KnnFloatVectorQuery knnQuery) { - Query returnedQuery = knnQuery; - if (indexOptions instanceof HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); - } else if (indexOptions instanceof Int8HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); - } else if (indexOptions instanceof Int4HnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); - } else if (indexOptions instanceof BBQHnswIndexOptions) { - returnedQuery = PatienceKnnVectorQuery.fromFloatQuery(knnQuery); - } - return returnedQuery; + private boolean canApplyPatienceQuery() { + return indexOptions instanceof HnswIndexOptions + || indexOptions instanceof Int8HnswIndexOptions + || indexOptions instanceof Int4HnswIndexOptions + || indexOptions instanceof BBQHnswIndexOptions; } private Query createKnnFloatQuery( From 768a513e5feeffee1938abb7bb3dd263aaf696b1 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 30 Jun 2025 17:06:40 +0200 Subject: [PATCH 45/45] revert unneeded change --- .../java/org/elasticsearch/index/mapper/MapperTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 8b0ada490a16b..7e127ba307942 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -519,7 +519,7 @@ public void testBoostNotAllowed() throws IOException { MapperParsingException e = expectThrows( MapperParsingException.class, () -> createMapperService(boostNotAllowedIndexVersion(), fieldMapping(b -> { - minimalMapping(b, boostNotAllowedIndexVersion()); + minimalMapping(b); b.field("boost", 2.0); })) );