add ord mapping from RAVV to graph creation

Signed-off-by: Samuel Herman <sherman8915@gmail.com>

Author: sam-herman

jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java

@@ -787,12 +787,27 @@ public synchronized long removeDeletedNodes() {
     /**
      * Convenience method to build a new graph from an existing one, with the addition of new nodes.
      * This is useful when we want to merge a new set of vectors into an existing graph that is already on disk.
+     *
+     * @param onDiskGraphIndex the on-disk representation of the graph index to be processed and converted.
+     * @param perLevelNeighborsScoreCache the cache containing pre-computed neighbor scores,
+     * @param newVectors a super set RAVV containing the new vectors to be added to the graph as well as the old ones that are already in the graph
+     * @param buildScoreProvider the provider responsible for calculating build scores.
+     * @param startingNodeOffset the offset in the newVectors RAVV where the new vectors start
+     * @param graphToRavvOrdMap a mapping from the old graph's node ids to the newVectors RAVV node ids
+     * @param beamWidth the width of the beam used during the graph building process.
+     * @param overflowRatio the ratio of extra neighbors to allow temporarily when inserting a node.
+     * @param alpha the weight factor for balancing score computations.
+     * @param addHierarchy whether to add hierarchical structures while building the graph.
+     *
+     * @return the in-memory representation of the graph index.
+     * @throws IOException if an I/O error occurs during the graph loading or conversion process.
      */
     public static OnHeapGraphIndex buildAndMergeNewNodes(OnDiskGraphIndex onDiskGraphIndex,
                                                          NeighborsScoreCache perLevelNeighborsScoreCache,
                                                          RandomAccessVectorValues newVectors,
                                                          BuildScoreProvider buildScoreProvider,
-                                                         int startingNodeId,
+                                                         int startingNodeOffset,
+                                                         int[] graphToRavvOrdMap,
                                                          int beamWidth,
                                                          float overflowRatio,
                                                          float alpha,
@@ -813,9 +828,9 @@ public static OnHeapGraphIndex buildAndMergeNewNodes(OnDiskGraphIndex onDiskGrap
 
             // Add each new vector incrementally
             final List<ForkJoinTask<?>> forkJoinTask = new ArrayList<>(newVectors.size());
-            for (int i = 0; i < newVectors.size(); i++) {
-                final int nodeId = startingNodeId + i;
-                final VectorFloat<?> vector = newVectors.getVector(i);
+            for (int i = startingNodeOffset; i < newVectors.size(); i++) {
+                final int nodeId = i;
+                final VectorFloat<?> vector = newVectors.getVector(graphToRavvOrdMap[nodeId]);
 
                 // The GraphIndexBuilder can add nodes to an existing index
                 forkJoinTask.add(PhysicalCoreExecutor.pool().submit(() -> builder.addGraphNode(nodeId, vector)));

jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/NeighborsScoreCache.java

@@ -40,7 +40,7 @@
  * @see OnHeapGraphIndex#convertToHeap(OnDiskGraphIndex, NeighborsScoreCache, BuildScoreProvider, float, float)
  *
  * This is particularly useful when merging new nodes into an existing graph.
- * @see GraphIndexBuilder#buildAndMergeNewNodes(OnDiskGraphIndex, NeighborsScoreCache, RandomAccessVectorValues, BuildScoreProvider, int, int, float, float, boolean)
+ * @see GraphIndexBuilder#buildAndMergeNewNodes(OnDiskGraphIndex, NeighborsScoreCache, RandomAccessVectorValues, BuildScoreProvider, int, int[], int, float, float, boolean)
  */
 public class NeighborsScoreCache {
     private final Map<Integer, Map<Integer, NodeArray>> perLevelNeighborsScoreCache;

jvector-tests/src/test/java/io/github/jbellis/jvector/graph/OnHeapGraphIndexTest.java

@@ -41,6 +41,7 @@
 import java.util.Arrays;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -175,7 +176,9 @@ public void testIncrementalInsertionFromOnDiskIndex() throws IOException {
         try (var readerSupplier = new SimpleMappedReader.Supplier(outputPath.toAbsolutePath());
              var onDiskGraph = OnDiskGraphIndex.load(readerSupplier)) {
             TestUtil.assertGraphEquals(baseGraphIndex, onDiskGraph);
-            OnHeapGraphIndex reconstructedAllNodeOnHeapGraphIndex = GraphIndexBuilder.buildAndMergeNewNodes(onDiskGraph, neighborsScoreCache, newVectorsRavv, allBuildScoreProvider, numBaseVectors, beamWidth, neighborOverflow, alpha, addHierarchy);
+            // We will create a trivial 1:1 mapping between the new graph and the ravv
+            final int[] graphToRavvOrdMap = IntStream.range(0, allVectorsRavv.size()).toArray();
+            OnHeapGraphIndex reconstructedAllNodeOnHeapGraphIndex = GraphIndexBuilder.buildAndMergeNewNodes(onDiskGraph, neighborsScoreCache, allVectorsRavv, allBuildScoreProvider, numBaseVectors, graphToRavvOrdMap, beamWidth, neighborOverflow, alpha, addHierarchy);
 
             try (GraphSearcher reconstructedAllGraphSearcher = new GraphSearcher(reconstructedAllNodeOnHeapGraphIndex);
                  GraphSearcher allGraphSearcher = new GraphSearcher(allGraphIndex)) {