Defer Sorter.DocMap packing until after flush (apache#16048)

* Defer Sorter.DocMap packing until after flush

IndexingChain calls sortMap.oldToNew heavily during flush — per posting
in FreqProxTermsWriter, per (field, doc) in DV writers, per vector in
KNN writers. The previous implementation built oldToNew as a
PackedLongValues, which is slow for that random-access hot path.

Build oldToNew as an int[] during flush, then pack to PackedLongValues
before storing on FlushedSegment, which is retained long-term on
ReadersAndUpdates for sorted segments with seg-private updates.

* Changes

Author: Tim-Brooks

lucene/CHANGES.txt

@@ -379,6 +379,7 @@ Optimizations
 * GITHUB#16061, GITHUB#16070: Improve cost estimation in SortedSetDocValuesRangeQuery when using DocValuesSkipper
   and the field is dense and is the primary sort of the index to reduce the number of doc values visited. (Ignacio Vera)
 
+* GITHUB#16048: Defer Sorter.DocMap packing until after flush (Tim Brooks)
 
 Bug Fixes
 ---------------------

lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java

@@ -481,15 +481,15 @@ FlushedSegment flush(DocumentsWriter.FlushNotifications flushNotifications) thro
           "DWPT",
           "flush postings as segment " + flushState.segmentInfo.name + " numDocs=" + numDocsInRAM);
     }
-    final Sorter.DocMap sortMap;
+    final Sorter.PackableDocMap packableSortMap;
     try {
       DocIdSetIterator softDeletedDocs;
       if (indexWriterConfig.getSoftDeletesField() != null) {
         softDeletedDocs = indexingChain.getHasDocValues(indexWriterConfig.getSoftDeletesField());
       } else {
         softDeletedDocs = null;
       }
-      sortMap = indexingChain.flush(flushState);
+      packableSortMap = indexingChain.flush(flushState);
       if (softDeletedDocs == null) {
         flushState.softDelCountOnFlush = 0;
       } else {
@@ -570,8 +570,10 @@ FlushedSegment flush(DocumentsWriter.FlushNotifications flushNotifications) thro
               segmentDeletes,
               flushState.liveDocs,
               flushState.delCountOnFlush,
-              sortMap);
-      sealFlushedSegment(fs, sortMap, flushNotifications);
+              packableSortMap != null
+                  ? packableSortMap.pack()
+                  : null); // Use a packed version as the lifetime of FlushedSegment is long.
+      sealFlushedSegment(fs, packableSortMap, flushNotifications);
       if (infoStream.isEnabled("DWPT")) {
         infoStream.message(
             "DWPT",

lucene/core/src/java/org/apache/lucene/index/IndexingChain.java

@@ -237,7 +237,7 @@ public FieldInfos getFieldInfos() {
     };
   }
 
-  private Sorter.DocMap maybeSortSegment(SegmentWriteState state) throws IOException {
+  private Sorter.PackableDocMap maybeSortSegment(SegmentWriteState state) throws IOException {
     Sort indexSort = state.segmentInfo.getIndexSort();
     if (indexSort == null) {
       return null;
@@ -283,15 +283,15 @@ private Sorter.DocMap maybeSortSegment(SegmentWriteState state) throws IOExcepti
     }
     Sorter sorter = new Sorter(indexSort);
     // returns null if the documents are already sorted
-    return sorter.sort(
+    return sorter.sortAndLeaveUnpacked(
         state.segmentInfo.maxDoc(), comparators.toArray(IndexSorter.DocComparator[]::new));
   }
 
-  Sorter.DocMap flush(SegmentWriteState state) throws IOException {
+  Sorter.PackableDocMap flush(SegmentWriteState state) throws IOException {
 
     // NOTE: caller (DocumentsWriterPerThread) handles
     // aborting on any exception from this method
-    Sorter.DocMap sortMap = maybeSortSegment(state);
+    Sorter.PackableDocMap sortMap = maybeSortSegment(state);
     int maxDoc = state.segmentInfo.maxDoc();
     long t0 = System.nanoTime();
     writeNorms(state, sortMap);

lucene/core/src/java/org/apache/lucene/index/Sorter.java

@@ -129,7 +129,7 @@ protected int compareSaved(int i, int j) {
   }
 
   /** Computes the old-to-new permutation over the given comparator. */
-  private static Sorter.DocMap sort(final int maxDoc, IndexSorter.DocComparator comparator) {
+  private static PackableDocMap sort(final int maxDoc, IndexSorter.DocComparator comparator) {
     // check if the index is sorted
     boolean sorted = true;
     for (int i = 1; i < maxDoc; ++i) {
@@ -168,30 +168,7 @@ private static Sorter.DocMap sort(final int maxDoc, IndexSorter.DocComparator co
       docs[(int) newToOld.get(i)] = i;
     } // docs is now the oldToNew mapping
 
-    final PackedLongValues.Builder oldToNewBuilder =
-        PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
-    for (int i = 0; i < maxDoc; ++i) {
-      oldToNewBuilder.add(docs[i]);
-    }
-    final PackedLongValues oldToNew = oldToNewBuilder.build();
-
-    return new Sorter.DocMap() {
-
-      @Override
-      public int oldToNew(int docID) {
-        return (int) oldToNew.get(docID);
-      }
-
-      @Override
-      public int newToOld(int docID) {
-        return (int) newToOld.get(docID);
-      }
-
-      @Override
-      public int size() {
-        return maxDoc;
-      }
-    };
+    return new PackableDocMap(docs, newToOld, maxDoc);
   }
 
   /**
@@ -241,6 +218,12 @@ DocMap sort(LeafReader reader) throws IOException {
   }
 
   DocMap sort(int maxDoc, IndexSorter.DocComparator[] comparators) throws IOException {
+    PackableDocMap packableDocMap = sortAndLeaveUnpacked(maxDoc, comparators);
+    return packableDocMap != null ? packableDocMap.pack() : packableDocMap;
+  }
+
+  PackableDocMap sortAndLeaveUnpacked(int maxDoc, IndexSorter.DocComparator[] comparators)
+      throws IOException {
     final IndexSorter.DocComparator comparator =
         (docID1, docID2) -> {
           for (int i = 0; i < comparators.length; i++) {
@@ -270,4 +253,60 @@ public String getID() {
   public String toString() {
     return getID();
   }
+
+  /** A {@link DocMap} that can keep oldToNew in either packed or unpacked form. */
+  static final class PackableDocMap extends DocMap {
+
+    private final PackedLongValues newToOld;
+    private final int maxDoc;
+
+    private final int[] oldToNewUnpacked;
+    private final PackedLongValues oldToNewPacked;
+
+    private PackableDocMap(int[] oldToNewUnpacked, PackedLongValues newToOld, int maxDoc) {
+      this.oldToNewUnpacked = oldToNewUnpacked;
+      this.oldToNewPacked = null;
+      this.newToOld = newToOld;
+      this.maxDoc = maxDoc;
+    }
+
+    private PackableDocMap(PackedLongValues oldToNewPacked, PackedLongValues newToOld, int maxDoc) {
+      this.oldToNewUnpacked = null;
+      this.oldToNewPacked = oldToNewPacked;
+      this.newToOld = newToOld;
+      this.maxDoc = maxDoc;
+    }
+
+    @Override
+    public int oldToNew(int docID) {
+      if (oldToNewUnpacked != null) {
+        return oldToNewUnpacked[docID];
+      } else {
+        return (int) oldToNewPacked.get(docID);
+      }
+    }
+
+    @Override
+    public int newToOld(int docID) {
+      return (int) newToOld.get(docID);
+    }
+
+    /** Pack the internal representations into more compact forms. No-op if already packed. */
+    DocMap pack() {
+      if (oldToNewPacked != null) {
+        return this;
+      }
+      final PackedLongValues.Builder oldToNewBuilder =
+          PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
+      for (int i = 0; i < maxDoc; ++i) {
+        oldToNewBuilder.add(oldToNewUnpacked[i]);
+      }
+      return new PackableDocMap(oldToNewBuilder.build(), newToOld, maxDoc);
+    }
+
+    @Override
+    public int size() {
+      return maxDoc;
+    }
+  }
 }