Skip to content

Commit a83a8a7

Browse files
forgespectrayansbharatjoshi
forgespectrayan
and
sbharatjoshi
authored
feat(memory): ADR-0004 Phase 2 — Bundle migration of CHECKPOINT, entity names, and BM25 (#508) (#511)
* feat(memory): add CHECKPOINT region and bundle-aware metadata persistence (#508) Phase 2a of ADR-0004 bundle migration: - Add CHECKPOINT (23) RegionSizeSpec to CognitiveCortexBuilder (128KB, growable) - Add bundle-aware writeCheckpointMeta in CheckpointDaemon via MemorySegment - Add checkpointRegion support to CoActivationRecordMemory for sidecar elimination - Retain V3 standalone file fallback for backward compatibility Co-authored-by: Bharat Joshi <bharatjoshi@spectrayan.com> * feat(memory): migrate entity name index to ENTITY_NAMES bundle region (#508) Phase 2b of ADR-0004 bundle migration: - Add saveNameIndexToRegion/loadNameIndexFromRegion to EntityDirectorySerializer - Modify EntityDirectory.save() to write name index to ENTITY_NAMES region after adjacency data - Modify EntityDirectory.fromBundle() to load name index from region, fallback to sidecar - Increase ENTITY_NAMES region size (+32B/entity) and set growable=true Co-authored-by: Bharat Joshi <bharatjoshi@spectrayan.com> * feat(memory): migrate BM25 inverted index to bundle region (#508) Phase 2c of ADR-0004 bundle migration (Option A: fixed-capacity + growable): - Add BM25 (22) RegionSizeSpec sized dynamically from workingCapacity - Add saveToRegion/loadFromRegion to BM25Index for MemorySegment I/O - Modify RetrievalIndexBuilder to load BM25 from bundle region first, file fallback - Add BM25 bundle region save on DefaultSpectorMemory close - Retain V3 bm25.bidx standalone file for backward compatibility Co-authored-by: Bharat Joshi <bharatjoshi@spectrayan.com> --------- Co-authored-by: Bharat Joshi <bharatjoshi@spectrayan.com>
1 parent c8b53ed commit a83a8a7

8 files changed

Lines changed: 522 additions & 37 deletions

File tree

‎memory/spector-index/src/main/java/com/spectrayan/spector/index/text/BM25Index.java‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -667,4 +667,161 @@ public static BM25Index load(java.nio.file.Path filePath) {
667667
return null;
668668
}
669669
}
670+
671+
// ── V4 Bundle Region I/O ──
672+
673+
/**
674+
* Saves this BM25 index to a V4 bundle region MemorySegment.
675+
*
676+
* <p>Uses the same binary format as {@link #save(java.nio.file.Path)} but writes
677+
* to a memory-mapped region instead of a file. The first 4 bytes store the
678+
* total payload length, followed by the binary index data.</p>
679+
*
680+
* @param region the BM25 region MemorySegment
681+
* @return number of bytes written, or -1 if region too small
682+
*/
683+
public int saveToRegion(java.lang.foreign.MemorySegment region) {
684+
rwLock.readLock().lock();
685+
try {
686+
java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream();
687+
java.io.DataOutputStream dos = new java.io.DataOutputStream(baos);
688+
689+
// Header
690+
dos.writeInt(MAGIC);
691+
dos.writeInt(FORMAT_VERSION);
692+
dos.writeInt(totalDocs);
693+
dos.writeInt(invertedIndex.size());
694+
dos.writeLong(totalDocLength);
695+
696+
// DocIds
697+
int docCount = docIds.size();
698+
dos.writeInt(docCount);
699+
for (String id : docIds) {
700+
byte[] idBytes = id.getBytes(java.nio.charset.StandardCharsets.UTF_8);
701+
dos.writeInt(idBytes.length);
702+
dos.write(idBytes);
703+
}
704+
705+
// DocLengths
706+
dos.writeInt(docCount);
707+
for (int i = 0; i < docCount; i++) {
708+
dos.writeInt(docLengthsArray[i]);
709+
}
710+
711+
// Terms + Postings
712+
dos.writeInt(invertedIndex.size());
713+
for (var entry : invertedIndex.entrySet()) {
714+
byte[] termBytes = entry.getKey().getBytes(java.nio.charset.StandardCharsets.UTF_8);
715+
PostingList pl = entry.getValue();
716+
dos.writeInt(termBytes.length);
717+
dos.write(termBytes);
718+
dos.writeInt(pl.size);
719+
for (int i = 0; i < pl.size; i++) {
720+
dos.writeInt(pl.docIndices[i]);
721+
dos.writeInt(pl.termFrequencies[i]);
722+
}
723+
}
724+
725+
dos.flush();
726+
byte[] data = baos.toByteArray();
727+
int totalBytes = 4 + data.length; // 4-byte length prefix + payload
728+
729+
if (totalBytes > region.byteSize()) {
730+
log.warn("BM25 index ({} bytes) exceeds region capacity ({}B)",
731+
totalBytes, region.byteSize());
732+
return -1;
733+
}
734+
735+
// Write length prefix then payload
736+
region.set(java.lang.foreign.ValueLayout.JAVA_INT, 0, data.length);
737+
java.lang.foreign.MemorySegment.copy(
738+
java.lang.foreign.MemorySegment.ofArray(data), 0,
739+
region, 4, data.length);
740+
741+
log.info("BM25 index saved to bundle region: {} docs, {} terms, {} bytes",
742+
totalDocs, invertedIndex.size(), totalBytes);
743+
return totalBytes;
744+
745+
} catch (java.io.IOException e) {
746+
log.error("Failed to save BM25 index to bundle region", e);
747+
return -1;
748+
} finally {
749+
rwLock.readLock().unlock();
750+
}
751+
}
752+
753+
/**
754+
* Loads a BM25 index from a V4 bundle region MemorySegment.
755+
*
756+
* @param region the BM25 region MemorySegment
757+
* @return the loaded BM25Index, or null if no valid data present
758+
*/
759+
public static BM25Index loadFromRegion(java.lang.foreign.MemorySegment region) {
760+
if (region == null || region.byteSize() < 28) return null; // 4B len + 24B min header
761+
762+
int payloadLen = region.get(java.lang.foreign.ValueLayout.JAVA_INT, 0);
763+
if (payloadLen <= 0 || 4 + payloadLen > region.byteSize()) return null;
764+
765+
byte[] data = new byte[payloadLen];
766+
java.lang.foreign.MemorySegment.copy(region, 4,
767+
java.lang.foreign.MemorySegment.ofArray(data), 0, payloadLen);
768+
769+
java.nio.ByteBuffer buf = java.nio.ByteBuffer.wrap(data);
770+
771+
// Header
772+
int magic = buf.getInt();
773+
if (magic != MAGIC) return null;
774+
int version = buf.getInt();
775+
if (version != FORMAT_VERSION) return null;
776+
int savedTotalDocs = buf.getInt();
777+
int termCount = buf.getInt();
778+
long savedTotalDocLength = buf.getLong();
779+
780+
BM25Index idx = new BM25Index();
781+
782+
// DocIds
783+
int docCount = buf.getInt();
784+
for (int i = 0; i < docCount; i++) {
785+
int idLen = buf.getInt();
786+
byte[] idBytes = new byte[idLen];
787+
buf.get(idBytes);
788+
String id = new String(idBytes, java.nio.charset.StandardCharsets.UTF_8);
789+
idx.docIds.add(id);
790+
idx.docIdToIndex.put(id, i);
791+
}
792+
793+
// DocLengths
794+
int docLenCount = buf.getInt();
795+
if (docLenCount > idx.docLengthsCapacity) {
796+
idx.docLengthsCapacity = docLenCount;
797+
idx.docLengthsArray = new int[docLenCount];
798+
}
799+
for (int i = 0; i < docLenCount; i++) {
800+
idx.docLengthsArray[i] = buf.getInt();
801+
}
802+
803+
// Terms + Postings
804+
int savedTermCount = buf.getInt();
805+
for (int t = 0; t < savedTermCount; t++) {
806+
int termLen = buf.getInt();
807+
byte[] termBytes = new byte[termLen];
808+
buf.get(termBytes);
809+
String term = new String(termBytes, java.nio.charset.StandardCharsets.UTF_8);
810+
811+
int postingsSize = buf.getInt();
812+
PostingList pl = new PostingList(Math.max(postingsSize, 16));
813+
for (int p = 0; p < postingsSize; p++) {
814+
pl.add(buf.getInt(), buf.getInt());
815+
}
816+
idx.invertedIndex.put(term, pl);
817+
}
818+
819+
idx.totalDocs = savedTotalDocs;
820+
idx.totalDocLength = savedTotalDocLength;
821+
idx.avgDocLength = savedTotalDocs > 0 ? (double) savedTotalDocLength / savedTotalDocs : 0;
822+
823+
log.info("BM25 index loaded from bundle region: {} docs, {} terms, {} bytes",
824+
savedTotalDocs, savedTermCount, payloadLen + 4);
825+
return idx;
826+
}
670827
}

‎memory/spector-memory/src/main/java/com/spectrayan/spector/memory/CognitiveCortexBuilder.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,9 @@ private static List<RegionSizeSpec> getRuntimeBundleSpecs(SpectorMemoryBuilder b
309309
long typeRegistrySize = builder.typeRegistrySize;
310310
long insulaSize = builder.insulaSize;
311311

312+
// BM25 region sizing: header(24) + docIds(~48B/doc) + docLengths(4B/doc) + terms+postings(~80B/doc)
313+
long bm25InitialSize = Math.max(64 * 1024, 24 + 132L * workingCap);
314+
312315
return List.of(
313316
new RegionSizeSpec(
314317
RegionId.WORKING,
@@ -384,12 +387,12 @@ private static List<RegionSizeSpec> getRuntimeBundleSpecs(SpectorMemoryBuilder b
384387
),
385388
new RegionSizeSpec(
386389
RegionId.ENTITY_NAMES,
387-
64 + 16 + 8L * hyperCap * 16,
390+
64 + 16 + 8L * hyperCap * 16 + 32L * hyperCap, // adjacency + name index space
388391
1,
389392
8,
390393
new com.spectrayan.spector.memory.kernel.layout.EntityDirectoryLayout().layoutId(),
391394
new com.spectrayan.spector.memory.kernel.layout.EntityDirectoryLayout().schemaVersion(),
392-
false
395+
true // growable — name index may exceed initial allocation
393396
),
394397
new RegionSizeSpec(
395398
RegionId.HYPERGRAPH,
@@ -426,6 +429,24 @@ private static List<RegionSizeSpec> getRuntimeBundleSpecs(SpectorMemoryBuilder b
426429
InsularLayout.LAYOUT_ID,
427430
InsularLayout.SCHEMA_VERSION,
428431
false
432+
),
433+
new RegionSizeSpec(
434+
RegionId.CHECKPOINT,
435+
128L * 1024,
436+
1,
437+
0,
438+
0x434B5054,
439+
1,
440+
true
441+
),
442+
new RegionSizeSpec(
443+
RegionId.BM25,
444+
bm25InitialSize,
445+
1,
446+
0,
447+
0x42494458, // "BIDX" magic
448+
1,
449+
true // growable — term/posting lists grow dynamically
429450
)
430451
);
431452
}

‎memory/spector-memory/src/main/java/com/spectrayan/spector/memory/DefaultSpectorMemory.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,6 +1429,18 @@ private void doClose() {
14291429
} catch (Exception e) {
14301430
log.warn("Failed to save BM25 index on close: {}", e.getMessage());
14311431
}
1432+
// V4 bundle: save to BM25 region
1433+
if (runtimeBundle != null) {
1434+
try {
1435+
java.lang.foreign.MemorySegment bm25Region = runtimeBundle.regionSegment(
1436+
com.spectrayan.spector.memory.kernel.bundle.RegionId.BM25);
1437+
if (bm25Region != null) {
1438+
bm25Index.partition(0).saveToRegion(bm25Region);
1439+
}
1440+
} catch (Exception e) {
1441+
log.debug("BM25 bundle region save on close failed: {}", e.getMessage());
1442+
}
1443+
}
14321444
}
14331445

14341446
PersistenceManager.flushAndClose(

‎memory/spector-memory/src/main/java/com/spectrayan/spector/memory/RetrievalIndexBuilder.java‎

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,42 @@ static RetrievalIndices build(SpectorMemoryBuilder builder,
6767
textDataStore.readAll();
6868
index.setTextDataStore(textDataStore);
6969

70-
java.nio.file.Path bm25Path = StorageLayout.bm25BidxRuntime(basePath);
71-
java.nio.file.Path v2Bm25 = resolvedPartitionDir != null ? resolvedPartitionDir.resolve(StorageLayout.FILE_BM25) : null;
72-
java.nio.file.Path loadFrom = MigrationPathResolver.getNewerPath(bm25Path, v2Bm25, null);
73-
if (loadFrom != null) {
74-
bm25Path = loadFrom;
70+
// V4 bundle path: try loading from BM25 region first
71+
BM25Index loadedBm25 = null;
72+
boolean usedBundleRegion = false;
73+
if (cortex.useBundleMode() && cortex.runtimeBundle() != null) {
74+
try {
75+
java.lang.foreign.MemorySegment bm25Region = cortex.runtimeBundle().regionSegment(
76+
com.spectrayan.spector.memory.kernel.bundle.RegionId.BM25);
77+
if (bm25Region != null) {
78+
loadedBm25 = BM25Index.loadFromRegion(bm25Region);
79+
if (loadedBm25 != null) {
80+
usedBundleRegion = true;
81+
log.info("BM25 loaded from bundle region: {} docs", loadedBm25.size());
82+
}
83+
}
84+
} catch (Exception e) {
85+
log.debug("BM25 bundle region load failed, falling back to file: {}", e.getMessage());
86+
}
87+
}
88+
89+
// V3 fallback: load from bm25.bidx file
90+
if (loadedBm25 == null) {
91+
java.nio.file.Path bm25Path = StorageLayout.bm25BidxRuntime(basePath);
92+
java.nio.file.Path v2Bm25 = resolvedPartitionDir != null ? resolvedPartitionDir.resolve(StorageLayout.FILE_BM25) : null;
93+
java.nio.file.Path loadFrom = MigrationPathResolver.getNewerPath(bm25Path, v2Bm25, null);
94+
if (loadFrom != null) {
95+
bm25Path = loadFrom;
96+
}
97+
loadedBm25 = BM25Index.load(bm25Path);
98+
if (loadedBm25 != null) {
99+
log.info("BM25 loaded from binary index: {} docs", loadedBm25.size());
100+
}
75101
}
76-
BM25Index loadedBm25 = BM25Index.load(bm25Path);
102+
77103
bm25Index = new MemoryBM25Index(1);
78104
if (loadedBm25 != null) {
79105
bm25Index.setPartition(0, loadedBm25);
80-
log.info("BM25 loaded from binary index: {} docs", loadedBm25.size());
81106
} else {
82107
Map<String, String> allTexts = new java.util.HashMap<>();
83108
for (var entry : index.locationMap().entrySet()) {
@@ -89,7 +114,20 @@ static RetrievalIndices build(SpectorMemoryBuilder builder,
89114
if (!allTexts.isEmpty()) {
90115
bm25Index.rebuildPartition(0, allTexts);
91116
log.info("Rebuilt BM25 index with {} documents from memory index", allTexts.size());
117+
// Save to file (V3 compat) and bundle region (V4)
118+
java.nio.file.Path bm25Path = StorageLayout.bm25BidxRuntime(basePath);
92119
bm25Index.partition(0).save(bm25Path);
120+
if (cortex.useBundleMode() && cortex.runtimeBundle() != null) {
121+
try {
122+
java.lang.foreign.MemorySegment bm25Region = cortex.runtimeBundle().regionSegment(
123+
com.spectrayan.spector.memory.kernel.bundle.RegionId.BM25);
124+
if (bm25Region != null) {
125+
bm25Index.partition(0).saveToRegion(bm25Region);
126+
}
127+
} catch (Exception e) {
128+
log.debug("BM25 bundle region save failed: {}", e.getMessage());
129+
}
130+
}
93131
}
94132
}
95133
} else {

‎memory/spector-memory/src/main/java/com/spectrayan/spector/memory/graph/EntityDirectory.java‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,15 +199,24 @@ private EntityDirectory(Arena arena, MemorySegment entityRegionSlice, MemorySegm
199199
this.adjHighWaterMark = 0;
200200
}
201201

202-
// Load names from sidecar
202+
// Load names: try V4 bundle region first, then V3 sidecar
203203
if (!isNew && bundlePath != null) {
204204
try {
205-
ConcurrentHashMap<String, Integer> names = EntityDirectorySerializer.loadNameIndexSidecar(bundlePath, null);
206-
if (names != null) {
205+
long nameIndexOffset = MemoryHeader.HEADER_BYTES + 16
206+
+ (long) adjSegmentCapacity * ADJ_ENTRY_BYTES;
207+
ConcurrentHashMap<String, Integer> names = EntityDirectorySerializer.loadNameIndexFromRegion(
208+
adjacencyRegionSlice, nameIndexOffset);
209+
if (names != null && !names.isEmpty()) {
207210
this.nameIndex.putAll(names);
211+
} else {
212+
// V3 fallback: load from sidecar file
213+
names = EntityDirectorySerializer.loadNameIndexSidecar(bundlePath, null);
214+
if (names != null) {
215+
this.nameIndex.putAll(names);
216+
}
208217
}
209218
} catch (Exception e) {
210-
log.warn("Failed to load EntityDirectory name index sidecar: {}", e.getMessage());
219+
log.warn("Failed to load EntityDirectory name index: {}", e.getMessage());
211220
}
212221
}
213222

@@ -1125,7 +1134,15 @@ public void save(Path filePath, DataEncryptor encryptor) {
11251134

11261135
Path path = filePath != null ? filePath : mmapFilePath;
11271136
if (path != null) {
1128-
EntityDirectorySerializer.saveNameIndexSidecar(this, path, encryptor);
1137+
// V4 bundle path: write name index to ENTITY_NAMES region after adjacency data
1138+
long nameIndexOffset = MemoryHeader.HEADER_BYTES + 16
1139+
+ (long) adjSegmentCapacity * ADJ_ENTRY_BYTES;
1140+
int written = EntityDirectorySerializer.saveNameIndexToRegion(
1141+
adjacencySegment, nameIndexOffset, nameIndex);
1142+
if (written < 0) {
1143+
// Fallback to sidecar file if region too small
1144+
EntityDirectorySerializer.saveNameIndexSidecar(this, path, encryptor);
1145+
}
11291146
}
11301147
} finally {
11311148
lock.unlockRead(stamp);

0 commit comments

Comments
 (0)