Skip to content

Commit 47d62b2

Browse files
CopilotPrashant Chinnam
authored andcommitted
Roaring Bitmaps as a Garnet module (issue #1270)
Addresses PR review feedback from @badrishc: - Move the extension from main/GarnetServer/Extensions/RoaringBitmap to modules/RoaringBitmap so it isn't bundled by default (mirrors GarnetJSON). - Retarget the PR to dev (companion change). Implementation changes for the move: - New modules/RoaringBitmap/GarnetRoaringBitmap.csproj (mirrors GarnetJSON.csproj, signs assembly, exposes InternalsVisibleTo Garnet.test). - New RoaringBitmapModule : ModuleBase entry point that registers the factory and the four R.SETBIT/R.GETBIT/R.BITCOUNT/R.BITPOS commands. - Renamed namespace Garnet.Extensions.RoaringBitmap -> GarnetRoaringBitmap to avoid the namespace/class collision with class RoaringBitmap. - Updated CustomObjectFunctions overrides to dev-branch scoped ReadOnlySpan<byte> signatures for NeedInitialUpdate / Updater. - Updated RoaringBitmapObject to dev-branch CustomObjectBase ctor and HeapMemorySize accounting. - Wired the module into Garnet.slnx and Garnet.test.csproj. - Tests still register via server.Register.NewCommand in [SetUp] (in-process), matching the existing custom-object test pattern. - Updated StringKeyAndCustomObjectKey_AreSeparate to expect WRONGTYPE on the unified store on dev.
1 parent 70b8da6 commit 47d62b2

13 files changed

Lines changed: 2188 additions & 0 deletions

Garnet.slnx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
</Folder>
4444
<Folder Name="/modules/">
4545
<Project Path="modules/GarnetJSON/GarnetJSON.csproj" />
46+
<Project Path="modules/RoaringBitmap/GarnetRoaringBitmap.csproj" />
4647
</Folder>
4748
<Folder Name="/playground/">
4849
<Project Path="modules/NoOpModule/NoOpModule.csproj" />
Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT license.
3+
4+
using System.Numerics;
5+
using System.Runtime.InteropServices;
6+
7+
namespace GarnetRoaringBitmap.Containers
8+
{
9+
/// <summary>
10+
/// Sorted-array container: stores up to <see cref="ArrayThreshold"/> 16-bit values in
11+
/// ascending order. Uses ~2 bytes per element; preferred representation when the
12+
/// container's cardinality is below the threshold because membership is O(log n)
13+
/// and the memory footprint scales with cardinality. Above the threshold the parent
14+
/// promotes the container to a <see cref="BitmapContainer"/> (constant 8 KiB).
15+
///
16+
/// Serialization layout (body only — caller writes kind+cardinality):
17+
/// ushort[cardinality] values (little-endian per BinaryWriter contract)
18+
/// </summary>
19+
internal sealed class ArrayContainer : IContainer
20+
{
21+
/// <summary>
22+
/// Crossover point at which an array-encoded container should be promoted to a bitmap.
23+
/// 4096 is the canonical Roaring threshold: at 4096 elements an array uses 8 KiB
24+
/// (same as a bitmap container) but membership is still O(log n); past 4096 the
25+
/// bitmap becomes strictly cheaper per element.
26+
/// </summary>
27+
public const int ArrayThreshold = 4096;
28+
29+
// Initial capacity is intentionally small to keep tiny containers cheap;
30+
// capacity grows geometrically up to ArrayThreshold via power-of-two rounding.
31+
private const int InitialCapacity = 4;
32+
33+
private ushort[] values;
34+
private int count;
35+
36+
public ArrayContainer()
37+
{
38+
values = new ushort[InitialCapacity];
39+
count = 0;
40+
}
41+
42+
public ArrayContainer(ushort[] sortedUnique, int count)
43+
{
44+
ArgumentNullException.ThrowIfNull(sortedUnique);
45+
if ((uint)count > (uint)sortedUnique.Length) throw new ArgumentOutOfRangeException(nameof(count));
46+
this.values = sortedUnique;
47+
this.count = count;
48+
}
49+
50+
public ContainerKind Kind => ContainerKind.Array;
51+
public int Cardinality => count;
52+
public long ByteSize => 16 + 2L * values.Length; // array header ~16B + 2B per slot
53+
54+
public bool Contains(ushort value)
55+
{
56+
return values.AsSpan(0, count).BinarySearch(value) >= 0;
57+
}
58+
59+
public IContainer Add(ushort value, out bool added)
60+
{
61+
var idx = values.AsSpan(0, count).BinarySearch(value);
62+
if (idx >= 0)
63+
{
64+
added = false;
65+
return this;
66+
}
67+
68+
var insert = ~idx;
69+
// Promote to bitmap before exceeding the threshold to avoid one wasteful array grow.
70+
if (count >= ArrayThreshold)
71+
{
72+
var bitmap = ToBitmap();
73+
bitmap.SetUnchecked(value);
74+
added = true;
75+
return bitmap;
76+
}
77+
78+
EnsureCapacity(count + 1);
79+
if (insert < count)
80+
{
81+
Array.Copy(values, insert, values, insert + 1, count - insert);
82+
}
83+
values[insert] = value;
84+
count++;
85+
added = true;
86+
return this;
87+
}
88+
89+
public IContainer Remove(ushort value, out bool removed)
90+
{
91+
var idx = values.AsSpan(0, count).BinarySearch(value);
92+
if (idx < 0)
93+
{
94+
removed = false;
95+
return this;
96+
}
97+
98+
if (idx < count - 1)
99+
{
100+
Array.Copy(values, idx + 1, values, idx, count - idx - 1);
101+
}
102+
count--;
103+
removed = true;
104+
return count == 0 ? null : this;
105+
}
106+
107+
public ushort First()
108+
{
109+
if (count == 0) throw new InvalidOperationException("empty container");
110+
return values[0];
111+
}
112+
113+
public ushort Last()
114+
{
115+
if (count == 0) throw new InvalidOperationException("empty container");
116+
return values[count - 1];
117+
}
118+
119+
public int NextSetBit(int from)
120+
{
121+
// Find first values[i] >= from
122+
var lo = 0;
123+
var hi = count;
124+
while (lo < hi)
125+
{
126+
var mid = (lo + hi) >>> 1;
127+
if (values[mid] < from) lo = mid + 1; else hi = mid;
128+
}
129+
return lo == count ? -1 : values[lo];
130+
}
131+
132+
public int NextUnsetBit(int from)
133+
{
134+
if (from < 0 || from > 65535) return -1;
135+
// Find first values[i] >= from. Then walk forward looking for a gap.
136+
var lo = 0;
137+
var hi = count;
138+
while (lo < hi)
139+
{
140+
var mid = (lo + hi) >>> 1;
141+
if (values[mid] < from) lo = mid + 1; else hi = mid;
142+
}
143+
// Walk from `from`. If values[lo] != from, then `from` itself is unset.
144+
var candidate = from;
145+
while (lo < count && values[lo] == candidate)
146+
{
147+
if (candidate == 65535) return -1;
148+
candidate++;
149+
lo++;
150+
}
151+
return candidate <= 65535 ? candidate : -1;
152+
}
153+
154+
public IContainer Clone()
155+
{
156+
var copy = new ushort[count]; // tight copy — clones don't preallocate growth
157+
Array.Copy(values, copy, count);
158+
return new ArrayContainer(copy, count);
159+
}
160+
161+
public void SerializeBody(BinaryWriter writer)
162+
{
163+
writer.Write(MemoryMarshal.AsBytes<ushort>(values.AsSpan(0, count)));
164+
}
165+
166+
public static ArrayContainer DeserializeBody(BinaryReader reader, int cardinality)
167+
{
168+
if (cardinality < 1 || cardinality > ArrayThreshold)
169+
throw new InvalidDataException($"ArrayContainer cardinality out of range: {cardinality}");
170+
var arr = new ushort[cardinality];
171+
reader.BaseStream.ReadExactly(MemoryMarshal.AsBytes(arr.AsSpan()));
172+
for (var i = 1; i < cardinality; i++)
173+
if (arr[i] <= arr[i - 1])
174+
throw new InvalidDataException("ArrayContainer values must be strictly ascending");
175+
return new ArrayContainer(arr, cardinality);
176+
}
177+
178+
private void EnsureCapacity(int required)
179+
{
180+
if (required <= values.Length) return;
181+
var newCap = Math.Min(ArrayThreshold, (int)BitOperations.RoundUpToPowerOf2((uint)required));
182+
Array.Resize(ref values, newCap);
183+
}
184+
185+
public BitmapContainer ToBitmap()
186+
{
187+
var bitmap = new BitmapContainer();
188+
for (var i = 0; i < count; i++)
189+
bitmap.SetUnchecked(values[i]);
190+
return bitmap;
191+
}
192+
193+
// Accessor for tests and RunContainer conversion.
194+
internal ushort GetAt(int index) => values[index];
195+
}
196+
}

0 commit comments

Comments
 (0)