public PtrSize Get(long id)
{
var readResult = _parent.Read(_nestedValueKey);
if (readResult == null)
{
throw new InvalidOperationException($"Could not find a map result wit id '{id}' within a nested values section stored under '{_nestedValueKey}' key");
}
var reader = readResult.Reader;
var entry = (ResultHeader *)reader.Base;
var end = reader.Base + reader.Length;
while (entry < end)
{
if (entry->Id == id)
{
return(PtrSize.Create((byte *)entry + sizeof(ResultHeader), entry->Size));
}
entry = (ResultHeader *)((byte *)entry + sizeof(ResultHeader) + entry->Size);
}
throw new InvalidOperationException($"Could not find a map result wit id '{id}' within a nested values section stored under '{_nestedValueKey}' key");
}
private void Add(ref PtrSize ptr)
{
#if DEBUG
if (ptr.Size < 0)
ThrowSizeCannotBeNegative(nameof(ptr.Size));
#endif
_values.Add(ptr);
_size += ptr.Size;
_isDirty = true;
}
public unsafe void Can_seek_and_read_from_chunked_mmap_file(int totalSize, int chunkSize, int seed)
{
var random = new Random(seed);
var buffer = new byte[totalSize];
random.NextBytes(buffer);
fixed(byte *ptr = buffer)
{
var ptrSize = new List <PtrSize>();
var numberOfBuffers = totalSize / chunkSize + (totalSize % chunkSize != 0 ? 1 : 0);
for (int i = 0; i < numberOfBuffers; i++)
{
ptrSize.Add(PtrSize.Create(
ptr + (i * chunkSize),
i == numberOfBuffers - 1 ? totalSize % chunkSize : chunkSize));
}
var sut = new ChunkedMmapStream(ptrSize.ToArray(), chunkSize);
// read all bytes
var allBytes = sut.ReadEntireBlock(totalSize);
Assert.Equal(buffer, allBytes);
// seek to beginning, read to end
sut.Position = 0;
allBytes = sut.ReadData();
Assert.Equal(buffer, allBytes);
// read all bytes one by one
sut.Position = 0;
for (int i = 0; i < allBytes.Length; i++)
{
Assert.Equal(allBytes[i], sut.ReadByte());
}
// random seeks
for (int i = 0; i < 1000; i++)
{
var pos = random.Next(0, totalSize - 1);
sut.Position = pos;
var result = new byte[Math.Min(totalSize - pos, random.Next(1, totalSize))];
sut.ReadEntireBlock(result, 0, result.Length);
Assert.Equal(buffer.Skip(pos).Take(result.Length).ToArray(), result);
}
}
}
private void Add(PtrSize ptr)
{
if (ptr.Size < 0)
{
throw new ArgumentException("Size cannot be negative", nameof(ptr.Size));
}
_values.Add(ptr);
_size += ptr.Size;
if (_size + _values.Count + 1 > byte.MaxValue)
{
_elementSize = 2;
}
if (_size + _values.Count * 2 + 1 > ushort.MaxValue)
{
_elementSize = 4;
}
}
private void OpenInternal()
{
var fileTree = _currentTransaction.Value.ReadTree(_name);
if (fileTree == null)
{
throw new FileNotFoundException("Could not find index input", _name);
}
var numberOfChunks = fileTree.State.NumberOfEntries;
_ptrs = new PtrSize[numberOfChunks];
int index = 0;
using (var it = fileTree.Iterate(prefetch: false))
{
if (it.Seek(Slices.BeforeAllKeys) == false)
{
throw new InvalidDataException("Could not seek to any chunk of this file");
}
do
{
var readResult = fileTree.Read(it.CurrentKey);
_ptrs[index] = PtrSize.Create(readResult.Reader.Base, readResult.Reader.Length);
index++;
} while (it.MoveNext());
}
if (numberOfChunks != index)
{
throw new InvalidDataException($"Read invalid number of file chunks. Expected {numberOfChunks}, read {index}.");
}
_stream = new ChunkedMmapStream(_ptrs, VoronIndexOutput.MaxFileChunkSize);
}
public ReadMapEntryScope Get(long id)
{
id = Bits.SwapBytes(id);
switch (Type)
{
case MapResultsStorageType.Tree:
Slice entrySlice;
using (Slice.External(_indexContext.Allocator, (byte *)&id, sizeof(long), out entrySlice))
{
if (Tree.IsLeafCompressionSupported)
{
var read = Tree.ReadDecompressed(entrySlice);
if (read == null)
{
throw new InvalidOperationException($"Could not find a map result with id '{id}' in '{Tree.Name}' tree");
}
return(new ReadMapEntryScope(read));
}
else
{
var read = Tree.Read(entrySlice);
if (read == null)
{
throw new InvalidOperationException($"Could not find a map result with id '{id}' in '{Tree.Name}' tree");
}
return(new ReadMapEntryScope(PtrSize.Create(read.Reader.Base, read.Reader.Length)));
}
}
case MapResultsStorageType.Nested:
var section = GetNestedResultsSection();
return(new ReadMapEntryScope(section.Get(id)));
default:
throw new ArgumentOutOfRangeException(Type.ToString());
}
}
private static unsafe bool ResultsBinaryEqual(BlittableJsonReaderObject newResult, PtrSize existingData)
{
return(newResult.Size == existingData.Size &&
Memory.CompareInline(newResult.BasePointer, existingData.Ptr, existingData.Size) == 0);
}
public void Add <T>(T value) where T : struct
{
PtrSize ptr = PtrSize.Create(value);
Add(ref ptr);
}
public void CopyTo(byte *ptr)
{
if (Compressed)
{
Compression.CopyTo(ptr);
return;
}
JsonParserState.WriteVariableSizeInt(ref ptr, _values.Count);
int elementSize = ElementSize;
int pos = _values.Count * elementSize;
byte *dataStart = ptr + pos;
switch (elementSize)
{
case 1:
byte *bytePtr = ptr;
for (int i = 0; i < _values.Count; i++)
{
bytePtr[i] = (byte)pos;
pos += _values[i].Size;
}
break;
case 2:
ushort *shortPtr = (ushort *)ptr;
for (int i = 0; i < _values.Count; i++)
{
shortPtr[i] = (ushort)pos;
pos += _values[i].Size;
}
break;
case 4:
int *intPtr = (int *)ptr;
for (int i = 0; i < _values.Count; i++)
{
intPtr[i] = pos;
pos += _values[i].Size;
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(ElementSize), "Unknown element size " + ElementSize);
}
ulong value; // Do not move inside because we require value to exist inside the for loop.
for (int i = 0; i < _values.Count; i++)
{
PtrSize p = _values[i];
byte *srcPtr;
if (_values[i].IsValue)
{
value = p.Value;
srcPtr = (byte *)&value; // This generates an alias on value
}
else
{
srcPtr = p.Ptr;
}
Memory.Copy(dataStart, srcPtr, p.Size);
dataStart += p.Size;
value = 0; // This ensures there cannot be any JIT optimization that could reuse the memory location.
}
}
public void Add(void *pointer, int size)
{
PtrSize ptr = PtrSize.Create(pointer, size);
Add(ref ptr);
}
public void Add(Slice buffer)
{
PtrSize ptr = PtrSize.Create(buffer.Content.Ptr, buffer.Content.Length);
Add(ref ptr);
}
public ReadMapEntryScope(DecompressedReadResult read)
{
_read = read;
Data = PtrSize.Create(read.Reader.Base, read.Reader.Length);
}
public ReadMapEntryScope(PtrSize data)
{
Data = data;
_read = null;
}
public ReadMapEntryScope(PtrSize data)
{
Data = data;
}
public void Add(void *ptr, int size)
{
Add(PtrSize.Create(ptr, size));
}
public void Add <T>(T value) where T : struct
{
Add(PtrSize.Create(value));
}
