public PtrSize Get(long id) { var readResult = _parent.Read(_nestedValueKey); if (readResult == null) { throw new InvalidOperationException($"Could not find a map result wit id '{id}' within a nested values section stored under '{_nestedValueKey}' key"); } var reader = readResult.Reader; var entry = (ResultHeader *)reader.Base; var end = reader.Base + reader.Length; while (entry < end) { if (entry->Id == id) { return(PtrSize.Create((byte *)entry + sizeof(ResultHeader), entry->Size)); } entry = (ResultHeader *)((byte *)entry + sizeof(ResultHeader) + entry->Size); } throw new InvalidOperationException($"Could not find a map result wit id '{id}' within a nested values section stored under '{_nestedValueKey}' key"); }
private void Add(ref PtrSize ptr) { #if DEBUG if (ptr.Size < 0) ThrowSizeCannotBeNegative(nameof(ptr.Size)); #endif _values.Add(ptr); _size += ptr.Size; _isDirty = true; }
public unsafe void Can_seek_and_read_from_chunked_mmap_file(int totalSize, int chunkSize, int seed) { var random = new Random(seed); var buffer = new byte[totalSize]; random.NextBytes(buffer); fixed(byte *ptr = buffer) { var ptrSize = new List <PtrSize>(); var numberOfBuffers = totalSize / chunkSize + (totalSize % chunkSize != 0 ? 1 : 0); for (int i = 0; i < numberOfBuffers; i++) { ptrSize.Add(PtrSize.Create( ptr + (i * chunkSize), i == numberOfBuffers - 1 ? totalSize % chunkSize : chunkSize)); } var sut = new ChunkedMmapStream(ptrSize.ToArray(), chunkSize); // read all bytes var allBytes = sut.ReadEntireBlock(totalSize); Assert.Equal(buffer, allBytes); // seek to beginning, read to end sut.Position = 0; allBytes = sut.ReadData(); Assert.Equal(buffer, allBytes); // read all bytes one by one sut.Position = 0; for (int i = 0; i < allBytes.Length; i++) { Assert.Equal(allBytes[i], sut.ReadByte()); } // random seeks for (int i = 0; i < 1000; i++) { var pos = random.Next(0, totalSize - 1); sut.Position = pos; var result = new byte[Math.Min(totalSize - pos, random.Next(1, totalSize))]; sut.ReadEntireBlock(result, 0, result.Length); Assert.Equal(buffer.Skip(pos).Take(result.Length).ToArray(), result); } } }
private void Add(PtrSize ptr) { if (ptr.Size < 0) { throw new ArgumentException("Size cannot be negative", nameof(ptr.Size)); } _values.Add(ptr); _size += ptr.Size; if (_size + _values.Count + 1 > byte.MaxValue) { _elementSize = 2; } if (_size + _values.Count * 2 + 1 > ushort.MaxValue) { _elementSize = 4; } }
private void OpenInternal() { var fileTree = _currentTransaction.Value.ReadTree(_name); if (fileTree == null) { throw new FileNotFoundException("Could not find index input", _name); } var numberOfChunks = fileTree.State.NumberOfEntries; _ptrs = new PtrSize[numberOfChunks]; int index = 0; using (var it = fileTree.Iterate(prefetch: false)) { if (it.Seek(Slices.BeforeAllKeys) == false) { throw new InvalidDataException("Could not seek to any chunk of this file"); } do { var readResult = fileTree.Read(it.CurrentKey); _ptrs[index] = PtrSize.Create(readResult.Reader.Base, readResult.Reader.Length); index++; } while (it.MoveNext()); } if (numberOfChunks != index) { throw new InvalidDataException($"Read invalid number of file chunks. Expected {numberOfChunks}, read {index}."); } _stream = new ChunkedMmapStream(_ptrs, VoronIndexOutput.MaxFileChunkSize); }
public ReadMapEntryScope Get(long id) { id = Bits.SwapBytes(id); switch (Type) { case MapResultsStorageType.Tree: Slice entrySlice; using (Slice.External(_indexContext.Allocator, (byte *)&id, sizeof(long), out entrySlice)) { if (Tree.IsLeafCompressionSupported) { var read = Tree.ReadDecompressed(entrySlice); if (read == null) { throw new InvalidOperationException($"Could not find a map result with id '{id}' in '{Tree.Name}' tree"); } return(new ReadMapEntryScope(read)); } else { var read = Tree.Read(entrySlice); if (read == null) { throw new InvalidOperationException($"Could not find a map result with id '{id}' in '{Tree.Name}' tree"); } return(new ReadMapEntryScope(PtrSize.Create(read.Reader.Base, read.Reader.Length))); } } case MapResultsStorageType.Nested: var section = GetNestedResultsSection(); return(new ReadMapEntryScope(section.Get(id))); default: throw new ArgumentOutOfRangeException(Type.ToString()); } }
private static unsafe bool ResultsBinaryEqual(BlittableJsonReaderObject newResult, PtrSize existingData) { return(newResult.Size == existingData.Size && Memory.CompareInline(newResult.BasePointer, existingData.Ptr, existingData.Size) == 0); }
public void Add <T>(T value) where T : struct { PtrSize ptr = PtrSize.Create(value); Add(ref ptr); }
public void CopyTo(byte *ptr) { if (Compressed) { Compression.CopyTo(ptr); return; } JsonParserState.WriteVariableSizeInt(ref ptr, _values.Count); int elementSize = ElementSize; int pos = _values.Count * elementSize; byte *dataStart = ptr + pos; switch (elementSize) { case 1: byte *bytePtr = ptr; for (int i = 0; i < _values.Count; i++) { bytePtr[i] = (byte)pos; pos += _values[i].Size; } break; case 2: ushort *shortPtr = (ushort *)ptr; for (int i = 0; i < _values.Count; i++) { shortPtr[i] = (ushort)pos; pos += _values[i].Size; } break; case 4: int *intPtr = (int *)ptr; for (int i = 0; i < _values.Count; i++) { intPtr[i] = pos; pos += _values[i].Size; } break; default: throw new ArgumentOutOfRangeException(nameof(ElementSize), "Unknown element size " + ElementSize); } ulong value; // Do not move inside because we require value to exist inside the for loop. for (int i = 0; i < _values.Count; i++) { PtrSize p = _values[i]; byte *srcPtr; if (_values[i].IsValue) { value = p.Value; srcPtr = (byte *)&value; // This generates an alias on value } else { srcPtr = p.Ptr; } Memory.Copy(dataStart, srcPtr, p.Size); dataStart += p.Size; value = 0; // This ensures there cannot be any JIT optimization that could reuse the memory location. } }
public void Add(void *pointer, int size) { PtrSize ptr = PtrSize.Create(pointer, size); Add(ref ptr); }
public void Add(Slice buffer) { PtrSize ptr = PtrSize.Create(buffer.Content.Ptr, buffer.Content.Length); Add(ref ptr); }
public ReadMapEntryScope(DecompressedReadResult read) { _read = read; Data = PtrSize.Create(read.Reader.Base, read.Reader.Length); }
public ReadMapEntryScope(PtrSize data) { Data = data; _read = null; }
public ReadMapEntryScope(PtrSize data) { Data = data; }
public void Add(void *ptr, int size) { Add(PtrSize.Create(ptr, size)); }
public void Add <T>(T value) where T : struct { Add(PtrSize.Create(value)); }