/// <summary> /// Deserializes an index from the given stream. /// </summary> /// <param name="stream">The stream to read from. Reading will start at position 0.</param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> /// <param name="copy">Flag to make an in-memory copy.</param> /// <returns></returns> public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy) { long size; return(MemoryMappedIndex <T> .Deserialize(stream, readFrom, writeTo, copy, out size)); }
/// <summary> /// Creates a new memory mapped file. /// </summary> /// <param name="file"></param> /// <param name="stream"></param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> internal MemoryMappedAccessorVariable(MemoryMappedFile file, Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo) : base(file, stream, -1) { _readFrom = readFrom; _writeTo = writeTo; }
/// <summary> /// Deserializes an index from the given stream. /// </summary> /// <returns></returns> public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy, out long size) { stream.Seek(0, System.IO.SeekOrigin.Begin); var longBytes = new byte[8]; stream.Read(longBytes, 0, 8); size = BitConverter.ToInt64(longBytes, 0) + 8; if (copy) { var bytes = (int)size - 8; var memoryStream = new MemoryStream(bytes); var buffer = new byte[32768]; int read; while ((read = stream.Read(buffer, 0, System.Math.Min(buffer.Length, bytes))) > 0) { memoryStream.Write(buffer, 0, read); bytes -= read; } memoryStream.Seek(0, SeekOrigin.Begin); stream = memoryStream; } var file = new MemoryMappedStream(new LimitedStream(stream)); return(new MemoryMappedIndex <T>(file, readFrom, writeTo, size - 8)); }
/// <summary> /// Deserializes an index from the given stream. /// </summary> /// <param name="stream">The stream to read from. Reading will start at position 0.</param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> /// <param name="copy">Flag to make an in-memory copy.</param> /// <param name="size">The total size in bytes of the deserialized data.</param> /// <returns></returns> public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy, out long size) { if (copy) { throw new NotSupportedException("Deserializing a memory mapped index with copy option is not supported."); } stream.Seek(0, System.IO.SeekOrigin.Begin); var longBytes = new byte[8]; stream.Read(longBytes, 0, 8); size = BitConverter.ToInt64(longBytes, 0) + 8; var file = new MemoryMappedStream(new LimitedStream(stream)); return(new MemoryMappedIndex <T>(file, readFrom, writeTo, size - 8)); }
/// <summary> /// Creates a new memory-mapped index. /// </summary> /// <param name="file"></param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> /// <param name="accessorSizeInBytes"></param> /// <param name="readOnly"></param> public MemoryMappedIndex(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo, long accessorSizeInBytes, bool readOnly) { if (file == null) { throw new ArgumentNullException("file"); } if (readFrom == null) { throw new ArgumentNullException("readFrom"); } if (writeTo == null) { throw new ArgumentNullException("writeTo"); } _file = file; _readFrom = readFrom; _writeTo = writeTo; _position = 0; _accessorSizeInBytes = accessorSizeInBytes; _accessors = new List <MemoryMappedAccessor <T> >(); _accessorSize = new List <long>(); _accessors.Add(this.CreateAccessor(_file, _accessorSizeInBytes)); if (readOnly) { // index is readonly, meaning it already contains data on construction, meaning accessor size is non-zero. _accessorSize.Add(_accessorSizeInBytes); } else { // is writable. _accessorSize.Add(0); } _readOnly = readOnly; }
public void TestHuge() { var randomGenerator = new RandomGenerator(66707770); // make this deterministic var buffer = new byte[255]; var readFrom = new MemoryMappedFile.ReadFromDelegate<string>((stream, position) => { stream.Seek(position, System.IO.SeekOrigin.Begin); var size = stream.ReadByte(); int pos = 0; stream.Read(buffer, pos, size); while (size == 255) { pos = pos + size; size = stream.ReadByte(); if (buffer.Length < size + pos) { Array.Resize(ref buffer, size + pos); } stream.Read(buffer, pos, size); } pos = pos + size; return System.Text.Encoding.Unicode.GetString(buffer, 0, pos); }); var writeTo = new MemoryMappedFile.WriteToDelegate<string>((stream, position, structure) => { stream.Seek(position, System.IO.SeekOrigin.Begin); var bytes = System.Text.Encoding.Unicode.GetBytes(structure); var length = bytes.Length; for (int idx = 0; idx <= bytes.Length; idx = idx + 255) { var size = bytes.Length - idx; if (size > 255) { size = 255; } if (stream.Length <= stream.Position + size + 1) { // past end of stream. return -1; } stream.WriteByte((byte)size); stream.Write(bytes, idx, size); length++; } return length; }); var dictionary = new MemoryMappedHugeDictionary<string, string>(new MemoryMappedStream(new MemoryStream()), readFrom, writeTo, readFrom, writeTo); var reference = new Dictionary<string, string>(); var keys = new List<string>(); for (int idx = 0; idx < 100; idx++) { keys.Add(string.Format("key{0}", idx)); } var testCount = 1000; while (testCount > 0) { var keyIdx = randomGenerator.Generate(keys.Count); var value = randomGenerator.GenerateString( randomGenerator.Generate(256) + 32); dictionary[keys[keyIdx]] = value; reference[keys[keyIdx]] = value; testCount--; } foreach (var pair in reference) { var value = dictionary[pair.Key]; Assert.AreEqual(pair.Value, value); Assert.IsTrue(dictionary.TryGetValue(pair.Key, out value)); Assert.AreEqual(pair.Value, value); } }
public void TestSmall() { var randomGenerator = new RandomGenerator(66707770); // make this deterministic var buffer = new byte[255]; var readFrom = new MemoryMappedFile.ReadFromDelegate <string>((stream, position) => { stream.Seek(position, System.IO.SeekOrigin.Begin); var size = stream.ReadByte(); int pos = 0; stream.Read(buffer, pos, size); while (size == 255) { pos = pos + size; size = stream.ReadByte(); if (buffer.Length < size + pos) { Array.Resize(ref buffer, size + pos); } stream.Read(buffer, pos, size); } pos = pos + size; return(System.Text.Encoding.Unicode.GetString(buffer, 0, pos)); }); var writeTo = new MemoryMappedFile.WriteToDelegate <string>((stream, position, structure) => { stream.Seek(position, System.IO.SeekOrigin.Begin); var bytes = System.Text.Encoding.Unicode.GetBytes(structure); var length = bytes.Length; for (int idx = 0; idx <= bytes.Length; idx = idx + 255) { var size = bytes.Length - idx; if (size > 255) { size = 255; } if (stream.Length <= stream.Position + size + 1) { // past end of stream. return(-1); } stream.WriteByte((byte)size); stream.Write(bytes, idx, size); length++; } return(length); }); var dictionary = new MemoryMappedHugeDictionary <string, string>(new MemoryMappedStream(new MemoryStream()), readFrom, writeTo, readFrom, writeTo); var reference = new Dictionary <string, string>(); var keys = new string[] { "key1", "key2", "key3", "key4", "key5", "key6", "key7", "key8" }; var testCount = 10; while (testCount > 0) { var keyIdx = randomGenerator.Generate(keys.Length); var value = randomGenerator.GenerateString( randomGenerator.Generate(256) + 32); dictionary[keys[keyIdx]] = value; reference[keys[keyIdx]] = value; testCount--; } foreach (var pair in reference) { var value = dictionary[pair.Key]; Assert.AreEqual(pair.Value, value); Assert.IsTrue(dictionary.TryGetValue(pair.Key, out value)); Assert.AreEqual(pair.Value, value); } }
/// <summary> /// Creates a new readonly fixed-size memory-mapped index. /// </summary> /// <param name="file"></param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> /// <param name="sizeInBytes"></param> public MemoryMappedIndex(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo, long sizeInBytes) : this(file, readFrom, writeTo, sizeInBytes, true) { }
/// <summary> /// Creates a new memory-mapped index. /// </summary> /// <param name="file"></param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> public MemoryMappedIndex(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo) : this(file, readFrom, writeTo, DefaultFileSize, false) { }
/// <summary> /// Creates a new memory mapped file based on the given stream and the given size in bytes. /// </summary> /// <param name="position">The position to start at.</param> /// <param name="sizeInBytes">The size.</param> /// <param name="readFrom"></param> /// <param name="writeTo"></param> /// <returns></returns> protected override MemoryMappedAccessor <T> DoCreateVariable <T>(long position, long sizeInBytes, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo) { return(new Accessors.MemoryMappedAccessorVariable <T>(this, new CappedStream(_stream, position, sizeInBytes), readFrom, writeTo)); }
/// <summary> /// Creates a new dictionary. /// </summary> /// <param name="file">A memory mapped file.</param> /// <param name="readKeyFrom"></param> /// <param name="writeToKey"></param> /// <param name="readValueFrom"></param> /// <param name="writeValueTo"></param> public MemoryMappedHugeDictionary(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <TKey> readKeyFrom, MemoryMappedFile.WriteToDelegate <TKey> writeToKey, MemoryMappedFile.ReadFromDelegate <TValue> readValueFrom, MemoryMappedFile.WriteToDelegate <TValue> writeValueTo) : this(file, readKeyFrom, writeToKey, readValueFrom, writeValueTo, null, null) { }
/// <summary> /// Creates a new dictionary. /// </summary> /// <param name="file">A memory mapped file.</param> /// <param name="readKeyFrom"></param> /// <param name="writeToKey"></param> /// <param name="readValueFrom"></param> /// <param name="writeValueTo"></param> /// <param name="hashKey"></param> /// <param name="compareKey"></param> public MemoryMappedHugeDictionary(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <TKey> readKeyFrom, MemoryMappedFile.WriteToDelegate <TKey> writeToKey, MemoryMappedFile.ReadFromDelegate <TValue> readValueFrom, MemoryMappedFile.WriteToDelegate <TValue> writeValueTo, Func <TKey, int> hashKey, Func <TKey, TKey, int> compareKey) { _file = file; _size = 0; _nextPairId = 0; _hashKey = hashKey; _compareKey = compareKey; _hashes = new MemoryMappedHugeArrayInt64(_file, 1024 * 1024, MemoryMappedHugeArrayInt64.DefaultFileElementSize, (int)MemoryMappedHugeArrayInt64.DefaultFileElementSize / MemoryMappedHugeArrayInt64.DefaultBufferSize, MemoryMappedHugeArrayInt64.DefaultCacheSize); for (int idx = 0; idx < _hashes.Length; idx++) { _hashes[idx] = -1; } _pairs = new MemoryMappedHugeArrayInt64(_file, 16 * 4); _keys = new MemoryMappedIndex <KeyStruct>(_file, (stream, position) => { return(new KeyStruct() { Key = readKeyFrom.Invoke(stream, position) }); }, (stream, position, structure) => { return(writeToKey.Invoke(stream, position, structure.Key)); }); _values = new MemoryMappedIndex <ValueStruct>(_file, (stream, position) => { return(new ValueStruct() { Value = readValueFrom.Invoke(stream, position) }); }, (stream, position, structure) => { return(writeValueTo.Invoke(stream, position, structure.Value)); }); }