/// <summary>
/// Deserializes an index from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
/// <param name="copy">Flag to make an in-memory copy.</param>
/// <returns></returns>
public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy)
{
long size;
return(MemoryMappedIndex <T> .Deserialize(stream, readFrom, writeTo, copy, out size));
}
/// <summary>
/// Creates a new memory mapped file.
/// </summary>
/// <param name="file"></param>
/// <param name="stream"></param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
internal MemoryMappedAccessorVariable(MemoryMappedFile file, Stream stream,
MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo)
: base(file, stream, -1)
{
_readFrom = readFrom;
_writeTo = writeTo;
}
/// <summary>
/// Deserializes an index from the given stream.
/// </summary>
/// <returns></returns>
public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy, out long size)
{
stream.Seek(0, System.IO.SeekOrigin.Begin);
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
size = BitConverter.ToInt64(longBytes, 0) + 8;
if (copy)
{
var bytes = (int)size - 8;
var memoryStream = new MemoryStream(bytes);
var buffer = new byte[32768];
int read;
while ((read = stream.Read(buffer, 0, System.Math.Min(buffer.Length, bytes))) > 0)
{
memoryStream.Write(buffer, 0, read);
bytes -= read;
}
memoryStream.Seek(0, SeekOrigin.Begin);
stream = memoryStream;
}
var file = new MemoryMappedStream(new LimitedStream(stream));
return(new MemoryMappedIndex <T>(file, readFrom, writeTo, size - 8));
}
/// <summary>
/// Deserializes an index from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
/// <param name="copy">Flag to make an in-memory copy.</param>
/// <param name="size">The total size in bytes of the deserialized data.</param>
/// <returns></returns>
public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy, out long size)
{
if (copy)
{
throw new NotSupportedException("Deserializing a memory mapped index with copy option is not supported.");
}
stream.Seek(0, System.IO.SeekOrigin.Begin);
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
size = BitConverter.ToInt64(longBytes, 0) + 8;
var file = new MemoryMappedStream(new LimitedStream(stream));
return(new MemoryMappedIndex <T>(file, readFrom, writeTo, size - 8));
}
/// <summary>
/// Creates a new memory-mapped index.
/// </summary>
/// <param name="file"></param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
/// <param name="accessorSizeInBytes"></param>
/// <param name="readOnly"></param>
public MemoryMappedIndex(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, long accessorSizeInBytes, bool readOnly)
{
if (file == null)
{
throw new ArgumentNullException("file");
}
if (readFrom == null)
{
throw new ArgumentNullException("readFrom");
}
if (writeTo == null)
{
throw new ArgumentNullException("writeTo");
}
_file = file;
_readFrom = readFrom;
_writeTo = writeTo;
_position = 0;
_accessorSizeInBytes = accessorSizeInBytes;
_accessors = new List <MemoryMappedAccessor <T> >();
_accessorSize = new List <long>();
_accessors.Add(this.CreateAccessor(_file, _accessorSizeInBytes));
if (readOnly)
{ // index is readonly, meaning it already contains data on construction, meaning accessor size is non-zero.
_accessorSize.Add(_accessorSizeInBytes);
}
else
{ // is writable.
_accessorSize.Add(0);
}
_readOnly = readOnly;
}
public void TestHuge()
{
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
var buffer = new byte[255];
var readFrom = new MemoryMappedFile.ReadFromDelegate<string>((stream, position) =>
{
stream.Seek(position, System.IO.SeekOrigin.Begin);
var size = stream.ReadByte();
int pos = 0;
stream.Read(buffer, pos, size);
while (size == 255)
{
pos = pos + size;
size = stream.ReadByte();
if (buffer.Length < size + pos)
{
Array.Resize(ref buffer, size + pos);
}
stream.Read(buffer, pos, size);
}
pos = pos + size;
return System.Text.Encoding.Unicode.GetString(buffer, 0, pos);
});
var writeTo = new MemoryMappedFile.WriteToDelegate<string>((stream, position, structure) =>
{
stream.Seek(position, System.IO.SeekOrigin.Begin);
var bytes = System.Text.Encoding.Unicode.GetBytes(structure);
var length = bytes.Length;
for (int idx = 0; idx <= bytes.Length; idx = idx + 255)
{
var size = bytes.Length - idx;
if (size > 255)
{
size = 255;
}
if (stream.Length <= stream.Position + size + 1)
{ // past end of stream.
return -1;
}
stream.WriteByte((byte)size);
stream.Write(bytes, idx, size);
length++;
}
return length;
});
var dictionary = new MemoryMappedHugeDictionary<string, string>(new MemoryMappedStream(new MemoryStream()),
readFrom, writeTo, readFrom, writeTo);
var reference = new Dictionary<string, string>();
var keys = new List<string>();
for (int idx = 0; idx < 100; idx++)
{
keys.Add(string.Format("key{0}", idx));
}
var testCount = 1000;
while (testCount > 0)
{
var keyIdx = randomGenerator.Generate(keys.Count);
var value = randomGenerator.GenerateString(
randomGenerator.Generate(256) + 32);
dictionary[keys[keyIdx]] = value;
reference[keys[keyIdx]] = value;
testCount--;
}
foreach (var pair in reference)
{
var value = dictionary[pair.Key];
Assert.AreEqual(pair.Value, value);
Assert.IsTrue(dictionary.TryGetValue(pair.Key, out value));
Assert.AreEqual(pair.Value, value);
}
}
public void TestSmall()
{
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
var buffer = new byte[255];
var readFrom = new MemoryMappedFile.ReadFromDelegate <string>((stream, position) =>
{
stream.Seek(position, System.IO.SeekOrigin.Begin);
var size = stream.ReadByte();
int pos = 0;
stream.Read(buffer, pos, size);
while (size == 255)
{
pos = pos + size;
size = stream.ReadByte();
if (buffer.Length < size + pos)
{
Array.Resize(ref buffer, size + pos);
}
stream.Read(buffer, pos, size);
}
pos = pos + size;
return(System.Text.Encoding.Unicode.GetString(buffer, 0, pos));
});
var writeTo = new MemoryMappedFile.WriteToDelegate <string>((stream, position, structure) =>
{
stream.Seek(position, System.IO.SeekOrigin.Begin);
var bytes = System.Text.Encoding.Unicode.GetBytes(structure);
var length = bytes.Length;
for (int idx = 0; idx <= bytes.Length; idx = idx + 255)
{
var size = bytes.Length - idx;
if (size > 255)
{
size = 255;
}
if (stream.Length <= stream.Position + size + 1)
{ // past end of stream.
return(-1);
}
stream.WriteByte((byte)size);
stream.Write(bytes, idx, size);
length++;
}
return(length);
});
var dictionary = new MemoryMappedHugeDictionary <string, string>(new MemoryMappedStream(new MemoryStream()),
readFrom, writeTo, readFrom, writeTo);
var reference = new Dictionary <string, string>();
var keys = new string[] { "key1", "key2", "key3", "key4", "key5", "key6", "key7", "key8" };
var testCount = 10;
while (testCount > 0)
{
var keyIdx = randomGenerator.Generate(keys.Length);
var value = randomGenerator.GenerateString(
randomGenerator.Generate(256) + 32);
dictionary[keys[keyIdx]] = value;
reference[keys[keyIdx]] = value;
testCount--;
}
foreach (var pair in reference)
{
var value = dictionary[pair.Key];
Assert.AreEqual(pair.Value, value);
Assert.IsTrue(dictionary.TryGetValue(pair.Key, out value));
Assert.AreEqual(pair.Value, value);
}
}
/// <summary>
/// Creates a new readonly fixed-size memory-mapped index.
/// </summary>
/// <param name="file"></param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
/// <param name="sizeInBytes"></param>
public MemoryMappedIndex(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, long sizeInBytes)
: this(file, readFrom, writeTo, sizeInBytes, true)
{
}
/// <summary>
/// Creates a new memory-mapped index.
/// </summary>
/// <param name="file"></param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
public MemoryMappedIndex(MemoryMappedFile file, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo)
: this(file, readFrom, writeTo, DefaultFileSize, false)
{
}
/// <summary>
/// Creates a new memory mapped file based on the given stream and the given size in bytes.
/// </summary>
/// <param name="position">The position to start at.</param>
/// <param name="sizeInBytes">The size.</param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
/// <returns></returns>
protected override MemoryMappedAccessor <T> DoCreateVariable <T>(long position, long sizeInBytes, MemoryMappedFile.ReadFromDelegate <T> readFrom, MemoryMappedFile.WriteToDelegate <T> writeTo)
{
return(new Accessors.MemoryMappedAccessorVariable <T>(this, new CappedStream(_stream, position, sizeInBytes), readFrom, writeTo));
}
/// <summary>
/// Creates a new dictionary.
/// </summary>
/// <param name="file">A memory mapped file.</param>
/// <param name="readKeyFrom"></param>
/// <param name="writeToKey"></param>
/// <param name="readValueFrom"></param>
/// <param name="writeValueTo"></param>
public MemoryMappedHugeDictionary(MemoryMappedFile file,
MemoryMappedFile.ReadFromDelegate <TKey> readKeyFrom, MemoryMappedFile.WriteToDelegate <TKey> writeToKey,
MemoryMappedFile.ReadFromDelegate <TValue> readValueFrom, MemoryMappedFile.WriteToDelegate <TValue> writeValueTo)
: this(file, readKeyFrom, writeToKey, readValueFrom, writeValueTo, null, null)
{
}
/// <summary>
/// Creates a new dictionary.
/// </summary>
/// <param name="file">A memory mapped file.</param>
/// <param name="readKeyFrom"></param>
/// <param name="writeToKey"></param>
/// <param name="readValueFrom"></param>
/// <param name="writeValueTo"></param>
/// <param name="hashKey"></param>
/// <param name="compareKey"></param>
public MemoryMappedHugeDictionary(MemoryMappedFile file,
MemoryMappedFile.ReadFromDelegate <TKey> readKeyFrom, MemoryMappedFile.WriteToDelegate <TKey> writeToKey,
MemoryMappedFile.ReadFromDelegate <TValue> readValueFrom, MemoryMappedFile.WriteToDelegate <TValue> writeValueTo,
Func <TKey, int> hashKey, Func <TKey, TKey, int> compareKey)
{
_file = file;
_size = 0;
_nextPairId = 0;
_hashKey = hashKey;
_compareKey = compareKey;
_hashes = new MemoryMappedHugeArrayInt64(_file, 1024 * 1024,
MemoryMappedHugeArrayInt64.DefaultFileElementSize,
(int)MemoryMappedHugeArrayInt64.DefaultFileElementSize / MemoryMappedHugeArrayInt64.DefaultBufferSize,
MemoryMappedHugeArrayInt64.DefaultCacheSize);
for (int idx = 0; idx < _hashes.Length; idx++)
{
_hashes[idx] = -1;
}
_pairs = new MemoryMappedHugeArrayInt64(_file, 16 * 4);
_keys = new MemoryMappedIndex <KeyStruct>(_file,
(stream, position) =>
{
return(new KeyStruct()
{
Key = readKeyFrom.Invoke(stream, position)
});
},
(stream, position, structure) =>
{
return(writeToKey.Invoke(stream, position, structure.Key));
});
_values = new MemoryMappedIndex <ValueStruct>(_file,
(stream, position) =>
{
return(new ValueStruct()
{
Value = readValueFrom.Invoke(stream, position)
});
},
(stream, position, structure) =>
{
return(writeValueTo.Invoke(stream, position, structure.Value));
});
}
