private void MultiThread(
Action <ConcurrentHashmapOfKeys, ExpandableArrayOfKeys, ulong, ulong, ulong> a,
ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, int nThreads, ulong count, ulong offset)
{
var tasks = new Task[nThreads];
var first = offset;
for (var i = 0; i < tasks.Length; i++)
{
var first1 = first;
tasks[i] = new Task(() => a(map, keys, first1, count, offset));
first += count;
}
foreach (var task in tasks)
{
task.Start();
}
foreach (var task in tasks)
{
task.Wait();
}
}
private unsafe void ReadAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset)
{
var watch = Stopwatch.StartNew();
for (var k = 0; k < 10000; k++)
{
for (var i = first; i < first + count; i++)
{
ulong val = 0;
if (!map.TryGetValue(keys.GetAt(i - offset), ref val))
{
throw new Exception("Failed to get at " + i + ", offset from " + offset);
}
if (val != i)
{
throw new Exception("Failed to validate at " + i + ", offset from " + offset);
}
//Console.WriteLine("Added {0} at {1}", val, i);
}
}
watch.Stop();
//Console.WriteLine("Elapsed: {0}, for {1}, {2}", watch.ElapsedMilliseconds, first, count);
}
public DocumentDataContainer(
DataContainerDescriptor dataContainerDescriptor,
DocumentTypeDescriptor documentTypeDescriptor,
IUnmanagedAllocator allocator,
ITracer tracer)
{
if (tracer == null)
{
throw new ArgumentNullException("tracer");
}
if (dataContainerDescriptor == null)
{
throw new ArgumentNullException("dataContainerDescriptor");
}
if (documentTypeDescriptor == null)
{
throw new ArgumentNullException("documentTypeDescriptor");
}
if (allocator == null)
{
throw new ArgumentNullException("allocator");
}
m_logger = tracer;
m_allocator = allocator;
DocDesc = documentTypeDescriptor;
DataContainerDescriptor = dataContainerDescriptor;
ColumnStores = new ColumnDataBase[DocDesc.Fields.Length];
DocumentKeys = new ExpandableArrayOfKeys(m_allocator);
FieldIdToColumnStore = new Dictionary <int, int>(ColumnStores.Length * 2);
PrimaryKeyFieldId = dataContainerDescriptor.RequireField(documentTypeDescriptor.DocumentType, documentTypeDescriptor.PrimaryKeyFieldName).FieldId;
for (var i = 0; i < DocDesc.Fields.Length; i++)
{
var field = dataContainerDescriptor.RequireField(DocDesc.Fields[i]);
ColumnStores[i] = CreateColumnStore(field.DbType, m_allocator, null);
FieldIdToColumnStore.Add(field.FieldId, i);
}
DocumentIdToIndex = new ConcurrentHashmapOfKeys(m_allocator);
ValidDocumentsBitmap = new BitVector(m_allocator);
SortIndexManager = new SortIndexManager(this);
StructureLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
public unsafe void Test()
{
const ulong nThreads = 4;
const ulong offset = 0;
const ulong count = 15000000;
for (var loop = 0; loop < 20; loop++)
{
using (var keys = GenerateKeys(nThreads * count))
{
Console.WriteLine("Generated " + count * nThreads);
using (var map = new ConcurrentHashmapOfKeys(_pool))
{
MultiThread(InsertAndReadAction, map, keys, (int) nThreads, count, offset);
//MultiThread(ReadAction, map, keys, (int) nThreads, count, offset);
//Console.WriteLine("Inserted " + count * nThreads);
}
//TestPersistence(count, keys);
Console.WriteLine("Completed test");
}
//_pool.Recycle();
//_pool.DeallocateGarbage();
Console.WriteLine("Deallocated garbage");
}
Console.WriteLine("Disposed keys");
//Console.ReadLine();
Console.WriteLine("Disposed map");
//Console.ReadLine();
_pool.DeallocateGarbage();
Console.WriteLine("Deallocated garbage");
Console.ReadLine();
//_pool.Recycle();
//Console.WriteLine("Recycled pool");
//Console.ReadLine();
_pool.Dispose();
Console.WriteLine("Disposed pool");
Console.ReadLine();
}
public unsafe void Test()
{
const ulong nThreads = 4;
const ulong offset = 0;
const ulong count = 15000000;
for (var loop = 0; loop < 20; loop++)
{
using (var keys = GenerateKeys(nThreads * count))
{
Console.WriteLine("Generated " + count * nThreads);
using (var map = new ConcurrentHashmapOfKeys(_pool))
{
MultiThread(InsertAndReadAction, map, keys, (int)nThreads, count, offset);
//MultiThread(ReadAction, map, keys, (int) nThreads, count, offset);
//Console.WriteLine("Inserted " + count * nThreads);
}
//TestPersistence(count, keys);
Console.WriteLine("Completed test");
}
//_pool.Recycle();
//_pool.DeallocateGarbage();
Console.WriteLine("Deallocated garbage");
}
Console.WriteLine("Disposed keys");
//Console.ReadLine();
Console.WriteLine("Disposed map");
//Console.ReadLine();
_pool.DeallocateGarbage();
Console.WriteLine("Deallocated garbage");
Console.ReadLine();
//_pool.Recycle();
//Console.WriteLine("Recycled pool");
//Console.ReadLine();
_pool.Dispose();
Console.WriteLine("Disposed pool");
Console.ReadLine();
}
private unsafe void BasicMemoryAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset)
{
for (var k = first; k < first + count; k++)
{
using (var map2 = new ConcurrentHashmapOfKeys(_pool))
{
map2.TryAdd(keys.GetAt(0), 0);
}
}
//var b = stackalloc void*[(int)100000];
//for (var k = 0; k < 10000000000; k++)
//{
// for (var i = 0; i < 100000; i++)
// {
// b[i] = _pool.Alloc((ulong) i % 10000);
// }
// for (var i = 0; i < 100000; i++)
// {
// _pool.Free(b[i]);
// }
//}
}
private unsafe void BasicMemoryAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset)
{
for (var k = first; k < first + count; k++)
{
using (var map2 = new ConcurrentHashmapOfKeys(_pool))
{
map2.TryAdd(keys.GetAt(0), 0);
}
}
//var b = stackalloc void*[(int)100000];
//for (var k = 0; k < 10000000000; k++)
//{
// for (var i = 0; i < 100000; i++)
// {
// b[i] = _pool.Alloc((ulong) i % 10000);
// }
// for (var i = 0; i < 100000; i++)
// {
// _pool.Free(b[i]);
// }
//}
}
private unsafe void ReadAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset)
{
var watch = Stopwatch.StartNew();
for (var k = 0; k < 10000; k++ )
for (var i = first; i < first + count; i++)
{
ulong val = 0;
if (!map.TryGetValue(keys.GetAt(i - offset), ref val))
{
throw new Exception("Failed to get at " + i + ", offset from " + offset);
}
if (val != i)
{
throw new Exception("Failed to validate at " + i + ", offset from " + offset);
}
//Console.WriteLine("Added {0} at {1}", val, i);
}
watch.Stop();
//Console.WriteLine("Elapsed: {0}, for {1}, {2}", watch.ElapsedMilliseconds, first, count);
}
private void MultiThread(
Action<ConcurrentHashmapOfKeys, ExpandableArrayOfKeys, ulong, ulong, ulong> a,
ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, int nThreads, ulong count, ulong offset)
{
var tasks = new Task[nThreads];
var first = offset;
for (var i = 0; i < tasks.Length; i++)
{
var first1 = first;
tasks[i] = new Task(() => a(map, keys, first1, count, offset));
first += count;
}
foreach (var task in tasks)
{
task.Start();
}
foreach (var task in tasks)
{
task.Wait();
}
}
public DocumentDataContainer(
DataContainerDescriptor dataContainerDescriptor,
DocumentTypeDescriptor documentTypeDescriptor,
IUnmanagedAllocator allocator,
ITracer tracer)
{
if (tracer == null)
{
throw new ArgumentNullException("tracer");
}
if (dataContainerDescriptor == null)
{
throw new ArgumentNullException("dataContainerDescriptor");
}
if (documentTypeDescriptor == null)
{
throw new ArgumentNullException("documentTypeDescriptor");
}
if (allocator == null)
{
throw new ArgumentNullException("allocator");
}
m_logger = tracer;
m_allocator = allocator;
DocDesc = documentTypeDescriptor;
DataContainerDescriptor = dataContainerDescriptor;
ColumnStores = new ColumnDataBase[DocDesc.Fields.Length];
DocumentKeys = new ExpandableArrayOfKeys(m_allocator);
FieldIdToColumnStore = new Dictionary<int, int>(ColumnStores.Length * 2);
PrimaryKeyFieldId = dataContainerDescriptor.RequireField(documentTypeDescriptor.DocumentType, documentTypeDescriptor.PrimaryKeyFieldName).FieldId;
for (var i = 0; i < DocDesc.Fields.Length; i++)
{
var field = dataContainerDescriptor.RequireField(DocDesc.Fields[i]);
ColumnStores[i] = CreateColumnStore(field.DbType, m_allocator, null);
FieldIdToColumnStore.Add(field.FieldId, i);
}
DocumentIdToIndex = new ConcurrentHashmapOfKeys(m_allocator);
ValidDocumentsBitmap = new BitVector(m_allocator);
SortIndexManager = new SortIndexManager(this);
StructureLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
/// <summary>
/// Reconstructs all unmanaged data in the new pool.
/// Side effect is that all fragmentation is removed.
/// </summary>
/// <param name="newpool">The new memory pool to use.</param>
public void MigrateRAM(IUnmanagedAllocator newpool)
{
CheckState();
var vdb = ValidDocumentsBitmap;
var dk = DocumentKeys;
var diti = DocumentIdToIndex;
var colstores = ColumnStores.ToArray();
StructureLock.EnterWriteLock();
try
{
// generate a new copy of the data
var tasks = new List<Task>();
tasks.Add(new Task<BitVector>(() => new BitVector(ValidDocumentsBitmap, newpool)));
tasks.Add(new Task<ExpandableArrayOfKeys>(() => new ExpandableArrayOfKeys(DocumentKeys, newpool)));
foreach (var c in ColumnStores)
{
tasks.Add(new Task<ColumnDataBase>(o => CreateColumnStore(((ColumnDataBase)o).DbType, newpool, (ColumnDataBase)o), c));
}
foreach (var t in tasks)
{
t.Start();
}
Task.WaitAll(tasks.ToArray());
var newvdb = ((Task<BitVector>) tasks[0]).Result;
var newdk = ((Task<ExpandableArrayOfKeys>) tasks[1]).Result;
var newditi = new ConcurrentHashmapOfKeys(DocumentIdToIndex, newdk, newpool);
// now, since no exception was thrown, let's consume results and dispose of old structures
try
{
ValidDocumentsBitmap = newvdb;
DocumentKeys = newdk;
DocumentIdToIndex = newditi;
for (var i = 2; i < tasks.Count; i++)
{
ColumnStores[i-2] = ((Task<ColumnDataBase>)tasks[i]).Result;
}
vdb.Dispose();
dk.Dispose();
diti.Dispose();
foreach (var c in colstores)
{
c.Dispose();
}
}
catch
{
m_stateBroken = true;
throw;
}
m_allocator = newpool;
}
finally
{
StructureLock.ExitWriteLock();
}
}
/// <summary>
/// Reconstructs all unmanaged data in the new pool.
/// Side effect is that all fragmentation is removed.
/// </summary>
/// <param name="newpool">The new memory pool to use.</param>
public void MigrateRAM(IUnmanagedAllocator newpool)
{
CheckState();
var vdb = ValidDocumentsBitmap;
var dk = DocumentKeys;
var diti = DocumentIdToIndex;
var colstores = ColumnStores.ToArray();
StructureLock.EnterWriteLock();
try
{
// generate a new copy of the data
var tasks = new List <Task>();
tasks.Add(new Task <BitVector>(() => new BitVector(ValidDocumentsBitmap, newpool)));
tasks.Add(new Task <ExpandableArrayOfKeys>(() => new ExpandableArrayOfKeys(DocumentKeys, newpool)));
foreach (var c in ColumnStores)
{
tasks.Add(new Task <ColumnDataBase>(o => CreateColumnStore(((ColumnDataBase)o).DbType, newpool, (ColumnDataBase)o), c));
}
foreach (var t in tasks)
{
t.Start();
}
Task.WaitAll(tasks.ToArray());
var newvdb = ((Task <BitVector>)tasks[0]).Result;
var newdk = ((Task <ExpandableArrayOfKeys>)tasks[1]).Result;
var newditi = new ConcurrentHashmapOfKeys(DocumentIdToIndex, newdk, newpool);
// now, since no exception was thrown, let's consume results and dispose of old structures
try
{
ValidDocumentsBitmap = newvdb;
DocumentKeys = newdk;
DocumentIdToIndex = newditi;
for (var i = 2; i < tasks.Count; i++)
{
ColumnStores[i - 2] = ((Task <ColumnDataBase>)tasks[i]).Result;
}
vdb.Dispose();
dk.Dispose();
diti.Dispose();
foreach (var c in colstores)
{
c.Dispose();
}
}
catch
{
m_stateBroken = true;
throw;
}
m_allocator = newpool;
}
finally
{
StructureLock.ExitWriteLock();
}
}
