private void TestPersistence(ulong count, ExpandableArrayOfKeys keys) { using (var validEntries = new BitVector(_pool)) { validEntries.EnsureCapacity(count); validEntries.ChangeAll(true); using (var stream = new FileStream(@"c:\temp\data.d", FileMode.Create, FileAccess.ReadWrite)) { using (var writer = new BinaryWriter(stream)) { keys.Write(writer, (ulong)count, validEntries); } } using (var keys2 = new ExpandableArrayOfKeys(_pool)) { using (var stream = new FileStream(@"c:\temp\data.d", FileMode.Open, FileAccess.Read)) { using (var reader = new BinaryReader(stream)) { { keys2.Read(reader, (ulong)count, validEntries); } } } } } }
private void MultiThread( Action <ConcurrentHashmapOfKeys, ExpandableArrayOfKeys, ulong, ulong, ulong> a, ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, int nThreads, ulong count, ulong offset) { var tasks = new Task[nThreads]; var first = offset; for (var i = 0; i < tasks.Length; i++) { var first1 = first; tasks[i] = new Task(() => a(map, keys, first1, count, offset)); first += count; } foreach (var task in tasks) { task.Start(); } foreach (var task in tasks) { task.Wait(); } }
private unsafe void ReadAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset) { var watch = Stopwatch.StartNew(); for (var k = 0; k < 10000; k++) { for (var i = first; i < first + count; i++) { ulong val = 0; if (!map.TryGetValue(keys.GetAt(i - offset), ref val)) { throw new Exception("Failed to get at " + i + ", offset from " + offset); } if (val != i) { throw new Exception("Failed to validate at " + i + ", offset from " + offset); } //Console.WriteLine("Added {0} at {1}", val, i); } } watch.Stop(); //Console.WriteLine("Elapsed: {0}, for {1}, {2}", watch.ElapsedMilliseconds, first, count); }
public DocumentDataContainer( DataContainerDescriptor dataContainerDescriptor, DocumentTypeDescriptor documentTypeDescriptor, IUnmanagedAllocator allocator, ITracer tracer) { if (tracer == null) { throw new ArgumentNullException("tracer"); } if (dataContainerDescriptor == null) { throw new ArgumentNullException("dataContainerDescriptor"); } if (documentTypeDescriptor == null) { throw new ArgumentNullException("documentTypeDescriptor"); } if (allocator == null) { throw new ArgumentNullException("allocator"); } m_logger = tracer; m_allocator = allocator; DocDesc = documentTypeDescriptor; DataContainerDescriptor = dataContainerDescriptor; ColumnStores = new ColumnDataBase[DocDesc.Fields.Length]; DocumentKeys = new ExpandableArrayOfKeys(m_allocator); FieldIdToColumnStore = new Dictionary <int, int>(ColumnStores.Length * 2); PrimaryKeyFieldId = dataContainerDescriptor.RequireField(documentTypeDescriptor.DocumentType, documentTypeDescriptor.PrimaryKeyFieldName).FieldId; for (var i = 0; i < DocDesc.Fields.Length; i++) { var field = dataContainerDescriptor.RequireField(DocDesc.Fields[i]); ColumnStores[i] = CreateColumnStore(field.DbType, m_allocator, null); FieldIdToColumnStore.Add(field.FieldId, i); } DocumentIdToIndex = new ConcurrentHashmapOfKeys(m_allocator); ValidDocumentsBitmap = new BitVector(m_allocator); SortIndexManager = new SortIndexManager(this); StructureLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); }
private unsafe void BasicMemoryAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset) { for (var k = first; k < first + count; k++) { using (var map2 = new ConcurrentHashmapOfKeys(_pool)) { map2.TryAdd(keys.GetAt(0), 0); } } //var b = stackalloc void*[(int)100000]; //for (var k = 0; k < 10000000000; k++) //{ // for (var i = 0; i < 100000; i++) // { // b[i] = _pool.Alloc((ulong) i % 10000); // } // for (var i = 0; i < 100000; i++) // { // _pool.Free(b[i]); // } //} }
unsafe ExpandableArrayOfKeys GenerateKeys(ulong count) { var result = new ExpandableArrayOfKeys(_pool); result.EnsureCapacity(count); var key = new byte[10]; for (ulong i = 1; i <= count; i++) { var val = i; byte pos = 1; while (val != 0) { key[pos++] = (byte)val; val >>= 8; } key[0] = (byte)(pos-1); //result[i-1] = key; if (!result.TrySetAt((int)i - 1, key)) { throw new Exception("Failed to set a key element at " + (i-1)); } } for (ulong i = 1; i <= count; i++) { var storedKey = result.GetAt(i-1); var val = i; byte pos = 1; while (val != 0) { key[pos++] = (byte)val; val >>= 8; } key[0] = (byte)(pos-1); if (storedKey[0] != key[0]) { throw new Exception("Length prefix broken at " + (i - 1)); } for (var j = 0; j <= key[0]; j++) { //Console.Write(storedKey[j]); //Console.Write(','); if (storedKey[j] != key[j]) { throw new Exception("Data broken at " + (i - 1) + ", offset " + j); } } //Console.WriteLine(); } return result; }
private void TestPersistence(ulong count, ExpandableArrayOfKeys keys) { using (var validEntries = new BitVector(_pool)) { validEntries.EnsureCapacity(count); validEntries.ChangeAll(true); using (var stream = new FileStream(@"c:\temp\data.d", FileMode.Create, FileAccess.ReadWrite)) { using (var writer = new BinaryWriter(stream)) { keys.Write(writer, (ulong) count, validEntries); } } using (var keys2 = new ExpandableArrayOfKeys(_pool)) { using (var stream = new FileStream(@"c:\temp\data.d", FileMode.Open, FileAccess.Read)) { using (var reader = new BinaryReader(stream)) { { keys2.Read(reader, (ulong) count, validEntries); } } } } } }
private unsafe void ReadAction(ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, ulong first, ulong count, ulong offset) { var watch = Stopwatch.StartNew(); for (var k = 0; k < 10000; k++ ) for (var i = first; i < first + count; i++) { ulong val = 0; if (!map.TryGetValue(keys.GetAt(i - offset), ref val)) { throw new Exception("Failed to get at " + i + ", offset from " + offset); } if (val != i) { throw new Exception("Failed to validate at " + i + ", offset from " + offset); } //Console.WriteLine("Added {0} at {1}", val, i); } watch.Stop(); //Console.WriteLine("Elapsed: {0}, for {1}, {2}", watch.ElapsedMilliseconds, first, count); }
private void MultiThread( Action<ConcurrentHashmapOfKeys, ExpandableArrayOfKeys, ulong, ulong, ulong> a, ConcurrentHashmapOfKeys map, ExpandableArrayOfKeys keys, int nThreads, ulong count, ulong offset) { var tasks = new Task[nThreads]; var first = offset; for (var i = 0; i < tasks.Length; i++) { var first1 = first; tasks[i] = new Task(() => a(map, keys, first1, count, offset)); first += count; } foreach (var task in tasks) { task.Start(); } foreach (var task in tasks) { task.Wait(); } }
public DocumentDataContainer( DataContainerDescriptor dataContainerDescriptor, DocumentTypeDescriptor documentTypeDescriptor, IUnmanagedAllocator allocator, ITracer tracer) { if (tracer == null) { throw new ArgumentNullException("tracer"); } if (dataContainerDescriptor == null) { throw new ArgumentNullException("dataContainerDescriptor"); } if (documentTypeDescriptor == null) { throw new ArgumentNullException("documentTypeDescriptor"); } if (allocator == null) { throw new ArgumentNullException("allocator"); } m_logger = tracer; m_allocator = allocator; DocDesc = documentTypeDescriptor; DataContainerDescriptor = dataContainerDescriptor; ColumnStores = new ColumnDataBase[DocDesc.Fields.Length]; DocumentKeys = new ExpandableArrayOfKeys(m_allocator); FieldIdToColumnStore = new Dictionary<int, int>(ColumnStores.Length * 2); PrimaryKeyFieldId = dataContainerDescriptor.RequireField(documentTypeDescriptor.DocumentType, documentTypeDescriptor.PrimaryKeyFieldName).FieldId; for (var i = 0; i < DocDesc.Fields.Length; i++) { var field = dataContainerDescriptor.RequireField(DocDesc.Fields[i]); ColumnStores[i] = CreateColumnStore(field.DbType, m_allocator, null); FieldIdToColumnStore.Add(field.FieldId, i); } DocumentIdToIndex = new ConcurrentHashmapOfKeys(m_allocator); ValidDocumentsBitmap = new BitVector(m_allocator); SortIndexManager = new SortIndexManager(this); StructureLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); }
/// <summary> /// Reconstructs all unmanaged data in the new pool. /// Side effect is that all fragmentation is removed. /// </summary> /// <param name="newpool">The new memory pool to use.</param> public void MigrateRAM(IUnmanagedAllocator newpool) { CheckState(); var vdb = ValidDocumentsBitmap; var dk = DocumentKeys; var diti = DocumentIdToIndex; var colstores = ColumnStores.ToArray(); StructureLock.EnterWriteLock(); try { // generate a new copy of the data var tasks = new List<Task>(); tasks.Add(new Task<BitVector>(() => new BitVector(ValidDocumentsBitmap, newpool))); tasks.Add(new Task<ExpandableArrayOfKeys>(() => new ExpandableArrayOfKeys(DocumentKeys, newpool))); foreach (var c in ColumnStores) { tasks.Add(new Task<ColumnDataBase>(o => CreateColumnStore(((ColumnDataBase)o).DbType, newpool, (ColumnDataBase)o), c)); } foreach (var t in tasks) { t.Start(); } Task.WaitAll(tasks.ToArray()); var newvdb = ((Task<BitVector>) tasks[0]).Result; var newdk = ((Task<ExpandableArrayOfKeys>) tasks[1]).Result; var newditi = new ConcurrentHashmapOfKeys(DocumentIdToIndex, newdk, newpool); // now, since no exception was thrown, let's consume results and dispose of old structures try { ValidDocumentsBitmap = newvdb; DocumentKeys = newdk; DocumentIdToIndex = newditi; for (var i = 2; i < tasks.Count; i++) { ColumnStores[i-2] = ((Task<ColumnDataBase>)tasks[i]).Result; } vdb.Dispose(); dk.Dispose(); diti.Dispose(); foreach (var c in colstores) { c.Dispose(); } } catch { m_stateBroken = true; throw; } m_allocator = newpool; } finally { StructureLock.ExitWriteLock(); } }
/// <summary> /// Reconstructs all unmanaged data in the new pool. /// Side effect is that all fragmentation is removed. /// </summary> /// <param name="newpool">The new memory pool to use.</param> public void MigrateRAM(IUnmanagedAllocator newpool) { CheckState(); var vdb = ValidDocumentsBitmap; var dk = DocumentKeys; var diti = DocumentIdToIndex; var colstores = ColumnStores.ToArray(); StructureLock.EnterWriteLock(); try { // generate a new copy of the data var tasks = new List <Task>(); tasks.Add(new Task <BitVector>(() => new BitVector(ValidDocumentsBitmap, newpool))); tasks.Add(new Task <ExpandableArrayOfKeys>(() => new ExpandableArrayOfKeys(DocumentKeys, newpool))); foreach (var c in ColumnStores) { tasks.Add(new Task <ColumnDataBase>(o => CreateColumnStore(((ColumnDataBase)o).DbType, newpool, (ColumnDataBase)o), c)); } foreach (var t in tasks) { t.Start(); } Task.WaitAll(tasks.ToArray()); var newvdb = ((Task <BitVector>)tasks[0]).Result; var newdk = ((Task <ExpandableArrayOfKeys>)tasks[1]).Result; var newditi = new ConcurrentHashmapOfKeys(DocumentIdToIndex, newdk, newpool); // now, since no exception was thrown, let's consume results and dispose of old structures try { ValidDocumentsBitmap = newvdb; DocumentKeys = newdk; DocumentIdToIndex = newditi; for (var i = 2; i < tasks.Count; i++) { ColumnStores[i - 2] = ((Task <ColumnDataBase>)tasks[i]).Result; } vdb.Dispose(); dk.Dispose(); diti.Dispose(); foreach (var c in colstores) { c.Dispose(); } } catch { m_stateBroken = true; throw; } m_allocator = newpool; } finally { StructureLock.ExitWriteLock(); } }
unsafe ExpandableArrayOfKeys GenerateKeys(ulong count) { var result = new ExpandableArrayOfKeys(_pool); result.EnsureCapacity(count); var key = new byte[10]; for (ulong i = 1; i <= count; i++) { var val = i; byte pos = 1; while (val != 0) { key[pos++] = (byte)val; val >>= 8; } key[0] = (byte)(pos - 1); //result[i-1] = key; if (!result.TrySetAt((int)i - 1, key)) { throw new Exception("Failed to set a key element at " + (i - 1)); } } for (ulong i = 1; i <= count; i++) { var storedKey = result.GetAt(i - 1); var val = i; byte pos = 1; while (val != 0) { key[pos++] = (byte)val; val >>= 8; } key[0] = (byte)(pos - 1); if (storedKey[0] != key[0]) { throw new Exception("Length prefix broken at " + (i - 1)); } for (var j = 0; j <= key[0]; j++) { //Console.Write(storedKey[j]); //Console.Write(','); if (storedKey[j] != key[j]) { throw new Exception("Data broken at " + (i - 1) + ", offset " + j); } } //Console.WriteLine(); } return(result); }