/// <summary>
/// Backup Data of a certain disk region onto the transaction log file
/// </summary>
internal void BackupData(List <KeyValuePair <RecordKey, Region> > dataRegions)
{
using (var writePool = new ConcurrentIOPoolManager())
{
using (var readPool = new ConcurrentIOPoolManager())
{
BackupData(dataRegions, readPool, writePool);
}
}
}
/// <summary>
/// Backup Data of a certain disk region onto the transaction log file
/// </summary>
internal void BackupData(List <KeyValuePair <RecordKey, Region> > dataRegions,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool)
{
LogTracer.Verbose("BackupData: Start for Thread {0}.", Thread.CurrentThread.ManagedThreadId);
foreach (KeyValuePair <RecordKey, Region> dataRegion in dataRegions)
{
RecordKey key = dataRegion.Key;
Region region = dataRegion.Value;
var f = (OnDisk.File.IFile)Server.GetFile(key.Filename);
string fFilename = key.Filename;
//** foreach disk area in region, copy it to transaction file
foreach (KeyValuePair <long, int> area in region)
{
// short circuit if IO exception was detected.
if (readPool.AsyncThreadException != null)
{
throw readPool.AsyncThreadException;
}
if (writePool.AsyncThreadException != null)
{
throw writePool.AsyncThreadException;
}
var logKey = new BackupDataLogKey();
logKey.SourceFilename = f == null ? fFilename : f.Filename;
logKey.SourceDataAddress = area.Key;
IEnumerable <KeyValuePair <BackupDataLogKey, BackupDataLogValue> > intersectingLogs;
long mergedBlockStartAddress, mergedBlockSize;
// todo: optimize LogCollection locking!
//LogCollection.Locker.Lock();
LogTracer.Verbose("Transactin.BackupData: Thread {0}, Locking LogCollection, count {1}.", Thread.CurrentThread.ManagedThreadId, LogCollection.Count);
bool isIntersectingLogs = GetIntersectingLogs(logKey, area.Value, out intersectingLogs, out mergedBlockStartAddress,
out mergedBlockSize);
if (isIntersectingLogs)
{
BackupDataWithIntersection(intersectingLogs, logKey, area, f, fFilename, readPool, writePool,
key);
}
else
{
BackupDataWithNoIntersection(intersectingLogs, logKey, area, f, fFilename, readPool, writePool,
key);
}
LogTracer.Verbose("Transactin.BackupData: Thread {0}, Unlocking LogCollection, count {1}.", Thread.CurrentThread.ManagedThreadId, LogCollection.Count);
//LogCollection.Locker.Unlock();
}
}
}
private static void CopyData(string serverRootPath, List <CopyParams> copyBlocks)
{
if (copyBlocks == null)
{
throw new ArgumentNullException("copyBlocks");
}
using (var writePool = new ConcurrentIOPoolManager())
{
using (var readPool = new ConcurrentIOPoolManager())
{
string targetFile = null;
long readerFileSize = 0;
for (int i = 0; i < copyBlocks.Count; i++)
{
var rb = copyBlocks[i];
int size = rb.DataSize;
if (targetFile == null || targetFile != rb.TargetFilename)
{
targetFile = rb.TargetFilename;
}
string systemBackupFilename = string.Format("{0}\\{1}", serverRootPath, rb.SourceFilename);
// copy backed up data onto the DB
var reader = readPool.GetInstance(systemBackupFilename, null, size);
var writer = writePool.GetInstance(targetFile, null, size);
if (reader == null || writer == null)
{
continue;
}
if (readerFileSize == 0)
{
readerFileSize = reader.FileStream.Length;
}
if (rb.SourceAddress + rb.DataSize <= readerFileSize)
{
reader.FileStream.Seek(rb.SourceAddress, SeekOrigin.Begin);
writer.FileStream.Seek(rb.TargetAddress, SeekOrigin.Begin);
reader.FileStream.BeginRead(reader.Buffer, 0, size,
ReadCallback,
new object[] { new[] { reader, writer }, false }
);
}
else
{
reader.Event.Set();
writer.Event.Set();
}
// end
}
}
}
}
private void BackupDataWithNoIntersection(
IEnumerable <KeyValuePair <BackupDataLogKey, BackupDataLogValue> > intersectingLogs,
BackupDataLogKey logKey, KeyValuePair <long, int> area, OnDisk.File.IFile f, string fFilename,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool,
RecordKey key)
{
string systemBackupFilename = Server.Path + DataBackupFilename;
int size = area.Value;
key.Address = area.Key;
// no intersection nor mergeable logs, add new log! backup and log the data area
ConcurrentIOData reader = f != null
? readPool.GetInstance(f, size)
: readPool.GetInstance(fFilename, null, size);
ConcurrentIOData writer = writePool.GetInstance(systemBackupFilename, (TransactionRoot)Root);
if (reader == null || writer == null)
{
throw new SopException("This program has a bug! 'didn't get reader or writer from Async IO Pool.");
}
LogTracer.Verbose("BackupDataWithNoIntersection: Start for Thread {0}.", Thread.CurrentThread.ManagedThreadId);
var logValue = new BackupDataLogValue();
logValue.DataSize = size;
logValue.TransactionId = Id;
logValue.BackupFileHandle = GetLogBackupFileHandle(DataBackupFilename);
// return the current backup file size and grow it to make room for data to be backed up...
logValue.BackupDataAddress = GrowBackupFile(size, writer.FileStream);
// save a record of the backed up data..
LogCollection.Add(logKey, logValue);
// log after data was backed up!!
Sop.VoidFunc logBackedupData = () =>
{
UpdateLogger.LogLine("{0}{1}:{2} to {3}:{4} Size={5}",
BackupFromToken, f != null ? f.Filename : fFilename, area.Key,
DataBackupFilename, logValue.BackupDataAddress, size);
};
writer.FileStream.Seek(logValue.BackupDataAddress, SeekOrigin.Begin, true);
reader.FileStream.Seek(area.Key, SeekOrigin.Begin, true);
reader.FileStream.BeginRead(
reader.Buffer, 0, size, ReadCallback,
new object[] { new[] { reader, writer }, true, logKey, logBackedupData });
}
protected internal override bool RegisterRecycle(CollectionOnDisk collection, long blockAddress,
int blockSize)
{
if (!Transaction.RegisterRecycle(_addBlocksStore, _recycledBlocksStore, collection, blockAddress, blockSize))
{
using (var writePool = new ConcurrentIOPoolManager())
{
using (var readPool = new ConcurrentIOPoolManager())
{
RegisterSave(collection, blockAddress, blockSize, readPool, writePool);
}
}
}
return(true);
}
private void RestoreData()
{
if (LogCollection == null)
{
return;
}
var lc = LogCollection;
using (var writePool = new ConcurrentIOPoolManager())
{
using (var readPool = new ConcurrentIOPoolManager())
{
OnDisk.File.IFile targetFile = null;
foreach (var de in lc)
{
var key = de.Key;
var value = de.Value;
string systemBackupFilename = Server.Path + GetLogBackupFilename(value.BackupFileHandle);
int size = value.DataSize;
if (targetFile == null || targetFile.Filename != key.SourceFilename)
{
targetFile = (OnDisk.File.IFile)Server.GetFile(key.SourceFilename);
}
if (targetFile == null)
{
continue;
}
ConcurrentIOData reader = readPool.GetInstance(systemBackupFilename, (TransactionRoot)Root,
size);
ConcurrentIOData writer = writePool.GetInstance(targetFile, size);
reader.FileStream.Seek(value.BackupDataAddress, SeekOrigin.Begin);
writer.FileStream.Seek(key.SourceDataAddress, SeekOrigin.Begin);
reader.FileStream.BeginRead(reader.Buffer, 0, size, ReadCallback,
new object[] { new[] { reader, writer }, false });
}
}
}
ClearLogs();
}
/// <summary>
/// Write a group of Blocks onto Disk. NOTE: it will be more optimal if Blocks
/// are sorted by its Data Address so this function can write contiguous blocks
/// in one async write.
/// </summary>
public int WriteBlocksToDisk(Algorithm.Collection.ICollectionOnDisk parent,
IDictionary <long, Sop.DataBlock> blocks, bool clear)
{
if (!parent.IsOpen)
{
return(0);
}
var blockSize = (int)parent.DataBlockSize;
int chunkSize = MaxSegmentSize / 2;
if (chunkSize > blocks.Count * blockSize)
{
chunkSize = blocks.Count * blockSize;
}
if (_writeBuffer == null || _writeBuffer.Length < chunkSize)
{
_writeBuffer = new byte[chunkSize];
}
Sop.Transaction.Transaction.LogTracer.Verbose("WriteBlocksToDisk: Start for Thread {0}.", Thread.CurrentThread.ManagedThreadId);
using (var backupWritePool = new ConcurrentIOPoolManager())
{
// Backup the target blocks on disk
using (var backupReadPool = new ConcurrentIOPoolManager())
{
WriteBlocksToDisk(backupReadPool, backupWritePool, parent, blocks);
}
}
// overwrite the target blocks on disk with source blocks.
using (var writePool = new ConcurrentIOPoolManager())
{
WriteBlocksToDisk(null, writePool, parent, blocks);
}
Sop.Transaction.Transaction.LogTracer.Verbose("WriteBlocksToDisk: End for Thread {0}.", Thread.CurrentThread.ManagedThreadId);
return(blocks.Count);
}
/// <summary>
/// Backup Data of a certain disk region onto the transaction log file
/// </summary>
internal void BackupData(List <KeyValuePair <RecordKey, Region> > dataRegions,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool)
{
foreach (KeyValuePair <RecordKey, Region> dataRegion in dataRegions)
{
RecordKey key = dataRegion.Key;
Region region = dataRegion.Value;
var f = (OnDisk.File.IFile)Server.GetFile(key.Filename);
string fFilename = key.Filename;
//** foreach disk area in region, copy it to transaction file
foreach (KeyValuePair <long, int> area in region)
{
var logKey = new BackupDataLogKey();
logKey.SourceFilename = f == null ? fFilename : f.Filename;
logKey.SourceDataAddress = area.Key;
IEnumerable <KeyValuePair <BackupDataLogKey, BackupDataLogValue> > intersectingLogs;
long mergedBlockStartAddress, mergedBlockSize;
if (GetIntersectingLogs(logKey, area.Value, out intersectingLogs, out mergedBlockStartAddress,
out mergedBlockSize))
{
BackupDataWithIntersection(intersectingLogs, logKey, area, f, fFilename, readPool, writePool,
key);
}
else
{
BackupDataWithNoIntersection(intersectingLogs, logKey, area, f, fFilename, readPool, writePool,
key);
}
}
//** Detect and Merge backed up blocks to minimize growth of Items stored in LogCollection,
//** impact of not merging is a slower rollback of an unfinished pending transaction in previous run.
//DetectAndMergeBlocks();
}
}
protected internal override bool RegisterSave(CollectionOnDisk collection, long blockAddress, int segmentSize,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool)
{
return(true);
}
/// <summary>
/// Try to register block for update and back up contents as needed.
/// </summary>
/// <param name="collection"></param>
/// <param name="blockAddress"></param>
/// <param name="segmentSize"></param>
/// <param name="readPool"></param>
/// <param name="writePool"></param>
/// <returns>true means block was registered and contents backed up,
/// false means block is either new or recycled and wasn't registered for update and no backup occurred.</returns>
protected internal abstract bool RegisterSave(CollectionOnDisk collection,
long blockAddress, int segmentSize,
ConcurrentIOPoolManager readPool,
ConcurrentIOPoolManager writePool);
/// <summary>
/// RegisterSave will be called when a block cache faulted from memory
/// onto Disk. Resolution of Added blocks will be done here and only
/// those "modified" blocks will be registered & backed up.
/// </summary>
/// <param name="collection">Collection that is saving the block</param>
/// <param name="blockAddress"></param>
/// <param name="segmentSize"></param>
/// <param name="readPool"> </param>
/// <param name="writePool"> </param>
protected internal override bool RegisterSave(CollectionOnDisk collection, long blockAddress,
int segmentSize, ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool)
{
if (IsTransactionStore(collection))
{
return(((TransactionBase)collection.ParentTransactionLogger).RegisterSave(collection, blockAddress,
segmentSize, readPool,
writePool));
}
if (LogCollection == null)
{
return(false);
}
LogTracer.Verbose("Transactin.RegisterSave: Start for Thread {0}.", Thread.CurrentThread.ManagedThreadId);
//Step 1. Remove Intersections with Added, Growth segments & Recycled Blocks from region as no need to backup
// new Blocks
//Step 2. Copy or backup remaining (Updated) blocks onto the Transaction Log file for restore on Rollback
RecordKey key = CreateKey(collection, blockAddress);
// if in file growth segments, don't register for save...
Region region = RegionLogic.RemoveIntersections(_fileGrowthStore, key, blockAddress, segmentSize);
if (region == null || region.Count == 0)
{
if (_inCommit == 0)
{
TrackModification(collection.GetTopParent());
}
return(false);
}
#region subtract any region intersecting with recycled and add Stores
int itemCount = region.Count / 2;
if (itemCount < 5)
{
itemCount = 5;
}
var regionsForBackup = new List <KeyValuePair <RecordKey, Region> >(itemCount);
foreach (KeyValuePair <long, int> area in region)
{
// subtract regions intersecting with recycled segments
key.Address = area.Key;
Region region2 = RegionLogic.RemoveIntersections(_recycledSegmentsStore,
key, area.Key, area.Value);
LogTracer.Verbose("Transactin.RegisterSave: Thread {0}, _recycledSegmentsStore count {1}.", Thread.CurrentThread.ManagedThreadId, _recycledSegmentsStore.Count);
if (region2 == null || region2.Count <= 0 ||
((LogCollection is SortedDictionaryOnDisk) &&
key.Filename == ((SortedDictionaryOnDisk)LogCollection).File.Filename))
{
continue;
}
// subtract regions intersecting with (new) add segments
foreach (KeyValuePair <long, int> area2 in region2)
{
key.Address = area2.Key;
var region3 = RegionLogic.RemoveIntersections(_addBlocksStore, key, area2.Key, area2.Value);
LogTracer.Verbose("Transactin.RegisterSave: Thread {0}, _addBlocksStore count {1}.", Thread.CurrentThread.ManagedThreadId, _addBlocksStore.Count);
if (region3 == null || region3.Count <= 0)
{
continue;
}
foreach (KeyValuePair <long, int> area3 in region3)
{
key.Address = area3.Key;
var region4 = RegionLogic.RemoveIntersections(_recycledBlocksStore, key, area3.Key, area3.Value);
LogTracer.Verbose("Transactin.RegisterSave: Thread {0}, _recycledBlocksStore count {1}.", Thread.CurrentThread.ManagedThreadId, _recycledBlocksStore.Count);
if (region4 == null || region4.Count <= 0)
{
continue;
}
// any remaining portions are marked for backup
if (_inCommit == 0)
{
TrackModification(collection.GetTopParent());
}
regionsForBackup.Add(new KeyValuePair <RecordKey, Region>(key, region4));
}
}
}
#endregion
if (readPool != null)
{
BackupData(regionsForBackup, readPool, writePool);
}
else
{
BackupData(regionsForBackup);
}
return(true);
}
/// <summary>
/// RegisterSave is the only one we do COW to be able to roll back from text log file
/// the last transaction action done.
/// </summary>
/// <param name="collection"></param>
/// <param name="blockAddress"></param>
/// <param name="segmentSize"></param>
/// <param name="readPool"> </param>
/// <param name="writePool"> </param>
protected internal override bool RegisterSave(CollectionOnDisk collection, long blockAddress, int segmentSize,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool)
{
if (!string.IsNullOrEmpty(_lastRegisterSaveFilename))
{
LoggerTransDetails.Log(string.Format("Successful RegisterSave {0}, {1}, {2}",
_lastRegisterSaveFilename, _lastRegisterSaveBlockAddress,
_lastRegisterSaveSegmentSize));
_lastRegisterSaveFilename = null;
_lastRegisterSaveBlockAddress = 0;
_lastRegisterSaveSegmentSize = 0;
}
/* Step 1. Remove Intersections with added/recycled Blocks from region as no need to backup
* new Blocks
* Step 2. Copy or backup (any) remaining blocks (the Updated blocks)
* onto the Transaction Log file for restore on Rollback
*/
Transaction.RecordKey key = Transaction.CreateKey(collection, blockAddress);
Region region = RegionLogic.RemoveIntersections(_fileGrowthStore, key,
blockAddress, segmentSize);
if (region == null || region.Count == 0)
{
return(false);
}
bool logOnce = false;
foreach (KeyValuePair <long, int> area2 in region)
{
key.Address = area2.Key;
Region region2 = RegionLogic.RemoveIntersections(_recycledCollectionStore, key,
area2.Key, area2.Value);
if (region2 != null && region.Count > 0)
{
foreach (KeyValuePair <long, int> area3 in region2)
{
key.Address = area3.Key;
Region region3 = RegionLogic.RemoveIntersections(_addStore, key,
area3.Key, area3.Value);
//** Step 2: Backup the "modified" portion(s) of data
if (region3 != null && region3.Count > 0)
{
//** foreach disk area in region, copy it to transaction log file
foreach (KeyValuePair <long, int> area4 in region3)
{
BackupData(collection, area4.Key, area4.Value);
}
if (!logOnce)
{
logOnce = true;
_lastRegisterSaveFilename = collection.File.Filename;
_lastRegisterSaveBlockAddress = blockAddress;
_lastRegisterSaveSegmentSize = segmentSize;
LoggerTransDetails.Log(
string.Format("RegisterSave {0}, {1}, {2}", collection.File.Filename, blockAddress,
segmentSize));
}
}
}
}
}
return(logOnce);
}
/// <summary>
/// RegisterSave will be called when a block cache faulted from memory
/// onto Disk. Resolution of Added blocks will be done here and only
/// those "modified" blocks will be saved. Newly added block(s) will
/// not be saved.
/// </summary>
/// <param name="collection">Collection that is saving the block</param>
/// <param name="blockAddress"></param>
/// <param name="segmentSize"></param>
/// <param name="readPool"> </param>
/// <param name="writePool"> </param>
protected internal override bool RegisterSave(CollectionOnDisk collection, long blockAddress,
int segmentSize, ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool)
{
if (IsTransactionStore(collection))
{
return(((TransactionBase)collection.ParentTransactionLogger).RegisterSave(collection, blockAddress,
segmentSize, readPool,
writePool));
}
if (LogCollection == null)
{
return(false);
}
/* Step 1. Remove Intersections with Added, Growth segments & Recycled Blocks from region as no need to backup
* new Blocks
* Step 2. Copy or backup (any) remaining blocks (the Updated blocks)
* onto the Transaction Log file for restore on Rollback
*/
RecordKey key = CreateKey(collection, blockAddress);
//// if in recycled or add store, don't register for save...
//if (RegionLogic.IsSegmentInStore(_recycledCollectionStore, key, segmentSize) || InAddStore(key, segmentSize))
// return false;
//** if in file growth segments, don't register for save...
Region region = RegionLogic.RemoveIntersections(_fileGrowthStore, key, blockAddress, segmentSize);
if (region == null || region.Count == 0)
{
if (_inCommit == 0)
{
TrackModification(collection.GetTopParent());
}
return(false);
}
//**
int itemCount = region.Count / 2;
if (itemCount < 5)
{
itemCount = 5;
}
var regionsForBackup = new List <KeyValuePair <RecordKey, Region> >(itemCount);
foreach (KeyValuePair <long, int> area in region)
{
key.Address = area.Key;
Region region2 = RegionLogic.RemoveIntersections(_recycledCollectionStore,
key, area.Key, area.Value);
if (region2 == null || region2.Count <= 0 ||
((LogCollection is SortedDictionaryOnDisk) &&
key.Filename == ((SortedDictionaryOnDisk)LogCollection).File.Filename))
{
continue;
}
foreach (KeyValuePair <long, int> area2 in region2)
{
key.Address = area2.Key;
Region region3 = RegionLogic.RemoveIntersections(_addStore,
key, area2.Key, area2.Value);
//** Step 2: Backup the "modified" portion(s) of data
if (region3 == null || region3.Count <= 0)
{
continue;
}
if (_inCommit == 0)
{
TrackModification(collection.GetTopParent());
}
regionsForBackup.Add(new KeyValuePair <RecordKey, Region>(key, region3));
}
}
if (readPool != null)
{
BackupData(regionsForBackup, readPool, writePool);
}
else
{
BackupData(regionsForBackup);
}
return(true);
}
private void WriteBlocksToDisk(ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool,
Algorithm.Collection.ICollectionOnDisk parent, IDictionary <long, Sop.DataBlock> blocks)
{
// todo: do this Async way, when time permits. :)
if (BinaryWriter == null)
{
if (BufferStream == null)
{
BufferStream = new MemoryStream();
}
BinaryWriter = new BinaryWriter(BufferStream, parent.File.Server.Encoding);
}
else
{
BinaryWriter.Seek(0, SeekOrigin.Begin);
}
const int sizeOfNumerics = Sop.DataBlock.OverheadSize;
var writeBuffer = _writeBuffer;
int bufferIndex = 0, currentTargetBufferIndex = 0;
long runningAddress = -1, startBlockAddress = -1;
var bulkWriter = new BulkWriter();
var dataChunks = new List <BulkWriter.DataChunk>(4);
#region resize data file before appending to it...
if (readPool == null || writePool == null)
{
long currentAddress = -1;
if (((Collections.Generic.ISortedDictionary <long, Sop.DataBlock>)blocks).MoveLast())
{
currentAddress = ((Collections.Generic.ISortedDictionary <long, Sop.DataBlock>)blocks).CurrentKey;
}
if (currentAddress > -1)
{
// thread safe increase File Size to accomodate data to be appended...
var FileSize = currentAddress + (int)parent.DataBlockSize;
if (parent.FileStream.Length < FileSize)
{
if (parent.Transaction != null)
{
lock (parent.Transaction)
{
if (parent.FileStream.Length < FileSize)
{
parent.FileStream.SetLength(FileSize);
}
}
}
else
{
parent.FileStream.SetLength(FileSize);
}
}
}
}
#endregion
Sop.DataBlock[] blocksCopy = new Sop.DataBlock[blocks.Count];
blocks.Values.CopyTo(blocksCopy, 0);
foreach (Sop.DataBlock block in blocksCopy)
{
SetIsDirty(block, false);
if (block.DataAddress >= 0)
{
#region Process special states, e.g. - buffer is full, current block Address is fragmented from previous block's
if (startBlockAddress == -1)
{
startBlockAddress = runningAddress = block.DataAddress;
}
else
{
bool bufferIsFull = bufferIndex + sizeOfNumerics + block.Data.Length +
currentTargetBufferIndex > writeBuffer.Length - block.Length;
if (block.DataAddress != runningAddress || bufferIsFull)
{
dataChunks.Add(new BulkWriter.DataChunk
{
TargetDataAddress = startBlockAddress,
Index = currentTargetBufferIndex, // Index in the buffer of 1st byte of this segment
Size = bufferIndex // size of the segment
});
if (bufferIsFull)
{
//** write to disk
if (readPool != null && writePool != null)
{
bulkWriter.Backup(readPool, writePool, parent, writeBuffer, dataChunks);
if (writePool.AsyncThreadException != null)
{
throw writePool.AsyncThreadException;
}
else if (readPool.AsyncThreadException != null)
{
throw readPool.AsyncThreadException;
}
}
else if (writePool != null)
{
bulkWriter.Write(writePool, parent, writeBuffer, dataChunks);
if (writePool.AsyncThreadException != null)
{
throw writePool.AsyncThreadException;
}
}
else
{
throw new SopException("WriteBlocksToDisk has a bug!");
}
// create new buffer for succeeding chunks...
dataChunks = new List <BulkWriter.DataChunk>(4);
writeBuffer = new byte[writeBuffer.Length];
currentTargetBufferIndex = 0;
}
else
{
currentTargetBufferIndex += bufferIndex;
}
bufferIndex = 0;
runningAddress = startBlockAddress = block.DataAddress;
}
}
#endregion
}
else
{
throw new InvalidOperationException("Invalid (-) Block.DataAddress detected.");
}
//**** write Block Header and Data to disk
BinaryWriter.Seek(0, SeekOrigin.Begin);
// Byte 0 to 7: Next Item Address (64 bit long int) = 0 (no next item)
BinaryWriter.Write(block.NextItemAddress);
// Byte 8 to 11: Size Occupied
BinaryWriter.Write(block.SizeOccupied);
// Byte 12 to 19: Low-level next datablock address
BinaryWriter.Write(block.InternalNextBlockAddress);
// Byte 20: count of member blocks, max is 65535.
ushort memberCount = 0;
if (block.IsHead)
{
int cm = block.CountMembers(true);
memberCount = cm > Sop.DataBlock.MaxChainMemberCount ? Sop.DataBlock.MaxChainMemberCount : (ushort)cm;
}
BinaryWriter.Write(memberCount);
byte[] b2 = BufferStream.GetBuffer();
Array.Copy(b2, 0, writeBuffer, currentTargetBufferIndex + bufferIndex, sizeOfNumerics);
bufferIndex += sizeOfNumerics;
//** Byte 20 to 20 + Data Length: USER DATA
int cs = block.Data.Length;
//if (currentTargetBufferIndex + cs + bufferIndex > writeBuffer.Length - block.Length)
// cs = writeBuffer.Length - (currentTargetBufferIndex + bufferIndex);
Array.Copy(block.Data, 0, writeBuffer, currentTargetBufferIndex + bufferIndex, cs);
bufferIndex += block.Data.Length;
runningAddress += block.Length;
}
// write the last chunk set to disk...
if (startBlockAddress != -1)
{
//** write to disk
dataChunks.Add(new BulkWriter.DataChunk
{
TargetDataAddress = startBlockAddress,
Index = currentTargetBufferIndex,
Size = bufferIndex
});
}
if (dataChunks.Count > 0)
{
if (readPool != null && writePool != null)
{
bulkWriter.Backup(readPool, writePool, parent, writeBuffer, dataChunks);
}
else if (writePool != null)
{
bulkWriter.Write(writePool, parent, writeBuffer, dataChunks);
}
else
{
throw new SopException("WriteBlocksToDisk has a bug!");
}
}
}
private void BackupDataWithIntersection(
IEnumerable <KeyValuePair <BackupDataLogKey, BackupDataLogValue> > intersectingLogs,
BackupDataLogKey logKey, KeyValuePair <long, int> area, OnDisk.File.IFile f, string fFilename,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool,
RecordKey key
)
{
if (intersectingLogs == null)
{
// process conflicts with other trans...
//ProcessTransactionConflicts(logKey, area.Value);
// area is within an already backed up area (intersectingLogs == null), do nothing...
return;
}
LogTracer.Verbose("BackupDataWithIntersection: Start for Thread {0}.", Thread.CurrentThread.ManagedThreadId);
// get area(s) outside each intersecting segment and back it up...
var newRegion = new Region(area.Key, area.Value);
#region for future implements... ?
//bool wasIntersected = false;
//foreach (KeyValuePair<BackupDataLogKey, BackupDataLogValue> entry in intersectingLogs)
//{
// // process conflicts with other trans...
// ProcessTransactionConflicts(entry.Key, entry.Value.DataSize);
// if (newRegion.Subtract(entry.Key.SourceDataAddress, entry.Value.DataSize))
// wasIntersected = true;
//}
//if (!wasIntersected) return;
#endregion
// copy modified blocks to the transaction backup file.
foreach (KeyValuePair <long, int> newArea in newRegion)
{
if (readPool.AsyncThreadException != null)
{
throw readPool.AsyncThreadException;
}
if (writePool.AsyncThreadException != null)
{
throw writePool.AsyncThreadException;
}
var logKey2 = new BackupDataLogKey();
logKey2.SourceFilename = logKey.SourceFilename;
logKey2.SourceDataAddress = newArea.Key;
var logValue = new BackupDataLogValue();
logValue.DataSize = newArea.Value;
logValue.TransactionId = Id;
int newSize = newArea.Value;
key.Address = newArea.Key;
//if (RegisterAdd(_addBlocksStore, null, null, key, newArea.Value, false))
// return;
logValue.BackupFileHandle = GetLogBackupFileHandle(DataBackupFilename);
ConcurrentIOData reader = f != null
? readPool.GetInstance(f, newArea.Value)
: readPool.GetInstance(fFilename, null, newArea.Value);
if (reader == null)
{
throw new InvalidOperationException("Can't get ConcurrentIOData from ReadPool");
}
string systemBackupFilename = Server.Path + DataBackupFilename;
ConcurrentIOData writer = writePool.GetInstance(systemBackupFilename, ((TransactionRoot)Root));
if (writer == null)
{
throw new InvalidOperationException("Can't get ConcurrentIOData from WritePool");
}
// return the current backup file size and grow it to make room for data to be backed up...
logValue.BackupDataAddress = GrowBackupFile(newSize, writer.FileStream);
// save a record of the backed up data..
LogCollection.Add(logKey2, logValue);
// prepare lambda expression to log after data was backed up!!
Sop.VoidFunc logBackedupData = () =>
{
UpdateLogger.LogLine(
"{0}{1}:{2} to {3}:{4} Size={5}", BackupFromToken, logKey2.SourceFilename,
logKey2.SourceDataAddress, DataBackupFilename, logValue.BackupDataAddress, newSize);
};
writer.FileStream.Seek(logValue.BackupDataAddress, SeekOrigin.Begin, true);
reader.FileStream.Seek(newArea.Key, SeekOrigin.Begin, true);
reader.FileStream.BeginRead(
reader.Buffer, 0, newSize, ReadCallback,
new object[] { new[] { reader, writer }, true, logKey2, logBackedupData });
}
}
private void BackupDataWithNoIntersection(
IEnumerable <KeyValuePair <BackupDataLogKey, BackupDataLogValue> > intersectingLogs,
BackupDataLogKey logKey, KeyValuePair <long, int> area, OnDisk.File.IFile f, string fFilename,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool,
RecordKey key)
{
string systemBackupFilename = Server.Path + DataBackupFilename;
int size = area.Value;
key.Address = area.Key;
//if (RegisterAdd(_addStore, null, null, key, size, false))
//{
// Logger.LogLine("Extending, skipping Backup...");
// return;
//}
//** no intersection nor mergeable logs, add new log!
//** backup and log the data area
ConcurrentIOData reader = f != null
? readPool.GetInstance(f, size)
: readPool.GetInstance(fFilename, null, size);
ConcurrentIOData writer = writePool.GetInstance(systemBackupFilename, (TransactionRoot)Root, size);
if (reader == null || writer == null)
{
return;
}
var logValue = new BackupDataLogValue();
logValue.DataSize = size;
logValue.TransactionId = Id;
//** todo: can we remove this block:
//long readerFileSize = reader.FileStream.Length;
//if (area.Key + size > readerFileSize)
//{
// int appendSize = (int)(area.Key + size - readerFileSize);
// key.Address = readerFileSize;
// RegisterAdd(_addStore, null, null, key, appendSize, false);
// size = (int)(readerFileSize - area.Key);
// logValue.DataSize = size;
// reader.Buffer = new byte[size];
//}
//**
reader.FileStream.Seek(area.Key, SeekOrigin.Begin);
logValue.BackupFileHandle = GetLogBackupFileHandle(DataBackupFilename);
logValue.BackupDataAddress = writer.FileStream.Seek(0, SeekOrigin.End);
UpdateLogger.LogLine("{0}{1}:{2} to {3}:{4} Size={5}",
BackupFromToken, f != null ? f.Filename : fFilename, area.Key,
DataBackupFilename, logValue.BackupDataAddress, size);
// resize target file to accomodate data to be copied.
writer.FileStream.Seek(size, SeekOrigin.End);
writer.FileStream.Seek(logValue.BackupDataAddress, SeekOrigin.Begin);
reader.FileStream.BeginRead(
reader.Buffer, 0, size, ReadCallback,
new object[] { new[] { reader, writer }, true, logKey }
);
//** save a record of the backed up data..
LogCollection.Add(logKey, logValue);
}
private void BackupDataWithIntersection(
IEnumerable <KeyValuePair <BackupDataLogKey, BackupDataLogValue> > intersectingLogs,
BackupDataLogKey logKey, KeyValuePair <long, int> area, OnDisk.File.IFile f, string fFilename,
ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool,
RecordKey key
)
{
if (intersectingLogs == null)
{
//** process conflicts with other trans...
ProcessTransactionConflicts(logKey, area.Value);
//** area is within an already backed up area (intersectingLogs == null), do nothing...
return;
}
//** get area(s) outside each intersecting segment and back it up...
var newRegion = new Region(area.Key, area.Value);
bool wasIntersected = false;
foreach (KeyValuePair <BackupDataLogKey, BackupDataLogValue> entry in intersectingLogs)
{
//** process conflicts with other trans...
ProcessTransactionConflicts(entry.Key, entry.Value.DataSize);
if (newRegion.Subtract(entry.Key.SourceDataAddress, entry.Value.DataSize))
{
wasIntersected = true;
}
}
//** copy
if (!wasIntersected)
{
return;
}
foreach (KeyValuePair <long, int> newArea in newRegion)
{
var logKey2 = new BackupDataLogKey();
logKey2.SourceFilename = logKey.SourceFilename;
logKey2.SourceDataAddress = newArea.Key;
var logValue = new BackupDataLogValue();
logValue.DataSize = newArea.Value;
logValue.TransactionId = Id;
int newSize = newArea.Value;
key.Address = newArea.Key;
//if (RegisterAdd(_addStore, null, null, key, newArea.Value, false))
// return;
logValue.BackupFileHandle = GetLogBackupFileHandle(DataBackupFilename);
ConcurrentIOData reader = f != null
? readPool.GetInstance(f, newArea.Value)
: readPool.GetInstance(fFilename, null, newArea.Value);
if (reader == null)
{
throw new InvalidOperationException("Can't get ConcurrentIOData from ReadPool");
}
string systemBackupFilename = Server.Path + DataBackupFilename;
ConcurrentIOData writer = writePool.GetInstance(systemBackupFilename, ((TransactionRoot)Root),
newArea.Value);
if (writer == null)
{
throw new InvalidOperationException("Can't get ConcurrentIOData from WritePool");
}
logValue.BackupDataAddress = writer.FileStream.Seek(0, SeekOrigin.End);
//** todo: can we remove this block:
//long readerFileSize = reader.FileStream.Length;
//if (newArea.Key + newArea.Value > readerFileSize)
//{
// int appendSize = (int)(newArea.Key + newArea.Value - readerFileSize);
// key.Address = readerFileSize;
// RegisterAdd(_addStore, null, null, key, appendSize, false);
// newSize = (int)(readerFileSize - newArea.Key);
// logValue.DataSize = newSize;
// reader.Buffer = new byte[newSize];
//}
//**
reader.FileStream.Seek(newArea.Key, SeekOrigin.Begin);
UpdateLogger.LogLine(
"{0}{1}:{2} to {3}:{4} Size={5}", BackupFromToken, logKey2.SourceFilename,
logKey2.SourceDataAddress,
DataBackupFilename, logValue.BackupDataAddress, newSize);
// resize target file to accomodate data to be copied.
writer.FileStream.Seek(newSize, SeekOrigin.End);
writer.FileStream.Seek(logValue.BackupDataAddress, SeekOrigin.Begin);
reader.FileStream.BeginRead(
reader.Buffer, 0, newSize, ReadCallback,
new object[] { new[] { reader, writer }, true, logKey2 }
);
//** save a record of the backed up data..
LogCollection.Add(logKey2, logValue);
}
}
