private void AllocateOnNextSegment(CollectionOnDisk parent, HeaderData hd, Sop.DataBlock block)
{
long g;
long segmentSize = parent.Grow(out g);
if (segmentSize > 0)
{
Transaction.ITransactionLogger trans = parent.Transaction;
if (trans != null)
{
((Transaction.TransactionBase)trans).RegisterFileGrowth(parent, g, segmentSize);
}
}
else
{
string s = string.Format("File '{0}' failed to grow.", parent.File.Filename);
//_logger.LogLine(s);
throw new InvalidOperationException(s);
}
hd.OnDiskLeftoverSegmentSize = (int)(hd.EndAllocatableAddress - hd.NextAllocatableAddress);
hd.StartAllocatableAddress = g;
hd.EndAllocatableAddress = hd.StartAllocatableAddress + segmentSize;
hd.NextAllocatableAddress = hd.StartAllocatableAddress;
block.DataAddress = hd.NextAllocatableAddress;
hd.NextAllocatableAddress += block.Length;
hd.DiskBuffer.IsDirty = true;
hd.IsModifiedInTransaction = true;
}
/// <summary>
/// Add to target block collection a certain Block
/// </summary>
/// <param name="blockSource"></param>
/// <param name="blocksDest"></param>
protected internal virtual void AddToBlocks(Sop.DataBlock blockSource,
IDictionary <long, Sop.DataBlock> blocksDest)
{
if (blockSource != null && !_inSaveBlocks) //(!InSaveBlocks || Blocks != BlocksDest))
{
Sop.DataBlock db = blockSource;
IDictionary <long, Sop.DataBlock> target = blocksDest;
while (db != null)
{
db.IsDirty = false;
long da = GetId(db);
Sop.DataBlock db2 = target[da];
if (db2 != null && db2 != db)
{
if (db2.InternalNextBlockAddress >= 0)
{
if (db2.InternalNextBlockAddress != db.InternalNextBlockAddress)
{
Log.Logger.Instance.Log(Log.LogLevels.Information,
"CollectionOnDisk.AddToBlocks(segment boundary related): db.InternalNextBlockAddress={0}, db2.InternalNextBlockAddress={1}",
db.InternalNextBlockAddress, db2.InternalNextBlockAddress);
db.InternalNextBlockAddress = db2.InternalNextBlockAddress;
}
}
}
target[da] = db;
db = db.Next;
}
}
}
/// <summary>
/// Read or DeSerialize Object from its Source Sop.DataBlock
/// </summary>
/// <param name="source"></param>
/// <param name="target">Destination of Object DeSerialization</param>
/// <returns></returns>
public virtual object ReadFromBlock(Sop.DataBlock source, object target = null)
{
if (source.SizeOccupied > 0)
{
if (OnDiskBinaryReader == null)
{
Open();
}
OnDiskBinaryReader.DataBlock = source;
object r = target;
bool? f = ReadPersistentData(this, OnDiskBinaryReader, ref r);
if (f != null && !f.Value)
{
throw new InvalidOperationException("Can't deserialize.");
}
if (!(r is IInternalPersistent))
{
return(r);
}
((IInternalPersistent)r).DiskBuffer = source;
((IInternalPersistent)r).IsDirty = false;
return(r);
}
return(target);
}
/// <summary>
/// Write Value to the Sop.DataBlock
/// </summary>
protected internal Sop.DataBlock WriteToBlock(object value)
{
if (!(value is IInternalPersistent))
{
var db = CreateBlock();
return(WriteToBlock(value, db, false));
}
int sizeOnDisk = ((IInternalPersistent)value).HintSizeOnDisk;
if (sizeOnDisk == 0 && this is BTreeAlgorithm)
{
sizeOnDisk = ((BTreeAlgorithm)this).HintValueSizeOnDisk;
}
if (((IInternalPersistent)value).DiskBuffer == null)
{
((IInternalPersistent)value).DiskBuffer = CreateBlock();
((IInternalPersistent)value).DiskBuffer.IsHead = true;
}
else
{
((IInternalPersistent)value).DiskBuffer.ClearData();
}
Sop.DataBlock r = WriteToBlock(value, ((IInternalPersistent)value).DiskBuffer);
int sizeWrittenOnStream = r.GetSizeOccupied();
if (sizeOnDisk > sizeWrittenOnStream)
{
var b = new byte[sizeOnDisk - sizeWrittenOnStream];
OnDiskBinaryWriter.Write(b);
}
return(r);
}
internal Sop.DataBlock ReadBlockFromDisk(Algorithm.Collection.ICollectionOnDisk parent, long dataAddress, bool getForRemoval,
Sop.DataBlock block)
{
Sop.DataBlock d = block;
ReadBlockFromDiskOrInitializeIfEmpty(parent, dataAddress, getForRemoval, d);
return(ReadBlock(parent, d, getForRemoval));
}
/// <summary>
/// Read InternalPersistent Value returns the Data (byte array) of a previously saved InternalPersistent.
/// </summary>
/// <returns></returns>
public byte[] ReadPersistent(Sop.DataBlock dataBlock)
{
DataBlock = dataBlock;
int size = ReadInt32();
return(ReadBytes(size));
}
public long GetId(Sop.DataBlock block)
{
if (block == null)
{
return(-1);
}
return(block.DataAddress);
}
/// <summary>
/// Returns flag that tells whether this Collection had been modified or not
/// </summary>
/// <param name="block"></param>
/// <returns></returns>
public bool GetIsDirty(Sop.DataBlock block)
{
if (DataBlockDriver == null)
{
return(block.IsDirty);
}
return(DataBlockDriver.GetIsDirty(block));
}
void IRecyclable.Initialize()
{
_diskBuffer = null;
ParentAddress = -1;
ResetArray(Slots, null);
ChildrenAddresses = null;
Count = 0;
IsDirty = true;
}
/// <summary>
/// Mark this Collection as modified or not
/// </summary>
/// <param name="block"></param>
/// <param name="newValue"></param>
public void SetIsDirty(Sop.DataBlock block, bool newValue)
{
if (DataBlockDriver == null)
{
block.IsDirty = newValue;
}
else
{
DataBlockDriver.SetIsDirty(block, newValue);
}
}
private void SetDiskAddress(Algorithm.Collection.ICollectionOnDisk parent, Sop.DataBlock block, bool addToMru)
{
if (block.DataAddress < 0)
{
// reserve the block chunk of space on disk to the block and put it in MRU,
// MRU Manager will take care of making another call to physically write the
// block when it's appropriate (during MRU fault).
block.DataAddress = parent.FileStream.Position;
parent.FileStream.Seek(block.Length, SeekOrigin.Current);
}
SetIsDirty(block, true); //ensure block will be written by MRUManager
}
/// <summary>
/// Assign block(s) of space on disk to the Data Block(s).
/// The assigned disk blocks' file offsets will be set as the blocks' DataAddresses.
/// </summary>
/// <param name="parent"></param>
/// <param name="block">Data Block which will be assigned a block of space on disk</param>
/// <param name="isCollectionBlock"> </param>
/// <param name="addToMru">true will put the block into MRU, else not</param>
public void SetDiskBlock(Algorithm.Collection.ICollectionOnDisk parent,
Sop.DataBlock block,
bool isCollectionBlock,
bool addToMru = true)
{
Sop.DataBlock prevBlock = null;
bool isHead = true;
bool blockExtended = false;
while (block != null)
{
if (block.DataAddress == -1)
{
if (prevBlock != null)
{
blockExtended = true;
}
AddBlockToDisk(parent, block, isHead, isCollectionBlock, addToMru);
}
if (prevBlock != null)
{
var db = ((CollectionOnDisk)parent).Blocks[GetId(prevBlock)];
if (db != null)
{
if (db.InternalNextBlockAddress != -1)
{
prevBlock.InternalNextBlockAddress = db.InternalNextBlockAddress;
}
((CollectionOnDisk)parent).Blocks[GetId(prevBlock)] = prevBlock;
}
if (prevBlock.NextItemAddress != block.DataAddress)
{
if (blockExtended)
{
Log.Logger.Instance.Log(Log.LogLevels.Verbose, "DataBlockDriver.SetDiskBlock: block({0}) got extended.",
prevBlock.DataAddress);
}
prevBlock.NextItemAddress = block.DataAddress;
SetDiskAddress(parent, prevBlock, addToMru);
}
else
{
//ensure blocks will be written by MRUManager
SetIsDirty(prevBlock, true);
SetIsDirty(block, true);
}
}
prevBlock = block;
block = block.Next;
isHead = false;
}
}
/// <summary>
/// Create a block of Data
/// </summary>
/// <param name="blockSize"></param>
/// <returns></returns>
public Sop.DataBlock CreateBlock(DataBlockSize blockSize)
{
Sop.DataBlock db = null;
if (BlockRecycler != null)
{
db = BlockRecycler.GetRecycledObject();
}
if (db == null)
{
db = new Sop.DataBlock(blockSize);
}
return(db);
}
/// <summary>
/// Clear the HeaderData member fields
/// </summary>
public void Clear()
{
StartAllocatableAddress = 0;
NextAllocatableAddress = -1;
EndAllocatableAddress = -1;
OccupiedBlocksHead = null;
OccupiedBlocksTail = null;
Count = 0;
if (diskBuffer != null)
{
diskBuffer.Initialize();
}
}
/// <summary>
/// Remove ObjectToRemove from the Collection if found, else throws an exception
/// </summary>
public void Remove(object item)
{
if (!Contains(item))
{
return;
}
Sop.DataBlock currBlock = this.GetCurrentDataBlock(true);
if (GetId(currBlock) >= 0)
{
MruManager.Remove(GetId(currBlock), true);
}
DataBlockDriver.Remove(this, currBlock);
}
/// Header:
/// OccupiedBlock Head
/// OccupiedBlock Tail
/// DeletedBlock Head
/// DeletedBlock Tail
///
/// Layout in Disk:
/// Byte 0: Available or Occupied flag
/// Byte 1 to 8: Next Item Address (64 bit long int)
/// Byte 9 to 10: Size Occupied
/// Byte 11 to 11 + Size Occupied: USER DATA
/// Disk Layout:
/// Block 1 -> Block 2 -> Block 3 -> Block n
private void AddBlockToDisk(Algorithm.Collection.ICollectionOnDisk parent, Sop.DataBlock block,
bool isHead, bool isCollectionBlock, bool addToMru)
{
parent.IsDirty = true;
if (block == null)
{
throw new ArgumentNullException("block");
}
Sop.DataBlock currentBlock = block, prevBlock = null;
//** save linked Blocks...
do
{
if (AllocateNextBlock(parent, currentBlock, isHead, isCollectionBlock, addToMru))
{
SetIsDirty(currentBlock, true); //ensure block will be written by MRUManager
Sop.DataBlock db = currentBlock.Next;
while (db != null)
{
SetIsDirty(db, true); //ensure block will be written by MRUManager
if (addToMru && db.SizeOccupied > 0)
{
((CollectionOnDisk)parent).Blocks.Add(db.DataAddress, db);
}
//MruManager.Add(db.DataAddress, db);
db = db.Next;
}
return;
}
SetIsDirty(currentBlock, true); //ensure block will be written by MRUManager
if (prevBlock != null)
{
long prevId = GetId(prevBlock);
Sop.DataBlock db;
if (((CollectionOnDisk)parent).Blocks.TryGetValue(prevId, out db))
{
if (db.InternalNextBlockAddress != -1)
{
prevBlock.InternalNextBlockAddress = db.InternalNextBlockAddress;
}
((CollectionOnDisk)parent).Blocks[prevId] = prevBlock;
}
prevBlock.NextItemAddress = currentBlock.DataAddress;
SetDiskAddress(parent, prevBlock, addToMru);
}
prevBlock = currentBlock;
currentBlock = currentBlock.Next;
isHead = false;
} while (currentBlock != null && currentBlock.DataAddress == -1);
SetDiskAddress(parent, prevBlock, addToMru);
}
/// <summary>
/// Add 'Value' to the Collection
/// </summary>
public long Add(object value)
{
var o = new LinkedItemOnDisk(this.DataBlockSize)
{
Data = value
};
WriteToDisk(o, false);
UpdateCount(UpdateCountType.Increment);
//*** update Current, Last & First
if (LastItem.DiskBuffer.DataAddress != -1)
{
o.PreviousItemAddress = LastItem.DiskBuffer.DataAddress;
WriteToDisk(o, false);
LinkedItemOnDisk biod = CurrentItem;
biod.NextItemAddress = o.DiskBuffer.DataAddress;
Sop.DataBlock db = DataBlockDriver.ReadBlockFromDisk(this, LastItem.DiskBuffer.DataAddress, true);
if (CurrentItem.DiskBuffer.DataAddress != LastItem.DiskBuffer.DataAddress)
{
biod = (LinkedItemOnDisk)ReadFromBlock(db);
biod.NextItemAddress = o.DiskBuffer.DataAddress;
}
else
{
PurifyMeta(biod, db);
if (db.SizeOccupied > 0)
{
biod.DiskBuffer = db;
}
}
WriteToDisk(biod, false);
}
else
{
FirstItem.DiskBuffer.DataAddress = o.DiskBuffer.DataAddress;
}
currentEntry = null;
CurrentItem = o;
CurrentEntryDataAddress = o.DiskBuffer.DataAddress;
currentDataBlock = o.DiskBuffer;
LastItem.DiskBuffer.DataAddress = o.DiskBuffer.DataAddress;
MruManager.Add(CurrentEntryDataAddress, o);
//** update the header
RegisterChange(true);
//IsDirty = true;
return(o.DiskBuffer.DataAddress);
}
private Sop.DataBlock ReadBlock(Algorithm.Collection.ICollectionOnDisk parent, Sop.DataBlock block, bool getForRemoval)
{
// Read the block
if (block.SizeAvailable == 0 || (block.Next == null && block.NextItemAddress >= 0))
{
Sop.DataBlock d = block;
//** read rest of blocks
while (d.NextItemAddress >= 0)
{
d = ReadNextBlock(parent, d, getForRemoval);
}
}
return(block);
}
/// <summary>
/// Read Block from Disk
/// </summary>
/// <returns></returns>
private Sop.DataBlock ReadBlockFromDiskOrInitializeIfEmpty(
Algorithm.Collection.ICollectionOnDisk parent,
long dataAddress, bool getForRemoval, Sop.DataBlock block)
{
if (parent.FileStream.Length > 0)
{
block = ReadBlockFromDisk(parent, dataAddress, getForRemoval, getForRemoval, block);
}
else
{
block.Initialize();
}
return(block);
}
/// <summary>
/// Shared Block on Disk purifies data and only saves one
/// copy of Meta data in it.
/// </summary>
/// <param name="biod"></param>
/// <param name="db"></param>
protected override void PurifyMeta(LinkedItemOnDisk biod, Sop.DataBlock db)
{
if (db.SizeOccupied > 0)
{
//** purify data by separating meta data from it so meta data won't be serialized more than once
int metaDataSize = LinkedListOnDisk.SizeOfMetaData(this);
int newSize = db.SizeOccupied - metaDataSize;
byte[] newData = new byte[newSize];
Array.Copy(db.Data, metaDataSize, newData, 0, newSize);
//** assign data to block for serialization
biod.Data = newData;
}
}
private void AllocateAvailableBlock(CollectionOnDisk parent, HeaderData hd, Sop.DataBlock block)
{
//** Allocate block from the "available" segment
block.DataAddress = hd.NextAllocatableAddress;
hd.NextAllocatableAddress += block.Length;
hd.DiskBuffer.IsDirty = true;
hd.IsModifiedInTransaction = true;
//** register the add of new block to transaction log
Transaction.ITransactionLogger trans = parent.Transaction;
if (trans == null)
{
return;
}
((Transaction.TransactionBase)trans).RegisterAdd(parent, block.DataAddress, block.Length);
}
// deserialize value from entry's disk buffer...
private object GetValue(BTreeItemOnDisk entry)
{
var itm = entry;
if (itm == null)
{
return(null);
}
if (!itm.ValueLoaded || itm.IsDisposed)
{
Sop.DataBlock d = itm.Value.DiskBuffer;
var itemOnDisk = ReadFromBlock(d);
if (itemOnDisk != null)
{
if (itemOnDisk is ItemOnDisk)
{
var iod = (ItemOnDisk)itemOnDisk;
iod.DiskBuffer = itm.Value.DiskBuffer;
itm.Value = iod;
if (iod.Data == null && iod.DataIsUserDefined && onValueUnpack != null)
{
iod.Data = onValueUnpack(OnDiskBinaryReader);
}
// fold DiskBuffer to save memory...
if (iod.Data != null && itm.Value != null && itm.Value.diskBuffer != null)
{
itm.Value.diskBuffer.Fold();
if (itm.Value.diskBuffer.CountMembers() >= File.Profile.BigDataBlockCount)
{
var r = itm.Value.Data;
// nullify big data to conserve memory
itm.Value.Data = null;
return(r);
}
}
}
else
{
throw new SopException(
string.Format("Unexpected item of type {0} was deserialized.",
itemOnDisk.GetType()));
}
}
itm.ValueLoaded = true;
}
return(itm.Value != null ? itm.Value.Data : null);
}
/// <summary>
/// Reads into memory the Collection On Disk's Header Block which contains
/// state information of the collection
/// </summary>
public virtual void Load()
{
if (DiskBuffer == null)
{
throw new InvalidOperationException("'DiskBuffer' is null.");
}
if (DataBlockDriver.GetId(DiskBuffer) < 0)
{
throw new InvalidOperationException("'DiskBuffer.DataAddress' is < 0.");
}
//Blocks.Clear();
//if (DataBlockDriver.MruManager != null)
// DataBlockDriver.MruManager.Clear();
//if (MruManager != null)
// MruManager.Clear();
currentDataBlock = null;
DataBlockDriver.MoveTo(this, DataBlockDriver.GetId(DiskBuffer));
Sop.DataBlock block = GetCurrentDataBlock();
if (block.SizeOccupied > 0)
{
if (block.DataAddress == DiskBuffer.DataAddress &&
DiskBuffer.Data == null)
{
DiskBuffer = block;
}
ReadFromBlock(block, this);
DiskBuffer = block;
}
if (IsDirty)
{
IsDirty = false;
}
//** allow deleted blocks to be loaded its Header and clear its MRU cache...
if (File.Server.HasTrashBin && _deletedBlocksAddress >= 0) // && deletedBlocks.DataAddress >= 0)
{
// ensure Deleted Blocks collection is loaded & initialized.
var o = DeletedBlocks;
//if (DeletedBlocksAddress != DeletedBlocks.DataAddress)
// DeletedBlocks.DataAddress = DeletedBlocksAddress;
deletedBlocks.Load();
}
}
/// <summary>
/// MoveFirst makes the first entry in the Collection the current one
/// </summary>
public override bool MoveFirst()
{
if (Count > 0 && CurrentItem != null)
{
if (FirstItem.DiskBuffer.DataAddress != CurrentEntryDataAddress ||
FirstItem.DiskBuffer.DataAddress != CurrentItem.DiskBuffer.DataAddress)
{
CurrentEntryDataAddress = -1;
this.DataBlockDriver.MoveTo(this, FirstItem.DiskBuffer.DataAddress);
Sop.DataBlock db = DataBlockDriver.ReadBlockFromDisk(this, FirstItem.DiskBuffer.DataAddress, false);
this.currentDataBlock = db;
CurrentItem = (LinkedItemOnDisk)ReadFromBlock(db);
}
return(true);
}
return(false);
}
/// <summary>
/// Get Total Size occupied by the chained Sop.DataBlock.
/// </summary>
/// <returns></returns>
public int GetSizeOccupied(int offset = 0)
{
int totalSize = 0;
Sop.DataBlock h = this;
do
{
if (h.Data != null && h.SizeOccupied > 0)
{
totalSize += h.SizeOccupied;
if (h == this && offset > 0)
totalSize -= offset;
h = h.Next;
}
else
break;
} while (h != null);
return totalSize;
}
/// <summary>
/// Serialize an Object to the target DataBlock
/// </summary>
/// <param name="file"></param>
/// <param name="value"></param>
/// <param name="dataBlock"></param>
public void WriteObject(File.IFile file, object value, Sop.DataBlock dataBlock)
{
if (value == null)
{
throw new ArgumentNullException("value");
}
this.DataBlock = dataBlock;
if (value is IInternalPersistent)
{
((IInternalPersistent)value).Pack(file, this);
}
else
{
_serializer.Serialize(this.BaseStream, value);
this.Write(_buffer.GetBuffer(), 0, (int)this.BaseStream.Position);
}
}
/// <summary>
/// Useful if wanting to DeSerialize back into IInternalPersistent type of object.
/// </summary>
/// <returns></returns>
public byte[] ReadBytes(Sop.DataBlock dataBlock)
{
DataBlock = dataBlock;
#region for removal
//long l = BaseStream.Position;
//var m = (MemoryStream) _reader.BaseStream;
//m.Seek(0, SeekOrigin.End);
//if (l < m.Position)
//{
// long c = m.Position - l;
// m.Seek(l, SeekOrigin.Begin);
// return ReadBytes((int) c);
//}
//return null;
#endregion
long n = _reader.BaseStream.Length - BaseStream.Position;
return(n > 0 ? ReadBytes((int)n) : null);
}
/// <summary>
/// Write Value to the Sop.DataBlock.
/// </summary>
protected internal virtual Sop.DataBlock WriteToBlock(object value,
Sop.DataBlock destination, bool clearBlock)
{
if (destination == null)
{
destination = CreateBlock(); // new Sop.DataBlock(this.DataBlockSize);
}
else if (clearBlock)
{
destination.ClearData();
}
if (OnDiskBinaryWriter != null)
{
OnDiskBinaryWriter.DataBlock = destination;
WritePersistentData(this, value, OnDiskBinaryWriter);
}
return(destination);
}
internal bool ResurfaceDeletedBlockNextSegment(CollectionOnDisk parent,
DeletedBlockInfo dbi, long segmentEnd)
{
//** read next segment of deleted collection
Sop.DataBlock db = CreateBlock(parent.DataBlockSize);
long address = segmentEnd - (int)parent.DataBlockSize;
db = ReadBlockFromDisk(parent, address, true, true, db);
if (db.InternalNextBlockAddress >= 0)
{
dbi.StartBlockAddress = db.InternalNextBlockAddress;
if (parent.File.DeletedCollections != null)
{
parent.File.DeletedCollections.SetTop(dbi);
}
return(true);
}
return(false);
}
public override void Unpack(IInternalPersistent parent,
BinaryReader reader)
{
if (DiskBuffer == null)
{
Sop.DataBlock db = ((OnDiskBinaryReader)reader).DataBlock;
DiskBuffer = db;
}
NextItemAddress = reader.ReadInt64();
PreviousItemAddress = reader.ReadInt64();
long l = reader.ReadInt64();
if (DiskBuffer == null || (l > -1 && DiskBuffer.DataAddress != l))
{
DiskBuffer = ((CollectionOnDisk)parent).DataBlockDriver.
CreateBlock(((CollectionOnDisk)parent).DataBlockSize);
this.DiskBuffer.DataAddress = l;
}
DiskBuffer.IsDirty = false;
base.Unpack(parent, reader);
}
