/// <summary>
/// Initialise the item data from an assembly config
/// </summary>
/// <param name="assemblyConfig"></param>
public DistributedItemData(ItemAssemblyConfig assemblyConfig)
{
this.DataBuildMode = assemblyConfig.ItemBuildMode;
this.TotalNumChunks = assemblyConfig.TotalNumChunks;
this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;
InitialiseChunkPositionLengthDict();
#region Build Internal Data Structure
//Create the data stores as per the assembly config
if (DataBuildMode == DataBuildMode.Memory_Single)
//ItemBuildMode == ItemBuildMode.Both_Single ||)
{
MemoryStream itemStream = new MemoryStream(0);
itemStream.SetLength(ItemBytesLength);
CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
}
else if (DataBuildMode == DataBuildMode.Disk_Single)
{
#region DiskSingle
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
FileStream fileStream;
if (File.Exists(fileName))
{
//If the file already exists the MD5 had better match otherwise we have a problem
try
{
fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
throw new Exception("Wrong place, wrong time, wrong file!");
}
catch (Exception)
{
try
{
File.Delete(fileName);
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
Directory.CreateDirectory(folderLocation);
}
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
fileStream.SetLength(ItemBytesLength);
fileStream.Flush();
}
CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);
if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
#endregion
}
else
{
#region Data Chunks
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
try
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
throw new Exception("Wrong place, wrong time, wrong file!");
}
catch (Exception)
{
try
{
File.Delete(fileName);
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
Directory.CreateDirectory(folderLocation);
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
else
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
//Ensure we start at the beginning of the stream
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
destinationStream.Seek(0, SeekOrigin.Begin);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
#endregion
}
#endregion
}
/// <summary>
/// Sets the item data using the provided data stream. Useful for setting data after deserialisation
/// </summary>
/// <param name="itemIdentifier"></param>
/// <param name="itemDataStream"></param>
public void SetData(string itemIdentifier, Stream itemDataStream)
{
lock (dataLocker)
{
if (itemIdentifier == null) throw new ArgumentNullException("itemIdentifier");
if (itemDataStream == null) throw new ArgumentNullException("itemDataStream");
#region Build Internal Data Structure
if (DataBuildMode == DataBuildMode.Disk_Single ||
//ItemBuildMode == ItemBuildMode.Both_Single ||
DataBuildMode == DataBuildMode.Memory_Single)
{
CompleteDataStream = new StreamTools.ThreadSafeStream(itemDataStream);
}
else
{
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
//If the itemDataStream is a memory stream we can try to access the buffer as it makes creating the streams more efficient
byte[] itemDataStreamBuffer = null;
if (itemDataStream is MemoryStream)
{
try
{
itemDataStreamBuffer = ((MemoryStream)itemDataStream).GetBuffer();
}
catch (UnauthorizedAccessException) { /* Ignore */ }
}
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
if (itemDataStreamBuffer != null)
//This is the fastest way to create a block of data streams
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(new MemoryStream(itemDataStreamBuffer, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length));
else
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, itemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
//if (StreamTools.MD5(destinationStream) != checksum)
// throw new Exception("Wrong place, wrong time, wrong file!");
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
Directory.CreateDirectory(folderLocation);
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
}
if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
else
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
//Ensure we start at the beginning of the stream
destinationStream.Seek(0, SeekOrigin.Begin);
//Copy over the correct part of the itemDataStream to the chunks
StreamTools.Write(itemDataStream, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length, destinationStream, 8192, double.MaxValue, int.MaxValue);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
}
}
#endregion
this.CompleteDataCheckSum = StreamTools.MD5(itemDataStream);
}
}
/// <summary>
/// Initialise the item data using existing chunk streams. Build mode must be to blocks.
/// </summary>
/// <param name="dataBuildMode"></param>
/// <param name="itemDataStreams"></param>
/// <param name="enableChunkChecksum"></param>
public DistributedItemData(DataBuildMode dataBuildMode, Dictionary<int, Stream> itemDataStreams, bool enableChunkChecksum = false)
{
this.DataBuildMode = DataBuildMode;
if (itemDataStreams == null) throw new ArgumentNullException("itemDataStreams", "itemDataStreams cannot be null.");
if (dataBuildMode == DataBuildMode.Disk_Single ||
//itemBuildMode == ItemBuildMode.Both_Single ||
dataBuildMode == DataBuildMode.Memory_Single)
throw new ArgumentException("Please use other constructor that takes a single input data stream.");
this.ItemBytesLength = itemDataStreams.Select(i => i.Value.Length).Sum();
//Calculate the exactChunkSize if we split everything up into 255 pieces
double exactChunkSize = (double)ItemBytesLength / 255.0;
//If the item is too small we just use the minimumChunkSize
//If we need something larger than MinChunkSizeInBytes we select appropriately
this.ChunkSizeInBytes = (exactChunkSize <= DFS.MinChunkSizeInBytes ? DFS.MinChunkSizeInBytes : (int)Math.Ceiling(exactChunkSize));
this.TotalNumChunks = (byte)(Math.Ceiling((double)ItemBytesLength / (double)ChunkSizeInBytes));
InitialiseChunkPositionLengthDict();
if (itemDataStreams.Count != ChunkPositionLengthDict.Count)
throw new ArgumentException("Number of streams should equal the number of chunks");
//Initialise the data streams
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
foreach (int chunkIndex in ChunkPositionLengthDict.Keys)
ChunkDataStreams[chunkIndex] = new StreamTools.ThreadSafeStream(itemDataStreams[chunkIndex]);
this.CompleteDataCheckSum = MD5();
if (enableChunkChecksum) BuildChunkCheckSums();
}