/// <summary>
/// Initialise the item data from an assembly config
/// </summary>
/// <param name="assemblyConfig"></param>
public DistributedItemData(ItemAssemblyConfig assemblyConfig)
{
this.DataBuildMode = assemblyConfig.ItemBuildMode;
this.TotalNumChunks = assemblyConfig.TotalNumChunks;
this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;
InitialiseChunkPositionLengthDict();
#region Build Internal Data Structure
//Create the data stores as per the assembly config
if (DataBuildMode == DataBuildMode.Memory_Single)
//ItemBuildMode == ItemBuildMode.Both_Single ||)
{
MemoryStream itemStream = new MemoryStream(0);
itemStream.SetLength(ItemBytesLength);
CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
}
else if (DataBuildMode == DataBuildMode.Disk_Single)
{
#region DiskSingle
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
FileStream fileStream;
if (File.Exists(fileName))
{
//If the file already exists the MD5 had better match otherwise we have a problem
try
{
fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
{
throw new Exception("Wrong place, wrong time, wrong file!");
}
}
catch (Exception)
{
try
{
File.Delete(fileName);
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
fileStream.SetLength(ItemBytesLength);
fileStream.Flush();
}
CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
#endregion
}
else
{
#region Data Chunks
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
try
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
{
throw new Exception("Wrong place, wrong time, wrong file!");
}
}
catch (Exception)
{
try
{
File.Delete(fileName);
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
}
else
{
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
}
//Ensure we start at the beginning of the stream
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
destinationStream.Seek(0, SeekOrigin.Begin);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
#endregion
}
#endregion
}
/// <summary>
/// Sets the item data using the provided data stream. Useful for setting data after deserialisation
/// </summary>
/// <param name="itemIdentifier"></param>
/// <param name="itemDataStream"></param>
public void SetData(string itemIdentifier, Stream itemDataStream)
{
lock (dataLocker)
{
if (itemIdentifier == null)
{
throw new ArgumentNullException("itemIdentifier");
}
if (itemDataStream == null)
{
throw new ArgumentNullException("itemDataStream");
}
#region Build Internal Data Structure
if (DataBuildMode == DataBuildMode.Disk_Single ||
//ItemBuildMode == ItemBuildMode.Both_Single ||
DataBuildMode == DataBuildMode.Memory_Single)
{
CompleteDataStream = new StreamTools.ThreadSafeStream(itemDataStream);
}
else
{
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
//If the itemDataStream is a memory stream we can try to access the buffer as it makes creating the streams more efficient
byte[] itemDataStreamBuffer = null;
if (itemDataStream is MemoryStream)
{
try
{
itemDataStreamBuffer = ((MemoryStream)itemDataStream).GetBuffer();
}
catch (UnauthorizedAccessException) { /* Ignore */ }
}
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
if (itemDataStreamBuffer != null)
{
//This is the fastest way to create a block of data streams
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(new MemoryStream(itemDataStreamBuffer, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length));
}
else
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, itemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
//if (StreamTools.MD5(destinationStream) != checksum)
// throw new Exception("Wrong place, wrong time, wrong file!");
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
}
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
}
else
{
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
}
//Ensure we start at the beginning of the stream
destinationStream.Seek(0, SeekOrigin.Begin);
//Copy over the correct part of the itemDataStream to the chunks
StreamTools.Write(itemDataStream, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length, destinationStream, 8192, double.MaxValue, int.MaxValue);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
}
}
#endregion
this.CompleteDataCheckSum = StreamTools.MD5(itemDataStream);
}
}
private void Constructor <payloadObjectType>(string sendingPacketTypeStr, string requestReturnPacketTypeStr, payloadObjectType payloadObject, SendReceiveOptions options, bool isNested)
{
if (sendingPacketTypeStr == null || sendingPacketTypeStr == "")
{
throw new ArgumentNullException("sendingPacketTypeStr", "The provided string can not be null or zero length.");
}
if (options == null)
{
throw new ArgumentNullException("options", "The provided SendReceiveOptions cannot be null.");
}
if (options.DataSerializer == null)
{
throw new ArgumentNullException("options", "The provided SendReceiveOptions.DataSerializer cannot be null. Consider using NullSerializer instead.");
}
//Check for security critical data processors
//There may be performance issues here
bool containsSecurityCritialDataProcessors = false;
if (!options.Options.ContainsKey("UseNestedPacketType") && //We only need to perform this check if we are not already using a nested packet
!isNested) //We do not perform this check within a nested packet
{
foreach (DataProcessor processor in options.DataProcessors)
{
if (processor.IsSecurityCritical)
{
containsSecurityCritialDataProcessors = true;
break;
}
}
}
//By default the object to serialise will be the payloadObject
object objectToSerialise = payloadObject;
bool objectToSerialiseIsNull = false;
//We only replace the null with an empty stream if this is either in the nested packet
//or we will not be nesting
if (objectToSerialise == null &&
((!options.Options.ContainsKey("UseNestedPacketType") &&
!containsSecurityCritialDataProcessors) || isNested))
{
#if NETFX_CORE
var emptyStream = new MemoryStream(new byte[0], 0, 0, false);
#else
var emptyStream = new MemoryStream(new byte[0], 0, 0, false, true);
#endif
//If the sending object is null we set objectToSerialiseIsNull and create a zero length StreamSendWrapper
//The zero length StreamSendWrapper can then be passed to any data serializers
objectToSerialiseIsNull = true;
objectToSerialise = new StreamTools.StreamSendWrapper(new StreamTools.ThreadSafeStream(emptyStream, true));
}
//If we need to nest this packet
if ((containsSecurityCritialDataProcessors || options.Options.ContainsKey("UseNestedPacketType")) && !isNested)
{
//We set the objectToSerialise to a nested packet
objectToSerialise = new Packet(sendingPacketTypeStr, requestReturnPacketTypeStr, payloadObject, options, true);
}
else if (isNested)
{
//Serialise the payload object into byte[]
//We do not use any data processors at this stage as the object will be processed again one level higher.
#if NETFX_CORE
_payloadObjectBytes = options.DataSerializer.SerialiseDataObject(payloadObject).ThreadSafeStream.ToArray();
_payloadSize = _payloadObjectBytes.Length;
#else
NetworkCommsDotNet.Tools.StreamTools.ThreadSafeStream tempStream = options.DataSerializer.SerialiseDataObject(objectToSerialise).ThreadSafeStream;
_payloadObjectBytes = tempStream.GetBuffer();
_payloadSize = (int)tempStream.Length;
#endif
//Set the packet header
//THe nulls represent internal SendReceiveOptions and no checksum
this._packetHeader = new PacketHeader(sendingPacketTypeStr, _payloadSize, null, requestReturnPacketTypeStr, null);
//Set the deserialiser information in the nested packet header, excluding data processors
this._packetHeader.SetOption(PacketHeaderLongItems.SerializerProcessors, DPSManager.CreateSerializerDataProcessorIdentifier(options.DataSerializer, null));
}
//If we are at the top level packet we can finish off the serialisation
if (!isNested)
{
//Set the payload stream data.
if (objectToSerialiseIsNull && options.DataProcessors.Count == 0)
{
//Only if there are no data processors can we use a zero length array for nulls
//This ensures that should there be any required padding we can include it
this.payloadStream = (StreamTools.StreamSendWrapper)objectToSerialise;
}
else
{
if (objectToSerialise is Packet)
{
//We have to use the internal explicit serializer for nested packets (the nested data is already byte[])
this.payloadStream = NetworkComms.InternalFixedSendReceiveOptions.DataSerializer.SerialiseDataObject(objectToSerialise, options.DataProcessors, options.Options);
}
else
{
this.payloadStream = options.DataSerializer.SerialiseDataObject(objectToSerialise, options.DataProcessors, options.Options);
}
}
//We only calculate the checkSum if we are going to use it
string hashStr = null;
if (NetworkComms.EnablePacketCheckSumValidation)
{
hashStr = StreamTools.MD5(payloadStream.ThreadSafeStream.ToArray(payloadStream.Start, payloadStream.Length));
}
//Choose the sending and receiving packet type depending on if it is being used with a nested packet
string _sendingPacketTypeStr;
string _requestReturnPacketTypeStr = null;
if (containsSecurityCritialDataProcessors || options.Options.ContainsKey("UseNestedPacketType"))
{
_sendingPacketTypeStr = Enum.GetName(typeof(ReservedPacketType), ReservedPacketType.NestedPacket);
}
else
{
_sendingPacketTypeStr = sendingPacketTypeStr;
_requestReturnPacketTypeStr = requestReturnPacketTypeStr;
}
this._packetHeader = new PacketHeader(_sendingPacketTypeStr, payloadStream.Length, options, _requestReturnPacketTypeStr, hashStr);
//Add an identifier specifying the serialisers and processors we have used
if (objectToSerialise is Packet)
{
this._packetHeader.SetOption(PacketHeaderLongItems.SerializerProcessors, DPSManager.CreateSerializerDataProcessorIdentifier(NetworkComms.InternalFixedSendReceiveOptions.DataSerializer, options.DataProcessors));
}
else
{
this._packetHeader.SetOption(PacketHeaderLongItems.SerializerProcessors, DPSManager.CreateSerializerDataProcessorIdentifier(options.DataSerializer, options.DataProcessors));
}
}
//Set the null data header section if required
if (objectToSerialiseIsNull &&
((!containsSecurityCritialDataProcessors && !options.Options.ContainsKey("UseNestedPacketType")) || isNested))
{
this._packetHeader.SetOption(PacketHeaderStringItems.NullDataSection, "");
}
if (NetworkComms.LoggingEnabled)
{
if (isNested)
{
NetworkComms.Logger.Trace(" ... created nested packet of type " + sendingPacketTypeStr);
}
else
{
NetworkComms.Logger.Trace(" ... created packet of type " + sendingPacketTypeStr + ". PacketObject data size is " + payloadStream.Length.ToString() + " bytes");
}
}
}