/// <summary>
/// Initialise the item data from an assembly config
/// </summary>
/// <param name="assemblyConfig"></param>
public DistributedItemData(ItemAssemblyConfig assemblyConfig)
{
this.DataBuildMode = assemblyConfig.ItemBuildMode;
this.TotalNumChunks = assemblyConfig.TotalNumChunks;
this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;
InitialiseChunkPositionLengthDict();
#region Build Internal Data Structure
//Create the data stores as per the assembly config
if (DataBuildMode == DataBuildMode.Memory_Single)
//ItemBuildMode == ItemBuildMode.Both_Single ||)
{
MemoryStream itemStream = new MemoryStream(0);
itemStream.SetLength(ItemBytesLength);
CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
}
else if (DataBuildMode == DataBuildMode.Disk_Single)
{
#region DiskSingle
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
FileStream fileStream;
if (File.Exists(fileName))
{
//If the file already exists the MD5 had better match otherwise we have a problem
try
{
fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
{
throw new Exception("Wrong place, wrong time, wrong file!");
}
}
catch (Exception)
{
try
{
File.Delete(fileName);
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
fileStream.SetLength(ItemBytesLength);
fileStream.Flush();
}
CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
#endregion
}
else
{
#region Data Chunks
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
try
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
{
throw new Exception("Wrong place, wrong time, wrong file!");
}
}
catch (Exception)
{
try
{
File.Delete(fileName);
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
}
else
{
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
}
//Ensure we start at the beginning of the stream
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
destinationStream.Seek(0, SeekOrigin.Begin);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
#endregion
}
#endregion
}
/// <summary>
/// TCP - Received by this DFS if a server is telling this instance to build a local file
/// </summary>
/// <param name="packetHeader"></param>
/// <param name="connection"></param>
/// <param name="assemblyConfig"></param>
private static void IncomingLocalItemBuild(PacketHeader packetHeader, Connection connection, ItemAssemblyConfig assemblyConfig)
{
//We start the build in the DFS task factory as it will be a long lived task
//BuildTaskFactory.StartNew(() =>
Action assembleAction = new Action(() =>
{
DistributedItem newItem = null;
byte[] itemBytes = null;
try
{
if (assemblyConfig == null)
throw new NullReferenceException("AssemblyConfig should not be null.");
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild from " + connection + " for item " + assemblyConfig.CompleteDataCheckSum + ".");
//We check to see if we already have the necessary file locally
lock (globalDFSLocker)
{
if (swarmedItemsDict.ContainsKey(assemblyConfig.CompleteDataCheckSum))
{
if (swarmedItemsDict[assemblyConfig.CompleteDataCheckSum].Data.ItemBytesLength != assemblyConfig.TotalItemSizeInBytes)
throw new Exception("Possible MD5 conflict detected.");
else
newItem = swarmedItemsDict[assemblyConfig.CompleteDataCheckSum];
}
else
{
newItem = new DistributedItem(assemblyConfig);
swarmedItemsDict.Add(assemblyConfig.CompleteDataCheckSum, newItem);
}
}
//Ensure all possible local listeners are added here
List<ConnectionInfo> seedConnectionInfoList = (from current in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP) select new ConnectionInfo(ConnectionType.TCP, NetworkComms.NetworkIdentifier, current, true)).ToList();
foreach (ConnectionInfo info in seedConnectionInfoList)
newItem.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(info, newItem.SwarmChunkAvailability.PeerChunkAvailability(NetworkComms.NetworkIdentifier), newItem.SwarmChunkAvailability.PeerIsSuperPeer(NetworkComms.NetworkIdentifier), false);
//Build the item from the swarm
//If the item is already complete this will return immediately
newItem.AssembleItem((int)(ItemBuildTimeoutSecsPerMB * (assemblyConfig.TotalItemSizeInBytes / (1024.0 * 1024.0))));
//Once complete we pass the item bytes back into network comms
//If an exception is thrown we will probably not call this method, timeouts in other areas should then handle and can restart the build.
if (newItem.LocalItemComplete() && assemblyConfig.CompletedPacketType != "")
{
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild completed for item with MD5 " + assemblyConfig.CompleteDataCheckSum + ". Item build target is " + assemblyConfig.ItemBuildMode + ".");
itemBytes = newItem.GetCompletedItemBytes();
}
else if (assemblyConfig.CompletedPacketType != "")
RemoveItem(assemblyConfig.CompleteDataCheckSum);
if (DFS.loggingEnabled)
{
Exception exceptionToLogWith = new Exception("Build completed successfully. Logging was enabled so saving build log.");
string fileName = "DFSItemBuildLog_" + newItem.ItemIdentifier + "_" + NetworkComms.NetworkIdentifier;
if (newItem != null)
LogTools.LogException(exceptionToLogWith, fileName, newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
else
LogTools.LogException(exceptionToLogWith, fileName, "newItem==null so no build log was available.");
}
}
catch (ObjectDisposedException)
{
//The item was closed during assemble, no need to log an errors here
RemoveItem(assemblyConfig.CompleteDataCheckSum);
}
catch (CommsException e)
{
//Crap an error has happened, let people know we probably don't have a good file
RemoveItem(assemblyConfig.CompleteDataCheckSum);
//connection.CloseConnection(true, 30);
//LogTools.LogException(e, "CommsError_IncomingLocalItemBuild");
if (newItem != null)
LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
else
LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", "newItem==null so no build log was available.");
}
catch (Exception e)
{
//Crap an error has happened, let people know we probably don't have a good file
RemoveItem(assemblyConfig.CompleteDataCheckSum);
//connection.CloseConnection(true, 31);
if (newItem != null)
LogTools.LogException(e, "Error_IncomingLocalItemBuild", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
else
LogTools.LogException(e, "Error_IncomingLocalItemBuild", "newItem==null so no build log was available.");
}
//finally
//{
//Putting any code here appears to cause a sigsegv fault on leaving the finally in mono
//Just moved the code out to below as it makes no difference
//}
//Regardless of if the item completed we call the necessary packet handlers
//If there was a build error we just pass null data to the handlers so that the errors can get called up the relevant stack traces.
try
{
PacketHeader itemPacketHeader = new PacketHeader(assemblyConfig.CompletedPacketType, newItem == null ? 0 : newItem.Data.ItemBytesLength);
//We set the item checksum so that the entire distributed item can be easily retrieved later
itemPacketHeader.SetOption(PacketHeaderStringItems.PacketIdentifier, newItem == null ? "" : newItem.ItemTypeStr + "|" + newItem.ItemIdentifier + "|" + newItem.Data.CompleteDataCheckSum);
var dataStream = (itemBytes == null ? new MemoryStream(new byte[0], 0, 0, false, true) : new MemoryStream(itemBytes, 0, itemBytes.Length, false, true));
var sendRecieveOptions = new SendReceiveOptions<NullSerializer>(new Dictionary<string, string>());
NetworkComms.TriggerAllPacketHandlers(itemPacketHeader, connection, dataStream, sendRecieveOptions);
}
catch (Exception ex)
{
LogTools.LogException(ex, "Error_IncomingLocalItemBuildFinal");
}
});
if (BuildTaskFactory == null)
LogTools.LogException(new NullReferenceException("BuildTaskFactory is null in IncomingLocalItemBuild"), "IncomingLocalBuildError");
else
//Thread buildThread = new Thread(buildAction);
//buildThread.Name = "DFS_" + assemblyConfig.ItemIdentifier + "_Build";
//buildThread.Start();
BuildTaskFactory.StartNew(assembleAction);
}
/// <summary>
/// Introduces a new item into the swarm and sends a build command to the originating requester
/// </summary>
/// <param name="peerConnection">The peer which requested the DFS item</param>
/// <param name="itemToDistribute">The item to be distributed</param>
/// <param name="completedPacketType">The packet type to use once the item has been fully assembled</param>
public static void PushItemToPeer(Connection peerConnection, DistributedItem itemToDistribute, string completedPacketType)
{
try
{
if (peerConnection.ConnectionInfo.ConnectionType != ConnectionType.TCP)
throw new Exception("Only able to push DFS item when the request is made via TCP.");
if (itemToDistribute.ItemClosed)
throw new ArgumentException("Unable to push a closed item.");
ItemAssemblyConfig assemblyConfig;
lock (globalDFSLocker)
{
//First double check to see if it's already in the swarm
if (!ItemAlreadyInLocalCache(itemToDistribute))
swarmedItemsDict.Add(itemToDistribute.Data.CompleteDataCheckSum, itemToDistribute);
else
itemToDistribute = swarmedItemsDict[itemToDistribute.Data.CompleteDataCheckSum];
}
itemToDistribute.IncrementPushCount();
//We add the requester to the item swarm at this point
itemToDistribute.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(peerConnection.ConnectionInfo, new ChunkFlags(0));
//There is a possibility when we create this assembly config that the peer has been removed again
//We handle this on the peer end
assemblyConfig = new ItemAssemblyConfig(itemToDistribute, completedPacketType);
//Send the config information to the client that wanted the file
peerConnection.SendObject("DFS_IncomingLocalItemBuild", assemblyConfig, nullCompressionSRO);
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("Pushed DFS item " + itemToDistribute.Data.CompleteDataCheckSum + " to peer " + peerConnection + ".");
}
catch (CommsException)
{
//LogTools.LogException(ex, "CommsError_AddItemToSwarm");
}
catch (Exception ex)
{
LogTools.LogException(ex, "Error_AddItemToSwarm");
}
//try { GC.Collect(); }
//catch (Exception) { }
}
/// <summary>
/// Initialise the item data from an assembly config
/// </summary>
/// <param name="assemblyConfig"></param>
public DistributedItemData(ItemAssemblyConfig assemblyConfig)
{
this.DataBuildMode = assemblyConfig.ItemBuildMode;
this.TotalNumChunks = assemblyConfig.TotalNumChunks;
this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;
InitialiseChunkPositionLengthDict();
#region Build Internal Data Structure
//Create the data stores as per the assembly config
if (DataBuildMode == DataBuildMode.Memory_Single)
//ItemBuildMode == ItemBuildMode.Both_Single ||)
{
MemoryStream itemStream = new MemoryStream(0);
itemStream.SetLength(ItemBytesLength);
CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
}
else if (DataBuildMode == DataBuildMode.Disk_Single)
{
#region DiskSingle
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
FileStream fileStream;
if (File.Exists(fileName))
{
//If the file already exists the MD5 had better match otherwise we have a problem
try
{
fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
throw new Exception("Wrong place, wrong time, wrong file!");
}
catch (Exception)
{
try
{
File.Delete(fileName);
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
Directory.CreateDirectory(folderLocation);
}
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
fileStream.SetLength(ItemBytesLength);
fileStream.Flush();
}
CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);
if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
#endregion
}
else
{
#region Data Chunks
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
try
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
throw new Exception("Wrong place, wrong time, wrong file!");
}
catch (Exception)
{
try
{
File.Delete(fileName);
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
Directory.CreateDirectory(folderLocation);
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
else
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
//Ensure we start at the beginning of the stream
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
destinationStream.Seek(0, SeekOrigin.Begin);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
#endregion
}
#endregion
}