/// <summary> /// Initialise the item data from an assembly config /// </summary> /// <param name="assemblyConfig"></param> public DistributedItemData(ItemAssemblyConfig assemblyConfig) { this.DataBuildMode = assemblyConfig.ItemBuildMode; this.TotalNumChunks = assemblyConfig.TotalNumChunks; this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes; this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum; this.ChunkCheckSums = assemblyConfig.ChunkCheckSums; this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes; InitialiseChunkPositionLengthDict(); #region Build Internal Data Structure //Create the data stores as per the assembly config if (DataBuildMode == DataBuildMode.Memory_Single) //ItemBuildMode == ItemBuildMode.Both_Single ||) { MemoryStream itemStream = new MemoryStream(0); itemStream.SetLength(ItemBytesLength); CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream); } else if (DataBuildMode == DataBuildMode.Disk_Single) { #region DiskSingle string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData"); FileStream fileStream; if (File.Exists(fileName)) { //If the file already exists the MD5 had better match otherwise we have a problem try { fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum) { throw new Exception("Wrong place, wrong time, wrong file!"); } } catch (Exception) { try { File.Delete(fileName); fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose); } catch (Exception) { throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue."); } } } else { lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) { Directory.CreateDirectory(folderLocation); } } fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose); fileStream.SetLength(ItemBytesLength); fileStream.Flush(); } CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream); if (!File.Exists(fileName)) { throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); } #endregion } else { #region Data Chunks //Break the itemDataStream into blocks ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count]; for (int i = 0; i < ChunkPositionLengthDict.Count; i++) { //We now need to available the respective data into blocks Stream destinationStream = null; if (DataBuildMode == DataBuildMode.Disk_Blocks) { string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString()); if (File.Exists(fileName)) { try { destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i]) { throw new Exception("Wrong place, wrong time, wrong file!"); } } catch (Exception) { try { File.Delete(fileName); destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); } catch (Exception) { throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue."); } } } else { //Create the folder if it does not exist yet lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) { Directory.CreateDirectory(folderLocation); } } destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); } if (!File.Exists(fileName)) { throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); } } else { //If we are not exclusively building to the disk we just use a memory stream at this stage destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length); } //Ensure we start at the beginning of the stream destinationStream.SetLength(ChunkPositionLengthDict[i].Length); destinationStream.Flush(); destinationStream.Seek(0, SeekOrigin.Begin); ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream); } #endregion } #endregion }
/// <summary> /// TCP - Received by this DFS if a server is telling this instance to build a local file /// </summary> /// <param name="packetHeader"></param> /// <param name="connection"></param> /// <param name="assemblyConfig"></param> private static void IncomingLocalItemBuild(PacketHeader packetHeader, Connection connection, ItemAssemblyConfig assemblyConfig) { //We start the build in the DFS task factory as it will be a long lived task //BuildTaskFactory.StartNew(() => Action assembleAction = new Action(() => { DistributedItem newItem = null; byte[] itemBytes = null; try { if (assemblyConfig == null) throw new NullReferenceException("AssemblyConfig should not be null."); if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild from " + connection + " for item " + assemblyConfig.CompleteDataCheckSum + "."); //We check to see if we already have the necessary file locally lock (globalDFSLocker) { if (swarmedItemsDict.ContainsKey(assemblyConfig.CompleteDataCheckSum)) { if (swarmedItemsDict[assemblyConfig.CompleteDataCheckSum].Data.ItemBytesLength != assemblyConfig.TotalItemSizeInBytes) throw new Exception("Possible MD5 conflict detected."); else newItem = swarmedItemsDict[assemblyConfig.CompleteDataCheckSum]; } else { newItem = new DistributedItem(assemblyConfig); swarmedItemsDict.Add(assemblyConfig.CompleteDataCheckSum, newItem); } } //Ensure all possible local listeners are added here List<ConnectionInfo> seedConnectionInfoList = (from current in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP) select new ConnectionInfo(ConnectionType.TCP, NetworkComms.NetworkIdentifier, current, true)).ToList(); foreach (ConnectionInfo info in seedConnectionInfoList) newItem.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(info, newItem.SwarmChunkAvailability.PeerChunkAvailability(NetworkComms.NetworkIdentifier), newItem.SwarmChunkAvailability.PeerIsSuperPeer(NetworkComms.NetworkIdentifier), false); //Build the item from the swarm //If the item is already complete this will return immediately newItem.AssembleItem((int)(ItemBuildTimeoutSecsPerMB * (assemblyConfig.TotalItemSizeInBytes / (1024.0 * 1024.0)))); //Once complete we pass the item bytes back into network comms //If an exception is thrown we will probably not call this method, timeouts in other areas should then handle and can restart the build. if (newItem.LocalItemComplete() && assemblyConfig.CompletedPacketType != "") { if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild completed for item with MD5 " + assemblyConfig.CompleteDataCheckSum + ". Item build target is " + assemblyConfig.ItemBuildMode + "."); itemBytes = newItem.GetCompletedItemBytes(); } else if (assemblyConfig.CompletedPacketType != "") RemoveItem(assemblyConfig.CompleteDataCheckSum); if (DFS.loggingEnabled) { Exception exceptionToLogWith = new Exception("Build completed successfully. Logging was enabled so saving build log."); string fileName = "DFSItemBuildLog_" + newItem.ItemIdentifier + "_" + NetworkComms.NetworkIdentifier; if (newItem != null) LogTools.LogException(exceptionToLogWith, fileName, newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; })); else LogTools.LogException(exceptionToLogWith, fileName, "newItem==null so no build log was available."); } } catch (ObjectDisposedException) { //The item was closed during assemble, no need to log an errors here RemoveItem(assemblyConfig.CompleteDataCheckSum); } catch (CommsException e) { //Crap an error has happened, let people know we probably don't have a good file RemoveItem(assemblyConfig.CompleteDataCheckSum); //connection.CloseConnection(true, 30); //LogTools.LogException(e, "CommsError_IncomingLocalItemBuild"); if (newItem != null) LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; })); else LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", "newItem==null so no build log was available."); } catch (Exception e) { //Crap an error has happened, let people know we probably don't have a good file RemoveItem(assemblyConfig.CompleteDataCheckSum); //connection.CloseConnection(true, 31); if (newItem != null) LogTools.LogException(e, "Error_IncomingLocalItemBuild", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; })); else LogTools.LogException(e, "Error_IncomingLocalItemBuild", "newItem==null so no build log was available."); } //finally //{ //Putting any code here appears to cause a sigsegv fault on leaving the finally in mono //Just moved the code out to below as it makes no difference //} //Regardless of if the item completed we call the necessary packet handlers //If there was a build error we just pass null data to the handlers so that the errors can get called up the relevant stack traces. try { PacketHeader itemPacketHeader = new PacketHeader(assemblyConfig.CompletedPacketType, newItem == null ? 0 : newItem.Data.ItemBytesLength); //We set the item checksum so that the entire distributed item can be easily retrieved later itemPacketHeader.SetOption(PacketHeaderStringItems.PacketIdentifier, newItem == null ? "" : newItem.ItemTypeStr + "|" + newItem.ItemIdentifier + "|" + newItem.Data.CompleteDataCheckSum); var dataStream = (itemBytes == null ? new MemoryStream(new byte[0], 0, 0, false, true) : new MemoryStream(itemBytes, 0, itemBytes.Length, false, true)); var sendRecieveOptions = new SendReceiveOptions<NullSerializer>(new Dictionary<string, string>()); NetworkComms.TriggerAllPacketHandlers(itemPacketHeader, connection, dataStream, sendRecieveOptions); } catch (Exception ex) { LogTools.LogException(ex, "Error_IncomingLocalItemBuildFinal"); } }); if (BuildTaskFactory == null) LogTools.LogException(new NullReferenceException("BuildTaskFactory is null in IncomingLocalItemBuild"), "IncomingLocalBuildError"); else //Thread buildThread = new Thread(buildAction); //buildThread.Name = "DFS_" + assemblyConfig.ItemIdentifier + "_Build"; //buildThread.Start(); BuildTaskFactory.StartNew(assembleAction); }
/// <summary> /// Introduces a new item into the swarm and sends a build command to the originating requester /// </summary> /// <param name="peerConnection">The peer which requested the DFS item</param> /// <param name="itemToDistribute">The item to be distributed</param> /// <param name="completedPacketType">The packet type to use once the item has been fully assembled</param> public static void PushItemToPeer(Connection peerConnection, DistributedItem itemToDistribute, string completedPacketType) { try { if (peerConnection.ConnectionInfo.ConnectionType != ConnectionType.TCP) throw new Exception("Only able to push DFS item when the request is made via TCP."); if (itemToDistribute.ItemClosed) throw new ArgumentException("Unable to push a closed item."); ItemAssemblyConfig assemblyConfig; lock (globalDFSLocker) { //First double check to see if it's already in the swarm if (!ItemAlreadyInLocalCache(itemToDistribute)) swarmedItemsDict.Add(itemToDistribute.Data.CompleteDataCheckSum, itemToDistribute); else itemToDistribute = swarmedItemsDict[itemToDistribute.Data.CompleteDataCheckSum]; } itemToDistribute.IncrementPushCount(); //We add the requester to the item swarm at this point itemToDistribute.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(peerConnection.ConnectionInfo, new ChunkFlags(0)); //There is a possibility when we create this assembly config that the peer has been removed again //We handle this on the peer end assemblyConfig = new ItemAssemblyConfig(itemToDistribute, completedPacketType); //Send the config information to the client that wanted the file peerConnection.SendObject("DFS_IncomingLocalItemBuild", assemblyConfig, nullCompressionSRO); if (DFS.loggingEnabled) DFS._DFSLogger.Debug("Pushed DFS item " + itemToDistribute.Data.CompleteDataCheckSum + " to peer " + peerConnection + "."); } catch (CommsException) { //LogTools.LogException(ex, "CommsError_AddItemToSwarm"); } catch (Exception ex) { LogTools.LogException(ex, "Error_AddItemToSwarm"); } //try { GC.Collect(); } //catch (Exception) { } }
/// <summary> /// Initialise the item data from an assembly config /// </summary> /// <param name="assemblyConfig"></param> public DistributedItemData(ItemAssemblyConfig assemblyConfig) { this.DataBuildMode = assemblyConfig.ItemBuildMode; this.TotalNumChunks = assemblyConfig.TotalNumChunks; this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes; this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum; this.ChunkCheckSums = assemblyConfig.ChunkCheckSums; this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes; InitialiseChunkPositionLengthDict(); #region Build Internal Data Structure //Create the data stores as per the assembly config if (DataBuildMode == DataBuildMode.Memory_Single) //ItemBuildMode == ItemBuildMode.Both_Single ||) { MemoryStream itemStream = new MemoryStream(0); itemStream.SetLength(ItemBytesLength); CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream); } else if (DataBuildMode == DataBuildMode.Disk_Single) { #region DiskSingle string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData"); FileStream fileStream; if (File.Exists(fileName)) { //If the file already exists the MD5 had better match otherwise we have a problem try { fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum) throw new Exception("Wrong place, wrong time, wrong file!"); } catch (Exception) { try { File.Delete(fileName); fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose); } catch (Exception) { throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue."); } } } else { lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) Directory.CreateDirectory(folderLocation); } fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose); fileStream.SetLength(ItemBytesLength); fileStream.Flush(); } CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream); if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); #endregion } else { #region Data Chunks //Break the itemDataStream into blocks ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count]; for (int i = 0; i < ChunkPositionLengthDict.Count; i++) { //We now need to available the respective data into blocks Stream destinationStream = null; if (DataBuildMode == DataBuildMode.Disk_Blocks) { string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString()); if (File.Exists(fileName)) { try { destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i]) throw new Exception("Wrong place, wrong time, wrong file!"); } catch (Exception) { try { File.Delete(fileName); destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); } catch (Exception) { throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue."); } } } else { //Create the folder if it does not exist yet lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) Directory.CreateDirectory(folderLocation); } destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); } if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); } else //If we are not exclusively building to the disk we just use a memory stream at this stage destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length); //Ensure we start at the beginning of the stream destinationStream.SetLength(ChunkPositionLengthDict[i].Length); destinationStream.Flush(); destinationStream.Seek(0, SeekOrigin.Begin); ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream); } #endregion } #endregion }