/// <summary> /// Initialise the item data using existing chunk streams. Build mode must be to blocks. /// </summary> /// <param name="dataBuildMode"></param> /// <param name="itemDataStreams"></param> /// <param name="enableChunkChecksum"></param> public DistributedItemData(DataBuildMode dataBuildMode, Dictionary <int, Stream> itemDataStreams, bool enableChunkChecksum = false) { this.DataBuildMode = DataBuildMode; if (itemDataStreams == null) { throw new ArgumentNullException("itemDataStreams", "itemDataStreams cannot be null."); } if (dataBuildMode == DataBuildMode.Disk_Single || //itemBuildMode == ItemBuildMode.Both_Single || dataBuildMode == DataBuildMode.Memory_Single) { throw new ArgumentException("Please use other constructor that takes a single input data stream."); } this.ItemBytesLength = itemDataStreams.Select(i => i.Value.Length).Sum(); //Calculate the exactChunkSize if we split everything up into 255 pieces double exactChunkSize = (double)ItemBytesLength / 255.0; //If the item is too small we just use the minimumChunkSize //If we need something larger than MinChunkSizeInBytes we select appropriately this.ChunkSizeInBytes = (exactChunkSize <= DFS.MinChunkSizeInBytes ? DFS.MinChunkSizeInBytes : (int)Math.Ceiling(exactChunkSize)); this.TotalNumChunks = (byte)(Math.Ceiling((double)ItemBytesLength / (double)ChunkSizeInBytes)); InitialiseChunkPositionLengthDict(); if (itemDataStreams.Count != ChunkPositionLengthDict.Count) { throw new ArgumentException("Number of streams should equal the number of chunks"); } //Initialise the data streams ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count]; foreach (int chunkIndex in ChunkPositionLengthDict.Keys) { ChunkDataStreams[chunkIndex] = new StreamTools.ThreadSafeStream(itemDataStreams[chunkIndex]); } this.CompleteDataCheckSum = MD5(); if (enableChunkChecksum) { BuildChunkCheckSums(); } }
private void SendFile(Connection connection, string fileName) { try { FileStream stream = new FileStream(fileName, FileMode.Open, FileAccess.Read); StreamTools.ThreadSafeStream safeStream = new StreamTools.ThreadSafeStream(stream); string shortFileName = System.IO.Path.GetFileName(fileName); long sendChunkSizeBytes = (long)(stream.Length / 20.0) + 1; long maxChunkSizeBytes = 500L * 1024L * 1024L; if (sendChunkSizeBytes > maxChunkSizeBytes) { sendChunkSizeBytes = maxChunkSizeBytes; } long totalBytesSent = 0; do { long bytesToSend = (totalBytesSent + sendChunkSizeBytes < stream.Length ? sendChunkSizeBytes : stream.Length - totalBytesSent); StreamTools.StreamSendWrapper streamWrapper = new StreamTools.StreamSendWrapper(safeStream, totalBytesSent, bytesToSend); long packetSequenceNumber; connection.SendObject("PartialFileData", streamWrapper, customOptions, out packetSequenceNumber); connection.SendObject("PartialFileDataInfo", new SendInfo(shortFileName, stream.Length, totalBytesSent, packetSequenceNumber), customOptions); totalBytesSent += bytesToSend; } while (totalBytesSent < stream.Length); GC.Collect(); } catch (CommunicationException) { } catch (Exception ex) { } }
/// <summary> /// Sends requested file to the remoteIP and port set in GUI /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void SendFileButton_Click(object sender, RoutedEventArgs e) { //Create an OpenFileDialog so that we can request the file to send OpenFileDialog openDialog = new OpenFileDialog(); openDialog.Multiselect = false; //If a file was selected if (openDialog.ShowDialog() == true) { //Disable the send and compression buttons sendFileButton.IsEnabled = false; UseCompression.IsEnabled = false; //Parse the necessary remote information string filename = openDialog.FileName; string remoteIP = this.remoteIP.Text; string remotePort = this.remotePort.Text; //Set the send progress bar to 0 UpdateSendProgress(0); //Perform the send in a task so that we don't lock the GUI Task.Factory.StartNew(() => { try { //Create a fileStream from the selected file FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read); //Wrap the fileStream in a threadSafeStream so that future operations are thread safe StreamTools.ThreadSafeStream safeStream = new StreamTools.ThreadSafeStream(stream); //Get the filename without the associated path information string shortFileName = System.IO.Path.GetFileName(filename); //Parse the remote connectionInfo //We have this in a separate try catch so that we can write a clear message to the log window //if there are problems ConnectionInfo remoteInfo; try { remoteInfo = new ConnectionInfo(remoteIP, int.Parse(remotePort)); } catch (Exception) { throw new InvalidDataException("Failed to parse remote IP and port. Check and try again."); } //Get a connection to the remote side Connection connection = TCPConnection.GetConnection(remoteInfo); //Break the send into 20 segments. The less segments the less overhead //but we still want the progress bar to update in sensible steps long sendChunkSizeBytes = (long)(stream.Length / 20.0) + 1; //Limit send chunk size to 500MB long maxChunkSizeBytes = 500L * 1024L * 1024L; if (sendChunkSizeBytes > maxChunkSizeBytes) { sendChunkSizeBytes = maxChunkSizeBytes; } long totalBytesSent = 0; do { //Check the number of bytes to send as the last one may be smaller long bytesToSend = (totalBytesSent + sendChunkSizeBytes < stream.Length ? sendChunkSizeBytes : stream.Length - totalBytesSent); //Wrap the threadSafeStream in a StreamSendWrapper so that we can get NetworkComms.Net //to only send part of the stream. StreamTools.StreamSendWrapper streamWrapper = new StreamTools.StreamSendWrapper(safeStream, totalBytesSent, bytesToSend); //We want to record the packetSequenceNumber long packetSequenceNumber; //Send the select data connection.SendObject("PartialFileData", streamWrapper, customOptions, out packetSequenceNumber); //Send the associated SendInfo for this send so that the remote can correctly rebuild the data connection.SendObject("PartialFileDataInfo", new SendInfo(shortFileName, stream.Length, totalBytesSent, packetSequenceNumber), customOptions); totalBytesSent += bytesToSend; //Update the GUI with our send progress UpdateSendProgress((double)totalBytesSent / stream.Length); } while (totalBytesSent < stream.Length); //Clean up any unused memory GC.Collect(); AddLineToLog("Completed file send to '" + connection.ConnectionInfo.ToString() + "'."); } catch (CommunicationException) { //If there is a communication exception then we just write a connection //closed message to the log window AddLineToLog("Failed to complete send as connection was closed."); } catch (Exception ex) { //If we get any other exception which is not an InvalidDataException //we log the error if (!windowClosing && ex.GetType() != typeof(InvalidDataException)) { AddLineToLog(ex.Message.ToString()); LogTools.LogException(ex, "SendFileError"); } } //Once the send is finished reset the send progress bar UpdateSendProgress(0); //Once complete enable the send button again sendFileButton.Dispatcher.BeginInvoke(new Action(() => { sendFileButton.IsEnabled = true; UseCompression.IsEnabled = true; })); }); } }
/// <summary> /// Sets the item data using the provided data stream. Useful for setting data after deserialisation /// </summary> /// <param name="itemIdentifier"></param> /// <param name="itemDataStream"></param> public void SetData(string itemIdentifier, Stream itemDataStream) { lock (dataLocker) { if (itemIdentifier == null) { throw new ArgumentNullException("itemIdentifier"); } if (itemDataStream == null) { throw new ArgumentNullException("itemDataStream"); } #region Build Internal Data Structure if (DataBuildMode == DataBuildMode.Disk_Single || //ItemBuildMode == ItemBuildMode.Both_Single || DataBuildMode == DataBuildMode.Memory_Single) { CompleteDataStream = new StreamTools.ThreadSafeStream(itemDataStream); } else { //Break the itemDataStream into blocks ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count]; //If the itemDataStream is a memory stream we can try to access the buffer as it makes creating the streams more efficient byte[] itemDataStreamBuffer = null; if (itemDataStream is MemoryStream) { try { itemDataStreamBuffer = ((MemoryStream)itemDataStream).GetBuffer(); } catch (UnauthorizedAccessException) { /* Ignore */ } } for (int i = 0; i < ChunkPositionLengthDict.Count; i++) { if (itemDataStreamBuffer != null) { //This is the fastest way to create a block of data streams ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(new MemoryStream(itemDataStreamBuffer, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length)); } else { //We now need to available the respective data into blocks Stream destinationStream = null; if (DataBuildMode == DataBuildMode.Disk_Blocks) { string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, itemIdentifier + ".DFSItemData_" + i.ToString()); if (File.Exists(fileName)) { destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); //if (StreamTools.MD5(destinationStream) != checksum) // throw new Exception("Wrong place, wrong time, wrong file!"); } else { //Create the folder if it does not exist yet lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) { Directory.CreateDirectory(folderLocation); } } destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); destinationStream.SetLength(ChunkPositionLengthDict[i].Length); destinationStream.Flush(); } if (!File.Exists(fileName)) { throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); } } else { //If we are not exclusively building to the disk we just use a memory stream at this stage destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length); } //Ensure we start at the beginning of the stream destinationStream.Seek(0, SeekOrigin.Begin); //Copy over the correct part of the itemDataStream to the chunks StreamTools.Write(itemDataStream, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length, destinationStream, 8192, double.MaxValue, int.MaxValue); ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream); } } } #endregion this.CompleteDataCheckSum = StreamTools.MD5(itemDataStream); } }
/// <summary> /// Initialise the item data from an assembly config /// </summary> /// <param name="assemblyConfig"></param> public DistributedItemData(ItemAssemblyConfig assemblyConfig) { this.DataBuildMode = assemblyConfig.ItemBuildMode; this.TotalNumChunks = assemblyConfig.TotalNumChunks; this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes; this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum; this.ChunkCheckSums = assemblyConfig.ChunkCheckSums; this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes; InitialiseChunkPositionLengthDict(); #region Build Internal Data Structure //Create the data stores as per the assembly config if (DataBuildMode == DataBuildMode.Memory_Single) //ItemBuildMode == ItemBuildMode.Both_Single ||) { MemoryStream itemStream = new MemoryStream(0); itemStream.SetLength(ItemBytesLength); CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream); } else if (DataBuildMode == DataBuildMode.Disk_Single) { #region DiskSingle string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData"); FileStream fileStream; if (File.Exists(fileName)) { //If the file already exists the MD5 had better match otherwise we have a problem try { fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum) { throw new Exception("Wrong place, wrong time, wrong file!"); } } catch (Exception) { try { File.Delete(fileName); fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose); } catch (Exception) { throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue."); } } } else { lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) { Directory.CreateDirectory(folderLocation); } } fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose); fileStream.SetLength(ItemBytesLength); fileStream.Flush(); } CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream); if (!File.Exists(fileName)) { throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); } #endregion } else { #region Data Chunks //Break the itemDataStream into blocks ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count]; for (int i = 0; i < ChunkPositionLengthDict.Count; i++) { //We now need to available the respective data into blocks Stream destinationStream = null; if (DataBuildMode == DataBuildMode.Disk_Blocks) { string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier; string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString()); if (File.Exists(fileName)) { try { destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read); if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i]) { throw new Exception("Wrong place, wrong time, wrong file!"); } } catch (Exception) { try { File.Delete(fileName); destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); } catch (Exception) { throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue."); } } } else { //Create the folder if it does not exist yet lock (DFS.globalDFSLocker) { if (!Directory.Exists(folderLocation)) { Directory.CreateDirectory(folderLocation); } } destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose); } if (!File.Exists(fileName)) { throw new Exception("At this point the item data file should have been created. This exception should not really be possible."); } } else { //If we are not exclusively building to the disk we just use a memory stream at this stage destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length); } //Ensure we start at the beginning of the stream destinationStream.SetLength(ChunkPositionLengthDict[i].Length); destinationStream.Flush(); destinationStream.Seek(0, SeekOrigin.Begin); ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream); } #endregion } #endregion }
/* * Transfer code partially retrieved from NetworkComms example https://www.networkcomms.net/creating-a-wpf-file-transfer-application/ * on 07.10.2020 */ private void btnUpload_Click(object sender, EventArgs e) { string filename = openFileDialog1.FileName; if (File.Exists(filename)) { btnUpload.Enabled = false; btnBrowse.Enabled = false; this.isBusy = true; progressBar1.Value = (int)0; Task.Factory.StartNew(() => { try { //Create a fileStream from the selected file FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read); //Wrap the fileStream in a threadSafeStream so that future operations are thread safe StreamTools.ThreadSafeStream safeStream = new StreamTools.ThreadSafeStream(stream); //Get the filename without the associated path information string shortFileName = System.IO.Path.GetFileName(filename); //Parse the remote connectionInfo //We have this in a separate try catch so that we can write a clear message to the log window //if there are problems //Get a connection to the remote side Connection connection = AdminNetwork.Connection(); //Break the send into 20 segments. The less segments the less overhead //but we still want the progress bar to update in sensible steps long sendChunkSizeBytes = (long)(stream.Length / 20.0) + 1; //Limit send chunk size to 500MB long maxChunkSizeBytes = 500L * 1024L * 1024L; if (sendChunkSizeBytes > maxChunkSizeBytes) { sendChunkSizeBytes = maxChunkSizeBytes; } long totalBytesSent = 0; do { //Check the number of bytes to send as the last one may be smaller long bytesToSend = (totalBytesSent + sendChunkSizeBytes < stream.Length ? sendChunkSizeBytes : stream.Length - totalBytesSent); //Wrap the threadSafeStream in a StreamSendWrapper so that we can get NetworkComms.Net //to only send part of the stream. StreamTools.StreamSendWrapper streamWrapper = new StreamTools.StreamSendWrapper(safeStream, totalBytesSent, bytesToSend); //We want to record the packetSequenceNumber long packetSequenceNumber; //Send the select data connection.SendObject("PartialFileData", streamWrapper, NetworkComms.DefaultSendReceiveOptions, out packetSequenceNumber); //Send the associated SendInfo for this send so that the remote can correctly rebuild the data connection.SendObject("PartialFileDataInfo", new SendInfo(shortFileName, stream.Length, totalBytesSent, packetSequenceNumber), NetworkComms.DefaultSendReceiveOptions); totalBytesSent += bytesToSend; //Update the GUI with our send progress UpdateSendProgress(shortFileName, (double)totalBytesSent / stream.Length); } while (totalBytesSent < stream.Length); //Clean up any unused memory GC.Collect(); UpdateSendProgress(shortFileName, 100); // AddLineToLog("Completed file send to '" + connection.ConnectionInfo.ToString() + "'."); } catch (CommunicationException) { //If there is a communication exception then we just write a connection //closed message to the log window // AddLineToLog("Failed to complete send as connection was closed."); MessageBox.Show("Failed to complete send as connection was closed."); } catch (Exception ex) { //If we get any other exception which is not an InvalidDataException //we log the error /* if (!windowClosing && ex.GetType() != typeof(InvalidDataException)) * { * AddLineToLog(ex.Message.ToString()); * LogTools.LogException(ex, "SendFileError"); * }*/ MessageBox.Show(ex.ToString()); } }); } }