/// <summary>
/// Initialise the item data using existing chunk streams. Build mode must be to blocks.
/// </summary>
/// <param name="dataBuildMode"></param>
/// <param name="itemDataStreams"></param>
/// <param name="enableChunkChecksum"></param>
public DistributedItemData(DataBuildMode dataBuildMode, Dictionary <int, Stream> itemDataStreams, bool enableChunkChecksum = false)
{
this.DataBuildMode = DataBuildMode;
if (itemDataStreams == null)
{
throw new ArgumentNullException("itemDataStreams", "itemDataStreams cannot be null.");
}
if (dataBuildMode == DataBuildMode.Disk_Single ||
//itemBuildMode == ItemBuildMode.Both_Single ||
dataBuildMode == DataBuildMode.Memory_Single)
{
throw new ArgumentException("Please use other constructor that takes a single input data stream.");
}
this.ItemBytesLength = itemDataStreams.Select(i => i.Value.Length).Sum();
//Calculate the exactChunkSize if we split everything up into 255 pieces
double exactChunkSize = (double)ItemBytesLength / 255.0;
//If the item is too small we just use the minimumChunkSize
//If we need something larger than MinChunkSizeInBytes we select appropriately
this.ChunkSizeInBytes = (exactChunkSize <= DFS.MinChunkSizeInBytes ? DFS.MinChunkSizeInBytes : (int)Math.Ceiling(exactChunkSize));
this.TotalNumChunks = (byte)(Math.Ceiling((double)ItemBytesLength / (double)ChunkSizeInBytes));
InitialiseChunkPositionLengthDict();
if (itemDataStreams.Count != ChunkPositionLengthDict.Count)
{
throw new ArgumentException("Number of streams should equal the number of chunks");
}
//Initialise the data streams
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
foreach (int chunkIndex in ChunkPositionLengthDict.Keys)
{
ChunkDataStreams[chunkIndex] = new StreamTools.ThreadSafeStream(itemDataStreams[chunkIndex]);
}
this.CompleteDataCheckSum = MD5();
if (enableChunkChecksum)
{
BuildChunkCheckSums();
}
}
private void SendFile(Connection connection, string fileName)
{
try
{
FileStream stream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
StreamTools.ThreadSafeStream safeStream = new StreamTools.ThreadSafeStream(stream);
string shortFileName = System.IO.Path.GetFileName(fileName);
long sendChunkSizeBytes = (long)(stream.Length / 20.0) + 1;
long maxChunkSizeBytes = 500L * 1024L * 1024L;
if (sendChunkSizeBytes > maxChunkSizeBytes)
{
sendChunkSizeBytes = maxChunkSizeBytes;
}
long totalBytesSent = 0;
do
{
long bytesToSend = (totalBytesSent + sendChunkSizeBytes < stream.Length ? sendChunkSizeBytes : stream.Length - totalBytesSent);
StreamTools.StreamSendWrapper streamWrapper = new StreamTools.StreamSendWrapper(safeStream, totalBytesSent, bytesToSend);
long packetSequenceNumber;
connection.SendObject("PartialFileData", streamWrapper, customOptions, out packetSequenceNumber);
connection.SendObject("PartialFileDataInfo", new SendInfo(shortFileName, stream.Length, totalBytesSent, packetSequenceNumber), customOptions);
totalBytesSent += bytesToSend;
} while (totalBytesSent < stream.Length);
GC.Collect();
}
catch (CommunicationException)
{
}
catch (Exception ex)
{
}
}
/// <summary>
/// Sends requested file to the remoteIP and port set in GUI
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void SendFileButton_Click(object sender, RoutedEventArgs e)
{
//Create an OpenFileDialog so that we can request the file to send
OpenFileDialog openDialog = new OpenFileDialog();
openDialog.Multiselect = false;
//If a file was selected
if (openDialog.ShowDialog() == true)
{
//Disable the send and compression buttons
sendFileButton.IsEnabled = false;
UseCompression.IsEnabled = false;
//Parse the necessary remote information
string filename = openDialog.FileName;
string remoteIP = this.remoteIP.Text;
string remotePort = this.remotePort.Text;
//Set the send progress bar to 0
UpdateSendProgress(0);
//Perform the send in a task so that we don't lock the GUI
Task.Factory.StartNew(() =>
{
try
{
//Create a fileStream from the selected file
FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read);
//Wrap the fileStream in a threadSafeStream so that future operations are thread safe
StreamTools.ThreadSafeStream safeStream = new StreamTools.ThreadSafeStream(stream);
//Get the filename without the associated path information
string shortFileName = System.IO.Path.GetFileName(filename);
//Parse the remote connectionInfo
//We have this in a separate try catch so that we can write a clear message to the log window
//if there are problems
ConnectionInfo remoteInfo;
try
{
remoteInfo = new ConnectionInfo(remoteIP, int.Parse(remotePort));
}
catch (Exception)
{
throw new InvalidDataException("Failed to parse remote IP and port. Check and try again.");
}
//Get a connection to the remote side
Connection connection = TCPConnection.GetConnection(remoteInfo);
//Break the send into 20 segments. The less segments the less overhead
//but we still want the progress bar to update in sensible steps
long sendChunkSizeBytes = (long)(stream.Length / 20.0) + 1;
//Limit send chunk size to 500MB
long maxChunkSizeBytes = 500L * 1024L * 1024L;
if (sendChunkSizeBytes > maxChunkSizeBytes)
{
sendChunkSizeBytes = maxChunkSizeBytes;
}
long totalBytesSent = 0;
do
{
//Check the number of bytes to send as the last one may be smaller
long bytesToSend = (totalBytesSent + sendChunkSizeBytes < stream.Length ? sendChunkSizeBytes : stream.Length - totalBytesSent);
//Wrap the threadSafeStream in a StreamSendWrapper so that we can get NetworkComms.Net
//to only send part of the stream.
StreamTools.StreamSendWrapper streamWrapper = new StreamTools.StreamSendWrapper(safeStream, totalBytesSent, bytesToSend);
//We want to record the packetSequenceNumber
long packetSequenceNumber;
//Send the select data
connection.SendObject("PartialFileData", streamWrapper, customOptions, out packetSequenceNumber);
//Send the associated SendInfo for this send so that the remote can correctly rebuild the data
connection.SendObject("PartialFileDataInfo", new SendInfo(shortFileName, stream.Length, totalBytesSent, packetSequenceNumber), customOptions);
totalBytesSent += bytesToSend;
//Update the GUI with our send progress
UpdateSendProgress((double)totalBytesSent / stream.Length);
} while (totalBytesSent < stream.Length);
//Clean up any unused memory
GC.Collect();
AddLineToLog("Completed file send to '" + connection.ConnectionInfo.ToString() + "'.");
}
catch (CommunicationException)
{
//If there is a communication exception then we just write a connection
//closed message to the log window
AddLineToLog("Failed to complete send as connection was closed.");
}
catch (Exception ex)
{
//If we get any other exception which is not an InvalidDataException
//we log the error
if (!windowClosing && ex.GetType() != typeof(InvalidDataException))
{
AddLineToLog(ex.Message.ToString());
LogTools.LogException(ex, "SendFileError");
}
}
//Once the send is finished reset the send progress bar
UpdateSendProgress(0);
//Once complete enable the send button again
sendFileButton.Dispatcher.BeginInvoke(new Action(() =>
{
sendFileButton.IsEnabled = true;
UseCompression.IsEnabled = true;
}));
});
}
}
/// <summary>
/// Sets the item data using the provided data stream. Useful for setting data after deserialisation
/// </summary>
/// <param name="itemIdentifier"></param>
/// <param name="itemDataStream"></param>
public void SetData(string itemIdentifier, Stream itemDataStream)
{
lock (dataLocker)
{
if (itemIdentifier == null)
{
throw new ArgumentNullException("itemIdentifier");
}
if (itemDataStream == null)
{
throw new ArgumentNullException("itemDataStream");
}
#region Build Internal Data Structure
if (DataBuildMode == DataBuildMode.Disk_Single ||
//ItemBuildMode == ItemBuildMode.Both_Single ||
DataBuildMode == DataBuildMode.Memory_Single)
{
CompleteDataStream = new StreamTools.ThreadSafeStream(itemDataStream);
}
else
{
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
//If the itemDataStream is a memory stream we can try to access the buffer as it makes creating the streams more efficient
byte[] itemDataStreamBuffer = null;
if (itemDataStream is MemoryStream)
{
try
{
itemDataStreamBuffer = ((MemoryStream)itemDataStream).GetBuffer();
}
catch (UnauthorizedAccessException) { /* Ignore */ }
}
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
if (itemDataStreamBuffer != null)
{
//This is the fastest way to create a block of data streams
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(new MemoryStream(itemDataStreamBuffer, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length));
}
else
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, itemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
//if (StreamTools.MD5(destinationStream) != checksum)
// throw new Exception("Wrong place, wrong time, wrong file!");
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
}
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
}
else
{
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
}
//Ensure we start at the beginning of the stream
destinationStream.Seek(0, SeekOrigin.Begin);
//Copy over the correct part of the itemDataStream to the chunks
StreamTools.Write(itemDataStream, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length, destinationStream, 8192, double.MaxValue, int.MaxValue);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
}
}
#endregion
this.CompleteDataCheckSum = StreamTools.MD5(itemDataStream);
}
}
/// <summary>
/// Initialise the item data from an assembly config
/// </summary>
/// <param name="assemblyConfig"></param>
public DistributedItemData(ItemAssemblyConfig assemblyConfig)
{
this.DataBuildMode = assemblyConfig.ItemBuildMode;
this.TotalNumChunks = assemblyConfig.TotalNumChunks;
this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;
InitialiseChunkPositionLengthDict();
#region Build Internal Data Structure
//Create the data stores as per the assembly config
if (DataBuildMode == DataBuildMode.Memory_Single)
//ItemBuildMode == ItemBuildMode.Both_Single ||)
{
MemoryStream itemStream = new MemoryStream(0);
itemStream.SetLength(ItemBytesLength);
CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
}
else if (DataBuildMode == DataBuildMode.Disk_Single)
{
#region DiskSingle
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
FileStream fileStream;
if (File.Exists(fileName))
{
//If the file already exists the MD5 had better match otherwise we have a problem
try
{
fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
{
throw new Exception("Wrong place, wrong time, wrong file!");
}
}
catch (Exception)
{
try
{
File.Delete(fileName);
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
fileStream.SetLength(ItemBytesLength);
fileStream.Flush();
}
CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
#endregion
}
else
{
#region Data Chunks
//Break the itemDataStream into blocks
ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
{
//We now need to available the respective data into blocks
Stream destinationStream = null;
if (DataBuildMode == DataBuildMode.Disk_Blocks)
{
string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());
if (File.Exists(fileName))
{
try
{
destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
{
throw new Exception("Wrong place, wrong time, wrong file!");
}
}
catch (Exception)
{
try
{
File.Delete(fileName);
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
catch (Exception)
{
throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
}
}
}
else
{
//Create the folder if it does not exist yet
lock (DFS.globalDFSLocker)
{
if (!Directory.Exists(folderLocation))
{
Directory.CreateDirectory(folderLocation);
}
}
destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
}
if (!File.Exists(fileName))
{
throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
}
}
else
{
//If we are not exclusively building to the disk we just use a memory stream at this stage
destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
}
//Ensure we start at the beginning of the stream
destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
destinationStream.Flush();
destinationStream.Seek(0, SeekOrigin.Begin);
ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
}
#endregion
}
#endregion
}
/*
* Transfer code partially retrieved from NetworkComms example https://www.networkcomms.net/creating-a-wpf-file-transfer-application/
* on 07.10.2020
*/
private void btnUpload_Click(object sender, EventArgs e)
{
string filename = openFileDialog1.FileName;
if (File.Exists(filename))
{
btnUpload.Enabled = false;
btnBrowse.Enabled = false;
this.isBusy = true;
progressBar1.Value = (int)0;
Task.Factory.StartNew(() =>
{
try
{
//Create a fileStream from the selected file
FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read);
//Wrap the fileStream in a threadSafeStream so that future operations are thread safe
StreamTools.ThreadSafeStream safeStream = new StreamTools.ThreadSafeStream(stream);
//Get the filename without the associated path information
string shortFileName = System.IO.Path.GetFileName(filename);
//Parse the remote connectionInfo
//We have this in a separate try catch so that we can write a clear message to the log window
//if there are problems
//Get a connection to the remote side
Connection connection = AdminNetwork.Connection();
//Break the send into 20 segments. The less segments the less overhead
//but we still want the progress bar to update in sensible steps
long sendChunkSizeBytes = (long)(stream.Length / 20.0) + 1;
//Limit send chunk size to 500MB
long maxChunkSizeBytes = 500L * 1024L * 1024L;
if (sendChunkSizeBytes > maxChunkSizeBytes)
{
sendChunkSizeBytes = maxChunkSizeBytes;
}
long totalBytesSent = 0;
do
{
//Check the number of bytes to send as the last one may be smaller
long bytesToSend = (totalBytesSent + sendChunkSizeBytes < stream.Length ? sendChunkSizeBytes : stream.Length - totalBytesSent);
//Wrap the threadSafeStream in a StreamSendWrapper so that we can get NetworkComms.Net
//to only send part of the stream.
StreamTools.StreamSendWrapper streamWrapper = new StreamTools.StreamSendWrapper(safeStream, totalBytesSent, bytesToSend);
//We want to record the packetSequenceNumber
long packetSequenceNumber;
//Send the select data
connection.SendObject("PartialFileData", streamWrapper, NetworkComms.DefaultSendReceiveOptions, out packetSequenceNumber);
//Send the associated SendInfo for this send so that the remote can correctly rebuild the data
connection.SendObject("PartialFileDataInfo", new SendInfo(shortFileName, stream.Length, totalBytesSent, packetSequenceNumber), NetworkComms.DefaultSendReceiveOptions);
totalBytesSent += bytesToSend;
//Update the GUI with our send progress
UpdateSendProgress(shortFileName, (double)totalBytesSent / stream.Length);
} while (totalBytesSent < stream.Length);
//Clean up any unused memory
GC.Collect();
UpdateSendProgress(shortFileName, 100);
// AddLineToLog("Completed file send to '" + connection.ConnectionInfo.ToString() + "'.");
}
catch (CommunicationException)
{
//If there is a communication exception then we just write a connection
//closed message to the log window
// AddLineToLog("Failed to complete send as connection was closed.");
MessageBox.Show("Failed to complete send as connection was closed.");
}
catch (Exception ex)
{
//If we get any other exception which is not an InvalidDataException
//we log the error
/* if (!windowClosing && ex.GetType() != typeof(InvalidDataException))
* {
* AddLineToLog(ex.Message.ToString());
* LogTools.LogException(ex, "SendFileError");
* }*/
MessageBox.Show(ex.ToString());
}
});
}
}