/// <summary>
/// Instantiate a new ItemAssemblyConfig
/// </summary>
/// <param name="itemToDistribute">The DFS item for which this ItemAssemblyConfig should be created.</param>
/// <param name="completedPacketType">The packet type to use once the item has been fully assembled</param>
public ItemAssemblyConfig(DistributedItem itemToDistribute, string completedPacketType)
{
this.CompleteDataCheckSum = itemToDistribute.Data.CompleteDataCheckSum;
this.ChunkCheckSums = itemToDistribute.Data.ChunkCheckSums;
this.TotalNumChunks = itemToDistribute.Data.TotalNumChunks;
this.ChunkSizeInBytes = itemToDistribute.Data.ChunkSizeInBytes;
this.TotalItemSizeInBytes = itemToDistribute.Data.ItemBytesLength;
this.SwarmChunkAvailabilityBytes = itemToDistribute.SwarmChunkAvailability.ThreadSafeSerialise();
this.CompletedPacketType = completedPacketType;
this.ItemBuildCascadeDepth = itemToDistribute.ItemBuildCascadeDepth;
this.ItemTypeStr = itemToDistribute.ItemTypeStr;
this.ItemIdentifier = itemToDistribute.ItemIdentifier;
this.ItemBuildMode = itemToDistribute.Data.DataBuildMode;
}
/// <summary>
/// The distributed file system (DFS) allows for the high performance distribution of large files
/// within a cluster of peers. This sytem replicates the behaviour the bitTorrent protocol by using
/// NetworkCommsDotNet. This example demonstrates the DFS in action.
/// </summary>
public static void RunExample()
{
//Select mode
Console.WriteLine("Distributed File System (DFS) Example ...\n");
Console.WriteLine("Please select host or peer mode:");
Console.WriteLine("1 - Host Mode (Original source of data)");
Console.WriteLine("2 - Peer Mode (Builds data and then acts as subhost)");
//Read in user choice
if (Console.ReadKey(true).Key == ConsoleKey.D1) hostMode = true;
else hostMode = false;
if (hostMode)
{
//Prepare DFS in host mode
#region ServerMode
Console.WriteLine("\n ... host mode selected.");
Console.WriteLine("\nPlease enter how large the test data packet should be in MB and press return, e.g. 50:");
int numberMegsToCreate = int.Parse(Console.ReadLine());
//Fill a byte[] with random data
DateTime startTime = DateTime.Now;
Random randGen = new Random();
byte[] someRandomData = new byte[numberMegsToCreate * 1024 * 1024];
randGen.NextBytes(someRandomData);
Console.WriteLine("\n ... successfully created a {0}MB test packet.", ((double)someRandomData.Length / (1024.0 * 1024.0)).ToString("0.###"));
object listLocker = new object();
List<IPEndPoint> connectedClients = new List<IPEndPoint>();
//Initialise the DFS before creating the test object to ensure the correct port and IP are used as the seed
DFS.Initialise(10000);
//Create the item to be distributed
List<ConnectionInfo> seedConnectionInfoList = (from current in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP) select new ConnectionInfo(ConnectionType.TCP, NetworkComms.NetworkIdentifier, current, true)).ToList();
MemoryStream itemData = new MemoryStream(someRandomData, 0, someRandomData.Length, false, true);
DistributedItem newItem = new DistributedItem("exampleItem", "exampleItem", itemData, seedConnectionInfoList, DataBuildMode.Disk_Blocks);
NetworkComms.ConnectionEstablishShutdownDelegate clientEstablishDelegate = (connection) =>
{
lock (listLocker)
connectedClients.Remove((IPEndPoint)connection.ConnectionInfo.RemoteEndPoint);
Console.WriteLine("Client " + connection.ConnectionInfo + " connected.");
};
NetworkComms.ConnectionEstablishShutdownDelegate clientShutdownDelegate = (connection) =>
{
lock (listLocker)
connectedClients.Remove((IPEndPoint)connection.ConnectionInfo.RemoteEndPoint);
Console.WriteLine("Client " + connection.ConnectionInfo + " disconnected.");
};
NetworkComms.PacketHandlerCallBackDelegate<int> ReplyDelegate = (packetHeader, connection, incomingObject) =>
{
//Push the item into the swarm
lock (listLocker)
if (!connectedClients.Contains(connection.ConnectionInfo.RemoteEndPoint))
connectedClients.Add((IPEndPoint)connection.ConnectionInfo.RemoteEndPoint);
DFS.PushItemToPeer(connection, newItem, "BigDataRequestResponse");
Console.WriteLine("Pushing item to " + connection.ConnectionInfo + " (" + connection.ConnectionInfo.NetworkIdentifier + "). {0} in swarm. P#={1}, S#={2}.", connectedClients.Count, newItem.PushCount, newItem.TotalChunkSupplyCount);
};
NetworkComms.PacketHandlerCallBackDelegate<string> InfoDelegate = (packetHeader, connectionId, incomingString) =>
{
Console.WriteLine(" ... " + connectionId + " - " + incomingString);
};
Console.WriteLine(" ... DFS has been initialised.");
NetworkComms.AppendGlobalConnectionEstablishHandler(clientEstablishDelegate);
NetworkComms.AppendGlobalConnectionCloseHandler(clientShutdownDelegate);
NetworkComms.AppendGlobalIncomingPacketHandler("BigDataRequest", ReplyDelegate);
NetworkComms.AppendGlobalIncomingPacketHandler("ClientInfo", InfoDelegate);
Console.WriteLine("\nListening for incoming connections on:");
foreach (IPEndPoint localEndPoint in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP))
Console.WriteLine("{0}:{1}", localEndPoint.Address, localEndPoint.Port);
Console.WriteLine("\nIdentifier - {0}", NetworkComms.NetworkIdentifier);
Console.WriteLine("\nPress 's' to write out stats, 'q' to close any connected peers, 'ctrl+q' to close this host.\n");
while (true)
{
ConsoleKeyInfo pressedKey = Console.ReadKey(true);
#region Host Shutdown
if (pressedKey.Modifiers != ConsoleModifiers.Control && pressedKey.Key == ConsoleKey.Q)
{
Console.WriteLine("Sending shutdown to clients...");
lock (listLocker)
{
for (int i = 0; i < connectedClients.Count; i++)
{
try
{
TCPConnection.GetConnection(new ConnectionInfo(connectedClients[i])).SendObject("ClientCommand", 0);
}
catch (Exception)
{
Console.WriteLine("Exception telling client to shutdown. Probably already disconnected.");
}
}
}
}
else if (pressedKey.Modifiers == ConsoleModifiers.Control && pressedKey.Key == ConsoleKey.Q)
{
Console.WriteLine("Sending shutdown to clients and closing local host...");
lock (listLocker)
{
for (int i = 0; i < connectedClients.Count; i++)
{
try
{
TCPConnection.GetConnection(new ConnectionInfo(connectedClients[i])).SendObject("ClientCommand", 0);
}
catch (Exception)
{
Console.WriteLine("Exception telling client to shutdown. Probably already disconnected.");
}
}
}
Console.WriteLine("Closing host.");
break;
}
else if (pressedKey.Key == ConsoleKey.S)
{
#region Stats
Console.WriteLine("\nCurrent Stats:");
Console.WriteLine("{0} comms connections.",NetworkComms.TotalNumConnections());
if (NetworkComms.TotalNumConnections() > 0)
{
Console.WriteLine("Connections with: ");
var connections = NetworkComms.GetExistingConnection();
foreach (var connection in connections)
Console.WriteLine("\t{0}", connection.ConnectionInfo);
}
#endregion
}
#endregion
}
#endregion
}
else if (!hostMode)
{
//Prepare DFS in peer mode
#region PeerMode
Console.WriteLine("\n ... peer mode selected.");
ConnectionInfo serverConnectionInfo = ExampleHelper.GetServerDetails();
DFS.Initialise(10000);
Console.WriteLine(" ... DFS has been initialised.");
bool shutDown = false;
bool buildComplete = true;
DateTime startTime = DateTime.Now;
int buildCount = 0;
NetworkComms.PacketHandlerCallBackDelegate<byte[]> ReplyDelegate = (packetHeader, connection, dataBytes) =>
{
try
{
buildCount++;
DistributedItem item = DFS.MostRecentlyCompletedItem();
Console.WriteLine(" ... full item build " + buildCount + " took {0} secs ({1} MB/s), using {2} total peers and {4} build mode. {3} builds completed.",
(DateTime.Now - startTime).TotalSeconds.ToString("0.00"),
(((double)dataBytes.Length / 1048576.0) / (DateTime.Now - startTime).TotalSeconds).ToString("0.0"),
item.SwarmChunkAvailability.NumPeersInSwarm(),
buildCount,
item.Data.DataBuildMode);
double speed = (((double)dataBytes.Length / 1048576.0) / (DateTime.Now - startTime).TotalSeconds);
connection.SendObject("ClientInfo", " ... build " + buildCount + " took " + (DateTime.Now - startTime).TotalSeconds.ToString("0.00") + " secs (" + speed.ToString("0.0") + " MB/s) using " + item.SwarmChunkAvailability.NumPeersInSwarm() + " peers. " + buildCount + " builds completed.");
buildComplete = true;
}
catch (Exception)
{
Console.WriteLine("Shutting down due to exception.");
shutDown = true;
}
};
NetworkComms.PacketHandlerCallBackDelegate<int> ShutdownDelegate = (packetHeader, connectionId, packetDataBytes) =>
{
shutDown = true;
};
NetworkComms.AppendGlobalIncomingPacketHandler("BigDataRequestResponse", ReplyDelegate);
NetworkComms.AppendGlobalIncomingPacketHandler("ClientCommand", ShutdownDelegate);
Console.WriteLine("\nIdentifier - {0}", NetworkComms.NetworkIdentifier);
Console.WriteLine("\nListening for incoming objects on:");
foreach (IPEndPoint localEndPoint in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP))
Console.WriteLine("{0}:{1}", localEndPoint.Address, localEndPoint.Port);
startTime = DateTime.Now;
while (true)
{
if (!shutDown && buildComplete)
{
Console.WriteLine("\nPress 'r' to rebuild or any other key to shutdown.");
var shutdownKey = Console.ReadKey(true).Key;
if (shutdownKey != ConsoleKey.R) shutDown = true;
if (!shutDown)
{
DistributedItem item = DFS.MostRecentlyCompletedItem();
if (item != null)
{
DFS.RemoveItem(item.Data.CompleteDataCheckSum);
Console.WriteLine("\n ... item removed from local and rebuilding at {0}.", DateTime.Now.ToString("HH:mm:ss.fff"));
startTime = DateTime.Now;
}
buildComplete = false;
TCPConnection.GetConnection(serverConnectionInfo).SendObject("BigDataRequest");
Console.WriteLine(" ... initiating item build ...");
}
}
else if (shutDown)
{
shutDown = true;
DFS.Shutdown();
break;
}
Thread.Sleep(250);
}
try
{
TCPConnection.GetConnection(serverConnectionInfo).SendObject("ClientInfo", "... shutting down, initiating DFS shutdown.");
}
catch (CommsException)
{
Console.WriteLine("... unable to inform local of shutdown. Connection probably already closed.");
}
Console.WriteLine("Done. Completed {0} builds.", buildCount);
#endregion
}
DFS.Shutdown();
NetworkComms.Shutdown();
}
/// <summary>
/// The distributed file system (DFS) allows for the high performance distribution of large files
/// within a cluster of peers. This sytem replicates the behaviour the bitTorrent protocol by using
/// NetworkCommsDotNet. This example demonstrates the DFS in action.
/// </summary>
public static void RunExample()
{
//Select launch mode
Console.WriteLine("Launching DFS system ...\n");
Console.WriteLine("Please select host or peer mode:");
Console.WriteLine("1 - Host Mode (Original source of data)");
Console.WriteLine("2 - Peer Mode (Builds data and then acts as subhost)");
//Read in user choice
//if (Console.ReadKey(true).Key == ConsoleKey.D1) hostMode = true;
//else hostMode = false;
hostMode = false;
if (hostMode)
{
//Prepare DFS in host mode
#region ServerMode
Console.WriteLine("\n ... host mode selected.");
Console.WriteLine("\nPlease enter how large the test data packet should be in MB and press return, e.g. 50:");
int numberMegsToCreate = int.Parse(Console.ReadLine());
//Fill a byte[] with random data
DateTime startTime = DateTime.Now;
Random randGen = new Random();
byte[] someRandomData = new byte[numberMegsToCreate * 1024 * 1024];
randGen.NextBytes(someRandomData);
Console.WriteLine("\n ... successfully created a {0}MB test packet.", ((double)someRandomData.Length / (1024.0 * 1024.0)).ToString("0.###"));
object listLocker = new object();
List <IPEndPoint> connectedClients = new List <IPEndPoint>();
//Initialise the DFS before creating the test object to ensure the correct port and IP are used as the seed
DFS.Initialise(10000);
//Create the item to be distributed
List <ConnectionInfo> seedConnectionInfoList = (from current in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP) select new ConnectionInfo(ConnectionType.TCP, NetworkComms.NetworkIdentifier, current, true)).ToList();
DistributedItem newItem = new DistributedItem("exampleItem", "example1", new MemoryStream(someRandomData), seedConnectionInfoList, DataBuildMode.Memory_Single);
NetworkComms.ConnectionEstablishShutdownDelegate clientEstablishDelegate = (connection) =>
{
lock (listLocker)
connectedClients.Remove((IPEndPoint)connection.ConnectionInfo.RemoteEndPoint);
Console.WriteLine("Client " + connection.ConnectionInfo + " connected.");
};
NetworkComms.ConnectionEstablishShutdownDelegate clientShutdownDelegate = (connection) =>
{
lock (listLocker)
connectedClients.Remove((IPEndPoint)connection.ConnectionInfo.RemoteEndPoint);
Console.WriteLine("Client " + connection.ConnectionInfo + " disconnected.");
};
NetworkComms.PacketHandlerCallBackDelegate <int> ReplyDelegate = (packetHeader, connection, incomingObject) =>
{
//Push the item into the swarm
lock (listLocker)
if (!connectedClients.Contains(connection.ConnectionInfo.RemoteEndPoint))
{
connectedClients.Add((IPEndPoint)connection.ConnectionInfo.RemoteEndPoint);
}
DFS.PushItemToPeer(connection, newItem, "BigDataRequestResponse");
Console.WriteLine("Pushing item to " + connection.ConnectionInfo + " (" + connection.ConnectionInfo.NetworkIdentifier + "). {0} in swarm. P#={1}, S#={2}.", connectedClients.Count, newItem.PushCount, newItem.TotalChunkSupplyCount);
};
NetworkComms.PacketHandlerCallBackDelegate <string> InfoDelegate = (packetHeader, connectionId, incomingString) =>
{
Console.WriteLine(" ... " + connectionId + " - " + incomingString);
};
Console.WriteLine(" ... DFS has been initialised.");
NetworkComms.AppendGlobalConnectionEstablishHandler(clientEstablishDelegate);
NetworkComms.AppendGlobalConnectionCloseHandler(clientShutdownDelegate);
NetworkComms.AppendGlobalIncomingPacketHandler("BigDataRequest", ReplyDelegate);
NetworkComms.AppendGlobalIncomingPacketHandler("ClientInfo", InfoDelegate);
Console.WriteLine("\nListening for incoming connections on:");
foreach (IPEndPoint localEndPoint in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP))
{
Console.WriteLine("{0}:{1}", localEndPoint.Address, localEndPoint.Port);
}
Console.WriteLine("\nIdentifier - {0}", NetworkComms.NetworkIdentifier);
Console.WriteLine("\nPress 's' to write out stats, 'q' to close any connected peers, 'ctrl+q' to close this host.\n");
while (true)
{
ConsoleKeyInfo pressedKey = Console.ReadKey(true);
#region Host Shutdown
if (pressedKey.Modifiers != ConsoleModifiers.Control && pressedKey.Key == ConsoleKey.Q)
{
Console.WriteLine("Sending shutdown to clients...");
lock (listLocker)
{
for (int i = 0; i < connectedClients.Count; i++)
{
try
{
TCPConnection.GetConnection(new ConnectionInfo(connectedClients[i])).SendObject("ClientCommand", 0);
}
catch (Exception)
{
Console.WriteLine("Exception telling client to shutdown. Probably already disconnected.");
}
}
}
}
else if (pressedKey.Modifiers == ConsoleModifiers.Control && pressedKey.Key == ConsoleKey.Q)
{
Console.WriteLine("Sending shutdown to clients and closing local host...");
lock (listLocker)
{
for (int i = 0; i < connectedClients.Count; i++)
{
try
{
TCPConnection.GetConnection(new ConnectionInfo(connectedClients[i])).SendObject("ClientCommand", 0);
}
catch (Exception)
{
Console.WriteLine("Exception telling client to shutdown. Probably already disconnected.");
}
}
}
Console.WriteLine("Closing host.");
break;
}
else if (pressedKey.Key == ConsoleKey.S)
{
#region Stats
Console.WriteLine("\nCurrent Stats:");
Console.WriteLine("{0} comms connections.", NetworkComms.TotalNumConnections());
if (NetworkComms.TotalNumConnections() > 0)
{
Console.WriteLine("Connections with: ");
var connections = NetworkComms.GetExistingConnection();
foreach (var connection in connections)
{
Console.WriteLine("\t{0}", connection.ConnectionInfo);
}
}
#endregion
}
#endregion
}
#endregion
}
else if (!hostMode)
{
//Prepare DFS in peer mode
#region PeerMode
Console.WriteLine("\n ... peer mode selected.");
try
{
ConnectionInfo serverConnectionInfo = new ConnectionInfo("192.168.0.105", 10000);
//ExampleHelper.GetServerDetails(out serverConnectionInfo);
DFS.Initialise(10000);
Console.WriteLine(" ... DFS has been initialised.");
bool shutDown = false;
bool buildComplete = true;
DateTime startTime = DateTime.Now;
int buildCount = 0;
NetworkComms.PacketHandlerCallBackDelegate <byte[]> ReplyDelegate = (packetHeader, connection, dataBytes) =>
{
try
{
buildCount++;
DistributedItem item = DFS.MostRecentlyCompletedItem();
Console.WriteLine(" ... full item build " + buildCount + " took {0} secs ({1} MB/s) using {2} total peers. {3} builds completed.", (DateTime.Now - startTime).TotalSeconds.ToString("0.00"), (((double)dataBytes.Length / 1048576.0) / (DateTime.Now - startTime).TotalSeconds).ToString("0.0"), item.SwarmChunkAvailability.NumPeersInSwarm(), buildCount);
double speed = (((double)dataBytes.Length / 1048576.0) / (DateTime.Now - startTime).TotalSeconds);
connection.SendObject("ClientInfo", " ... build " + buildCount + " took " + (DateTime.Now - startTime).TotalSeconds.ToString("0.00") + " secs (" + speed.ToString("0.0") + " MB/s) using " + item.SwarmChunkAvailability.NumPeersInSwarm() + " peers. " + buildCount + " builds completed.");
buildComplete = true;
dataBytes = null;
GC.Collect();
}
catch (Exception ex)
{
LogTools.LogException(ex, "DFSTestCallbackError");
Console.WriteLine("Shutting down due to exception.");
shutDown = true;
}
};
NetworkComms.PacketHandlerCallBackDelegate <int> ShutdownDelegate = (packetHeader, connectionId, packetDataBytes) =>
{
shutDown = true;
};
NetworkComms.AppendGlobalIncomingPacketHandler("BigDataRequestResponse", ReplyDelegate);
NetworkComms.AppendGlobalIncomingPacketHandler("ClientCommand", ShutdownDelegate);
Console.WriteLine("\nIdentifier - {0}", NetworkComms.NetworkIdentifier);
Console.WriteLine("\nListening for incoming objects on:");
foreach (IPEndPoint localEndPoint in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP))
{
Console.WriteLine("{0}:{1}", localEndPoint.Address, localEndPoint.Port);
}
startTime = DateTime.Now;
//while (true)
//{
if (!shutDown && buildComplete)
{
//Console.WriteLine("\nPress 'r' to rebuild or any other key to shutdown.");
//var shutdownKey = Console.ReadKey(true).Key;
//if (shutdownKey != ConsoleKey.R) shutDown = true;
//if (!shutDown)
//{
DistributedItem item = DFS.MostRecentlyCompletedItem();
if (item != null)
{
DFS.RemoveItem(item.Data.CompleteDataCheckSum);
Console.WriteLine("\n ... item removed from local and rebuilding at {0}.", DateTime.Now.ToString("HH:mm:ss.fff"));
startTime = DateTime.Now;
}
buildComplete = false;
TCPConnection.GetConnection(serverConnectionInfo).SendObject("BigDataRequest");
Console.WriteLine(" ... initiating item build ...");
//}
}
else if (shutDown)
{
shutDown = true;
DFS.Shutdown();
//break;
}
Thread.Sleep(250);
//}
while (!buildComplete)
{
Thread.Sleep(250);
}
try
{
TCPConnection.GetConnection(serverConnectionInfo).SendObject("ClientInfo", "... shutting down, initiating DFS shutdown.");
}
catch (CommsException)
{
Console.WriteLine("... unable to inform local of shutdown. Connection probably already closed.");
}
Console.WriteLine("Done. Completed {0} builds.", buildCount);
}
catch (Exception ex)
{
Console.WriteLine("Bad Error!");
LogTools.LogException(ex, "DFSTestError_" + NetworkComms.NetworkIdentifier);
}
#endregion
}
DFS.Shutdown();
NetworkComms.Shutdown();
}
/// <summary>
/// Instantiate a new ItemAssemblyConfig
/// </summary>
/// <param name="itemToDistribute">The DFS item for which this ItemAssemblyConfig should be created.</param>
/// <param name="completedPacketType">The packet type to use once the item has been fully assembled</param>
public ItemAssemblyConfig(DistributedItem itemToDistribute, string completedPacketType)
{
this.CompleteDataCheckSum = itemToDistribute.Data.CompleteDataCheckSum;
this.ChunkCheckSums = itemToDistribute.Data.ChunkCheckSums;
this.TotalNumChunks = itemToDistribute.Data.TotalNumChunks;
this.ChunkSizeInBytes = itemToDistribute.Data.ChunkSizeInBytes;
this.TotalItemSizeInBytes = itemToDistribute.Data.ItemBytesLength;
this.SwarmChunkAvailabilityBytes = itemToDistribute.SwarmChunkAvailability.ThreadSafeSerialise();
this.CompletedPacketType = completedPacketType;
this.ItemBuildCascadeDepth = itemToDistribute.ItemBuildCascadeDepth;
this.ItemTypeStr = itemToDistribute.ItemTypeStr;
this.ItemIdentifier = itemToDistribute.ItemIdentifier;
this.ItemBuildMode = itemToDistribute.Data.DataBuildMode;
}
/// <summary>
/// Adds a distributed item to the local cache and informs any known peers of the item availability
/// </summary>
/// <param name="itemToAdd">The item to add</param>
/// <returns>The actual item added to the local cache. May not be the provided itemToAdd if an item with the same
/// checksum already existed.</returns>
public static DistributedItem AddItem(DistributedItem itemToAdd)
{
try
{
lock (globalDFSLocker)
{
//First double check to see if it's already in the swarm
if (!ItemAlreadyInLocalCache(itemToAdd))
{
swarmedItemsDict.Add(itemToAdd.Data.CompleteDataCheckSum, itemToAdd);
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("... added existing item to DFS (" + itemToAdd.Data.CompleteDataCheckSum + ").");
}
else
{
itemToAdd = swarmedItemsDict[itemToAdd.Data.CompleteDataCheckSum];
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("... added new item to DFS (" + itemToAdd.Data.CompleteDataCheckSum + ").");
}
}
//Send the config information to the client that wanted the file
//NetworkComms.SendObject("DFS_IncomingLocalItemBuild, requestOriginConnectionId, false, new ItemAssemblyConfig(itemToDistribute, completedPacketType));
itemToAdd.SwarmChunkAvailability.BroadcastLocalAvailability(itemToAdd.Data.CompleteDataCheckSum);
}
catch (CommsException)
{
//LogTools.LogException(ex, "CommsError_AddItemToSwarm");
}
catch (Exception ex)
{
LogTools.LogException(ex, "Error_AddItemToSwarm");
}
//try { GC.Collect(); }
//catch (Exception) { }
return itemToAdd;
}
/// <summary>
/// Introduces a new item into the swarm and sends a build command to the originating requester
/// </summary>
/// <param name="peerConnection">The peer which requested the DFS item</param>
/// <param name="itemToDistribute">The item to be distributed</param>
/// <param name="completedPacketType">The packet type to use once the item has been fully assembled</param>
public static void PushItemToPeer(Connection peerConnection, DistributedItem itemToDistribute, string completedPacketType)
{
try
{
if (peerConnection.ConnectionInfo.ConnectionType != ConnectionType.TCP)
throw new Exception("Only able to push DFS item when the request is made via TCP.");
if (itemToDistribute.ItemClosed)
throw new ArgumentException("Unable to push a closed item.");
ItemAssemblyConfig assemblyConfig;
lock (globalDFSLocker)
{
//First double check to see if it's already in the swarm
if (!ItemAlreadyInLocalCache(itemToDistribute))
swarmedItemsDict.Add(itemToDistribute.Data.CompleteDataCheckSum, itemToDistribute);
else
itemToDistribute = swarmedItemsDict[itemToDistribute.Data.CompleteDataCheckSum];
}
itemToDistribute.IncrementPushCount();
//We add the requester to the item swarm at this point
itemToDistribute.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(peerConnection.ConnectionInfo, new ChunkFlags(0));
//There is a possibility when we create this assembly config that the peer has been removed again
//We handle this on the peer end
assemblyConfig = new ItemAssemblyConfig(itemToDistribute, completedPacketType);
//Send the config information to the client that wanted the file
peerConnection.SendObject("DFS_IncomingLocalItemBuild", assemblyConfig, nullCompressionSRO);
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("Pushed DFS item " + itemToDistribute.Data.CompleteDataCheckSum + " to peer " + peerConnection + ".");
}
catch (CommsException)
{
//LogTools.LogException(ex, "CommsError_AddItemToSwarm");
}
catch (Exception ex)
{
LogTools.LogException(ex, "Error_AddItemToSwarm");
}
//try { GC.Collect(); }
//catch (Exception) { }
}
/// <summary>
/// Returns true if the provided item is already present within the swarm
/// </summary>
/// <param name="item">The relevant DFS item</param>
/// <returns></returns>
public static bool ItemAlreadyInLocalCache(DistributedItem item)
{
lock (globalDFSLocker)
{
if (swarmedItemsDict.ContainsKey(item.Data.CompleteDataCheckSum))
{
if (swarmedItemsDict[item.Data.CompleteDataCheckSum].Data.ItemBytesLength == item.Data.ItemBytesLength)
return true;
else
throw new Exception("Potential Md5 conflict detected in DFS.");
}
return false;
}
}
/// <summary>
/// TCP - Received by this DFS if a server is telling this instance to build a local file
/// </summary>
/// <param name="packetHeader"></param>
/// <param name="connection"></param>
/// <param name="assemblyConfig"></param>
private static void IncomingLocalItemBuild(PacketHeader packetHeader, Connection connection, ItemAssemblyConfig assemblyConfig)
{
//We start the build in the DFS task factory as it will be a long lived task
//BuildTaskFactory.StartNew(() =>
Action assembleAction = new Action(() =>
{
DistributedItem newItem = null;
byte[] itemBytes = null;
try
{
if (assemblyConfig == null)
throw new NullReferenceException("AssemblyConfig should not be null.");
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild from " + connection + " for item " + assemblyConfig.CompleteDataCheckSum + ".");
//We check to see if we already have the necessary file locally
lock (globalDFSLocker)
{
if (swarmedItemsDict.ContainsKey(assemblyConfig.CompleteDataCheckSum))
{
if (swarmedItemsDict[assemblyConfig.CompleteDataCheckSum].Data.ItemBytesLength != assemblyConfig.TotalItemSizeInBytes)
throw new Exception("Possible MD5 conflict detected.");
else
newItem = swarmedItemsDict[assemblyConfig.CompleteDataCheckSum];
}
else
{
newItem = new DistributedItem(assemblyConfig);
swarmedItemsDict.Add(assemblyConfig.CompleteDataCheckSum, newItem);
}
}
//Ensure all possible local listeners are added here
List<ConnectionInfo> seedConnectionInfoList = (from current in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP) select new ConnectionInfo(ConnectionType.TCP, NetworkComms.NetworkIdentifier, current, true)).ToList();
foreach (ConnectionInfo info in seedConnectionInfoList)
newItem.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(info, newItem.SwarmChunkAvailability.PeerChunkAvailability(NetworkComms.NetworkIdentifier), newItem.SwarmChunkAvailability.PeerIsSuperPeer(NetworkComms.NetworkIdentifier), false);
//Build the item from the swarm
//If the item is already complete this will return immediately
newItem.AssembleItem((int)(ItemBuildTimeoutSecsPerMB * (assemblyConfig.TotalItemSizeInBytes / (1024.0 * 1024.0))));
//Once complete we pass the item bytes back into network comms
//If an exception is thrown we will probably not call this method, timeouts in other areas should then handle and can restart the build.
if (newItem.LocalItemComplete() && assemblyConfig.CompletedPacketType != "")
{
if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild completed for item with MD5 " + assemblyConfig.CompleteDataCheckSum + ". Item build target is " + assemblyConfig.ItemBuildMode + ".");
itemBytes = newItem.GetCompletedItemBytes();
}
else if (assemblyConfig.CompletedPacketType != "")
RemoveItem(assemblyConfig.CompleteDataCheckSum);
if (DFS.loggingEnabled)
{
Exception exceptionToLogWith = new Exception("Build completed successfully. Logging was enabled so saving build log.");
string fileName = "DFSItemBuildLog_" + newItem.ItemIdentifier + "_" + NetworkComms.NetworkIdentifier;
if (newItem != null)
LogTools.LogException(exceptionToLogWith, fileName, newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
else
LogTools.LogException(exceptionToLogWith, fileName, "newItem==null so no build log was available.");
}
}
catch (ObjectDisposedException)
{
//The item was closed during assemble, no need to log an errors here
RemoveItem(assemblyConfig.CompleteDataCheckSum);
}
catch (CommsException e)
{
//Crap an error has happened, let people know we probably don't have a good file
RemoveItem(assemblyConfig.CompleteDataCheckSum);
//connection.CloseConnection(true, 30);
//LogTools.LogException(e, "CommsError_IncomingLocalItemBuild");
if (newItem != null)
LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
else
LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", "newItem==null so no build log was available.");
}
catch (Exception e)
{
//Crap an error has happened, let people know we probably don't have a good file
RemoveItem(assemblyConfig.CompleteDataCheckSum);
//connection.CloseConnection(true, 31);
if (newItem != null)
LogTools.LogException(e, "Error_IncomingLocalItemBuild", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
else
LogTools.LogException(e, "Error_IncomingLocalItemBuild", "newItem==null so no build log was available.");
}
//finally
//{
//Putting any code here appears to cause a sigsegv fault on leaving the finally in mono
//Just moved the code out to below as it makes no difference
//}
//Regardless of if the item completed we call the necessary packet handlers
//If there was a build error we just pass null data to the handlers so that the errors can get called up the relevant stack traces.
try
{
PacketHeader itemPacketHeader = new PacketHeader(assemblyConfig.CompletedPacketType, newItem == null ? 0 : newItem.Data.ItemBytesLength);
//We set the item checksum so that the entire distributed item can be easily retrieved later
itemPacketHeader.SetOption(PacketHeaderStringItems.PacketIdentifier, newItem == null ? "" : newItem.ItemTypeStr + "|" + newItem.ItemIdentifier + "|" + newItem.Data.CompleteDataCheckSum);
var dataStream = (itemBytes == null ? new MemoryStream(new byte[0], 0, 0, false, true) : new MemoryStream(itemBytes, 0, itemBytes.Length, false, true));
var sendRecieveOptions = new SendReceiveOptions<NullSerializer>(new Dictionary<string, string>());
NetworkComms.TriggerAllPacketHandlers(itemPacketHeader, connection, dataStream, sendRecieveOptions);
}
catch (Exception ex)
{
LogTools.LogException(ex, "Error_IncomingLocalItemBuildFinal");
}
});
if (BuildTaskFactory == null)
LogTools.LogException(new NullReferenceException("BuildTaskFactory is null in IncomingLocalItemBuild"), "IncomingLocalBuildError");
else
//Thread buildThread = new Thread(buildAction);
//buildThread.Name = "DFS_" + assemblyConfig.ItemIdentifier + "_Build";
//buildThread.Start();
BuildTaskFactory.StartNew(assembleAction);
}
