/// <summary>
        /// Initialise the item data from an assembly config
        /// </summary>
        /// <param name="assemblyConfig"></param>
        public DistributedItemData(ItemAssemblyConfig assemblyConfig)
        {
            this.DataBuildMode        = assemblyConfig.ItemBuildMode;
            this.TotalNumChunks       = assemblyConfig.TotalNumChunks;
            this.ChunkSizeInBytes     = assemblyConfig.ChunkSizeInBytes;
            this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
            this.ChunkCheckSums       = assemblyConfig.ChunkCheckSums;
            this.ItemBytesLength      = assemblyConfig.TotalItemSizeInBytes;

            InitialiseChunkPositionLengthDict();

            #region Build Internal Data Structure
            //Create the data stores as per the assembly config
            if (DataBuildMode == DataBuildMode.Memory_Single)
            //ItemBuildMode == ItemBuildMode.Both_Single ||)
            {
                MemoryStream itemStream = new MemoryStream(0);
                itemStream.SetLength(ItemBytesLength);
                CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
            }
            else if (DataBuildMode == DataBuildMode.Disk_Single)
            {
                #region DiskSingle
                string     folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                string     fileName       = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
                FileStream fileStream;
                if (File.Exists(fileName))
                {
                    //If the file already exists the MD5 had better match otherwise we have a problem
                    try
                    {
                        fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                        if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
                        {
                            throw new Exception("Wrong place, wrong time, wrong file!");
                        }
                    }
                    catch (Exception)
                    {
                        try
                        {
                            File.Delete(fileName);
                            fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
                        }
                        catch (Exception)
                        {
                            throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
                        }
                    }
                }
                else
                {
                    lock (DFS.globalDFSLocker)
                    {
                        if (!Directory.Exists(folderLocation))
                        {
                            Directory.CreateDirectory(folderLocation);
                        }
                    }

                    fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
                    fileStream.SetLength(ItemBytesLength);
                    fileStream.Flush();
                }

                CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);

                if (!File.Exists(fileName))
                {
                    throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                }
                #endregion
            }
            else
            {
                #region Data Chunks
                //Break the itemDataStream into blocks
                ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];

                for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
                {
                    //We now need to available the respective data into blocks
                    Stream destinationStream = null;
                    if (DataBuildMode == DataBuildMode.Disk_Blocks)
                    {
                        string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                        string fileName       = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());

                        if (File.Exists(fileName))
                        {
                            try
                            {
                                destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                                if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
                                {
                                    throw new Exception("Wrong place, wrong time, wrong file!");
                                }
                            }
                            catch (Exception)
                            {
                                try
                                {
                                    File.Delete(fileName);
                                    destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                                }
                                catch (Exception)
                                {
                                    throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
                                }
                            }
                        }
                        else
                        {
                            //Create the folder if it does not exist yet
                            lock (DFS.globalDFSLocker)
                            {
                                if (!Directory.Exists(folderLocation))
                                {
                                    Directory.CreateDirectory(folderLocation);
                                }
                            }

                            destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                        }

                        if (!File.Exists(fileName))
                        {
                            throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                        }
                    }
                    else
                    {
                        //If we are not exclusively building to the disk we just use a memory stream at this stage
                        destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);
                    }

                    //Ensure we start at the beginning of the stream
                    destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
                    destinationStream.Flush();
                    destinationStream.Seek(0, SeekOrigin.Begin);
                    ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
                }
                #endregion
            }
            #endregion
        }
Example #2
0
        /// <summary>
        /// TCP - Received by this DFS if a server is telling this instance to build a local file
        /// </summary>
        /// <param name="packetHeader"></param>
        /// <param name="connection"></param>
        /// <param name="assemblyConfig"></param>
        private static void IncomingLocalItemBuild(PacketHeader packetHeader, Connection connection, ItemAssemblyConfig assemblyConfig)
        {
            //We start the build in the DFS task factory as it will be a long lived task
            //BuildTaskFactory.StartNew(() =>
            Action assembleAction = new Action(() =>
                {
                    DistributedItem newItem = null;
                    byte[] itemBytes = null;

                    try
                    {
                        if (assemblyConfig == null)
                            throw new NullReferenceException("AssemblyConfig should not be null.");

                        if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild from " + connection + " for item " + assemblyConfig.CompleteDataCheckSum + ".");

                        //We check to see if we already have the necessary file locally
                        lock (globalDFSLocker)
                        {
                            if (swarmedItemsDict.ContainsKey(assemblyConfig.CompleteDataCheckSum))
                            {
                                if (swarmedItemsDict[assemblyConfig.CompleteDataCheckSum].Data.ItemBytesLength != assemblyConfig.TotalItemSizeInBytes)
                                    throw new Exception("Possible MD5 conflict detected.");
                                else
                                    newItem = swarmedItemsDict[assemblyConfig.CompleteDataCheckSum];
                            }
                            else
                            {
                                newItem = new DistributedItem(assemblyConfig);
                                swarmedItemsDict.Add(assemblyConfig.CompleteDataCheckSum, newItem);
                            }
                        }

                        //Ensure all possible local listeners are added here
                        List<ConnectionInfo> seedConnectionInfoList = (from current in Connection.ExistingLocalListenEndPoints(ConnectionType.TCP) select new ConnectionInfo(ConnectionType.TCP, NetworkComms.NetworkIdentifier, current, true)).ToList();
                        foreach (ConnectionInfo info in seedConnectionInfoList)
                            newItem.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(info, newItem.SwarmChunkAvailability.PeerChunkAvailability(NetworkComms.NetworkIdentifier), newItem.SwarmChunkAvailability.PeerIsSuperPeer(NetworkComms.NetworkIdentifier), false);

                        //Build the item from the swarm
                        //If the item is already complete this will return immediately
                        newItem.AssembleItem((int)(ItemBuildTimeoutSecsPerMB * (assemblyConfig.TotalItemSizeInBytes / (1024.0 * 1024.0))));

                        //Once complete we pass the item bytes back into network comms
                        //If an exception is thrown we will probably not call this method, timeouts in other areas should then handle and can restart the build.
                        if (newItem.LocalItemComplete() && assemblyConfig.CompletedPacketType != "")
                        {
                            if (DFS.loggingEnabled) DFS._DFSLogger.Debug("IncomingLocalItemBuild completed for item with MD5 " + assemblyConfig.CompleteDataCheckSum + ". Item build target is " + assemblyConfig.ItemBuildMode + ".");

                            itemBytes = newItem.GetCompletedItemBytes();
                        }
                        else if (assemblyConfig.CompletedPacketType != "")
                            RemoveItem(assemblyConfig.CompleteDataCheckSum);

                        if (DFS.loggingEnabled)
                        {
                            Exception exceptionToLogWith = new Exception("Build completed successfully. Logging was enabled so saving build log.");
                            string fileName = "DFSItemBuildLog_" + newItem.ItemIdentifier + "_" + NetworkComms.NetworkIdentifier;
                            if (newItem != null)
                                LogTools.LogException(exceptionToLogWith, fileName, newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
                            else
                                LogTools.LogException(exceptionToLogWith, fileName, "newItem==null so no build log was available.");
                        }
                    }
                    catch (ObjectDisposedException)
                    {
                        //The item was closed during assemble, no need to log an errors here
                        RemoveItem(assemblyConfig.CompleteDataCheckSum);
                    }
                    catch (CommsException e)
                    {
                        //Crap an error has happened, let people know we probably don't have a good file
                        RemoveItem(assemblyConfig.CompleteDataCheckSum);
                        //connection.CloseConnection(true, 30);
                        //LogTools.LogException(e, "CommsError_IncomingLocalItemBuild");

                        if (newItem != null)
                            LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
                        else
                            LogTools.LogException(e, "Error_IncomingLocalItemBuildComms", "newItem==null so no build log was available.");
                    }
                    catch (Exception e)
                    {
                        //Crap an error has happened, let people know we probably don't have a good file
                        RemoveItem(assemblyConfig.CompleteDataCheckSum);
                        //connection.CloseConnection(true, 31);

                        if (newItem != null)
                            LogTools.LogException(e, "Error_IncomingLocalItemBuild", newItem.BuildLog().Aggregate(Environment.NewLine, (p, q) => { return p + Environment.NewLine + q; }));
                        else
                            LogTools.LogException(e, "Error_IncomingLocalItemBuild", "newItem==null so no build log was available.");
                    }
                    //finally
                    //{
                    //Putting any code here appears to cause a sigsegv fault on leaving the finally in mono
                    //Just moved the code out to below as it makes no difference
                    //}

                    //Regardless of if the item completed we call the necessary packet handlers
                    //If there was a build error we just pass null data to the handlers so that the errors can get called up the relevant stack traces.
                    try
                    {
                        PacketHeader itemPacketHeader = new PacketHeader(assemblyConfig.CompletedPacketType, newItem == null ? 0 : newItem.Data.ItemBytesLength);
                        //We set the item checksum so that the entire distributed item can be easily retrieved later
                        itemPacketHeader.SetOption(PacketHeaderStringItems.PacketIdentifier, newItem == null ? "" :  newItem.ItemTypeStr + "|" + newItem.ItemIdentifier + "|" + newItem.Data.CompleteDataCheckSum);

                        var dataStream = (itemBytes == null ? new MemoryStream(new byte[0], 0, 0, false, true) : new MemoryStream(itemBytes, 0, itemBytes.Length, false, true));
                        var sendRecieveOptions = new SendReceiveOptions<NullSerializer>(new Dictionary<string, string>());
                        NetworkComms.TriggerAllPacketHandlers(itemPacketHeader, connection, dataStream, sendRecieveOptions);                            
                    }
                    catch (Exception ex)
                    {
                        LogTools.LogException(ex, "Error_IncomingLocalItemBuildFinal");
                    }
                });

            if (BuildTaskFactory == null)
                LogTools.LogException(new NullReferenceException("BuildTaskFactory is null in IncomingLocalItemBuild"), "IncomingLocalBuildError");
            else
                //Thread buildThread = new Thread(buildAction);
                //buildThread.Name = "DFS_" + assemblyConfig.ItemIdentifier + "_Build";
                //buildThread.Start();
                BuildTaskFactory.StartNew(assembleAction);
        }
Example #3
0
        /// <summary>
        /// Introduces a new item into the swarm and sends a build command to the originating requester
        /// </summary>
        /// <param name="peerConnection">The peer which requested the DFS item</param>
        /// <param name="itemToDistribute">The item to be distributed</param>
        /// <param name="completedPacketType">The packet type to use once the item has been fully assembled</param>
        public static void PushItemToPeer(Connection peerConnection, DistributedItem itemToDistribute, string completedPacketType)
        {
            try
            {
                if (peerConnection.ConnectionInfo.ConnectionType != ConnectionType.TCP)
                    throw new Exception("Only able to push DFS item when the request is made via TCP.");


                if (itemToDistribute.ItemClosed)
                    throw new ArgumentException("Unable to push a closed item.");

                ItemAssemblyConfig assemblyConfig;
                lock (globalDFSLocker)
                {
                    //First double check to see if it's already in the swarm
                    if (!ItemAlreadyInLocalCache(itemToDistribute))
                        swarmedItemsDict.Add(itemToDistribute.Data.CompleteDataCheckSum, itemToDistribute);
                    else
                        itemToDistribute = swarmedItemsDict[itemToDistribute.Data.CompleteDataCheckSum];
                }

                itemToDistribute.IncrementPushCount();

                //We add the requester to the item swarm at this point
                itemToDistribute.SwarmChunkAvailability.AddOrUpdateCachedPeerChunkFlags(peerConnection.ConnectionInfo, new ChunkFlags(0));

                //There is a possibility when we create this assembly config that the peer has been removed again
                //We handle this on the peer end
                assemblyConfig = new ItemAssemblyConfig(itemToDistribute, completedPacketType);

                //Send the config information to the client that wanted the file
                peerConnection.SendObject("DFS_IncomingLocalItemBuild", assemblyConfig, nullCompressionSRO);

                if (DFS.loggingEnabled) DFS._DFSLogger.Debug("Pushed DFS item " + itemToDistribute.Data.CompleteDataCheckSum + " to peer " + peerConnection + ".");
            }
            catch (CommsException)
            {
                //LogTools.LogException(ex, "CommsError_AddItemToSwarm");
            }
            catch (Exception ex)
            {
                LogTools.LogException(ex, "Error_AddItemToSwarm");
            }

            //try { GC.Collect(); }
            //catch (Exception) { }
        }
        /// <summary>
        /// Initialise the item data from an assembly config
        /// </summary>
        /// <param name="assemblyConfig"></param>
        public DistributedItemData(ItemAssemblyConfig assemblyConfig)
        {
            this.DataBuildMode = assemblyConfig.ItemBuildMode;
            this.TotalNumChunks = assemblyConfig.TotalNumChunks;
            this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
            this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
            this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
            this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;

            InitialiseChunkPositionLengthDict();

            #region Build Internal Data Structure
            //Create the data stores as per the assembly config
            if (DataBuildMode == DataBuildMode.Memory_Single)
                //ItemBuildMode == ItemBuildMode.Both_Single ||)
            {
                MemoryStream itemStream = new MemoryStream(0);
                itemStream.SetLength(ItemBytesLength);
                CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
            }
            else if (DataBuildMode == DataBuildMode.Disk_Single)
            {
                #region DiskSingle
                string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
                FileStream fileStream;
                if (File.Exists(fileName))
                {
                    //If the file already exists the MD5 had better match otherwise we have a problem
                    try
                    {
                        fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                        if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
                            throw new Exception("Wrong place, wrong time, wrong file!");
                    }
                    catch (Exception)
                    {
                        try
                        {
                            File.Delete(fileName);
                            fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
                        }
                        catch (Exception)
                        {
                            throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
                        }
                    }
                }
                else
                {
                    lock (DFS.globalDFSLocker)
                    {
                        if (!Directory.Exists(folderLocation))
                            Directory.CreateDirectory(folderLocation);
                    }

                    fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
                    fileStream.SetLength(ItemBytesLength);
                    fileStream.Flush();
                }

                CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);

                if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                #endregion
            }
            else
            {
                #region Data Chunks
                //Break the itemDataStream into blocks
                ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];

                for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
                {
                    //We now need to available the respective data into blocks
                    Stream destinationStream = null;
                    if (DataBuildMode == DataBuildMode.Disk_Blocks)
                    {
                        string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                        string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());

                        if (File.Exists(fileName))
                        {
                            try
                            {
                                destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                                if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
                                    throw new Exception("Wrong place, wrong time, wrong file!");
                            }
                            catch (Exception)
                            {
                                try
                                {
                                    File.Delete(fileName);
                                    destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                                }
                                catch (Exception)
                                {
                                    throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
                                }
                            }
                        }
                        else
                        {
                            //Create the folder if it does not exist yet
                            lock (DFS.globalDFSLocker)
                            {
                                if (!Directory.Exists(folderLocation))
                                    Directory.CreateDirectory(folderLocation);
                            }

                            destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                        }

                        if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                    }
                    else
                        //If we are not exclusively building to the disk we just use a memory stream at this stage
                        destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);

                    //Ensure we start at the beginning of the stream
                    destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
                    destinationStream.Flush();
                    destinationStream.Seek(0, SeekOrigin.Begin);
                    ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
                }
                #endregion
            }
            #endregion
        }