A wrapper around a stream to ensure it can be accessed in a thread safe way. The .net implementation of Stream.Synchronized is not suitable on its own.
Inheritance: Stream, IDisposable
コード例 #1
0
        /// <summary>
        /// Initialise the item data from an assembly config
        /// </summary>
        /// <param name="assemblyConfig"></param>
        public DistributedItemData(ItemAssemblyConfig assemblyConfig)
        {
            this.DataBuildMode = assemblyConfig.ItemBuildMode;
            this.TotalNumChunks = assemblyConfig.TotalNumChunks;
            this.ChunkSizeInBytes = assemblyConfig.ChunkSizeInBytes;
            this.CompleteDataCheckSum = assemblyConfig.CompleteDataCheckSum;
            this.ChunkCheckSums = assemblyConfig.ChunkCheckSums;
            this.ItemBytesLength = assemblyConfig.TotalItemSizeInBytes;

            InitialiseChunkPositionLengthDict();

            #region Build Internal Data Structure
            //Create the data stores as per the assembly config
            if (DataBuildMode == DataBuildMode.Memory_Single)
                //ItemBuildMode == ItemBuildMode.Both_Single ||)
            {
                MemoryStream itemStream = new MemoryStream(0);
                itemStream.SetLength(ItemBytesLength);
                CompleteDataStream = new StreamTools.ThreadSafeStream(itemStream);
            }
            else if (DataBuildMode == DataBuildMode.Disk_Single)
            {
                #region DiskSingle
                string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData");
                FileStream fileStream;
                if (File.Exists(fileName))
                {
                    //If the file already exists the MD5 had better match otherwise we have a problem
                    try
                    {
                        fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                        if (StreamTools.MD5(fileStream) != assemblyConfig.CompleteDataCheckSum)
                            throw new Exception("Wrong place, wrong time, wrong file!");
                    }
                    catch (Exception)
                    {
                        try
                        {
                            File.Delete(fileName);
                            fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
                        }
                        catch (Exception)
                        {
                            throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
                        }
                    }
                }
                else
                {
                    lock (DFS.globalDFSLocker)
                    {
                        if (!Directory.Exists(folderLocation))
                            Directory.CreateDirectory(folderLocation);
                    }

                    fileStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose);
                    fileStream.SetLength(ItemBytesLength);
                    fileStream.Flush();
                }

                CompleteDataStream = new StreamTools.ThreadSafeStream(fileStream);

                if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                #endregion
            }
            else
            {
                #region Data Chunks
                //Break the itemDataStream into blocks
                ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];

                for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
                {
                    //We now need to available the respective data into blocks
                    Stream destinationStream = null;
                    if (DataBuildMode == DataBuildMode.Disk_Blocks)
                    {
                        string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                        string fileName = Path.Combine(folderLocation, assemblyConfig.ItemIdentifier + ".DFSItemData_" + i.ToString());

                        if (File.Exists(fileName))
                        {
                            try
                            {
                                destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                                if (assemblyConfig.ChunkCheckSums != null && StreamTools.MD5(destinationStream) != assemblyConfig.ChunkCheckSums[i])
                                    throw new Exception("Wrong place, wrong time, wrong file!");
                            }
                            catch (Exception)
                            {
                                try
                                {
                                    File.Delete(fileName);
                                    destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                                }
                                catch (Exception)
                                {
                                    throw new Exception("File with name '" + fileName + "' already exists. Unfortunately the MD5 does match the expected DFS item. Unable to delete in order to continue.");
                                }
                            }
                        }
                        else
                        {
                            //Create the folder if it does not exist yet
                            lock (DFS.globalDFSLocker)
                            {
                                if (!Directory.Exists(folderLocation))
                                    Directory.CreateDirectory(folderLocation);
                            }

                            destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                        }

                        if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                    }
                    else
                        //If we are not exclusively building to the disk we just use a memory stream at this stage
                        destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);

                    //Ensure we start at the beginning of the stream
                    destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
                    destinationStream.Flush();
                    destinationStream.Seek(0, SeekOrigin.Begin);
                    ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
                }
                #endregion
            }
            #endregion
        }
コード例 #2
0
        /// <summary>
        /// Sets the item data using the provided data stream. Useful for setting data after deserialisation
        /// </summary>
        /// <param name="itemIdentifier"></param>
        /// <param name="itemDataStream"></param>
        public void SetData(string itemIdentifier, Stream itemDataStream)
        {
            lock (dataLocker)
            {
                if (itemIdentifier == null) throw new ArgumentNullException("itemIdentifier");
                if (itemDataStream == null) throw new ArgumentNullException("itemDataStream");

                #region Build Internal Data Structure
                if (DataBuildMode == DataBuildMode.Disk_Single ||
                    //ItemBuildMode == ItemBuildMode.Both_Single ||
                    DataBuildMode == DataBuildMode.Memory_Single)
                {
                    CompleteDataStream = new StreamTools.ThreadSafeStream(itemDataStream);
                }
                else
                {
                    //Break the itemDataStream into blocks
                    ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];

                    //If the itemDataStream is a memory stream we can try to access the buffer as it makes creating the streams more efficient
                    byte[] itemDataStreamBuffer = null;
                    if (itemDataStream is MemoryStream)
                    {
                        try
                        {
                            itemDataStreamBuffer = ((MemoryStream)itemDataStream).GetBuffer();
                        }
                        catch (UnauthorizedAccessException) { /* Ignore */ }
                    }

                    for (int i = 0; i < ChunkPositionLengthDict.Count; i++)
                    {
                        if (itemDataStreamBuffer != null)
                            //This is the fastest way to create a block of data streams
                            ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(new MemoryStream(itemDataStreamBuffer, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length));
                        else
                        {
                            //We now need to available the respective data into blocks
                            Stream destinationStream = null;
                            if (DataBuildMode == DataBuildMode.Disk_Blocks)
                            {
                                string folderLocation = "DFS_" + NetworkComms.NetworkIdentifier;
                                string fileName = Path.Combine(folderLocation, itemIdentifier + ".DFSItemData_" + i.ToString());

                                if (File.Exists(fileName))
                                {
                                    destinationStream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                                    //if (StreamTools.MD5(destinationStream) != checksum)
                                    //    throw new Exception("Wrong place, wrong time, wrong file!");
                                }
                                else
                                {
                                    //Create the folder if it does not exist yet
                                    lock (DFS.globalDFSLocker)
                                    {
                                        if (!Directory.Exists(folderLocation))
                                            Directory.CreateDirectory(folderLocation);
                                    }

                                    destinationStream = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 8192, FileOptions.DeleteOnClose);
                                    destinationStream.SetLength(ChunkPositionLengthDict[i].Length);
                                    destinationStream.Flush();
                                }

                                if (!File.Exists(fileName)) throw new Exception("At this point the item data file should have been created. This exception should not really be possible.");
                            }
                            else
                                //If we are not exclusively building to the disk we just use a memory stream at this stage
                                destinationStream = new MemoryStream(ChunkPositionLengthDict[i].Length);

                            //Ensure we start at the beginning of the stream
                            destinationStream.Seek(0, SeekOrigin.Begin);

                            //Copy over the correct part of the itemDataStream to the chunks
                            StreamTools.Write(itemDataStream, ChunkPositionLengthDict[i].Position, ChunkPositionLengthDict[i].Length, destinationStream, 8192, double.MaxValue, int.MaxValue);
                            ChunkDataStreams[i] = new StreamTools.ThreadSafeStream(destinationStream);
                        }
                    }
                }
                #endregion

                this.CompleteDataCheckSum = StreamTools.MD5(itemDataStream);
            }
        }
コード例 #3
0
        /// <summary>
        /// Initialise the item data using existing chunk streams. Build mode must be to blocks.
        /// </summary>
        /// <param name="dataBuildMode"></param>
        /// <param name="itemDataStreams"></param>
        /// <param name="enableChunkChecksum"></param>
        public DistributedItemData(DataBuildMode dataBuildMode, Dictionary<int, Stream> itemDataStreams, bool enableChunkChecksum = false)
        {
            this.DataBuildMode = DataBuildMode;
            
            if (itemDataStreams == null) throw new ArgumentNullException("itemDataStreams", "itemDataStreams cannot be null.");

            if (dataBuildMode == DataBuildMode.Disk_Single ||
                //itemBuildMode == ItemBuildMode.Both_Single ||
                dataBuildMode == DataBuildMode.Memory_Single)
                throw new ArgumentException("Please use other constructor that takes a single input data stream.");

            this.ItemBytesLength = itemDataStreams.Select(i => i.Value.Length).Sum();

            //Calculate the exactChunkSize if we split everything up into 255 pieces
            double exactChunkSize = (double)ItemBytesLength / 255.0;

            //If the item is too small we just use the minimumChunkSize
            //If we need something larger than MinChunkSizeInBytes we select appropriately
            this.ChunkSizeInBytes = (exactChunkSize <= DFS.MinChunkSizeInBytes ? DFS.MinChunkSizeInBytes : (int)Math.Ceiling(exactChunkSize));
            this.TotalNumChunks = (byte)(Math.Ceiling((double)ItemBytesLength / (double)ChunkSizeInBytes));

            InitialiseChunkPositionLengthDict();

            if (itemDataStreams.Count != ChunkPositionLengthDict.Count)
                throw new ArgumentException("Number of streams should equal the number of chunks");

            //Initialise the data streams
            ChunkDataStreams = new StreamTools.ThreadSafeStream[ChunkPositionLengthDict.Count];
            foreach (int chunkIndex in ChunkPositionLengthDict.Keys)
                ChunkDataStreams[chunkIndex] = new StreamTools.ThreadSafeStream(itemDataStreams[chunkIndex]);

            this.CompleteDataCheckSum = MD5();

            if (enableChunkChecksum) BuildChunkCheckSums();
        }