Example #1
0
        private long RecoverFreeBlocks()
        {
            Trace.TraceWarning("The file store was not closed properly, recovering free blocks.");

            using (Stream s = _streamCache.Open(FileAccess.ReadWrite))
                using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
                {
                    long lastBlock = LastAllocated(s);
                    long last      = 0;
                    for (long blk = lastBlock; blk > 0; blk -= _blockSize)
                    {
                        s.Position = blk & _maskOffset;
                        block.Read(s, FileBlock.HeaderSize);
                        Check.Assert <InvalidDataException>((block.BlockId & _maskOffset) == blk);
                        if ((block.Flags & BlockFlags.BlockDeleted) == BlockFlags.BlockDeleted)
                        {
                            block.NextBlockId = last;
                            last       = block.BlockId;
                            s.Position = blk & _maskOffset;
                            block.Write(s, FileBlock.HeaderSize);
                        }
                    }
                    return(last);
                }
        }
Example #2
0
        /// <summary>
        ///
        /// </summary>
        /// <remarks>Exposed for testability.</remarks>
        internal static async Task PopulateMetadataAsync(MetadataTable metadataTable, Stream stream, string traceType)
        {
            // Read and validate the Footer section.  The footer is always at the end of the stream, minus space for the checksum.
            var footerHandle = new BlockHandle(stream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
            var footer       = await FileBlock.ReadBlockAsync(stream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);

            // Verify we know how to deserialize this version of the file.
            if (footer.Version != FileVersion)
            {
                throw new InvalidDataException(SR.Error_MetadataManager_Deserialized);
            }

            // Read and validate the Properties section.
            var propertiesHandle = footer.PropertiesHandle;
            var properties       =
                await
                FileBlock.ReadBlockAsync(
                    stream,
                    propertiesHandle,
                    (sectionReader, sectionHandle) => FilePropertySection.Read <MetadataManagerFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);

            // Read disk metadata into memory.
            await ReadDiskMetadataAsync(metadataTable.Table, stream, properties, traceType).ConfigureAwait(false);

            metadataTable.CheckpointLSN = properties.CheckpointLSN;
        }
Example #3
0
        private bool FreeBlock(long blockId, BlockFlags expected)
        {
            using (FileBlock first = new FileBlock(_blockSize, _useAlignedIo))
            {
                ReadBlock(blockId, first, FileBlock.HeaderSize, expected | BlockFlags.BlockDeleted);

                if ((first.Flags & expected) != expected)
                {
                    return(false);
                }

                using (FileBlock last = first.Clone())
                {
                    while (last.NextBlockId != 0)
                    {
                        last.Flags = BlockFlags.BlockDeleted;

                        WriteBlock(last.BlockId, last, FileBlock.HeaderSize);
                        ReadBlock(last.NextBlockId, last, FileBlock.HeaderSize, BlockFlags.InternalBlock);
                    }

                    using (new SafeLock(_syncFreeBlock))
                    {
                        last.Flags       = BlockFlags.BlockDeleted;
                        last.NextBlockId = _nextFree;
                        WriteBlock(last.BlockId, last, FileBlock.HeaderSize);
                        _nextFree = first.BlockId;
                    }
                    return(true);
                }
            }
        }
Example #4
0
        public void TestBlockReadError()
        {
            // Create a memory stream to generate the data to work with
            var memStream = new MemoryStream();
            var writer    = new BinaryWriter(memStream);

            byte[] contents    = { 0xFF, 0xFE, 0xFD, 0xFC };
            byte[] offsetBytes = { 0, 0, 0, 0 };

            // Write some data off so the memory stream starts with an offset
            writer.Write(offsetBytes, 0, 4);

            // Block ID
            writer.Write(FileBlock.BLOCKID_NULL);
            // Block length
            writer.Write(contents.LongLength + 1); // Write a block length that is larger than the real block
            // Block version
            writer.Write((short)1);

            // Block contents
            memStream.Write(contents, 0, 4);

            // Reset memory stream position and read a block
            // We reset to just past the offset so the block offset property can be tested as well
            memStream.Position = offsetBytes.Length;

            FileBlock.FromStream(memStream, null);
        }
Example #5
0
        public void TestFileBlockIds()
        {
            var fileBlock       = new FileBlock();
            var animBlock       = new AnimationBlock();
            var animHeaderBlock = new AnimationHeaderBlock();
            var sheetBlock      = new AnimationSheetBlock();
            var frameBlock      = new FrameBlock();
            var treeBlock       = new ProjectTreeBlock();

            // Assert block IDs
            Assert.AreEqual(FileBlock.BLOCKID_NULL, fileBlock.BlockID,
                            "The ID for the FileBlock must equal to FileBlock.BLOCKID_NULL");

            Assert.AreEqual(FileBlock.BLOCKID_ANIMATION, animBlock.BlockID,
                            "The ID for the AnimationBlock instance does not match the ID specified for it on the FileBlock class");

            Assert.AreEqual(FileBlock.BLOCKID_ANIMATION_HEADER, animHeaderBlock.BlockID,
                            "The ID for the AnimationHeaderBlock instance does not match the ID specified for it on the FileBlock class");

            Assert.AreEqual(FileBlock.BLOCKID_ANIMATIONSHEET, sheetBlock.BlockID,
                            "The ID for the AnimationSheetBlock instance does not match the ID specified for it on the FileBlock class");

            Assert.AreEqual(FileBlock.BLOCKID_FRAME, frameBlock.BlockID,
                            "The ID for the frameBlock instance does not match the ID specified for it on the FileBlock class");

            Assert.AreEqual(FileBlock.BLOCKID_PROJECTTREE, treeBlock.BlockID,
                            "The ID for the treeBlock instance does not match the ID specified for it on the FileBlock class");
        }
Example #6
0
            public BlockStreamWriter(FragmentedFile file, long ordinal)
                : this(file)
            {
                _current = _first = new FileBlock(file._blockSize, file._useAlignedIo);
                _file.ReadBlock(ordinal, _current, file._blockSize, BlockFlags.ExternalBlock);

                if ((_current.Flags & BlockFlags.ExternalBlock) != BlockFlags.ExternalBlock)
                {
                    throw new InvalidOperationException();
                }

                if ((_current.Flags & BlockFlags.Temporary) == BlockFlags.Temporary)
                {
                    _isNew          = true;
                    _restore        = null;
                    _current.Flags &= ~BlockFlags.Temporary;
                }
                else
                {
                    _isNew               = false;
                    _restore             = _current.Clone();
                    _current.NextBlockId = 0;
                }

                _current.Length = 0;
            }
Example #7
0
        public void TestBlockWrite()
        {
            // Create a memory stream to generate the data to work with
            var memStream      = new MemoryStream();
            var writeMemStream = new MemoryStream();
            var writer         = new BinaryWriter(memStream);

            byte[] contents    = { 0xFF, 0xFE, 0xFD, 0xFC };
            byte[] offsetBytes = { 0, 0, 0, 0 };

            // Write some data off so the memory stream starts with an offset
            writer.Write(offsetBytes, 0, 4);
            writeMemStream.Write(offsetBytes, 0, 4);

            // Block ID
            writer.Write(FileBlock.BLOCKID_NULL);
            // Block length
            writer.Write(contents.LongLength);
            // Block version
            writer.Write((short)1);

            // Block contents
            memStream.Write(contents, 0, 4);

            // Reset memory stream position and read a block
            // We reset to just past the offset so the block offset property can be tested as well
            memStream.Position = offsetBytes.Length;

            var block = FileBlock.FromStream(memStream, null);

            block.SaveToStream(writeMemStream);

            Assert.IsTrue(memStream.GetBuffer().SequenceEqual(writeMemStream.GetBuffer()),
                          "After writing the contents of a block back to a stream, its contents must exactly match the original contents that were read");
        }
Example #8
0
        public bool UploadCloudFile(FileBlock file, string destinationPath = null)
        {
            bool status = false;

            try
            {
                using (var client = DriveClient.GetInfo())
                {
                    using (var stream = new FileStream(file.path, FileMode.OpenOrCreate))
                    {
                        FilesResource.CreateMediaUpload request = new FilesResource.CreateMediaUpload(client,
                                                                                                      new Google.Apis.Drive.v3.Data.File
                        {
                            Name    = file.Name,
                            Parents = file.Parent,
                        },
                                                                                                      stream,
                                                                                                      file.mimeType);
                        request.ChunkSize = 1024 * 1024;
                        Task initiateSessionTask = Task.Run(() => request.InitiateSessionAsync());
                        initiateSessionTask.Wait();
                        Task <Google.Apis.Upload.IUploadProgress> uploadTask = Task.Run(() => request.UploadAsync());
                        uploadTask.Wait();
                        status = true;
                    }
                }
            }
            catch (Exception e)
            {
                throw new Exception(e.Message + " " + e.StackTrace);
            }

            return(status);
        }
        /// <summary>
        /// Deserializes the metadata (footer, properties, etc.) for this checkpoint file.
        /// </summary>
        private async Task ReadMetadataAsync()
        {
            Stream filestream = null;

            try
            {
                filestream = this.ReaderPool.AcquireStream();

                // Read and validate the Footer section.  The footer is always at the end of the stream, minus space for the checksum.
                var footerHandle = new BlockHandle(filestream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
                this.Footer =
                    await FileBlock.ReadBlockAsync(filestream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);

                // Verify we know how to deserialize this version of the checkpoint file.
                if (this.Footer.Version != FileVersion)
                {
                    throw new InvalidDataException(SR.Error_KeyCheckpointFile_Deserialized);
                }

                // Read and validate the Properties section.
                var propertiesHandle = this.Footer.PropertiesHandle;
                this.Properties =
                    await
                    FileBlock.ReadBlockAsync(
                        filestream,
                        propertiesHandle,
                        (sectionReader, sectionHandle) => FilePropertySection.Read <KeyCheckpointFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
            }
            finally
            {
                this.ReaderPool.ReleaseStream(filestream, true);
            }
        }
Example #10
0
        public bool UploadCloudFile(FileBlock file, string destinationPath = null)
        {
            bool status = false;

            try
            {
                using (var stream = new FileStream(file.path, FileMode.OpenOrCreate))
                {
                    BoxFileRequest request = new BoxFileRequest()
                    {
                        Name   = file.id,
                        Parent = new BoxRequestEntity()
                        {
                            Id = file.Parent[0]
                        }
                    };
                    Task task = Task.Run(() => client.FilesManager.UploadAsync(request, stream));
                    task.Wait();
                    status = true;
                }
            }
            catch (Exception e)
            {
                throw e;
            }
            return(status);
        }
Example #11
0
        internal FileBlock CreateFileObject(Box.V2.Models.BoxItem item, DirectoryBlock parentDirectory)
        {
            string parent = null;

            if (item.Parent == null)
            {
                parent = "";
            }
            FileBlock file = new FileBlock()
            {
                id           = item.Id,
                Name         = item.Name,
                Description  = item.Description,
                Size         = item.Size,
                SharedLink   = GetSharedLink(item),
                Owners       = GetOwners(item),
                CreationTime = item.CreatedAt.ToString(),
                // Users = GetUsers(item),
                Parent = new List <string> {
                    parent
                },
                path = parentDirectory.path + "\\" + item.Name,
            };

            return(file);
        }
Example #12
0
        private List <FileBlock> ReadFileBlocks(string directoryName)
        {
            List <FileBlock> fileBlocks = new List <FileBlock>();

            foreach (var file in Directory.GetFiles(directoryName))
            {
                try
                {
                    FileInfo  f         = new FileInfo(file);
                    FileBlock fileBlock = new FileBlock(SelectedFile.FileContentHashID,
                                                        int.Parse(Path.GetFileNameWithoutExtension(file)),
                                                        f.Length);


                    fileBlock.ContentHash = Utility.ToString(Utility.ComputeHashSumForFile(file), true);

                    fileBlocks.Add(fileBlock);
                }
                catch (Exception ex)
                {
                    MessageBox.Show(ex.ToString(), "Error", MessageBoxButton.OK, MessageBoxImage.Error);
                }
            }

            return(fileBlocks);
        }
Example #13
0
        public IActionResult Add([FromRoute] long fileId, [FromBody] FileBlockInfo fileBlockInfo)
        {
            FileBlock fileBlock = _db.FileBlocks.FirstOrDefault(fb => fb.FileId == fileId && fb.Number == fileBlockInfo.Number);

            using (MD5 md5Hash = MD5.Create())
            {
                byte[] rawContent = Convert.FromBase64String(fileBlockInfo.Value);

                if (fileBlock == null) //Create new fileBlock
                {
                    _db.FileBlocks.Add(new FileBlock()
                    {
                        FileId   = fileId,
                        Number   = fileBlockInfo.Number,
                        Content  = rawContent,
                        Checksum = MD5Utils.GetMd5Hash(md5Hash, rawContent.ToString())
                    });
                }
                else //Update existing fileBlock
                {
                    fileBlock.Content  = rawContent;
                    fileBlock.Checksum = MD5Utils.GetMd5Hash(md5Hash, rawContent.ToString());
                    _db.FileBlocks.Update(fileBlock);
                }

                _db.SaveChanges();
            }

            return(StatusCode((int)HttpStatusCode.OK));
        }
Example #14
0
 public FolderEntry(FileBlock fileBlock)
 {
     Type           = EntryType.file;
     Name           = fileBlock.FileName;
     Size           = fileBlock.Size;
     this.fileBlock = fileBlock;
 }
Example #15
0
        private void OnDownload(object state)
        {
            FrmProgress frm = (FrmProgress)state;

            using (FileHelper fh = new FileHelper(mSaveFileName, false))
            {
                while (true)
                {
                    FileBlock fb = mClient.Read <FileBlock>();
                    fh.Write(fb.Data, 0, fb.Data.Length);

                    Invoke(new Action <FrmProgress>(o =>
                    {
                        frm.ChangeProgress(fb.Data.Length);
                        if (fb.Eof)
                        {
                            frm.Hide();
                            cmdDownload.Enabled = true;
                        }
                    }), frm);
                    if (fb.Eof)
                    {
                        break;
                    }
                }
            }
        }
Example #16
0
        public void TestBlockRead()
        {
            // Create a memory stream to generate the data to work with
            var memStream = new MemoryStream();
            var writer    = new BinaryWriter(memStream);

            byte[] contents    = { 0xFF, 0xFE, 0xFD, 0xFC };
            byte[] offsetBytes = { 0, 0, 0, 0 };

            // Write some data off so the memory stream starts with an offset
            writer.Write(offsetBytes, 0, 4);

            // Block ID
            writer.Write(FileBlock.BLOCKID_NULL);
            // Block length
            writer.Write(contents.LongLength);
            // Block version
            writer.Write((short)1);

            // Block contents
            memStream.Write(contents, 0, 4);

            // Reset memory stream position and read a block
            // We reset to just past the offset so the block offset property can be tested as well
            memStream.Position = offsetBytes.Length;

            var block = FileBlock.FromStream(memStream, null);

            Assert.AreEqual(FileBlock.BLOCKID_NULL, block.BlockID, "The block's ID must match the data that was read off the stream");
            Assert.AreEqual(offsetBytes.Length, block.BlockOffset, "The block's length must match the block content's size in bytes");
            Assert.AreEqual(contents.Length, block.BlockLength, "The block's length must match the block content's size in bytes");
            Assert.AreEqual(1, block.BlockVersion, "The block's ID must match the data that was read off the stream");
            Assert.IsTrue(contents.SequenceEqual(block.GetBlockBuffer()), "The buffered data on the block must match the data that was read off the stream");
        }
Example #17
0
            public FileBlock Clone()
            {
                FileBlock copy = new FileBlock(BlockSize, _alignedIo);

                Array.Copy(BlockData, _baseOffset, copy.BlockData, copy._baseOffset, BlockSize);
                return(copy);
            }
        public void WriteFileBlock(byte[] blockContent, long blockIndex)
        {
            var fileBlock = new FileBlock(blockContent, blockIndex);

            AddBlockToList(fileBlock);

            ExecuteWrite();
        }
Example #19
0
 /// <summary> Creates a new allocation block within the file </summary>
 /// <returns> A unique integer id for the block to be used with Open/Delete </returns>
 public long Create()
 {
     using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
     {
         long identity = AllocBlock(block, BlockFlags.ExternalBlock);
         return(identity);
     }
 }
 public NormalResponse UploadFileBlock(FileBlock file)
 {
     if (file == null)
     {
         return(new NormalResponse(false, "文件块为空"));
     }
     return(FileBlock.WritrBuffer(file));
 }
Example #21
0
 /// <summary> Creates a new allocation block within the file </summary>
 /// <param name="identity">A unique integer id for the block to be used with Open/Delete</param>
 /// <returns>The stream to write to the newly created block</returns>
 public Stream Create(out long identity)
 {
     using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
     {
         identity = AllocBlock(block, BlockFlags.ExternalBlock);
         return(new BlockStreamWriter(this, block));
     }
 }
Example #22
0
        /// <summary>
        /// Create a new <see cref="RecoveryPointMetadataFile"/> and write it to the given file.
        /// </summary>
        /// <returns>Task that represents the asynchronous operation.</returns>
        public static async Task <RecoveryPointMetadataFile> CreateAsync(
            string fileName,
            DateTime backupTime,
            Guid backupId,
            Guid parentBackupId,
            Guid backupChainId,
            Epoch epochOfLastBackupRecord,
            long lsnOfLastBackupRecord,
            string backupLocation,
            string parentBackupLocation,
            ServicePartitionInformation partitionInformation,
            string serviceManifestVersion,
            CancellationToken cancellationToken)
        {
            // Create the file with asynchronous flag and 4096 cache size (C# default).
            using (var filestream = FabricFile.Open(
                       fileName,
                       FileMode.CreateNew,
                       FileAccess.Write,
                       FileShare.Read,
                       4096,
                       FileOptions.Asynchronous))
            {
                BackupRestoreUtility.SetIoPriorityHint(filestream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintLow);

                var recoveryPointMetadataFile = new RecoveryPointMetadataFile(fileName);
                recoveryPointMetadataFile.properties = new RecoveryPointMetadataFileProperties
                {
                    BackupTime              = backupTime,
                    BackupLocation          = backupLocation,
                    ParentBackupLocation    = parentBackupLocation,
                    BackupId                = backupId,
                    ParentBackupId          = parentBackupId,
                    BackupChainId           = backupChainId,
                    EpochOfLastBackupRecord = epochOfLastBackupRecord,
                    LsnOfLastBackupRecord   = lsnOfLastBackupRecord,
                    PartitionInformation    = partitionInformation,
                    ServiceManifestVersion  = serviceManifestVersion,
                };

                // Write the properties.
                var propertiesHandle =
                    await FileBlock.WriteBlockAsync(filestream, writer => recoveryPointMetadataFile.properties.Write(writer)).ConfigureAwait(false);

                cancellationToken.ThrowIfCancellationRequested();

                // Write the footer.
                recoveryPointMetadataFile.footer = new FileFooter(propertiesHandle, Version);
                await FileBlock.WriteBlockAsync(filestream, writer => recoveryPointMetadataFile.footer.Write(writer)).ConfigureAwait(false);

                cancellationToken.ThrowIfCancellationRequested();

                // Flush to underlying stream.
                await filestream.FlushAsync(cancellationToken).ConfigureAwait(false);

                return(recoveryPointMetadataFile);
            }
        }
Example #23
0
 public BlockStreamWriter(FragmentedFile file, FileBlock block)
     : this(file)
 {
     _current        = _first = block;
     _current.Flags &= ~BlockFlags.Temporary;
     _restore        = null;
     _temp           = null;
     _isNew          = true;
 }
        private void ResolveOutputTargetProjectItem(FileBlock fileBlock)
        {
            OutputTarget outputTarget = fileBlock.Target;

            Project project = string.IsNullOrEmpty(outputTarget.ProjectName)
                ? null
                : FindProjectByName(dte, outputTarget.ProjectName);

            ProjectItems targetProjectItems = null;

            if (project != null)
            {
                if (!string.IsNullOrEmpty(outputTarget.ProjectPath))
                {
                    var pathItems    = outputTarget.ProjectPath.Split('\\');
                    var projectItems = project.ProjectItems;
                    foreach (var pathItem in pathItems)
                    {
                        var projItem = FindProjectItem(dte, projectItems, pathItem, Constants.vsProjectItemKindPhysicalFolder);
                        if (projItem != null)
                        {
                            if (projItem.ProjectItems.Count > 0)
                            {
                                projectItems = projItem.ProjectItems;
                            }

                            if (pathItem == pathItems.Last())
                            {
                                targetProjectItems = projItem.ProjectItems;
                            }
                        }
                    }
                }
                else
                {
                    var dir             = Path.GetDirectoryName(project.FileName);
                    var templateFileDir = TemplateFileAsOutputTarget.OutputDirectory;

                    if (dir != templateFileDir)
                    {
                        targetProjectItems = project.ProjectItems;
                    }
                }
            }

            //outputTarget.ProjectItem = targetProjectItem ?? TemplateFileAsOutputTarget;
            if (targetProjectItems != null)
            {
                outputTarget.Update(targetProjectItems);
            }
            else
            {
                fileBlock.Target = TemplateFileAsOutputTarget;
            }
        }
    private void EndFileBlock()
    {
        if (CurrentFileBlock == null)
        {
            return;
        }

        EndBlockData(CurrentFileBlock); //.Length = this.Template.Length - CurrentBlock.Start;
        //if (CurrentBlock != header && CurrentBlock != footer)
        files.Add(CurrentFileBlock);
        currentFileBlock = null;
    }
Example #26
0
        private FileBlock ReadFileBlock(XPathNavigator nav)
        {
            FileBlock result = new FileBlock();

            result.Begin        = ReadIntAttribute(nav, "begin");
            result.Count        = ReadIntAttribute(nav, "count");
            result.EnableFileId = ReadIntAttribute(nav, "enable");
            result.OffsetFileId = ReadIntAttribute(nav, "offset");
            result.FileId       = nav.ValueAsInt;

            return(result);
        }
    public virtual void StartNewFile(String name, OutputTarget outputTarget)
    {
        if (name == null)
        {
            throw new ArgumentNullException("name");
        }

        CurrentFileBlock = new FileBlock
        {
            Name   = name,
            Target = outputTarget
        };
    }
Example #28
0
        internal FileBlock CreateFileObject(IListBlobItem item, DirectoryBlock directory)
        {
            FileBlock file = new FileBlock()
            {
                Name = GetName(item),
                //Size = blob.Properties.Length,
                container  = item.Container.Name,
                SharedLink = item.Uri.ToString(),
                path       = directory.path + "\\" + GetName(item),
                CloudUri   = item.Uri,
            };

            return(file);
        }
        private void RemoveFromBlockList(FileBlock fileBlock)
        {
            _listLock.EnterWriteLock();

            try
            {
                _pendingFileBlocks.Remove(fileBlock);
            }
            finally
            {
                if (_listLock.IsWriteLockHeld)
                {
                    _listLock.ExitWriteLock();
                }
            }
        }
Example #30
0
        public static JsonData ReadPAC(FileBlock fb)
        {
            var pac = new Pac(fb);

            return(new JsonData {
                Meshes = pac.Meshes.Select(m => new Mesh {
                    TextureName = m.TextureName,
                    LODs = m.LOD.Select(l => new Geometry {
                        Positions = l.Vertexes.SelectMany(v => v.Position).ToList(),
                        UVs = l.Vertexes.SelectMany(v => v.UV.Select(u => (float)Half.ToHalf(u))).ToList(),
                        FaceIndices = l.FaceIndices,
                        Normals = TangentsToNormals(l.Vertexes.SelectMany(v => v.Normals)).ToList()
                    }).ToList()
                }).ToList()
            });
        }
            private void PrepareWrite()
            {
                if (_saved || _reverted) throw new ObjectDisposedException(GetType().FullName);
                
                if (_current.Length == FileBlockDataSize)//full
                {
                    FileBlock next = _temp ?? new FileBlock(_file._blockSize, _file._useAlignedIo);
                    _current.NextBlockId = _file.AllocBlock(next, BlockFlags.InternalBlock);

                    if (!ReferenceEquals(_current, _first))
                    {
                        _file.WriteBlock(_current.BlockId, _current, _current.Length + FileBlock.HeaderSize);
                        _temp = _current;
                    }
                    _current = next;
                }
            }
        /// <summary> Internal use to specify aligned IO when using NoBuffering file option </summary>
        protected FragmentedFile(IFactory<Stream> streamFactory, int blockSize, int growthRate, int cacheLimit, FileOptions options)
        {
            _useAlignedIo = (options & NoBuffering) == NoBuffering;
            _streamCache = new StreamCache(streamFactory, cacheLimit);
            _header = new FileBlock(blockSize, _useAlignedIo);
            _syncFreeBlock = new object();
            try
            {
                long fallocated;
                bool canWrite;

                using (Stream s = _streamCache.Open(FileAccess.ReadWrite))
                {
                    canWrite = s.CanWrite;
                    if (!s.CanRead)
                        throw new InvalidOperationException("The stream does not support Read access.");

                    _header.Read(s, blockSize);

                    if ((_header.Flags & ~BlockFlags.HeaderFilter) != BlockFlags.HeaderFlags)
                        throw new InvalidDataException();

                    _nextFree = _header.NextBlockId;
                    SetBlockSize(_header.Length, out _blockSize, out _maskVersion, out _maskOffset);
                    if (blockSize != _blockSize) throw new ArgumentOutOfRangeException("blockSize");
                    fallocated = LastAllocated(s);
                    _reallocSize = growthRate * _blockSize;

                    if (canWrite)
                    {
                        s.Position = 0;
                        _header.NextBlockId = long.MinValue;
                        _header.Write(s, FileBlock.HeaderSize);
                    }
                }

                if (canWrite)
                {
                    if ((_header.Flags & BlockFlags.ResizingFile) == BlockFlags.ResizingFile && _nextFree > 0)
                        ResizeFile(_nextFree, Math.Max(fallocated, _nextFree + _reallocSize));

                    if (_nextFree == long.MinValue)
                        _nextFree = RecoverFreeBlocks();
                }
            }
            catch
            {
                _streamCache.Dispose();
                throw;
            }
        }
        /// <summary>
        /// Creates a new file (or truncates an existing one) that stores multiple smaller files
        /// </summary>
        private static void WriteEmtpy(IFactory<Stream> streamFactory, int blockSize)
        {
            long mask;
            SetBlockSize(blockSize, out blockSize, out mask, out mask);
            using (FileBlock block = new FileBlock(Check.InRange(blockSize, 512, /*65,536*/0x10000), false))
            {
                block.Length = blockSize;
                block.Flags = BlockFlags.HeaderFlags;

                using (Stream f = streamFactory.Create())
                {
                    f.Position = 0;
                    block.Write(f, blockSize);
                    f.SetLength(f.Position);
                }
            }
        }
 /// <summary> Creates a new allocation block within the file </summary>
 /// <returns> A unique integer id for the block to be used with Open/Delete </returns>
 public long Create()
 {
     using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
     {
         long identity = AllocBlock(block, BlockFlags.ExternalBlock);
         return identity;
     }
 }
 /// <summary> Creates a new allocation block within the file </summary>
 /// <param name="identity">A unique integer id for the block to be used with Open/Delete</param>
 /// <returns>The stream to write to the newly created block</returns>
 public Stream Create(out long identity)
 {
     using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
     {
         identity = AllocBlock(block, BlockFlags.ExternalBlock);
         return new BlockStreamWriter(this, block);
     }
 }
        private void WriteBlock(long ordinal, FileBlock block, int length)
        {
            if (block.BlockId != ordinal)
                throw new InvalidDataException();

            if (block.Length < 0 || block.Length > (_blockSize - FileBlock.HeaderSize))
                throw new InvalidDataException();

            try { } 
            finally 
            {
                using (Stream io = OpenBlock(FileAccess.Write, ordinal))
                    block.Write(io, length);
            }
        }
        private bool FreeBlock(long blockId, BlockFlags expected)
        {
            using (FileBlock first = new FileBlock(_blockSize, _useAlignedIo))
            {
                ReadBlock(blockId, first, FileBlock.HeaderSize, expected | BlockFlags.BlockDeleted);

                if ((first.Flags & expected) != expected)
                    return false;

                using (FileBlock last = first.Clone())
                {
                    while (last.NextBlockId != 0)
                    {
                        last.Flags = BlockFlags.BlockDeleted;

                        WriteBlock(last.BlockId, last, FileBlock.HeaderSize);
                        ReadBlock(last.NextBlockId, last, FileBlock.HeaderSize, BlockFlags.InternalBlock);
                    }

                    using (new SafeLock(_syncFreeBlock))
                    {
                        last.Flags = BlockFlags.BlockDeleted;
                        last.NextBlockId = _nextFree;
                        WriteBlock(last.BlockId, last, FileBlock.HeaderSize);
                        _nextFree = first.BlockId;
                    }
                    return true;
                }
            }
        }
Example #38
0
        public void Init(IApplication application)
        {
            application[BASE_PATH] = System.Configuration.ConfigurationManager.AppSettings["path"];
            application.SendCompleted += (o, e) => {
                object tag = e.Info.Data.Tag;
                if (tag is FileInfo || tag is FileBlock)
                {
                    FileHelper fh = (FileHelper)e.Session["DOWNLOAD_TAG"];
                    if (fh != null)
                    {
                       
                        FileBlock fb = new FileBlock();
                        fb.Data = fh.Read();
                        fb.Eof = fh.Eof;
                        if (fb.Eof)
                        {
                            fh.Dispose();
                            e.Session["DOWNLOAD_TAG"] = null;
                        }
                        e.Application.Server.Send(fb, e.Session.Channel);
                    }

                }
            };
        }
        private long AllocBlock(FileBlock block, BlockFlags type)
        {
            using (new SafeLock(_syncFreeBlock))
            {
                long blockId = _nextFree;
                if (blockId == 0 && _reallocSize > 0)
                {
                    long fsize;
                    using (Stream s = _streamCache.Open(FileAccess.Read))
                        fsize = LastAllocated(s);
                    ResizeFile(fsize + _blockSize, fsize + _reallocSize);
                    blockId = _nextFree;
                }

                if (blockId <= 0)
                    throw new IOException();

                using (Stream io = OpenBlock(FileAccess.Read, blockId))
                    block.Read(io, FileBlock.HeaderSize);

                if ((block.BlockId & _maskOffset) != (blockId & _maskOffset) || (block.Flags & BlockFlags.BlockDeleted) == 0)
                    throw new InvalidDataException();

                _nextFree = block.NextBlockId;
            
                block.BlockId = blockId;
                block.IncrementId(_maskVersion);
                block.NextBlockId = 0;
                block.Flags = type == BlockFlags.ExternalBlock ? (type | BlockFlags.Temporary) : type;
                block.Length = 0;
                WriteBlock(block.BlockId, block, FileBlock.HeaderSize); 
                return block.BlockId;
            }
        }
        private void ResizeFile(long startBlock, long endBlock)
        {
            if ((startBlock & _maskVersion) != 0 || (endBlock & _maskVersion) != 0)
                throw new InvalidDataException();

            using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
            using (Stream io = _streamCache.Open(FileAccess.Write))
            {
                _header.Flags |= BlockFlags.ResizingFile;
                _header.NextBlockId = startBlock;
                io.Position = 0;
                _header.Write(io, FileBlock.HeaderSize);
                try
                {
                    block.Clear();
                    block.Flags = BlockFlags.BlockDeleted;
                    _nextFree = 0;

                    for (long ix = endBlock; ix >= startBlock; ix -= _blockSize)
                    {
                        block.BlockId = ix;
                        block.NextBlockId = _nextFree;
                        _nextFree = block.BlockId;

                        io.Position = ix & _maskOffset;
                        block.Write(io, FileBlock.HeaderSize);
                    }
                }
                finally
                {
                    _header.Flags &= ~BlockFlags.ResizingFile;
                    _header.NextBlockId = long.MinValue;
                    io.Position = 0;
                    _header.Write(io, FileBlock.HeaderSize);
                }
            }
        }
 private void ReadBlock(long ordinal, FileBlock block, int length, BlockFlags type)
 { ReadBlock(ordinal, block, length, type, true); }
        private void ReadBlock(long ordinal, FileBlock block, int length, BlockFlags type, bool exactId)
        {
            using (Stream io = OpenBlock(FileAccess.Read, ordinal))
                block.Read(io, length);

            if (exactId && block.BlockId != ordinal)
                throw new InvalidDataException();

            if ((block.Flags & type) == 0 && type != 0)
                throw new InvalidDataException();

            if (block.Length < 0 || block.Length > (_blockSize - FileBlock.HeaderSize))
                throw new InvalidDataException();
        }
Example #43
0
        /// <summary>
        /// 校验文件
        /// </summary>
        /// <returns>损坏或尚未下载的区块序号列表</returns>
        public List<int> HashFile()
        {
            FileStream.Position = 0;
            ExistBlock.Clear();
            for (int count = 0; FileStream.Position < FileStream.Length && count < TotalBlock; count++)
            {
//校验已存在的区块
                var testBlock = new FileBlock(this, count, true);
                SendString(string.Format("GET BlockHash {0}", count));
                string[] msg = ReceiveString().Split(' ');
                if (msg[0] == "BlockHash")
                {
                    if (Convert.ToInt32(msg[1]) == count)
                    {
                        if (BitConverter.ToInt32(testBlock.DataHash, 0) == Convert.ToInt32(msg[2]))
                            ExistBlock.Add(count);
                    }
                }
                if (BlockHashed != null)
                    BlockHashed(this, new BlockFinishedEventArgs(count));
            }
            int maxExistBlockIndex; //已存在的区块最大序号
            try
            {
                maxExistBlockIndex = ExistBlock.Max();
            }
            catch
            {
                maxExistBlockIndex = 0;
            }
            var blockRemaining = new List<int>();
            for (int index = 0; index < TotalBlock;)
            {
//计算仍需传输的区块
                if (index <= maxExistBlockIndex)
                {
                    if (ExistBlock.Exists(a => a == index))
                    {
                        index++;
                        continue;
                    }
                }
                blockRemaining.Add(index++);
            }
            return blockRemaining;
        }
        /// <summary>
        /// 读取或写入区块
        /// </summary>
        /// <param name="blockIndex">区块序号</param>
        public FileBlock this[int blockIndex]
        {
            get
            {
                FileBlock output;
                if (EnabledIoBuffer)
                {

                    bool isInBuf;
                    lock (IoBuffer)
                        isInBuf = IoBuffer.TryGetValue(blockIndex, out output);
                    if (isInBuf)
                        return output;
                    output = new FileBlock(Task, blockIndex, true);
                    BeginFillIoBuffer(blockIndex + 1, null, null);
                }
                else
                    output = new FileBlock(Task, blockIndex, true);
                return output;
            }
            set
            {
                if (blockIndex != value.Index)
                    throw new FileBlockException("Bad Index!", FileBlockException.ErrorCode.BadIndex);
                Write(value);
            }
        }
            public BlockStreamWriter(FragmentedFile file, long ordinal)
                : this(file)
            {
                _current = _first = new FileBlock(file._blockSize, file._useAlignedIo);
                _file.ReadBlock(ordinal, _current, file._blockSize, BlockFlags.ExternalBlock);

                if ((_current.Flags & BlockFlags.ExternalBlock) != BlockFlags.ExternalBlock)
                    throw new InvalidOperationException();

                if ((_current.Flags & BlockFlags.Temporary) == BlockFlags.Temporary)
                {
                    _isNew = true;
                    _restore = null;
                    _current.Flags &= ~BlockFlags.Temporary;
                }
                else
                {
                    _isNew = false;
                    _restore = _current.Clone();
                    _current.NextBlockId = 0;
                }

                _current.Length = 0;
            }
 /// <summary>
 /// 写入区块
 /// </summary>
 /// <param name="value">区块对象</param>
 public void Write(FileBlock value)
 {
     if (EnabledIoBuffer)
     {
         if (IoBuffer.Count >= IoBufferSize)
             WriteAllBlock();
         lock (IoBuffer)
             IoBuffer.Add(value.Index, value);
     }
     else
         value.Write();
 }
        private long RecoverFreeBlocks()
        {
            Trace.TraceWarning("The file store was not closed properly, recovering free blocks.");

            using (Stream s = _streamCache.Open(FileAccess.ReadWrite))
            using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
            {
                long lastBlock = LastAllocated(s);
                long last = 0;
                for (long blk = lastBlock; blk > 0; blk -= _blockSize)
                {
                    s.Position = blk & _maskOffset;
                    block.Read(s, FileBlock.HeaderSize);
                    Check.Assert<InvalidDataException>((block.BlockId & _maskOffset) == blk);
                    if ((block.Flags & BlockFlags.BlockDeleted) == BlockFlags.BlockDeleted)
                    {
                        block.NextBlockId = last;
                        last = block.BlockId;
                        s.Position = blk & _maskOffset;
                        block.Write(s, FileBlock.HeaderSize);
                    }
                }
                return last;
            }
        }
        /// <summary> Used for enumeration of the storage blocks in the file. </summary>
        /// <param name="allocatedOnly"> Allows enumeration of all stream, or of just the externally allocated streams </param>
        /// <param name="verifyReads"> Determines if the checksum should be verified while reading the block bytes </param>
        /// <param name="ignoreException"> A method that returns true to ignore the exception and continue processing </param>
        /// <returns>Enumeration of the identity and data stream of each block in the file</returns>
        public IEnumerable<KeyValuePair<long, Stream>> ForeachBlock(bool allocatedOnly, bool verifyReads, Converter<Exception, bool> ignoreException)
        {
            using (Stream s = _streamCache.Open(FileAccess.ReadWrite))
            using (FileBlock block = new FileBlock(_blockSize, _useAlignedIo))
            {
                long lastBlock = LastAllocated(s);
                for (long blk = lastBlock; blk > 0; blk -= _blockSize)
                {
                    s.Position = blk & _maskOffset;
                    block.Read(s, FileBlock.HeaderSize);

                    byte[] bytes;
                    try
                    {
                        if ((block.BlockId & _maskOffset) != blk)
                            throw new InvalidDataException();
                        if (allocatedOnly && (block.Flags & BlockFlags.ExternalBlock) != BlockFlags.ExternalBlock)
                            continue;

                        using (Stream reader = new BlockStreamReader(this, block.BlockId, BlockFlags.None, verifyReads))
                            bytes = IOStream.ReadAllBytes(reader);
                    }
                    catch (Exception error)
                    {
                        if (ignoreException != null && ignoreException(error))
                            continue;
                        throw;
                    }

                    using (Stream ms = new MemoryStream(bytes, false))
                        yield return new KeyValuePair<long, Stream>(block.BlockId, ms);
                }
            }
        }
 public FileBlock Clone()
 {
     FileBlock copy = new FileBlock(BlockSize, _alignedIo);
     Array.Copy(BlockData, _baseOffset, copy.BlockData, copy._baseOffset, BlockSize);
     return copy;
 }
            public BlockStreamReader(FragmentedFile file, long ordinal, BlockFlags typeExpected, bool validated)
            {
                _file = file;
                _blockPos = 0;
                _validated = validated;
                _block = new FileBlock(file._blockSize, file._useAlignedIo);

                _file.ReadBlock(ordinal, _block, file._blockSize, typeExpected, _validated);
                if (_validated)
                {
                    _expectedSum = _block.CheckSum;
                    _checksum = new Crc32();
                    _checksum.Add(_block.BlockData, _block.DataOffset, _block.Length);
                    if (_block.NextBlockId == 0 && _checksum != _expectedSum)
                        throw new InvalidDataException();
                }
            }
 public BlockStreamWriter(FragmentedFile file, FileBlock block)
     : this(file)
 {
     _current = _first = block;
     _current.Flags &= ~BlockFlags.Temporary;
     _restore = null;
     _temp = null;
     _isNew = true;
 }