예제 #1
0
        /// <summary>
        /// Orchestrates common block reading procedures for the <see cref="WriteBlock"/> and
        /// <see cref="WriteBlockStreamed"/> methods.
        /// </summary>
        protected virtual void OrchestrateBlockWriting <T>(T dataBlock, Action <TTransfer, T> writerImplFunc) where T : class, IDataBlock
        {
            Ensure.ArgumentNotNull(dataBlock, "dataBlock");

            const FileSystemTask context  = FileSystemTask.DataBlockUploadRequest;
            TTransfer            transfer = GetCachedTransfer(dataBlock.TransferTokenId, true, context);

            lock (transfer.SyncRoot)
            {
                //make sure the transfer is active
                if (!transfer.Status.Is(TransferStatus.Starting, TransferStatus.Running, TransferStatus.Paused))
                {
                    string msg = String.Format("Transfer [{0}] is not active anymore - status is [{1}].", transfer.TransferId,
                                               transfer.Status);

                    if (transfer.AbortReason.HasValue)
                    {
                        msg += String.Format(" Transfer abort reason: [{0}].", transfer.AbortReason);
                    }

                    AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.UploadNoLongerActive, transfer.FileItem, msg);
                    throw new TransferStatusException(msg)
                          {
                              IsAudited = true, EventId = (int)AuditEvent.UploadNoLongerActive
                          };
                }

                if (!transfer.HasUploadStarted)
                {
                    if (transfer.FileItem.Exists)
                    {
                        //overwrite an existing file
                        DeleteExistingFileImpl(transfer);
                    }

                    InitializeFileUploadImpl(transfer);
                    transfer.HasUploadStarted = true;
                }

                //update status
                transfer.Status = TransferStatus.Running;

                ValidateBlockSizeAndSettings(transfer, dataBlock);

                //write the data to the file system
                writerImplFunc(transfer, dataBlock);

                //store copy of the uploaded block that doesn't contain any data
                transfer.RegisterBlock(DataBlockInfo.FromDataBlock(dataBlock));

                //audit uploaded block
                AuditHelper.AuditResourceOperation(Auditor, context, AuditEvent.FileBlockUploaded, transfer.FileItem);

                //if we just read the last block, close implicitly
                if (dataBlock.IsLastBlock)
                {
                    CompleteTransfer(transfer.TransferId);
                }
            }
        }
예제 #2
0
        public void EmptyTest()
        {
            DataBlockInfo dataBlockInfo = new DataBlockInfo(0, null);

            for (int i = 0; i < 10; i++)
            {
                Console.WriteLine(dataBlockInfo.AddData());
            }
            dataBlockInfo.RemoveData(9);
            dataBlockInfo.RemoveData(7);
            dataBlockInfo.RemoveData(8);
            dataBlockInfo.RemoveData(0);
            dataBlockInfo.RemoveData(5);
            dataBlockInfo.RemoveData(6);
            dataBlockInfo.RemoveData(4);
            dataBlockInfo.RemoveData(1);
            dataBlockInfo.RemoveData(2);
            dataBlockInfo.RemoveData(3);
            //Console.WriteLine(dataBlockInfo.IsEmpty);
            Assert.AreEqual(dataBlockInfo.IsEmpty, true);

            for (int i = 0; i < 15; i++)
            {
                Console.WriteLine(dataBlockInfo.AddData());
            }
        }
예제 #3
0
 /// <summary>
 /// Private helper to request a new data block from the native stream.
 ///  - it releases the current data block back to the native stream code
 ///    (which owns the lifecycle of read buffers),
 ///  - then requests a new buffer from the native stream
 ///  - and updates the internal read buffer pointer (m_curDataBlock), size (m_curDataSize)
 ///    and position, all of which are needed by subsequent Read*() calls coming from the user
 /// </summary>
 private unsafe void GetNextDataBlock()
 {
     this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
     this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
     this.m_curDataBlockInfo            = this.m_nativeStream.ReadDataBlock();
     this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
     this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
     this.m_curBlockPos  = 0;
 }
예제 #4
0
 /// <summary>
 /// Writes out the current data buffer (equivalent of FlushDataBlock), and calls
 /// Flush on the native stream to ensure all the data makes its way to the disk
 /// </summary>
 internal void Flush()
 {
     if (this.m_curRecordEnd > 0)
     {
         this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordEnd);
         this.m_numBytesWritten += this.m_curRecordEnd;
         this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
         this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
         this.m_curDataBlockInfo            = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
         this.m_curDataBlock   = this.m_curDataBlockInfo.dataBlock;
         this.m_curBlockSize   = this.m_curDataBlockInfo.blockSize;
         this.m_curRecordStart = 0;
         this.m_curRecordEnd   = 0;
     }
     this.m_nativeStream.Flush();
 }
예제 #5
0
 public EntityBlock(int id, EntityType entityType)
 {
     this.entityType = entityType;
     this.id         = id;
     int[] datas = new int[DataManager.Count];
     for (int i = 0; i < DataManager.Count; i++)
     {
         if (entityType.DataInfo[i])
         {
             datas[i] = DataManager.DataBlockManagers[i].AddBlock();
         }
         else
         {
             datas[i] = -1;
         }
     }
     dataBlockInfo = new DataBlockInfo(id, datas);
 }
예제 #6
0
        public void Refreshing_Token_After_Multiple_Read_Should_Still_Indicate_The_Last_Block_Number()
        {
            for (int i = 0; i < 10; i++)
            {
                DownloadService.ReadBlock(Token.TransferId, i);
            }

            DataBlockInfo lastBlock = null;

            //redownload a few of these blocks
            for (int i = 0; i < 10; i++)
            {
                if (i % 2 == 0)
                {
                    lastBlock = DownloadService.ReadBlock(Token.TransferId, i);
                }
            }

            DownloadToken copy = DownloadService.RenewToken(Token, false);

            Assert.AreEqual(lastBlock.BlockNumber, copy.LastTransmittedBlockInfo.BlockNumber);
        }
예제 #7
0
        public void BlockTest()
        {
            DataBlockInfo dataBlockInfo = new DataBlockInfo(0, null);

            for (int i = 0; i < 70; i++)
            {
                Console.WriteLine(dataBlockInfo.AddData());
            }
            dataBlockInfo.RemoveData(10);
            dataBlockInfo.RemoveData(7);
            dataBlockInfo.RemoveData(20);
            dataBlockInfo.RemoveData(50);
            dataBlockInfo.RemoveData(50);
            dataBlockInfo.RemoveData(-1);
            dataBlockInfo.RemoveData(70);
            dataBlockInfo.RemoveData(6);
            dataBlockInfo.RemoveData(6);
            dataBlockInfo.RemoveData(-1);
            dataBlockInfo.RemoveData(70);
            Assert.AreEqual(false, dataBlockInfo[10]);
            Assert.AreEqual(false, dataBlockInfo[7]);
            Assert.AreEqual(false, dataBlockInfo[20]);
            Assert.AreEqual(false, dataBlockInfo[50]);
            Assert.AreEqual(false, dataBlockInfo[6]);
            Assert.AreEqual(false, dataBlockInfo[-1]);
            Assert.AreEqual(false, dataBlockInfo[70]);
            Assert.AreEqual(true, dataBlockInfo[8]);
            Assert.AreEqual(true, dataBlockInfo[14]);
            Assert.AreEqual(true, dataBlockInfo[35]);
            Assert.AreEqual(true, dataBlockInfo[49]);
            Assert.AreEqual(10, dataBlockInfo.AddData());
            Assert.AreEqual(7, dataBlockInfo.AddData());
            Assert.AreEqual(20, dataBlockInfo.AddData());
            Assert.AreEqual(50, dataBlockInfo.AddData());
            Assert.AreEqual(6, dataBlockInfo.AddData());
            Assert.AreEqual(-1, dataBlockInfo.AddData());
            Assert.AreEqual(-1, dataBlockInfo.AddData());
        }
예제 #8
0
        /// <summary>
        /// Private helper to write the current block out to the native stream.
        ///  - it writes out the current data buffer up to the point it was filled
        ///  - it releases the current data block back to the native stream code (which owns the lifecycle of read buffers),
        ///  - then allocated a new buffer from the native stream
        ///  - and updates the internal read buffer pointer and position members
        /// </summary>
        private void FlushDataBlock()
        {
            DataBlockInfo newDataBlockInfo;

            if (this.m_curRecordStart <= 16)
            {
                // The current block is too small for a single record, augment it
                if (this.m_curBlockSize == this.m_nextBlockSize)
                {
                    throw new DryadLinqException(HpcLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
                }
                newDataBlockInfo     = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
                this.m_nextBlockSize = this.m_nextBlockSize * 2;
                if (this.m_nextBlockSize < 0)
                {
                    this.m_nextBlockSize = 0x7FFFFFF8;
                }
                HpcLinqUtil.memcpy(this.m_curDataBlock, newDataBlockInfo.dataBlock, this.m_curRecordEnd);
            }
            else
            {
                // Write all the complete records in the block, put the partial record in the new block
                newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
                HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curRecordStart,
                                   newDataBlockInfo.dataBlock,
                                   this.m_curRecordEnd - this.m_curRecordStart);
                this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordStart);
                this.m_numBytesWritten += this.m_curRecordStart;
                this.m_curRecordEnd    -= this.m_curRecordStart;
                this.m_curRecordStart   = 0;
            }
            this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
            this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
            this.m_curDataBlockInfo            = newDataBlockInfo;
            this.m_curDataBlock = newDataBlockInfo.dataBlock;
            this.m_curBlockSize = newDataBlockInfo.blockSize;
        }
예제 #9
0
        /// <summary>
        /// Handles the creation of an actual data block based on the underlying resource.
        /// </summary>
        /// <param name="transfer">The processed transfer.</param>
        /// <param name="blockNumber">The number of the downloaded block.</param>
        /// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
        /// contains the information about the block. Can be used in order to ensure proper retransmission in case
        /// of variable block sizes.</param>
        /// <returns>A data block which exposes the data as a resource-friendly stream
        /// (<see cref="BufferedDataBlock.Data"/>).</returns>
        protected override StreamedDataBlock CreateStreamedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
        {
            //this func creates the returned DataBlock by reading a chunk of data
            //from the underlying stream.
            Func <Stream, long, StreamedDataBlock> func = (stream, position) =>
            {
                DownloadToken token = transfer.Token;

                //check if we can use the max block size
                long streamLength = stream.Length;
                int  blockLength  = (int)Math.Min(token.DownloadBlockSize,
                                                  streamLength - position);
                if (blockLength < 0)
                {
                    blockLength = 0;
                }

                var chunkStream = new ChunkStream(stream, blockLength, position, false);


                return(new StreamedDataBlock
                {
                    TransferTokenId = transfer.TransferId,
                    BlockNumber = blockNumber,
                    BlockLength = blockLength,
                    Offset = position,
                    Data = chunkStream,
                    IsLastBlock = blockNumber == token.TotalBlockCount - 1
                });
            };

            return(PrepareAndRunBlockReading(transfer, blockNumber, func));
        }
예제 #10
0
        /// <summary>
        /// Handles the creation of an actual data block based on the underlying resource.
        /// </summary>
        /// <param name="transfer">The processed transfer.</param>
        /// <param name="blockNumber">The number of the downloaded block.</param>
        /// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
        /// contains the information about the block. Can be used in order to ensure proper retransmission in case
        /// of variable block sizes.</param>
        /// <returns>A data block which contains the data as an in-memory buffer
        /// (<see cref="BufferedDataBlock.Data"/>).</returns>
        protected override BufferedDataBlock CreateBufferedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
        {
            //this func creates the returned DataBlock by reading a chunk of data
            //from the underlying stream.
            Func <Stream, long, BufferedDataBlock> func = (stream, position) =>
            {
                DownloadToken token;
                byte[]        data;
                try
                {
                    token = transfer.Token;

                    //read data
                    data = new byte[token.DownloadBlockSize];
                    int read = stream.Read(data, 0, data.Length);
                    if (read < token.DownloadBlockSize)
                    {
                        Array.Resize(ref data, read);
                    }
                }
                finally
                {
                    //TODO should change once simplistic implementation was replaced
                    stream.Close();
                }


                return(new BufferedDataBlock
                {
                    TransferTokenId = transfer.TransferId,
                    BlockNumber = blockNumber,
                    BlockLength = data.Length,
                    Offset = position,
                    Data = data,
                    IsLastBlock = blockNumber == token.TotalBlockCount - 1
                });
            };

            BufferedDataBlock dataBlock = PrepareAndRunBlockReading(transfer, blockNumber, func);

            return(dataBlock);
        }
예제 #11
0
        /// <summary>
        /// Handles the creation of an actual data block based on the underlying resource.
        /// </summary>
        /// <param name="transfer">The processed transfer.</param>
        /// <param name="blockNumber">The number of the downloaded block.</param>
        /// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
        /// contains the information about the block. Can be used in order to ensure proper retransmission in case
        /// of variable block sizes.</param>
        /// <returns>A data block which contains the data as an in-memory buffer
        /// (<see cref="BufferedDataBlock.Data"/>).</returns>
        protected override BufferedDataBlock CreateBufferedDataBlockImpl(LocalDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
        {
            //this func creates the returned DataBlock by reading a chunk of data
            //from the underlying stream.
            Func <long, BufferedDataBlock> func = position =>
            {
                DownloadToken token = transfer.Token;

                //read data
                byte[] data = new byte[token.DownloadBlockSize];
                int    read = transfer.Stream.Read(data, 0, data.Length);
                if (read < token.DownloadBlockSize)
                {
                    Array.Resize(ref data, read);
                }

                return(new BufferedDataBlock
                {
                    TransferTokenId = transfer.TransferId,
                    BlockNumber = blockNumber,
                    BlockLength = data.Length,
                    Offset = position,
                    Data = data,
                    IsLastBlock = blockNumber == token.TotalBlockCount - 1
                });
            };

            var dataBlock = PrepareAndRunBlockReading(transfer, blockNumber, func);

            if (dataBlock.IsLastBlock)
            {
                //assume the last block will be processed successfully and
                //already close the stream (don't wait for the transfer to be
                //closed by client). This would still reopen the stream if
                //necessary
                CloseStream(transfer);
            }

            return(dataBlock);
        }