public void Writing_Last_Buffered_Block_Should_Complete_Transfer()
{
using (var fs = SourceFile.OpenRead())
{
fs.WriteTo(Token, SourceFile.Length, 2048, b =>
{
b.IsLastBlock = false;
Uploads.WriteBlockStreamed(b);
});
}
Assert.AreEqual(TransferStatus.Running, Uploads.GetTransferStatus(Token.TransferId));
//write an empty last block
BufferedDataBlock db = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 0,
Offset = SourceFile.Length,
Data = new byte[0],
BlockNumber = 0,
IsLastBlock = true
};
Uploads.WriteBlock(db);
CompareUploadToSourceFile();
TransferStatus status = Uploads.GetTransferStatus(Token.TransferId);
Assert.IsTrue(status.Is(TransferStatus.Completed, TransferStatus.UnknownTransfer));
}
/// <summary>
/// Handles the actual writing of the submitted data to the file system.
/// </summary>
/// <param name="transfer">The processed transfer.</param>
/// <param name="dataBlock">A data block that contains a chunk of data
/// which should be written to the file system.</param>
protected override void WriteBufferedDataBlockImpl(TTransfer transfer, BufferedDataBlock dataBlock)
{
TempStream stream = GetCachedTempData(transfer, dataBlock.Offset);
byte[] data = dataBlock.Data;
stream.Write(data, 0, data.Length);
}
public void Writing_Last_Buffered_Block_Should_Complete_Transfer()
{
BufferedBlockOutputStream os = new BufferedBlockOutputStream(Token, 15000, b => UploadHandler.WriteBlock(b));
MemoryStream ms = new MemoryStream(SourceFileContents);
ms.WriteTo(os);
Assert.AreEqual(TransferStatus.Running, UploadHandler.GetTransferStatus(Token.TransferId));
//write an empty last block
BufferedDataBlock db = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 0,
Offset = SourceFile.Length,
Data = new byte[0],
BlockNumber = 0,
IsLastBlock = true
};
UploadHandler.WriteBlock(db);
Assert.AreEqual(TransferStatus.Completed, UploadHandler.GetTransferStatus(Token.TransferId));
FileAssert.AreEqual(SourceFilePath, TargetFilePath);
}
public void Init()
{
RootDir = FileUtil.CreateTempFolder("_" + GetType().Name);
var provider = new LocalFileSystemProvider(RootDir, true);
//prepare test dir and paths
DownloadService = (LocalFileSystemDownloadService)provider.DownloadTransfers;
SourceFilePath = FileUtil.CreateTempFilePath(RootDir.FullName, "source", "bin");
//create file
SourceFileContents = new byte[5 * 1024 * 1024];
new Random(DateTime.Now.Millisecond).NextBytes(SourceFileContents);
File.WriteAllBytes(SourceFilePath, SourceFileContents);
SourceFile = new FileInfo(SourceFilePath);
//prepare target buffer
ReceivingBuffer = new List <byte>();
LastBlock = null;
//get token
Token = DownloadService.RequestDownloadToken(SourceFilePath, false);
InitInternal();
}
/// <summary>
/// Uploads a given data block that contains a chunk of data for an uploaded file.
/// </summary>
/// <param name="block">The block to be written.</param>
/// <exception cref="DataBlockException">If the data block's contents cannot be stored,
/// either because it's an invalid number, or because only sequential downloads
/// are possible, and the block does not refer to the current download
/// position.
/// </exception>
/// <exception cref="TransferStatusException">If the transfer has already expired.</exception>
public void WriteBlock(BufferedDataBlock block)
{
Ensure.ArgumentNotNull(block, "block");
//switch to streaming
MemoryStream stream = new MemoryStream(block.Data);
if (block.BlockLength != block.Data.Length)
{
string msg = "Block number {0} of transfer [{1}] has a buffer of size [{2}] that doesn't match the indicated block size of [{3}].";
msg = String.Format(msg, block.BlockNumber, block.TransferTokenId, block.Data.Length, block.BlockLength);
throw new DataBlockException(msg);
}
var streamingBlock = new StreamedDataBlock
{
TransferTokenId = block.TransferTokenId,
BlockNumber = block.BlockNumber,
BlockLength = block.BlockLength,
Offset = block.Offset,
IsLastBlock = block.IsLastBlock,
Data = stream
};
WriteBlockStreamed(streamingBlock);
}
public void If_File_Size_Is_Not_Multitude_Of_Block_Size_Then_Last_Block_Should_Be_Smaller_Then_Others()
{
//make file of 10 blocks
var blockSize = (int)Token.DownloadBlockSize;
SourceFileContents = new byte[blockSize * 10 + blockSize / 2]; //add half a block
File.WriteAllBytes(SourceFilePath, SourceFileContents);
Token = DownloadHandler.RequestDownloadToken(SourceFilePath, false);
List <BufferedDataBlock> blockList = new List <BufferedDataBlock>();
int blockNumber = 0;
while (true)
{
BufferedDataBlock block = DownloadHandler.ReadBlock(Token.TransferId, blockNumber++);
blockList.Add(block);
if (block.IsLastBlock)
{
break;
}
}
Assert.AreEqual(11, blockList.Count);
Assert.AreEqual(blockSize / 2, blockList[10].BlockLength);
}
/// <summary>
/// Updates the <see cref="ReceivingBuffer"/> with the data
/// of a given block, and performs a few simple tests.
/// </summary>
/// <param name="block"></param>
private void ReceiveBlock(BufferedDataBlock block)
{
using (var stream = File.Open(TargetFilePath, FileMode.Append))
{
stream.Write(block.Data, 0, block.Data.Length);
}
}
public void If_File_Size_Is_Not_Multitude_Of_Block_Size_Then_Last_Block_Should_Be_Smaller_Then_Others()
{
var maxSize = Downloads.MaxBlockSize ?? 2048;
//upload a file that consists of a total of 5.5 (6) blocks
File.WriteAllBytes(Context.DownloadFile0Template.FullName, new byte[maxSize * 5 + maxSize / 2]);
Context.DownloadFile0Template.Refresh();
SourceFile = Context.DownloadFolder.AddFile(Context.DownloadFile0Template.FullName, "multiple.bin", false);
Downloads.CancelTransfer(Token.TransferId, AbortReason.ClientAbort);
Token = Downloads.RequestDownloadToken(SourceFileInfo.FullName, false, maxSize);
Assert.AreEqual(maxSize, Token.DownloadBlockSize);
Assert.AreEqual(6, Token.TotalBlockCount);
List <BufferedDataBlock> blockList = new List <BufferedDataBlock>();
int blockNumber = 0;
while (true)
{
BufferedDataBlock block = Downloads.ReadBlock(Token.TransferId, blockNumber++);
blockList.Add(block);
if (block.IsLastBlock)
{
break;
}
}
Assert.AreEqual(6, blockList.Count);
Assert.AreEqual(maxSize / 2, blockList[5].BlockLength);
}
/// <summary>
/// Uploads a given data block that contains a chunk of data for an uploaded file.
/// </summary>
/// <param name="block">The block to be written.</param>
/// <exception cref="DataBlockException">If the data block's contents cannot be stored,
/// either because it's an invalid number, or because only sequential downloads
/// are possible, and the block does not refer to the current download
/// position.
/// </exception>
/// <exception cref="TransferStatusException">If the transfer has already expired.</exception>
public virtual void WriteBlock(BufferedDataBlock block)
{
SecureAction(FileSystemTask.DataBlockUploadRequest,
() => OrchestrateBlockWriting(block, WriteBufferedDataBlockImpl),
() =>
String.Format("Could not write buffered data block [{0}] for transfer [{1}]", block.BlockNumber,
block.TransferTokenId));
}
private void WriteBlock(BufferedDataBlock block)
{
Assert.AreEqual(block.BlockLength, block.Data.Length);
Assert.AreEqual(token.TransferId, block.TransferTokenId);
receivedBlocks.Add(block);
target.AddRange(block.Data);
}
/// <summary>
/// Updates the <see cref="ReceivingBuffer"/> with the data
/// of a given block, and performs a few simple tests.
/// </summary>
/// <param name="block"></param>
protected virtual void ReceiveBlock(BufferedDataBlock block)
{
Assert.AreEqual(Token.TransferId, block.TransferTokenId);
Assert.AreEqual(ReceivingBuffer.Count, block.Offset);
Assert.AreEqual(block.BlockLength, block.Data.Length);
ReceivingBuffer.AddRange(block.Data);
LastBlock = block;
}
public void Requesting_Token_With_Block_Size_Above_Default_Should_Return_Bigger_Blocks()
{
var blockSize = FileSystemConfiguration.DefaultDownloadBlockSize + 50;
Token = DownloadHandler.RequestDownloadToken(SourceFileInfo.FullName, false, blockSize);
Assert.AreEqual(blockSize, Token.DownloadBlockSize);
BufferedDataBlock block = DownloadHandler.ReadBlock(Token.TransferId, 0);
Assert.AreEqual(blockSize, block.BlockLength);
Assert.AreEqual(blockSize, block.Data.Length);
}
public void Requesting_Token_With_Block_Size_Below_Default_Should_Return_Small_Blocks()
{
var blockSize = 20480;
Token = DownloadHandler.RequestDownloadToken(SourceFileInfo.FullName, false, blockSize);
Assert.AreEqual(blockSize, Token.DownloadBlockSize);
BufferedDataBlock block = DownloadHandler.ReadBlock(Token.TransferId, 0);
Assert.AreEqual(blockSize, block.BlockLength);
Assert.AreEqual(blockSize, block.Data.Length);
}
private void FlushInternal(bool isAutoFlush)
{
//on autoflush, do not send blocks smaller than the threshold
long minBufferSize = 0;
if (isAutoFlush)
{
//if auto-flushing, flush as long as we can create full-sized blocks,
//OR are above the flushing threshold
minBufferSize = Math.Min(AutoFlushThreshold, Token.MaxBlockSize ?? 0);
//make sure we don't have a negative value
minBufferSize = Math.Max(minBufferSize, 0);
}
lock (syncRoot)
{
if (transmissionBuffer.Count == 0)
{
return;
}
while (transmissionBuffer.Count > 0 && transmissionBuffer.Count >= minBufferSize)
{
//the blockSize is an int - the buffer can't be bigger anyway
int blockSize = (int)(Math.Min(transmissionBuffer.Count, Token.MaxBlockSize ?? transmissionBuffer.Count));
byte[] data = new byte[blockSize];
transmissionBuffer.CopyTo(0, data, 0, blockSize);
long blockNumber = LastTransmittedBlockNumber.HasValue
? LastTransmittedBlockNumber.Value + 1
: 0;
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockNumber = blockNumber,
BlockLength = blockSize,
Data = data,
Offset = InitialOffset + WrittenBytes
};
//write block
OutputAction(block);
//if not exception occurred, update state, remove data from buffer and continue
transmissionBuffer.RemoveRange(0, blockSize);
WrittenBytes += blockSize;
LastTransmittedBlockNumber = blockNumber;
}
}
}
public void Token_And_Blocks_Should_Match_Default_Block_Size()
{
var blockSize = FileSystemConfiguration.DefaultDownloadBlockSize;
Token = DownloadHandler.RequestDownloadToken(SourceFileInfo.FullName, false);
Assert.AreEqual(blockSize, Token.DownloadBlockSize);
BufferedDataBlock block = DownloadHandler.ReadBlock(Token.TransferId, 5);
Assert.AreEqual(blockSize, block.BlockLength);
Assert.AreEqual(blockSize, block.Data.Length);
}
/// <summary>
/// Parses the received buffer and wraps it into a memory stream.
/// </summary>
/// <param name="bufferedBlock">A buffered block.</param>
private void ParseBuffer(BufferedDataBlock bufferedBlock)
{
StreamedDataBlock streamedBlock = new StreamedDataBlock
{
TransferTokenId = bufferedBlock.TransferTokenId,
BlockLength = bufferedBlock.BlockLength,
BlockNumber = bufferedBlock.BlockNumber,
IsLastBlock = bufferedBlock.IsLastBlock,
Offset = bufferedBlock.Offset,
Data = new MemoryStream(bufferedBlock.Data)
};
OutputAction(streamedBlock);
}
public void Requesting_Token_With_Block_Size_Above_Max_Size_Should_Fall_Back_To_Maxium()
{
var blockSize = FileSystemConfiguration.MaxDownloadBlockSize.Value + 10000;
var expectedSize = FileSystemConfiguration.MaxDownloadBlockSize.Value;
Token = DownloadHandler.RequestDownloadToken(SourceFileInfo.FullName, false, blockSize);
Assert.AreEqual(expectedSize, Token.DownloadBlockSize);
BufferedDataBlock block = DownloadHandler.ReadBlock(Token.TransferId, 0);
Assert.AreEqual(expectedSize, block.BlockLength);
Assert.AreEqual(expectedSize, block.Data.Length);
}
public void Read_Blocks_Should_Be_Reflected_In_Block_Table()
{
BufferedDataBlock block1 = DownloadHandler.ReadBlock(Token.TransferId, 5);
BufferedDataBlock block2 = DownloadHandler.ReadBlock(Token.TransferId, 7);
IEnumerable <DataBlockInfo> transferTable = DownloadHandler.GetTransferredBlocks(Token.TransferId);
Assert.AreEqual(2, transferTable.Count());
Assert.AreEqual(block1.BlockNumber, transferTable.First().BlockNumber);
Assert.AreEqual(block1.Offset, transferTable.First().Offset);
Assert.AreEqual(block2.BlockNumber, transferTable.Last().BlockNumber);
Assert.AreEqual(block2.Offset, transferTable.Last().Offset);
}
public void Number_Of_Available_Blocks_Should_Match_Token()
{
long blockNumber = 0;
//request data using block numbers according to token info
for (int i = 0; i < Token.TotalBlockCount; i++)
{
BufferedDataBlock block = Downloads.ReadBlock(Token.TransferId, blockNumber++);
ReceiveBlock(block);
}
//make sure we got all the data
FileAssert.AreEqual(Context.DownloadFile0Template.FullName, TargetFilePath);
}
/// <summary>
/// Parses the received buffer and wraps it into a memory stream.
/// </summary>
/// <param name="bufferedBlock">A buffered block.</param>
private void ParseBuffer(BufferedDataBlock bufferedBlock)
{
StreamedDataBlock streamedBlock = new StreamedDataBlock
{
TransferTokenId = bufferedBlock.TransferTokenId,
BlockLength = bufferedBlock.BlockLength,
BlockNumber = bufferedBlock.BlockNumber,
IsLastBlock = bufferedBlock.IsLastBlock,
Offset = bufferedBlock.Offset,
Data = new MemoryStream(bufferedBlock.Data)
};
OutputAction(streamedBlock);
}
public void Number_Of_Available_Blocks_Should_Match_Token()
{
long blockNumber = 0;
for (int i = 0; i < Token.TotalBlockCount; i++)
{
BufferedDataBlock block = DownloadHandler.ReadBlock(Token.TransferId, blockNumber++);
//the receive method checks offsets and block numbers
base.ReceiveBlock(block);
}
//make sure we got all the data
CollectionAssert.AreEqual(SourceFileContents, ReceivingBuffer);
}
public void Writing_An_Empty_Block_Should_Be_Supported()
{
//write an empty last block
BufferedDataBlock db = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 0,
Offset = 0,
Data = new byte[0],
BlockNumber = 0,
IsLastBlock = false
};
Uploads.WriteBlock(db);
}
public void Requesting_Block_Size_With_Specified_Value_Should_Be_Allowed_If_Service_Does_Not_Have_Limit()
{
if (Downloads.MaxBlockSize.HasValue)
{
Assert.Inconclusive("Download service has a maximum block size - did not run test.");
}
var blockSize = 2048;
Token = Downloads.RequestDownloadToken(SourceFileInfo.FullName, false, blockSize);
Assert.AreEqual(blockSize, Token.DownloadBlockSize);
BufferedDataBlock block = Downloads.ReadBlock(Token.TransferId, 0);
Assert.AreEqual(blockSize, block.BlockLength);
Assert.AreEqual(blockSize, block.Data.Length);
}
public void Requesting_Token_With_Block_Size_Below_Maximum_Should_Return_Small_Blocks()
{
if (!Downloads.MaxBlockSize.HasValue)
{
Assert.Inconclusive("Download service does not provide maximum block size");
}
var blockSize = Downloads.MaxBlockSize.Value / 2;
Token = Downloads.RequestDownloadToken(SourceFileInfo.FullName, false, blockSize);
Assert.AreEqual(blockSize, Token.DownloadBlockSize);
BufferedDataBlock block = Downloads.ReadBlock(Token.TransferId, 0);
Assert.AreEqual(blockSize, block.BlockLength);
Assert.AreEqual(blockSize, block.Data.Length);
}
public void WriteDataBlock(string transferId, int blockNumber, int blockLength, long offset, byte[] data)
{
Action action = () =>
{
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = transferId,
BlockNumber = blockNumber,
BlockLength = blockLength,
Offset = offset,
Data = data
};
Decorated.UploadTransfers.WriteBlock(block);
};
FaultUtil.SecureAction(FileSystemTask.DataBlockUploadRequest, action);
}
public void Requesting_Token_With_Block_Size_Above_Max_Size_Should_Fall_Back_To_Maxium()
{
if (!Downloads.MaxBlockSize.HasValue)
{
Assert.Inconclusive("Download service does not provide maximum block size");
}
var expectedSize = Downloads.MaxBlockSize.Value;
var blockSize = expectedSize + 10000;
Token = Downloads.RequestDownloadToken(SourceFileInfo.FullName, false, blockSize);
Assert.AreEqual(expectedSize, Token.DownloadBlockSize);
BufferedDataBlock block = Downloads.ReadBlock(Token.TransferId, 0);
Assert.AreEqual(expectedSize, block.BlockLength);
Assert.AreEqual(expectedSize, block.Data.Length);
}
public void Writing_A_Single_Empty_Block_Should_Create_File()
{
UploadHandler.CancelTransfer(Token.TransferId, AbortReason.ClientAbort);
//get token for empty file
Token = UploadHandler.RequestUploadToken(TargetFilePath, false, 0, "");
TargetFile.Refresh();
Assert.IsFalse(TargetFile.Exists);
var block = new BufferedDataBlock {
TransferTokenId = Token.TransferId, BlockLength = 0, BlockNumber = 0, Data = new byte[0], IsLastBlock = true
};
UploadHandler.WriteBlock(block);
TargetFile.Refresh();
Assert.IsTrue(TargetFile.Exists);
}
protected override void InitInternal()
{
SourceFileContents = new byte[12345678];
new Random(DateTime.Now.Millisecond).NextBytes(SourceFileContents);
//create parent folder
ParentDirectory = rootDirectory.CreateSubdirectory("Dwnld_Parent");
//create file
SourceFilePath = FileUtil.CreateTempFilePath(ParentDirectory.FullName, "_dwlndsource", "bin");
File.WriteAllBytes(SourceFilePath, SourceFileContents);
SourceFile = new FileInfo(SourceFilePath);
SourceFileInfo = provider.GetFileInfo(SourceFilePath);
//prepare target buffer
ReceivingBuffer = new List <byte>();
LastBlock = null;
}
public void Reading_Blocks_Should_Deliver_All_Data()
{
long blockNumber = 0;
//read blocks until last block is indicated
while (true)
{
BufferedDataBlock block = Downloads.ReadBlock(Token.TransferId, blockNumber++);
ReceiveBlock(block);
if (block.IsLastBlock)
{
break;
}
}
//make sure we got all the data
FileAssert.AreEqual(Context.DownloadFile0Template.FullName, TargetFilePath);
}
/// <summary>
/// Creates a list of data blocks based on the source file, which
/// can be uploaded to the target.
/// </summary>
protected List <BufferedDataBlock> CreateBufferedBlocks()
{
var list = new List <BufferedDataBlock>();
long offset = 0;
int blockCount = 0;
byte[] buffer = new byte[Token.MaxBlockSize ?? 2048];
using (var fs = SourceFile.OpenRead())
{
while (true)
{
//long offset = fs.Position;
var read = fs.Read(buffer, 0, buffer.Length);
//we're done
if (read == 0)
{
break;
}
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = read,
BlockNumber = blockCount++,
Offset = offset,
Data = buffer.CreateCopy(read)
};
offset += read;
list.Add(block);
}
}
var bytesCount = list.Select(b => b.BlockLength).Sum();
Assert.AreEqual(bytesCount.Value, SourceFile.Length);
return(list);
}
/// <summary>
/// Handles the creation of an actual data block based on the underlying resource.
/// </summary>
/// <param name="transfer">The processed transfer.</param>
/// <param name="blockNumber">The number of the downloaded block.</param>
/// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
/// contains the information about the block. Can be used in order to ensure proper retransmission in case
/// of variable block sizes.</param>
/// <returns>A data block which contains the data as an in-memory buffer
/// (<see cref="BufferedDataBlock.Data"/>).</returns>
protected override BufferedDataBlock CreateBufferedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
{
//this func creates the returned DataBlock by reading a chunk of data
//from the underlying stream.
Func <Stream, long, BufferedDataBlock> func = (stream, position) =>
{
DownloadToken token;
byte[] data;
try
{
token = transfer.Token;
//read data
data = new byte[token.DownloadBlockSize];
int read = stream.Read(data, 0, data.Length);
if (read < token.DownloadBlockSize)
{
Array.Resize(ref data, read);
}
}
finally
{
//TODO should change once simplistic implementation was replaced
stream.Close();
}
return(new BufferedDataBlock
{
TransferTokenId = transfer.TransferId,
BlockNumber = blockNumber,
BlockLength = data.Length,
Offset = position,
Data = data,
IsLastBlock = blockNumber == token.TotalBlockCount - 1
});
};
BufferedDataBlock dataBlock = PrepareAndRunBlockReading(transfer, blockNumber, func);
return(dataBlock);
}
private void FlushInternal(bool isAutoFlush)
{
//on autoflush, do not send blocks smaller than the threshold
long minBufferSize = 0;
if (isAutoFlush)
{
//if auto-flushing, flush as long as we can create full-sized blocks,
//OR are above the flushing threshold
minBufferSize = Math.Min(AutoFlushThreshold, Token.MaxBlockSize ?? 0);
//make sure we don't have a negative value
minBufferSize = Math.Max(minBufferSize, 0);
}
lock (syncRoot)
{
if (transmissionBuffer.Count == 0) return;
while (transmissionBuffer.Count > 0 && transmissionBuffer.Count >= minBufferSize)
{
//the blockSize is an int - the buffer can't be bigger anyway
int blockSize = (int)(Math.Min(transmissionBuffer.Count, Token.MaxBlockSize ?? transmissionBuffer.Count));
byte[] data = new byte[blockSize];
transmissionBuffer.CopyTo(0, data, 0, blockSize);
long blockNumber = LastTransmittedBlockNumber.HasValue
? LastTransmittedBlockNumber.Value + 1
: 0;
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockNumber = blockNumber,
BlockLength = blockSize,
Data = data,
Offset = InitialOffset + WrittenBytes
};
//write block
OutputAction(block);
//if not exception occurred, update state, remove data from buffer and continue
transmissionBuffer.RemoveRange(0, blockSize);
WrittenBytes += blockSize;
LastTransmittedBlockNumber = blockNumber;
}
}
}