/// <summary>
/// Updates the <see cref="ReceivingBuffer"/> with the data
/// of a given block, and performs a few simple tests.
/// </summary>
/// <param name="block"></param>
private void ReceiveBlock(BufferedDataBlock block)
{
using (var stream = File.Open(TargetFilePath, FileMode.Append))
{
stream.Write(block.Data, 0, block.Data.Length);
}
}
public void Init()
{
RootDir = FileUtil.CreateTempFolder("_" + GetType().Name);
var provider = new LocalFileSystemProvider(RootDir, true);
//prepare test dir and paths
DownloadService = (LocalFileSystemDownloadService) provider.DownloadTransfers;
SourceFilePath = FileUtil.CreateTempFilePath(RootDir.FullName, "source", "bin");
//create file
SourceFileContents = new byte[5 * 1024 * 1024];
new Random(DateTime.Now.Millisecond).NextBytes(SourceFileContents);
File.WriteAllBytes(SourceFilePath, SourceFileContents);
SourceFile = new FileInfo(SourceFilePath);
//prepare target buffer
ReceivingBuffer = new List<byte>();
LastBlock = null;
//get token
Token = DownloadService.RequestDownloadToken(SourceFilePath, false);
InitInternal();
}
private void WriteBlock(BufferedDataBlock block)
{
Assert.AreEqual(block.BlockLength, block.Data.Length);
Assert.AreEqual(token.TransferId, block.TransferTokenId);
receivedBlocks.Add(block);
target.AddRange(block.Data);
}
/// <summary>
/// Reads a data block's buffer and writes it to a given <paramref name="targetStream"/>.
/// </summary>
/// <param name="dataBlock">The data block that provides a chunk of data that should
/// be written to the <paramref name="targetStream"/>.</param>
/// <param name="targetStream">The target stream that receives the block's
/// <see cref="BufferedDataBlock.Data"/>.</param>
public static void WriteTo(this BufferedDataBlock dataBlock, Stream targetStream)
{
//TODO support variable block length that differs from buffer length:
//targetStream.Write(dataBlock.Data, 0, dataBlock.BlockLength ?? dataBlock.Data.Length);
//-> means we also have to change block validation
targetStream.Position = dataBlock.Offset;
targetStream.Write(dataBlock.Data, 0, dataBlock.Data.Length);
}
private void FlushInternal(bool isAutoFlush)
{
//on autoflush, do not send blocks smaller than the threshold
long minBufferSize = 0;
if (isAutoFlush)
{
//if auto-flushing, flush as long as we can create full-sized blocks,
//OR are above the flushing threshold
minBufferSize = Math.Min(AutoFlushThreshold, Token.MaxBlockSize ?? 0);
//make sure we don't have a negative value
minBufferSize = Math.Max(minBufferSize, 0);
}
lock (syncRoot)
{
if (transmissionBuffer.Count == 0) return;
while (transmissionBuffer.Count > 0 && transmissionBuffer.Count >= minBufferSize)
{
//the blockSize is an int - the buffer can't be bigger anyway
int blockSize = (int)(Math.Min(transmissionBuffer.Count, Token.MaxBlockSize ?? transmissionBuffer.Count));
byte[] data = new byte[blockSize];
transmissionBuffer.CopyTo(0, data, 0, blockSize);
long blockNumber = LastTransmittedBlockNumber.HasValue
? LastTransmittedBlockNumber.Value + 1
: Token.TransmittedBlockCount;
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockNumber = blockNumber,
BlockLength = blockSize,
Data = data,
Offset = InitialOffset + WrittenBytes
};
//write block
OutputAction(block);
//if not exception occurred, update state, remove data from buffer and continue
transmissionBuffer.RemoveRange(0, blockSize);
WrittenBytes += blockSize;
LastTransmittedBlockNumber = blockNumber;
}
}
}
public void WriteBlock(BufferedDataBlock block)
{
lock (this)
{
if (block.BlockNumber != LastProcessedBlockNumber + 1)
{
throw new NotImplementedException(""); //TODO provide implementation
}
WriteDataImpl(block);
if (block.IsLastBlock)
{
FinalizeTransfer();
}
}
}
public void WriteBlock(BufferedDataBlock block)
{
lock(this)
{
if (block.BlockNumber != LastProcessedBlockNumber + 1)
{
throw new NotImplementedException(""); //TODO provide implementation
}
WriteDataImpl(block);
if (block.IsLastBlock)
{
FinalizeTransfer();
}
}
}
public void StartDownload()
{
if (Token == null)
{
Token = TransferService.RequestDownloadToken(ResourceId, false);
}
while (true)
{
IDataBlockInfo blockInfo;
string transferId = Token.TransferId;
long nextBlock = 0;
if (Token.LastTransmittedBlockInfo != null)
{
nextBlock = Token.LastTransmittedBlockInfo.BlockNumber + 1;
}
if (UseStreamingTransfer)
{
StreamedDataBlock block = TransferService.ReadBlockStreamed(transferId, nextBlock);
block.Data.WriteTo(OutputStream);
blockInfo = block;
}
else
{
BufferedDataBlock block = TransferService.ReadBlock(transferId, nextBlock);
OutputStream.Write(block.Data, 0, block.Data.Length);
blockInfo = block;
}
//update the token, get independent file info without data or stream
Token.LastTransmittedBlockInfo = DataBlockInfo.FromDataBlock(blockInfo);
Token.LastBlockTransmissionTime = SystemTime.Now();
if (blockInfo.IsLastBlock)
{
//we're done - update token data
Token.CompletionTime = Token.LastBlockTransmissionTime;
TransferService.CompleteTransfer(transferId);
Token.Status = TransferStatus.Completed;
break;
}
}
}
protected override void InitInternal()
{
SourceFileContents = new byte[12345678];
new Random(DateTime.Now.Millisecond).NextBytes(SourceFileContents);
//create parent folder
ParentDirectory = rootDirectory.CreateSubdirectory("Dwnld_Parent");
//create file
SourceFilePath = FileUtil.CreateTempFilePath(ParentDirectory.FullName, "_dwlndsource", "bin");
File.WriteAllBytes(SourceFilePath, SourceFileContents);
SourceFile = new FileInfo(SourceFilePath);
SourceFileInfo = provider.GetFileInfo(SourceFilePath);
//prepare target buffer
ReceivingBuffer = new List<byte>();
LastBlock = null;
}
protected override void InitInternal()
{
base.InitInternal();
InitToken();
byte[] buffer = new byte[10000];
Array.Copy(SourceFileContents, buffer, 10000);
BufferedBlock = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 10000,
Data = buffer
};
StreamedBlock = new StreamedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 10000,
Data = new ChunkStream(new MemoryStream(SourceFileContents), 10000, 0, false)
};
}
protected override void InitInternal()
{
base.InitInternal();
InitToken();
//read first 10000 bytes
byte[] buffer = GetBuffer(10000);
BufferedBlock = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 10000,
Data = buffer
};
StreamedBlock = new StreamedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 10000,
Data = new ChunkStream(new MemoryStream(buffer), 10000, 0, false)
};
}
public void Init()
{
Input = new List<BufferedDataBlock>();
for(int i=0; i<10; i++)
{
byte[] data = new byte[200000];
new Random().NextBytes(data);
BufferedDataBlock db = new BufferedDataBlock
{
BlockLength = 200000,
BlockNumber = i,
Data = data,
IsLastBlock = i == 9,
Offset = i*200000,
TransferTokenId = "xxx"
};
Input.Add(db);
}
ReceivedData = new List<byte>();
InputStream = new BufferedBlockInputStream((i) => Input[(int)i], 200000*10);
}
public void Writing_Last_Buffered_Block_Should_Complete_Transfer()
{
BufferedBlockOutputStream os = new BufferedBlockOutputStream(Token, 15000, b => UploadHandler.WriteBlock(b));
MemoryStream ms = new MemoryStream(SourceFileContents);
ms.WriteTo(os);
Assert.AreEqual(TransferStatus.Running, UploadHandler.GetTransferStatus(Token.TransferId));
//write an empty last block
BufferedDataBlock db = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 0,
Offset = SourceFile.Length,
Data = new byte[0],
BlockNumber = 0,
IsLastBlock = true
};
UploadHandler.WriteBlock(db);
Assert.AreEqual(TransferStatus.Completed, UploadHandler.GetTransferStatus(Token.TransferId));
FileAssert.AreEqual(SourceFilePath, TargetFilePath);
}
/// <summary>
/// Uploads a given data block that contains a chunk of data for an uploaded file.
/// </summary>
/// <param name="block">The block to be written.</param>
/// <exception cref="DataBlockException">If the data block's contents cannot be stored,
/// either because it's an invalid number, or because only sequential downloads
/// are possible, and the block does not refer to the current download
/// position.
/// </exception>
/// <exception cref="TransferStatusException">If the transfer has already expired.</exception>
public void WriteBlock(BufferedDataBlock block)
{
throw new NotImplementedException();
}
/// <summary>
/// Uploads a given data block that contains a chunk of data for an uploaded file.
/// </summary>
/// <param name="block">The block to be written.</param>
/// <exception cref="DataBlockException">If the data block's contents cannot be stored,
/// either because it's an invalid number, or because only sequential downloads
/// are possible, and the block does not refer to the current download
/// position.
/// </exception>
/// <exception cref="TransferStatusException">If the transfer has already expired.</exception>
void IFSDataUploadService.WriteDataBlock(BufferedDataBlock block)
{
throw new NotImplementedException();
}
public void WriteDataBlock(string transferId, int blockNumber, int blockLength, long offset, byte[] data)
{
Action action = () =>
{
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = transferId,
BlockNumber = blockNumber,
BlockLength = blockLength,
Offset = offset,
Data = data
};
Decorated.UploadTransfers.WriteBlock(block);
};
FaultUtil.SecureAction(FileSystemTask.DataBlockUploadRequest, action);
}
public void Writing_A_Single_Empty_Block_Should_Create_File()
{
UploadHandler.CancelTransfer(Token.TransferId, AbortReason.ClientAbort);
//get token for empty file
Token = UploadHandler.RequestUploadToken(TargetFilePath, false, 0, "");
TargetFile.Refresh();
Assert.IsFalse(TargetFile.Exists);
var block = new BufferedDataBlock { TransferTokenId = Token.TransferId, BlockLength = 0, BlockNumber = 0, Data = new byte[0], IsLastBlock = true};
UploadHandler.WriteBlock(block);
TargetFile.Refresh();
Assert.IsTrue(TargetFile.Exists);
}
/// <summary>
/// Creates a list of data blocks based on the source file, which
/// can be uploaded to the target.
/// </summary>
protected List<BufferedDataBlock> CreateBufferedBlocks()
{
var list = new List<BufferedDataBlock>();
long offset = 0;
int blockCount = 0;
byte[] buffer = new byte[Token.MaxBlockSize ?? 2048];
using (var fs = SourceFile.OpenRead())
{
while (true)
{
//long offset = fs.Position;
var read = fs.Read(buffer, 0, buffer.Length);
//we're done
if (read == 0) break;
BufferedDataBlock block = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = read,
BlockNumber = blockCount++,
Offset = offset,
Data = buffer.CreateCopy(read)
};
offset += read;
list.Add(block);
}
}
var bytesCount = list.Select(b => b.BlockLength).Sum();
Assert.AreEqual(bytesCount.Value, SourceFile.Length);
return list;
}
public void Writing_Last_Buffered_Block_Should_Complete_Transfer()
{
using (var fs = SourceFile.OpenRead())
{
fs.WriteTo(Token, SourceFile.Length, 2048, b =>
{
b.IsLastBlock = false;
Uploads.WriteBlockStreamed(b);
});
}
Assert.AreEqual(TransferStatus.Running, Uploads.GetTransferStatus(Token.TransferId));
//write an empty last block
BufferedDataBlock db = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 0,
Offset = SourceFile.Length,
Data = new byte[0],
BlockNumber = 0,
IsLastBlock = true
};
Uploads.WriteBlock(db);
CompareUploadToSourceFile();
TransferStatus status = Uploads.GetTransferStatus(Token.TransferId);
Assert.IsTrue(status.Is(TransferStatus.Completed, TransferStatus.UnknownTransfer));
}
public void Writing_An_Empty_Block_Should_Be_Supported()
{
//write an empty last block
BufferedDataBlock db = new BufferedDataBlock
{
TransferTokenId = Token.TransferId,
BlockLength = 0,
Offset = 0,
Data = new byte[0],
BlockNumber = 0,
IsLastBlock = false
};
Uploads.WriteBlock(db);
}
protected override void WriteDataImpl(BufferedDataBlock block)
{
DestinationStream.Seek(block.Offset, SeekOrigin.Begin);
DestinationStream.Write(block.Data, 0, block.Data.Length);
DestinationStream.Flush();
}
protected override void WriteDataImpl(BufferedDataBlock block)
{
DestinationStream.Seek(block.Offset, SeekOrigin.Begin);
DestinationStream.Write(block.Data, 0, block.Data.Length);
DestinationStream.Flush();
}
protected override void InitInternal()
{
base.InitInternal();
InitToken();
recentBlock = null;
}
/// <summary> /// /// </summary> /// <param name="block"></param> protected abstract void WriteDataImpl(BufferedDataBlock block);
/// <summary>
/// Updates the <see cref="ReceivingBuffer"/> with the data
/// of a given block, and performs a few simple tests.
/// </summary>
/// <param name="block"></param>
protected virtual void ReceiveBlock(BufferedDataBlock block)
{
Assert.AreEqual(Token.TransferId, block.TransferTokenId);
Assert.AreEqual(ReceivingBuffer.Count, block.Offset);
Assert.AreEqual(block.BlockLength, block.Data.Length);
ReceivingBuffer.AddRange(block.Data);
LastBlock = block;
}
/// <summary> /// /// </summary> /// <param name="block"></param> protected abstract void WriteDataImpl(BufferedDataBlock block);