/// <summary>
/// Orchestrates common block reading procedures for the <see cref="WriteBlock"/> and
/// <see cref="WriteBlockStreamed"/> methods.
/// </summary>
protected virtual void OrchestrateBlockWriting <T>(T dataBlock, Action <TTransfer, T> writerImplFunc) where T : class, IDataBlock
{
Ensure.ArgumentNotNull(dataBlock, "dataBlock");
const FileSystemTask context = FileSystemTask.DataBlockUploadRequest;
TTransfer transfer = GetCachedTransfer(dataBlock.TransferTokenId, true, context);
lock (transfer.SyncRoot)
{
//make sure the transfer is active
if (!transfer.Status.Is(TransferStatus.Starting, TransferStatus.Running, TransferStatus.Paused))
{
string msg = String.Format("Transfer [{0}] is not active anymore - status is [{1}].", transfer.TransferId,
transfer.Status);
if (transfer.AbortReason.HasValue)
{
msg += String.Format(" Transfer abort reason: [{0}].", transfer.AbortReason);
}
AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.UploadNoLongerActive, transfer.FileItem, msg);
throw new TransferStatusException(msg)
{
IsAudited = true, EventId = (int)AuditEvent.UploadNoLongerActive
};
}
if (!transfer.HasUploadStarted)
{
if (transfer.FileItem.Exists)
{
//overwrite an existing file
DeleteExistingFileImpl(transfer);
}
InitializeFileUploadImpl(transfer);
transfer.HasUploadStarted = true;
}
//update status
transfer.Status = TransferStatus.Running;
ValidateBlockSizeAndSettings(transfer, dataBlock);
//write the data to the file system
writerImplFunc(transfer, dataBlock);
//store copy of the uploaded block that doesn't contain any data
transfer.RegisterBlock(DataBlockInfo.FromDataBlock(dataBlock));
//audit uploaded block
AuditHelper.AuditResourceOperation(Auditor, context, AuditEvent.FileBlockUploaded, transfer.FileItem);
//if we just read the last block, close implicitly
if (dataBlock.IsLastBlock)
{
CompleteTransfer(transfer.TransferId);
}
}
}
public void EmptyTest()
{
DataBlockInfo dataBlockInfo = new DataBlockInfo(0, null);
for (int i = 0; i < 10; i++)
{
Console.WriteLine(dataBlockInfo.AddData());
}
dataBlockInfo.RemoveData(9);
dataBlockInfo.RemoveData(7);
dataBlockInfo.RemoveData(8);
dataBlockInfo.RemoveData(0);
dataBlockInfo.RemoveData(5);
dataBlockInfo.RemoveData(6);
dataBlockInfo.RemoveData(4);
dataBlockInfo.RemoveData(1);
dataBlockInfo.RemoveData(2);
dataBlockInfo.RemoveData(3);
//Console.WriteLine(dataBlockInfo.IsEmpty);
Assert.AreEqual(dataBlockInfo.IsEmpty, true);
for (int i = 0; i < 15; i++)
{
Console.WriteLine(dataBlockInfo.AddData());
}
}
/// <summary>
/// Private helper to request a new data block from the native stream.
/// - it releases the current data block back to the native stream code
/// (which owns the lifecycle of read buffers),
/// - then requests a new buffer from the native stream
/// - and updates the internal read buffer pointer (m_curDataBlock), size (m_curDataSize)
/// and position, all of which are needed by subsequent Read*() calls coming from the user
/// </summary>
private unsafe void GetNextDataBlock()
{
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.ReadDataBlock();
this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
this.m_curBlockPos = 0;
}
/// <summary>
/// Writes out the current data buffer (equivalent of FlushDataBlock), and calls
/// Flush on the native stream to ensure all the data makes its way to the disk
/// </summary>
internal void Flush()
{
if (this.m_curRecordEnd > 0)
{
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordEnd);
this.m_numBytesWritten += this.m_curRecordEnd;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
this.m_curDataBlock = this.m_curDataBlockInfo.dataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.blockSize;
this.m_curRecordStart = 0;
this.m_curRecordEnd = 0;
}
this.m_nativeStream.Flush();
}
public EntityBlock(int id, EntityType entityType)
{
this.entityType = entityType;
this.id = id;
int[] datas = new int[DataManager.Count];
for (int i = 0; i < DataManager.Count; i++)
{
if (entityType.DataInfo[i])
{
datas[i] = DataManager.DataBlockManagers[i].AddBlock();
}
else
{
datas[i] = -1;
}
}
dataBlockInfo = new DataBlockInfo(id, datas);
}
public void Refreshing_Token_After_Multiple_Read_Should_Still_Indicate_The_Last_Block_Number()
{
for (int i = 0; i < 10; i++)
{
DownloadService.ReadBlock(Token.TransferId, i);
}
DataBlockInfo lastBlock = null;
//redownload a few of these blocks
for (int i = 0; i < 10; i++)
{
if (i % 2 == 0)
{
lastBlock = DownloadService.ReadBlock(Token.TransferId, i);
}
}
DownloadToken copy = DownloadService.RenewToken(Token, false);
Assert.AreEqual(lastBlock.BlockNumber, copy.LastTransmittedBlockInfo.BlockNumber);
}
public void BlockTest()
{
DataBlockInfo dataBlockInfo = new DataBlockInfo(0, null);
for (int i = 0; i < 70; i++)
{
Console.WriteLine(dataBlockInfo.AddData());
}
dataBlockInfo.RemoveData(10);
dataBlockInfo.RemoveData(7);
dataBlockInfo.RemoveData(20);
dataBlockInfo.RemoveData(50);
dataBlockInfo.RemoveData(50);
dataBlockInfo.RemoveData(-1);
dataBlockInfo.RemoveData(70);
dataBlockInfo.RemoveData(6);
dataBlockInfo.RemoveData(6);
dataBlockInfo.RemoveData(-1);
dataBlockInfo.RemoveData(70);
Assert.AreEqual(false, dataBlockInfo[10]);
Assert.AreEqual(false, dataBlockInfo[7]);
Assert.AreEqual(false, dataBlockInfo[20]);
Assert.AreEqual(false, dataBlockInfo[50]);
Assert.AreEqual(false, dataBlockInfo[6]);
Assert.AreEqual(false, dataBlockInfo[-1]);
Assert.AreEqual(false, dataBlockInfo[70]);
Assert.AreEqual(true, dataBlockInfo[8]);
Assert.AreEqual(true, dataBlockInfo[14]);
Assert.AreEqual(true, dataBlockInfo[35]);
Assert.AreEqual(true, dataBlockInfo[49]);
Assert.AreEqual(10, dataBlockInfo.AddData());
Assert.AreEqual(7, dataBlockInfo.AddData());
Assert.AreEqual(20, dataBlockInfo.AddData());
Assert.AreEqual(50, dataBlockInfo.AddData());
Assert.AreEqual(6, dataBlockInfo.AddData());
Assert.AreEqual(-1, dataBlockInfo.AddData());
Assert.AreEqual(-1, dataBlockInfo.AddData());
}
/// <summary>
/// Private helper to write the current block out to the native stream.
/// - it writes out the current data buffer up to the point it was filled
/// - it releases the current data block back to the native stream code (which owns the lifecycle of read buffers),
/// - then allocated a new buffer from the native stream
/// - and updates the internal read buffer pointer and position members
/// </summary>
private void FlushDataBlock()
{
DataBlockInfo newDataBlockInfo;
if (this.m_curRecordStart <= 16)
{
// The current block is too small for a single record, augment it
if (this.m_curBlockSize == this.m_nextBlockSize)
{
throw new DryadLinqException(HpcLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
}
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
this.m_nextBlockSize = this.m_nextBlockSize * 2;
if (this.m_nextBlockSize < 0)
{
this.m_nextBlockSize = 0x7FFFFFF8;
}
HpcLinqUtil.memcpy(this.m_curDataBlock, newDataBlockInfo.dataBlock, this.m_curRecordEnd);
}
else
{
// Write all the complete records in the block, put the partial record in the new block
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curRecordStart,
newDataBlockInfo.dataBlock,
this.m_curRecordEnd - this.m_curRecordStart);
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordStart);
this.m_numBytesWritten += this.m_curRecordStart;
this.m_curRecordEnd -= this.m_curRecordStart;
this.m_curRecordStart = 0;
}
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = newDataBlockInfo;
this.m_curDataBlock = newDataBlockInfo.dataBlock;
this.m_curBlockSize = newDataBlockInfo.blockSize;
}
/// <summary>
/// Handles the creation of an actual data block based on the underlying resource.
/// </summary>
/// <param name="transfer">The processed transfer.</param>
/// <param name="blockNumber">The number of the downloaded block.</param>
/// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
/// contains the information about the block. Can be used in order to ensure proper retransmission in case
/// of variable block sizes.</param>
/// <returns>A data block which exposes the data as a resource-friendly stream
/// (<see cref="BufferedDataBlock.Data"/>).</returns>
protected override StreamedDataBlock CreateStreamedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
{
//this func creates the returned DataBlock by reading a chunk of data
//from the underlying stream.
Func <Stream, long, StreamedDataBlock> func = (stream, position) =>
{
DownloadToken token = transfer.Token;
//check if we can use the max block size
long streamLength = stream.Length;
int blockLength = (int)Math.Min(token.DownloadBlockSize,
streamLength - position);
if (blockLength < 0)
{
blockLength = 0;
}
var chunkStream = new ChunkStream(stream, blockLength, position, false);
return(new StreamedDataBlock
{
TransferTokenId = transfer.TransferId,
BlockNumber = blockNumber,
BlockLength = blockLength,
Offset = position,
Data = chunkStream,
IsLastBlock = blockNumber == token.TotalBlockCount - 1
});
};
return(PrepareAndRunBlockReading(transfer, blockNumber, func));
}
/// <summary>
/// Handles the creation of an actual data block based on the underlying resource.
/// </summary>
/// <param name="transfer">The processed transfer.</param>
/// <param name="blockNumber">The number of the downloaded block.</param>
/// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
/// contains the information about the block. Can be used in order to ensure proper retransmission in case
/// of variable block sizes.</param>
/// <returns>A data block which contains the data as an in-memory buffer
/// (<see cref="BufferedDataBlock.Data"/>).</returns>
protected override BufferedDataBlock CreateBufferedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
{
//this func creates the returned DataBlock by reading a chunk of data
//from the underlying stream.
Func <Stream, long, BufferedDataBlock> func = (stream, position) =>
{
DownloadToken token;
byte[] data;
try
{
token = transfer.Token;
//read data
data = new byte[token.DownloadBlockSize];
int read = stream.Read(data, 0, data.Length);
if (read < token.DownloadBlockSize)
{
Array.Resize(ref data, read);
}
}
finally
{
//TODO should change once simplistic implementation was replaced
stream.Close();
}
return(new BufferedDataBlock
{
TransferTokenId = transfer.TransferId,
BlockNumber = blockNumber,
BlockLength = data.Length,
Offset = position,
Data = data,
IsLastBlock = blockNumber == token.TotalBlockCount - 1
});
};
BufferedDataBlock dataBlock = PrepareAndRunBlockReading(transfer, blockNumber, func);
return(dataBlock);
}
/// <summary>
/// Handles the creation of an actual data block based on the underlying resource.
/// </summary>
/// <param name="transfer">The processed transfer.</param>
/// <param name="blockNumber">The number of the downloaded block.</param>
/// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
/// contains the information about the block. Can be used in order to ensure proper retransmission in case
/// of variable block sizes.</param>
/// <returns>A data block which contains the data as an in-memory buffer
/// (<see cref="BufferedDataBlock.Data"/>).</returns>
protected override BufferedDataBlock CreateBufferedDataBlockImpl(LocalDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
{
//this func creates the returned DataBlock by reading a chunk of data
//from the underlying stream.
Func <long, BufferedDataBlock> func = position =>
{
DownloadToken token = transfer.Token;
//read data
byte[] data = new byte[token.DownloadBlockSize];
int read = transfer.Stream.Read(data, 0, data.Length);
if (read < token.DownloadBlockSize)
{
Array.Resize(ref data, read);
}
return(new BufferedDataBlock
{
TransferTokenId = transfer.TransferId,
BlockNumber = blockNumber,
BlockLength = data.Length,
Offset = position,
Data = data,
IsLastBlock = blockNumber == token.TotalBlockCount - 1
});
};
var dataBlock = PrepareAndRunBlockReading(transfer, blockNumber, func);
if (dataBlock.IsLastBlock)
{
//assume the last block will be processed successfully and
//already close the stream (don't wait for the transfer to be
//closed by client). This would still reopen the stream if
//necessary
CloseStream(transfer);
}
return(dataBlock);
}