/// <summary>
/// Creates a new downloader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="downloadMetadata">The metadata for the entire download.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="token">The cancellation token to use</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null)
{
_metadata = downloadMetadata;
_segmentMetadata = downloadMetadata.Segments[segmentNumber];
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseBackOffRetryStrategy = true;
}
/// <summary>
/// Calculates the value by which we'd need to adjust the length of the given segment, by searching for the nearest newline around it (before and after),
/// and returning the distance to it (which can be positive, if after, or negative, if before).
/// </summary>
/// <param name="segment"></param>
/// <param name="stream"></param>
/// <param name="encoding"></param>
/// <param name="delimiter"></param>
/// <returns></returns>
/// <exception cref="Microsoft.Azure.Management.DataLake.Store.TransferFailedException">If no record boundary could be located on either side of the segment end offset within the allowed distance.</exception>
private int DetermineLengthAdjustment(TransferSegmentMetadata segment, FileStream stream, Encoding encoding, string delimiter = null)
{
long referenceFileOffset = segment.Offset + segment.Length;
byte[] buffer = new byte[SingleSegmentUploader.MaxRecordLength];
//read 2MB before the segment boundary and 2MB after (for a total of 4MB = max append length)
int bytesRead = ReadIntoBufferAroundReference(stream, buffer, referenceFileOffset);
if (bytesRead > 0)
{
int middlePoint = bytesRead / 2;
//search for newline in it
int newLinePosBefore = StringExtensions.FindNewline(buffer, middlePoint + 1, middlePoint + 1, true, encoding, delimiter);
//in some cases, we may have a newline that is 2 characters long, and it occurrs exactly on the midpoint, which means we won't be able to find its end.
//see if that's the case, and then search for a new candidate before it.
if (string.IsNullOrEmpty(delimiter) && newLinePosBefore == middlePoint + 1 && buffer[newLinePosBefore] == (byte)'\r')
{
int newNewLinePosBefore = StringExtensions.FindNewline(buffer, middlePoint, middlePoint, true, encoding);
if (newNewLinePosBefore >= 0)
{
newLinePosBefore = newNewLinePosBefore;
}
}
int newLinePosAfter = StringExtensions.FindNewline(buffer, middlePoint, middlePoint, false, encoding, delimiter);
if (string.IsNullOrEmpty(delimiter) && newLinePosAfter == buffer.Length - 1 && buffer[newLinePosAfter] == (byte)'\r' && newLinePosBefore >= 0)
{
newLinePosAfter = -1;
}
int closestNewLinePos = FindClosestToCenter(newLinePosBefore, newLinePosAfter, middlePoint);
//middle point of the buffer corresponds to the reference file offset, so all we need to do is return the difference between the closest newline and the center of the buffer
if (closestNewLinePos >= 0)
{
return(closestNewLinePos - middlePoint);
}
}
//if we get this far, we were unable to find a record boundary within our limits => fail the upload
throw new TransferFailedException(
string.Format(
"Unable to locate a record boundary within {0}MB on either side of segment {1} (offset {2}). This means the record at that offset is larger than {0}MB.",
SingleSegmentUploader.MaxRecordLength / 1024 / 1024 / 2,
segment.SegmentNumber,
segment.Offset,
SingleSegmentUploader.MaxRecordLength / 1024 / 1024));
}
/// <summary>
/// Constructs a new TransferMetadata from the given parameters.
/// </summary>
/// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param>
/// <param name="transferParameters">The parameters to use for constructing this metadata.</param>
/// <param name="frontEnd">The front end. This is used only in the constructor for determining file length</param>
internal TransferMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontEnd, long fileSize = -1)
{
this.MetadataFilePath = metadataFilePath;
this.TransferId = Guid.NewGuid().ToString("N");
this.InputFilePath = transferParameters.InputFilePath;
this.TargetStreamPath = transferParameters.TargetStreamPath;
this.IsDownload = transferParameters.IsDownload;
this.SegmentStreamDirectory = GetSegmentStreamDirectory();
this.IsBinary = transferParameters.IsBinary;
this.FileLength = fileSize < 0 ? frontEnd.GetStreamLength(transferParameters.InputFilePath, !IsDownload) : fileSize;
this.EncodingCodePage = transferParameters.FileEncoding.CodePage;
// we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic.
// this protects us against agressive increase of thread count resulting in far more segments than
// is reasonable for a given file size. We also ensure that each segment is at least 256mb in size.
// This is the size that ensures we have the optimal storage creation in the store.
var preliminarySegmentCount = (int)Math.Ceiling((double)this.FileLength / transferParameters.MaxSegementLength);
this.SegmentCount = Math.Min(preliminarySegmentCount, TransferSegmentMetadata.CalculateSegmentCount(this.FileLength));
this.SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(this.FileLength, this.SegmentCount);
this.Segments = new TransferSegmentMetadata[this.SegmentCount];
for (int i = 0; i < this.SegmentCount; i++)
{
this.Segments[i] = new TransferSegmentMetadata(i, this);
}
if (!transferParameters.IsBinary && this.SegmentCount > 1 && !this.IsDownload)
{
this.AlignSegmentsToRecordBoundaries();
// ensure that nothing strange happened during alignment
this.ValidateConsistency();
}
// initialize the status to pending, since it is not yet done.
this.Status = SegmentTransferStatus.Pending;
}
