/// <summary> /// Determines the upload cutoff for text file. /// </summary> /// <param name="buffer">The buffer.</param> /// <param name="bufferDataLength">Length of the buffer data.</param> /// <param name="inputStream">The input stream.</param> /// <returns></returns> /// <exception cref="TransferFailedException"></exception> private int DetermineUploadCutoffForTextFile(byte[] buffer, int bufferDataLength, Stream inputStream) { var encoding = Encoding.GetEncoding(_metadata.EncodingCodePage); //NOTE: we return an offset, but everywhere else below we treat it as a byte count; in order for that to work, we need to add 1 to the result of FindNewLine. int uploadCutoff = StringExtensions.FindNewline(buffer, bufferDataLength - 1, bufferDataLength, true, encoding, _metadata.Delimiter) + 1; if (uploadCutoff <= 0 && (_metadata.SegmentCount > 1 || bufferDataLength >= MaxRecordLength)) { var ex = new TransferFailedException(string.Format("Found a record that exceeds the maximum allowed record length around offset {0}", inputStream.Position)); TracingHelper.LogError(ex); throw ex; } //a corner case here is when the newline is 2 chars long, and the first of those lands on the last byte of the buffer. If so, let's try to find another //newline inside the buffer, because we might be splitting this wrongly. if (string.IsNullOrEmpty(_metadata.Delimiter) && uploadCutoff == buffer.Length && buffer[buffer.Length - 1] == (byte)'\r') { int newCutoff = StringExtensions.FindNewline(buffer, bufferDataLength - 2, bufferDataLength - 1, true, encoding) + 1; if (newCutoff > 0) { uploadCutoff = newCutoff; } } return(uploadCutoff); }
/// <summary> /// Verifies the downloaded stream. /// </summary> /// <exception cref="TransferFailedException"></exception> internal void VerifyDownloadedStream() { //verify that the remote stream has the length we expected. var retryCount = 0; long remoteLength = -1; while (retryCount < MaxBufferDownloadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { remoteLength = _frontEnd.GetStreamLength(_segmentMetadata.Path, _metadata.IsDownload); break; } catch (Exception e) { _token.ThrowIfCancellationRequested(); if (retryCount >= MaxBufferDownloadAttemptCount) { TracingHelper.LogError(e); throw e; } var waitTime = WaitForRetry(retryCount, this.UseBackOffRetryStrategy, _token); TracingHelper.LogInfo("VerifyDownloadedStream: GetStreamLength at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}", _segmentMetadata.Path, retryCount, e, waitTime); } } if (_segmentMetadata.Length != remoteLength) { var ex = new TransferFailedException(string.Format("Post-download stream verification failed: target stream has a length of {0}, expected {1}", remoteLength, _segmentMetadata.Length)); TracingHelper.LogError(ex); throw ex; } }
/// <summary> /// Downloads the segment contents. /// </summary> private void DownloadSegmentContents() { // set the current offset in the stream we are reading to the offset // that this segment starts at. long curOffset = _segmentMetadata.Offset; // set the offset of the local file that we are creating to the beginning of the local stream. // this value will be used to ensure that we are always reporting the right progress and that, // in the event of faiure, we reset the local stream to the proper location. long localOffset = 0; // determine the number of requests made based on length of file divded by 32MB max size requests var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength); // set the length remaining to ensure that only the exact number of bytes is ultimately downloaded // for this segment. var lengthRemaining = _segmentMetadata.Length; // for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use. // This also protects the user from unintentionally using the file after a failed download. var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath; using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { outputStream.Seek(curOffset, SeekOrigin.Begin); for (int i = 0; i < numRequests; i++) { _token.ThrowIfCancellationRequested(); int attemptCount = 0; int partialDataAttempts = 0; bool downloadCompleted = false; long dataReceived = 0; bool modifyLengthAndOffset = false; while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount) { _token.ThrowIfCancellationRequested(); try { long lengthToDownload = (long)BufferLength; // in the case where we got less than the expected amount of data, // only download the rest of the data from the previous request // instead of a new full buffer. if (modifyLengthAndOffset) { lengthToDownload -= dataReceived; } // test to make sure that the remaining length is larger than the max size, // otherwise just download the remaining length. if (lengthRemaining - lengthToDownload < 0) { lengthToDownload = lengthRemaining; } using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload)) { readStream.CopyTo(outputStream, (int)lengthToDownload); } var lengthReturned = outputStream.Position - curOffset; // if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data if (lengthReturned > lengthToDownload) { var ex = new TransferFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - curOffset, curOffset, _metadata.InputFilePath, DateTime.Now.ToString())); TracingHelper.LogError(ex); throw ex; } // we need to validate how many bytes have actually been copied to the read stream if (lengthReturned < lengthToDownload) { partialDataAttempts++; lengthRemaining -= lengthReturned; curOffset += lengthReturned; localOffset += lengthReturned; modifyLengthAndOffset = true; dataReceived += lengthReturned; ReportProgress(localOffset, false); // we will wait before the next iteration, since something went wrong and we did not receive enough data. // this could be a throttling issue or an issue with the service itself. Either way, waiting should help // reduce the liklihood of additional failures. if (partialDataAttempts >= MaxBufferDownloadAttemptCount) { var ex = new TransferFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path)); TracingHelper.LogError(ex); throw ex; } var waitTime = WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token); TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} returned: {1} bytes. Expected: {2} bytes. Attempt: {3}. Wait time in ms before retry: {4}", _metadata.InputFilePath, lengthReturned, lengthToDownload, partialDataAttempts, waitTime); } else { downloadCompleted = true; lengthRemaining -= lengthToDownload; curOffset += lengthToDownload; localOffset += lengthToDownload; ReportProgress(localOffset, false); } } catch (Exception ex) { // update counts and reset for internal attempts attemptCount++; partialDataAttempts = 0; //if we tried more than the number of times we were allowed to, give up and throw the exception if (attemptCount >= MaxBufferDownloadAttemptCount) { ReportProgress(localOffset, true); TracingHelper.LogError(ex); throw ex; } else { var waitTime = WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token); TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}", _metadata.InputFilePath, attemptCount, ex, waitTime); // forcibly put the stream back to where it should be based on where we think we are in the download. outputStream.Seek(curOffset, SeekOrigin.Begin); } } } } // full validation of the segment. if (outputStream.Position - _segmentMetadata.Offset != _segmentMetadata.Length) { var ex = new TransferFailedException(string.Format("Post-download stream segment verification failed for file {2}: target stream has a length of {0}, expected {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", outputStream.Position - _segmentMetadata.Offset, _segmentMetadata.Length, _segmentMetadata.Path)); TracingHelper.LogError(ex); throw ex; } } }