public void TransferStarted(object sender, TransferFileProgressArgs args)
{
string key = args.FilePath;
TransferEntry entry = new TransferEntry(args);
Debug.Assert(entry.Data.State == TransferState.STARTED);
AddItemToList(entry);
OnChangedTransferState(entry);
}
public void TransferCompleted(object sender, TransferFileProgressArgs args)
{
string key = args.FilePath;
TransferEntry entry = Transfers[key];
if (entry == null)
{
return;
}
Debug.Assert(entry.Data.State == TransferState.COMPLETED);
OnChangedTransferState(entry);
}
public void TransferCanceled(object sender, TransferFileProgressArgs args)
{
string key = args.FilePath;
TransferEntry entry = Transfers[key];
if (entry == null)
{
return;
}
Debug.Assert(entry.Data.State == TransferState.CANCELED);
entry.Exception = args.Exception;
OnChangedTransferState(entry);
}
public void TransferProgress(object sender, TransferFileProgressArgs args)
{
string key = args.FilePath;
TransferEntry entry = Transfers[key];
if (entry == null)
{
return;
}
Debug.Assert(entry.Data.State == TransferState.TRANSFERRING);
//entry.Remaining.Text = FileSizeToString(entry.Data.TotalBytes - entry.Data.TransferredBytes);
OnChangedTransferState(entry);
}
public TransferEntry(TransferFileProgressArgs other)
{
Data = other;
Path = new Label();
ControlDefaults(Path);
Path.Name = "Path";
Path.Text = Data.FilePath;
// NOTE: Cannot use DataBindings because the change event is raised on another thread.
Path.DataBindings.Add(new Binding("Text", Data,
this.GetPropertyName((TransferFileProgressArgs x) => x.FilePath)));
Remaining = new Label();
ControlDefaults(Remaining);
Remaining.Name = "Remaining";
Remaining.Text = "";
Binding RemainingBinding = new Binding("Text", Data,
this.GetPropertyName((TransferFileProgressArgs x) => x.RemainingBytes));
RemainingBinding.Format += FileSizeUtils.FileSizeToString;
Remaining.DataBindings.Add(RemainingBinding);
Message = new Label();
ControlDefaults(Message);
Message.Name = "Message";
Message.Text = "";
Message.ForeColor = Color.Red;
Progress = new ExtendedProgressBar();
ControlDefaults(Progress);
Progress.Name = "Progress";
Progress.Minimum = 0;
Progress.Maximum = 100;
// NOTE: Cannot use DataBindings because the change event is raised on another thread.
Progress.DataBindings.Add(new Binding("Value", Data,
this.GetPropertyName((TransferFileProgressArgs x) => x.PercentDone)));
}
// REFERENCE: http://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingNetSDK.html
public override void DownloadFile(string filePath, string keyName, object userData, CancellationToken cancellationToken)
{
if (cancellationToken != null)
{
cancellationToken.ThrowIfCancellationRequested();
}
TransferFileProgressArgs reusedProgressArgs = new TransferFileProgressArgs
{
UserData = userData,
State = TransferState.PENDING,
TotalBytes = 0,
TransferredBytes = 0,
FilePath = filePath,
};
// Download request.
GetObjectRequest downloadRequest = new GetObjectRequest
{
BucketName = this._awsBuckeName,
Key = keyName,
};
try
{
// Attempt to create any intermediary directories before anything else.
string fileDirectoryName = FileManager.UnsafeGetDirectoryName(filePath);
FileManager.UnsafeCreateDirectory(fileDirectoryName);
// Report start - before any possible failures.
if (DownloadStarted != null)
{
DownloadStarted(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.STARTED;
});
}
const int DefaultBufferSize = 8192; // S3Constants.DefaultBufferSize
// REFERENCE: https://github.com/aws/aws-sdk-net/blob/5f19301ee9fa1ec29b11b3dfdee82071a04ed5ae/AWSSDK_DotNet35/Amazon.S3/Model/GetObjectResponse.cs
// Download.
// Create the file. If the file already exists, it will be overwritten.
using (GetObjectResponse downloadResponse = this._s3Client.GetObject(downloadRequest))
using (BufferedStream bufferedStream = new BufferedStream(downloadResponse.ResponseStream))
using (Stream fileStream = new BufferedStream(new FileStream(filePath, FileMode.Create)))
{
// Report 0% progress.
if (DownloadProgressed != null)
{
DownloadProgressed(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.TRANSFERRING;
reusedProgressArgs.TotalBytes = downloadResponse.ContentLength;
});
}
string requestId = downloadResponse.ResponseMetadata.RequestId;
string amzId2;
downloadResponse.ResponseMetadata.Metadata.TryGetValue(HeaderKeys.XAmzId2Header, out amzId2);
amzId2 = amzId2 ?? string.Empty;
long filePosition = 0;
int bytesRead = 0;
byte[] buffer = new byte[DefaultBufferSize];
while ((bytesRead = bufferedStream.Read(buffer, 0, buffer.Length)) > 0)
{
if (cancellationToken != null)
{
cancellationToken.ThrowIfCancellationRequested();
}
fileStream.Write(buffer, 0, bytesRead);
filePosition += bytesRead;
// Report progress.
if (DownloadProgressed != null)
{
DownloadProgressed(reusedProgressArgs, () =>
{
reusedProgressArgs.DeltaTransferredBytes = bytesRead;
reusedProgressArgs.TransferredBytes = filePosition;
});
}
}
// Validate transferred size.
if (reusedProgressArgs.TransferredBytes != reusedProgressArgs.TotalBytes)
{
var message = string.Format(CultureInfo.InvariantCulture,
"The total bytes read {0} from response stream is not equal to the Content-Length {1} for the object {2} in bucket {3}."
+ " Request ID = {4} , AmzId2 = {5}.",
reusedProgressArgs.TransferredBytes,
reusedProgressArgs.TotalBytes,
keyName, this._awsBuckeName, requestId, amzId2);
throw new StreamSizeMismatchException(message, reusedProgressArgs.TotalBytes, reusedProgressArgs.TransferredBytes, requestId, amzId2);
}
}
// Report completion.
if (DownloadCompleted != null)
{
DownloadCompleted(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.COMPLETED;
});
}
}
catch (OperationCanceledException exception)
{
logger.Info("Download canceled: {0}", filePath);
// Report cancelation.
if (DownloadCanceled != null)
{
DownloadCanceled(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.CANCELED;
reusedProgressArgs.Exception = exception;
});
}
}
catch (Exception exception)
{
if (exception is AmazonS3Exception)
{
AmazonS3Exception amznException = exception as AmazonS3Exception;
if (amznException.ErrorCode != null && (amznException.ErrorCode.Equals("InvalidAccessKeyId") || amznException.ErrorCode.Equals("InvalidSecurity")))
{
logger.Log(LogLevel.Warn, "Check the provided AWS Credentials.");
}
else
{
logger.Log(LogLevel.Warn, "Error occurred during the download of {0}\nMessage:'{1}'", filePath, amznException.Message);
}
}
else
{
logger.Log(LogLevel.Warn, "Exception occurred during the download of {0}\nException: {1}", filePath, exception.Message);
}
// Report failure.
if (DownloadFailed != null)
{
DownloadFailed(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.FAILED;
reusedProgressArgs.Exception = exception;
});
}
}
}
public override void UploadFile(string filePath, string keyName, object userData, CancellationToken cancellationToken)
{
if (cancellationToken != null)
{
cancellationToken.ThrowIfCancellationRequested();
}
TransferFileProgressArgs reusedProgressArgs = new TransferFileProgressArgs
{
UserData = userData,
State = TransferState.PENDING,
TotalBytes = 0,
TransferredBytes = 0,
FilePath = filePath,
};
TransferUtility fileTransferUtility = null; // IDisposable
// REFERENCES:
// https://docs.aws.amazon.com/AmazonS3/latest/dev/HLTrackProgressMPUDotNet.html
// https://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
try
{
long fileLength = ZetaLongPaths.ZlpIOHelper.GetFileLength(filePath);
reusedProgressArgs.TotalBytes = fileLength;
TransferUtilityConfig xferConfig = new TransferUtilityConfig
{
ConcurrentServiceRequests = 10, // Maximum allowed concurrent requests for this file alone.
MinSizeBeforePartUpload = AbsoluteMinPartSize,
};
fileTransferUtility = new TransferUtility(this._s3Client, xferConfig);
// Step 1: Initialize.
// Use TransferUtilityUploadRequest to configure options.
TransferUtilityUploadRequest uploadRequest = new TransferUtilityUploadRequest
{
BucketName = this._awsBuckeName,
Key = keyName,
FilePath = filePath,
CannedACL = S3CannedACL.Private,
StorageClass = S3StorageClass.ReducedRedundancy,
PartSize = CalculatePartSize(fileLength),
};
uploadRequest.UploadProgressEvent += new EventHandler <UploadProgressArgs>(
(object sender, UploadProgressArgs e) =>
{
// Process event.
//logger.Debug("PROGRESS {0} -> {1}", filePath, e.ToString());
long delta = e.TransferredBytes - reusedProgressArgs.TransferredBytes;
// Report progress.
if (UploadProgressed != null)
{
UploadProgressed(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.TRANSFERRING;
reusedProgressArgs.DeltaTransferredBytes = delta;
reusedProgressArgs.TransferredBytes = e.TransferredBytes;
});
}
});
// Report start - before any possible failures.
if (UploadStarted != null)
{
UploadStarted(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.STARTED;
});
}
// TODO(jweyrich): Make it interruptible - use UploadAsync?
fileTransferUtility.Upload(uploadRequest);
// Report completion.
if (UploadCompleted != null)
{
UploadCompleted(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.COMPLETED;
});
}
}
catch (OperationCanceledException exception)
{
logger.Info("Upload canceled: {0}", filePath);
// Report cancelation.
if (UploadCanceled != null)
{
UploadCanceled(reusedProgressArgs, () =>
{
reusedProgressArgs.Exception = exception;
reusedProgressArgs.State = TransferState.CANCELED;
});
}
}
catch (Exception exception)
{
if (exception is AmazonS3Exception)
{
AmazonS3Exception amznException = exception as AmazonS3Exception;
if (amznException.ErrorCode != null && (amznException.ErrorCode.Equals("InvalidAccessKeyId") || amznException.ErrorCode.Equals("InvalidSecurity")))
{
logger.Log(LogLevel.Warn, "Check the provided AWS Credentials.");
}
else
{
logger.Log(LogLevel.Warn, "Error occurred during the upload of {0}\nMessage:'{1}'", filePath, amznException.Message);
}
}
else
{
logger.Log(LogLevel.Warn, "Exception occurred during the upload of {0}\nException: {1}", filePath, exception.Message);
}
// Report failure.
if (UploadFailed != null)
{
UploadFailed(reusedProgressArgs, () =>
{
reusedProgressArgs.State = TransferState.FAILED;
reusedProgressArgs.Exception = exception;
});
}
}
finally
{
if (fileTransferUtility != null)
{
fileTransferUtility.Dispose();
}
}
}
