internal GetJobOutputResponse GetJobOutput(GetJobOutputRequest request)
{
var marshaller = new GetJobOutputRequestMarshaller();
var unmarshaller = GetJobOutputResponseUnmarshaller.Instance;
return Invoke<GetJobOutputRequest,GetJobOutputResponse>(request, marshaller, unmarshaller);
}
/// <summary>
/// Initiates the asynchronous execution of the GetJobOutput operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetJobOutput operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<GetJobOutputResponse> GetJobOutputAsync(GetJobOutputRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new GetJobOutputRequestMarshaller();
var unmarshaller = GetJobOutputResponseUnmarshaller.Instance;
return InvokeAsync<GetJobOutputRequest,GetJobOutputResponse>(request, marshaller,
unmarshaller, cancellationToken);
}
/// <summary>
/// Initiates the asynchronous execution of the GetJobOutput operation.
/// <seealso cref="Amazon.Glacier.IAmazonGlacier.GetJobOutput"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetJobOutput operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<GetJobOutputResponse> GetJobOutputAsync(GetJobOutputRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new GetJobOutputRequestMarshaller();
var unmarshaller = GetJobOutputResponseUnmarshaller.GetInstance();
return Invoke<IRequest, GetJobOutputRequest, GetJobOutputResponse>(request, marshaller, unmarshaller, signer, cancellationToken);
}
/// <summary>
/// Initiates the asynchronous execution of the GetJobOutput operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetJobOutput operation on AmazonGlacierClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndGetJobOutput
/// operation.</returns>
public IAsyncResult BeginGetJobOutput(GetJobOutputRequest request, AsyncCallback callback, object state)
{
var marshaller = new GetJobOutputRequestMarshaller();
var unmarshaller = GetJobOutputResponseUnmarshaller.Instance;
return BeginInvoke<GetJobOutputRequest>(request, marshaller, unmarshaller,
callback, state);
}
public byte[] GetVaultInventory(string jobId)
{
GetJobOutputRequest inventoryRequest = new GetJobOutputRequest()
{
JobId = jobId,
VaultName = _vault
};
GetJobOutputResponse response = _amazonGlacierClient.GetJobOutput(inventoryRequest);
using (Stream webStream = response.GetJobOutputResult.Body)
{
List<byte> data = new List<byte>();
int newByte;
while ((newByte = webStream.ReadByte()) != -1)
data.Add((byte)newByte);
return data.ToArray();
}
}
/// <summary>
/// <para>This operation downloads the output of the job you initiated using InitiateJob. Depending on the job type you specified when you
/// initiated the job, the output will be either the content of an archive or a vault inventory.</para> <para>A job ID will not expire for at
/// least 24 hours after Amazon Glacier completes the job. That is, you can download the job output within the 24 hours period after Amazon
/// Glacier completes the job.</para> <para>If the job output is large, then you can use the <c>Range</c> request header to retrieve a portion
/// of the output. This allows you to download the entire output in smaller chunks of bytes. For example, suppose you have 1 GB of job output
/// you want to download and you decide to download 128 MB chunks of data at a time, which is a total of eight Get Job Output requests. You use
/// the following process to download the job output:</para> <ol> <li> <para>Download a 128 MB chunk of output by specifying the appropriate
/// byte range using the <c>Range</c> header.</para> </li>
/// <li> <para>Along with the data, the response includes a checksum of the payload. You compute the checksum of the payload on the client and
/// compare it with the checksum you received in the response to ensure you received all the expected data.</para> </li>
/// <li> <para>Repeat steps 1 and 2 for all the eight 128 MB chunks of output data, each time specifying the appropriate byte range.</para>
/// </li>
/// <li> <para>After downloading all the parts of the job output, you have a list of eight checksum values. Compute the tree hash of these
/// values to find the checksum of the entire output. Using the DescribeJob API, obtain job information of the job that provided you the output.
/// The response includes the checksum of the entire archive stored in Amazon Glacier. You compare this value with the checksum you computed to
/// ensure you have downloaded the entire archive content with no errors.</para> </li>
/// </ol> <para>An AWS account has full permission to perform all operations (actions). However, AWS Identity and Access Management (IAM) users
/// don't have any permissions by default. You must grant them explicit permission to perform specific actions. For more information, see <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/using-iam-with-amazon-glacier.html">Access Control Using AWS Identity and Access
/// Management (IAM)</a> .</para> <para>For conceptual information and the underlying REST API, go to <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/vault-inventory.html">Downloading a Vault Inventory</a> ,
///
/// <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/downloading-an-archive.html">Downloading an Archive</a> ,
/// and <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/api-job-output-get.html">Get Job Output </a> </para>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetJobOutput service method on
/// AmazonGlacier.</param>
///
/// <returns>The response from the GetJobOutput service method, as returned by AmazonGlacier.</returns>
///
/// <exception cref="T:Amazon.Glacier.Model.ResourceNotFoundException" />
/// <exception cref="T:Amazon.Glacier.Model.MissingParameterValueException" />
/// <exception cref="T:Amazon.Glacier.Model.ServiceUnavailableException" />
/// <exception cref="T:Amazon.Glacier.Model.InvalidParameterValueException" />
public GetJobOutputResponse GetJobOutput(GetJobOutputRequest request)
{
var task = GetJobOutputAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return null;
}
}
/// <summary>
/// Initiates the asynchronous execution of the GetJobOutput operation.
/// <seealso cref="Amazon.Glacier.AmazonGlacier.GetJobOutput"/>
/// </summary>
///
/// <param name="getJobOutputRequest">Container for the necessary parameters to execute the GetJobOutput operation on AmazonGlacier.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndGetJobOutput
/// operation.</returns>
public IAsyncResult BeginGetJobOutput(GetJobOutputRequest getJobOutputRequest, AsyncCallback callback, object state)
{
return invokeGetJobOutput(getJobOutputRequest, callback, state, false);
}
public ArchivePartInfo DownloadArchivePart(string jobId, long start, long end)
{
GetJobOutputRequest downloadRequest = new GetJobOutputRequest()
{
JobId = jobId,
VaultName = _vault
};
downloadRequest.SetRange(start, end);
GetJobOutputResponse response = _amazonGlacierClient.GetJobOutput(downloadRequest);
GetJobOutputResult result = response.GetJobOutputResult;
ArchivePartInfo info = new ArchivePartInfo(result.Body, result.Checksum);
return info;
}
/// <summary>
/// <para>This operation downloads the output of the job you initiated using InitiateJob. Depending on the job type you specified when you
/// initiated the job, the output will be either the content of an archive or a vault inventory.</para> <para>A job ID will not expire for at
/// least 24 hours after Amazon Glacier completes the job. That is, you can download the job output within the 24 hours period after Amazon
/// Glacier completes the job.</para> <para>If the job output is large, then you can use the <c>Range</c> request header to retrieve a portion
/// of the output. This allows you to download the entire output in smaller chunks of bytes. For example, suppose you have 1 GB of job output
/// you want to download and you decide to download 128 MB chunks of data at a time, which is a total of eight Get Job Output requests. You use
/// the following process to download the job output:</para> <ol> <li> <para>Download a 128 MB chunk of output by specifying the appropriate
/// byte range using the <c>Range</c> header.</para> </li>
/// <li> <para>Along with the data, the response includes a checksum of the payload. You compute the checksum of the payload on the client and
/// compare it with the checksum you received in the response to ensure you received all the expected data.</para> </li>
/// <li> <para>Repeat steps 1 and 2 for all the eight 128 MB chunks of output data, each time specifying the appropriate byte range.</para>
/// </li>
/// <li> <para>After downloading all the parts of the job output, you have a list of eight checksum values. Compute the tree hash of these
/// values to find the checksum of the entire output. Using the Describe Job API, obtain job information of the job that provided you the
/// output. The response includes the checksum of the entire archive stored in Amazon Glacier. You compare this value with the checksum you
/// computed to ensure you have downloaded the entire archive content with no errors.</para> </li>
/// </ol> <para>An AWS account has full permission to perform all operations (actions). However, AWS Identity and Access Management (IAM) users
/// don't have any permissions by default. You must grant them explicit permission to perform specific actions. For more information, see Access
/// Control Using AWS Identity and Access Management (IAM).</para> <para>For conceptual information and the underlying REST API, go to
/// Downloading a Vault Inventory, Downloading an Archive, and Get Job Output </para>
/// </summary>
///
/// <param name="getJobOutputRequest">Container for the necessary parameters to execute the GetJobOutput service method on
/// AmazonGlacier.</param>
///
/// <returns>The response from the GetJobOutput service method, as returned by AmazonGlacier.</returns>
///
/// <exception cref="ResourceNotFoundException"/>
/// <exception cref="MissingParameterValueException"/>
/// <exception cref="ServiceUnavailableException"/>
/// <exception cref="InvalidParameterValueException"/>
public GetJobOutputResponse GetJobOutput(GetJobOutputRequest getJobOutputRequest)
{
IAsyncResult asyncResult = invokeGetJobOutput(getJobOutputRequest, null, null, true);
return EndGetJobOutput(asyncResult);
}
IAsyncResult invokeGetJobOutput(GetJobOutputRequest getJobOutputRequest, AsyncCallback callback, object state, bool synchronized)
{
IRequest irequest = new GetJobOutputRequestMarshaller().Marshall(getJobOutputRequest);
var unmarshaller = GetJobOutputResponseUnmarshaller.GetInstance();
AsyncResult result = new AsyncResult(irequest, callback, state, synchronized, signer, unmarshaller);
Invoke(result);
return result;
}
internal async Task ExecuteAsync()
{
long contentLength = -1;
string glacierProvidedCheckSum = null;
string rangeValue = null;
Stream input = null;
Stream output = null;
try
{
// Make sure the directory exists to write too.
FileInfo fi = new FileInfo(filePath);
Directory.CreateDirectory(fi.DirectoryName);
FileMode fileMode = FileMode.Create;
int retryAttempts = 0;
byte[] buffer = new byte[1024 * 1024 * 5];
long transferredBytes = 0;
MemoryStream partStream = new MemoryStream(new byte[PART_STREAM_HASH_SIZE]);
LinkedList<string> hashes = new LinkedList<string>();
while (true)
{
try
{
output = File.Open(filePath, fileMode, FileAccess.Write, FileShare.None);
try
{
GetJobOutputRequest getJobOutputRequest = new GetJobOutputRequest()
{
AccountId = this.options.AccountId,
VaultName = this.vaultName,
JobId = jobId,
Range = rangeValue
};
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)getJobOutputRequest).AddBeforeRequestHandler(new ArchiveTransferManager.UserAgentPostFix("DownloadArchive").UserAgentRequestEventHandlerSync);
GetJobOutputResponse jobOutputResponse = await this.manager.GlacierClient.GetJobOutputAsync(getJobOutputRequest).ConfigureAwait(false);
if (contentLength < 0)
{
contentLength = jobOutputResponse.ContentLength;
glacierProvidedCheckSum = jobOutputResponse.Checksum;
}
#if CORECLR
input = jobOutputResponse.Body;
#else
input = new BufferedStream(jobOutputResponse.Body);
#endif
long totalBytesFromGetJobOutput = jobOutputResponse.ContentLength;
long bytesReadFromGetJobOutput = 0;
int bytesRead = 0;
do
{
bytesRead = input.Read(buffer, 0, buffer.Length);
if (bytesRead > 0)
{
bytesReadFromGetJobOutput += bytesRead;
output.Write(buffer, 0, bytesRead);
transferredBytes += bytesRead;
int offset = 0;
if (partStream.Position + bytesRead > PART_STREAM_HASH_SIZE)
{
var length = PART_STREAM_HASH_SIZE - (int)partStream.Position;
partStream.Write(buffer, 0, length);
offset = length;
}
else
{
partStream.Write(buffer, 0, bytesRead);
offset = bytesRead;
}
if (partStream.Position == PART_STREAM_HASH_SIZE)
{
partStream.Position = 0;
hashes.AddLast(TreeHashGenerator.CalculateTreeHash(partStream));
}
if (offset != bytesRead)
{
partStream.Write(buffer, offset, bytesRead - offset);
}
// Make callback on progress
AWSSDKUtils.InvokeInBackground(
this.options.StreamTransferProgress,
new Runtime.StreamTransferProgressArgs(bytesRead, transferredBytes, contentLength),
this.manager);
}
if (retryAttempts > 0)
retryAttempts = 0; // Reset retry attempts back to 0 since we able to successfully write more data to disk.
} while (bytesReadFromGetJobOutput < totalBytesFromGetJobOutput);
// Compute hash of the last remaining bytes
if (partStream.Position != 0)
{
partStream.SetLength(partStream.Position);
partStream.Position = 0;
hashes.AddLast(TreeHashGenerator.CalculateTreeHash(partStream));
}
break;
}
finally
{
output.Dispose();
output = null;
try { if (input != null) input.Dispose(); }
catch (Exception) { }
}
}
catch (Exception e)
{
var age = e as AmazonGlacierException;
if (age != null && age.StatusCode == HttpStatusCode.NotFound)
{
throw;
}
fileMode = FileMode.Append;
rangeValue = string.Format(CultureInfo.InvariantCulture, "bytes={0}-", new FileInfo(filePath).Length);
retryAttempts++;
if (retryAttempts <= DownloadFileCommand.MAX_OPERATION_RETRY)
{
Console.WriteLine("Error and going to retry: {0}", e.Message);
Console.WriteLine(e.StackTrace);
Thread.Sleep(60 * 1000);
}
else
throw;
}
}
var computedCheckSum = TreeHashGenerator.CalculateTreeHash(hashes);
if (!string.Equals(glacierProvidedCheckSum, computedCheckSum, StringComparison.OrdinalIgnoreCase))
{
throw new AmazonGlacierException("Checksum of the downloaded file does not match the checksum reported by Amazon Glacier.");
}
}
catch (IOException e)
{
throw new IOException("Unable to save archive to disk", e);
}
finally
{
try { if (input != null) input.Dispose(); }
catch (Exception) { }
try { if (output != null) output.Dispose(); }
catch (Exception) { }
}
}
/// <summary>
/// <para>This operation downloads the output of the job you initiated using InitiateJob. Depending on the job type you specified when you
/// initiated the job, the output will be either the content of an archive or a vault inventory.</para> <para>A job ID will not expire for at
/// least 24 hours after Amazon Glacier completes the job. That is, you can download the job output within the 24 hours period after Amazon
/// Glacier completes the job.</para> <para>If the job output is large, then you can use the <c>Range</c> request header to retrieve a portion
/// of the output. This allows you to download the entire output in smaller chunks of bytes. For example, suppose you have 1 GB of job output
/// you want to download and you decide to download 128 MB chunks of data at a time, which is a total of eight Get Job Output requests. You use
/// the following process to download the job output:</para> <ol> <li> <para>Download a 128 MB chunk of output by specifying the appropriate
/// byte range using the <c>Range</c> header.</para> </li>
/// <li> <para>Along with the data, the response includes a checksum of the payload. You compute the checksum of the payload on the client and
/// compare it with the checksum you received in the response to ensure you received all the expected data.</para> </li>
/// <li> <para>Repeat steps 1 and 2 for all the eight 128 MB chunks of output data, each time specifying the appropriate byte range.</para>
/// </li>
/// <li> <para>After downloading all the parts of the job output, you have a list of eight checksum values. Compute the tree hash of these
/// values to find the checksum of the entire output. Using the DescribeJob API, obtain job information of the job that provided you the output.
/// The response includes the checksum of the entire archive stored in Amazon Glacier. You compare this value with the checksum you computed to
/// ensure you have downloaded the entire archive content with no errors.</para> </li>
/// </ol> <para>An AWS account has full permission to perform all operations (actions). However, AWS Identity and Access Management (IAM) users
/// don't have any permissions by default. You must grant them explicit permission to perform specific actions. For more information, see <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/using-iam-with-amazon-glacier.html">Access Control Using AWS Identity and Access
/// Management (IAM)</a> .</para> <para>For conceptual information and the underlying REST API, go to <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/vault-inventory.html">Downloading a Vault Inventory</a> ,
///
/// <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/downloading-an-archive.html">Downloading an Archive</a> ,
/// and <a href="http://docs.aws.amazon.com/amazonglacier/latest/dev/api-job-output-get.html">Get Job Output </a> </para>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetJobOutput service method on
/// AmazonGlacier.</param>
///
/// <returns>The response from the GetJobOutput service method, as returned by AmazonGlacier.</returns>
///
/// <exception cref="T:Amazon.Glacier.Model.ResourceNotFoundException" />
/// <exception cref="T:Amazon.Glacier.Model.MissingParameterValueException" />
/// <exception cref="T:Amazon.Glacier.Model.ServiceUnavailableException" />
/// <exception cref="T:Amazon.Glacier.Model.InvalidParameterValueException" />
public GetJobOutputResponse GetJobOutput(GetJobOutputRequest request)
{
var task = GetJobOutputAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
throw e.InnerException;
}
}
private static void GetInventory(string jobId, string vaultName, string filename)
{
using (IAmazonGlacier client = new AmazonGlacierClient(RegionEndpoint.EUWest1))
{
GetJobOutputRequest getJobOutputRequest = new GetJobOutputRequest()
{
JobId = jobId,
VaultName = vaultName,
};
GetJobOutputResponse getJobOutputResponse = client.GetJobOutput(getJobOutputRequest);
using (Stream webStream = getJobOutputResponse.Body)
{
using (Stream fileToSave = File.OpenWrite(filename))
{
CopyStream(webStream, fileToSave);
}
}
}
}
/// <summary>
/// Initiates the asynchronous execution of the GetJobOutput operation.
/// <seealso cref="Amazon.Glacier.IAmazonGlacier.GetJobOutput"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the GetJobOutput operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public async Task<GetJobOutputResponse> GetJobOutputAsync(GetJobOutputRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new GetJobOutputRequestMarshaller();
var unmarshaller = GetJobOutputResponseUnmarshaller.GetInstance();
var response = await Invoke<IRequest, GetJobOutputRequest, GetJobOutputResponse>(request, marshaller, unmarshaller, signer, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
return response;
}
private void completeJob(Dictionary<string, string> properties, string action)
{
FolderVaultMapping mapping = Context.Mapping;
string jobId = properties.ContainsKey("JobId") ? properties["JobId"] : null;
if (jobId != null)
{
using (AmazonGlacierClient client = new AmazonGlacierClient(mapping.AccessKey, mapping.SecretKey, mapping.Endpoint))
{
GetJobOutputRequest gReq = new GetJobOutputRequest();
GetJobOutputResponse gResp;
GetJobOutputResult gResult;
gReq.AccountId = "-";
gReq.JobId = jobId;
gReq.VaultName = mapping.VaultName;
gResp = client.GetJobOutput(gReq);
gResult = gResp.GetJobOutputResult;
using (Stream input = gResult.Body)
{
if (action == "InventoryRetrieval")
{
handleInventory(properties, input);
}
else if (action == "ArchiveRetrieval")
{
handleArchive(properties, input);
}
}
}
}
}
