public async Task <string> FinishUploadAsync(UploadJob job, UploadItem item) { var checksum = TreeHashGenerator.CalculateTreeHash(job.ChunkChecksums); //prepare request var request = new CompleteMultipartUploadRequest { UploadId = job.UploadId, ArchiveSize = item.ContentLength.ToString(), Checksum = checksum, VaultName = job.VaultName }; //finish up multipart upload var response = await _client.CompleteMultipartUploadAsync(request); var achiveId = response.ArchiveId; return(achiveId); }
private async Task <string> UploadArchiveAsync(Stream stream, string archiveDescription) { await TestConnectionAsync(); var streamSize = new Size(stream.Length, SizeUnit.Bytes); if (streamSize > TotalArchiveSizeLimit) { throw new InvalidOperationException($@"Can't upload more than 40TB to AWS Glacier, current upload size: {streamSize}"); } var streamLength = streamSize.GetValue(SizeUnit.Bytes); try { _progress?.UploadProgress.SetTotal(streamLength); if (streamSize > MaxUploadArchiveSize) { var partSize = GetPartSize(streamLength); _progress?.UploadProgress.ChangeType(UploadType.Chunked); var initiateResponse = await _client.InitiateMultipartUploadAsync(new InitiateMultipartUploadRequest { ArchiveDescription = archiveDescription, VaultName = _vaultName, AccountId = "-", PartSize = partSize }, _cancellationToken); var partChecksums = new List <string>(); var currentPosition = 0L; while (stream.Position < streamLength) { var partStream = GlacierUtils.CreatePartStream(stream, partSize); var partChecksum = TreeHashGenerator.CalculateTreeHash(partStream); partChecksums.Add(partChecksum); var uploadRequest = new UploadMultipartPartRequest { UploadId = initiateResponse.UploadId, VaultName = _vaultName, AccountId = "-", Body = partStream, StreamTransferProgress = (_, args) => { _progress?.UploadProgress.ChangeState(UploadState.Uploading); _progress?.UploadProgress.UpdateUploaded(args.IncrementTransferred); _progress?.OnUploadProgress?.Invoke(); }, Checksum = partChecksum }; uploadRequest.SetRange(currentPosition, currentPosition + partStream.Length - 1); await _client.UploadMultipartPartAsync(uploadRequest, _cancellationToken); currentPosition += partStream.Length; } var completeResponse = await _client.CompleteMultipartUploadAsync(new CompleteMultipartUploadRequest { AccountId = "-", VaultName = _vaultName, ArchiveSize = streamLength.ToString(), UploadId = initiateResponse.UploadId, Checksum = TreeHashGenerator.CalculateTreeHash(partChecksums) }, _cancellationToken); return(completeResponse.ArchiveId); } var response = await _client.UploadArchiveAsync(new UploadArchiveRequest { AccountId = "-", ArchiveDescription = archiveDescription, Body = stream, VaultName = _vaultName, StreamTransferProgress = (_, args) => { _progress?.UploadProgress.ChangeState(UploadState.Uploading); _progress?.UploadProgress.UpdateUploaded(args.IncrementTransferred); _progress?.OnUploadProgress?.Invoke(); }, Checksum = TreeHashGenerator.CalculateTreeHash(stream) }, _cancellationToken); return(response.ArchiveId); } finally { _progress?.UploadProgress.ChangeState(UploadState.Done); } }
private static void HandleUpload(string[] args) { var region = RegionEndpoint.EnumerableAllRegions.SingleOrDefault(reg => reg.SystemName == args[2]); if (args.Length < 6 || region == null) { Console.WriteLine($"args should be aws_key aws_secret region vault_name description filename"); return; } var aws_key = args[0]; var aws_secret = args[1]; var vault_name = args[3]; var description = args[4]; var filename = args[5]; var creds = new BasicAWSCredentials(aws_key, aws_secret); var config = new AmazonGlacierConfig { RegionEndpoint = region, Timeout = TimeSpan.FromDays(10) }; var client = new AmazonGlacierClient(creds, config); var initReq = new InitiateMultipartUploadRequest(vault_name, description, PartSize); var ts = new CancellationTokenSource(); var completed = 0; var started = 0; try { var res = client.InitiateMultipartUploadAsync(initReq, ts.Token).Result; var promises = new List <Task <UploadMultipartPartResponse> >(); Task <UploadMultipartPartResponse> lastPart = null; var sem = new SemaphoreSlim(ConcurrencyLimit); long totalSize = 0; int totalParts = 0; using (var fs = new FileStream(filename, FileMode.Open)) { totalSize = fs.Length; Console.WriteLine($"Preparing to upload {ByteSize.FromBytes(totalSize)}"); totalParts = (int)(fs.Length / PartSize) + 1; bool noErrors = true; while (noErrors) { sem.Wait(); var arr = new byte[PartSize]; var start = fs.Position; var read = fs.Read(arr, 0, (int)PartSize); var check = TreeHasher.ComputeArrayHashString(arr, read); var partReq = new UploadMultipartPartRequest(vault_name, res.UploadId, check, $"bytes {start}-{start + read - 1}/*", new MemoryStream(arr, 0, read)); var promise = client.UploadMultipartPartAsync(partReq, ts.Token); Interlocked.Increment(ref started); Console.WriteLine($"Started {started} out of {totalParts}"); promise.ContinueWith(tsk => { if (tsk.IsFaulted) { Console.WriteLine($"Exception encountered: {tsk.Exception.ToString()}"); noErrors = false; throw tsk.Exception; } Interlocked.Increment(ref completed); Console.WriteLine($"{completed} out of {totalParts} completed."); sem.Release(); }); promises.Add(promise); if (read < PartSize || fs.Position >= fs.Length - 1) { lastPart = promise; break; } } } Task.WaitAll(promises.ToArray()); using (var fs = new FileStream(filename, FileMode.Open)) { var check = TreeHasher.ComputeHashString(fs); var finisher = new CompleteMultipartUploadRequest(vault_name, res.UploadId, totalSize.ToString(), check); Console.WriteLine("Finishing up"); Console.WriteLine($"Computed checksum {check}"); var result = client.CompleteMultipartUploadAsync(finisher, ts.Token).Result; Console.WriteLine($"Completed: {result.Checksum}"); Console.WriteLine($"Calculated: {check}"); var match = string.Equals(result.Checksum, check, StringComparison.InvariantCultureIgnoreCase) ? "" : "not "; Console.WriteLine($"Checksums do {match}match."); Console.WriteLine($"Archive ID: {result.ArchiveId} Location: {result.Location}"); } } catch (Exception ex) { Console.WriteLine($"Exception thrown: {ex.GetType().Name} - {ex.Message}"); Console.WriteLine($"Full exception: {ex.ToString()}"); } }