public IDisposable SetUploadProgress([CanBeNull] IProgress <AsyncProgressEntry> progress, long?suggestedTotal = null, Action callback = null) { var s = Stopwatch.StartNew(); void Handler(object sender, UploadProgressChangedEventArgs args) { if (s.Elapsed.TotalMilliseconds < 20) { return; } callback?.Invoke(); s.Restart(); var total = args.TotalBytesToSend; if (total == -1 && suggestedTotal != null && suggestedTotal > 0) { total = Math.Max(suggestedTotal.Value, args.BytesSent); } progress?.Report(AsyncProgressEntry.CreateUploading(args.BytesSent, total)); } UploadProgressChanged += Handler; return(new ActionAsDisposable(() => { UploadProgressChanged -= Handler; })); }
public static async Task <T> PostMultipart <T>(string url, object metadata, string authToken, byte[] data, string contentType, IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default(CancellationToken)) { try { const string boundary = "--fdfmkj4ixeyfzuxr6q3yp66ry53lerk98g33ow29e0khjjor"; var prefix = Encoding.UTF8.GetBytes(boundary + "\nContent-Type: application/json; charset=UTF-8\n\n" + JsonConvert.SerializeObject(metadata) + "\n\n" + boundary + "\nContent-Type: " + contentType + "\n\n"); var postfix = Encoding.UTF8.GetBytes("\n" + boundary + "--"); var total = prefix.Length + data.Length + postfix.Length; var request = (HttpWebRequest)WebRequest.Create(url); request.Method = "POST"; request.UserAgent = InternalUtils.GetKunosUserAgent(); request.ContentType = "multipart/related; boundary=" + boundary.Substring(2); request.ContentLength = total; request.Headers["Authorization"] = "Bearer " + authToken; using (var stream = await request.GetRequestStreamAsync()) { if (cancellation.IsCancellationRequested) { return(default(T)); } progress?.Report(AsyncProgressEntry.CreateUploading(0, total)); await stream.WriteAsync(prefix, 0, prefix.Length, cancellation); if (cancellation.IsCancellationRequested) { return(default(T)); } const int blockSize = 10240; for (var i = 0; i < data.Length; i += blockSize) { progress?.Report(AsyncProgressEntry.CreateUploading(prefix.Length + i, total)); await stream.WriteAsync(data, i, Math.Min(blockSize, data.Length - i), cancellation); if (cancellation.IsCancellationRequested) { return(default(T)); } } progress?.Report(AsyncProgressEntry.CreateUploading(prefix.Length + data.Length, total)); await stream.WriteAsync(postfix, 0, postfix.Length, cancellation); if (cancellation.IsCancellationRequested) { return(default(T)); } } string result; using (var response = (HttpWebResponse)await request.GetResponseAsync()) using (var stream = response.GetResponseStream()) { if (cancellation.IsCancellationRequested) { return(default(T)); } if (stream == null) { return(default(T)); } using (var reader = new StreamReader(stream, Encoding.UTF8)) { result = await reader.ReadToEndAsync(); if (cancellation.IsCancellationRequested) { return(default(T)); } } } // Logging.Write("Upload result: " + result); return(JsonConvert.DeserializeObject <T>(result)); } catch (WebException e) { using (var stream = e.Response.GetResponseStream()) { Logging.Warning("Prepare() error: " + e); if (stream != null) { Logging.Warning("Prepare() response: " + new StreamReader(stream, Encoding.UTF8).ReadToEnd()); } } return(default(T)); } catch (Exception e) { Logging.Warning("Prepare() error: " + e); return(default(T)); } }
public async Task <string> PostMultipart(string url, object metadata, string authToken, [NotNull] byte[] data, string contentType, IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default(CancellationToken), NameValueCollection extraHeaders = null) { try { const string boundary = "--fdfmkj4ixeyfzuxr6q3yp66ry53lerk98g33ow29e0khjjor"; var prefix = Encoding.UTF8.GetBytes(boundary + "\nContent-Type: application/json; charset=UTF-8\n\n" + JsonConvert.SerializeObject(metadata) + "\n\n" + boundary + "\nContent-Type: " + contentType + "\n\n"); var postfix = Encoding.UTF8.GetBytes("\n" + boundary + "--"); var total = prefix.Length + data.Length + postfix.Length; var request = (HttpWebRequest)WebRequest.Create(url); request.Method = "POST"; request.UserAgent = InternalUtils.GetKunosUserAgent(); request.ContentType = "multipart/related; boundary=" + boundary.Substring(2); request.ContentLength = total; request.Headers["Authorization"] = "Bearer " + authToken; if (extraHeaders != null) { foreach (string header in extraHeaders) { request.Headers[header] = extraHeaders[header]; } } var stopwatch = new AsyncProgressBytesStopwatch(); using (var stream = await request.GetRequestStreamAsync()) { if (cancellation.IsCancellationRequested) { return(null); } progress?.Report(AsyncProgressEntry.CreateUploading(0, total, stopwatch)); await stream.WriteAsync(prefix, 0, prefix.Length, cancellation); if (cancellation.IsCancellationRequested) { return(null); } const int blockSize = 10240; for (var i = 0; i < data.Length; i += blockSize) { progress?.Report(AsyncProgressEntry.CreateUploading(prefix.Length + i, total, stopwatch)); await stream.WriteAsync(data, i, Math.Min(blockSize, data.Length - i), cancellation); if (cancellation.IsCancellationRequested) { return(null); } } progress?.Report(AsyncProgressEntry.CreateUploading(prefix.Length + data.Length, total, stopwatch)); await stream.WriteAsync(postfix, 0, postfix.Length, cancellation); if (cancellation.IsCancellationRequested) { return(null); } } string result; using (var response = (HttpWebResponse)await request.GetResponseAsync()) using (var stream = response.GetResponseStream()) { if (cancellation.IsCancellationRequested) { return(null); } if (stream == null) { return(null); } using (var reader = new StreamReader(stream, Encoding.UTF8)) { result = await reader.ReadToEndAsync(); if (cancellation.IsCancellationRequested) { return(null); } } } return(result); } catch (Exception e) { var wrapped = ApiException.Wrap(e, cancellation); if (wrapped == null) { throw; } throw wrapped; } }
public async Task <WorkshopUploadResult> UploadAsync(byte[] data, string group, string name, IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default) { progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Authorizing…")); await Authorize().WithCancellation(cancellation).ConfigureAwait(false); cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Finding a vault to upload…")); var uploadUrl = await _b2Client.Files.GetUploadUrl(cancelToken : cancellation); cancellation.ThrowIfCancellationRequested(); for (var i = 0; i < 4; ++i) { progress?.Report(AsyncProgressEntry.FromStringIndetermitate(i == 0 ? "Starting upload…" : $"Trying again, {(i + 1).ToOrdinal("attempt").ToSentenceMember()} attempt")); try { return(await TryToUploadAsync(uploadUrl)); } catch (B2Exception e) when(e.Code == "bad_auth_token" || e.ShouldRetryRequest) { cancellation.ThrowIfCancellationRequested(); uploadUrl = await _b2Client.Files.GetUploadUrl(cancelToken : cancellation); } catch (HttpRequestException e) { Logging.Warning(e); cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Target vault is not available, waiting a bit before the next attempt…")); await Task.Delay(TimeSpan.FromSeconds(i + 1d)); cancellation.ThrowIfCancellationRequested(); } catch (WebException e) { Logging.Warning(e); cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Target vault is not available, waiting a bit before the next attempt…")); await Task.Delay(TimeSpan.FromSeconds(i + 1d)); cancellation.ThrowIfCancellationRequested(); } catch (B2Exception e) when(e.Status == "500" || e.Status == "503") { Logging.Warning(e); cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Target vault is not full, waiting a bit before the next attempt…")); await Task.Delay(TimeSpan.FromSeconds(i + 1d)); cancellation.ThrowIfCancellationRequested(); } catch (B2Exception e) { Logging.Warning("B2Exception.Code=" + e.Code); Logging.Warning("B2Exception.Status=" + e.Status); Logging.Warning("B2Exception.Message=" + e.Message); Logging.Warning("B2Exception.ShouldRetryRequest=" + e.ShouldRetryRequest); throw; } } cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate($"Trying again, last attempt")); return(await TryToUploadAsync(uploadUrl)); async Task <WorkshopUploadResult> TryToUploadAsync(B2UploadUrl url) { var fileName = _b2ClientPrefix + group + "/" + name; var stopwatch = new AsyncProgressBytesStopwatch(); var file = await _b2Client.Files.Upload(data, fileName, url, "", "", new Dictionary <string, string> { ["b2-content-disposition"] = Regex.IsMatch(name, @"\.(png|jpg)$") ? "inline" : "attachment", ["b2-cache-control"] = "immutable" }, progress == null?null : new Progress <long>(x => progress.Report(AsyncProgressEntry.CreateUploading(x, data.Length, stopwatch))), cancellation).ConfigureAwait(false); return(new WorkshopUploadResult { Tag = JsonConvert.SerializeObject(new { fileName = file.FileName, fileID = file.FileId }), Size = data.LongLength }); } }
public async Task <WorkshopUploadResult> UploadAsync(byte[] data, string group, string name, IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default) { for (var i = 0; i < 3; ++i) { progress?.Report(AsyncProgressEntry.FromStringIndetermitate(i == 0 ? "Starting upload…" : $"Trying again, {(i + 1).ToOrdinal("attempt").ToSentenceMember()} attempt")); try { return(await TryToUploadAsync()); } catch (HttpRequestException e) { Logging.Warning(e); cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Upload is failed, waiting a bit before the next attempt…")); await Task.Delay(TimeSpan.FromSeconds(i + 1d)); cancellation.ThrowIfCancellationRequested(); } catch (WebException e) { Logging.Warning(e); cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Upload is failed, waiting a bit before the next attempt…")); await Task.Delay(TimeSpan.FromSeconds(i + 1d)); cancellation.ThrowIfCancellationRequested(); } } cancellation.ThrowIfCancellationRequested(); progress?.Report(AsyncProgressEntry.FromStringIndetermitate($"Trying again, last attempt")); return(await TryToUploadAsync()); async Task <WorkshopUploadResult> TryToUploadAsync() { var request = new HttpRequestMessage(HttpMethod.Post, _endpoint); request.Headers.TryAddWithoutValidation("X-Data-File-Group", group); request.Headers.TryAddWithoutValidation("X-Data-File-Name", name); request.Headers.TryAddWithoutValidation("X-Data-Checksum", _checksum); var stopwatch = new AsyncProgressBytesStopwatch(); request.Content = progress == null ? (HttpContent) new ByteArrayContent(data) : new ProgressableByteArrayContent(data, 8192, new Progress <long>(x => progress.Report(AsyncProgressEntry.CreateUploading(x, data.Length, stopwatch)))); using (var response = await HttpClientHolder.Get().SendAsync(request, cancellation).ConfigureAwait(false)) { if (response.StatusCode != HttpStatusCode.OK) { throw new WebException($"Failed to upload: {response.StatusCode}, response: {await LoadContent()}"); } var result = JObject.Parse(await LoadContent()); return(new WorkshopUploadResult { Size = data.Length, Tag = result["key"].ToString() }); ConfiguredTaskAwaitable <string> LoadContent() { return(response.Content.ReadAsStringAsync().WithCancellation(cancellation).ConfigureAwait(false)); } } } }