public static async Task <string> LoadAsyncTo(string argument,
FlexibleLoaderGetPreferredDestinationCallback getPreferredDestination, [CanBeNull] FlexibleLoaderReportDestinationCallback reportDestination,
Action <FlexibleLoaderMetaInformation> reportMetaInformation = null, Func <bool> checkIfPaused = null,
IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default)
{
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Finding fitting loader…"));
var loader = await CreateLoaderAsync(argument, cancellation) ?? throw new OperationCanceledException();
try {
using (var order = KillerOrder.Create(new CookieAwareWebClient(), TimeSpan.FromMinutes(10))) {
var client = order.Victim;
if (_proxy != null)
{
client.Proxy = _proxy;
}
progress?.Report(AsyncProgressEntry.Indetermitate);
cancellation.ThrowIfCancellationRequested();
cancellation.Register(client.CancelAsync);
if (!await loader.PrepareAsync(client, cancellation))
{
throw new InformativeException("Can’t load file", "Loader preparation failed.");
}
cancellation.ThrowIfCancellationRequested();
reportMetaInformation?.Invoke(FlexibleLoaderMetaInformation.FromLoader(loader));
var initialProgressCallback = true;
var reportStopwatch = Stopwatch.StartNew();
var progressStopwatch = new AsyncProgressBytesStopwatch();
if (loader.UsesClientToDownload)
{
client.DownloadProgressChanged += (sender, args) => {
if (initialProgressCallback)
{
reportMetaInformation?.Invoke(FlexibleLoaderMetaInformation.FromLoader(loader));
initialProgressCallback = false;
}
if (reportStopwatch.Elapsed.TotalMilliseconds < 20)
{
return;
}
order.Delay();
reportStopwatch.Restart();
progress?.Report(AsyncProgressEntry.CreateDownloading(args.BytesReceived, args.TotalBytesToReceive == -1 &&
loader.TotalSize.HasValue ? Math.Max(loader.TotalSize.Value, args.BytesReceived) : args.TotalBytesToReceive,
progressStopwatch));
};
}
var loaded = await loader.DownloadAsync(client, getPreferredDestination, reportDestination, checkIfPaused,
loader.UsesClientToDownload?null : new Progress <long>(p => {
if (initialProgressCallback)
{
reportMetaInformation?.Invoke(FlexibleLoaderMetaInformation.FromLoader(loader));
initialProgressCallback = false;
}
if (reportStopwatch.Elapsed.TotalMilliseconds < 20)
{
return;
}
order.Delay();
reportStopwatch.Restart();
progress?.Report(loader.TotalSize.HasValue ? AsyncProgressEntry.CreateDownloading(p, loader.TotalSize.Value, progressStopwatch)
: new AsyncProgressEntry(string.Format(UiStrings.Progress_Downloading, p.ToReadableSize(1)), null));
}), cancellation);
cancellation.ThrowIfCancellationRequested();
Logging.Write("Loaded: " + loaded);
return(loaded);
}
} catch (Exception e) when(cancellation.IsCancellationRequested || e.IsCancelled())
{
Logging.Warning("Cancelled");
throw new OperationCanceledException();
} catch (Exception e) {
Logging.Warning(e);
throw;
}
}
public async Task <WorkshopUploadResult> UploadAsync(byte[] data, string group, string name,
IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default)
{
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Authorizing…"));
await Authorize().WithCancellation(cancellation).ConfigureAwait(false);
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Finding a vault to upload…"));
var uploadUrl = await _b2Client.Files.GetUploadUrl(cancelToken : cancellation);
cancellation.ThrowIfCancellationRequested();
for (var i = 0; i < 4; ++i)
{
progress?.Report(AsyncProgressEntry.FromStringIndetermitate(i == 0
? "Starting upload…"
: $"Trying again, {(i + 1).ToOrdinal("attempt").ToSentenceMember()} attempt"));
try {
return(await TryToUploadAsync(uploadUrl));
} catch (B2Exception e) when(e.Code == "bad_auth_token" || e.ShouldRetryRequest)
{
cancellation.ThrowIfCancellationRequested();
uploadUrl = await _b2Client.Files.GetUploadUrl(cancelToken : cancellation);
} catch (HttpRequestException e) {
Logging.Warning(e);
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Target vault is not available, waiting a bit before the next attempt…"));
await Task.Delay(TimeSpan.FromSeconds(i + 1d));
cancellation.ThrowIfCancellationRequested();
} catch (WebException e) {
Logging.Warning(e);
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Target vault is not available, waiting a bit before the next attempt…"));
await Task.Delay(TimeSpan.FromSeconds(i + 1d));
cancellation.ThrowIfCancellationRequested();
} catch (B2Exception e) when(e.Status == "500" || e.Status == "503")
{
Logging.Warning(e);
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Target vault is not full, waiting a bit before the next attempt…"));
await Task.Delay(TimeSpan.FromSeconds(i + 1d));
cancellation.ThrowIfCancellationRequested();
} catch (B2Exception e) {
Logging.Warning("B2Exception.Code=" + e.Code);
Logging.Warning("B2Exception.Status=" + e.Status);
Logging.Warning("B2Exception.Message=" + e.Message);
Logging.Warning("B2Exception.ShouldRetryRequest=" + e.ShouldRetryRequest);
throw;
}
}
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate($"Trying again, last attempt"));
return(await TryToUploadAsync(uploadUrl));
async Task <WorkshopUploadResult> TryToUploadAsync(B2UploadUrl url)
{
var fileName = _b2ClientPrefix + group + "/" + name;
var stopwatch = new AsyncProgressBytesStopwatch();
var file = await _b2Client.Files.Upload(data, fileName, url, "", "", new Dictionary <string, string> {
["b2-content-disposition"] = Regex.IsMatch(name, @"\.(png|jpg)$") ? "inline" : "attachment",
["b2-cache-control"] = "immutable"
}, progress == null?null : new Progress <long>(x => progress.Report(AsyncProgressEntry.CreateUploading(x, data.Length, stopwatch))),
cancellation).ConfigureAwait(false);
return(new WorkshopUploadResult {
Tag = JsonConvert.SerializeObject(new { fileName = file.FileName, fileID = file.FileId }),
Size = data.LongLength
});
}
}
public async Task <string> PostMultipart(string url, object metadata, string authToken, [NotNull] byte[] data, string contentType,
IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default(CancellationToken),
NameValueCollection extraHeaders = null)
{
try {
const string boundary = "--fdfmkj4ixeyfzuxr6q3yp66ry53lerk98g33ow29e0khjjor";
var prefix = Encoding.UTF8.GetBytes(boundary + "\nContent-Type: application/json; charset=UTF-8\n\n" +
JsonConvert.SerializeObject(metadata) + "\n\n" + boundary + "\nContent-Type: " + contentType + "\n\n");
var postfix = Encoding.UTF8.GetBytes("\n" + boundary + "--");
var total = prefix.Length + data.Length + postfix.Length;
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "POST";
request.UserAgent = InternalUtils.GetKunosUserAgent();
request.ContentType = "multipart/related; boundary=" + boundary.Substring(2);
request.ContentLength = total;
request.Headers["Authorization"] = "Bearer " + authToken;
if (extraHeaders != null)
{
foreach (string header in extraHeaders)
{
request.Headers[header] = extraHeaders[header];
}
}
var stopwatch = new AsyncProgressBytesStopwatch();
using (var stream = await request.GetRequestStreamAsync()) {
if (cancellation.IsCancellationRequested)
{
return(null);
}
progress?.Report(AsyncProgressEntry.CreateUploading(0, total, stopwatch));
await stream.WriteAsync(prefix, 0, prefix.Length, cancellation);
if (cancellation.IsCancellationRequested)
{
return(null);
}
const int blockSize = 10240;
for (var i = 0; i < data.Length; i += blockSize)
{
progress?.Report(AsyncProgressEntry.CreateUploading(prefix.Length + i, total, stopwatch));
await stream.WriteAsync(data, i, Math.Min(blockSize, data.Length - i), cancellation);
if (cancellation.IsCancellationRequested)
{
return(null);
}
}
progress?.Report(AsyncProgressEntry.CreateUploading(prefix.Length + data.Length, total, stopwatch));
await stream.WriteAsync(postfix, 0, postfix.Length, cancellation);
if (cancellation.IsCancellationRequested)
{
return(null);
}
}
string result;
using (var response = (HttpWebResponse)await request.GetResponseAsync())
using (var stream = response.GetResponseStream()) {
if (cancellation.IsCancellationRequested)
{
return(null);
}
if (stream == null)
{
return(null);
}
using (var reader = new StreamReader(stream, Encoding.UTF8)) {
result = await reader.ReadToEndAsync();
if (cancellation.IsCancellationRequested)
{
return(null);
}
}
}
return(result);
} catch (Exception e) {
var wrapped = ApiException.Wrap(e, cancellation);
if (wrapped == null)
{
throw;
}
throw wrapped;
}
}
public async Task <WorkshopUploadResult> UploadAsync(byte[] data, string group, string name,
IProgress <AsyncProgressEntry> progress = null, CancellationToken cancellation = default)
{
for (var i = 0; i < 3; ++i)
{
progress?.Report(AsyncProgressEntry.FromStringIndetermitate(i == 0
? "Starting upload…"
: $"Trying again, {(i + 1).ToOrdinal("attempt").ToSentenceMember()} attempt"));
try {
return(await TryToUploadAsync());
} catch (HttpRequestException e) {
Logging.Warning(e);
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Upload is failed, waiting a bit before the next attempt…"));
await Task.Delay(TimeSpan.FromSeconds(i + 1d));
cancellation.ThrowIfCancellationRequested();
} catch (WebException e) {
Logging.Warning(e);
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate("Upload is failed, waiting a bit before the next attempt…"));
await Task.Delay(TimeSpan.FromSeconds(i + 1d));
cancellation.ThrowIfCancellationRequested();
}
}
cancellation.ThrowIfCancellationRequested();
progress?.Report(AsyncProgressEntry.FromStringIndetermitate($"Trying again, last attempt"));
return(await TryToUploadAsync());
async Task <WorkshopUploadResult> TryToUploadAsync()
{
var request = new HttpRequestMessage(HttpMethod.Post, _endpoint);
request.Headers.TryAddWithoutValidation("X-Data-File-Group", group);
request.Headers.TryAddWithoutValidation("X-Data-File-Name", name);
request.Headers.TryAddWithoutValidation("X-Data-Checksum", _checksum);
var stopwatch = new AsyncProgressBytesStopwatch();
request.Content = progress == null
? (HttpContent) new ByteArrayContent(data)
: new ProgressableByteArrayContent(data, 8192,
new Progress <long>(x => progress.Report(AsyncProgressEntry.CreateUploading(x, data.Length, stopwatch))));
using (var response = await HttpClientHolder.Get().SendAsync(request, cancellation).ConfigureAwait(false)) {
if (response.StatusCode != HttpStatusCode.OK)
{
throw new WebException($"Failed to upload: {response.StatusCode}, response: {await LoadContent()}");
}
var result = JObject.Parse(await LoadContent());
return(new WorkshopUploadResult {
Size = data.Length,
Tag = result["key"].ToString()
});
ConfiguredTaskAwaitable <string> LoadContent()
{
return(response.Content.ReadAsStringAsync().WithCancellation(cancellation).ConfigureAwait(false));
}
}
}
}
