public async Task <HtmlDocumentResponse> UrlGetToHtmlDocument(string request, string referer = null,
ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || maxTries == 0)
{
Logger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
StringResponse response =
await UrlGetToString(request, referer, requestOptions, maxTries).ConfigureAwait(false);
return(response != null ? new HtmlDocumentResponse(response) : null);
}
internal async Task <StringResponse> UrlGetToString(string request, string referer = null,
ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || maxTries == 0)
{
Logger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
StringResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
using HttpResponseMessage response = await InternalGet(request, referer).ConfigureAwait(false);
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new StringResponse(response);
}
break;
}
return(new StringResponse(response, await response.Content.ReadAsStringAsync().ConfigureAwait(false)));
}
if (maxTries > 1)
{
Logger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
Logger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
internal async Task <BinaryResponse> UrlGetToBinaryWithProgress(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
BinaryResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
const byte printPercentage = 10;
const byte maxBatches = 99 / printPercentage;
await using StreamResponse response = await UrlGetToStream(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new BinaryResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
#if !NETFRAMEWORK
await
#endif
using MemoryStream ms = new MemoryStream((int)response.Length);
try {
byte batch = 0;
uint readThisBatch = 0;
byte[] buffer = new byte[8192]; // This is HttpClient's buffer, using more doesn't make sense
while (response.Content.CanRead)
{
int read = await response.Content.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
break;
}
await ms.WriteAsync(buffer, 0, read).ConfigureAwait(false);
if ((response.Length == 0) || (batch >= maxBatches))
{
continue;
}
readThisBatch += (uint)read;
if (readThisBatch < response.Length / printPercentage)
{
continue;
}
readThisBatch -= response.Length / printPercentage;
}
} catch (Exception) {
return(null);
}
return(new BinaryResponse(response, ms.ToArray()));
}
if (maxTries > 1)
{
}
return(result);
}
public async Task <ObjectResponse <TResult> > UrlPostToJsonObject <TResult, TData>(string request, TData data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where TResult : class where TData : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
ObjectResponse <TResult> result = null;
for (byte i = 0; i < maxTries; i++)
{
await using StreamResponse response = await UrlPostToStream(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new ObjectResponse <TResult>(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
TResult obj;
try {
using StreamReader steamReader = new StreamReader(response.Content);
using JsonReader jsonReader = new JsonTextReader(steamReader);
JsonSerializer serializer = new JsonSerializer();
obj = serializer.Deserialize <TResult>(jsonReader);
} catch (Exception) {
continue;
}
return(new ObjectResponse <TResult>(response, obj));
}
return(result);
}
public async Task <HtmlDocumentResponse> UrlPostToHtmlDocument <T>(string request, T data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
HtmlDocumentResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
await using StreamResponse response = await UrlPostToStream(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new HtmlDocumentResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
try {
result = await HtmlDocumentResponse.Create(response).ConfigureAwait(false);
} catch (Exception) {
continue;
}
return(result);
}
return(result);
}
public async Task <ObjectResponse <T> > UrlPostToJsonObject <T>(string request, IReadOnlyCollection <KeyValuePair <string, string> > data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
ObjectResponse <T> result = null;
for (byte i = 0; i < maxTries; i++)
{
StringResponse response = await UrlPostToString(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response == null)
{
return(null);
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new ObjectResponse <T>(response);
}
break;
}
if (string.IsNullOrEmpty(response.Content))
{
continue;
}
T obj;
try {
obj = JsonConvert.DeserializeObject <T>(response.Content);
} catch (JsonException e) {
ArchiLogger.LogGenericWarningException(e);
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, response.Content));
}
continue;
}
return(new ObjectResponse <T>(response, obj));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <BasicResponse> UrlPost(string request, IReadOnlyCollection <KeyValuePair <string, string> > data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
BasicResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
using (HttpResponseMessage response = await InternalPost(request, data, referer).ConfigureAwait(false)) {
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new BasicResponse(response);
}
break;
}
return(new BasicResponse(response));
}
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <ObjectResponse <T> > UrlGetToJsonObject <T>(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
ObjectResponse <T> result = null;
for (byte i = 0; i < maxTries; i++)
{
using StreamResponse response = await UrlGetToStream(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
// ReSharper disable once UseNullPropagationWhenPossible - false check
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new ObjectResponse <T>(response);
}
break;
}
if (response.Content == null)
{
continue;
}
T obj;
try {
using StreamReader streamReader = new StreamReader(response.Content);
using JsonReader jsonReader = new JsonTextReader(streamReader);
JsonSerializer serializer = new JsonSerializer();
obj = serializer.Deserialize <T>(jsonReader);
} catch (JsonException e) {
ArchiLogger.LogGenericWarningException(e);
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, response.Content));
}
continue;
}
return(new ObjectResponse <T>(response, obj));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
private async Task <HttpResponseMessage?> InternalPost <T>(string request, T?data = null, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead) where T : class
{
if (string.IsNullOrEmpty(request))
{
throw new ArgumentNullException(nameof(request));
}
return(await InternalRequest(new Uri(request), HttpMethod.Post, data, referer, requestOptions, httpCompletionOption).ConfigureAwait(false));
}
private async Task <HttpResponseMessage?> InternalHead(string request, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead)
{
if (string.IsNullOrEmpty(request))
{
throw new ArgumentNullException(nameof(request));
}
return(await InternalRequest <object>(new Uri(request), HttpMethod.Head, null, referer, requestOptions, httpCompletionOption).ConfigureAwait(false));
}
internal async Task <BinaryResponse?> UrlGetToBinary(string request, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, IProgress <byte>?progressReporter = null)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
throw new ArgumentNullException(nameof(request) + " || " + nameof(maxTries));
}
BinaryResponse?result = null;
for (byte i = 0; i < maxTries; i++)
{
await using StreamResponse? response = await UrlGetToStream(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new BinaryResponse(response);
}
break;
}
if (response?.StatusCode.IsServerErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors))
{
result = new BinaryResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
progressReporter?.Report(0);
#if NETFRAMEWORK
using MemoryStream ms = new MemoryStream((int)response.Length);
#else
await using MemoryStream ms = new MemoryStream((int)response.Length);
#endif
try {
byte batch = 0;
uint readThisBatch = 0;
uint batchIncreaseSize = response.Length / 100;
byte[] buffer = new byte[8192]; // This is HttpClient's buffer, using more doesn't make sense
while (response.Content.CanRead)
{
int read = await response.Content.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
break;
}
await ms.WriteAsync(buffer, 0, read).ConfigureAwait(false);
if ((batchIncreaseSize == 0) || (batch >= 99))
{
continue;
}
readThisBatch += (uint)read;
if (readThisBatch < batchIncreaseSize)
{
continue;
}
readThisBatch -= batchIncreaseSize;
progressReporter?.Report(++batch);
}
} catch (Exception e) {
ArchiLogger.LogGenericDebuggingException(e);
return(null);
}
progressReporter?.Report(100);
return(new BinaryResponse(response, ms.ToArray()));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <ObjectResponse <TResult>?> UrlPostToJsonObject <TResult, TData>(string request, TData?data = null, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where TResult : class where TData : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
throw new ArgumentNullException(nameof(request) + " || " + nameof(maxTries));
}
ObjectResponse <TResult>?result = null;
for (byte i = 0; i < maxTries; i++)
{
await using StreamResponse? response = await UrlPostToStream(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new ObjectResponse <TResult>(response);
}
break;
}
if (response?.StatusCode.IsServerErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors))
{
result = new ObjectResponse <TResult>(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
TResult?obj;
try {
using StreamReader steamReader = new StreamReader(response.Content);
using JsonReader jsonReader = new JsonTextReader(steamReader);
JsonSerializer serializer = new JsonSerializer();
obj = serializer.Deserialize <TResult>(jsonReader);
if (obj == null)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorIsEmpty, nameof(obj)));
continue;
}
} catch (Exception e) {
ArchiLogger.LogGenericWarningException(e);
continue;
}
return(new ObjectResponse <TResult>(response, obj));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <HtmlDocumentResponse?> UrlPostToHtmlDocument <T>(string request, T?data = null, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
throw new ArgumentNullException(nameof(request) + " || " + nameof(maxTries));
}
HtmlDocumentResponse?result = null;
for (byte i = 0; i < maxTries; i++)
{
await using StreamResponse? response = await UrlPostToStream(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new HtmlDocumentResponse(response);
}
break;
}
if (response?.StatusCode.IsServerErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors))
{
result = new HtmlDocumentResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
try {
result = await HtmlDocumentResponse.Create(response).ConfigureAwait(false);
} catch (Exception e) {
ArchiLogger.LogGenericWarningException(e);
continue;
}
return(result);
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <BasicResponse?> UrlPost <T>(string request, T?data = null, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
throw new ArgumentNullException(nameof(request) + " || " + nameof(maxTries));
}
BasicResponse?result = null;
for (byte i = 0; i < maxTries; i++)
{
using HttpResponseMessage? response = await InternalPost(request, data, referer).ConfigureAwait(false);
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new BasicResponse(response);
}
break;
}
if (response.StatusCode.IsServerErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors))
{
result = new BasicResponse(response);
}
break;
}
return(new BasicResponse(response));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <HtmlDocumentResponse> UrlPostToHtmlDocument(string request, IReadOnlyCollection <KeyValuePair <string, string> > data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
HtmlDocumentResponse result = null;
for (int i = 0; i < maxTries; i++)
{
await using StreamResponse response = await UrlPostToStream(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response == null)
{
return(null);
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new HtmlDocumentResponse(response);
}
break;
}
if (response.Content == null)
{
continue;
}
try {
result = await HtmlDocumentResponse.Create(response).ConfigureAwait(false);
} catch (Exception e) {
ArchiLogger.LogGenericWarningException(e);
continue;
}
return(result);
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
internal async Task <StringResponse> UrlGetToString(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
StringResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
using HttpResponseMessage response = await InternalGet(request, referer).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new StringResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
return(new StringResponse(response, await response.Content.ReadAsStringAsync().ConfigureAwait(false)));
}
return(result);
}
private async Task <StreamResponse> UrlPostToStream <T>(string request, T data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
StreamResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
HttpResponseMessage response = await InternalPost(request, data, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new StreamResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
return(new StreamResponse(response, await response.Content.ReadAsStreamAsync().ConfigureAwait(false)));
}
return(result);
}
private async Task <HttpResponseMessage?> InternalRequest <T>(Uri requestUri, HttpMethod httpMethod, T?data = null, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, byte maxRedirections = MaxTries) where T : class
{
if ((requestUri == null) || (httpMethod == null))
{
throw new ArgumentNullException(nameof(requestUri) + " || " + nameof(httpMethod));
}
HttpResponseMessage response;
using (HttpRequestMessage request = new HttpRequestMessage(httpMethod, requestUri)) {
#if !NETFRAMEWORK
request.Version = HttpClient.DefaultRequestVersion;
#endif
if (data != null)
{
switch (data)
{
case HttpContent content:
request.Content = content;
break;
case IReadOnlyCollection <KeyValuePair <string, string> > dictionary:
try {
request.Content = new FormUrlEncodedContent(dictionary);
} catch (UriFormatException) {
request.Content = new StringContent(string.Join("&", dictionary.Select(kv => WebUtility.UrlEncode(kv.Key) + "=" + WebUtility.UrlEncode(kv.Value))), null, "application/x-www-form-urlencoded");
}
break;
case string text:
request.Content = new StringContent(text);
break;
default:
request.Content = new StringContent(JsonConvert.SerializeObject(data), Encoding.UTF8, "application/json");
break;
}
}
if (!string.IsNullOrEmpty(referer))
{
request.Headers.Referrer = new Uri(referer !);
}
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(httpMethod + " " + requestUri);
}
try {
response = await HttpClient.SendAsync(request, httpCompletionOption).ConfigureAwait(false);
} catch (Exception e) {
ArchiLogger.LogGenericDebuggingException(e);
return(null);
}
}
if (response == null)
{
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug("null <- " + httpMethod + " " + requestUri);
}
return(null);
}
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(response.StatusCode + " <- " + httpMethod + " " + requestUri);
}
if (response.IsSuccessStatusCode)
{
return(response);
}
// WARNING: We still have not disposed response by now, make sure to dispose it ASAP if we're not returning it!
if ((response.StatusCode >= HttpStatusCode.Ambiguous) && (response.StatusCode < HttpStatusCode.BadRequest) && (maxRedirections > 0))
{
Uri redirectUri = response.Headers.Location;
if (redirectUri.IsAbsoluteUri)
{
switch (redirectUri.Scheme)
{
case "http":
case "https":
break;
case "steammobile":
// Those redirections are invalid, but we're aware of that and we have extra logic for them
return(response);
default:
// We have no clue about those, but maybe HttpClient can handle them for us
ArchiLogger.LogGenericError(string.Format(Strings.WarningUnknownValuePleaseReport, nameof(redirectUri.Scheme), redirectUri.Scheme));
break;
}
}
else
{
redirectUri = new Uri(requestUri, redirectUri);
}
switch (response.StatusCode)
{
case HttpStatusCode.SeeOther:
// Per https://tools.ietf.org/html/rfc7231#section-6.4.4, a 303 redirect should be performed with a GET request
httpMethod = HttpMethod.Get;
// Data doesn't make any sense for a GET request, clear it
data = null;
break;
}
response.Dispose();
// Per https://tools.ietf.org/html/rfc7231#section-7.1.2, a redirect location without a fragment should inherit the fragment from the original URI
if (!string.IsNullOrEmpty(requestUri.Fragment) && string.IsNullOrEmpty(redirectUri.Fragment))
{
redirectUri = new UriBuilder(redirectUri)
{
Fragment = requestUri.Fragment
}.Uri;
}
return(await InternalRequest(redirectUri, httpMethod, data, referer, requestOptions, httpCompletionOption, --maxRedirections).ConfigureAwait(false));
}
if (!Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(response.StatusCode + " <- " + httpMethod + " " + requestUri);
}
if (response.StatusCode.IsClientErrorCode())
{
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, await response.Content.ReadAsStringAsync().ConfigureAwait(false)));
}
// Do not retry on client errors
return(response);
}
if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors) && response.StatusCode.IsServerErrorCode())
{
// Do not retry on server errors in this case
return(response);
}
using (response) {
if (Debugging.IsUserDebugging)
{
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, await response.Content.ReadAsStringAsync().ConfigureAwait(false)));
}
return(null);
}
}
public async Task <XmlDocumentResponse> UrlGetToXmlDocument(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
XmlDocumentResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
StringResponse response = await UrlGetToString(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
// ReSharper disable once UseNullPropagationWhenPossible - false check
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new XmlDocumentResponse(response);
}
break;
}
if (string.IsNullOrEmpty(response.Content))
{
continue;
}
XmlDocument xmlDocument = new XmlDocument();
try {
xmlDocument.LoadXml(response.Content);
} catch (XmlException e) {
ArchiLogger.LogGenericWarningException(e);
continue;
}
return(new XmlDocumentResponse(response, xmlDocument));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
private async Task <StreamResponse?> UrlPostToStream <T>(string request, T?data = null, string?referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
throw new ArgumentNullException(nameof(request) + " || " + nameof(maxTries));
}
StreamResponse?result = null;
for (byte i = 0; i < maxTries; i++)
{
HttpResponseMessage?response = await InternalPost(request, data, referer, requestOptions, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new StreamResponse(response);
}
break;
}
if (response?.StatusCode.IsServerErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors))
{
result = new StreamResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
return(new StreamResponse(response, await response.Content.ReadAsStreamAsync().ConfigureAwait(false)));
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <HtmlDocumentResponse> UrlPostToHtmlDocument(string request, IReadOnlyCollection <KeyValuePair <string, string> > data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
StringResponse response = await UrlPostToString(request, data, referer, requestOptions, maxTries).ConfigureAwait(false);
return(response != null ? new HtmlDocumentResponse(response) : null);
}
public async Task <XmlDocumentResponse> UrlGetToXmlDocument(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
XmlDocumentResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
await using StreamResponse response = await UrlGetToStream(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false);
if (response?.StatusCode.IsClientErrorCode() == true)
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new XmlDocumentResponse(response);
}
break;
}
if (response?.Content == null)
{
continue;
}
XmlDocument xmlDocument = new XmlDocument();
try {
xmlDocument.Load(response.Content);
} catch (XmlException) {
continue;
}
return(new XmlDocumentResponse(response, xmlDocument));
}
return(result);
}
internal async Task <BinaryResponse> UrlGetToBinaryWithProgress(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries)
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries));
return(null);
}
BinaryResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
const byte printPercentage = 10;
const byte maxBatches = 99 / printPercentage;
using (HttpResponseMessage response = await InternalGet(request, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false)) {
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new BinaryResponse(response);
}
break;
}
ArchiLogger.LogGenericDebug("0%...");
uint contentLength = (uint)response.Content.Headers.ContentLength.GetValueOrDefault();
using (MemoryStream ms = new MemoryStream((int)contentLength)) {
try {
using (Stream contentStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) {
byte batch = 0;
uint readThisBatch = 0;
byte[] buffer = new byte[8192]; // This is HttpClient's buffer, using more doesn't make sense
while (contentStream.CanRead)
{
int read = await contentStream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
break;
}
await ms.WriteAsync(buffer, 0, read).ConfigureAwait(false);
if ((contentLength == 0) || (batch >= maxBatches))
{
continue;
}
readThisBatch += (uint)read;
if (readThisBatch < contentLength / printPercentage)
{
continue;
}
readThisBatch -= contentLength / printPercentage;
ArchiLogger.LogGenericDebug((++batch * printPercentage) + "%...");
}
}
} catch (Exception e) {
ArchiLogger.LogGenericDebuggingException(e);
return(null);
}
ArchiLogger.LogGenericDebug("100%");
return(new BinaryResponse(response, ms.ToArray()));
}
}
}
if (maxTries > 1)
{
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
}
return(result);
}
public async Task <BasicResponse> UrlPost <T>(string request, T data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class
{
if (string.IsNullOrEmpty(request) || (maxTries == 0))
{
return(null);
}
BasicResponse result = null;
for (byte i = 0; i < maxTries; i++)
{
using HttpResponseMessage response = await InternalPost(request, data, referer).ConfigureAwait(false);
if (response == null)
{
continue;
}
if (response.StatusCode.IsClientErrorCode())
{
if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors))
{
result = new BasicResponse(response);
}
break;
}
return(new BasicResponse(response));
}
return(result);
}
