public LinkPreviewResponse(LinkPreviewRequest request)
{
this.Error = request.Error;
this.Result = request.Result;
this.Redirects = new List <RedirectInfo>();
if (request.Redirects?.Any() ?? false)
{
var urls = request.Redirects.Keys.ToList();
for (int i = 0; i < urls.Count; i++)
{
if (i == 0)
{
this.Redirects.Add(new RedirectInfo()
{
RequestedUrl = request.OriginalUrl,
RedirectUrl = new Uri(urls[i])
});
}
else if (i > 0)
{
this.Redirects.Add(new RedirectInfo()
{
RequestedUrl = new Uri(urls[i - 1]),
RedirectUrl = new Uri(urls[i])
});
}
}
}
}
private async Task <LinkPreviewRequest> HandleFacebookExitLink(LinkPreviewRequest previewRequest)
{
var correctLink = previewRequest.CurrentRequestedUrl.TryGetLinkFromFacebookExitLink();
if (correctLink != null)
{
previewRequest.CurrentRequestedUrl = correctLink;
}
return(await GetLinkDataAsync(previewRequest, false));
}
private string TryExtractCookieValueFromLastResponse(LinkPreviewRequest previewRequest)
{
HttpResponseMessage cookieContainingResponse = null;
var lastResponse = previewRequest.Redirects.LastOrDefault();
cookieContainingResponse = lastResponse.Value == null ? previewRequest.OriginalResponse : lastResponse.Value;
var cookies = cookieContainingResponse.Headers.SingleOrDefault(header => header.Key == "Set-Cookie");
var cookieValues = cookies.Value.Select(cookie => cookie.Substring(0, cookie.IndexOf(";") + 1)).ToList();
StringBuilder cookieValueStringBuilder = new StringBuilder();
foreach (var value in cookieValues)
{
cookieValueStringBuilder.Append($"{value} ");
}
return(cookieValueStringBuilder.ToString().Trim());
}
private async Task TryGetLinkDataFrom302Redirects(LinkPreviewRequest previewRequest)
{
if (previewRequest.OriginalResponse.StatusCode == HttpStatusCode.Found)
{
var linkPreview = await TryGetLinkPreview(previewRequest.OriginalResponse);
previewRequest.Result = linkPreview;
}
else if (previewRequest.Redirects.Values.Any(r => r.StatusCode == HttpStatusCode.Found))
{
var linkPreviewTasks = new List <Task <LinkInfo> >();
foreach (var response in previewRequest.Redirects.Values.Where(r => r.StatusCode == HttpStatusCode.Found))
{
linkPreviewTasks.Add(TryGetLinkPreview(response));
}
var linkPreviews = await Task.WhenAll(linkPreviewTasks).ConfigureAwait(false);
var linkWithTitleAndImage = linkPreviews.FirstOrDefault(p => !string.IsNullOrEmpty(p.Title) && p.ImageUrl != null);
previewRequest.Result = linkWithTitleAndImage ?? linkPreviews.FirstOrDefault(p => !string.IsNullOrEmpty(p.Title));
}
}
public async Task <LinkPreviewRequest> GetLinkDataAsync(LinkPreviewRequest previewRequest, bool isCircleRedirect = false, bool retryWithoutCompressionOnFailure = true, bool noCompression = false, bool addCookieToRedirectedRequest = false)
{
try
{
var currentRequestedUrlString = previewRequest.CurrentRequestedUrl.ToString();
string cookieHeaderValue = null;
if (addCookieToRedirectedRequest)
{
cookieHeaderValue = TryExtractCookieValueFromLastResponse(previewRequest);
}
if (currentRequestedUrlString.Contains("facebook.com") &&
previewRequest.CurrentRequestedUrl.ContainsParameter("u"))
{
return(await HandleFacebookExitLink(previewRequest));
}
//todo: add other special cases (twitch?, youtube?) here...
else
{
if (!isCircleRedirect)
{
Console.WriteLine($"sending request for url {previewRequest.CurrentRequestedUrl}");
var request = new HttpRequestMessage(currentRequestedUrlString.IsHttps() ? HttpMethod.Get : HttpMethod.Head, previewRequest.CurrentRequestedUrl);
request.Headers.Host = previewRequest.CurrentRequestedUrl.Host;
if (!string.IsNullOrWhiteSpace(cookieHeaderValue))
{
request.Headers.Add("Cookie", cookieHeaderValue);
}
HttpCompletionOption completionOption = HttpCompletionOption.ResponseHeadersRead;
var response = noCompression ?
await _httpClientInstance.SendAsync(request, completionOption) :
await TryGetResponseMessageWithoutCompressionAsync(request, completionOption);
if (previewRequest.OriginalResponse == null)
{
previewRequest.OriginalResponse = response;
}
else
{
previewRequest.Redirects.Add(previewRequest.CurrentRequestedUrl.ToString(), response);
}
var statusCode = (int)response.StatusCode;
Console.WriteLine($"received response {statusCode} from url {request.RequestUri}");
if (statusCode >= 300 && statusCode <= 399)
{
return(await HandleRedirect(response, previewRequest));
}
else if (statusCode >= 400)
{
var message = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
previewRequest.Error = new RequestError(statusCode, message);
Console.WriteLine($"got error response ({statusCode}) from {previewRequest.CurrentRequestedUrl}\nmessage: {message}");
}
else
{
var linkPreview = await TryGetLinkPreview(response);
previewRequest.Result = linkPreview;
}
}
else
{
await TryGetLinkDataFrom302Redirects(previewRequest).ConfigureAwait(false);
}
return(previewRequest);
}
}
catch (Exception ex)
{
if (ex is HttpRequestException requestException)
{
//socket exceptions get wrapped in http request exceptions
//avoiding circular requests by explicitly returning
if (ex.InnerException is SocketException socketException)
{
previewRequest.Error = new RequestError(socketException);
return(previewRequest);
}
else
{
//in many cases, this leads to a success
if (retryWithoutCompressionOnFailure)
{
await GetLinkDataAsync(previewRequest, false, true);
}
}
}
previewRequest.Error = new RequestError(ex);
Console.WriteLine($"{ex.GetType()}:{ex.Message} for url {previewRequest.CurrentRequestedUrl} in {nameof(GetLinkDataAsync)}");
return(previewRequest);
}
}
private async Task <LinkPreviewRequest> HandleRedirect(HttpResponseMessage response, LinkPreviewRequest previewRequest)
{
var redirectUri = response.Headers.Location;
if (redirectUri != null)
{
Console.WriteLine($"got redirect ({response.StatusCode}) from {previewRequest.CurrentRequestedUrl} to {redirectUri} (https: {redirectUri.ToString().IsHttps()})");
if (redirectUri.ToString() == previewRequest.CurrentRequestedUrl.ToString())
{
if (!response.Headers.Any(header => header.Key == "Set-Cookie"))
{
return(await GetLinkDataAsync(previewRequest, true));
}
else
{
return(await GetLinkDataAsync(previewRequest, false, false, false, true));
}
}
var redirectUriString = redirectUri.ToString();
if (!redirectUriString.IsHttps())
{
//supporting also relative urls
if (!redirectUri.IsAbsoluteUri)
{
if (redirectUriString.StartsWith("//"))
{
redirectUriString = $"{response.RequestMessage.RequestUri.Scheme}:{redirectUriString}";
}
else
{
redirectUriString = $"{response.RequestMessage.RequestUri.GetLeftPart(UriPartial.Authority)}{redirectUriString}";
}
}
}
if (!previewRequest.Redirects.ContainsKey(redirectUriString))
{
previewRequest.CurrentRequestedUrl = new Uri(redirectUriString);
return(await GetLinkDataAsync(previewRequest));
}
else
{
return(await GetLinkDataAsync(previewRequest, true));
}
}
Console.WriteLine($"got redirect ({response.StatusCode}) from {previewRequest.CurrentRequestedUrl} with no location header)");
return(null);
}
