예제 #1
0
        public LinkPreviewResponse(LinkPreviewRequest request)
        {
            this.Error  = request.Error;
            this.Result = request.Result;

            this.Redirects = new List <RedirectInfo>();

            if (request.Redirects?.Any() ?? false)
            {
                var urls = request.Redirects.Keys.ToList();

                for (int i = 0; i < urls.Count; i++)
                {
                    if (i == 0)
                    {
                        this.Redirects.Add(new RedirectInfo()
                        {
                            RequestedUrl = request.OriginalUrl,
                            RedirectUrl  = new Uri(urls[i])
                        });
                    }
                    else if (i > 0)
                    {
                        this.Redirects.Add(new RedirectInfo()
                        {
                            RequestedUrl = new Uri(urls[i - 1]),
                            RedirectUrl  = new Uri(urls[i])
                        });
                    }
                }
            }
        }
예제 #2
0
        private async Task <LinkPreviewRequest> HandleFacebookExitLink(LinkPreviewRequest previewRequest)
        {
            var correctLink = previewRequest.CurrentRequestedUrl.TryGetLinkFromFacebookExitLink();

            if (correctLink != null)
            {
                previewRequest.CurrentRequestedUrl = correctLink;
            }

            return(await GetLinkDataAsync(previewRequest, false));
        }
예제 #3
0
        private string TryExtractCookieValueFromLastResponse(LinkPreviewRequest previewRequest)
        {
            HttpResponseMessage cookieContainingResponse = null;
            var lastResponse = previewRequest.Redirects.LastOrDefault();

            cookieContainingResponse = lastResponse.Value == null ? previewRequest.OriginalResponse : lastResponse.Value;

            var cookies = cookieContainingResponse.Headers.SingleOrDefault(header => header.Key == "Set-Cookie");

            var cookieValues = cookies.Value.Select(cookie => cookie.Substring(0, cookie.IndexOf(";") + 1)).ToList();

            StringBuilder cookieValueStringBuilder = new StringBuilder();

            foreach (var value in cookieValues)
            {
                cookieValueStringBuilder.Append($"{value} ");
            }

            return(cookieValueStringBuilder.ToString().Trim());
        }
예제 #4
0
        private async Task TryGetLinkDataFrom302Redirects(LinkPreviewRequest previewRequest)
        {
            if (previewRequest.OriginalResponse.StatusCode == HttpStatusCode.Found)
            {
                var linkPreview = await TryGetLinkPreview(previewRequest.OriginalResponse);

                previewRequest.Result = linkPreview;
            }
            else if (previewRequest.Redirects.Values.Any(r => r.StatusCode == HttpStatusCode.Found))
            {
                var linkPreviewTasks = new List <Task <LinkInfo> >();
                foreach (var response in previewRequest.Redirects.Values.Where(r => r.StatusCode == HttpStatusCode.Found))
                {
                    linkPreviewTasks.Add(TryGetLinkPreview(response));
                }

                var linkPreviews = await Task.WhenAll(linkPreviewTasks).ConfigureAwait(false);

                var linkWithTitleAndImage = linkPreviews.FirstOrDefault(p => !string.IsNullOrEmpty(p.Title) && p.ImageUrl != null);
                previewRequest.Result = linkWithTitleAndImage ?? linkPreviews.FirstOrDefault(p => !string.IsNullOrEmpty(p.Title));
            }
        }
예제 #5
0
        public async Task <LinkPreviewRequest> GetLinkDataAsync(LinkPreviewRequest previewRequest, bool isCircleRedirect = false, bool retryWithoutCompressionOnFailure = true, bool noCompression = false, bool addCookieToRedirectedRequest = false)
        {
            try
            {
                var currentRequestedUrlString = previewRequest.CurrentRequestedUrl.ToString();

                string cookieHeaderValue = null;
                if (addCookieToRedirectedRequest)
                {
                    cookieHeaderValue = TryExtractCookieValueFromLastResponse(previewRequest);
                }

                if (currentRequestedUrlString.Contains("facebook.com") &&
                    previewRequest.CurrentRequestedUrl.ContainsParameter("u"))
                {
                    return(await HandleFacebookExitLink(previewRequest));
                }
                //todo: add other special cases (twitch?, youtube?) here...
                else
                {
                    if (!isCircleRedirect)
                    {
                        Console.WriteLine($"sending request for url {previewRequest.CurrentRequestedUrl}");

                        var request = new HttpRequestMessage(currentRequestedUrlString.IsHttps() ? HttpMethod.Get : HttpMethod.Head, previewRequest.CurrentRequestedUrl);
                        request.Headers.Host = previewRequest.CurrentRequestedUrl.Host;

                        if (!string.IsNullOrWhiteSpace(cookieHeaderValue))
                        {
                            request.Headers.Add("Cookie", cookieHeaderValue);
                        }

                        HttpCompletionOption completionOption = HttpCompletionOption.ResponseHeadersRead;

                        var response = noCompression ?
                                       await _httpClientInstance.SendAsync(request, completionOption) :
                                       await TryGetResponseMessageWithoutCompressionAsync(request, completionOption);

                        if (previewRequest.OriginalResponse == null)
                        {
                            previewRequest.OriginalResponse = response;
                        }
                        else
                        {
                            previewRequest.Redirects.Add(previewRequest.CurrentRequestedUrl.ToString(), response);
                        }

                        var statusCode = (int)response.StatusCode;
                        Console.WriteLine($"received response {statusCode} from url {request.RequestUri}");

                        if (statusCode >= 300 && statusCode <= 399)
                        {
                            return(await HandleRedirect(response, previewRequest));
                        }
                        else if (statusCode >= 400)
                        {
                            var message = await response.Content.ReadAsStringAsync().ConfigureAwait(false);

                            previewRequest.Error = new RequestError(statusCode, message);

                            Console.WriteLine($"got error response ({statusCode}) from {previewRequest.CurrentRequestedUrl}\nmessage: {message}");
                        }
                        else
                        {
                            var linkPreview = await TryGetLinkPreview(response);

                            previewRequest.Result = linkPreview;
                        }
                    }
                    else
                    {
                        await TryGetLinkDataFrom302Redirects(previewRequest).ConfigureAwait(false);
                    }

                    return(previewRequest);
                }
            }
            catch (Exception ex)
            {
                if (ex is HttpRequestException requestException)
                {
                    //socket exceptions get wrapped in http request exceptions
                    //avoiding circular requests by explicitly returning
                    if (ex.InnerException is SocketException socketException)
                    {
                        previewRequest.Error = new RequestError(socketException);
                        return(previewRequest);
                    }
                    else
                    {
                        //in many cases, this leads to a success
                        if (retryWithoutCompressionOnFailure)
                        {
                            await GetLinkDataAsync(previewRequest, false, true);
                        }
                    }
                }

                previewRequest.Error = new RequestError(ex);

                Console.WriteLine($"{ex.GetType()}:{ex.Message} for url {previewRequest.CurrentRequestedUrl} in {nameof(GetLinkDataAsync)}");
                return(previewRequest);
            }
        }
예제 #6
0
        private async Task <LinkPreviewRequest> HandleRedirect(HttpResponseMessage response, LinkPreviewRequest previewRequest)
        {
            var redirectUri = response.Headers.Location;

            if (redirectUri != null)
            {
                Console.WriteLine($"got redirect ({response.StatusCode}) from {previewRequest.CurrentRequestedUrl} to {redirectUri} (https: {redirectUri.ToString().IsHttps()})");


                if (redirectUri.ToString() == previewRequest.CurrentRequestedUrl.ToString())
                {
                    if (!response.Headers.Any(header => header.Key == "Set-Cookie"))
                    {
                        return(await GetLinkDataAsync(previewRequest, true));
                    }
                    else
                    {
                        return(await GetLinkDataAsync(previewRequest, false, false, false, true));
                    }
                }

                var redirectUriString = redirectUri.ToString();
                if (!redirectUriString.IsHttps())
                {
                    //supporting also relative urls
                    if (!redirectUri.IsAbsoluteUri)
                    {
                        if (redirectUriString.StartsWith("//"))
                        {
                            redirectUriString = $"{response.RequestMessage.RequestUri.Scheme}:{redirectUriString}";
                        }
                        else
                        {
                            redirectUriString = $"{response.RequestMessage.RequestUri.GetLeftPart(UriPartial.Authority)}{redirectUriString}";
                        }
                    }
                }

                if (!previewRequest.Redirects.ContainsKey(redirectUriString))
                {
                    previewRequest.CurrentRequestedUrl = new Uri(redirectUriString);

                    return(await GetLinkDataAsync(previewRequest));
                }
                else
                {
                    return(await GetLinkDataAsync(previewRequest, true));
                }
            }

            Console.WriteLine($"got redirect ({response.StatusCode}) from {previewRequest.CurrentRequestedUrl} with no location header)");

            return(null);
        }