private void ProcessRedirectedLocation(TcpResponse page, Uri baseUrl, int retries)
{
var newLocation = page.Headers["Location"];
var absoluteUrl = UrlFrontier.GetUrlFromAnchor(baseUrl, newLocation)?.AbsoluteUri ?? string.Empty;
if (!UrlFrontier.CanBeSkipped(absoluteUrl))
{
Console.WriteLine($"Processing redirect");
ProcessUrl(absoluteUrl, retries);
}
}
private static TcpResponse BuildTcpResponse(string downloaded)
{
var response = new TcpResponse();
var headers = GetHeaders(downloaded, out int index);
response.Html = index < downloaded.Length ? downloaded.Substring(index, downloaded.Length - index) : string.Empty;
response.StatusCode = (HttpStatusCode)Convert.ToInt32(headers[0].Substring(9, 3));
for (int i = 1; i < headers.Count - 1; i++)
{
var items = headers[i].Split(':', 2, StringSplitOptions.RemoveEmptyEntries);
response.Headers[items[0]] = items[1];
}
return(response);
}
