public static HttpFetchResponse Fetch(string url)
{
try
{
using (var f = new HttpFetcher())
{
return f.Get(new Uri(url));
}
}
catch (Exception e)
{
Console.WriteLine(e);
var resp = new HttpFetchResponse();
resp.Exception = e;
return resp;
}
}
public HttpFetchResponse ReadHeaders()
{
//Read headers
var resp = new HttpFetchResponse();
//HTTP response
string line = reader.ReadLine();
if (line == null)
throw new EndOfStreamException();
string[] parts = line.Split(new char[] { ' ' }, 3);
resp.HttpVersion = parts[0];
try
{
resp.StatusCode = (HttpStatusCode)int.Parse(parts[1]);
}
catch (FormatException)
{
resp.StatusCode = (HttpStatusCode)(-1);
resp.StatusMessage = "Invalid HTTP StatusCode: " + line;
return resp;
}
if (parts.Length == 3)
resp.StatusMessage = parts[2];
else if (parts.Length == 2)
resp.StatusMessage = "";
else
{
resp.StatusCode = (HttpStatusCode)(-1);
resp.StatusMessage = "Invalid HTTP response: " + line;
return resp;
}
DebugLine(line);
//Remaining headers
while (true)
{
line = reader.ReadLine();
DebugLine(line);
if (line == "")
return resp;
int sep = line.IndexOf(':');
if (sep < 0)
continue;
string key = line.Substring(0, sep).ToLowerInvariant();
string val = line.Substring(sep + 1).Trim();
#if DEBUG
if (key.StartsWith("x-"))
continue;
#endif
switch (key)
{
case "location":
resp.Location = val;
break;
case "connection":
val = val.ToLowerInvariant();
if (val == "close")
resp.KeepAlive = false;
else if (val.Contains("keep-alive"))
resp.KeepAlive = true;
else
Console.WriteLine("ResponseHeader: unknown Connection: " + val);
break;
case "content-length":
long.TryParse(val, out resp.ContentLength);
break;
case "last-modified":
DateTime dt;
if (SilentOrbit.Parsing.DateParser.TryParse(val, out dt))
resp.LastModified = dt;
else
Console.Error.WriteLine("Parse failed: " + line);
break;
case "transfer-encoding":
val = val.ToLowerInvariant().Trim();
if (val == "chunked")
resp.ChunkedTransferEncoding = true;
else
throw new NotImplementedException(line);
break;
#if DEBUG
case "content-type": //text/html
case "date":
case "frame-options": //DENY
case "server":
case "set-cookie":
case "vary":
case "accept-ranges":
case "etag":
case "p3p":
case "expires":
case "pragma":
case "cache-control":
case "microsoftofficewebserver":
case "wp-super-cache":
case "via":
case "age":
case "status":
case "communityserver":
case "gdata-version":
case "strict-transport-security":
case "cf-ray":
case "content-language":
case "keep-alive":
case "content-disposition":
case "mw-webserver":
case "telligent-evolution":
case "host-header":
case "link":
case "ms-author-via":
case "alternate-protocol":
case "pool-info":
case "edge-control":
case "content-security-policy":
break; //Ignore
default:
throw new NotImplementedException(line);
#endif
}
}
}
public void ReadBody(HttpFetchResponse resp)
{
resp.Stream = new MemoryStream();
if (resp.StatusCode == HttpStatusCode.NotModified)
return;
if (resp.ContentLength == 0)
return;
using (Timer timeout = new Timer(ReadTimeout, null, TimeSpan.FromMinutes(3), Timeout.InfiniteTimeSpan))
{
if (resp.ChunkedTransferEncoding)
{
ReadChunkedBody(resp.Stream);
resp.Stream.Seek(0, SeekOrigin.Begin);
return;
}
if (resp.ContentLength < 0)
{
byte[] buffer = reader.GetBuffered();
resp.Stream.Write(buffer, 0, buffer.Length);
reader.ReadToEnd(resp.Stream, (int)MaxContentLength);
resp.Stream.Seek(0, SeekOrigin.Begin);
return;
}
if (resp.ContentLength > MaxContentLength)
throw new InvalidDataException("Content-Length(" + resp.ContentLength + ") > MaxContentLength(" + MaxContentLength + ")");
//Read given length
byte[] data = reader.ReadBytes((int)resp.ContentLength);
resp.Stream = new MemoryStream(data);
}
}