public static HttpFetchResponse Fetch(string url) { try { using (var f = new HttpFetcher()) { return f.Get(new Uri(url)); } } catch (Exception e) { Console.WriteLine(e); var resp = new HttpFetchResponse(); resp.Exception = e; return resp; } }
public HttpFetchResponse ReadHeaders() { //Read headers var resp = new HttpFetchResponse(); //HTTP response string line = reader.ReadLine(); if (line == null) throw new EndOfStreamException(); string[] parts = line.Split(new char[] { ' ' }, 3); resp.HttpVersion = parts[0]; try { resp.StatusCode = (HttpStatusCode)int.Parse(parts[1]); } catch (FormatException) { resp.StatusCode = (HttpStatusCode)(-1); resp.StatusMessage = "Invalid HTTP StatusCode: " + line; return resp; } if (parts.Length == 3) resp.StatusMessage = parts[2]; else if (parts.Length == 2) resp.StatusMessage = ""; else { resp.StatusCode = (HttpStatusCode)(-1); resp.StatusMessage = "Invalid HTTP response: " + line; return resp; } DebugLine(line); //Remaining headers while (true) { line = reader.ReadLine(); DebugLine(line); if (line == "") return resp; int sep = line.IndexOf(':'); if (sep < 0) continue; string key = line.Substring(0, sep).ToLowerInvariant(); string val = line.Substring(sep + 1).Trim(); #if DEBUG if (key.StartsWith("x-")) continue; #endif switch (key) { case "location": resp.Location = val; break; case "connection": val = val.ToLowerInvariant(); if (val == "close") resp.KeepAlive = false; else if (val.Contains("keep-alive")) resp.KeepAlive = true; else Console.WriteLine("ResponseHeader: unknown Connection: " + val); break; case "content-length": long.TryParse(val, out resp.ContentLength); break; case "last-modified": DateTime dt; if (SilentOrbit.Parsing.DateParser.TryParse(val, out dt)) resp.LastModified = dt; else Console.Error.WriteLine("Parse failed: " + line); break; case "transfer-encoding": val = val.ToLowerInvariant().Trim(); if (val == "chunked") resp.ChunkedTransferEncoding = true; else throw new NotImplementedException(line); break; #if DEBUG case "content-type": //text/html case "date": case "frame-options": //DENY case "server": case "set-cookie": case "vary": case "accept-ranges": case "etag": case "p3p": case "expires": case "pragma": case "cache-control": case "microsoftofficewebserver": case "wp-super-cache": case "via": case "age": case "status": case "communityserver": case "gdata-version": case "strict-transport-security": case "cf-ray": case "content-language": case "keep-alive": case "content-disposition": case "mw-webserver": case "telligent-evolution": case "host-header": case "link": case "ms-author-via": case "alternate-protocol": case "pool-info": case "edge-control": case "content-security-policy": break; //Ignore default: throw new NotImplementedException(line); #endif } } }
public void ReadBody(HttpFetchResponse resp) { resp.Stream = new MemoryStream(); if (resp.StatusCode == HttpStatusCode.NotModified) return; if (resp.ContentLength == 0) return; using (Timer timeout = new Timer(ReadTimeout, null, TimeSpan.FromMinutes(3), Timeout.InfiniteTimeSpan)) { if (resp.ChunkedTransferEncoding) { ReadChunkedBody(resp.Stream); resp.Stream.Seek(0, SeekOrigin.Begin); return; } if (resp.ContentLength < 0) { byte[] buffer = reader.GetBuffered(); resp.Stream.Write(buffer, 0, buffer.Length); reader.ReadToEnd(resp.Stream, (int)MaxContentLength); resp.Stream.Seek(0, SeekOrigin.Begin); return; } if (resp.ContentLength > MaxContentLength) throw new InvalidDataException("Content-Length(" + resp.ContentLength + ") > MaxContentLength(" + MaxContentLength + ")"); //Read given length byte[] data = reader.ReadBytes((int)resp.ContentLength); resp.Stream = new MemoryStream(data); } }