public static void Assert()
{
var input = new MemoryStream(Encoding.ASCII.GetBytes("xxaa\rbbb\n ?ccc\r\nuuuu"));
var r = new SmartStreamReader(input);
var a = r.ReadLine();
var b = r.ReadLine();
var buffer = new byte[5];
var c = r.Read(buffer, 0, buffer.Length);
var u = r.ReadToEnd();
}
public void Crawl(string path)
{
var t = new TcpClient();
var Host = this.Host;
const string CoralSuffix = ".nyud.net";
if (this.CoralEnabled)
{
if (!Host.EndsWith(CoralSuffix))
{
Host += CoralSuffix;
}
}
t.Connect(Host, this.Port);
var w = new StreamWriter(t.GetStream());
w.WriteLine(this.Method + " " + path + " HTTP/1.1");
w.WriteLine("Host: " + Host);
w.WriteLine("Connection: Close");
this.Range.Apply(k => w.WriteLine(k.ToString()));
if (this.HeaderWriter != null)
{
this.HeaderWriter(w);
}
w.WriteLine();
w.Flush();
if (this.AllHeadersSent != null)
{
this.AllHeadersSent();
}
if (this.StreamWriter != null)
{
this.StreamWriter(w);
w.Flush();
}
if (DiscardResponse)
{
t.Close();
return;
}
// http://bytes.com/groups/net-c/266557-problem-accessing-streamreaders-basestream
var r = new SmartStreamReader(t.GetStream());
var TransferEncoding = default(string);
this.ForHeaderReceived("transfer-encoding:", value => TransferEncoding = value);
#region ReadingHeaders
var ReadingHeaders = true;
while (ReadingHeaders)
{
var Header = r.ReadLine();
//Console.WriteLine("-- Header:" + Header);
if (string.IsNullOrEmpty(Header))
{
ReadingHeaders = false;
}
else
if (this.HeaderReceived != null)
{
this.HeaderReceived(Header);
}
}
#endregion
if (Diagnostics != null)
{
Diagnostics("SmartStreamReader AllHeadersReceived");
}
if (AllHeadersReceived != null)
{
AllHeadersReceived();
}
string Data = null;
var BufferSize = -1;
var s = new Stopwatch();
if (!DiscardData)
{
s.Start();
if (this.StreamReader != null)
{
this.StreamReader(r);
}
else if (this.Buffer != null)
{
//throw new NotSupportedException("");
BufferSize = r.Read(this.Buffer, 0, this.Buffer.Length);
}
else
{
if (string.IsNullOrEmpty(TransferEncoding))
{
Data = r.ReadToEnd();
}
else
{
if (TransferEncoding == "chunked")
{
var ww = new StringBuilder();
var ChunkedLengthString = r.ReadLine();
var ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);
while (ChunkedLength > 0)
{
r.ReadBlockTo(ChunkedLength, ww);
var Separator = r.ReadLine();
ChunkedLengthString = r.ReadLine();
if (ChunkedLengthString == "")
{
ChunkedLength = 0;
}
else
{
ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);
}
}
Data = ww.ToString();
}
}
}
s.Stop();
}
t.Close();
if (!DiscardData)
{
if (DataReceived != null)
{
DataReceived(Data);
}
if (DataReceivedWithTimeSpan != null)
{
DataReceivedWithTimeSpan(Data, s.Elapsed);
}
}
if (BufferSize > 0)
{
// http://drupal.org/node/195642
var u = new byte[BufferSize];
for (int i = 0; i < BufferSize; i++)
{
u[i] = this.Buffer[i];
}
if (this.BinaryDataReceived != null)
{
this.BinaryDataReceived(u);
}
if (this.BinaryDataReceivedWithTimeSpan != null)
{
this.BinaryDataReceivedWithTimeSpan(u, s.Elapsed);
}
}
}
public void Crawl(string path)
{
var t = new TcpClient();
var Host = this.Host;
const string CoralSuffix = ".nyud.net";
if (this.CoralEnabled)
if (!Host.EndsWith(CoralSuffix))
{
Host += CoralSuffix;
}
t.Connect(Host, this.Port);
var w = new StreamWriter(t.GetStream());
w.WriteLine(this.Method + " " + path + " HTTP/1.1");
w.WriteLine("Host: " + Host);
w.WriteLine("Connection: Close");
this.Range.Apply(k => w.WriteLine(k.ToString()));
if (this.HeaderWriter != null)
this.HeaderWriter(w);
w.WriteLine();
w.Flush();
if (this.AllHeadersSent != null)
this.AllHeadersSent();
if (this.StreamWriter != null)
{
this.StreamWriter(w);
w.Flush();
}
if (DiscardResponse)
{
t.Close();
return;
}
// http://bytes.com/groups/net-c/266557-problem-accessing-streamreaders-basestream
var r = new SmartStreamReader(t.GetStream());
var TransferEncoding = default(string);
this.ForHeaderReceived("transfer-encoding:", value => TransferEncoding = value);
#region ReadingHeaders
var ReadingHeaders = true;
while (ReadingHeaders)
{
var Header = r.ReadLine();
//Console.WriteLine("-- Header:" + Header);
if (string.IsNullOrEmpty(Header))
ReadingHeaders = false;
else
if (this.HeaderReceived != null)
this.HeaderReceived(Header);
}
#endregion
if (Diagnostics != null)
{
Diagnostics("SmartStreamReader AllHeadersReceived");
}
if (AllHeadersReceived != null)
AllHeadersReceived();
string Data = null;
var BufferSize = -1;
var s = new Stopwatch();
if (!DiscardData)
{
s.Start();
if (this.StreamReader != null)
{
this.StreamReader(r);
}
else if (this.Buffer != null)
{
//throw new NotSupportedException("");
BufferSize = r.Read(this.Buffer, 0, this.Buffer.Length);
}
else
{
if (string.IsNullOrEmpty(TransferEncoding))
{
Data = r.ReadToEnd();
}
else
{
if (TransferEncoding == "chunked")
{
var ww = new StringBuilder();
var ChunkedLengthString = r.ReadLine();
var ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);
while (ChunkedLength > 0)
{
r.ReadBlockTo(ChunkedLength, ww);
var Separator = r.ReadLine();
ChunkedLengthString = r.ReadLine();
if (ChunkedLengthString == "")
ChunkedLength = 0;
else
ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);
}
Data = ww.ToString();
}
}
}
s.Stop();
}
t.Close();
if (!DiscardData)
{
if (DataReceived != null)
DataReceived(Data);
if (DataReceivedWithTimeSpan != null)
DataReceivedWithTimeSpan(Data, s.Elapsed);
}
if (BufferSize > 0)
{
// http://drupal.org/node/195642
var u = new byte[BufferSize];
for (int i = 0; i < BufferSize; i++)
{
u[i] = this.Buffer[i];
}
if (this.BinaryDataReceived != null)
this.BinaryDataReceived(u);
if (this.BinaryDataReceivedWithTimeSpan != null)
this.BinaryDataReceivedWithTimeSpan(u, s.Elapsed);
}
}
public static void Assert()
{
var input = new MemoryStream(Encoding.ASCII.GetBytes("xxaa\rbbb\n ?ccc\r\nuuuu"));
var r = new SmartStreamReader(input);
var a = r.ReadLine();
var b = r.ReadLine();
var buffer = new byte[5];
var c = r.Read(buffer, 0, buffer.Length);
var u = r.ReadToEnd();
}