Beispiel #1
0
        public static void Assert()
        {
            var input = new MemoryStream(Encoding.ASCII.GetBytes("xxaa\rbbb\n ?ccc\r\nuuuu"));
            var r     = new SmartStreamReader(input);
            var a     = r.ReadLine();
            var b     = r.ReadLine();

            var buffer = new byte[5];

            var c = r.Read(buffer, 0, buffer.Length);

            var u = r.ReadToEnd();
        }
Beispiel #2
0
        public void Crawl(string path)
        {
            var t = new TcpClient();

            var Host = this.Host;

            const string CoralSuffix = ".nyud.net";

            if (this.CoralEnabled)
            {
                if (!Host.EndsWith(CoralSuffix))
                {
                    Host += CoralSuffix;
                }
            }

            t.Connect(Host, this.Port);

            var w = new StreamWriter(t.GetStream());

            w.WriteLine(this.Method + " " + path + " HTTP/1.1");
            w.WriteLine("Host: " + Host);
            w.WriteLine("Connection: Close");

            this.Range.Apply(k => w.WriteLine(k.ToString()));

            if (this.HeaderWriter != null)
            {
                this.HeaderWriter(w);
            }

            w.WriteLine();
            w.Flush();

            if (this.AllHeadersSent != null)
            {
                this.AllHeadersSent();
            }

            if (this.StreamWriter != null)
            {
                this.StreamWriter(w);
                w.Flush();
            }

            if (DiscardResponse)
            {
                t.Close();
                return;
            }

            // http://bytes.com/groups/net-c/266557-problem-accessing-streamreaders-basestream

            var r = new SmartStreamReader(t.GetStream());

            var TransferEncoding = default(string);

            this.ForHeaderReceived("transfer-encoding:", value => TransferEncoding = value);

            #region ReadingHeaders
            var ReadingHeaders = true;

            while (ReadingHeaders)
            {
                var Header = r.ReadLine();

                //Console.WriteLine("-- Header:" + Header);

                if (string.IsNullOrEmpty(Header))
                {
                    ReadingHeaders = false;
                }
                else
                if (this.HeaderReceived != null)
                {
                    this.HeaderReceived(Header);
                }
            }
            #endregion

            if (Diagnostics != null)
            {
                Diagnostics("SmartStreamReader AllHeadersReceived");
            }

            if (AllHeadersReceived != null)
            {
                AllHeadersReceived();
            }

            string Data       = null;
            var    BufferSize = -1;
            var    s          = new Stopwatch();

            if (!DiscardData)
            {
                s.Start();
                if (this.StreamReader != null)
                {
                    this.StreamReader(r);
                }
                else if (this.Buffer != null)
                {
                    //throw new NotSupportedException("");

                    BufferSize = r.Read(this.Buffer, 0, this.Buffer.Length);
                }
                else
                {
                    if (string.IsNullOrEmpty(TransferEncoding))
                    {
                        Data = r.ReadToEnd();
                    }
                    else
                    {
                        if (TransferEncoding == "chunked")
                        {
                            var ww = new StringBuilder();
                            var ChunkedLengthString = r.ReadLine();
                            var ChunkedLength       = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);

                            while (ChunkedLength > 0)
                            {
                                r.ReadBlockTo(ChunkedLength, ww);
                                var Separator = r.ReadLine();
                                ChunkedLengthString = r.ReadLine();

                                if (ChunkedLengthString == "")
                                {
                                    ChunkedLength = 0;
                                }
                                else
                                {
                                    ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);
                                }
                            }

                            Data = ww.ToString();
                        }
                    }
                }
                s.Stop();
            }

            t.Close();

            if (!DiscardData)
            {
                if (DataReceived != null)
                {
                    DataReceived(Data);
                }

                if (DataReceivedWithTimeSpan != null)
                {
                    DataReceivedWithTimeSpan(Data, s.Elapsed);
                }
            }

            if (BufferSize > 0)
            {
                // http://drupal.org/node/195642

                var u = new byte[BufferSize];

                for (int i = 0; i < BufferSize; i++)
                {
                    u[i] = this.Buffer[i];
                }

                if (this.BinaryDataReceived != null)
                {
                    this.BinaryDataReceived(u);
                }

                if (this.BinaryDataReceivedWithTimeSpan != null)
                {
                    this.BinaryDataReceivedWithTimeSpan(u, s.Elapsed);
                }
            }
        }
        public void Crawl(string path)
        {
            var t = new TcpClient();

            var Host = this.Host;

            const string CoralSuffix = ".nyud.net";

            if (this.CoralEnabled)
                if (!Host.EndsWith(CoralSuffix))
                {
                    Host += CoralSuffix;
                }

            t.Connect(Host, this.Port);

            var w = new StreamWriter(t.GetStream());

            w.WriteLine(this.Method + " " + path + " HTTP/1.1");
            w.WriteLine("Host: " + Host);
            w.WriteLine("Connection: Close");

            this.Range.Apply(k => w.WriteLine(k.ToString()));

            if (this.HeaderWriter != null)
                this.HeaderWriter(w);

            w.WriteLine();
            w.Flush();

            if (this.AllHeadersSent != null)
                this.AllHeadersSent();

            if (this.StreamWriter != null)
            {
                this.StreamWriter(w);
                w.Flush();
            }

            if (DiscardResponse)
            {
                t.Close();
                return;
            }

            // http://bytes.com/groups/net-c/266557-problem-accessing-streamreaders-basestream

            var r = new SmartStreamReader(t.GetStream());

            var TransferEncoding = default(string);

            this.ForHeaderReceived("transfer-encoding:", value => TransferEncoding = value);

            #region ReadingHeaders
            var ReadingHeaders = true;

            while (ReadingHeaders)
            {
                var Header = r.ReadLine();

                //Console.WriteLine("-- Header:" + Header);

                if (string.IsNullOrEmpty(Header))
                    ReadingHeaders = false;
                else
                    if (this.HeaderReceived != null)
                        this.HeaderReceived(Header);
            }
            #endregion

            if (Diagnostics != null)
            {
                Diagnostics("SmartStreamReader AllHeadersReceived");
            }

            if (AllHeadersReceived != null)
                AllHeadersReceived();

            string Data = null;
            var BufferSize = -1;
            var s = new Stopwatch();

            if (!DiscardData)
            {
                s.Start();
                if (this.StreamReader != null)
                {
                    this.StreamReader(r);
                }
                else if (this.Buffer != null)
                {
                    //throw new NotSupportedException("");

                    BufferSize = r.Read(this.Buffer, 0, this.Buffer.Length);

                }
                else
                {
                    if (string.IsNullOrEmpty(TransferEncoding))
                    {
                        Data = r.ReadToEnd();
                    }
                    else
                    {
                        if (TransferEncoding == "chunked")
                        {
                            var ww = new StringBuilder();
                            var ChunkedLengthString = r.ReadLine();
                            var ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);

                            while (ChunkedLength > 0)
                            {
                                r.ReadBlockTo(ChunkedLength, ww);
                                var Separator = r.ReadLine();
                                ChunkedLengthString = r.ReadLine();

                                if (ChunkedLengthString == "")
                                    ChunkedLength = 0;
                                else
                                    ChunkedLength = int.Parse(ChunkedLengthString, System.Globalization.NumberStyles.HexNumber);
                            }

                            Data = ww.ToString();
                        }
                    }

                }
                s.Stop();
            }

            t.Close();

            if (!DiscardData)
            {
                if (DataReceived != null)
                    DataReceived(Data);

                if (DataReceivedWithTimeSpan != null)
                    DataReceivedWithTimeSpan(Data, s.Elapsed);
            }

            if (BufferSize > 0)
            {
                // http://drupal.org/node/195642

                var u = new byte[BufferSize];

                for (int i = 0; i < BufferSize; i++)
                {
                    u[i] = this.Buffer[i];
                }

                if (this.BinaryDataReceived != null)
                    this.BinaryDataReceived(u);

                if (this.BinaryDataReceivedWithTimeSpan != null)
                    this.BinaryDataReceivedWithTimeSpan(u, s.Elapsed);
            }
        }
        public static void Assert()
        {
            var input = new MemoryStream(Encoding.ASCII.GetBytes("xxaa\rbbb\n ?ccc\r\nuuuu"));
            var r = new SmartStreamReader(input);
            var a = r.ReadLine();
            var b = r.ReadLine();

            var buffer = new byte[5];

            var c = r.Read(buffer, 0, buffer.Length);

            var u = r.ReadToEnd();
        }