public void TextIterator_Simple()
        {
            CharacterStream ti = new CharacterStream(new StringTextProvider("abcd"));

            Assert.AreEqual(4, ti.TextProvider.Length);
            Assert.AreEqual(0, ti.Position);
            Assert.AreEqual('a', ti.CurrentChar);
            Assert.AreEqual(new DecodedChar('a', 1), TextHelpers.DecodeCurrentChar(ti));

            Assert.IsTrue(ti.TextProvider.CompareTo(ti.Position, "ab", ignoreCase: false));
            Assert.IsFalse(ti.TextProvider.CompareTo(ti.Position, "abcde", ignoreCase: false));

            Assert.IsTrue(TextHelpers.CompareCurrentDecodedString(ti, "ab", ignoreCase: false, matchLength: out int matchLength));
            Assert.AreEqual(2, matchLength);
            Assert.IsFalse(TextHelpers.CompareCurrentDecodedString(ti, "abcde", ignoreCase: false, matchLength: out _));

            Assert.IsFalse(ti.IsAtEnd);
            Assert.IsTrue(ti.Advance(1));
            Assert.AreEqual(1, ti.Position);
            Assert.AreEqual('b', ti.CurrentChar);
            Assert.AreEqual('a', ti.Peek(-1));
            Assert.AreEqual('c', ti.Peek(1));
            Assert.AreEqual('d', ti.Peek(2));
            Assert.AreEqual(0, ti.Peek(3));
            Assert.AreEqual(0, ti.Peek(4));

            Assert.IsTrue(ti.Advance(3));
            Assert.IsTrue(ti.IsAtEnd);

            Assert.IsFalse(ti.Advance(1));
        }
Example #2
0
        private static void HandleString(int separatorLength, CharacterStream cs, Func <CharacterStream, bool> terminatorCheck)
        {
            cs.Advance(separatorLength);

            if (!cs.IsEndOfStream())
            {
                while (true)
                {
                    if (terminatorCheck(cs))
                    {
                        cs.Advance(separatorLength);
                        break;
                    }

                    if (cs.CurrentChar == '\\')
                    {
                        cs.MoveToNextChar();
                    }

                    if (!cs.MoveToNextChar())
                    {
                        break;
                    }
                }
            }
        }
Example #3
0
        public void TextHelpers_Decode1()
        {
            // Try parsing a simple unicode char and escaped char

            string          text = @"u\52 \l(foo.jpg)";
            CharacterStream cs   = new CharacterStream(new StringTextProvider(text));

            Assert.IsFalse(TextHelpers.AtEscape(cs));
            Assert.IsFalse(TextHelpers.AtUnicodeEscape(cs));
            Assert.AreEqual(new DecodedChar('u', 1), TextHelpers.DecodeCurrentChar(cs));
            Assert.IsTrue(cs.Advance(1));

            Assert.IsTrue(TextHelpers.AtEscape(cs));
            Assert.IsTrue(TextHelpers.AtUnicodeEscape(cs));
            Assert.AreEqual('R', TextHelpers.DecodeCurrentChar(cs).Char);
            Assert.AreEqual(4, TextHelpers.DecodeCurrentChar(cs).EncodedLength);
            Assert.IsTrue(cs.Advance(4));

            Assert.IsTrue(TextHelpers.AtEscape(cs));
            Assert.IsFalse(TextHelpers.AtUnicodeEscape(cs));
            Assert.AreEqual('l', TextHelpers.DecodeCurrentChar(cs).Char);
            Assert.AreEqual(2, TextHelpers.DecodeCurrentChar(cs).EncodedLength);
            Assert.IsTrue(cs.Advance(2));

            Assert.IsFalse(TextHelpers.AtEscape(cs));
            Assert.IsFalse(TextHelpers.AtUnicodeEscape(cs));
            Assert.AreEqual(new DecodedChar('(', 1), TextHelpers.DecodeCurrentChar(cs));

            Assert.AreEqual(@"uRl(foo.jpg)", TextHelpers.DecodeText(cs.TextProvider, 0, text.Length, forStringToken: false));
        }
Example #4
0
        public void TextHelpers_Decode2()
        {
            // Try parsing a unicode char that's larger than 0xFFFF

            CharacterStream cs = new CharacterStream(new StringTextProvider(@"\abcd1234"));

            Assert.IsTrue(TextHelpers.AtEscape(cs));
            Assert.IsTrue(TextHelpers.AtUnicodeEscape(cs));

            DecodedChar dc = TextHelpers.DecodeCurrentChar(cs);

            Assert.AreEqual(7, dc.EncodedLength);
            Assert.IsTrue(dc.RequiresUtf32);
            Assert.AreEqual(0xABCD12, dc.CharUtf32);
            Assert.AreEqual('\0', dc.Char);
            Assert.IsTrue(cs.Advance(dc.EncodedLength));
            Assert.AreEqual('3', cs.CurrentChar);
        }
Example #5
0
        // public static object CharacterSteam { get; private set; }

        public static int HandleNumber(CharacterStream cs)
        {
            int start = cs.Position;

            if (cs.CurrentChar == '-' || cs.CurrentChar == '+')
            {
                cs.MoveToNextChar();
            }

            if (cs.CurrentChar == '0' && cs.NextChar == 'x')
            {
                cs.Advance(2);
                return(HandleHex(cs, start));
            }

            if (cs.CurrentChar == 'x' && CharacterStream.IsHex(cs.NextChar))
            {
                cs.MoveToNextChar();
                return(HandleHex(cs, start));
            }

            int  integerPartStart   = cs.Position;
            int  integerPartLength  = 0;
            int  fractionPartLength = 0;
            bool isDouble           = false;

            // collect decimals (there may be none like in .1e+20
            while (cs.IsDecimal())
            {
                cs.MoveToNextChar();
                integerPartLength++;
            }

            if (cs.CurrentChar == '.')
            {
                isDouble = true;

                // float/double
                cs.MoveToNextChar();

                // If we've seen don we need to collect factional part of any
                while (cs.IsDecimal())
                {
                    cs.MoveToNextChar();
                    fractionPartLength++;
                }
            }

            if (integerPartLength + fractionPartLength == 0)
            {
                return(0); // +e or +.e is not a number and neither is lonely + or -
            }

            int numberLength;

            if (cs.CurrentChar == 'e' || cs.CurrentChar == 'E')
            {
                isDouble     = true;
                numberLength = HandleExponent(cs, start);
            }
            else
            {
                numberLength = cs.Position - start;
            }

            // Verify double format
            if (isDouble && !IsValidDouble(cs, start, cs.Position))
            {
                numberLength = 0;
            }

            if (numberLength > 0)
            {
                // skip over trailing 'L' if any
                if (cs.CurrentChar == 'L')
                {
                    cs.MoveToNextChar();
                    numberLength++;
                }
            }

            return(numberLength);
        }