private static void Read(Scanner scanner, ref List <Token> tokens)
        {
            scanner.SkipEmptyLines();
            scanner.SkipWhiteSpace();

            var indent = scanner.Column;

            if (SkipComment(scanner) && scanner.ReadLineBreak() != null)
            {
                scanner.SkipEmptyLines();
                indent = scanner.SkipWhiteSpace();
            }

            if (!scanner.IsFlowContent)
            {
                if (scanner.ReadLineBreak() != null)
                {
                    scanner.MaybeSimpleKey = true;
                    scanner.SkipEmptyLines();
                    indent = scanner.SkipWhiteSpace();
                }

                if (scanner.Index > 0 && scanner.IsLineBreak(-1))
                {
                    while (indent < scanner.CurrentIndent)
                    {
                        scanner.PopIndent();
                        tokens.Add(new Token(scanner, TokenKind.Unindent, length: 0));
                    }
                }

                if (indent == 0)
                {
                    switch (scanner.Current)
                    {
                    case '%':
                        var begin = scanner.Index;

                        while (scanner.CurrentIndent >= 0)
                        {
                            scanner.PopIndent();
                            tokens.Add(new Token(scanner, TokenKind.Unindent, length: 0));
                        }

                        scanner.MaybeSimpleKey = false;
                        tokens.Add(new Token(scanner, TokenKind.Directive, scanner.ReadUntilLineBreakOrEof().Substring(1), begin));

                        return;

                    default:
                        if (!scanner.IsCurrent("---", "...") || !scanner.IsWhiteSpaceOrLineBreakOrEof(3))
                        {
                            break;
                        }

                        while (scanner.CurrentIndent >= 0)
                        {
                            scanner.PopIndent();
                            tokens.Add(new Token(scanner, TokenKind.Unindent, length: 0));
                        }

                        scanner.MaybeSimpleKey = false;
                        tokens.Add(NewToken(scanner.Current == '-' ? TokenKind.Document : TokenKind.Eof, 3));

                        return;
                    }
                }
            }

            switch (scanner.Current)
            {
            case '\0':
                tokens.Add(new Token(scanner, TokenKind.Eof, length: 0));

                return;

            case '!':
            {
                var begin = scanner.Index++;
                var value = scanner.ReadUntilWhiteSpaceOrEof();

                scanner.MaybeSimpleKey = false;
                tokens.Add(new Token(scanner, TokenKind.Tag, value, begin, scanner.Index - begin));

                return;
            }

            case '[':
            case '{':
            {
                var kind = scanner.Current;

                scanner.FlowLevel++;
                scanner.MaybeSimpleKey = true;
                tokens.Add(NewToken(kind == '[' ? TokenKind.SequenceBegin : TokenKind.MappingBegin));

                return;
            }

            case ']':
            case '}':
            {
                var kind = scanner.Current;

                scanner.FlowLevel--;
                scanner.MaybeSimpleKey = false;
                tokens.Add(NewToken(kind == ']' ? TokenKind.SequenceEnd : TokenKind.MappingEnd));

                return;
            }

            case ',':
                scanner.MaybeSimpleKey = true;
                tokens.Add(NewToken(TokenKind.ItemDelimiter));

                return;

            case '-' when scanner.IsWhiteSpaceOrLineBreakOrEof(1):
            {
                if (scanner.IsFlowContent)
                {
                    throw ParseException.TokenNotAllowed(scanner);
                }

                if (!scanner.MaybeSimpleKey)
                {
                    throw ParseException.TokenNotAllowed(scanner);
                }

                var column = scanner.Column;

                if (column > scanner.CurrentIndent)
                {
                    scanner.PushIndent(column);
                    tokens.Add(new Token(scanner, TokenKind.Indent, scanner.Index - column, column));
                }

                var begin = scanner.Index++;

                scanner.SkipWhiteSpace();
                scanner.MaybeSimpleKey = true;
                tokens.Add(new Token(scanner, TokenKind.SequenceValue, begin));

                return;
            }

            case '?' when scanner.IsFlowContent || scanner.IsWhiteSpaceOrLineBreakOrEof(1):
                if (!scanner.IsFlowContent)
                {
                    if (!scanner.MaybeSimpleKey)
                    {
                        throw ParseException.TokenNotAllowed(scanner);
                    }

                    if (indent > scanner.CurrentIndent)
                    {
                        scanner.PushIndent(indent);
                        tokens.Add(new Token(scanner, TokenKind.Indent, scanner.Index - indent, indent));
                    }
                }

                scanner.MaybeSimpleKey = !scanner.IsFlowContent;
                tokens.Add(NewToken(TokenKind.MappingKey));

                return;

            case ':' when scanner.IsFlowContent || scanner.IsWhiteSpaceOrLineBreakOrEof(1):
                if (!scanner.IsFlowContent && tokens.Any())
                {
                    if (!scanner.MaybeSimpleKey)
                    {
                        throw ParseException.TokenNotAllowed(scanner);
                    }

                    indent = scanner.GetColumn(tokens.Last().Index);

                    if (indent > scanner.CurrentIndent)
                    {
                        scanner.PushIndent(indent);
                        tokens.Insert(tokens.Count - 1, new Token(scanner, TokenKind.Indent, scanner.Index - indent, indent));
                    }
                }

                scanner.MaybeSimpleKey = !scanner.IsFlowContent;
                tokens.Add(NewToken(TokenKind.MappingValue));

                return;

            case '&':
            case '*':
            {
                var kind  = scanner.Current;
                var begin = scanner.Index++;
                var value = scanner.ReadWhile(i => char.IsLetterOrDigit(i) || i == '-' || i == '_');

                if (string.IsNullOrEmpty(value))
                {
                    throw ParseException.UnexpectedToken(scanner, "identifier");
                }

                //                    if (!scanner.IsWhiteSpaceOrEof() && "]}?:,%@`".IndexOf(scanner.Current) == -1)
                //                    {
                //                        throw ParseException.UnexpectedToken(scanner, ' ');
                //                    }

                scanner.MaybeSimpleKey = kind == '*';
                tokens.Add(new Token(scanner, kind == '*' ? TokenKind.Alias : TokenKind.Anchor, value, begin, scanner.Index - begin));

                return;
            }

            case '|' when !scanner.IsFlowContent:
            case '>' when !scanner.IsFlowContent:
            {
                var kind            = scanner.Current;
                var begin           = scanner.Index++;
                var currentIndent   = 0;
                var indentIndicator = 0;

                scanner.SkipWhiteSpace();
                SkipComment(scanner);

                if (!scanner.IsLineBreakOrEof())
                {
                    throw ParseException.UnexpectedToken(scanner, '\n');
                }

                scanner.ReadLineBreak();
                scanner.SkipEmptyLines();

                if (indentIndicator > 0)
                {
                    currentIndent = scanner.CurrentIndent + indentIndicator;
                }

                var sb = new StringBuilder();
                var wasLineBreakLine = false;

                var appendNewLine = false;
                while (!scanner.IsEnd)
                {
                    var lineIndent = scanner.PeekWhiteSpace();

                    if (currentIndent > 0 && lineIndent.Length < currentIndent)
                    {
                        break;
                    }

                    if (appendNewLine)
                    {
                        sb.Append(Environment.NewLine);
                        appendNewLine = false;
                    }

                    if (indentIndicator == 0)
                    {
                        indentIndicator = lineIndent.Length - currentIndent;
                        currentIndent  += indentIndicator;
                    }

                    if (scanner.IsLineBreak(currentIndent))
                    {
                        scanner.Index += currentIndent;

                        if (scanner.IsLineBreak())
                        {
                            if (kind != '>' || wasLineBreakLine)
                            {
                                if (sb.Length > 0 && sb[sb.Length - 1] == ' ')
                                {
                                    sb.Remove(sb.Length - 1, 1);
                                }

                                appendNewLine = true;
                            }

                            scanner.SkipEmptyLines();
                        }

                        wasLineBreakLine = true;
                    }
                    else
                    {
                        sb.Append(scanner.ReadUntilLineBreakOrEof().Substring(currentIndent));
                        wasLineBreakLine = false;

                        if (scanner.IsLineBreak())
                        {
                            if (kind != '>' || lineIndent.Length > currentIndent)
                            {
                                appendNewLine = true;
                            }
                            else if (sb.Length > 0 && sb[sb.Length - 1] != ' ')
                            {
                                sb.Append(" ");
                            }
                        }
                    }

                    scanner.SkipEmptyLines();
                }

                var value = sb.ToString();
                scanner.MaybeSimpleKey = false;
                tokens.Add(new Token(scanner, kind == '>' ? TokenKind.StringFolding : TokenKind.StringLiteral, value, begin,
                                     scanner.Index - begin));

                return;
            }

            case '"':
            case '\'':
            {
                var kind  = scanner.Current;
                var begin = scanner.Index++;
                var sb    = new StringBuilder();

                while (!scanner.IsEnd)
                {
                    if (scanner.IsLineBreak())
                    {
                        scanner.ReadLineBreak();
                        scanner.SkipWhiteSpace();

                        if (scanner.IsLineBreak())
                        {
                            do
                            {
                                sb.Append(scanner.ReadLineBreak());
                                scanner.SkipWhiteSpace();
                            } while (scanner.IsLineBreak());
                        }
                        else if (sb.Length > 0 && sb[sb.Length - 1] != ' ')
                        {
                            sb.Append(" ");
                        }
                    }

                    if (kind == '\'' && scanner.Current == kind && scanner.PeekChar(1) == kind)
                    {
                        scanner.Index++;
                    }
                    else if (scanner.Current == '\\')
                    {
                        sb.Append(scanner.Current);
                        scanner.Index++;
                    }
                    else if (scanner.Current == kind)
                    {
                        scanner.Index++;

                        break;
                    }

                    sb.Append(scanner.Current);
                    scanner.Index++;
                }

                scanner.MaybeSimpleKey = true;
                tokens.Add(new Token(scanner, kind == '"' ? TokenKind.StringDouble : TokenKind.StringSingle, sb.ToString(), begin,
                                     scanner.Index - begin));

                return;
            }

            default:
            {
                if (scanner.IsCurrent('-', '?', ':', '|', '>', '%', '@', '`'))
                {
                    throw ParseException.UnexpectedToken(scanner, "Identifier");
                }

                var begin          = scanner.Index;
                var trailingSpaces = 0;
                var sb             = new StringBuilder();

                while (!scanner.IsEnd)
                {
                    if (scanner.Current == ':' && scanner.IsWhiteSpaceOrLineBreakOrEof(1) ||
                        scanner.IsFlowContent && scanner.IsCurrent('?', ':', ',', '[', ']', '{', '}') ||
                        scanner.Current == '#' ||
                        scanner.IsCurrent("---", "...") && scanner.IsLineBreakOrEof(-1))
                    {
                        break;
                    }

                    if (scanner.IsLineBreak())
                    {
                        if (trailingSpaces > 0)
                        {
                            sb.Remove(sb.Length - trailingSpaces, trailingSpaces);
                        }

                        trailingSpaces         = 0;
                        scanner.MaybeSimpleKey = false;
                        scanner.ReadLineBreak();

                        var newIndent = scanner.SkipWhiteSpace();

                        if (scanner.IsLineBreak())
                        {
                            sb.Append(scanner.SkipEmptyLines());
                            newIndent = scanner.SkipWhiteSpace();
                        }

                        if (newIndent <= scanner.CurrentIndent)
                        {
                            break;
                        }

                        if (sb.Length > 0 && sb[sb.Length - 1] != ' ')
                        {
                            sb.Append(" ");
                        }

                        if (!scanner.IsFlowContent && scanner.CurrentIndent > 0 && newIndent < scanner.CurrentIndent)
                        {
                            break;
                        }

                        continue;
                    }

                    if (scanner.IsWhiteSpace())
                    {
                        trailingSpaces++;
                    }
                    else
                    {
                        trailingSpaces = 0;
                    }

                    sb.Append(scanner.Current);
                    scanner.Index++;
                }

                scanner.MaybeSimpleKey = true;
                tokens.Add(new Token(scanner, TokenKind.StringPlain, sb.ToString().TrimEnd(), begin, scanner.Index - begin));

                return;
            }
            }

            Token NewToken(TokenKind kind, int?length = null)
            {
                var rt = new Token(scanner, kind, length: length);

                scanner.Index += rt.Length;

                return(rt);
            }
        }
 public Tokenizer(Scanner scanner)
 {
     this.Scanner     = scanner;
     this.Tokens      = new LinkedList <Token>(Tokenize(scanner));
     this.currentNode = this.Tokens.First;
 }