示例#1
0
        public static Sentence Tokenize(string ini, DelimiterDictionary delimiters)
        {
            var lines = ini.Split(new[] { Environment.NewLine }, StringSplitOptions.None);

            var globalSection = SectionFactory.CreateSection(Grammar.GlobalSectionName, delimiters);

            var appendSentence = new Action <Sentence>(next =>
            {
                globalSection.After.Last().Next = next;
            });

            for (var i = 0; i < lines.Length; i++)
            {
                var line = lines[i].TrimStart();

                var tokens = TokenizeLine(line, delimiters);
                appendSentence(new Sentence
                {
                    Line   = i,
                    Tokens = tokens
                });
            }

            return(globalSection);
        }
示例#2
0
        public static Sentence Tokenize(string ini, DelimiterDictionary delimiters)
        {
            var lines = ini.Split(new[] { Environment.NewLine }, StringSplitOptions.None);

            var globalSection = SectionFactory.CreateSection(Grammar.GlobalSectionName, delimiters);

            var appendSentence = new Action<Sentence>(next =>
            {
                globalSection.After.Last().Next = next;
            });

            for (var i = 0; i < lines.Length; i++)
            {
                var line = lines[i].TrimStart();

                var tokens = TokenizeLine(line, delimiters);
                appendSentence(new Sentence
                {
                    Line = i,
                    Tokens = tokens
                });
            }

            return globalSection;
        }
示例#3
0
 public static Sentence CreateComment(string text, DelimiterDictionary delimiters)
 {
     var section = new Sentence
     {
         Type = SentenceType.Comment,
         Tokens = new List<Token>
         {
             new Token(TokenType.CommentIndicator, delimiters.TokenTypes[TokenType.CommentIndicator]),
             new Token(TokenType.Comment, text),
         }
     };
     return section;
 }
示例#4
0
 public static Sentence CreateSection(string name, DelimiterDictionary delimiters)
 {
     var section = new Sentence
     {
         Type = SentenceType.Section,
         Tokens = new List<Token>
         {
             new Token(TokenType.LeftSectionDelimiter, delimiters.TokenTypes[TokenType.LeftSectionDelimiter]),
             new Token(TokenType.Section, name),
             new Token(TokenType.RightSectionDelimiter, delimiters.TokenTypes[TokenType.RightSectionDelimiter]),
         }
     };
     return section;
 }
示例#5
0
        public static Sentence CreateComment(string text, DelimiterDictionary delimiters)
        {
            var section = new Sentence
            {
                Type   = SentenceType.Comment,
                Tokens = new List <Token>
                {
                    new Token(TokenType.CommentIndicator, delimiters.TokenTypes[TokenType.CommentIndicator]),
                    new Token(TokenType.Comment, text),
                }
            };

            return(section);
        }
示例#6
0
 public static Sentence CreateProperty(string name, string value, DelimiterDictionary delimiters)
 {
     var property = new Sentence
     {
         Type = SentenceType.Property,
         Tokens = new List<Token>
         {
             new Token(TokenType.Property, name),
             new Token(TokenType.ProperetyValueDelimiter, delimiters.TokenTypes[TokenType.ProperetyValueDelimiter]),
             new Token(TokenType.Value, value),
         }
     };
     return property;
 }
示例#7
0
        public static Sentence CreateSection(string name, DelimiterDictionary delimiters)
        {
            var section = new Sentence
            {
                Type   = SentenceType.Section,
                Tokens = new List <Token>
                {
                    new Token(TokenType.LeftSectionDelimiter, delimiters.TokenTypes[TokenType.LeftSectionDelimiter]),
                    new Token(TokenType.Section, name),
                    new Token(TokenType.RightSectionDelimiter, delimiters.TokenTypes[TokenType.RightSectionDelimiter]),
                }
            };

            return(section);
        }
示例#8
0
        public static Sentence CreateProperty(string name, string value, DelimiterDictionary delimiters)
        {
            var property = new Sentence
            {
                Type   = SentenceType.Property,
                Tokens = new List <Token>
                {
                    new Token(TokenType.Property, name),
                    new Token(TokenType.ProperetyValueDelimiter, delimiters.TokenTypes[TokenType.ProperetyValueDelimiter]),
                    new Token(TokenType.Value, value),
                }
            };

            return(property);
        }
示例#9
0
        internal static List<Token> TokenizeLine(string line, DelimiterDictionary delimiters)
        {
            var isEmptyLine = string.IsNullOrWhiteSpace(line);
            if (isEmptyLine)
            {
                return new List<Token>();
            }

            // initialize tokens with start-of-line
            var delimiterTokens = new List<Token>
            {
                new Token(Grammar.Space)
                {
                    Type = TokenType.StartOfLine,
                }
            };

            // determine delimiter tokens
            for (var i = 0; i < line.Length; i++)
            {
                var c = line[i];

                TokenType tokenType;
                var isDelimiterToken = delimiters.Delimiters.TryGetValue(c, out tokenType);
                if (!isDelimiterToken)
                {
                    continue;
                }

                var isEscapedToken = Grammar.EscapableTokens.Contains(tokenType) && i > 0 && line[i - 1] == Grammar.Backslash;
                if (isEscapedToken)
                {
                    continue;
                }

                var isInlineCommentIndicator = tokenType == TokenType.CommentIndicator && i > 0;
                if (isInlineCommentIndicator)
                {
                    continue;
                }

                // ignore inline section delimiters
                var isInlineLeftSectionDelimiter = tokenType == TokenType.LeftSectionDelimiter && i > 0;
                var isInlineRightSectionDelimiter = tokenType == TokenType.RightSectionDelimiter && i <= line.Length - 1 && delimiterTokens.Skip(1).FirstOrDefault()?.Type != TokenType.LeftSectionDelimiter;
                var ignoreInlineSectionDelimiter = isInlineLeftSectionDelimiter || isInlineRightSectionDelimiter;
                if (ignoreInlineSectionDelimiter)
                {
                    continue;
                }

                // ignore property/value delimiter if first or already found
                var startsWithPropertyValueDelimiter = tokenType == TokenType.ProperetyValueDelimiter && i == 0;
                var containsPropertyValueDelimiter = delimiterTokens.Any(t => t.Type == TokenType.ProperetyValueDelimiter);
                var ignorePropertyValueDelimiter = startsWithPropertyValueDelimiter || containsPropertyValueDelimiter;
                if (ignorePropertyValueDelimiter)
                {
                    continue;
                }

                // collect token
                var token = new Token(c)
                {
                    FromColumn = i,
                    Type = tokenType
                };
                delimiterTokens.Add(token);
            }

            // add end-of-line token
            delimiterTokens.Add(new Token(Grammar.Space)
            {
                Type = TokenType.EndOfLine,
                FromColumn = line.Length
            });

            var lineTokens = CollectText(line, delimiterTokens);
            return lineTokens;
        }
示例#10
0
        internal static List <Token> TokenizeLine(string line, DelimiterDictionary delimiters)
        {
            var isEmptyLine = string.IsNullOrWhiteSpace(line);

            if (isEmptyLine)
            {
                return(new List <Token>());
            }

            // initialize tokens with start-of-line
            var delimiterTokens = new List <Token>
            {
                new Token(Grammar.Space)
                {
                    Type = TokenType.StartOfLine,
                }
            };

            // determine delimiter tokens
            for (var i = 0; i < line.Length; i++)
            {
                var c = line[i];

                TokenType tokenType;
                var       isDelimiterToken = delimiters.Delimiters.TryGetValue(c, out tokenType);
                if (!isDelimiterToken)
                {
                    continue;
                }

                var isEscapedToken = Grammar.EscapableTokens.Contains(tokenType) && i > 0 && line[i - 1] == Grammar.Backslash;
                if (isEscapedToken)
                {
                    continue;
                }

                var isInlineCommentIndicator = tokenType == TokenType.CommentIndicator && i > 0;
                if (isInlineCommentIndicator)
                {
                    continue;
                }

                // ignore inline section delimiters
                var isInlineLeftSectionDelimiter  = tokenType == TokenType.LeftSectionDelimiter && i > 0;
                var isInlineRightSectionDelimiter = tokenType == TokenType.RightSectionDelimiter && i <= line.Length - 1 && delimiterTokens.Skip(1).FirstOrDefault()?.Type != TokenType.LeftSectionDelimiter;
                var ignoreInlineSectionDelimiter  = isInlineLeftSectionDelimiter || isInlineRightSectionDelimiter;
                if (ignoreInlineSectionDelimiter)
                {
                    continue;
                }

                // ignore property/value delimiter if first or already found
                var startsWithPropertyValueDelimiter = tokenType == TokenType.ProperetyValueDelimiter && i == 0;
                var containsPropertyValueDelimiter   = delimiterTokens.Any(t => t.Type == TokenType.ProperetyValueDelimiter);
                var ignorePropertyValueDelimiter     = startsWithPropertyValueDelimiter || containsPropertyValueDelimiter;
                if (ignorePropertyValueDelimiter)
                {
                    continue;
                }

                // collect token
                var token = new Token(c)
                {
                    FromColumn = i,
                    Type       = tokenType
                };
                delimiterTokens.Add(token);
            }

            // add end-of-line token
            delimiterTokens.Add(new Token(Grammar.Space)
            {
                Type       = TokenType.EndOfLine,
                FromColumn = line.Length
            });

            var lineTokens = CollectText(line, delimiterTokens);

            return(lineTokens);
        }