Exemplo n.º 1
0
        public void Dispatch(WhitespaceToken token)
        {
            if (this.currentWordToken.Length == 0)
            {
                // We haven't read any actual data from the current line yet so lets ignore leading spaces
                return;
            }

            if (this.isWhitespace)
            {
                // The last token was whitespace too so let's skip this one
                return;
            }

            // Increment the line word count
            this.currentLineWordCount++;

            // Save off the current word token and create a new instance for the next word.
            var wordToken = this.currentWordToken;

            this.currentWordToken = new WordToken();

            // Dispatch the current word token so it can be processed
            this.Dispatch(wordToken);

            this.isWhitespace = true;
        }
Exemplo n.º 2
0
        private Statement VariableDeclaration()
        {
            // TODO : multiple declarations style int a, b, c;

            TypeToken type            = Type();
            WordToken identifierToken = look as WordToken;

            Match(Tags.ID);

            Expression initialValue = null;

            if (look.Tag == '=')
            {
                Match('=');
                initialValue = Bool();
            }

            Match(';');

            LocalVariableSymbol symbol = symbolTable.PutVar(identifierToken.Lexeme, type.DotNetType);

            memoryUsed += type.Width;

            return(new VariableDeclarationStatement(symbol, initialValue));
        }
Exemplo n.º 3
0
        private static IToken TokenizeParameters(MethodDefinition method)
        {
            IToken[] list;
            int      listIndex;

            if (method.IsStatic || method.IsConstructor)
            {
                if (method.Parameters.Length == 0)
                {
                    return(NullToken.Instance);
                }
                list      = new IToken[method.Parameters.Length];
                listIndex = 0;
            }
            else
            {
                list      = new IToken[method.Parameters.Length + 1];
                list[0]   = new WordToken("Native");
                listIndex = 1;
            }
            for (int i = 0; i < method.Parameters.Length; i++)
            {
                ParameterDefinition parameter = method.Parameters[i];
                list[listIndex++] = new WordToken(parameter.Name);
            }
            return(new ListToken(list));
        }
Exemplo n.º 4
0
        void edit_IrregularSet(object sender, EventArgs e)
        {
            ControlNounCaseEdit edit  = (ControlNounCaseEdit)sender;
            WordToken           token = new WordToken(edit.Value, edit.InflectionCase, edit.DecliantionNumber);

            NounGrammar.UpdateIrregular(this.edited, token);
            this.RefreshFields();
        }
Exemplo n.º 5
0
        public void EscapeStateTokenizeWord()
        {
            var lexer    = new Lexer();
            var tokens   = lexer.Tokenize("\\w");
            var expected = new WordToken();

            Assert.AreEqual(tokens.Last(), expected);
        }
Exemplo n.º 6
0
 public WordToken this[int index]
 {
     get
     {
         var token = new WordToken();
         tokens.Add(index, token);
         return(tokens[index]);
     }
 }
Exemplo n.º 7
0
 public WordToken this[int index]
 {
     get
     {
         var wt = new WordToken();
         this._WordTokens.Add(index, wt);
         return(_WordTokens[index]);
     }
 }
Exemplo n.º 8
0
        public static string GetForm(Noun noun, InflectionCase aCase, DecliantionNumber amount)
        {
            WordToken token = new WordToken(null, aCase, amount);
            foreach (WordToken tok in noun.Irregulars)
                if (tok.Is(token))
                    return tok.Text;

            return null;
        }
 public WordToken this[int index]
 {
     get
     {
         WordToken wt = new WordToken();
         tokens.Add(index, wt);
         return(tokens[index]);
     }
 }
Exemplo n.º 10
0
        public Sentence(string plainText)
        {
            _words      = plainText.Split(' ');
            _wordTokens = new WordToken[_words.Length];

            for (int index = 0; index < _words.Length; index++)
            {
                _wordTokens[index] = new WordToken();
            }
        }
Exemplo n.º 11
0
        public void Dispatch(WordToken token)
        {
            // Convert word token to string and look it up to record the current line number
            string word = token.ToString();

            this.results.AddOrUpdate(
                word,
                key => new[] { this.currentLineCount },
                (key, existing) => existing.Concat(new[] { this.currentLineCount }));
        }
Exemplo n.º 12
0
        private void GenerateSentenceToken()
        {
            foreach (var paragraph in _paragraphTokens)
            {
                string[] sentence = GetSentences(paragraph.TextValue);
                SentenceCount += sentence.Length;
                for (int i = 0; i < sentence.Length; i++)
                {
                    if (!String.IsNullOrEmpty(sentence[i]))
                    {
                        SentenceToken sentenceToken = new SentenceToken
                        {
                            TextValue = sentence[i]
                        };


                        var words = _rsw.Execute(sentence[i].Tokenize());
                        WordTokenCount += words.Count;
                        for (int j = 0; j < words.Count; j++)
                        {
                            if (!String.IsNullOrEmpty(words[j]))
                            {
                                WordToken wt = new WordToken
                                {
                                    TextValue       = words[j],
                                    Lemma           = _rusStemmer.Stem(words[j]),
                                    ParagraphNumber = paragraph.Number
                                                      //NumberOfSentence = i
                                };
                                sentenceToken.ListOfWord.Add(wt);
                            }
                            if (BagOfLemm.ContainsKey(_rusStemmer.Stem(words[j])))
                            {
                                (BagOfLemm[_rusStemmer.Stem(words[j])] as WordCounter).CountInAllText++;
                                var n = sentenceToken.ListOfWord.Where(w => w.Lemma == _rusStemmer.Stem(words[j])).Count();
                                if (sentenceToken.ListOfWord.Where(w => w.Lemma == _rusStemmer.Stem(words[j])).Count() <= 1)
                                {
                                    BagOfLemm[_rusStemmer.Stem(words[j])].CountSentenceForThisWord++;
                                }
                            }
                            else
                            {
                                BagOfLemm.Add(_rusStemmer.Stem(words[j]), new WordCounter()
                                {
                                    CountInAllText = 1, CountSentenceForThisWord = 1, CounterParagraphForThisWord = 1
                                });
                            }
                            //TODO переделать
                        }
                        paragraph.ListOfSentence.Add(sentenceToken);
                    }
                }
            }
            //CalculateCountSentenceForWordToken();
        }
Exemplo n.º 13
0
        public void WordTokenCtor()
        {
            string value       = "mouse";
            int    position    = 22;
            int    orderNumber = 1;
            string tokenValue  = $"<{value}:{position}:{orderNumber}>";

            WordToken token = new WordToken(position, value, orderNumber);

            Assert.AreEqual(tokenValue, token.TokenValue);
        }
 public WordToken this[int index]
 {
     get
     {
         if (!wordTokens.ContainsKey(index))
         {
             wordTokens[index] = new WordToken();
         }
         return(wordTokens[index]);
     }
 }
Exemplo n.º 15
0
 public WordToken this[int index]
 {
     get
     {
         var token = new WordToken {
             index = index
         };
         words.Add(token);
         return(token);
     }
 }
Exemplo n.º 16
0
 public WordToken this[int index]
 {
     get
     {
         var wordToken = new WordToken {
             Index = index
         };
         _wordTokens.Add(wordToken);
         return(wordToken);
     }
 }
        private WordToken GetOrAddTokenFromCache(int index)
        {
            if (_wordTokens.ContainsKey(index))
            {
                return(_wordTokens[index]);
            }

            var t = new WordToken();

            _wordTokens.Add(index, t);
            return(t);
        }
Exemplo n.º 18
0
        public void WordTokenCtor_WithСapitalLetter_ValueHasLowerCase()
        {
            string input       = "Mouse";
            string value       = "mouse";
            int    position    = 22;
            int    orderNumber = 1;
            string tokenValue  = $"<{value}:{position}:{orderNumber}>";

            WordToken token = new WordToken(position, input, orderNumber);

            Assert.AreEqual(value, token.Value);
        }
Exemplo n.º 19
0
        private static bool TokenSetContainsWordInTheMiddle([NotNull] IReadOnlyList <WordToken> tokenSet, [NotNull] string word)
        {
            for (int index = 1; index < tokenSet.Count - 1; index++)
            {
                WordToken token = tokenSet[index];
                if (string.Equals(word, token.Text, StringComparison.OrdinalIgnoreCase))
                {
                    return(true);
                }
            }

            return(false);
        }
Exemplo n.º 20
0
        public Sentence(string plainText)
        {
            tokens = new List <WordToken>();
            var words = plainText.Split(' ');

            foreach (var word in words)
            {
                var token = new WordToken {
                    Value = word
                };
                tokens.Add(token);
            }
        }
Exemplo n.º 21
0
        public void TestWord()
        {
            string    input     = "WORD1 WORD2";
            Tokenizer tokenizer = new Tokenizer(input);

            WordToken tok = (WordToken)tokenizer.NextToken();

            Assert.AreEqual(TokenType.WORD, tok.Type);
            Assert.AreEqual("WORD1", tok.Text);

            tok = (WordToken)tokenizer.NextToken();
            Assert.AreEqual(TokenType.WORD, tok.Type);
            Assert.AreEqual("WORD2", tok.Text);
        }
Exemplo n.º 22
0
        private WordToken CreateFromLexeme(string lexeme)
        {
            WordToken wordToken;

            if (words.TryGetValue(lexeme, out wordToken))
            {
                return(wordToken);
            }

            wordToken = new WordToken(Tags.ID, lexeme);
            ReserveWord(wordToken);

            return(wordToken);
        }
Exemplo n.º 23
0
        public static void UpdateIrregular(Noun noun, WordToken aToken)
        {
            // find and update existing one
            foreach (WordToken token in noun.Irregulars)
                if (token.DecliantionNumber == aToken.DecliantionNumber &&
                    token.InflectionCase == aToken.InflectionCase)
                {
                    token.Text = aToken.Text;
                    return;
                }

            // not found - add new
            noun.Irregulars.Add(aToken);
        }
Exemplo n.º 24
0
        private static string buildMessage(TextBuffer textBuffer, string errorDescription, bool undoLastRead)
        {
            if (undoLastRead)
            {
                textBuffer.UndoRead();
            }

            WordToken wordToken = TokenReader.ReadWordToken(textBuffer);

            int    arrowsIndex = textBuffer.PreviousIndex + textBuffer.LastRead.Length - wordToken.Name.Length;
            string arrowsLine  = Regex.Replace(textBuffer.Text.Substring(0, arrowsIndex), @"\S", " ", RegexOptions.Singleline);

            return(string.Format("{0}\n{1}\n{2}", errorDescription, textBuffer.Text, arrowsLine + new string('^', wordToken.Name.Length)));
        }
Exemplo n.º 25
0
        private static WordToken WriteEntry(Region resourceHeader, int valueLength, int type, string value)
        {
            WordToken result = resourceHeader.InsertWordToken(); // wLength

            resourceHeader.WriteInt16((short)valueLength);       // w
            resourceHeader.WriteInt16((short)type);              // type
            resourceHeader.WriteAsUtf16NullTerminated2(value);
            //padding
            if ((resourceHeader.CurrentLocation.Offset % 4) != 0)
            {
                resourceHeader.WriteInt16(0);
            }
            return(result);
        }
Exemplo n.º 26
0
        public static string GetForm(Noun noun, InflectionCase aCase, DecliantionNumber amount)
        {
            WordToken token = new WordToken(null, aCase, amount);

            foreach (WordToken tok in noun.Irregulars)
            {
                if (tok.Is(token))
                {
                    return(tok.Text);
                }
            }

            return(null);
        }
Exemplo n.º 27
0
 public WordToken this[int index]
 {
     get
     {
         var wordToken = wordTokens.Find(token => token.Index == index);
         if (wordToken != null)
         {
             return(wordToken);
         }
         wordToken       = new WordToken();
         wordToken.Index = index;
         wordTokens.Add(wordToken);
         return(wordToken);
     }
 }
Exemplo n.º 28
0
        public static void UpdateIrregular(Noun noun, WordToken aToken)
        {
            // find and update existing one
            foreach (WordToken token in noun.Irregulars)
            {
                if (token.DecliantionNumber == aToken.DecliantionNumber &&
                    token.InflectionCase == aToken.InflectionCase)
                {
                    token.Text = aToken.Text;
                    return;
                }
            }

            // not found - add new
            noun.Irregulars.Add(aToken);
        }
Exemplo n.º 29
0
        public void InterpretLine(string line)
        {
            try
            {
                Environment.TextBuffer = new TextBuffer(line);

                while (!Environment.TextBuffer.EndOfBuffer)
                {
                    if (isInsideMultilineComment(Environment) || TokenReader.ReadEmptyLineToken(Environment.TextBuffer) != null)
                    {
                        continue;
                    }

                    WordToken wordToken = TokenReader.ReadWordToken(Environment.TextBuffer);

                    if (Environment.Words.ContainsKey(wordToken.Name))
                    {
                        Environment.Words[wordToken.Name].Interpret(Environment);
                    }
                    else
                    {
                        Environment.TextBuffer.UndoRead();

                        SignedIntegerToken signedIntegerToken = TokenReader.ReadSignedIntegerToken(Environment.TextBuffer);
                        if (signedIntegerToken != null)
                        {
                            (new LiteralWord(signedIntegerToken.Value)).Interpret(Environment);
                        }
                        else
                        {
                            throw new InvalidWordException(Environment.TextBuffer, false);
                        }
                    }
                }

                if (Environment.ActiveExitWordName == "abort")
                {
                    throw new InvalidWordException(Environment.TextBuffer, "Aborted.");
                }
                Environment.ActiveExitWordName = null;
            }
            catch
            {
                Environment.Reset();
                throw;
            }
        }
Exemplo n.º 30
0
        public WordToken this[int index]
        {
            get
            {
                var result = _tokens.SingleOrDefault(t => t.Index == index);
                if (result == null)
                {
                    result = new WordToken()
                    {
                        Index = index
                    };
                    _tokens.Add(result);
                }

                return(result);
            }
        }
Exemplo n.º 31
0
        private FuncCallExpression FuncCallExpression()
        {
            WordToken idToken = prev as WordToken;

            Match('(');
            Expression[] parameters = Parameters();
            Match(')');
            Type[] parameterTypes = ParseUtils.ExpressionArrayToTypes(parameters);

            FunctionSymbol func = symbolTable.GetFunc(idToken.Lexeme, parameterTypes);

            if (func == null)
            {
                Error("Undeclared identifier : " + idToken);
            }

            return(new FuncCallExpression(func, parameters));
        }
Exemplo n.º 32
0
        private static void WriteEntry2(Region resourceHeader, string type, string data)
        {
            WordToken length = resourceHeader.InsertWordToken();

            resourceHeader.WriteInt16((short)(data.Length + 1)); // w
            resourceHeader.WriteInt16(1);                        // type
            resourceHeader.WriteAsUtf16NullTerminated2(type);
            //padding
            if ((resourceHeader.CurrentLocation.Offset % 4) != 0)
            {
                resourceHeader.WriteInt16(0);
            }
            resourceHeader.WriteAsUtf16NullTerminated2(data);
            //padding
            if ((resourceHeader.CurrentLocation.Offset % 4) != 0)
            {
                resourceHeader.WriteInt16(0);
            }
            length.SetDistanceSinceTaken();
        }
Exemplo n.º 33
0
 void edit_IrregularSet(object sender, EventArgs e)
 {
     ControlNounCaseEdit edit = (ControlNounCaseEdit)sender;
     WordToken token = new WordToken(edit.Value, edit.InflectionCase, edit.DecliantionNumber);
     NounGrammar.UpdateIrregular(this.edited, token);
     RefreshFields();
 }