public void Dispatch(WhitespaceToken token)
{
if (this.currentWordToken.Length == 0)
{
// We haven't read any actual data from the current line yet so lets ignore leading spaces
return;
}
if (this.isWhitespace)
{
// The last token was whitespace too so let's skip this one
return;
}
// Increment the line word count
this.currentLineWordCount++;
// Save off the current word token and create a new instance for the next word.
var wordToken = this.currentWordToken;
this.currentWordToken = new WordToken();
// Dispatch the current word token so it can be processed
this.Dispatch(wordToken);
this.isWhitespace = true;
}
private Statement VariableDeclaration()
{
// TODO : multiple declarations style int a, b, c;
TypeToken type = Type();
WordToken identifierToken = look as WordToken;
Match(Tags.ID);
Expression initialValue = null;
if (look.Tag == '=')
{
Match('=');
initialValue = Bool();
}
Match(';');
LocalVariableSymbol symbol = symbolTable.PutVar(identifierToken.Lexeme, type.DotNetType);
memoryUsed += type.Width;
return(new VariableDeclarationStatement(symbol, initialValue));
}
private static IToken TokenizeParameters(MethodDefinition method)
{
IToken[] list;
int listIndex;
if (method.IsStatic || method.IsConstructor)
{
if (method.Parameters.Length == 0)
{
return(NullToken.Instance);
}
list = new IToken[method.Parameters.Length];
listIndex = 0;
}
else
{
list = new IToken[method.Parameters.Length + 1];
list[0] = new WordToken("Native");
listIndex = 1;
}
for (int i = 0; i < method.Parameters.Length; i++)
{
ParameterDefinition parameter = method.Parameters[i];
list[listIndex++] = new WordToken(parameter.Name);
}
return(new ListToken(list));
}
void edit_IrregularSet(object sender, EventArgs e)
{
ControlNounCaseEdit edit = (ControlNounCaseEdit)sender;
WordToken token = new WordToken(edit.Value, edit.InflectionCase, edit.DecliantionNumber);
NounGrammar.UpdateIrregular(this.edited, token);
this.RefreshFields();
}
public void EscapeStateTokenizeWord()
{
var lexer = new Lexer();
var tokens = lexer.Tokenize("\\w");
var expected = new WordToken();
Assert.AreEqual(tokens.Last(), expected);
}
public WordToken this[int index]
{
get
{
var token = new WordToken();
tokens.Add(index, token);
return(tokens[index]);
}
}
public WordToken this[int index]
{
get
{
var wt = new WordToken();
this._WordTokens.Add(index, wt);
return(_WordTokens[index]);
}
}
public static string GetForm(Noun noun, InflectionCase aCase, DecliantionNumber amount)
{
WordToken token = new WordToken(null, aCase, amount);
foreach (WordToken tok in noun.Irregulars)
if (tok.Is(token))
return tok.Text;
return null;
}
public WordToken this[int index]
{
get
{
WordToken wt = new WordToken();
tokens.Add(index, wt);
return(tokens[index]);
}
}
public Sentence(string plainText)
{
_words = plainText.Split(' ');
_wordTokens = new WordToken[_words.Length];
for (int index = 0; index < _words.Length; index++)
{
_wordTokens[index] = new WordToken();
}
}
public void Dispatch(WordToken token)
{
// Convert word token to string and look it up to record the current line number
string word = token.ToString();
this.results.AddOrUpdate(
word,
key => new[] { this.currentLineCount },
(key, existing) => existing.Concat(new[] { this.currentLineCount }));
}
private void GenerateSentenceToken()
{
foreach (var paragraph in _paragraphTokens)
{
string[] sentence = GetSentences(paragraph.TextValue);
SentenceCount += sentence.Length;
for (int i = 0; i < sentence.Length; i++)
{
if (!String.IsNullOrEmpty(sentence[i]))
{
SentenceToken sentenceToken = new SentenceToken
{
TextValue = sentence[i]
};
var words = _rsw.Execute(sentence[i].Tokenize());
WordTokenCount += words.Count;
for (int j = 0; j < words.Count; j++)
{
if (!String.IsNullOrEmpty(words[j]))
{
WordToken wt = new WordToken
{
TextValue = words[j],
Lemma = _rusStemmer.Stem(words[j]),
ParagraphNumber = paragraph.Number
//NumberOfSentence = i
};
sentenceToken.ListOfWord.Add(wt);
}
if (BagOfLemm.ContainsKey(_rusStemmer.Stem(words[j])))
{
(BagOfLemm[_rusStemmer.Stem(words[j])] as WordCounter).CountInAllText++;
var n = sentenceToken.ListOfWord.Where(w => w.Lemma == _rusStemmer.Stem(words[j])).Count();
if (sentenceToken.ListOfWord.Where(w => w.Lemma == _rusStemmer.Stem(words[j])).Count() <= 1)
{
BagOfLemm[_rusStemmer.Stem(words[j])].CountSentenceForThisWord++;
}
}
else
{
BagOfLemm.Add(_rusStemmer.Stem(words[j]), new WordCounter()
{
CountInAllText = 1, CountSentenceForThisWord = 1, CounterParagraphForThisWord = 1
});
}
//TODO переделать
}
paragraph.ListOfSentence.Add(sentenceToken);
}
}
}
//CalculateCountSentenceForWordToken();
}
public void WordTokenCtor()
{
string value = "mouse";
int position = 22;
int orderNumber = 1;
string tokenValue = $"<{value}:{position}:{orderNumber}>";
WordToken token = new WordToken(position, value, orderNumber);
Assert.AreEqual(tokenValue, token.TokenValue);
}
public WordToken this[int index]
{
get
{
if (!wordTokens.ContainsKey(index))
{
wordTokens[index] = new WordToken();
}
return(wordTokens[index]);
}
}
public WordToken this[int index]
{
get
{
var token = new WordToken {
index = index
};
words.Add(token);
return(token);
}
}
public WordToken this[int index]
{
get
{
var wordToken = new WordToken {
Index = index
};
_wordTokens.Add(wordToken);
return(wordToken);
}
}
private WordToken GetOrAddTokenFromCache(int index)
{
if (_wordTokens.ContainsKey(index))
{
return(_wordTokens[index]);
}
var t = new WordToken();
_wordTokens.Add(index, t);
return(t);
}
public void WordTokenCtor_WithСapitalLetter_ValueHasLowerCase()
{
string input = "Mouse";
string value = "mouse";
int position = 22;
int orderNumber = 1;
string tokenValue = $"<{value}:{position}:{orderNumber}>";
WordToken token = new WordToken(position, input, orderNumber);
Assert.AreEqual(value, token.Value);
}
private static bool TokenSetContainsWordInTheMiddle([NotNull] IReadOnlyList <WordToken> tokenSet, [NotNull] string word)
{
for (int index = 1; index < tokenSet.Count - 1; index++)
{
WordToken token = tokenSet[index];
if (string.Equals(word, token.Text, StringComparison.OrdinalIgnoreCase))
{
return(true);
}
}
return(false);
}
public Sentence(string plainText)
{
tokens = new List <WordToken>();
var words = plainText.Split(' ');
foreach (var word in words)
{
var token = new WordToken {
Value = word
};
tokens.Add(token);
}
}
public void TestWord()
{
string input = "WORD1 WORD2";
Tokenizer tokenizer = new Tokenizer(input);
WordToken tok = (WordToken)tokenizer.NextToken();
Assert.AreEqual(TokenType.WORD, tok.Type);
Assert.AreEqual("WORD1", tok.Text);
tok = (WordToken)tokenizer.NextToken();
Assert.AreEqual(TokenType.WORD, tok.Type);
Assert.AreEqual("WORD2", tok.Text);
}
private WordToken CreateFromLexeme(string lexeme)
{
WordToken wordToken;
if (words.TryGetValue(lexeme, out wordToken))
{
return(wordToken);
}
wordToken = new WordToken(Tags.ID, lexeme);
ReserveWord(wordToken);
return(wordToken);
}
public static void UpdateIrregular(Noun noun, WordToken aToken)
{
// find and update existing one
foreach (WordToken token in noun.Irregulars)
if (token.DecliantionNumber == aToken.DecliantionNumber &&
token.InflectionCase == aToken.InflectionCase)
{
token.Text = aToken.Text;
return;
}
// not found - add new
noun.Irregulars.Add(aToken);
}
private static string buildMessage(TextBuffer textBuffer, string errorDescription, bool undoLastRead)
{
if (undoLastRead)
{
textBuffer.UndoRead();
}
WordToken wordToken = TokenReader.ReadWordToken(textBuffer);
int arrowsIndex = textBuffer.PreviousIndex + textBuffer.LastRead.Length - wordToken.Name.Length;
string arrowsLine = Regex.Replace(textBuffer.Text.Substring(0, arrowsIndex), @"\S", " ", RegexOptions.Singleline);
return(string.Format("{0}\n{1}\n{2}", errorDescription, textBuffer.Text, arrowsLine + new string('^', wordToken.Name.Length)));
}
private static WordToken WriteEntry(Region resourceHeader, int valueLength, int type, string value)
{
WordToken result = resourceHeader.InsertWordToken(); // wLength
resourceHeader.WriteInt16((short)valueLength); // w
resourceHeader.WriteInt16((short)type); // type
resourceHeader.WriteAsUtf16NullTerminated2(value);
//padding
if ((resourceHeader.CurrentLocation.Offset % 4) != 0)
{
resourceHeader.WriteInt16(0);
}
return(result);
}
public static string GetForm(Noun noun, InflectionCase aCase, DecliantionNumber amount)
{
WordToken token = new WordToken(null, aCase, amount);
foreach (WordToken tok in noun.Irregulars)
{
if (tok.Is(token))
{
return(tok.Text);
}
}
return(null);
}
public WordToken this[int index]
{
get
{
var wordToken = wordTokens.Find(token => token.Index == index);
if (wordToken != null)
{
return(wordToken);
}
wordToken = new WordToken();
wordToken.Index = index;
wordTokens.Add(wordToken);
return(wordToken);
}
}
public static void UpdateIrregular(Noun noun, WordToken aToken)
{
// find and update existing one
foreach (WordToken token in noun.Irregulars)
{
if (token.DecliantionNumber == aToken.DecliantionNumber &&
token.InflectionCase == aToken.InflectionCase)
{
token.Text = aToken.Text;
return;
}
}
// not found - add new
noun.Irregulars.Add(aToken);
}
public void InterpretLine(string line)
{
try
{
Environment.TextBuffer = new TextBuffer(line);
while (!Environment.TextBuffer.EndOfBuffer)
{
if (isInsideMultilineComment(Environment) || TokenReader.ReadEmptyLineToken(Environment.TextBuffer) != null)
{
continue;
}
WordToken wordToken = TokenReader.ReadWordToken(Environment.TextBuffer);
if (Environment.Words.ContainsKey(wordToken.Name))
{
Environment.Words[wordToken.Name].Interpret(Environment);
}
else
{
Environment.TextBuffer.UndoRead();
SignedIntegerToken signedIntegerToken = TokenReader.ReadSignedIntegerToken(Environment.TextBuffer);
if (signedIntegerToken != null)
{
(new LiteralWord(signedIntegerToken.Value)).Interpret(Environment);
}
else
{
throw new InvalidWordException(Environment.TextBuffer, false);
}
}
}
if (Environment.ActiveExitWordName == "abort")
{
throw new InvalidWordException(Environment.TextBuffer, "Aborted.");
}
Environment.ActiveExitWordName = null;
}
catch
{
Environment.Reset();
throw;
}
}
public WordToken this[int index]
{
get
{
var result = _tokens.SingleOrDefault(t => t.Index == index);
if (result == null)
{
result = new WordToken()
{
Index = index
};
_tokens.Add(result);
}
return(result);
}
}
private FuncCallExpression FuncCallExpression()
{
WordToken idToken = prev as WordToken;
Match('(');
Expression[] parameters = Parameters();
Match(')');
Type[] parameterTypes = ParseUtils.ExpressionArrayToTypes(parameters);
FunctionSymbol func = symbolTable.GetFunc(idToken.Lexeme, parameterTypes);
if (func == null)
{
Error("Undeclared identifier : " + idToken);
}
return(new FuncCallExpression(func, parameters));
}
private static void WriteEntry2(Region resourceHeader, string type, string data)
{
WordToken length = resourceHeader.InsertWordToken();
resourceHeader.WriteInt16((short)(data.Length + 1)); // w
resourceHeader.WriteInt16(1); // type
resourceHeader.WriteAsUtf16NullTerminated2(type);
//padding
if ((resourceHeader.CurrentLocation.Offset % 4) != 0)
{
resourceHeader.WriteInt16(0);
}
resourceHeader.WriteAsUtf16NullTerminated2(data);
//padding
if ((resourceHeader.CurrentLocation.Offset % 4) != 0)
{
resourceHeader.WriteInt16(0);
}
length.SetDistanceSinceTaken();
}
void edit_IrregularSet(object sender, EventArgs e)
{
ControlNounCaseEdit edit = (ControlNounCaseEdit)sender;
WordToken token = new WordToken(edit.Value, edit.InflectionCase, edit.DecliantionNumber);
NounGrammar.UpdateIrregular(this.edited, token);
RefreshFields();
}