public void Login(string username, string password)
{
Sentence initLogonMessage = new Sentence();
initLogonMessage.Add(new CommandWord("/login"));
SendMessage(initLogonMessage);
Response initLogonResponse = ReceiveResponse();
string challenge = ((AttributeWord)initLogonResponse.First()[1]).Value;
string response = EncodePassword(password, challenge);
Sentence completeLogonMessage = new Sentence();
completeLogonMessage.Add(new CommandWord("/login"));
completeLogonMessage.Add(new AttributeWord("name", username));
completeLogonMessage.Add(new AttributeWord("response", response));
SendMessage(completeLogonMessage);
Response completeLogonResponse = ReceiveResponse();
if (!completeLogonResponse.First().Done)
{
throw new Exception("Logon failed");
}
}
private void CreateGap(IToken gap)
{
if (gap != null)
{
if (gap.Value != null)
{
_sentence.Add(new Gap(gap.Value));
}
}
}
public Text Parse(List <string> fromReader)
{
Text finishedText = new Text();
foreach (string sentence in fromReader)
{
string inline = sentence.Substring(1, sentence.Length - 1);
string pattern = @"\s+|\t+";
_buffer = new Regex(pattern).Replace(inline, " ");
ISentence objectSentence = new Sentence();
while (_buffer.Length > 0)
{
char[] sentenceCharArray = _buffer.ToCharArray(0, _buffer.Length);
List <int> separatorsIndex = new List <int>();
foreach (var item in new SeparatorWord().FindSeparator())
{
for (int i = 0; i < sentenceCharArray.Length; i++)
{
if (item == sentenceCharArray[i])
{
separatorsIndex.Add(i + 1);
}
}
}
separatorsIndex.Sort();
if (separatorsIndex.Count >= 1)
{
objectSentence.Add(new SentenceItem(_buffer.Substring(0, separatorsIndex.FirstOrDefault(x => x >= 0) - 1), TypeOfItem.Word));
_buffer = _buffer.Substring(separatorsIndex.FirstOrDefault(x => x >= 0));
}
else
{
objectSentence.Add(new SentenceItem(_buffer.Substring(0, _buffer.Length - 1), TypeOfItem.Word));
objectSentence.Add(new SentenceItem(_buffer.Substring(_buffer.Length - 1, 1), TypeOfItem.Punctuation));
_buffer = string.Empty;
}
}
//foreach (Match regex in Regex.Matches(sentence, pattern))
//{
// if (Regex.IsMatch(regex.Value, @"\w+"))
// {
// objectSentence.Add(new SentenceItem(regex.Value, TypeOfItem.Word));
// }
// if (Regex.IsMatch(regex.Value, @"\p{P}"))
// {
// objectSentence.Add(new SentenceItem(regex.Value, TypeOfItem.Punctuation));
// }
//}
finishedText.Add(objectSentence);
}
return(finishedText);
}
public Production RandomProduction(int maxProductionLength, int numNonterminals, IList<Terminal> terminals, bool useNull = true) {
var lhs = RandomNonterminal(numNonterminals);
var weight = 100 * _rand.NextDouble() + 1.0;
var productionLength = useNull ? _rand.Next(maxProductionLength + 1) + 0: _rand.Next(maxProductionLength + 0) + 1;
Sentence rhs = new Sentence();
for (int i = 0; i < productionLength; i++) {
if (_rand.Next(2) == 0) {
rhs.Add(RandomNonterminal(numNonterminals));
} else {
rhs.Add(RandomTerminal(terminals));
}
}
return new Production(lhs, rhs, weight);
}
public ResponseData DoCommand(string command, Dictionary <string, string> attributes)
{
Sentence commandMessage = new Sentence();
commandMessage.Add(new CommandWord(command));
foreach (KeyValuePair <string, string> attribute in attributes)
{
commandMessage.Add(new AttributeWord(attribute));
}
SendMessage(commandMessage);
return(ResponseDataFactory.createResponseData(ReceiveResponse()));
}
private Sentence GoThrough(string currentHtml)
{
Sentence sentence = new Sentence();
int length = currentHtml.Length;
int index = 0;
while (index != length)
{
if (currentHtml[index] == '<')
{
inTag = true;
}
if (currentHtml[index] == '>')
{
inTag = false;
}
//InTag == true的项表示在标签内
sentence.Add(new Char()
{
Body = currentHtml[index], InTag = inTag
});
index++;
}
return(sentence);
}
public Text Parse(string txt)
{
bool newSentence = false;
Text text = new Text();
Sentence sentence = new Sentence();
Word word = new Word();
for (int i = 0; i < txt.Length; i++)
{
if (!CheckSeparator(punctuationSeparator.MyPunctuationSeparator(), txt[i]) &&
!CheckSeparator(punctuationSeparator.Separator(), txt[i]) &&
!CheckSeparator(punctuationSeparator.CodeSymbolSeparator(), txt[i]))
{
word.symbol.Content += txt[i];
}
else if (!CheckSeparator(punctuationSeparator.MyPunctuationSeparator(), txt[i]))
{
if (word.symbol.Content == null)
{
word.punctuationMarkBefore = txt[i];
}
else
{
word.punctuationMarkAfter.Add(txt[i]);
}
if (CheckSeparator(punctuationSeparator.EndSentenceSeparator(), txt[i]) && word.symbol.Content != null)
{
sentence.GetType(txt[i]);
newSentence = true;
}
}
else
{
if (txt[i] == '\r' && word.symbol.Content != null)
{
word.presenceOfLineFeed = true;
newSentence = true;
}
}
if (((CheckSeparator(punctuationSeparator.Separator(), txt[i]) ||
i == (txt.Length - 1) ||
(CheckSeparator(punctuationSeparator.CodeSymbolSeparator(), txt[i])) ||
txt[i] == '.') && word.symbol.Content != null))
{
sentence.Add(word);
if (newSentence)
{
text.Add(sentence);
sentence = new Sentence();
}
word = new Word();
newSentence = false;
}
}
return(text);
}
private static Sentence AddWordsToSentence(List<Word> listWords)
{
Sentence sentence = new Sentence();
foreach (var word in listWords)
{
sentence.Add(word);
}
return sentence;
}
public ResponseData DoCommand(string command)
{
Sentence commandMessage = new Sentence();
commandMessage.Add(new CommandWord(command));
SendMessage(commandMessage);
return(ResponseDataFactory.createResponseData(ReceiveResponse()));
}
private ISentence ParseSentence(Tuple <string, string> sentenceTuple)
{
ISentence sentence = new Sentence();
var words = sentenceTuple.Item1.Split(new string[] { " ", "\t", "\n" }, StringSplitOptions.RemoveEmptyEntries);
foreach (string word in words)
{
var temporaryWord = word;
var separator = _wordSeparators.GetSeparators().Where(x => word.IndexOf(x) >= 0).FirstOrDefault();
var openSeparator = _openingSeparators.GetSeparators().Where(x => word.IndexOf(x) >= 0).FirstOrDefault();
var closeSeparator = _closingSeparators.GetSeparators().Where(x => word.IndexOf(x) >= 0).FirstOrDefault();
if (openSeparator != null)
{
temporaryWord = temporaryWord.Replace(openSeparator, "");
sentence.Add(new Punctuation(openSeparator));
}
if (separator == word.ToString())
{
sentence.Add(new Punctuation(word));
continue;
}
else
{
sentence.Add(new Word(word.Replace(openSeparator ?? " ", "").Replace(separator ?? " ", "").Replace(closeSeparator ?? " ", "")));
}
if (separator != null)
{
temporaryWord = temporaryWord.Replace(separator, "");
sentence.Add(new Punctuation(separator));
}
if (closeSeparator != null)
{
temporaryWord = temporaryWord.Replace(closeSeparator, "");
sentence.Add(new Punctuation(closeSeparator));
}
}
SetSentenceType(sentence, sentenceTuple.Item2);
sentence.Add(new Punctuation(sentenceTuple.Item2));
return(sentence);
}
public void Run()
{
IWord[] words = new Word[5]
{
new Word("hello"),
new Word("world"),
new Word("how"),
new Word("are"),
new Word("you")
};
Gap space = new Gap(" ");
Gap space1 = new Gap(" ");
IGap tab = new Gap("\t");
IGap newLine = new Gap("\n");
IPunctuationSign comma = new PunctuationSign(", ");
IPunctuationSign dash = new PunctuationSign("-");
IPunctuationSign dot = new PunctuationSign(".");
IPunctuationSign mark = new PunctuationSign("! ");
IPunctuationSign q = new PunctuationSign("?");
IToken q1 = new PunctuationSign("?");
ISentence sentence = new Sentence(new List <IToken>());
var any = sentence.Last();
sentence.Add(tab);
sentence.Add(dash);
sentence.Add(words[0]);
sentence.Add(space);
sentence.Add(words[4]);
sentence.Add(dot);
sentence.Add(tab);
sentence.Add(words[2]);
sentence.Add(space);
sentence.Add(words[3]);
sentence.Add(mark);
Console.WriteLine(sentence.ToString());
sentence.Remove(words[3]);
sentence.Remove(words[4]);
Console.WriteLine(sentence.ToString());
}
public static Sentence ParseSentenceString(string source, PunctuationMark endMark)
{
var sentence = new Sentence();
// Get split parts of the sentence
var splitParts = source.Split(new char[] { ' ' });
foreach (var part in splitParts)
{
var sPart = part;
// Get inner punctuation marks in the split part
var innerPunctuationMarks = sPart.GetPunctuationMarks(DefaultPunctuationMarks.InternalPunctuationMarks);
if (innerPunctuationMarks.Any())
{
var sentenceParts = new List<ISingleTextElement>();
var compositeWord = new CompositeWord();
// If split part contains punctuation marks, split this part to single text elements
foreach (var mark in innerPunctuationMarks)
{
var markLenght = mark.StringValue.Length;
var index = sPart.IndexOfPunctuationMark(mark);
var leftPart = sPart.Substring(0, index);
if (!string.IsNullOrEmpty(leftPart))
{
var word = new Word() { StringValue = leftPart };
compositeWord.Add(word);
if (!sentenceParts.Contains(compositeWord))
sentenceParts.Add(compositeWord);
}
if(index == 0)
{
sentenceParts.Add(mark);
}
else if(index + markLenght < sPart.Length)
{
compositeWord.Add(mark);
if (!sentenceParts.Contains(compositeWord))
sentenceParts.Add(compositeWord);
}
else
{
sentenceParts.Add(mark);
}
sPart = sPart.Substring(index + markLenght);
}
if (!string.IsNullOrEmpty(sPart))
{
var word = new Word() { StringValue = sPart };
compositeWord.Add(word);
if (!sentenceParts.Contains(compositeWord))
sentenceParts.Add(compositeWord);
}
// Add single text elements to sentence
foreach(var sentencePart in sentenceParts)
{
if (sentencePart == sentenceParts.Last())
sentencePart.InnerOption = SingleTextElementInnerOption.RightSpace;
sentence.Add(sentencePart);
}
}
else
{
var word = new Word() { StringValue = sPart, InnerOption = SingleTextElementInnerOption.RightSpace };
sentence.Add(word);
}
}
if (endMark.HasValue)
{
if(sentence.Last().InnerOption == SingleTextElementInnerOption.RightSpace)
{
sentence.Last().InnerOption = SingleTextElementInnerOption.None;
}
sentence.Add(endMark);
}
return sentence;
}
private Sentence[] LoadUnanalyzedCorpus(string resourceName, bool ignoreIncorrect = false)
{
LuMiiCorpus corpus = new LuMiiCorpus();
LuMiiMorphology morphology = new LuMiiMorphology();
Sentence[] sentences = null;
using (Stream stream = this.GetType().Assembly.GetManifestResourceStream(resourceName))
sentences = corpus.Load(stream).ToArray();
List<Sentence> goodSentences = new List<Sentence>();
List<Sentence> ignoredSentences = new List<Sentence>();
List<Token> ignoredTokens = new List<Token>();
Stopwatch watch = new Stopwatch();
watch.Start();
foreach (Sentence sentence in sentences)
{
bool ignore = false;
Sentence analyzedSentence = new Sentence();
foreach (Token token in sentence)
{
Tag[] possibleTags = morphology.Analyze(token.TextTrueCase).ToArray();
if (!possibleTags.Any(t => t.Equals(token.CorrectTag)))
ignore = true;
Token analyzedToken = new Token(token.TextTrueCase, possibleTags, token.CorrectTag, analyzedSentence);
analyzedSentence.Add(analyzedToken);
}
if (!ignoreIncorrect || !ignore)
{
goodSentences.Add(analyzedSentence);
}
else
{
ignoredSentences.Add(analyzedSentence);
}
}
watch.Stop();
Debug.WriteLine(watch.Elapsed);
return goodSentences.ToArray();
}
//reads in the file and populates the script structure with it
static Script parseFile(string fname)
{
Script script = new Script();
script.sentences = new List <Sentence>();
script.roomToSentenceNumber = new Dictionary <string, int>();
script.currentPositions = new List <Coord>();
script.done = false;
script.lastInput = "";
script.switches = new Dictionary <string, bool>();
using (StreamReader reader = new StreamReader(fname))
{
string line;
int lineno; //used for storing rooms
for (line = reader.ReadLine(), lineno = 0; line != null; line = reader.ReadLine(), ++lineno)
{
if (line == "") //ignore empty lines
{
lineno = lineno - 1;
continue;
}
if (line[0] == '>') //auto-replace the ">" "with println "
{
line = line.Replace(' ', '+');
line = "println " + line.Substring(1) + "+++++"; //so that you can '>' an empty line
}
string[] linearray = line.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (linearray.Length % 2 == 0) //make sure there are only valid sentences with valid phrases
{
//proceed as normal, the sentence is well formed
Sentence s = new Sentence();
for (int i = 0; i < linearray.Length; i = i + 2) //add all the phrases to the sentence
{
Phrase p = new Phrase();
p.command = linearray[i];
p.info = linearray[i + 1].Replace('+', ' ');
s.Add(p);
if (p.command != "room")
{
;
}
else
{
script.roomToSentenceNumber.Add(p.info, lineno); //add rooms to the rooms dictionary
}
}
script.sentences.Add(s);
}
else
{
//oops, the sentence has an odd number of words. Die gracefully...
InvalidSentenceException e = new InvalidSentenceException();
e.sentence = line;
throw e;
}
}
}
Coord c = new Coord();
c.sentenceIndex = 0;
c.phraseIndex = 0;
script.currentPositions.Add(c);
return(script);
}
public void TagSpeed()
{
string trainResource = Analyzed2Train;
string testResource = Analyzed2Test;
int maxTokenCount = 1000000;
double minAccuracy = 0.93;
Sentence[] train = LoadAnalyzedCorpus(trainResource);
Sentence[] test = LoadAnalyzedCorpus(testResource);
List<Sentence> all = new List<Sentence>();
int allTokenCount = 0;
while (allTokenCount < maxTokenCount)
{
Sentence s = new Sentence();
foreach (Sentence sentence in test)
{
foreach (Token token in sentence)
{
s.Add(new Token(token));
allTokenCount++;
if (allTokenCount >= maxTokenCount) break;
}
if (allTokenCount >= maxTokenCount) break;
}
all.Add(s);
if (allTokenCount >= maxTokenCount) break;
}
Assert.AreEqual(maxTokenCount, all.SelectMany(t => t).Count());
LuMiiTagger tagger = new LuMiiTagger();
tagger.Train(train);
Stopwatch timer = new Stopwatch();
timer.Start();
tagger.Tag(all);
timer.Stop();
Token[] tokens = all.SelectMany(t => t).ToArray();
double accuracy = (double)tokens.Count(t => t.IsTagCorrect) / tokens.Count();
Assert.AreEqual(maxTokenCount, tokens.Length);
Debug.WriteLine("Accuracy: {0:0.00}%", accuracy * 100);
Debug.WriteLine("Tokens: {0}%", tokens.Length);
Debug.WriteLine("Tag duration: {0} or {1:0} ms", timer.Elapsed, timer.ElapsedMilliseconds);
Debug.WriteLine("Tag speed: {0:0.00} tokens/s", tokens.Length / timer.Elapsed.TotalSeconds);
Assert.Greater(accuracy, minAccuracy);
Assert.Less(accuracy, 0.97);
}
public void AddClause(params int[] variables)
{
Sentence.Add(variables);
}
public virtual ITextItem ParseTextItem(string sentence)
{
var result = new Sentence();
var matches = _sentenceParserRegex.Matches(sentence);
foreach (Match match in matches)
result.Add(ParseSentenceItem(match.Value));
return result;
}
