public Classification(string modelName, string dictionary, int dictionarySize, float cutoff)
{
//--quiet
_vw = new VowpalWabbit(string.Format("-t -i {0} --quiet", modelName));
_dictionary = new WordDictionary(dictionary, dictionarySize);
_cutoff = cutoff;
}
public DisclaimerPredict(string modelName, string dictionary, int dictionarySize, float cutoff)
{
//--quiet
// _vw = new VowpalWabbit(string.Format("-t -i {0} --quiet", modelName));
_vw = new VowpalWabbit(string.Format("-f {0} --loss_function logistic --passes 25 -c -l2", modelName));
_dictionary = new WordDictionary(dictionary, dictionarySize);
_cutoff = cutoff;
}
public Environment()
{
Words = new WordDictionary();
Memory = new Memory(100000);
DataStack = new DataStack();
ReturnStack = new DataStack("Return Stack");
ControlFlowStack = new ControlFlowStack();
}
public static void FindPath(string dictFilePath, string startWord, string endWord, string resultFilePath)
{
if (string.IsNullOrEmpty(dictFilePath) || string.IsNullOrEmpty(startWord) || string.IsNullOrEmpty(endWord) || string.IsNullOrEmpty(resultFilePath))
throw new ArgumentException("Missing arguments");
var wordSet = File.ReadAllLines(dictFilePath);
var wordDict = new WordDictionary(wordSet);
var resultSet = GetPathResult(wordDict, startWord, endWord);
File.WriteAllLines(resultFilePath, resultSet);
}
public void GetPathResultTestSpecialCase()
{
var stringArraySpecial = new string[] { "abaa", "aaaa", "abza", "abzz", "aazz" };
var wordDictSpecial = new WordDictionary(stringArraySpecial);
var result = WordSearchEngine.GetPathResult(wordDictSpecial, "aaaa", "aazz");
Assert.IsTrue(result.Contains("aaaa"));
Assert.IsTrue(result.Contains("abaa"));
Assert.IsTrue(result.Contains("abza"));
Assert.IsTrue(result.Contains("abzz"));
Assert.IsTrue(result.Contains("aazz"));
}
public TrainingSet()
{
countWordGroup = new Dictionary<string, Dictionary<string, int>>();
countCharacterGroup = new Dictionary<string, Dictionary<string, int>>();
//countHead = new Dictionary<string, int>();
segmenter = new JiebaSegmenter();
trie = new WordDictionary();
Console.WriteLine(System.DateTime.Now.ToString() + "词库加载完毕，开始加载训练集...");
LoadTrainingSet();
OutputDictionary();
Console.WriteLine(System.DateTime.Now.ToString() + "训练集加载完毕...");
}
public static void ShowWordsInfo(WordDictionary dict, char c)
{
int ccid = Utility.CC_ID(c);
Console.WriteLine("====================================\r\n汉字:{0}, ID ：{1}\r\n", Utility.CC_ID2Char(ccid), ccid);
Console.WriteLine(" 词长 频率 词性 词");
for (int i = 0; i < dict.indexTable[ccid].nCount; i++)
Console.WriteLine("{0,5} {1,6} {2,5} ({3}){4}",
dict.indexTable[ccid].WordItems[i].nWordLen,
dict.indexTable[ccid].WordItems[i].nFrequency,
Utility.GetPOSString(dict.indexTable[ccid].WordItems[i].nPOS),
Utility.CC_ID2Char(ccid),
dict.indexTable[ccid].WordItems[i].sWord);
}
public static List<string> GetPathResult(WordDictionary wordDict, string startWord, string endWord)
{
var resultSet = new List<string> { { "No results" } };
if (wordDict.WordSet.ContainsKey(startWord))
{
resultSet.Clear();
resultSet.Add(startWord);
if (startWord == endWord || !wordDict.WordSet.ContainsKey(endWord)) return resultSet;
var explored = new HashSet<string>(new string[] { startWord });
var frontier = wordDict.WordSet[startWord].Except(explored).ToList();
while (frontier.Any())
{
var newStartWord = frontier.OrderBy(w => WordDictionary.GetWordDistance(w, endWord)).FirstOrDefault();
frontier.Remove(newStartWord);
explored.Add(newStartWord);
if (newStartWord == endWord)
{
resultSet.Add(endWord);
break;
}
var newNeighbors = wordDict.WordSet[newStartWord].Except(explored).Except(frontier).ToList();
if (newNeighbors.Any())
{
resultSet.Add(newStartWord);
frontier.AddRange(newNeighbors);
}
else
{
}
}
if (!resultSet.Contains(endWord))
{
resultSet.Clear();
resultSet.Add(startWord);
}
}
return resultSet;
}
public void TestMethod(int opCount, int maxWordLength)
{
var dict = new WordDictionary();
var list = new List<string>();
while (opCount-- > 0)
{
var word = GenerateString(1, maxWordLength, 'a', 'z');
switch (Random.Next(2))
{
case 0:
dict.AddWord(word);
list.Add(word);
break;
case 1:
Assert.AreEqual(list.Contains(word), dict.Search(word));
break;
}
}
}
static void Main(string[] args)
{
string DictPath = Path.Combine(Environment.CurrentDirectory, "Data") + Path.DirectorySeparatorChar;
Console.WriteLine("正在读入字典，请稍候...");
WordDictionary dict = new WordDictionary();
dict.Load(DictPath + "coreDict.dct");
ShowWordsInfo(dict, '设');
Console.WriteLine("\r\n向字典库插入“设计模式”一词...");
dict.AddItem("设计模式", Utility.GetPOSValue("n"), 10);
Console.WriteLine("\r\n修改完成，将字典写入磁盘文件coreDictNew.dct，请稍候...");
dict.Save(DictPath + "coreDictNew.dct");
Console.WriteLine("\r\n打开已写入的字典，请稍候...");
dict.Load(DictPath + "coreDictNew.dct");
ShowWordsInfo(dict, '设');
Console.Write("按下回车键退出......");
Console.ReadLine();
}
public ActionResult StartTextMining()
{
WordDictionary englishDictionary = new WordDictionary {
DictionaryFile = "en-US.dic"
};
englishDictionary.Initialize();
Spelling englishSpeller = new Spelling {
Dictionary = englishDictionary
};
EnglishStemmer englishStemmer = new EnglishStemmer();
var cristinIDList = textMining.getCristinID();
var stopWords = textMining.getStopWords();
var addedStopWords = new List <string> {
"studi", "base", "analysi", "effect", "activ", "high",
"factor", "year", "express", "cross", "experi", "level", "assess", "case", "cohort", "impact",
"term", "low", "process", "long", "depend", "perform", "type", "increas", "outcom", "evalu",
"top", "larg", "comparison", "section", "central", "person", "problem", "rate", "general"
};
Int32 counter = 0;
Int32 total = cristinIDList.Count();
foreach (var cristinID in cristinIDList)
{
var titles = textMining.getTitles(cristinID);
if (titles == null)
{
continue;
}
var tokenizedTitles = textMining.tokenizeTitles(titles);
textMining.removeLanguages(tokenizedTitles, englishSpeller);
if (textMining.isActive(tokenizedTitles))
{
textMining.removeStopWords(tokenizedTitles, stopWords);
textMining.stemTitles(tokenizedTitles, englishStemmer);
textMining.removeStopWords(tokenizedTitles, addedStopWords); // sikrer at de stemmete ordene også fjernes
short totalTitles = (short)tokenizedTitles.Count();
var groupedWords = textMining.groupTitles(tokenizedTitles);
if (textMining.saveWordCloud(groupedWords, cristinID, totalTitles))
{
Debug.WriteLine("Saving: (" + (++counter) + "/" + total + ")");
}
else
{
Debug.WriteLine("An unexpected error has occured at cristinId: " + cristinID);
return(View());
}
}
else
{
Debug.WriteLine("Saving: (" + (++counter) + "/" + total + ")");
}
}
Debug.WriteLine("Text Mining Complete");
return(View());
}
private IEnumerable <string> FindWords_Search(char[][] board, int i, int j, StringBuilder sb, WordDictionary dict)
{
WordDictionary childDict;
if (dict.Keys.Count > 0 && char.IsLower(board[i][j]) && dict.TryGetValue(board[i][j], out childDict))
{
sb.Append(board[i][j]);
if (childDict.CanEnd)
{
yield return(sb.ToString());
}
board[i][j] = char.ToUpper(board[i][j]);
for (var k = 0; k < 4; ++k)
{
var newI = i + FindWords_SearchPath[k, 0];
var newJ = j + FindWords_SearchPath[k, 1];
if (newI >= 0 && newI < board.Length && newJ >= 0 && newJ < board[0].Length)
{
foreach (var word in FindWords_Search(board, newI, newJ, sb, childDict))
{
yield return(word);
}
}
}
sb.Remove(sb.Length - 1, 1);
board[i][j] = char.ToLower(board[i][j]);
}
}
private void TableHeader_Click(object sender, RoutedEventArgs e)
{
OrderAsc = !OrderAsc;
var bindingInfo = (sender as GridViewColumnHeader).Column.DisplayMemberBinding as System.Windows.Data.Binding;
string bindingPropertyName = bindingInfo.Path.Path;
Func <Word, object> keySelector = x => x.GetPropertyValue(bindingPropertyName);
var sortedWordDictionary = OrderAsc ? WordDictionary.OrderBy(keySelector) : WordDictionary.OrderByDescending(keySelector);
WordDictionary.Update(sortedWordDictionary.ToList());
}
public Chemin(char[,] lettersGrid, WordDictionary dictionary)
{
m_grid = lettersGrid;
m_dictionary = dictionary;
}
/// <summary>
/// 找到导入库和现有库的不同
/// </summary>
/// <param name="NewDicFile">导入库文件</param>
/// <param name="Encoding">导入库文件编码</param>
/// <param name="DicFormat">导入库文件格式</param>
/// <param name="SourceDictFileName">原库文件</param>
/// <param name="OddLines">输出没有词性标注且现有库中也没有的词行</param>
/// <param name="NewWords">输出新词或现有词的新词性</param>
/// <param name="ExistWords">输出重复词，且词性也相同</param>
/// <param name="MaxFrqRate">重复词的最大词频比例</param>
/// <param name="MinFrqRate">重复词的最小词频比例</param>
/// <param name="AvgFrqRate">重复词的平均词频比例</param>
public static void FindDifferent(string NewDicFile, Encoding Encoding, DictionaryFormat DicFormat, string SourceDictFileName,
out string[] OddLines, out WordDictionary.DicWordInfo[] NewWords, out WordDictionary.DicWordInfo[] ExistWords,
out double MaxFrqRate, out double MinFrqRate, out double AvgFrqRate)
{
WordDictionary SourceDict = new WordDictionary();
if (!SourceDict.Load(SourceDictFileName))
throw new Exception("load source dic file fail");
FindDifferent(NewDicFile, Encoding, DicFormat, SourceDict, out OddLines, out NewWords, out ExistWords, out MaxFrqRate, out MinFrqRate, out AvgFrqRate);
SourceDict.ReleaseDict();
}
protected void Page_Init(object sender, EventArgs e)
{
string cacheName = "WordDictionary|" + CMSContext.PreferredCultureCode;
// Get dictionary from cache
this.WordDictionary = (WordDictionary)HttpContext.Current.Cache[cacheName];
if (this.WordDictionary == null)
{
// If not in cache, create new
this.WordDictionary = new NetSpell.SpellChecker.Dictionary.WordDictionary();
this.WordDictionary.EnableUserFile = false;
// Getting folder for dictionaries
string folderName = "~/App_Data/Dictionaries";
folderName = this.MapPath(folderName);
string culture = CMSContext.PreferredCultureCode;
string dictFile = culture + ".dic";
string dictPath = Path.Combine(folderName, dictFile);
if (!File.Exists(dictPath))
{
// Get default culture
string defaultCulture = ValidationHelper.GetString(SettingsHelper.AppSettings["CMSDefaultSpellCheckerCulture"], "");
if (defaultCulture != "")
{
culture = defaultCulture;
dictFile = defaultCulture + ".dic";
dictPath = Path.Combine(folderName, dictFile);
}
}
if (!File.Exists(dictPath))
{
this.lblError.Text = string.Format(GetString("SpellCheck.DictionaryNotExists"), culture);
this.lblError.Visible = true;
return;
}
mDictExists = true;
this.WordDictionary.DictionaryFolder = folderName;
this.WordDictionary.DictionaryFile = dictFile;
// Load and initialize the dictionary
this.WordDictionary.Initialize();
// Store the Dictionary in cache
HttpContext.Current.Cache.Insert(cacheName, this.WordDictionary, new CacheDependency(Path.Combine(folderName, this.WordDictionary.DictionaryFile)));
}
else
{
mDictExists = true;
}
// Create spell checker
this.SpellChecker = new NetSpell.SpellChecker.Spelling();
this.SpellChecker.ShowDialog = false;
this.SpellChecker.Dictionary = this.WordDictionary;
// Adding events
this.SpellChecker.MisspelledWord += new NetSpell.SpellChecker.Spelling.MisspelledWordEventHandler(this.SpellChecker_MisspelledWord);
this.SpellChecker.EndOfText += new NetSpell.SpellChecker.Spelling.EndOfTextEventHandler(this.SpellChecker_EndOfText);
this.SpellChecker.DoubledWord += new NetSpell.SpellChecker.Spelling.DoubledWordEventHandler(this.SpellChecker_DoubledWord);
}
private async Task BotTranslate(IDialogContext context, UserMessage userText)
{
bool isSave = true;
string _text = userText.Text.GetFromBeginTo("?").Trim();
string _token = userText.Text.GetFromEndTo("?").Trim();
string messageText = userText.Text.GetFromBeginTo("?").Trim();
var _results = SearchInDictionary(_text);
if (_results != null)
{
isSave = false;
await context.PostAsync(_results);
return;
}
else
{
Translator translator = new Translator();
WordDictionary _newWord = new WordDictionary();
_newWord.CountViews = 0;
_newWord.IsRemembered = false;
if (_token.IsEnglishLangCode())
{
_newWord.English = _text;
_newWord.Vietnameses = translator.Translate(_text, "vi", "en");
_newWord.Vietnameses = System.Web.HttpUtility.HtmlDecode(_newWord.Vietnameses);
await context.PostAsync(_newWord.Vietnameses);
}
else if (_token.IsVietnameseLangCode())
{
_newWord.Vietnameses = _text;
_newWord.English = translator.Translate(_text, "en", "vi");
_newWord.English = System.Web.HttpUtility.HtmlDecode(_newWord.English);
await context.PostAsync(_newWord.English);
}
else
{
isSave = false;
await context.PostAsync("Sorry, i don't understand this language");
return;
}
if (isSave)
{
using (MathBotDataContext dbContext = new MathBotDataContext())
{
dbContext.Dictionaries.Add(_newWord);
try
{
dbContext.SaveChanges();
return;
}
catch (Exception ex)
{
await context.PostAsync(ex.Message);
return;
}
}
}
}
}
public void Dispose()
{
this._CachedDictionary = null;
this.Host = null;
GC.SuppressFinalize(this);
}
static void Main(string[] args)
{
//var x = "Hello World!".ReverseWordsInString();
//Console.WriteLine(x);
// MinMaxStack stack = new MinMaxStack();
// stack.Push(5);
// stack.Push(2);
// stack.Push(3);
// stack.Push(1);
// stack.Push(4);
// stack.Print();
// Console.WriteLine(stack.Min);
// Console.WriteLine(stack.Max);
// Console.WriteLine(stack.Pop());
// stack.Print();
// Console.WriteLine(stack.Min);
// Console.WriteLine(stack.Max);
// new int[] {2, 7, 11, 15}.TwoSum(9).Print();
//Console.WriteLine("vamsiv".HasUniqueCharacters());
// Console.WriteLine("vamgsi".IsPermutation("aitsv"));
// Console.WriteLine("aabcccccaaa".CompressString());
// SingleLinkedList list = new SingleLinkedList();
// list.Add(5);
// list.Add(10);
// list.Add(11);
// list.Add(4);
// list.Add(11);
// list.Add(13);
// list.Reverse();
// list.RemoveDuplicates();
// new int[]{2,2,1}.GetUniqueNumber();
// //var perm = "vgamsi".Permute();
// Console.WriteLine(list.KthToTheLast(5));
// LRUCache cache = new LRUCache(3);
// cache.Put(1,858);
// cache.Put(2,8);
// cache.Put(4,558);
// cache.Get(2);
// cache.Put(8,789);
// new int[] {1,2,6,0,0,0}.MergeSortedArray(3, new int[] {2,3,5}, 3);
// new int[] {1}.SearchInRotatedArray(0);
// new int[] {2, 7, 11, 15}.Rotate(2);
// new int[] {0,1,0,2,1,0,1,3,2,1,2,1}.TrapRainWater();
// new int[,] {
// {1, 2, 3},
// {4, 5, 6},
// {7, 8, 9}
// }.PrintSpiral();
// new int[,] {
// {1, 2, 3},
// {4, 5, 6},
// {7, 8, 9}
// }.Rotate();
// Console.WriteLine(new int[,] {
// {0,30},
// {5,16},
// {15,20}
// }.MeetingRoomsII());
// Console.WriteLine("ezupkr".LongestCommonSubSequence("ubmrapg"));
// Console.WriteLine("abcde".LexicographicallySmallestString());
// Console.WriteLine("baaabbaabbba".MinMovesToObtainStringWithaAandBWithOut3ConsecutiveLetters());
// var node = tree.Serialize().Deserilize();
// BSTIterator iter = new BSTIterator(tree);
// string[] array = new string[]{"cat","cats","catsdogcats","dog","dogcatsdog","hippopotamuses","rat","ratcatdogcat"};
// Trie trie = array.ToTrie();
// var cw = trie.ConcatnatedWords(array);
// new int[] {9,9}.ToSingleLinkedList().Add(new int[] {1}.ToSingleLinkedList()).Print();
// new int[] {1,2,3,4}.ToSingleLinkedList().ReorderList().Print();
// Console.WriteLine("-2147483649".ToInteger());
Console.WriteLine(" hello world! ".ReverseWordsInString());
// var graph = new UndirectedGraph<int>();
// graph.Add(1,2);
// graph.Add(1,3);
// graph.Add(3,4);
// graph.Add(1,5);
// var graph2 = new UndirectedGraph<string>();
// graph2.Add("v","a");
// graph2.Add("am","si");
// graph2.Add("b","v");
// graph2.Add("v","b");
// var tree = new int[] { 5, 2, 12, 1, 3, 9, 21, 19, 25 }.ToBSTree();
// Console.WriteLine(tree.DistanceBetweenNodes(9, 25));
// var x = "mouse".Suggestions(new string[] { "mobile", "mouse", "moneypot", "monitor", "mousepad" });
// var x = new MinHeap();
// x.Add(3);
// x.Add(2);
// x.Add(3);
// x.Add(1);
// x.Add(2);
// x.Add(4);
var d = new int[] { 1, 1 }.MinSizeSubArraySum(3);
//var y = "ADOBECODEBANC".MinWindow("ABC");
var dit = new WordDictionary();
dit.AddWord("bad");
dit.AddWord("mad");
//x.Pop();
Console.WriteLine(dit.Search("mad"));
Console.ReadLine();
}
public void GenerateCache()
{
// if saved and words > 0
if (_Words.Count == 0)
{
MessageBox.Show(this, "Dictionary contains no words!", "No Words", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
if (this.Changed)
{
if (MessageBox.Show(this, "Dictionary should be saved before phonetic cache is added. \r\n \r\n Save Dictonary Now?",
"Save Dictonary", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes)
{
this.SaveDictionary();
}
else
{
return;
}
}
this.Cursor = Cursors.WaitCursor;
// load dictionary
WordDictionary dict = new WordDictionary();
dict.DictionaryFile = this.FileName;
dict.Initialize();
this.Cursor = Cursors.Default;
if (dict.PhoneticRules.Count == 0)
{
MessageBox.Show(this, "Dictionary does not contain phonetic rules!", "No Phonetic Rules", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
MainForm main = (MainForm)this.MdiParent;
this.Cursor = Cursors.WaitCursor;
for (int i = 0; i < _Words.Count; i++)
{
if (i % 1000 == 0)
{
main.statusBar.Text = string.Format("Word {0} of {1}", i, _Words.Count);
Application.DoEvents();
}
string[] parts = _Words[i].ToString().Split('/');
// part 1 = base word
string tempWord = parts[0];
// part 2 = affix keys
string tempKeys = "";
if (parts.Length >= 2)
{
tempKeys = parts[1];
}
// part 3 = phonetic code
string tempCode = dict.PhoneticCode(tempWord);
if (tempCode.Length > 0)
{
_Words[i] = string.Format("{0}/{1}/{2}", tempWord, tempKeys, tempCode);
}
}
main.statusBar.Text = "";
this.Changed = true;
this.Cursor = Cursors.Default;
MessageBox.Show(this, "Cache created successfully", "Successful", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
//public static Dictionary<BoardTile, List<CharTile>> GetValidCrossChecksOneWay(BoardTile[,] boardArray, WordDictionary dictionary)
//{
// var dawg = LoadDawg(dictionary.GameLanguage);
// Dictionary<BoardTile, List<CharTile>> validCrossChecks = new Dictionary<BoardTile, List<CharTile>>();
// List<int[]> tilesAlreadyChecked = new List<int[]>();
// StringBuilder sb = new StringBuilder();
// for (int i = 0; i < boardArray.GetLength(0); i++)
// {
// for (int j = 0; j < boardArray.GetLength(1); j++)
// {
// sb.Clear();
// if (boardArray[i, j].CharTile == null)
// {
// var upIndexCounter = i;
// var downIndexCounter = i;
// while (upIndexCounter > 0 && boardArray[upIndexCounter - 1, j].CharTile != null)
// {
// sb.Insert(0, boardArray[upIndexCounter - 1, j].CharTile.Letter);
// upIndexCounter--;
// }
// sb.Append("_");
// while (downIndexCounter < boardArray.GetLength(0) - 1 && boardArray[downIndexCounter + 1, j].CharTile != null)
// {
// sb.Append(boardArray[downIndexCounter + 1, j].CharTile.Letter);
// downIndexCounter++;
// }
// if (sb.Length == 1)
// {
// continue;
// }
// else
// {
// var wordWithUnderscore = sb.ToString();
// var sbTemp = new StringBuilder(wordWithUnderscore);
// if (!validCrossChecks.ContainsKey(boardArray[i, j]))
// {
// validCrossChecks.Add(boardArray[i, j], new List<CharTile>());
// }
// foreach (var c in dictionary.CharTiles)
// {
// if (c.Score == 0) continue;
// sbTemp.Replace('_', c.Letter);
// if (CheckWordValidity(dawg, sbTemp.ToString()))
// {
// if (validCrossChecks.ContainsKey(boardArray[i, j]))
// {
// if (!validCrossChecks[boardArray[i, j]].Contains(c))
// {
// validCrossChecks[boardArray[i, j]].Add(c);
// }
// }
// }
// sbTemp.Clear();
// sbTemp.Append(wordWithUnderscore);
// }
// }
// }
// }
// }
// return validCrossChecks;
//}
/// <summary>
/// Goes through a board and gets all anchors and crosschecks for board tiles
/// </summary>
/// <param name="boardArray">Array to go over. May be untransposed or transposed</param>
/// <param name="dictionary">Collection of CharTiles to look up to</param>
/// <param name="validCrossChecks">List of valid cross checks to update</param>
/// <param name="anchors">List of anchors to update</param>
public static void GetValidCrossChecksAndAnchors(BoardTile[,] boardArray, WordDictionary dictionary, Dictionary <BoardTile, List <CharTile> > validCrossChecks, List <int[]> anchors)
{
var dawg = LoadDawg(dictionary.GameLanguage);
StringBuilder sb = new StringBuilder();
var noMovesMadeYet = false;
var startTileRow = 0;
var startTileColumn = 0;
for (int i = 0; i < boardArray.GetLength(0); i++)
{
for (int j = 0; j < boardArray.GetLength(1); j++)
{
///If board tile has been played on the Start tile, the game hasn't started
if (boardArray[i, j].CharTile == null)
{
if (boardArray[i, j].BoardTileTypeID == BoardTileTypeEnum.Start)
{
startTileRow = i;
startTileColumn = j;
noMovesMadeYet = true;
}
///Gets letters from above and below current tile to form a word with a gap from the currently iterated board tile
sb.Clear();
var upIndexCounter = i;
var downIndexCounter = i;
while (upIndexCounter > 0 && boardArray[upIndexCounter - 1, j].CharTile != null)
{
sb.Insert(0, boardArray[upIndexCounter - 1, j].CharTile.Letter);
upIndexCounter--;
}
sb.Append("_");
while (downIndexCounter < boardArray.GetLength(0) - 1 && boardArray[downIndexCounter + 1, j].CharTile != null)
{
sb.Append(boardArray[downIndexCounter + 1, j].CharTile.Letter);
downIndexCounter++;
}
//If no word is formed, continue to next board tile
if (sb.Length == 1)
{
continue;
}
//Else section replaces gap with all letters in alphabet to see which letters would fit to make a valid word
//Letters form the valid cross check set
//Cross check is stored as a dictionary object of board tile and a collection of char tiles
//This means for a given board tile, the following letters from the char tiles are valid
//If the collection is empty, no letter is a valid fit for the board tile
//If board tile (key) entry doesn't exist in the list, all letters are valid for that board tile (for now)
else
{
var wordWithUnderscore = sb.ToString();
var sbTemp = new StringBuilder(wordWithUnderscore);
if (!validCrossChecks.ContainsKey(boardArray[i, j]))
{
validCrossChecks.Add(boardArray[i, j], new List <CharTile>());
}
foreach (var c in dictionary.CharTiles)
{
if (c.Score == 0)
{
continue;
}
sbTemp.Replace('_', c.Letter);
if (CheckWordValidity(dawg, sbTemp.ToString()))
{
if (validCrossChecks.ContainsKey(boardArray[i, j]))
{
if (!validCrossChecks[boardArray[i, j]].Contains(c))
{
validCrossChecks[boardArray[i, j]].Add(c);
}
}
}
sbTemp.Clear();
sbTemp.Append(wordWithUnderscore);
}
}
//Checks surrounding tiles if they can be used as anchors
}
else
{
if (i > 0 && boardArray[i - 1, j].CharTile == null)
{
if (anchors != null)
{
var coordinates = new int[] { i - 1, j };
if (!anchors.Any(c => c[0] == coordinates[0] && c[1] == coordinates[1]))
{
anchors.Add(coordinates);
}
}
}
if (i < boardArray.GetLength(0) - 1 && boardArray[i + 1, j].CharTile == null)
{
if (anchors != null)
{
var coordinates = new int[] { i + 1, j };
if (!anchors.Any(c => c[0] == coordinates[0] && c[1] == coordinates[1]))
{
anchors.Add(coordinates);
}
}
}
if (j > 0 && boardArray[i, j - 1].CharTile == null)
{
{
var coordinates = new int[] { i, j - 1 };
if (!anchors.Any(c => c[0] == coordinates[0] && c[1] == coordinates[1]))
{
anchors.Add(coordinates);
}
}
}
if (j < boardArray.GetLength(1) - 1 && boardArray[i, j + 1].CharTile == null)
{
var coordinates = new int[] { i, j + 1 };
if (!anchors.Any(c => c[0] == coordinates[0] && c[1] == coordinates[1]))
{
anchors.Add(coordinates);
}
}
}
}
}
if (noMovesMadeYet)
{
anchors.Clear();
anchors.Add(new int[] { startTileRow, startTileColumn });
}
return;
}
public void SelectDictionary(WordDictionary dict)
{
}
protected void Page_Init(object sender, EventArgs e)
{
string cacheName = "WordDictionary|" + LocalizationContext.PreferredCultureCode;
// Get dictionary from cache
WordDictionary = (WordDictionary)CacheHelper.GetItem(cacheName);
if (WordDictionary == null)
{
// If not in cache, create new
WordDictionary = new WordDictionary();
WordDictionary.EnableUserFile = false;
// Getting folder for dictionaries
string folderName = "~/App_Data/Dictionaries";
folderName = MapPath(folderName);
string culture = LocalizationContext.PreferredCultureCode;
string dictFile = culture + ".dic";
string dictPath = Path.Combine(folderName, dictFile);
if (!File.Exists(dictPath))
{
// Get default culture
string defaultCulture = ValidationHelper.GetString(SettingsHelper.AppSettings["CMSDefaultSpellCheckerCulture"], String.Empty);
if (!String.IsNullOrEmpty(defaultCulture))
{
culture = defaultCulture;
dictFile = defaultCulture + ".dic";
dictPath = Path.Combine(folderName, dictFile);
}
}
if (!File.Exists(dictPath))
{
lblError.Text = string.Format(GetString("SpellCheck.DictionaryNotExists"), culture);
lblError.Visible = true;
return;
}
mDictExists = true;
WordDictionary.DictionaryFolder = folderName;
WordDictionary.DictionaryFile = dictFile;
// Load and initialize the dictionary
WordDictionary.Initialize();
// Store the Dictionary in cache
var path = Path.Combine(folderName, WordDictionary.DictionaryFile);
var cd = CacheHelper.GetFileCacheDependency(path);
CacheHelper.Add(cacheName, WordDictionary, cd, Cache.NoAbsoluteExpiration, Cache.NoSlidingExpiration);
}
else
{
mDictExists = true;
}
// Create spell checker
SpellChecker = new Spelling();
SpellChecker.ShowDialog = false;
SpellChecker.Dictionary = WordDictionary;
SpellChecker.IgnoreAllCapsWords = false;
// Adding events
SpellChecker.MisspelledWord += SpellChecker_MisspelledWord;
SpellChecker.EndOfText += SpellChecker_EndOfText;
SpellChecker.DoubledWord += SpellChecker_DoubledWord;
}
protected void Page_Init(object sender, EventArgs e)
{
string cacheName = "WordDictionary|" + LocalizationContext.PreferredCultureCode;
// Get dictionary from cache
WordDictionary = (WordDictionary)CacheHelper.GetItem(cacheName);
if (WordDictionary == null)
{
// If not in cache, create new
WordDictionary = new WordDictionary();
WordDictionary.EnableUserFile = false;
// Getting folder for dictionaries
string folderName = "~/App_Data/Dictionaries";
folderName = MapPath(folderName);
string culture = LocalizationContext.PreferredCultureCode;
string dictFile = culture + ".dic";
string dictPath = Path.Combine(folderName, dictFile);
if (!File.Exists(dictPath))
{
// Get default culture
string defaultCulture = ValidationHelper.GetString(SettingsHelper.AppSettings["CMSDefaultSpellCheckerCulture"], String.Empty);
if (!String.IsNullOrEmpty(defaultCulture))
{
culture = defaultCulture;
dictFile = defaultCulture + ".dic";
dictPath = Path.Combine(folderName, dictFile);
}
}
if (!File.Exists(dictPath))
{
lblError.Text = string.Format(GetString("SpellCheck.DictionaryNotExists"), culture);
lblError.Visible = true;
return;
}
mDictExists = true;
WordDictionary.DictionaryFolder = folderName;
WordDictionary.DictionaryFile = dictFile;
// Load and initialize the dictionary
WordDictionary.Initialize();
// Store the Dictionary in cache
var path = Path.Combine(folderName, WordDictionary.DictionaryFile);
var cd = CacheHelper.GetFileCacheDependency(path);
CacheHelper.Add(cacheName, WordDictionary, cd, Cache.NoAbsoluteExpiration, Cache.NoSlidingExpiration);
}
else
{
mDictExists = true;
}
// Create spell checker
SpellChecker = new Spelling();
SpellChecker.ShowDialog = false;
SpellChecker.Dictionary = WordDictionary;
SpellChecker.IgnoreAllCapsWords = false;
// Adding events
SpellChecker.MisspelledWord += SpellChecker_MisspelledWord;
SpellChecker.EndOfText += SpellChecker_EndOfText;
SpellChecker.DoubledWord += SpellChecker_DoubledWord;
}
public void Setup()
{
fileSystem = new FileSystem();
dict = new WordDictionary(fileSystem);
}
private bool FillCells(List <CellInfo> startCells)
{
if (Stopping)
{
return(false);
}
if (startCells.Count == 0)
{
return(true);
}
CellInfo startCell = startCells[0];
if ((startCell.isAcross && startCell.cell.hasAcrossWord) ||
(!startCell.isAcross && startCell.cell.hasDownWord))
{
List <CellInfo> newStartCells = new List <CellInfo>(startCells);
newStartCells.RemoveAt(0);
return(FillCells(newStartCells));
}
int iStart = startCell.cell.i;
int jStart = startCell.cell.j;
int len = startCell.isAcross ? startCell.cell.acrossLen : startCell.cell.downLen;
bool isAcross = startCell.isAcross;
List <string> possibleWords = new List <string>(WordDictionary.GetPossibleWords(cells, iStart, jStart, len, isAcross));
if (possibleWords != null)
{
for (int i = 0; i < possibleWords.Count; i++)
{
if (Stopping)
{
return(false);
}
// Get a random word to try from the list of possible words
int randIndex = rng.Next(i, possibleWords.Count);
string wordToTry = possibleWords[randIndex];
// Swap the words so if it does fit we wont pick it again in the next iteration of the for loop
possibleWords[randIndex] = possibleWords[i];
possibleWords[i] = wordToTry;
if (usedWords.Contains(wordToTry))
{
continue;
}
List <CellInfo> newStartCells = new List <CellInfo>();
List <Cell> changedCells = new List <Cell>();
bool canWordFit = true;
// Place the words characters in the cells
for (int j = 0; j < len; j++)
{
int iCell = iStart + (isAcross ? 0 : j);
int jCell = jStart + (isAcross ? j : 0);
Cell cell = cells[iCell][jCell];
if (cell.character == ' ')
{
cell.character = wordToTry[j];
changedCells.Add(cell);
// If the crossing word is only length one then we dont need to check it
if ((isAcross && cell.downStart.downLen == 1) ||
(!isAcross && cell.acrossStart.acrossLen == 1))
{
continue;
}
newStartCells.Add(new CellInfo(isAcross ? cell.downStart : cell.acrossStart, !isAcross));
// Now check that there is aleast on possible word for the crossing cells
Cell crossStart = isAcross ? cell.downStart : cell.acrossStart;
int crossLen = isAcross ? crossStart.downLen : crossStart.acrossLen;
List <string> possibleCrossWords = WordDictionary.GetPossibleWords(cells, crossStart.i, crossStart.j, crossLen, !isAcross);
if (possibleCrossWords == null || possibleCrossWords.Count == 0)
{
canWordFit = false;
break;
}
}
}
if (canWordFit)
{
if (isAcross)
{
startCell.cell.hasAcrossWord = true;
}
else
{
startCell.cell.hasDownWord = true;
}
// Add the start cells that still need words on them
for (int j = 1; j < startCells.Count; j++)
{
newStartCells.Add(startCells[j]);
}
usedWords.Add(wordToTry);
// Try filling the rest of the cells
if (FillCells(newStartCells))
{
return(true);
}
usedWords.RemoveAt(usedWords.Count - 1);
if (isAcross)
{
startCell.cell.hasAcrossWord = false;
}
else
{
startCell.cell.hasDownWord = false;
}
}
// Set the changed cells character back to blank
for (int j = 0; j < changedCells.Count; j++)
{
changedCells[j].character = ' ';
}
}
}
// Non of the possible words worked
return(false);
}
public int GetUniqueWordCount()
{
return(WordDictionary?.Select(x => x.Key).Distinct().Count() ?? 0);
}
public void ConfigureServices(IServiceCollection services)
{
services.AddSingleton <IConfiguration>(Configuration);
services
.AddAuthentication(o =>
{
o.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme;
})
.AddCookie(o =>
{
o.LoginPath = "/signin";
o.LogoutPath = "/signout";
o.Cookie.HttpOnly = true;
o.Cookie.MaxAge = TimeSpan.FromDays(1);
o.ExpireTimeSpan = TimeSpan.FromDays(1);
o.SlidingExpiration = true;
})
.AddOpenIdConnect(options =>
{
options.ClientId = "keepitsafer";
options.ClientSecret = "16bc0837-a54d-4bf1-88b7-923724ce7e63";
options.GetClaimsFromUserInfoEndpoint = true;
options.SaveTokens = true;
options.ResponseType = OpenIdConnectResponseType.Code;
options.AuthenticationMethod = OpenIdConnectRedirectBehavior.RedirectGet;
options.Authority = "https://smallauth.nosuchblogger.com/";
options.Scope.Add("roles");
options.SecurityTokenValidator = new JwtSecurityTokenHandler
{
InboundClaimTypeMap = new Dictionary <string, string>()
};
options.TokenValidationParameters.NameClaimType = "name";
options.TokenValidationParameters.RoleClaimType = "role";
options.AccessDeniedPath = "/";
});
services
.AddDataProtection()
.SetApplicationName(typeof(Startup).Namespace)
.PersistKeysToFileSystem(new DirectoryInfo(Path.Combine(hostingEnvironment.ContentRootPath, ".keys")));
services.AddLogging(logging =>
{
logging.AddConsole();
logging.AddDebug();
logging.SetMinimumLevel(LogLevel.Trace);
});
services.Configure <CookiePolicyOptions>(o =>
{
o.CheckConsentNeeded = context => false;
o.MinimumSameSitePolicy = SameSiteMode.None;
});
services.Configure <DropboxConfig>(Configuration.GetSection("Dropbox"));
services.AddDbContext <SqliteDataContext>();
services.AddMvc().AddSessionStateTempDataProvider();
services.AddRazorPages();
services.AddCors();
services.AddDistributedMemoryCache();
services.AddSession(options => options.IdleTimeout = TimeSpan.FromMinutes(5));
services.AddScoped <IUserAccountRepository, UserAccountRepository>();
services.AddSingleton <WordDictionary>(serviceProvider =>
{
var dict = new WordDictionary();
Task.Run(() => dict.LoadAsync());
return(dict);
});
services.AddTransient <RandomPasswordGenerator>();
}
/// <summary>
/// 找到导入库和现有库的不同
/// </summary>
/// <param name="NewDicFile">导入库文件</param>
/// <param name="Encoding">导入库文件编码</param>
/// <param name="DicFormat">导入库文件格式</param>
/// <param name="SourceDict">原库对象</param>
/// <param name="OddLines">输出没有词性标注且现有库中也没有的词行</param>
/// <param name="NewWords">输出新词或现有词的新词性</param>
/// <param name="ExistWords">输出重复词，且词性也相同</param>
/// <param name="MaxFrqRate">重复词的最大词频比例</param>
/// <param name="MinFrqRate">重复词的最小词频比例</param>
/// <param name="AvgFrqRate">重复词的平均词频比例</param>
public static void FindDifferent(string NewDicFile, Encoding Encoding, DictionaryFormat DicFormat, WordDictionary SourceDict,
out string[] OddLines, out WordDictionary.DicWordInfo[] NewWords, out WordDictionary.DicWordInfo[] ExistWords,
out double MaxFrqRate, out double MinFrqRate, out double AvgFrqRate)
{
//初始化
MaxFrqRate = double.MinValue; MinFrqRate = double.MaxValue; decimal SumFrqRate = 0;
//const string[] CheckPos = new string[] { "n", "ns", "nr", "ng", "v", "j", "m", "vn", "a", "q" };
//准备词性转换
Dictionary<string, string> PosTrans = getPosTransformMap(DicFormat);
//加载词库
Dictionary<string, WordDictionary.DicWordInfo> OldWords = SourceDict.ToWordDictionary(); ;
//内存词组
List<string> Odds = new List<string>(OldWords.Count / 2);
List<WordDictionary.DicWordInfo> Exists = new List<SharpICTCLAS.WordDictionary.DicWordInfo>(OldWords.Count / 2);
List<WordDictionary.DicWordInfo> News = new List<WordDictionary.DicWordInfo>(OldWords.Count / 2);
//加载词库并统计库内有的词的词频，以估算词频转换的比例关系
foreach (string Line in File.ReadAllLines(NewDicFile, Encoding))
{
string Word;
int Frq;
string Poses;
switch (DicFormat)
{
case DictionaryFormat.SogouW2006:
string[] s = Line.Split('\t', ' ');
Word = s[0];
Frq = s.Length == 1 ? -1 : int.Parse(s[1]);
Poses = s.Length < 2 ? null : s[2];
break;
case DictionaryFormat.ExcelCSV:
default:
int p1 = Line.IndexOf(',');
int p2 = Line.IndexOf(',', p1 + 1);
Word = Line.Substring(0, p1);
Frq = int.Parse(Line.Substring(p1 + 1, p2 - p1 - 1));
Poses = Line.Substring(p2 + 1).Trim('"').Trim();
break;
}
if (string.IsNullOrEmpty(Poses))
{
if (!OldWords.ContainsKey(Word.ToLower())) Odds.Add(Line);
continue;
}
foreach (string InputPos in Poses.TrimEnd(',').Split(','))
{
if (string.IsNullOrEmpty(InputPos)) continue;
//如果映射表中没有，则保留原始词性字母
string Pos = PosTrans.ContainsKey(InputPos.ToLower()) ? PosTrans[InputPos.ToLower()] : InputPos.ToLower();
//是否存在
if (OldWords.ContainsKey(Word.ToLower()) && OldWords[Word.ToLower()].Pos.Contains(Pos))
{
int SourceFrq = OldWords[Word.ToLower()].Frequence;
double FrqR = SourceFrq == 0 ? Frq : (double)Frq / SourceFrq;
if (FrqR > MaxFrqRate) MaxFrqRate = FrqR;
if (FrqR < MinFrqRate) MinFrqRate = FrqR;
SumFrqRate += (decimal)FrqR;
Exists.Add(new WordDictionary.DicWordInfo(Word, Pos, Frq));
}
else //新词或新词性
{
News.Add(new WordDictionary.DicWordInfo(Word, Pos, Frq));
}
}
}
//平均频度转换倍数
AvgFrqRate = Exists.Count > 0 ? Convert.ToDouble(SumFrqRate / Exists.Count) : 0;
OddLines = Odds.ToArray();
NewWords = News.ToArray();
ExistWords = Exists.ToArray();
}
public static WordDictionary ExtractWords(string path)
{
WordDictionary dict = new WordDictionary();
int lineCount = 0;
StringBuilder defBuilder = new StringBuilder();
DictionaryEntry currentEntry = null;
int id = 0;
foreach (string line in File.ReadLines(path))
{
if (currentEntry == null)
{
if (String.IsNullOrEmpty(line))
{
goto next;
}
foreach (char c in line)
{
if (c == ' ' || c == '.')
{
goto next;
}
if (!Char.IsUpper(c))
{
goto next;
}
}
defBuilder.Clear();
currentEntry = new DictionaryEntry();
currentEntry.Word = line.ToLowerInvariant();
lineCount = 0;
}
else
{
// Extract word type information
if (lineCount == 1)
{
string[] parts = line.Split(',', '.');
currentEntry.Type = parts[1].Trim();
}
else
{
if (line.StartsWith("Defn: "))
{
defBuilder.Append(line.Substring(6));
}
else
{
if (line.Trim() == String.Empty && defBuilder.Length != 0)
{
currentEntry.Definition = defBuilder.ToString();
currentEntry.Id = id;
id++;
dict[currentEntry.Word] = currentEntry;
Console.WriteLine(currentEntry.Word);
currentEntry = null;
}
else if (line.Trim() != String.Empty && defBuilder.Length != 0)
{
defBuilder.Append(line);
}
}
}
}
next:
id = id;
}
return(dict);
}
// To DO List:
// 1. Maybe needed to refactor code
// 2. Decrypt string 6 by some way
#endregion
static void Main(string[] args)
{
#region Task2
CeasarDecrypt ceasarDecrypt = new CeasarDecrypt
(File.ReadAllText("../../../Decryption/SubstitutionDecript/tasks/task2.txt"));
string result2 = ceasarDecrypt.Decrypt();
Console.WriteLine("Ceasar Decrypt:\n" + result2 + "\n");
#endregion
#region Task3
VigenereDecrypt vigenereDecrypt = new VigenereDecrypt(DecodeHexToUTF8
(File.ReadAllText("../../../Decryption/SubstitutionDecript/tasks/task3.txt")), null);
string result3 = vigenereDecrypt.Decrypt();
Console.WriteLine("Vigenere Decrypt:\n" + result3 + "\n");
Console.WriteLine("Press any key to start mono substitution decrypt");
Console.ReadKey();
#endregion
#region Task4
Console.Clear();
Console.WriteLine("Initializing dictionaries...");
Stopwatch task4Stopwatch = new Stopwatch();
task4Stopwatch.Start();
string task4 = File.ReadAllText("../../../Decryption/SubstitutionDecript/tasks/task4.txt").ToLower();
GeneticModel gm = new GeneticModel(task4);
string result4 = gm.Run();
task4Stopwatch.Stop();
Console.WriteLine("Result:\n");
List <string> output = Decryption.WordNinja.WordNinja.Split(result4);
foreach (var elem in output)
{
Console.Write(elem + " ");
}
Console.WriteLine("\n");
Console.WriteLine("Execution time: " + (task4Stopwatch.ElapsedMilliseconds / 1000.0).ToString("F3") + " seconds\n");
Console.WriteLine("Press any key to start poly substitution decrypt");
Console.ReadKey();
#endregion
#region Task5
Console.Clear();
Console.WriteLine("Initializing dictionaries...");
Stopwatch task5Stopwatch = new Stopwatch();
task5Stopwatch.Start();
int[] ngrams = { 2, 3, 4 };
var dict = DictionaryNGrams.GetDictionaryNGrams().Dictionary;
var words = WordDictionary.GetWordDictionary().Dictionary;
var smallWords = WordDictionary.GetWordDictionary().SmallDictionary;
string task5 = File.ReadAllText("../../../Decryption/SubstitutionDecript/tasks/task5.txt").ToLower();
string result5 = NewGeneticAlgo.GeneticAlgo(task5, 6, ngrams, dict, words, smallWords);
task5Stopwatch.Stop();
Console.WriteLine(result5);
Console.WriteLine("Execution time: " + task5Stopwatch.Elapsed.Minutes + " minutes " + task5Stopwatch.Elapsed.Seconds + " seconds");
#endregion
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
public override void AddWordTemplate(Random.RandomSourceBase randomness, WordDictionary dictionary, IList <WordTemplate.Template> currentTemplate)
{
_ = randomness ?? throw new ArgumentNullException(nameof(randomness));
_ = dictionary ?? throw new ArgumentNullException(nameof(dictionary));
_ = currentTemplate ?? throw new ArgumentNullException(nameof(currentTemplate));
this._LastVerbTemplateIndex = -1;
// Figuring out if the verb will be plural or not is... well... complicated.
bool subjectIsPlural;
int speechVerbIdx = -1, indefiniteNounIdx = -1;
bool containsDirectSpeech = currentTemplate.OfType <SpeechVerbTemplate>().Any();
bool containsIndefiniteNoun = currentTemplate.OfType <IndefinitePronounTemplate>().Any();
if (containsDirectSpeech)
{
speechVerbIdx = CollectionHelper.IndexOfType(currentTemplate, typeof(SpeechVerbTemplate));
}
if (containsIndefiniteNoun)
{
indefiniteNounIdx = CollectionHelper.IndexOfType(currentTemplate, typeof(IndefinitePronounTemplate));
}
var ntemp = !containsDirectSpeech?currentTemplate.OfType <NounTemplate>().FirstOrDefault()
: currentTemplate.Skip(speechVerbIdx).OfType <NounTemplate>().FirstOrDefault();
var ptemp = containsIndefiniteNoun ? (currentTemplate[indefiniteNounIdx] as IndefinitePronounTemplate) : null;
if (ptemp == null && ntemp == null)
{
subjectIsPlural = false; // Proper nouns are never plural.
}
else if (ptemp == null && ntemp != null)
{
subjectIsPlural = ntemp.IsPlural;
}
else if (ptemp != null && ntemp == null)
{
subjectIsPlural = ptemp.IsPlural;
}
else if (ptemp != null && ntemp != null)
{
// This probably shouldn't happen, but if it does, we'll take the noun.
subjectIsPlural = ntemp.IsPlural;
}
else
{
throw new Exception("Unexpected state.");
}
var verbFormToBePlural = subjectIsPlural;
// Choose how to handle intransitive verbs.
bool selectTransitive = true;
bool removeAccusativeNoun = false;
bool addPreposition = false;
int choice = 0;
if (IntransitiveByNoNounClauseFactor > 0 || IntransitiveByPrepositionFactor > 0)
{
choice = randomness.Next(dictionary.CountOf <Verb>()); // Choose between transitive or not by the number of trans/intrans verbs in dictionary.
selectTransitive = choice < dictionary.CountOfTransitiveVerbs();
if (!selectTransitive)
{
// OK, so we chose an intransitive verb, how will we handle that?
// Note that we've established either IntransitiveByNoNounClauseFactor or IntransitiveByPrepositionFactor is greater than zero, so WeightedCoinFlip() will never throw.
bool handleIntransByNoObjectNoun = randomness.WeightedCoinFlip(IntransitiveByNoNounClauseFactor, IntransitiveByPrepositionFactor);
if (handleIntransByNoObjectNoun)
{
// Remove the noun clause.
removeAccusativeNoun = true;
}
else
{
// Add a preposition.
addPreposition = true;
}
}
}
bool makeInterrogative = (InterrogativeFactor + NoInterrogativeFactor > 0) && // Case where neither is specified: assume no interrogative.
randomness.WeightedCoinFlip(InterrogativeFactor, NoInterrogativeFactor);
if (makeInterrogative && containsDirectSpeech)
{
// Insert an interrogative template after the direct speech verb.
currentTemplate.Insert(speechVerbIdx + 1, new InterrogativeTemplate(subjectIsPlural));
}
else if (makeInterrogative && !containsDirectSpeech)
{
// Insert an interrogative template at the start of the phrase.
currentTemplate.Insert(0, new InterrogativeTemplate(subjectIsPlural));
}
// Include adverb?
bool includeAdverb = (AdverbFactor + NoAdverbFactor > 0) && // Case where neither is specified: assume no adverb.
randomness.WeightedCoinFlip(AdverbFactor, NoAdverbFactor);
bool includeAdverbBeforeVerb = randomness.CoinFlip();
if (includeAdverb && includeAdverbBeforeVerb)
{
currentTemplate.Add(new AdverbTemplate());
}
// Select a verb tense form.
this.BuildTable();
choice = randomness.Next(this.DistributionMax);
var tense = VerbTense.Present;
if (!makeInterrogative)
{
// The the verb form becomes present plural whenever an interrogative is used.
tense = this.LookupTenseFromChoice(choice);
}
if (makeInterrogative)
{
verbFormToBePlural = true;
}
currentTemplate.Add(new VerbTemplate(tense, verbFormToBePlural, selectTransitive));
// Include adverb after the verb.
if (includeAdverb && !includeAdverbBeforeVerb)
{
currentTemplate.Add(new AdverbTemplate());
}
// Add a preposition to make the intransitive verb work?
if (addPreposition)
{
currentTemplate.Add(new PrepositionTemplate());
}
// Signal to the second pass we're going to remove the accusative noun clause.
if (removeAccusativeNoun)
{
_LastVerbTemplateIndex = currentTemplate.Count - 1;
}
}
private void OpenFile_Click(object sender, RoutedEventArgs e)
{
var openFileDialog = new OpenFileDialog();
if (openFileDialog.ShowDialog() == true)
{
string filePath = openFileDialog.FileName;
string text = File.ReadAllText(filePath);
if (NLPEngine.TextIsTagged(text))
{
db.AddText(new Text(filePath));
int counter = 0;
var matches = Regex.Matches(text, @"(?<word>[a-zA-Z][a-zA-Z-—']*)_(?<tag>[a-zA-Z?$]*)").Cast <Match>().ToArray();
var fileName = Path.GetFileName(filePath);
string previousTag = "NN";
foreach (var match in matches)
{
string word = match.Groups["word"].Value;
string tag = match.Groups["tag"].Value;
var currentWord = WordDictionary.FirstOrDefault(x => x.Name == word);
if (currentWord == null)
{
WordDictionary.Add(new Word(word, 1, tag + "_1", fileName));
}
else
{
currentWord.IncrementAmountAndAddNewTagAndFile(tag, fileName);
}
Tags.First(x => x.Name == previousTag).AddNewTag(tag);
var pair = TagPairs.FirstOrDefault(x => x.FirstTag == previousTag && x.SecondTag == tag);
if (pair == null)
{
pair = new TagPair
{
FirstTag = previousTag,
SecondTag = tag,
Amount = 0
};
TagPairs.Add(pair);
}
pair.Amount++;
previousTag = tag;
counter++;
if (counter % 100 == 0)
{
var progress = counter * 100.0 / matches.Length;
ProgressBar.Dispatcher.Invoke(() => ProgressBar.Value = progress, DispatcherPriority.Background);
}
}
ProgressBar.Dispatcher.Invoke(() => ProgressBar.Value = 0.0, DispatcherPriority.Background);
Task.Factory.StartNew(() => SaveWordDictionary());
StatusLine.Text = $"{openFileDialog.FileName} has been parsed.";
}
else
{
MessageBox.Show("Please select new tagged text \nor tag current text using \"Tag text\" window.",
"This text is not tagged", MessageBoxButton.OK, MessageBoxImage.Warning);
}
}
}
public override void SecondPassOfWordTemplate(Random.RandomSourceBase randomness, WordDictionary dictionary, IList <WordTemplate.Template> currentTemplate)
{
_ = randomness ?? throw new ArgumentNullException(nameof(randomness));
_ = dictionary ?? throw new ArgumentNullException(nameof(dictionary));
_ = currentTemplate ?? throw new ArgumentNullException(nameof(currentTemplate));
// If we deal with intransitive by having no object, remove the object!
if (this._LastVerbTemplateIndex >= 0)
{
while (currentTemplate.Count > this._LastVerbTemplateIndex + 1)
{
currentTemplate.RemoveAt(this._LastVerbTemplateIndex + 1);
}
}
}
public override void SecondPassOfWordTemplate(Random.RandomSourceBase randomness, WordDictionary dictionary, IList <WordTemplate.Template> currentTemplate)
{
// No-op.
}
/// <summary>
/// Did you mean suggestion.
/// </summary>
private static string DidYouMean(string dictionaryFile, string searchQuery, List<string> searchTerms)
{
if (searchTerms != null)
{
Spelling SpellChecker = null;
WordDictionary WordDictionary = null;
#region "Word dictionary"
// If not in cache, create new
WordDictionary = new WordDictionary();
WordDictionary.EnableUserFile = false;
// Getting folder for dictionaries
string folderName = HttpContext.Current.Request.MapPath("~/App_Data/Dictionaries/");
// Check if dictionary file exists
string fileName = Path.Combine(folderName, dictionaryFile);
if (!File.Exists(fileName))
{
EventLogProvider eventLog = new EventLogProvider();
eventLog.LogEvent(EventLogProvider.EVENT_TYPE_ERROR, DateTime.Now, "DidYouMean webpart", "Dictionary file not found!");
return String.Empty;
}
WordDictionary.DictionaryFolder = folderName;
WordDictionary.DictionaryFile = dictionaryFile;
// Load and initialize the dictionary
WordDictionary.Initialize();
#endregion
#region "SpellCheck"
// Prepare spellchecker
SpellChecker = new Spelling();
SpellChecker.Dictionary = WordDictionary;
SpellChecker.SuggestionMode = Spelling.SuggestionEnum.NearMiss;
bool suggest = false;
// Check all searched terms
foreach (string term in searchTerms)
{
if (term.Length > 2)
{
SpellChecker.Suggest(term);
ArrayList al = SpellChecker.Suggestions;
// If there are some suggestions
if ((al != null) && (al.Count > 0))
{
suggest = true;
// Expression to find term
Regex regex = RegexHelper.GetRegex("([\\+\\-\\s\\(]|^)" + term + "([\\s\\)]|$)");
// Change term in original search query
string suggestion = "$1" + startTag + al[0] + endTag + "$2";
searchQuery = regex.Replace(searchQuery, suggestion);
}
}
}
#endregion
if (suggest)
{
return searchQuery;
}
}
return String.Empty;
}
/// <summary>
/// Did you mean suggestion.
/// </summary>
private static string DidYouMean(string dictionaryFile, string searchQuery, IEnumerable <string> searchTerms)
{
if (searchTerms != null)
{
Spelling SpellChecker = null;
WordDictionary WordDictionary = null;
#region "Word dictionary"
// If not in cache, create new
WordDictionary = new WordDictionary();
WordDictionary.EnableUserFile = false;
// Getting folder for dictionaries
string folderName = HttpContext.Current.Request.MapPath("~/App_Data/Dictionaries/");
// Check if dictionary file exists
string fileName = Path.Combine(folderName, dictionaryFile);
if (!File.Exists(fileName))
{
EventLogProvider.LogEvent(EventType.ERROR, "DidYouMean webpart", "Dictionary file not found!");
return(String.Empty);
}
WordDictionary.DictionaryFolder = folderName;
WordDictionary.DictionaryFile = dictionaryFile;
// Load and initialize the dictionary
WordDictionary.Initialize();
#endregion
#region "SpellCheck"
// Prepare spellchecker
SpellChecker = new Spelling();
SpellChecker.Dictionary = WordDictionary;
SpellChecker.SuggestionMode = Spelling.SuggestionEnum.NearMiss;
bool suggest = false;
// Check all searched terms
foreach (string term in searchTerms)
{
if (term.Length > 2)
{
SpellChecker.Suggest(term);
ArrayList al = SpellChecker.Suggestions;
// If there are some suggestions
if ((al != null) && (al.Count > 0))
{
suggest = true;
// Expression to find term
Regex regex = RegexHelper.GetRegex("([\\+\\-\\s\\(]|^)" + term + "([\\s\\)]|$)");
// Change term in original search query
string suggestion = "$1" + startTag + al[0] + endTag + "$2";
searchQuery = regex.Replace(searchQuery, suggestion);
}
}
}
#endregion
if (suggest)
{
return(searchQuery);
}
}
return(String.Empty);
}
public static void TestBiSegment()
{
List <string> sentence = new List <string>();
List <string> description = new List <string>();
sentence.Add(@"他说的的确实在理");
description.Add(@"普通分词测试");
sentence.Add(@"张华平3－4月份来北京开会");
description.Add(@"数字切分");
sentence.Add(@"1.加强管理");
description.Add(@"剔除多余的“.”");
sentence.Add(@"他出生于1980年1月1日10点");
description.Add(@"日期合并");
sentence.Add(@"他出生于甲子年");
description.Add(@"年份识别");
sentence.Add(@"馆内陈列周恩来和邓颖超生前使用过的物品");
description.Add(@"姓名识别");
WordDictionary coreDict = new WordDictionary();
if (!coreDict.Load(coreDictFile))
{
Console.WriteLine("coreDict 字典装入错误！");
return;
}
WordDictionary biDict = new WordDictionary();
if (!biDict.Load(biDictFile))
{
Console.WriteLine("字典装入错误！");
return;
}
string sSentence;
string sDescription;
for (int i = 0; i < sentence.Count; i++)
{
sSentence = sentence[i];
sDescription = description[i];
Console.WriteLine("\r\n============ {0} ============", sDescription);
sSentence = Predefine.SENTENCE_BEGIN + sSentence + Predefine.SENTENCE_END;
List <AtomNode> nodes = Segment.AtomSegment(sSentence);
Console.WriteLine("原子切分：");
for (int j = 0; j < nodes.Count; j++)
{
Console.Write("{0}, ", nodes[j].sWord);
}
Console.WriteLine("\r\n\r\n实际切分：");
Segment segment = new Segment(biDict, coreDict);
segment.BiSegment(sSentence, 0.1, 1);
for (int k = 0; k < segment.m_pWordSeg.Count; k++)
{
for (int j = 0; j < segment.m_pWordSeg[k].Length; j++)
{
Console.Write("{0}, ", segment.m_pWordSeg[k][j].sWord);
}
Console.WriteLine();
}
}
}
public WordDictionary AddWord(string original, string translate)
{
var requestEngInsert =
$"INSERT INTO [dbo].[Words] ([value],[language]) VALUES (@original, 'eng'); select scope_identity() as id;";
var requestRusInsert =
$"INSERT INTO [dbo].[Words] ([value],[language]) VALUES (@translate, 'ru'); select scope_identity() as id;";
int engId = 0;
int rusId = 0;
using (var connection = Connection)
{
var responseObject = new WordDictionary();
connection.Open();
SqlCommand command = new SqlCommand(requestEngInsert, connection);
command.Parameters.AddWithValue("@original", original);
var response = command.ExecuteReader();
if (response.HasRows)
{
while (response.Read())
{
string respId = "";
// Каст (int) невозможен, поэтому костыль
respId = response["id"].ToString();
engId = Int32.Parse(respId);
}
}
else
{
_logger.Error($"[WordDictionaryRepository] Cant add word {original}");
throw new ArgumentException($"Cant add word {original}");
}
response.Close();
command = new SqlCommand(requestRusInsert, connection);
command.Parameters.AddWithValue("@translate", translate);
response = command.ExecuteReader();
if (response.HasRows)
{
while (response.Read())
{
string respId = "";
// Каст (int) невозможен, поэтому костыль
respId = response["id"].ToString();
rusId = Int32.Parse(respId);
}
}
else
{
_logger.Error($"[WordDictionaryRepository] Cant add word {translate}");
throw new ArgumentException($"Cant add word {translate}");
}
response.Close();
if (engId == 0 || rusId == 0)
{
throw new ArgumentException();
}
var requestDictionaryInsert =
$"INSERT INTO [dbo].[Dictionary] ([english_id],[russian_id]) VALUES (@engId, @rusId); select scope_identity() as id;";
command = new SqlCommand(requestDictionaryInsert, connection);
command.Parameters.AddWithValue("@engId", engId);
command.Parameters.AddWithValue("@rusId", rusId);
response = command.ExecuteReader();
if (response.HasRows)
{
while (response.Read())
{
string respId = "";
// Каст (int) невозможен, поэтому костыль
respId = response["id"].ToString();
responseObject.Id = Int32.Parse(respId);
}
}
else
{
_logger.Error($"[WordDictionaryRepository] Cand add id {engId} and {rusId} in dictionary ");
throw new ArgumentException($"Cand add id {engId} and {rusId} in dictionary ");
}
response.Close();
responseObject.EnglishValue = original;
responseObject.RussianValue = translate;
return(responseObject);
}
}
public TrainedDataSet()
{
_wordDictionary = new WordDictionary();
}
public void AfterDictionaryReordered(
WordDictionary reorderedDictionary, WsdProject project, GenerationInfo info,
IProgressHandle progress)
{
project.PluginData.SetData <StatisticsPlugin, WordDictionary>(string.Empty, reorderedDictionary);
}
public void SetUpTest()
{
wordDict = new WordDictionary(stringArray);
}
protected void Page_Init(object sender, EventArgs e)
{
string cacheName = "WordDictionary|" + CMSContext.PreferredCultureCode;
// Get dictionary from cache
this.WordDictionary = (WordDictionary)HttpContext.Current.Cache[cacheName];
if (this.WordDictionary == null)
{
// If not in cache, create new
this.WordDictionary = new NetSpell.SpellChecker.Dictionary.WordDictionary();
this.WordDictionary.EnableUserFile = false;
// Getting folder for dictionaries
string folderName = "~/App_Data/Dictionaries";
folderName = this.MapPath(folderName);
string culture = CMSContext.PreferredCultureCode;
string dictFile = culture + ".dic";
string dictPath = Path.Combine(folderName, dictFile);
if (!File.Exists(dictPath))
{
// Get default culture
string defaultCulture = ValidationHelper.GetString(SettingsHelper.AppSettings["CMSDefaultSpellCheckerCulture"], "");
if (defaultCulture != "")
{
culture = defaultCulture;
dictFile = defaultCulture + ".dic";
dictPath = Path.Combine(folderName, dictFile);
}
}
if (!File.Exists(dictPath))
{
this.lblError.Text = string.Format(GetString("SpellCheck.DictionaryNotExists"), culture);
this.lblError.Visible = true;
return;
}
mDictExists = true;
this.WordDictionary.DictionaryFolder = folderName;
this.WordDictionary.DictionaryFile = dictFile;
// Load and initialize the dictionary
this.WordDictionary.Initialize();
// Store the Dictionary in cache
HttpContext.Current.Cache.Insert(cacheName, this.WordDictionary, new CacheDependency(Path.Combine(folderName, this.WordDictionary.DictionaryFile)));
}
else
{
mDictExists = true;
}
// Create spell checker
this.SpellChecker = new NetSpell.SpellChecker.Spelling();
this.SpellChecker.ShowDialog = false;
this.SpellChecker.Dictionary = this.WordDictionary;
// Adding events
this.SpellChecker.MisspelledWord += new NetSpell.SpellChecker.Spelling.MisspelledWordEventHandler(this.SpellChecker_MisspelledWord);
this.SpellChecker.EndOfText += new NetSpell.SpellChecker.Spelling.EndOfTextEventHandler(this.SpellChecker_EndOfText);
this.SpellChecker.DoubledWord += new NetSpell.SpellChecker.Spelling.DoubledWordEventHandler(this.SpellChecker_DoubledWord);
}
public static void Run()
{
WordDictionary wordDictionary = new WordDictionary();
wordDictionary.AddWord("word");
Console.WriteLine(wordDictionary.Search("w.rd"));
Console.WriteLine(wordDictionary.Search("w"));
Console.WriteLine(wordDictionary.Search("wo"));
Console.WriteLine(wordDictionary.Search("."));
Console.WriteLine(wordDictionary.Search(".d"));
Console.WriteLine(wordDictionary.Search("w."));
wordDictionary.AddWord("a");
Console.WriteLine(wordDictionary.Search("."));
Console.WriteLine(wordDictionary.Search(".a"));
Console.WriteLine(wordDictionary.Search("a."));
}
/// <summary>
/// Initialises the object with a preloaded dictionary
/// </summary>
public ReadablePassphraseGenerator(WordDictionary words)
: this(words, new CryptoRandomSource())
{
}
// foreach (CheckBox checkBox in chkList)
// {
// checkBox.IsChecked = true;
// })
// this.xdgFocusWords.EndEdit();
// }
// private void RemoveRowHighlights(object item)
// {
// Xceed.Wpf.DataGrid.DataRow row = this.xdgFocusWords.GetContainerFromItem(item) as Xceed.Wpf.DataGrid.DataRow;
// Xceed.Wpf.DataGrid.DataCell c;
// if (row != null)
// {
// c = row.Cells[0];
// Xceed.Wpf.Controls.CheckBox chk = (Xceed.Wpf.Controls.CheckBox)c;
// chk.IsChecked = true;
// }
// }
private void btnNext15_Click(object sender, RoutedEventArgs e)
{
xdgFocusWords.ItemsSource = WordDictionary.getNUnmasteredWords(NumberOfWords, ++focusCounter);
}
public TrainedDataSet(WordDictionary dictionary)
{
_wordDictionary = dictionary;
}
/// <summary>
/// 导入外部词库，词频按照重合词频比例平均值
/// </summary>
/// <param name="ImportDicFile">外部词库文件名</param>
/// <param name="ImportEncoding">外部词库文件编码</param>
/// <param name="SourceDicFile">源dct文件名</param>
/// <param name="DestDicFile">目标dct文件名</param>
/// <param name="DicFormat">外部词库类型</param>
/// <param name="OddLines">导入的库中无效且不在源库中的数据</param>
/// <param name="ImportFrqRate">设置固定的导入文件频度比例（除以此数字后入库,小于等于0则按照AvgFrqRate入库）</param>
/// <param name="AvgFrqRate">导入文件的平均频度比例</param>
/// <returns>导入的条数</returns>
public static int ImportDictionary(string ImportDicFile, Encoding ImportEncoding, string SourceDicFile, string DestDicFile, DictionaryFormat DicFormat, out string[] OddLines, out double AvgFrqRate, double ImportFrqRate = 0)
{
//初始化
double MaxFrqRate, MinFrqRate;
WordDictionary.DicWordInfo[] NewWords;
WordDictionary.DicWordInfo[] ExistWords;
FindDifferent(ImportDicFile, ImportEncoding, DicFormat, SourceDicFile, out OddLines, out NewWords, out ExistWords, out MaxFrqRate, out MinFrqRate, out AvgFrqRate);
//加载词库
WordDictionary dict = new WordDictionary();
if (!dict.Load(SourceDicFile))
throw new Exception("load source dic file fail");
//加入新词
foreach (WordDictionary.DicWordInfo Word in NewWords)
{
int Frq = Convert.ToInt32(ImportFrqRate <= 0 ? Word.Frequence / AvgFrqRate : Word.Frequence / ImportFrqRate);
dict.AddWord(Word.Word, Word.Pos, Frq);
}
//保存
dict.Save(DestDicFile);
dict.ReleaseDict();
return NewWords.Length;
}
public SpellChecker(Stream dictionaryResource)
{
_dictionary = new WordDictionary(dictionaryResource);
}
public abstract WordAndString ChooseWord(WordDictionary words, Random.RandomSourceBase randomness, IEnumerable <Word> alreadyChosen);
