public void BuildFromXml_IsNull_ReturnWordCollection() { _wordCollection = WordCollection.BuildFromXml(Path); Assert.IsNotNull(_wordCollection); Assert.IsInstanceOf <WordCollection>(_wordCollection); }
public WordCollection GetWordDictionaryFromFile(int maxCodeLength) { var wordCollection = new WordCollection(); if (!File.Exists(_file)) { throw new InvalidOperationException($"Unable to find {_file}"); } using (var fileStream = new FileStream(_file, FileMode.OpenOrCreate, FileAccess.Read)) { using (var reader = new StreamReader(fileStream, Encoding.UTF8)) { string line; while ((line = reader.ReadLine()) != null) { wordCollection.AddWords(line, maxCodeLength); if (reader.EndOfStream) { break; } } } } return(wordCollection); }
public WordCollection PopAssignmentDestStr() { WordCollection ret = assigndest; assigndest = null; return(ret); }
public DimLineWC(WordCollection wc, bool isString, bool isPrivate, ScriptPosition position) { WC = wc; Dims = isString; IsPrivate = isPrivate; SC = position; }
public static void ParseTextAsset(TextAsset text) { List <string>[] dictionaryArray = new List <string> [4]; for (int i = 0; i < 4; i++) { dictionaryArray[i] = new List <string>(); } string[] words = text.text.Split(','); foreach (var w in words) { if (w.Length > 2 && w.Length < 7) { dictionaryArray[w.Length - 3].Add(w); Debug.Log(w.Length + "-" + w); } } WordCollection dic = ScriptableObject.CreateInstance <WordCollection>(); for (int i = 0; i < 4; i++) { dic.WordDictionary.Add(i + 3, new WordArray(dictionaryArray[i].ToArray())); } UnityEditor.AssetDatabase.CreateAsset(dic, "Assets/Dictionary/words.asset"); UnityEditor.AssetDatabase.SaveAssets(); }
/// <summary> /// Create unigram table for random sub-sampling. /// Frequent words will occupy more positions in the table. /// </summary> public static int[] GetUnigramTable(this WordCollection wordCollection, int tableSize, double power = 0.75) { if (wordCollection.GetNumberOfUniqueWords() == 0) { // maybe throw? return(Array.Empty <int>()); } var table = new int[tableSize]; var sumOfOccurenceOfWordsRaisedToPower = wordCollection.GetSumOfOccurenceOfWordsRaisedToPower(power); var words = wordCollection.GetWords().ToArray(); var indexOfCurrentWord = -1; var highestPositionOfWordInTable = -1; for (var tablePosition = 0; tablePosition < tableSize; tablePosition++) { if (tablePosition > highestPositionOfWordInTable) { indexOfCurrentWord++; highestPositionOfWordInTable += (int)Math.Ceiling(Math.Pow(wordCollection.GetOccurrenceOfWord(words[indexOfCurrentWord]), power) / sumOfOccurenceOfWordsRaisedToPower * tableSize); } table[tablePosition] = indexOfCurrentWord; if (indexOfCurrentWord >= wordCollection.GetNumberOfUniqueWords()) { indexOfCurrentWord = wordCollection.GetNumberOfUniqueWords() - 1; } } return(table); }
private static void AddWords(int numberOfCopies, WordCollection wordCollection, string inputCharacter) { for (var i = 0; i < numberOfCopies; i++) { wordCollection.AddWords(inputCharacter, 4); } }
public WordCollection PopRowArgs() { WordCollection ret = wc; wc = null; return(ret); }
public void CanGetSentence() { long wordCount = 0; const string input = "This is a string. The String to test, the string to prevail.\r\nWhat is the string?"; var wordCollection = new WordCollection(); wordCollection.AddWords(input, 11); wordCollection.InitWordPositions(); const int maxSentenceLength = 50; var sentence = new long?[maxSentenceLength + 1]; ulong nextRandom = 1; const float thresholdForOccurrenceOfWords = 0; long sentenceLength = 0; string[] lastLine = null; var reader = new StreamReader(new MemoryStream(Encoding.ASCII.GetBytes(input))); wordCount = NLP.Word2Vec.Word2Vec.SetSentence(reader, wordCount, sentence, ref nextRandom, ref sentenceLength, ref lastLine, wordCollection, thresholdForOccurrenceOfWords); Assert.Equal(16, wordCount); Assert.Equal(16, sentenceLength); Assert.NotNull(sentence[15]); Assert.Null(sentence[16]); }
private static void Preorder(WordCollection wordCollection, Node root) { if (root == null) { return; } if (root.Left != null) { root.Left.Code = root.Code + "0"; if (root.Left.WordInfo != null) { wordCollection.SetCode(root.Left.Word, root.Left.Code.ToCharArray()); SetPoint(wordCollection, root.Left.Word, root.Left.Code.Length, root, 1); } } if (root.Right != null) { root.Right.Code = root.Code + "1"; if (root.Right.WordInfo != null) { wordCollection.SetCode(root.Right.Word, root.Right.Code.ToCharArray()); SetPoint(wordCollection, root.Right.Word, root.Right.Code.Length, root, 1); } } Preorder(wordCollection, root.Right); Preorder(wordCollection, root.Left); }
private string getValueString(string str) { if ((emuera == null) || (GlobalStatic.EMediator == null)) { return(""); } if (string.IsNullOrEmpty(str)) { return(""); } mainConsole.RunERBFromMemory = true; try { StringStream st = new StringStream(str); WordCollection wc = LexicalAnalyzer.Analyse(st, LexEndWith.EoL, LexAnalyzeFlag.None); IOperandTerm term = ExpressionParser.ReduceExpressionTerm(wc, TermEndWith.EoL); SingleTerm value = term.GetValue(GlobalStatic.EMediator); return(value.ToString()); } catch (CodeEE e) { return(e.Message); } catch (Exception e) { return(e.GetType().ToString() + ":" + e.Message); } finally { mainConsole.RunERBFromMemory = false; } }
private static CaseExpression reduceCaseExpression(WordCollection wc) { CaseExpression ret = new CaseExpression(); IdentifierWord id = wc.Current as IdentifierWord; if ((id != null) && (id.Code.Equals("IS", Config.SCVariable))) { wc.ShiftNext(); ret.CaseType = CaseExpressionType.Is; OperatorWord opWT = wc.Current as OperatorWord; if (opWT == null) { throw new CodeEE("ISキーワードの後に演算子がありません"); } OperatorCode op = opWT.Code; if (!OperatorManager.IsBinary(op)) { throw new CodeEE("ISキーワードの後の演算子が2項演算子ではありません"); } wc.ShiftNext(); ret.Operator = op; ret.LeftTerm = reduceTerm(wc, false, TermEndWith.Comma, VariableCode.__NULL__); if (ret.LeftTerm == null) { throw new CodeEE("ISキーワードの後に式がありません"); } Type type = ret.LeftTerm.GetOperandType(); return(ret); } ret.LeftTerm = reduceTerm(wc, true, TermEndWith.Comma, VariableCode.__NULL__); if (ret.LeftTerm == null) { throw new CodeEE("CASEの引数は省略できません"); } id = wc.Current as IdentifierWord; if ((id != null) && (id.Code.Equals("TO", Config.SCVariable))) { ret.CaseType = CaseExpressionType.To; wc.ShiftNext(); ret.RightTerm = reduceTerm(wc, true, TermEndWith.Comma, VariableCode.__NULL__); if (ret.RightTerm == null) { throw new CodeEE("TOキーワードの後に式がありません"); } id = wc.Current as IdentifierWord; if ((id != null) && (id.Code.Equals("TO", Config.SCVariable))) { throw new CodeEE("TOキーワードが2度使われています"); } if (ret.LeftTerm.GetOperandType() != ret.RightTerm.GetOperandType()) { throw new CodeEE("TOキーワードの前後の型が一致していません"); } return(ret); } ret.CaseType = CaseExpressionType.Normal; return(ret); }
private void CheckIndexer_Index( WordCollection words, UInt16 ui16Index, Boolean success, String message) { MemoryOffset index = new MemoryOffset(ui16Index); CheckGetIndexer_Index(words, index, success, message); CheckSetIndexer_Index(words, index, success, message); }
public void BuildFromXml_BadXML_throwXmlException() { string pathToBug = @"Data/Base/WordBase_BUG.xml"; Assert.Throws <XmlException>( () => WordCollection.BuildFromXml(pathToBug) ); }
public InstructionLine(ScriptPosition thePosition, FunctionIdentifier functionIdentifier, OperatorCode assignOP, WordCollection dest, StringStream theArgPrimitive) { base.position = thePosition; func = functionIdentifier; AssignOperator = assignOP; assigndest = dest; this.argprimitive = theArgPrimitive; }
public Variety(string name) { _name = name; _words = new WordCollection(this); _varietyPairs = new VarietyVarietyPairCollection(this); _affixes = new BulkObservableList<Affix>(); _regions = new BulkObservableList<GeographicRegion>(); _syllablePositionSegmentFreqDists = new ObservableDictionary<FeatureSymbol, FrequencyDistribution<Segment>>(); }
// Action when user click Del button private void DelAction() { if (SelectedWord != null) { settings.WordsAmount -= 1; dataAccess.RemoveWord(SelectedWord); WordCollection.Remove(SelectedWord); } }
private static void Main(string[] args) { var lists = new List<WordCollection>(); var subject = new WordCollection(); subject.Add(new Word("I")); subject.Add(new Word("They")); subject.Add(new Word("She")); subject.Add(new Word("Jerry")); subject.Type = "SUBJECT"; var verb = new WordCollection(); verb.Add(new Word("is")); verb.Add(new Word("eat")); verb.Add(new Word("book")); verb.Type = "VERB"; var noun = new WordCollection(); noun.Add(new Word("cake")); noun.Add(new Word("rice")); noun.Type = "NOUN"; ConstantValue.GetGrammars().ForEach(c => { foreach (var symbol in c.Value.Split('+')) { var trimedSymbol = symbol.Trim(); if (HasDefiniton(trimedSymbol)) { trimedSymbol = trimedSymbol.Split('(')[0]; } if (trimedSymbol == "NOUN") { lists.Add(noun); } else if (trimedSymbol == "SUBJECT") { lists.Add(subject); } else if (trimedSymbol == "VERB") { lists.Add(verb); } else { lists.Add(new WordCollection() {new Word(trimedSymbol)}); } } new ListGenerator(lists).Generate().ForEach(Console.WriteLine); Console.WriteLine("---"); lists.Clear(); }); Console.ReadKey(); }
void LoadWordDict() { wordList = JsonUtility.FromJson <WordCollection>(wordDictFile.ToString()); wordDict = new Dictionary <string, Word>(); for (int i = 0; i < wordList.words.Length; i++) { wordDict.Add(wordList.words[i].word, wordList.words[i]); } }
public AlphabeticalOrderIterator(WordCollection collection, bool reverse = false) { _collection = collection; _reverse = reverse; if (reverse) { _position = collection.getItems().Count; } }
public static IOperandTerm ReduceVariableArgument(WordCollection wc, VariableCode varCode) { IOperandTerm ret = reduceTerm(wc, false, TermEndWith.EoL, varCode); if (ret == null) { throw new CodeEE("変数の:の後に引数がありません"); } return(ret); }
public User() { Statistics = new UserStatistics(); Achievements = new Achievements(); Collections = new WordCollection(); Information = new Settings(); System.Windows.Application.Current.Exit += Serialization; Deserialization(); }
private decimal creditvalue = 0.01m; //TODO Segun IP creditvalue puede variar public void AddItem(Word word) { Word w = WordCollection.Where(p => p.Character == word.Character).FirstOrDefault(); //Evitar duplicados en las palabras memorizadas if (w == null) { WordCollection.Add(word); } }
private void processJsonData(string url) { WordCollection wordCollection = JsonUtility.FromJson <WordCollection>(url); for (int i = 0; i < wordCollection.words.Length; i++) { words.Add(wordCollection.words[i].word); //Debug.Log(wordCollection.words[i].word); } }
/// <summary> /// Returns the Term Frequency-Inverse Document Frequency (TF-IDF) of a word in a document in relation to a collection of documents. /// </summary> public static double CalculateTFIDF(this WordCollection document, string word, List <WordCollection> allDocuments) { var termFrequency = (double)document.GetOccurrenceOfWord(word) / document.GetTotalNumberOfWords(); var totalDocuments = allDocuments.Count; var totalDocumentsWithWord = allDocuments.Count(wc => wc.GetWords().Contains(word)); // adjust denominator to avoid division by 0. var inverseDocumentFrequency = Math.Log((double)totalDocuments / totalDocumentsWithWord); return(termFrequency * inverseDocumentFrequency); }
// Action when user click Add button private void AddAction() { WordModel word = new WordModel(Word, Definition, Part); settings.WordsAmount += 1; WordCollection.Add(word); dataAccess.SaveWord(word); // this is for update the search request SearchRequest = _searchRequest; }
private static void VerifyWordInfo(WordCollection wordCollection, string word, char[] expectedCode, long[] expectedPoints, int expectedCodeLength) { var position = wordCollection[word].Value; var code = wordCollection[position].Code; var points = wordCollection[position].Point; var codeLength = wordCollection[position].CodeLength; Assert.Equal(expectedCode, code); Assert.Equal(expectedPoints, points); Assert.Equal(expectedCodeLength, codeLength); }
private static void IterateQueue(WordCollection wordCollection, List <Node> queue) { var numberOfInteriorNodes = 0; for (var a = 0; a < wordCollection.GetNumberOfUniqueWords() - 1; a++) { var node = CreateInteriorNode(queue, numberOfInteriorNodes); numberOfInteriorNodes++; InsertNodeInQueue(queue, node); } }
/// <summary> /// Create binary Huffman tree using the word counts. /// Frequent words will have short unique binary codes. /// Huffman encoding is used for loss-less compression. /// The vocab_word structure contains a field for the 'code' for the word. /// </summary> public static void CreateBinaryTree(this WordCollection wordCollection) { var queue = GetQueueSortedByWordFrequencyAscending(wordCollection); IterateQueue(wordCollection, queue); var root = queue.Single(); root.Code = ""; Preorder(wordCollection, root); GC.Collect(); }
private static void SetPoint(WordCollection wordCollection, string word, int codeLength, Node root, int i) { wordCollection.SetPoint(word, codeLength - i, root.IndexOfLeafNodeThisInteriorNodePretendsToBe.Value); if (codeLength - i == 0 || root.Parent == null) { return; } SetPoint(wordCollection, word, codeLength, root.Parent, ++i); }
/// <summary> /// カンマで区切られたCASEの引数を一括して取得。行端で終わる。 /// </summary> /// <param name="st"></param> /// <returns></returns> public static CaseExpression[] ReduceCaseExpressions(WordCollection wc) { List <CaseExpression> terms = new List <CaseExpression>(); while (!wc.EOL) { terms.Add(reduceCaseExpression(wc)); wc.ShiftNext(); } CaseExpression[] ret = new CaseExpression[terms.Count]; terms.CopyTo(ret); return(ret); }
private void CheckSetIndexer_Index( WordCollection words, MemoryOffset index, Boolean success, String message) { try { words[index] = Word.Zero; Assert.IsTrue(success, message); } catch (Casl2SimulatorException) { Assert.IsFalse(success, message); } }
private char[] TextArray; // The text of the document. #endregion Fields #region Constructors public Document(string initialText) { TextArray = initialText.ToCharArray(); Words = new WordCollection(this); Characters = new CharacterCollection(this); }
public void Clear() { mWords = new WordCollection(); }
protected internal EditMeaningWordLookupValidator(IWord parentWord, IMeaning meaning, WordCollection childrenWords, LookupData lookup) : base(parentWord, meaning, childrenWords, lookup) { WordItem item = (WordItem)this.lookup.Data; this.originalWord = item.Word; Validate(); }
protected internal InsertMeaningWordLookupValidator(IWord parentWord, IMeaning meaning, WordCollection childrenWords, LookupData lookup) : base(parentWord, meaning, childrenWords, lookup) { Validate(); }
protected internal MeaningWordLookupValidator(IWord parentWord, IMeaning meaning, WordCollection childrenWords, LookupData lookup) : base(lookup) { Debug.Assert(parentWord != null); Debug.Assert(meaning != null); Debug.Assert(childrenWords != null); this.parentWord = parentWord; this.meaning = meaning; this.childrenWords = childrenWords; }