public void BuildFromXml_IsNull_ReturnWordCollection()
{
_wordCollection = WordCollection.BuildFromXml(Path);
Assert.IsNotNull(_wordCollection);
Assert.IsInstanceOf <WordCollection>(_wordCollection);
}
public WordCollection GetWordDictionaryFromFile(int maxCodeLength)
{
var wordCollection = new WordCollection();
if (!File.Exists(_file))
{
throw new InvalidOperationException($"Unable to find {_file}");
}
using (var fileStream = new FileStream(_file, FileMode.OpenOrCreate, FileAccess.Read))
{
using (var reader = new StreamReader(fileStream, Encoding.UTF8))
{
string line;
while ((line = reader.ReadLine()) != null)
{
wordCollection.AddWords(line, maxCodeLength);
if (reader.EndOfStream)
{
break;
}
}
}
}
return(wordCollection);
}
public WordCollection PopAssignmentDestStr()
{
WordCollection ret = assigndest;
assigndest = null;
return(ret);
}
public DimLineWC(WordCollection wc, bool isString, bool isPrivate, ScriptPosition position)
{
WC = wc;
Dims = isString;
IsPrivate = isPrivate;
SC = position;
}
public static void ParseTextAsset(TextAsset text)
{
List <string>[] dictionaryArray = new List <string> [4];
for (int i = 0; i < 4; i++)
{
dictionaryArray[i] = new List <string>();
}
string[] words = text.text.Split(',');
foreach (var w in words)
{
if (w.Length > 2 && w.Length < 7)
{
dictionaryArray[w.Length - 3].Add(w);
Debug.Log(w.Length + "-" + w);
}
}
WordCollection dic = ScriptableObject.CreateInstance <WordCollection>();
for (int i = 0; i < 4; i++)
{
dic.WordDictionary.Add(i + 3, new WordArray(dictionaryArray[i].ToArray()));
}
UnityEditor.AssetDatabase.CreateAsset(dic, "Assets/Dictionary/words.asset");
UnityEditor.AssetDatabase.SaveAssets();
}
/// <summary>
/// Create unigram table for random sub-sampling.
/// Frequent words will occupy more positions in the table.
/// </summary>
public static int[] GetUnigramTable(this WordCollection wordCollection, int tableSize, double power = 0.75)
{
if (wordCollection.GetNumberOfUniqueWords() == 0)
{
// maybe throw?
return(Array.Empty <int>());
}
var table = new int[tableSize];
var sumOfOccurenceOfWordsRaisedToPower = wordCollection.GetSumOfOccurenceOfWordsRaisedToPower(power);
var words = wordCollection.GetWords().ToArray();
var indexOfCurrentWord = -1;
var highestPositionOfWordInTable = -1;
for (var tablePosition = 0; tablePosition < tableSize; tablePosition++)
{
if (tablePosition > highestPositionOfWordInTable)
{
indexOfCurrentWord++;
highestPositionOfWordInTable += (int)Math.Ceiling(Math.Pow(wordCollection.GetOccurrenceOfWord(words[indexOfCurrentWord]), power) / sumOfOccurenceOfWordsRaisedToPower * tableSize);
}
table[tablePosition] = indexOfCurrentWord;
if (indexOfCurrentWord >= wordCollection.GetNumberOfUniqueWords())
{
indexOfCurrentWord = wordCollection.GetNumberOfUniqueWords() - 1;
}
}
return(table);
}
private static void AddWords(int numberOfCopies, WordCollection wordCollection, string inputCharacter)
{
for (var i = 0; i < numberOfCopies; i++)
{
wordCollection.AddWords(inputCharacter, 4);
}
}
public WordCollection PopRowArgs()
{
WordCollection ret = wc;
wc = null;
return(ret);
}
public void CanGetSentence()
{
long wordCount = 0;
const string input = "This is a string. The String to test, the string to prevail.\r\nWhat is the string?";
var wordCollection = new WordCollection();
wordCollection.AddWords(input, 11);
wordCollection.InitWordPositions();
const int maxSentenceLength = 50;
var sentence = new long?[maxSentenceLength + 1];
ulong nextRandom = 1;
const float thresholdForOccurrenceOfWords = 0;
long sentenceLength = 0;
string[] lastLine = null;
var reader = new StreamReader(new MemoryStream(Encoding.ASCII.GetBytes(input)));
wordCount = NLP.Word2Vec.Word2Vec.SetSentence(reader, wordCount, sentence, ref nextRandom, ref sentenceLength,
ref lastLine, wordCollection, thresholdForOccurrenceOfWords);
Assert.Equal(16, wordCount);
Assert.Equal(16, sentenceLength);
Assert.NotNull(sentence[15]);
Assert.Null(sentence[16]);
}
private static void Preorder(WordCollection wordCollection, Node root)
{
if (root == null)
{
return;
}
if (root.Left != null)
{
root.Left.Code = root.Code + "0";
if (root.Left.WordInfo != null)
{
wordCollection.SetCode(root.Left.Word, root.Left.Code.ToCharArray());
SetPoint(wordCollection, root.Left.Word, root.Left.Code.Length, root, 1);
}
}
if (root.Right != null)
{
root.Right.Code = root.Code + "1";
if (root.Right.WordInfo != null)
{
wordCollection.SetCode(root.Right.Word, root.Right.Code.ToCharArray());
SetPoint(wordCollection, root.Right.Word, root.Right.Code.Length, root, 1);
}
}
Preorder(wordCollection, root.Right);
Preorder(wordCollection, root.Left);
}
private string getValueString(string str)
{
if ((emuera == null) || (GlobalStatic.EMediator == null))
{
return("");
}
if (string.IsNullOrEmpty(str))
{
return("");
}
mainConsole.RunERBFromMemory = true;
try
{
StringStream st = new StringStream(str);
WordCollection wc = LexicalAnalyzer.Analyse(st, LexEndWith.EoL, LexAnalyzeFlag.None);
IOperandTerm term = ExpressionParser.ReduceExpressionTerm(wc, TermEndWith.EoL);
SingleTerm value = term.GetValue(GlobalStatic.EMediator);
return(value.ToString());
}
catch (CodeEE e)
{
return(e.Message);
}
catch (Exception e)
{
return(e.GetType().ToString() + ":" + e.Message);
}
finally
{
mainConsole.RunERBFromMemory = false;
}
}
private static CaseExpression reduceCaseExpression(WordCollection wc)
{
CaseExpression ret = new CaseExpression();
IdentifierWord id = wc.Current as IdentifierWord;
if ((id != null) && (id.Code.Equals("IS", Config.SCVariable)))
{
wc.ShiftNext();
ret.CaseType = CaseExpressionType.Is;
OperatorWord opWT = wc.Current as OperatorWord;
if (opWT == null)
{
throw new CodeEE("ISキーワードの後に演算子がありません");
}
OperatorCode op = opWT.Code;
if (!OperatorManager.IsBinary(op))
{
throw new CodeEE("ISキーワードの後の演算子が2項演算子ではありません");
}
wc.ShiftNext();
ret.Operator = op;
ret.LeftTerm = reduceTerm(wc, false, TermEndWith.Comma, VariableCode.__NULL__);
if (ret.LeftTerm == null)
{
throw new CodeEE("ISキーワードの後に式がありません");
}
Type type = ret.LeftTerm.GetOperandType();
return(ret);
}
ret.LeftTerm = reduceTerm(wc, true, TermEndWith.Comma, VariableCode.__NULL__);
if (ret.LeftTerm == null)
{
throw new CodeEE("CASEの引数は省略できません");
}
id = wc.Current as IdentifierWord;
if ((id != null) && (id.Code.Equals("TO", Config.SCVariable)))
{
ret.CaseType = CaseExpressionType.To;
wc.ShiftNext();
ret.RightTerm = reduceTerm(wc, true, TermEndWith.Comma, VariableCode.__NULL__);
if (ret.RightTerm == null)
{
throw new CodeEE("TOキーワードの後に式がありません");
}
id = wc.Current as IdentifierWord;
if ((id != null) && (id.Code.Equals("TO", Config.SCVariable)))
{
throw new CodeEE("TOキーワードが2度使われています");
}
if (ret.LeftTerm.GetOperandType() != ret.RightTerm.GetOperandType())
{
throw new CodeEE("TOキーワードの前後の型が一致していません");
}
return(ret);
}
ret.CaseType = CaseExpressionType.Normal;
return(ret);
}
private void CheckIndexer_Index(
WordCollection words, UInt16 ui16Index, Boolean success, String message)
{
MemoryOffset index = new MemoryOffset(ui16Index);
CheckGetIndexer_Index(words, index, success, message);
CheckSetIndexer_Index(words, index, success, message);
}
public void BuildFromXml_BadXML_throwXmlException()
{
string pathToBug = @"Data/Base/WordBase_BUG.xml";
Assert.Throws <XmlException>(
() => WordCollection.BuildFromXml(pathToBug)
);
}
public InstructionLine(ScriptPosition thePosition, FunctionIdentifier functionIdentifier, OperatorCode assignOP, WordCollection dest, StringStream theArgPrimitive)
{
base.position = thePosition;
func = functionIdentifier;
AssignOperator = assignOP;
assigndest = dest;
this.argprimitive = theArgPrimitive;
}
public Variety(string name)
{
_name = name;
_words = new WordCollection(this);
_varietyPairs = new VarietyVarietyPairCollection(this);
_affixes = new BulkObservableList<Affix>();
_regions = new BulkObservableList<GeographicRegion>();
_syllablePositionSegmentFreqDists = new ObservableDictionary<FeatureSymbol, FrequencyDistribution<Segment>>();
}
// Action when user click Del button
private void DelAction()
{
if (SelectedWord != null)
{
settings.WordsAmount -= 1;
dataAccess.RemoveWord(SelectedWord);
WordCollection.Remove(SelectedWord);
}
}
private static void Main(string[] args)
{
var lists = new List<WordCollection>();
var subject = new WordCollection();
subject.Add(new Word("I"));
subject.Add(new Word("They"));
subject.Add(new Word("She"));
subject.Add(new Word("Jerry"));
subject.Type = "SUBJECT";
var verb = new WordCollection();
verb.Add(new Word("is"));
verb.Add(new Word("eat"));
verb.Add(new Word("book"));
verb.Type = "VERB";
var noun = new WordCollection();
noun.Add(new Word("cake"));
noun.Add(new Word("rice"));
noun.Type = "NOUN";
ConstantValue.GetGrammars().ForEach(c =>
{
foreach (var symbol in c.Value.Split('+'))
{
var trimedSymbol = symbol.Trim();
if (HasDefiniton(trimedSymbol))
{
trimedSymbol = trimedSymbol.Split('(')[0];
}
if (trimedSymbol == "NOUN")
{
lists.Add(noun);
}
else if (trimedSymbol == "SUBJECT")
{
lists.Add(subject);
}
else if (trimedSymbol == "VERB")
{
lists.Add(verb);
}
else
{
lists.Add(new WordCollection() {new Word(trimedSymbol)});
}
}
new ListGenerator(lists).Generate().ForEach(Console.WriteLine);
Console.WriteLine("---");
lists.Clear();
});
Console.ReadKey();
}
void LoadWordDict()
{
wordList = JsonUtility.FromJson <WordCollection>(wordDictFile.ToString());
wordDict = new Dictionary <string, Word>();
for (int i = 0; i < wordList.words.Length; i++)
{
wordDict.Add(wordList.words[i].word, wordList.words[i]);
}
}
public AlphabeticalOrderIterator(WordCollection collection, bool reverse = false)
{
_collection = collection;
_reverse = reverse;
if (reverse)
{
_position = collection.getItems().Count;
}
}
public static IOperandTerm ReduceVariableArgument(WordCollection wc, VariableCode varCode)
{
IOperandTerm ret = reduceTerm(wc, false, TermEndWith.EoL, varCode);
if (ret == null)
{
throw new CodeEE("変数の:の後に引数がありません");
}
return(ret);
}
public User()
{
Statistics = new UserStatistics();
Achievements = new Achievements();
Collections = new WordCollection();
Information = new Settings();
System.Windows.Application.Current.Exit += Serialization;
Deserialization();
}
private decimal creditvalue = 0.01m; //TODO Segun IP creditvalue puede variar
public void AddItem(Word word)
{
Word w = WordCollection.Where(p => p.Character == word.Character).FirstOrDefault();
//Evitar duplicados en las palabras memorizadas
if (w == null)
{
WordCollection.Add(word);
}
}
private void processJsonData(string url)
{
WordCollection wordCollection = JsonUtility.FromJson <WordCollection>(url);
for (int i = 0; i < wordCollection.words.Length; i++)
{
words.Add(wordCollection.words[i].word);
//Debug.Log(wordCollection.words[i].word);
}
}
/// <summary>
/// Returns the Term Frequency-Inverse Document Frequency (TF-IDF) of a word in a document in relation to a collection of documents.
/// </summary>
public static double CalculateTFIDF(this WordCollection document, string word, List <WordCollection> allDocuments)
{
var termFrequency = (double)document.GetOccurrenceOfWord(word) / document.GetTotalNumberOfWords();
var totalDocuments = allDocuments.Count;
var totalDocumentsWithWord = allDocuments.Count(wc => wc.GetWords().Contains(word));
// adjust denominator to avoid division by 0.
var inverseDocumentFrequency = Math.Log((double)totalDocuments / totalDocumentsWithWord);
return(termFrequency * inverseDocumentFrequency);
}
// Action when user click Add button
private void AddAction()
{
WordModel word = new WordModel(Word, Definition, Part);
settings.WordsAmount += 1;
WordCollection.Add(word);
dataAccess.SaveWord(word);
// this is for update the search request
SearchRequest = _searchRequest;
}
private static void VerifyWordInfo(WordCollection wordCollection, string word, char[] expectedCode, long[] expectedPoints, int expectedCodeLength)
{
var position = wordCollection[word].Value;
var code = wordCollection[position].Code;
var points = wordCollection[position].Point;
var codeLength = wordCollection[position].CodeLength;
Assert.Equal(expectedCode, code);
Assert.Equal(expectedPoints, points);
Assert.Equal(expectedCodeLength, codeLength);
}
private static void IterateQueue(WordCollection wordCollection, List <Node> queue)
{
var numberOfInteriorNodes = 0;
for (var a = 0; a < wordCollection.GetNumberOfUniqueWords() - 1; a++)
{
var node = CreateInteriorNode(queue, numberOfInteriorNodes);
numberOfInteriorNodes++;
InsertNodeInQueue(queue, node);
}
}
/// <summary>
/// Create binary Huffman tree using the word counts.
/// Frequent words will have short unique binary codes.
/// Huffman encoding is used for loss-less compression.
/// The vocab_word structure contains a field for the 'code' for the word.
/// </summary>
public static void CreateBinaryTree(this WordCollection wordCollection)
{
var queue = GetQueueSortedByWordFrequencyAscending(wordCollection);
IterateQueue(wordCollection, queue);
var root = queue.Single();
root.Code = "";
Preorder(wordCollection, root);
GC.Collect();
}
private static void SetPoint(WordCollection wordCollection, string word, int codeLength, Node root, int i)
{
wordCollection.SetPoint(word, codeLength - i, root.IndexOfLeafNodeThisInteriorNodePretendsToBe.Value);
if (codeLength - i == 0 || root.Parent == null)
{
return;
}
SetPoint(wordCollection, word, codeLength, root.Parent, ++i);
}
/// <summary>
/// カンマで区切られたCASEの引数を一括して取得。行端で終わる。
/// </summary>
/// <param name="st"></param>
/// <returns></returns>
public static CaseExpression[] ReduceCaseExpressions(WordCollection wc)
{
List <CaseExpression> terms = new List <CaseExpression>();
while (!wc.EOL)
{
terms.Add(reduceCaseExpression(wc));
wc.ShiftNext();
}
CaseExpression[] ret = new CaseExpression[terms.Count];
terms.CopyTo(ret);
return(ret);
}
private void CheckSetIndexer_Index(
WordCollection words, MemoryOffset index, Boolean success, String message)
{
try
{
words[index] = Word.Zero;
Assert.IsTrue(success, message);
}
catch (Casl2SimulatorException)
{
Assert.IsFalse(success, message);
}
}
private char[] TextArray; // The text of the document.
#endregion Fields
#region Constructors
public Document(string initialText)
{
TextArray = initialText.ToCharArray();
Words = new WordCollection(this);
Characters = new CharacterCollection(this);
}
public void Clear()
{
mWords = new WordCollection();
}
protected internal EditMeaningWordLookupValidator(IWord parentWord, IMeaning meaning, WordCollection childrenWords, LookupData lookup)
: base(parentWord, meaning, childrenWords, lookup)
{
WordItem item = (WordItem)this.lookup.Data;
this.originalWord = item.Word;
Validate();
}
protected internal InsertMeaningWordLookupValidator(IWord parentWord, IMeaning meaning, WordCollection childrenWords, LookupData lookup)
: base(parentWord, meaning, childrenWords, lookup)
{
Validate();
}
protected internal MeaningWordLookupValidator(IWord parentWord, IMeaning meaning, WordCollection childrenWords, LookupData lookup)
: base(lookup)
{
Debug.Assert(parentWord != null);
Debug.Assert(meaning != null);
Debug.Assert(childrenWords != null);
this.parentWord = parentWord;
this.meaning = meaning;
this.childrenWords = childrenWords;
}
