public static void BuildAllLemmatizers(bool bIncludeExamples, bool bCompress, LemmatizerSettings lsett, string sNewFileMask) { DateTime dtStartAll = DateTime.Now; foreach (LanguagePrebuilt lp in Enum.GetValues(typeof(LanguagePrebuilt))) { DateTime dtStart = DateTime.Now; string sDataFileName = GetResourceFileName(sFileMask, lp); string sLemFileName = GetResourceFileName(sNewFileMask, lp); Console.WriteLine("Building lemmatizer for {0} from {1}", lp, sDataFileName); LemmatizerPrebuiltData lemPrebuild = new LemmatizerPrebuiltData(lp, lsett); Console.WriteLine(" Building lemmatizer completed in: {0}", new TimeSpan(DateTime.Now.Ticks - dtStart.Ticks).ToString()); dtStart = DateTime.Now; Console.WriteLine(" Serializing {0} lemmatizer to file {1}", (bIncludeExamples ? "full (examples included)" : "compact (examples excluded)"), sLemFileName); Stream binStreamOut = File.Open(sLemFileName, FileMode.Create); lemPrebuild.Serialize(binStreamOut, bIncludeExamples, bCompress); binStreamOut.Close(); Console.WriteLine(" Successfully completed, time needed: {0}", new TimeSpan(DateTime.Now.Ticks - dtStart.Ticks).ToString()); } Console.WriteLine("Time for building all lemmatizers: " + new TimeSpan(DateTime.Now.Ticks - dtStartAll.Ticks).ToString()); }
public ExampleList(LemmatizerSettings lsett) : base() { this.lsett = lsett; this.dictExamples = new Dictionary<string, LemmaExample>(); this.lstExamples = null; this.rlRules = new RuleList(lsett); }
public LemmatizerPrebuiltData(LanguagePrebuilt lang, LemmatizerSettings lsett) : base(lang,lsett) { Stream stream = GetResourceStream(ResourceFileName); StreamReader srIn = new StreamReader(stream, LanguageEncoding); AddMultextFile(srIn, "WLM"); srIn.Close(); BuildModel(); }
public ExampleList(SerializationInfo info, StreamingContext context) { lsett = (LemmatizerSettings)info.GetValue("lsett", typeof(LemmatizerSettings)); this.dictExamples = new Dictionary<string, LemmaExample>(); this.lstExamples = null; this.rlRules = new RuleList(lsett); string[] aWords = (string[])info.GetValue("aWords", typeof(string[])); string[] aLemmas = (string[])info.GetValue("aLemmas", typeof(string[])); double[] aWeights = (double[])info.GetValue("aWeights", typeof(double[])); string[] aMsds = (string[])info.GetValue("aMsds", typeof(string[])); for (int iExm = 0; iExm < aWords.Length; iExm++) AddExample(aWords[iExm], aLemmas[iExm], aWeights[iExm], aMsds[iExm]); }
public LemmaRule(string sWord, string sLemma, int iId, LemmatizerSettings lsett) { this.lsett = lsett; this.iId = iId; int iSameStem = SameStem(sWord, sLemma); sTo = sLemma.Substring(iSameStem); iFrom = sWord.Length - iSameStem; if (lsett.bUseFromInRules) { sFrom = sWord.Substring(iSameStem); sSignature = "[" + sFrom + "]==>[" + sTo + "]"; } else { sFrom = null; sSignature = "[#" + iFrom + "]==>[" + sTo + "]"; } }
/// <summary> /// /// </summary> /// <param name="lsett"></param> /// <param name="elExamples"></param> /// <param name="iStart">Index of the first word of the current group</param> /// <param name="iEnd">Index of the last word of the current group</param> /// <param name="ltnParentNode"></param> private LemmaTreeNode(LemmatizerSettings lsett, ExampleList elExamples, int iStart, int iEnd, LemmaTreeNode ltnParentNode) : this(lsett) { this.ltnParentNode = ltnParentNode; this.dictSubNodes = null; this.iStart = iStart; this.iEnd = iEnd; this.elExamples = elExamples; if (iStart >= elExamples.Count || iEnd >= elExamples.Count || iStart > iEnd) { lrBestRule = elExamples.Rules.DefaultRule; aBestRules = new RuleWeighted[1]; aBestRules[0] = new RuleWeighted(lrBestRule, 0); dWeight = 0; return; } int iConditionLength = Math.Min(ltnParentNode == null ? 0 : ltnParentNode.iSimilarity + 1, elExamples[iStart].Word.Length); this.sCondition = elExamples[iStart].Word.Substring(elExamples[iStart].Word.Length - iConditionLength); this.iSimilarity = elExamples[iStart].Similarity(elExamples[iEnd]); this.bWholeWord = ltnParentNode == null ? false : elExamples[iEnd].Word.Length == ltnParentNode.iSimilarity; FindBestRules(); AddSubAll(); //TODO check this heuristics, can be problematic when there are more applicable rules if (dictSubNodes != null) { List<KeyValuePair<char, LemmaTreeNode>> lReplaceNodes = new List<KeyValuePair<char, LemmaTreeNode>>(); foreach (KeyValuePair<char, LemmaTreeNode> kvpChild in dictSubNodes) if (kvpChild.Value.dictSubNodes != null && kvpChild.Value.dictSubNodes.Count == 1) { IEnumerator<LemmaTreeNode> enumChildChild = kvpChild.Value.dictSubNodes.Values.GetEnumerator(); enumChildChild.MoveNext(); LemmaTreeNode ltrChildChild = enumChildChild.Current; if (kvpChild.Value.lrBestRule == lrBestRule) lReplaceNodes.Add(new KeyValuePair<char, LemmaTreeNode>(kvpChild.Key, ltrChildChild)); } foreach (KeyValuePair<char, LemmaTreeNode> kvpChild in lReplaceNodes) { dictSubNodes[kvpChild.Key] = kvpChild.Value; kvpChild.Value.ltnParentNode = this; } } }
public void Deserialize(BinaryReader binRead, LemmatizerSettings lsett) { //load metadata bool bThisTopObject = binRead.ReadBoolean(); //load value types -------------------------------------- iId = binRead.ReadInt32(); iFrom = binRead.ReadInt32(); if (binRead.ReadBoolean()) sFrom = binRead.ReadString(); else sFrom = null; sTo = binRead.ReadString(); sSignature = binRead.ReadString(); //load refernce types if needed ------------------------- if (bThisTopObject) this.lsett = new LemmatizerSettings(binRead); else this.lsett = lsett; }
public LemmaTreeNode(BinaryReader binRead, LemmatizerSettings lsett, ExampleList elExamples, LemmaTreeNode ltnParentNode) { Deserialize(binRead, lsett, elExamples, ltnParentNode); }
public void Load(Latino.BinarySerializer binRead, LemmatizerSettings lsett, ExampleList elExamples, LemmaTreeNode ltnParentNode) { this.lsett = lsett; if (binRead.ReadBool()) { dictSubNodes = new Dictionary<char, LemmaTreeNode>(); int iCount = binRead.ReadInt(); for (int i = 0; i < iCount; i++) { char cKey = binRead.ReadChar(); LemmaTreeNode ltrSub = new LemmaTreeNode(binRead, this.lsett, elExamples, this); dictSubNodes.Add(cKey, ltrSub); } } else dictSubNodes = null; this.ltnParentNode = ltnParentNode; iSimilarity = binRead.ReadInt(); sCondition = binRead.ReadString(); bWholeWord = binRead.ReadBool(); lrBestRule = elExamples.Rules[binRead.ReadString()]; int iCountBest = binRead.ReadInt(); aBestRules = new RuleWeighted[iCountBest]; for (int i = 0; i < iCountBest; i++) aBestRules[i] = new RuleWeighted(elExamples.Rules[binRead.ReadString()], binRead.ReadDouble()); dWeight = binRead.ReadDouble(); iStart = binRead.ReadInt(); iEnd = binRead.ReadInt(); this.elExamples = elExamples; }
public LemmaTreeNode(Latino.BinarySerializer binRead, LemmatizerSettings lsett, ExampleList elExamples, LemmaTreeNode ltnParentNode) { Load(binRead, lsett, elExamples, ltnParentNode); }
private string sCondition; //suffix that must match in order to lemmatize #endregion Fields #region Constructors public LemmaTreeNode(LemmatizerSettings lsett, ExampleList elExamples) : this(lsett, elExamples, 0, elExamples.Count-1, null) { }
public LemmatizerPrebuilt(LanguagePrebuilt lang, LemmatizerSettings lsett) : base(lsett) { this.lang = lang; }
public LemmaRule(System.IO.BinaryReader binRead, LemmatizerSettings lsett) { this.Deserialize(binRead, lsett); }
public ExampleList(BinaryReader binRead, LemmatizerSettings lsett) { Deserialize(binRead, lsett); }
public LemmaRule(Latino.BinarySerializer binRead, LemmatizerSettings lsett) { Load(binRead, lsett); }
public ExampleList(StreamReader srIn, string sFormat, LemmatizerSettings lsett) : this(lsett) { AddMultextFile(srIn, sFormat); }
public void Load(Latino.BinarySerializer binRead) { lsett = new LemmatizerSettings(binRead); elExamples = new ExampleList(binRead, lsett); ltnRootNode = new LemmaTreeNode(binRead, lsett, elExamples, null); }
public void Load(Latino.BinarySerializer binRead, LemmatizerSettings lsett) { //load metadata bool bThisTopObject = binRead.ReadBool(); //load refernce types if needed ------------------------- if (bThisTopObject) this.lsett = new LemmatizerSettings(binRead); else this.lsett = lsett; rlRules = new RuleList(binRead, this.lsett); bool bCreateLstExamples = binRead.ReadBool(); lstExamples = bCreateLstExamples ? new List<LemmaExample>() : null; dictExamples = new Dictionary<string, LemmaExample>(); //load dictionary items int iCount = binRead.ReadInt(); for (int iId = 0; iId < iCount; iId++) { LemmaRule lrRule = rlRules[binRead.ReadString()]; LemmaExample le = new LemmaExample(binRead, this.lsett, lrRule); dictExamples.Add(le.Signature, le); if (bCreateLstExamples) lstExamples.Add(le); } }
public ExampleList(Latino.BinarySerializer binRead, LemmatizerSettings lsett) { Load(binRead, lsett); }
public Lemmatizer(StreamReader srIn, string sFormat, LemmatizerSettings lsett) : this(lsett) { AddMultextFile(srIn, sFormat); }
private LemmaTreeNode(LemmatizerSettings lsett) { this.lsett = lsett; }
public Lemmatizer(LemmatizerSettings lsett) { this.lsett = lsett; this.elExamples = new ExampleList(lsett); this.ltnRootNode = null; }
public Lemmatizer(SerializationInfo info, StreamingContext context) : this() { lsett = (LemmatizerSettings)info.GetValue("lsett", typeof(LemmatizerSettings)); elExamples = (ExampleList)info.GetValue("elExamples", typeof(ExampleList)); this.BuildModel(); }