public static void BuildAllLemmatizers(bool bIncludeExamples, bool bCompress, LemmatizerSettings lsett, string sNewFileMask)
        {
            DateTime dtStartAll = DateTime.Now;
            foreach (LanguagePrebuilt lp in Enum.GetValues(typeof(LanguagePrebuilt))) {
                DateTime dtStart = DateTime.Now;

                string sDataFileName = GetResourceFileName(sFileMask, lp);
                string sLemFileName = GetResourceFileName(sNewFileMask, lp);

                Console.WriteLine("Building lemmatizer for {0} from {1}", lp, sDataFileName);

                LemmatizerPrebuiltData lemPrebuild = new LemmatizerPrebuiltData(lp, lsett);

                Console.WriteLine("  Building lemmatizer completed in: {0}", new TimeSpan(DateTime.Now.Ticks - dtStart.Ticks).ToString());
                dtStart = DateTime.Now;
                Console.WriteLine("  Serializing {0} lemmatizer to file {1}", (bIncludeExamples ? "full (examples included)" : "compact (examples excluded)"), sLemFileName);

                Stream binStreamOut = File.Open(sLemFileName, FileMode.Create);
                lemPrebuild.Serialize(binStreamOut, bIncludeExamples, bCompress);
                binStreamOut.Close();

                Console.WriteLine("  Successfully completed, time needed: {0}", new TimeSpan(DateTime.Now.Ticks - dtStart.Ticks).ToString());
            }
            Console.WriteLine("Time for building all lemmatizers: " + new TimeSpan(DateTime.Now.Ticks - dtStartAll.Ticks).ToString());
        }
Beispiel #2
0
        public ExampleList(LemmatizerSettings lsett)
            : base()
        {
            this.lsett = lsett;

            this.dictExamples = new Dictionary<string, LemmaExample>();
            this.lstExamples = null;
            this.rlRules = new RuleList(lsett);
        }
        public LemmatizerPrebuiltData(LanguagePrebuilt lang, LemmatizerSettings lsett)
            : base(lang,lsett)
        {
            Stream stream = GetResourceStream(ResourceFileName);
            StreamReader srIn = new StreamReader(stream, LanguageEncoding);
            AddMultextFile(srIn, "WLM");
            srIn.Close();

            BuildModel();
        }
Beispiel #4
0
        public ExampleList(SerializationInfo info, StreamingContext context)
        {
            lsett = (LemmatizerSettings)info.GetValue("lsett", typeof(LemmatizerSettings));

            this.dictExamples = new Dictionary<string, LemmaExample>();
            this.lstExamples = null;
            this.rlRules = new RuleList(lsett);

            string[] aWords = (string[])info.GetValue("aWords", typeof(string[]));
            string[] aLemmas = (string[])info.GetValue("aLemmas", typeof(string[]));
            double[] aWeights = (double[])info.GetValue("aWeights", typeof(double[]));
            string[] aMsds = (string[])info.GetValue("aMsds", typeof(string[]));

            for (int iExm = 0; iExm < aWords.Length; iExm++)
                AddExample(aWords[iExm], aLemmas[iExm], aWeights[iExm], aMsds[iExm]);
        }
Beispiel #5
0
        public LemmaRule(string sWord, string sLemma, int iId, LemmatizerSettings lsett)
        {
            this.lsett = lsett;
            this.iId = iId;

            int iSameStem = SameStem(sWord, sLemma);
            sTo = sLemma.Substring(iSameStem);
            iFrom = sWord.Length - iSameStem;

            if (lsett.bUseFromInRules) {
                sFrom = sWord.Substring(iSameStem);
                sSignature = "[" + sFrom + "]==>[" + sTo + "]";
            }
            else {
                sFrom = null;
                sSignature = "[#" + iFrom + "]==>[" + sTo + "]";
            }
        }
Beispiel #6
0
        /// <summary>
        /// 
        /// </summary>
        /// <param name="lsett"></param>
        /// <param name="elExamples"></param>
        /// <param name="iStart">Index of the first word of the current group</param>
        /// <param name="iEnd">Index of the last word of the current group</param>
        /// <param name="ltnParentNode"></param>
        private LemmaTreeNode(LemmatizerSettings lsett, ExampleList elExamples, int iStart, int iEnd, LemmaTreeNode ltnParentNode)
            : this(lsett)
        {
            this.ltnParentNode = ltnParentNode;
            this.dictSubNodes = null;

            this.iStart = iStart;
            this.iEnd = iEnd;
            this.elExamples = elExamples;

            if (iStart >= elExamples.Count || iEnd >= elExamples.Count || iStart > iEnd) {
                lrBestRule = elExamples.Rules.DefaultRule;
                aBestRules = new RuleWeighted[1];
                aBestRules[0] = new RuleWeighted(lrBestRule, 0);
                dWeight = 0;
                return;
            }

            int iConditionLength = Math.Min(ltnParentNode == null ? 0 : ltnParentNode.iSimilarity + 1, elExamples[iStart].Word.Length);
            this.sCondition = elExamples[iStart].Word.Substring(elExamples[iStart].Word.Length - iConditionLength);
            this.iSimilarity = elExamples[iStart].Similarity(elExamples[iEnd]);
            this.bWholeWord = ltnParentNode == null ? false : elExamples[iEnd].Word.Length == ltnParentNode.iSimilarity;

            FindBestRules();
            AddSubAll();

            //TODO check this heuristics, can be problematic when there are more applicable rules
            if (dictSubNodes != null) {
                List<KeyValuePair<char, LemmaTreeNode>> lReplaceNodes = new List<KeyValuePair<char, LemmaTreeNode>>();
                foreach (KeyValuePair<char, LemmaTreeNode> kvpChild in dictSubNodes)
                    if (kvpChild.Value.dictSubNodes != null && kvpChild.Value.dictSubNodes.Count == 1) {
                        IEnumerator<LemmaTreeNode> enumChildChild = kvpChild.Value.dictSubNodes.Values.GetEnumerator();
                        enumChildChild.MoveNext();
                        LemmaTreeNode ltrChildChild = enumChildChild.Current;
                        if (kvpChild.Value.lrBestRule == lrBestRule)
                            lReplaceNodes.Add(new KeyValuePair<char, LemmaTreeNode>(kvpChild.Key, ltrChildChild));
                    }
                foreach (KeyValuePair<char, LemmaTreeNode> kvpChild in lReplaceNodes) {
                    dictSubNodes[kvpChild.Key] = kvpChild.Value;
                    kvpChild.Value.ltnParentNode = this;
                }

            }
        }
Beispiel #7
0
        public void Deserialize(BinaryReader binRead, LemmatizerSettings lsett)
        {
            //load metadata
            bool bThisTopObject = binRead.ReadBoolean();

            //load value types --------------------------------------
            iId = binRead.ReadInt32();
            iFrom = binRead.ReadInt32();
            if (binRead.ReadBoolean())
                sFrom = binRead.ReadString();
            else
                sFrom = null;
            sTo = binRead.ReadString();
            sSignature = binRead.ReadString();

            //load refernce types if needed -------------------------
            if (bThisTopObject)
                this.lsett = new LemmatizerSettings(binRead);
            else
                this.lsett = lsett;
        }
Beispiel #8
0
 public LemmaTreeNode(BinaryReader binRead, LemmatizerSettings lsett, ExampleList elExamples, LemmaTreeNode ltnParentNode)
 {
     Deserialize(binRead, lsett, elExamples, ltnParentNode);
 }
Beispiel #9
0
        public void Load(Latino.BinarySerializer binRead, LemmatizerSettings lsett, ExampleList elExamples, LemmaTreeNode ltnParentNode)
        {
            this.lsett = lsett;

            if (binRead.ReadBool()) {
                dictSubNodes = new Dictionary<char, LemmaTreeNode>();
                int iCount = binRead.ReadInt();
                for (int i = 0; i < iCount; i++) {
                    char cKey = binRead.ReadChar();
                    LemmaTreeNode ltrSub = new LemmaTreeNode(binRead, this.lsett, elExamples, this);
                    dictSubNodes.Add(cKey, ltrSub);
                }
            }
            else
                dictSubNodes = null;

            this.ltnParentNode = ltnParentNode;

            iSimilarity = binRead.ReadInt();
            sCondition = binRead.ReadString();
            bWholeWord = binRead.ReadBool();

            lrBestRule = elExamples.Rules[binRead.ReadString()];

            int iCountBest = binRead.ReadInt();
            aBestRules = new RuleWeighted[iCountBest];
            for (int i = 0; i < iCountBest; i++)
                aBestRules[i] = new RuleWeighted(elExamples.Rules[binRead.ReadString()], binRead.ReadDouble());

            dWeight = binRead.ReadDouble();

            iStart = binRead.ReadInt();
            iEnd = binRead.ReadInt();
            this.elExamples = elExamples;
        }
Beispiel #10
0
 public LemmaTreeNode(Latino.BinarySerializer binRead, LemmatizerSettings lsett, ExampleList elExamples, LemmaTreeNode ltnParentNode)
 {
     Load(binRead, lsett, elExamples, ltnParentNode);
 }
Beispiel #11
0
        private string sCondition; //suffix that must match in order to lemmatize

        #endregion Fields

        #region Constructors

        public LemmaTreeNode(LemmatizerSettings lsett, ExampleList elExamples)
            : this(lsett, elExamples, 0, elExamples.Count-1, null)
        {
        }
Beispiel #12
0
 public LemmatizerPrebuilt(LanguagePrebuilt lang, LemmatizerSettings lsett)
     : base(lsett)
 {
     this.lang = lang;
 }
Beispiel #13
0
 public LemmaRule(System.IO.BinaryReader binRead, LemmatizerSettings lsett)
 {
     this.Deserialize(binRead, lsett);
 }
Beispiel #14
0
 public ExampleList(BinaryReader binRead, LemmatizerSettings lsett)
 {
     Deserialize(binRead, lsett);
 }
Beispiel #15
0
 public LemmaRule(Latino.BinarySerializer binRead, LemmatizerSettings lsett)
 {
     Load(binRead, lsett);
 }
Beispiel #16
0
 public ExampleList(StreamReader srIn, string sFormat, LemmatizerSettings lsett)
     : this(lsett)
 {
     AddMultextFile(srIn, sFormat);
 }
Beispiel #17
0
 public void Load(Latino.BinarySerializer binRead)
 {
     lsett       = new LemmatizerSettings(binRead);
     elExamples  = new ExampleList(binRead, lsett);
     ltnRootNode = new LemmaTreeNode(binRead, lsett, elExamples, null);
 }
Beispiel #18
0
        public void Load(Latino.BinarySerializer binRead, LemmatizerSettings lsett)
        {
            //load metadata
            bool bThisTopObject = binRead.ReadBool();

            //load refernce types if needed -------------------------
            if (bThisTopObject)
                this.lsett = new LemmatizerSettings(binRead);
            else
                this.lsett = lsett;

            rlRules = new RuleList(binRead, this.lsett);

            bool bCreateLstExamples = binRead.ReadBool();

            lstExamples = bCreateLstExamples ? new List<LemmaExample>() : null;
            dictExamples = new Dictionary<string, LemmaExample>();

            //load dictionary items
            int iCount = binRead.ReadInt();
            for (int iId = 0; iId < iCount; iId++) {
                LemmaRule lrRule = rlRules[binRead.ReadString()];
                LemmaExample le = new LemmaExample(binRead, this.lsett, lrRule);

                dictExamples.Add(le.Signature, le);
                if (bCreateLstExamples) lstExamples.Add(le);
            }
        }
Beispiel #19
0
 public ExampleList(Latino.BinarySerializer binRead, LemmatizerSettings lsett)
 {
     Load(binRead, lsett);
 }
Beispiel #20
0
 public Lemmatizer(StreamReader srIn, string sFormat, LemmatizerSettings lsett) : this(lsett) {
     AddMultextFile(srIn, sFormat);
 }
Beispiel #21
0
 private LemmaTreeNode(LemmatizerSettings lsett)
 {
     this.lsett = lsett;
 }
Beispiel #22
0
 public Lemmatizer(LemmatizerSettings lsett)
 {
     this.lsett       = lsett;
     this.elExamples  = new ExampleList(lsett);
     this.ltnRootNode = null;
 }
Beispiel #23
0
 public Lemmatizer(SerializationInfo info, StreamingContext context) : this()
 {
     lsett      = (LemmatizerSettings)info.GetValue("lsett", typeof(LemmatizerSettings));
     elExamples = (ExampleList)info.GetValue("elExamples", typeof(ExampleList));
     this.BuildModel();
 }
Beispiel #24
0
 public LemmatizerPrebuilt(LanguagePrebuilt lang, LemmatizerSettings lsett)
     : base(lsett)
 {
     this.lang = lang;
 }