private void Save(List <GameObjectRefs> refs) { BinarySerializer writer = new BinarySerializer(GetPath(backupName), FileAccess.Write); Exception ex = writer.Serialize(refs); if (ex == null) { ex = writer.Close(); } else { writer.Close(); } }
public static void Save(List <ModelImport> settings) { CustomAssetPostprocessor.modelSettings = null; BinarySerializer serializer = new BinarySerializer(GetStorePath(), FileAccess.Write); serializer.Serialize(settings); serializer.Close(); }
static CategorizerDemo() { string fileName = HttpContext.Current.Server.MapPath("App_Data\\model.bin"); BinarySerializer binReader = new BinarySerializer(fileName, FileMode.Open); mBowSpace = new BowSpace(binReader); mBowSpace.CutLowWeightsPerc = 0.2; mCategorizer = Utils.LoadDictionary<string, IModel<string>>(binReader); binReader.Close(); mReady = true; }
static TwitterSentimentDemo() { string modelFileName = HttpContext.Current.Server.MapPath("App_Data\\AdCfy.bin"); string bowSpcFileName = HttpContext.Current.Server.MapPath("App_Data\\AdCfyBowSpc.bin"); mClassifier = new SvmBinaryClassifier<int>(); mClassifier.LoadModel(modelFileName); BinarySerializer bs = new BinarySerializer(bowSpcFileName, FileMode.Open); mBowSpace = new BowSpace(bs); bs.Close(); mReady = true; }
static RssFeedComponent() { Assembly assembly = Assembly.GetExecutingAssembly(); foreach (string resName in assembly.GetManifestResourceNames()) { if (resName.EndsWith(".ldp")) { BinarySerializer ser = new BinarySerializer(assembly.GetManifestResourceStream(resName)); LanguageProfile langProfile = new LanguageProfile(ser); ser.Close(); mCodePageDetector.AddLanguageProfile(langProfile); } } }
static DocumentCategorizerComponent() { Logger.GetLogger(typeof(DocumentCategorizerComponent)).Info("CategorizerComponent", "Loading model ..."); string fileName = Utils.GetConfigValue("CategorizationModel", ".\\CategorizationModel.bin"); BinarySerializer binReader = new BinarySerializer(fileName, FileMode.Open); mBowSpace = new BowSpace(binReader); mBowSpace.CutLowWeightsPerc = 0.2; mCategorizer = Utils.LoadDictionary<string, IModel<string>>(binReader); binReader.Close(); //using (BinarySerializer writer = new BinarySerializer(@"C:\Work\AchimPipe\DocumentCategorizerModel.bin", FileMode.Create)) //{ // mBowSpace.Save(writer); // Utils.SaveDictionary<string, IModel<string>>(mCategorizer, writer); //} Logger.GetLogger(typeof(DocumentCategorizerComponent)).Info("CategorizerComponent", "Done."); }
static DocumentCategorizerComponent() { Logger.GetLogger(typeof(DocumentCategorizerComponent)).Info("CategorizerComponent", "Loading model ..."); string fileName = Utils.GetConfigValue("CategorizationModel", ".\\CategorizationModel.bin"); BinarySerializer binReader = new BinarySerializer(fileName, FileMode.Open); mBowSpace = new BowSpace(binReader); mBowSpace.CutLowWeightsPerc = 0.2; mCategorizer = Utils.LoadDictionary <string, IModel <string> >(binReader); binReader.Close(); //using (BinarySerializer writer = new BinarySerializer(@"C:\Work\AchimPipe\DocumentCategorizerModel.bin", FileMode.Create)) //{ // mBowSpace.Save(writer); // Utils.SaveDictionary<string, IModel<string>>(mCategorizer, writer); //} Logger.GetLogger(typeof(DocumentCategorizerComponent)).Info("CategorizerComponent", "Done."); }
public static LanguageDetector GetLanguageDetectorPrebuilt() { LanguageDetector ld = new LanguageDetector(); Assembly assembly = Assembly.GetExecutingAssembly(); foreach (string resName in assembly.GetManifestResourceNames()) { if (resName.EndsWith(".ldp")) { // load language detector profile BinarySerializer ser = new BinarySerializer(assembly.GetManifestResourceStream(resName)); LanguageProfile langProfile = new LanguageProfile(ser); ser.Close(); ld.AddLanguageProfile(langProfile); mLogger.Debug("GetLanguageDetectorPrebuilt", "Loaded resource {0}.", resName); } } return(ld); }
private void SaveSettings(object sender, EventArgs e) { var sfd = new SaveFileDialog { DefaultExt = ".tgc", Filter = "Файлы настроек (*.tgc)|*.tgc|Все файлы (*.*)|*.*" }; if (sfd.ShowDialog() != DialogResult.OK) { return; } var s = new BinarySerializer(sfd.FileName); if (!s.Open()) { MessageBox.Show("Не удалось открыть файл"); return; } if (!_scene.Settings.Serialize(s)) { MessageBox.Show("Не удалось сохранить настройки"); } s.Close(); }
private void Load() { ClearResults(); refDiffs.Clear(); BinarySerializer reader = new BinarySerializer(fileToLoad.FullName, FileAccess.Read); List <GameObjectRefs> backup = reader.Deserialize <List <GameObjectRefs> >(); foreach (GameObjectRefs refs in backup) { Transform t = TransformUtil.Search(refs.path); if (t != null) { Component[] comps = t.GetComponents(typeof(Component)); for (int i = 0; i < comps.Length && i < refs.refs.Count; ++i) { Component c = comps[i]; if (c == null) { continue; } CompRefs compRefs = refs.refs[i]; if (compRefs.compType == c.GetType().FullName) { foreach (FieldInfo f in registry.GetFields(c.GetType())) { Object backupVal = compRefs.GetValue(f.Name); if (backupVal != null) { refDiffs.Add(new RefDiff(c, f, backupVal)); } } } } } } reader.Close(); }
static void Main(string[] args) { LanguageDetector langDet = LanguageDetector.GetLanguageDetectorPrebuilt(); //LanguageDetector langDet = new LanguageDetector(); //langDet.ReadCorpus(@"C:\Users\mIHA\Desktop\langdet"); LanguageProfile p = langDet.FindMatchingLanguage("To je slovenski stavek. Čeprav ga naš detektor ne zazna pravilno. Mogoče šumniki pomagajo..."); Console.WriteLine(p.Language); p = langDet.FindMatchingLanguage("I love you."); Console.WriteLine(p.Language); p = langDet.FindMatchingLanguage("Baš te volim."); Console.WriteLine(p.Language); p = langDet.FindMatchingLanguage("Je t'aime."); Console.WriteLine(p.Language); foreach (LanguageProfile pr in langDet.LanguageProfiles) { BinarySerializer ser = new BinarySerializer(string.Format(@"C:\Users\mIHA\Desktop\langdet\{0}.ldp", pr.Language), FileMode.Create); pr.Save(ser); ser.Close(); } //Console.WriteLine(langDet.GetLanguageProfile("et")); //StreamWriter w = new StreamWriter("c:\\krneki\\langSim.txt"); //foreach (LanguageProfile p in langDet.LanguageProfiles) //{ // w.Write("{0}\t", p.Code); //} //w.WriteLine(); //foreach (LanguageProfile p in langDet.LanguageProfiles) //{ // foreach (LanguageProfile p2 in langDet.LanguageProfiles) // { // //w.Write("{0}\t", Math.Max(p.CalcSpearman(p2), p2.CalcSpearman(p))); // } // w.WriteLine(); //} //w.Close(); }
public static SparseMatrix <double> Gis <LblT>(ILabeledExampleCollection <LblT, BinaryVector> dataset, int cutOff, int numIter, bool clearDataset, string mtxFileName, ref LblT[] idxToLbl, int numThreads, double allowedDiff, IEqualityComparer <LblT> lblCmp, Logger logger) { logger.Info("Gis", "Creating observation matrix ..."); SparseMatrix <double> observations = null; if (Utils.VerifyFileNameOpen(mtxFileName)) { BinarySerializer reader = new BinarySerializer(mtxFileName, FileMode.Open); idxToLbl = new ArrayList <LblT>(reader).ToArray(); observations = new SparseMatrix <double>(reader); reader.Close(); } else { observations = CreateObservationMatrix(dataset, ref idxToLbl, lblCmp, logger); if (Utils.VerifyFileNameCreate(mtxFileName)) { BinarySerializer writer = new BinarySerializer(mtxFileName, FileMode.Create); new ArrayList <LblT>(idxToLbl).Save(writer); observations.Save(writer); writer.Close(); } } int numClasses = observations.GetLastNonEmptyRowIdx() + 1; int numExamples = dataset.Count; if (cutOff > 0) { logger.Info("Gis", "Performing cut-off ..."); observations = CutOff(observations, cutOff); } logger.Info("Gis", "Preparing structures ..."); SparseMatrix <double> lambda = CopyStructure(observations); SparseMatrix <double> expectations = CopyStructure(observations); double f = GisFindMaxF(dataset); SparseMatrix <double> trainMtxTr = TransposeDataset(dataset, clearDataset); logger.Info("Gis", "Entering main loop ..."); double[] oldLambda = null; if (allowedDiff > 0) { oldLambda = new double[lambda.CountValues()]; } for (int i = 0; i < numIter; i++) { logger.Info("Gis", "Iteration {0} / {1} ...", i + 1, numIter); logger.Info("Gis", "Updating expectations ..."); if (numThreads > 1) { UpdateExpectationMatrix(numClasses, numExamples, trainMtxTr, lambda, expectations, numThreads, logger); } else { UpdateExpectationMatrix(numClasses, numExamples, trainMtxTr, lambda, expectations, logger); } logger.Info("Gis", "Updating lambdas ..."); GisUpdate(lambda, expectations, observations, f); Reset(expectations); // check lambda change if (allowedDiff > 0) { int j = 0; double maxDiff = 0; foreach (IdxDat <SparseVector <double> > row in lambda) { foreach (IdxDat <double> item in row.Dat) { double diff = Math.Abs(item.Dat - oldLambda[j]); if (diff > maxDiff) { maxDiff = diff; } oldLambda[j] = item.Dat; j++; } } logger.Info("Gis", "Max lambda diff: {0:0.0000}", maxDiff); if (maxDiff <= allowedDiff) { logger.Info("Gis", "Max lambda diff is small enough. Exiting optimization loop."); break; } } } return(lambda); }
public static SparseMatrix <double> Gis <LblT>(IExampleCollection <LblT, BinaryVector <int> .ReadOnly> dataset, int cut_off, int num_iter, bool clear_dataset, string mtx_file_name, ref LblT[] idx_to_lbl, int num_threads) { Utils.VerboseLine("Creating observation matrix ..."); SparseMatrix <double> observations = null; if (Utils.VerifyFileNameOpen(mtx_file_name)) { BinarySerializer reader = new BinarySerializer(mtx_file_name, FileMode.Open); idx_to_lbl = new ArrayList <LblT>(reader).ToArray(); observations = new SparseMatrix <double>(reader); reader.Close(); } else { observations = CreateObservationMatrix2(dataset, ref idx_to_lbl); //SparseMatrix<double> test = CreateObservationMatrix(dataset, ref idx_to_lbl); //Console.WriteLine(test.ContentEquals(observations)); if (Utils.VerifyFileNameCreate(mtx_file_name)) { BinarySerializer writer = new BinarySerializer(mtx_file_name, FileMode.Create); new ArrayList <LblT>(idx_to_lbl).Save(writer); observations.Save(writer); writer.Close(); } } int num_classes = observations.GetLastNonEmptyRowIdx() + 1; int num_examples = dataset.Count; if (cut_off > 0) { Utils.VerboseLine("Performing cut-off ..."); observations = CutOff(observations, cut_off); } Utils.VerboseLine("Preparing structures ..."); SparseMatrix <double> lambda = CopyStructure(observations); SparseMatrix <double> expectations = CopyStructure(observations); double f = GisFindMaxF(dataset); SparseMatrix <double> train_mtx_tr = TransposeDataset(dataset, clear_dataset); Utils.VerboseLine("Entering main loop ..."); for (int i = 0; i < num_iter; i++) { Utils.VerboseLine("Iteration {0} / {1} ...", i + 1, num_iter); Utils.VerboseLine("Updating expectations ..."); if (num_threads > 1) { UpdateExpectationMatrix(num_classes, num_examples, train_mtx_tr, lambda, expectations, num_threads); } else { UpdateExpectationMatrix(num_classes, num_examples, train_mtx_tr, lambda, expectations); } Utils.VerboseLine("Updating lambdas ..."); GisUpdate(lambda, expectations, observations, f); //SaveForMatlab(expectations, "c:\\mec\\old\\expem.txt"); Reset(expectations); } //SaveForMatlab(lambda, "c:\\mec\\old\\lamem.txt"); //SaveForMatlab(observations, "c:\\mec\\old\\obsem.txt"); return(lambda); }
static void Main(string[] args) { try { if (args.Length < 2) { OutputHelp(); } else { string corpusFileName = null, modelFileName = null, lexiconFileName = null; bool considerTag = false; bool treeOpt = false; bool verbose = false; if (ParseParams(args, ref verbose, ref considerTag, ref treeOpt, ref corpusFileName, ref modelFileName, ref lexiconFileName)) { Logger logger = Logger.GetRootLogger(); if (!verbose) { logger.LocalLevel = Logger.Level.Off; logger.LocalProgressOutputType = Logger.ProgressOutputType.Off; } else { logger.LocalOutputType = Logger.OutputType.Custom; Logger.CustomOutput = new Logger.CustomOutputDelegate(delegate(string loggerName, Logger.Level level, string funcName, Exception e, string message, object[] msgArgs) { Console.WriteLine(message, msgArgs); }); } Corpus corpus = new Corpus(); logger.Info(/*funcName=*/ null, "Nalagam učni korpus ..."); corpus.LoadFromXmlFile(corpusFileName, /*tagLen=*/ int.MaxValue); LemmatizerSettings lemmatizerSettings = new LemmatizerSettings(); lemmatizerSettings.eMsdConsider = considerTag ? LemmatizerSettings.MsdConsideration.Distinct : LemmatizerSettings.MsdConsideration.Ignore; lemmatizerSettings.bUseFromInRules = true; lemmatizerSettings.iMaxRulesPerNode = 0; lemmatizerSettings.bBuildFrontLemmatizer = false; lemmatizerSettings.bStoreAllFullKnownWords = false; lemmatizerSettings.bUseMsdSplitTreeOptimization = treeOpt; Lemmatizer lemmatizer = new Lemmatizer(lemmatizerSettings); for (int i = 0; i < corpus.TaggedWords.Count; i++) { logger.ProgressFast(Logger.Level.Info, /*funcName=*/ null, "{0} / {1}", i + 1, corpus.TaggedWords.Count); TaggedWord word = corpus.TaggedWords[i]; if (!word.MoreInfo.Punctuation) { lemmatizer.AddExample(word.WordLower, word.Lemma.ToLower(), 1, word.Tag); } } if (lexiconFileName != null) { logger.Info(/*funcName=*/ null, "Nalagam leksikon ..."); StreamReader lexReader = new StreamReader(lexiconFileName); string lexLine; int i = 0; while ((lexLine = lexReader.ReadLine()) != null) { // lexicon format: word \t lemma \t tag \t freq logger.ProgressFast(Logger.Level.Info, /*funcName=*/ null, "{0}", ++i, /*numSteps=*/ 0); string[] lexData = lexLine.Split('\t'); string word = lexData[0]; string lemma = lexData[1]; string tag = lexData[2]; double freq = Math.Max(0.1, Convert.ToDouble(lexData[3])); lemmatizer.AddExample(word.ToLower(), lemma.ToLower(), freq, tag); } logger.ProgressFast(Logger.Level.Info, /*funcName=*/ null, "{0}", i, i); lexReader.Close(); } logger.Info(/*funcName=*/ null, "Gradim model za lematizacijo ..."); if (treeOpt) { string msdSpec = Utils.GetManifestResourceString(typeof(Program), "MsdSpecsSloSloCodes.txt"); MsdSplitTree.BeamSearchParams beamSearchParams = new MsdSplitTree.BeamSearchParams(); beamSearchParams.beamsPerLevel[0] = 2; lemmatizer.BuildModel(msdSpec, beamSearchParams); } else { lemmatizer.BuildModel(); } logger.Info(/*funcName=*/ null, "Optimiram lematizacijsko drevo ..."); lemmatizer.OptimizeMemorySize(); logger.Info(/*funcName=*/ null, "Zapisujem model ..."); BinarySerializer writer = new BinarySerializer(modelFileName, FileMode.Create); writer.WriteBool(considerTag); lemmatizer.Save(writer); writer.Close(); logger.Info(/*funcName=*/ null, "Končano."); } } } catch (Exception exception) { Console.WriteLine(); Console.WriteLine("*** Nepričakovana napaka. Podrobnosti: {0}\r\n{1}", exception, exception.StackTrace); } }
public override void Run(object[] args) { // Create an ArrayList of sets of numbers and populate it. ArrayList <Set <int> > array = new ArrayList <Set <int> >( new Set <int>[] { new Set <int>(new int[] { 1, 3, 5 }), new Set <int>(new int[] { 2, 4, 6 }), new Set <int>(new int[] { 1, 2, 3 }) }); Output.WriteLine(array); // Output: ( { 1 3 5 } { 2 4 6 } { 1 2 3 } ) // Serialize the instance into memory. BinarySerializer memSer = new BinarySerializer(); array.Save(memSer); array = null; // Loose the instance. // Deserialize the instance. memSer.Stream.Position = 0; array = new ArrayList <Set <int> >(memSer); Output.WriteLine(array); // Output: ( { 1 3 5 } { 2 4 6 } { 1 2 3 } ) // Serialize the instance into a file. BinarySerializer fileSer = new BinarySerializer("array.bin", FileMode.Create); array.Save(fileSer); fileSer.Close(); array = null; // Loose the instance. // Deserialize the instance from the file. fileSer = new BinarySerializer("array.bin", FileMode.Open); array = new ArrayList <Set <int> >(fileSer); fileSer.Close(); Output.WriteLine(array); // Output: ( { 1 3 5 } { 2 4 6 } { 1 2 3 } ) // Create an instance of SerializableObject and assign the // Tag member variable. SerializableObject serObj = new SerializableObject(); serObj.Tag = array; // Serialize serObj into memory and deserialize it again. Output.WriteLine(serObj.Tag); // Output: ( { 1 3 5 } { 2 4 6 } { 1 2 3 } ) memSer = new BinarySerializer(); serObj.Save(memSer); serObj = null; // Loose the instance. memSer.Stream.Position = 0; serObj = new SerializableObject(memSer); Output.WriteLine(serObj.Tag); // Output: ( { 1 3 5 } { 2 4 6 } { 1 2 3 } ) }
public static LanguageDetector GetLanguageDetectorPrebuilt() { LanguageDetector ld = new LanguageDetector(); Assembly assembly = Assembly.GetExecutingAssembly(); foreach (string resName in assembly.GetManifestResourceNames()) { if (resName.EndsWith(".ldp")) { // load language detector profile BinarySerializer ser = new BinarySerializer(assembly.GetManifestResourceStream(resName)); LanguageProfile langProfile = new LanguageProfile(ser); ser.Close(); ld.AddLanguageProfile(langProfile); mLogger.Debug("GetLanguageDetectorPrebuilt", "Loaded resource {0}.", resName); } } return ld; }
static void Main(string[] args) { try { if (args.Length < 2) { OutputHelp(); } else { int cutOff = 2; int numIter = 50; int numThreads = 1; string corpusFileName = null, modelFileName = null, lexiconFileName = null; bool verbose = false; if (ParseParams(args, ref verbose, ref cutOff, ref numIter, ref numThreads, ref corpusFileName, ref modelFileName, ref lexiconFileName)) { Logger logger = Logger.GetRootLogger(); if (!verbose) { logger.LocalLevel = Logger.Level.Off; logger.LocalProgressOutputType = Logger.ProgressOutputType.Off; } else { logger.LocalOutputType = Logger.OutputType.Custom; Logger.CustomOutput = new Logger.CustomOutputDelegate(delegate(string loggerName, Logger.Level level, string funcName, Exception e, string message, object[] msgArgs) { Console.WriteLine(message, msgArgs); }); } Corpus corpus = new Corpus(); logger.Info(/*funcName=*/ null, "Nalagam učni korpus ..."); corpus.LoadFromXmlFile(corpusFileName, /*tagLen=*/ -1); GC.Collect(); long oldMemUse = Process.GetCurrentProcess().PrivateMemorySize64; PatriciaTree suffixTree = new PatriciaTree(); foreach (TaggedWord word in corpus.TaggedWords) { suffixTree.AddWordTagPair(word.WordLower, word.Tag); } if (lexiconFileName != null) { logger.Info(/*funcName=*/ null, "Nalagam leksikon ..."); StreamReader lexReader = new StreamReader(lexiconFileName); string lexLine; while ((lexLine = lexReader.ReadLine()) != null) { string[] lexData = lexLine.Split('\t'); suffixTree.AddWordTagPair(lexData[0].ToLower(), lexData[2]); } lexReader.Close(); } GC.Collect(); long memUse = Process.GetCurrentProcess().PrivateMemorySize64; Console.WriteLine("Poraba pomnilnika (drevo končnic): {0:0.00} MB", (double)(memUse - oldMemUse) / 1048576.0); oldMemUse = memUse; suffixTree.PropagateTags(); GC.Collect(); memUse = Process.GetCurrentProcess().PrivateMemorySize64; Console.WriteLine("Poraba pomnilnika (propagirane oznake): {0:0.00} MB", (double)(memUse - oldMemUse) / 1048576.0); MaximumEntropyClassifierFast <string> model = new MaximumEntropyClassifierFast <string>(); LabeledDataset <string, BinaryVector> dataset = new LabeledDataset <string, BinaryVector>(); Dictionary <string, int> featureSpace = new Dictionary <string, int>(); logger.Info(/*funcName=*/ null, "Pripravljam vektorje značilk ..."); for (int i = 0; i < corpus.TaggedWords.Count; i++) { logger.ProgressFast(Logger.Level.Info, /*funcName=*/ null, "{0} / {1}", i + 1, corpus.TaggedWords.Count); BinaryVector featureVector = corpus.GenerateFeatureVector(i, featureSpace, /*extendFeatureSpace=*/ true, suffixTree); dataset.Add(corpus.TaggedWords[i].Tag, featureVector); } logger.Info(/*funcName=*/ null, "Gradim model ..."); DateTime startTime = DateTime.Now; model.CutOff = cutOff; model.NumThreads = numThreads; model.NumIter = numIter; model.Train(dataset); TimeSpan span = DateTime.Now - startTime; logger.Info(/*funcName=*/ null, "Trajanje gradnje modela: {0:00}:{1:00}:{2:00}.{3:000}.", span.Hours, span.Minutes, span.Seconds, span.Milliseconds); logger.Info(/*funcName=*/ null, "Zapisujem model ..."); BinarySerializer writer = new BinarySerializer(modelFileName, FileMode.Create); suffixTree.Save(writer); Utils.SaveDictionary(featureSpace, writer); model.Save(writer); writer.Close(); logger.Info(/*funcName=*/ null, "Končano."); } } } catch (Exception exception) { Console.WriteLine(); Console.WriteLine("*** Nepričakovana napaka. Podrobnosti: {0}\r\n{1}", exception, exception.StackTrace); } }