public DefaultControlBuilderFactory(RedwoodConfiguration configuration) { this.configuration = configuration; ViewCompilerFactory = () => configuration.ServiceLocator.GetService<IViewCompiler>(); markupFileLoader = configuration.ServiceLocator.GetService<IMarkupFileLoader>(); }
public static void MinimalSetup() { Properties props = new Properties(); props.SetProperty("log.output", "stderr"); RedwoodConfiguration.Apply(props); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> public static void Main(string[] args) { RedwoodConfiguration.Standard().Apply(); // Disable SLF4J crap. ArgumentParser.FillOptions(typeof(Edu.Stanford.Nlp.IE.KBPEnsembleExtractor), args); object @object = IOUtils.ReadObjectFromURLOrClasspathOrFileSystem(StatisticalModel); IKBPRelationExtractor statisticalExtractor; if (@object is LinearClassifier) { //noinspection unchecked statisticalExtractor = new KBPStatisticalExtractor((IClassifier <string, string>)@object); } else { if (@object is KBPStatisticalExtractor) { statisticalExtractor = (KBPStatisticalExtractor)@object; } else { throw new InvalidCastException(@object.GetType() + " cannot be cast into a " + typeof(KBPStatisticalExtractor)); } } logger.Info("Read statistical model from " + StatisticalModel); IKBPRelationExtractor extractor = new Edu.Stanford.Nlp.IE.KBPEnsembleExtractor(new KBPTokensregexExtractor(TokensregexDir), new KBPSemgrexExtractor(SemgrexDir), statisticalExtractor); IList <Pair <KBPRelationExtractor.KBPInput, string> > testExamples = IKBPRelationExtractor.ReadDataset(TestFile); extractor.ComputeAccuracy(testExamples.Stream(), Predictions.Map(null)); }
public static void Demonstrate(string text, POSMode partOfSpeechMode, NERMode namedEntityRecognitionMode, bool disableLogging = true) { if (disableLogging) { RedwoodConfiguration.current().clear().apply(); } //Input //Console.WriteLine("Input: {0}\n\n\n", text); ////Tokenization //Console.WriteLine("Tokenization:"); //Tokenisation.TokenizeText(text); //Console.WriteLine("\n\n\n"); ////POS //Console.WriteLine("Part Of Speech:"); //PartOfSpeech.Tag(text, partOfSpeechMode); //Console.WriteLine("\n\n\n"); ////NER //Console.WriteLine("Named Entity Recognition:"); //var ner = new NER(namedEntityRecognitionMode); //Console.WriteLine(ner.classifyToString(text)); //Console.WriteLine("\n\n\n"); ////Parser //Console.WriteLine("Parsed Text:"); //Parser.ParseString(text); //Console.WriteLine("\n\n\n"); //Find co-reference CorefAnnotator.FindCoreferenceResolution(text); }
/// <summary> /// Setup extended tagger that includes POS, lemma and entity analysis /// </summary> private void SetupExtendedTagger() { PerformanceTester.StartMET("NLP"); // Get path to Stanford NLP models var jarRoot = Path.Combine(Utility.GetResourcesFolder(), @"stanford-corenlp-3.9.2-models"); // Turn off logging RedwoodConfiguration.current().clear().apply(); var props = new java.util.Properties(); props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner"); // Makes Named Entity Recognition work in the library props.setProperty("ner.useSUTime", "0"); props.put("ner.applyFineGrained", "0"); props.put("ner.fine.regexner.mapping", jarRoot + @"\edu\stanford\nlp\models\kbp\english\"); // Set current directory var curDir = Environment.CurrentDirectory; var modelsDirectory = curDir + "\\" + jarRoot + @"\edu\stanford\nlp\models"; Directory.SetCurrentDirectory(jarRoot); // Load Stanford NLP Tagger = new StanfordCoreNLP(props); PerformanceTester.StopMET("NLP"); }
// Logging to file facilities. // The prefix is used to append stuff in front of the logging messages /// <exception cref="System.IO.IOException"/> public virtual void InitLog(File logFilePath) { RedwoodConfiguration.Empty().Handlers(RedwoodConfiguration.Handlers.Chain(RedwoodConfiguration.Handlers.ShowAllChannels(), RedwoodConfiguration.Handlers.stderr), RedwoodConfiguration.Handlers.File(logFilePath.ToString())).Apply(); // fh.setFormatter(new NewlineLogFormatter()); System.Console.Out.WriteLine("Starting Ssurgeon log, at " + logFilePath.GetAbsolutePath() + " date=" + DateFormat.GetDateInstance(DateFormat.Full).Format(new DateTime())); log.Info("Starting Ssurgeon log, date=" + DateFormat.GetDateInstance(DateFormat.Full).Format(new DateTime())); }
/// <summary> /// Initializes a new instance of the <see cref="RedwoodPresenter"/> class. /// </summary> public RedwoodPresenter(RedwoodConfiguration configuration) { RedwoodViewBuilder = configuration.ServiceLocator.GetService<IRedwoodViewBuilder>(); ViewModelLoader = configuration.ServiceLocator.GetService<IViewModelLoader>(); ViewModelSerializer = configuration.ServiceLocator.GetService<IViewModelSerializer>(); OutputRenderer = configuration.ServiceLocator.GetService<IOutputRenderer>(); CsrfProtector = configuration.ServiceLocator.GetService<ICsrfProtector>(); }
public static void Execute(string option, string text, bool disableLogging = true) { if (disableLogging) { RedwoodConfiguration.current().clear().apply(); } var jarRoot = @"../../../data/paket-files/stanford-corenlp-3.9.1-models/"; var props = new Properties(); props.setProperty("annotators", option); props.setProperty("ner.useSUTime", "0"); // We should change current directory, so StanfordCoreNLP could find all the model files automatically var curDir = Environment.CurrentDirectory; Directory.SetCurrentDirectory(jarRoot); var pipeline = new StanfordNLP.StanfordCoreNLP(props); Directory.SetCurrentDirectory(curDir); // Annotation var annotation = new StanfordNLP.Annotation(text); pipeline.annotate(annotation); //get sentencesAnnotation to get sentences var sentencesAnnotation = new CoreAnnotations.SentencesAnnotation().getClass(); //get tokensAnnotaion to get tokens in each sentence var tokensAnnotaion = new CoreAnnotations.TokensAnnotation().getClass(); //get posAnnotation to get POS result of each token var posAnnotation = new CoreAnnotations.PartOfSpeechAnnotation().getClass(); //get nerAnnotation to get NER result of each token var nerAnnotaion = new CoreAnnotations.NamedEntityTagAnnotation().getClass(); var deparseAnnotation = new TreeCoreAnnotations.TreeAnnotation().getClass(); //deparseAnnotation = new TypedDependency().getClass(); var sentences = annotation.get(sentencesAnnotation) as ArrayList; foreach (CoreMap sentence in sentences.toArray()) { var tokens = (ArrayList)sentence.get(tokensAnnotaion); Console.WriteLine("Token-POS-NER: "); foreach (CoreLabel token in tokens) { Console.Write($"{token.value()}-{token.get(posAnnotation)}-{token.get(nerAnnotaion)} "); } Console.WriteLine("\n\n\n"); var parsedText = (Tree)sentence.get(deparseAnnotation); if (parsedText != null) { Console.WriteLine("Parsed Text: "); new TreePrint("penn,typedDependenciesCollapsed").printTree(parsedText); } } }
/// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { RedwoodConfiguration.Standard().Apply(); // Disable SLF4J crap. ArgumentParser.FillOptions(typeof(Edu.Stanford.Nlp.IE.KBPSemgrexExtractor), args); Edu.Stanford.Nlp.IE.KBPSemgrexExtractor extractor = new Edu.Stanford.Nlp.IE.KBPSemgrexExtractor(Dir); IList <Pair <KBPRelationExtractor.KBPInput, string> > testExamples = IKBPRelationExtractor.ReadDataset(TestFile); extractor.ComputeAccuracy(testExamples.Stream(), Predictions.Map(null)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> public static void Main(string[] args) { RedwoodConfiguration.Standard().Apply(); // Disable SLF4J crap. ArgumentParser.FillOptions(typeof(Edu.Stanford.Nlp.IE.KBPStatisticalExtractor), args); // Fill command-line options // Load the test (or dev) data Redwood.Util.ForceTrack("Test data"); IList <Pair <KBPRelationExtractor.KBPInput, string> > testExamples = IKBPRelationExtractor.ReadDataset(TestFile); log.Info("Read " + testExamples.Count + " examples"); Redwood.Util.EndTrack("Test data"); // If we can't find an existing model, train one if (!IOUtils.ExistsInClasspathOrFileSystem(ModelFile)) { Redwood.Util.ForceTrack("Training data"); IList <Pair <KBPRelationExtractor.KBPInput, string> > trainExamples = IKBPRelationExtractor.ReadDataset(TrainFile); log.Info("Read " + trainExamples.Count + " examples"); log.Info(string.Empty + trainExamples.Stream().Map(null).Filter(null).Count() + " are " + KBPRelationExtractorConstants.NoRelation); Redwood.Util.EndTrack("Training data"); // Featurize + create the dataset Redwood.Util.ForceTrack("Creating dataset"); RVFDataset <string, string> dataset = new RVFDataset <string, string>(); AtomicInteger i = new AtomicInteger(0); long beginTime = Runtime.CurrentTimeMillis(); trainExamples.Stream().Parallel().ForEach(null); // This takes a while per example trainExamples.Clear(); // Free up some memory Redwood.Util.EndTrack("Creating dataset"); // Train the classifier log.Info("Training classifier:"); IClassifier <string, string> classifier = TrainMultinomialClassifier(dataset, FeatureThreshold, Sigma); dataset.Clear(); // Free up some memory // Save the classifier IOUtils.WriteObjectToFile(new Edu.Stanford.Nlp.IE.KBPStatisticalExtractor(classifier), ModelFile); } // Read either a newly-trained or pre-trained model object model = IOUtils.ReadObjectFromURLOrClasspathOrFileSystem(ModelFile); Edu.Stanford.Nlp.IE.KBPStatisticalExtractor classifier_1; if (model is IClassifier) { //noinspection unchecked classifier_1 = new Edu.Stanford.Nlp.IE.KBPStatisticalExtractor((IClassifier <string, string>)model); } else { classifier_1 = ((Edu.Stanford.Nlp.IE.KBPStatisticalExtractor)model); } // Evaluate the model classifier_1.ComputeAccuracy(testExamples.Stream(), Predictions.Map(null)); }
/// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { RedwoodConfiguration.Standard().Apply(); Redwood.Util.StartTrack("main"); // Read the data IStream <SimpleSentiment.SentimentDatum> data = IStream.Concat(IStream.Concat(IStream.Concat(Imdb("/users/gabor/tmp/aclImdb/train/pos", SentimentClass.Positive), Imdb("/users/gabor/tmp/aclImdb/train/neg", SentimentClass.Negative)), IStream.Concat (Imdb("/users/gabor/tmp/aclImdb/test/pos", SentimentClass.Positive), Imdb("/users/gabor/tmp/aclImdb/test/neg", SentimentClass.Negative))), IStream.Concat(IStream.Concat(Stanford("/users/gabor/tmp/train.tsv"), Stanford("/users/gabor/tmp/test.tsv" )), IStream.Concat(Twitter("/users/gabor/tmp/twitter.csv"), Unlabelled("/users/gabor/tmp/wikipedia")))); // Train the model OutputStream stream = IOUtils.GetFileOutputStream("/users/gabor/tmp/model.ser.gz"); SimpleSentiment classifier = SimpleSentiment.Train(data, Optional.Of(stream)); stream.Close(); log.Info(classifier.Classify("I think life is great")); Redwood.Util.EndTrack("main"); }
/// <summary> /// Configures the Redwood logger using a reasonable set of defaults, /// which can be overruled by the supplied Properties file. /// </summary> /// <param name="props">The properties file to overrule or augment the default configuration</param> public static void Apply(Properties props) { //--Tweak Properties //(output to stderr) if (props.GetProperty("log.output") == null) { props.SetProperty("log.output", "stderr"); } //(capture system streams) if (props.GetProperty("log.captureStderr") == null) { props.SetProperty("log.captureStderr", "true"); } //(apply properties) RedwoodConfiguration.Apply(props); //--Strange Tweaks //(adapt legacy logging systems) JavaUtilLoggingAdaptor.Adapt(); }
//*************************************************/ // METHODS //*************************************************/ #region Methods /// <summary> /// Setup tagger including POS /// </summary> private void SetupTagger() { PerformanceTester.StartMET("NLP"); // Get path to Stanford NLP models var jarRoot = Path.Combine(Utility.GetResourcesFolder(), @"stanford-corenlp-3.9.2-models"); // Turn off logging RedwoodConfiguration.current().clear().apply(); // Set properties var props = new java.util.Properties(); props.setProperty("annotators", "tokenize, ssplit, pos"); // Set current directory var curDir = Environment.CurrentDirectory; var modelsDirectory = curDir + "\\" + jarRoot + @"\edu\stanford\nlp\models"; Directory.SetCurrentDirectory(jarRoot); // Load Stanford NLP Tagger = new StanfordCoreNLP(props); PerformanceTester.StopMET("NLP"); }
/// <exception cref="System.Exception"/> public static void RunCoref(Properties props) { /* * property, environment setting */ Redwood.HideChannelsEverywhere("debug-cluster", "debug-mention", "debug-preprocessor", "debug-docreader", "debug-mergethres", "debug-featureselection", "debug-md"); int nThreads = HybridCorefProperties.GetThreadCounts(props); string timeStamp = Calendar.GetInstance().GetTime().ToString().ReplaceAll("\\s", "-").ReplaceAll(":", "-"); Logger logger = Logger.GetLogger(typeof(Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem).FullName); // set log file path if (props.Contains(HybridCorefProperties.LogProp)) { File logFile = new File(props.GetProperty(HybridCorefProperties.LogProp)); RedwoodConfiguration.Current().Handlers(RedwoodConfiguration.Handlers.File(logFile)).Apply(); Redwood.Log("Starting coref log"); } log.Info(props.ToString()); if (HybridCorefProperties.CheckMemory(props)) { CheckMemoryUsage(); } Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem cs = new Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem(props); /* * output setting */ // prepare conll output string goldOutput = null; string beforeCorefOutput = null; string afterCorefOutput = null; PrintWriter writerGold = null; PrintWriter writerBeforeCoref = null; PrintWriter writerAfterCoref = null; if (HybridCorefProperties.DoScore(props)) { string pathOutput = CorefProperties.ConllOutputPath(props); (new File(pathOutput)).Mkdir(); goldOutput = pathOutput + "output-" + timeStamp + ".gold.txt"; beforeCorefOutput = pathOutput + "output-" + timeStamp + ".predicted.txt"; afterCorefOutput = pathOutput + "output-" + timeStamp + ".coref.predicted.txt"; writerGold = new PrintWriter(new FileOutputStream(goldOutput)); writerBeforeCoref = new PrintWriter(new FileOutputStream(beforeCorefOutput)); writerAfterCoref = new PrintWriter(new FileOutputStream(afterCorefOutput)); } // run coref MulticoreWrapper <Pair <Document, Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem>, StringBuilder[]> wrapper = new MulticoreWrapper <Pair <Document, Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem>, StringBuilder[]>(nThreads, new _IThreadsafeProcessor_134 ()); // conll output and logs DateTime startTime = null; if (HybridCorefProperties.CheckTime(props)) { startTime = new DateTime(); System.Console.Error.Printf("END-TO-END COREF Start time: %s\n", startTime); } // run processes int docCnt = 0; while (true) { Document document = cs.docMaker.NextDoc(); if (document == null) { break; } wrapper.Put(Pair.MakePair(document, cs)); docCnt = LogOutput(wrapper, writerGold, writerBeforeCoref, writerAfterCoref, docCnt); } // Finished reading the input. Wait for jobs to finish wrapper.Join(); docCnt = LogOutput(wrapper, writerGold, writerBeforeCoref, writerAfterCoref, docCnt); IOUtils.CloseIgnoringExceptions(writerGold); IOUtils.CloseIgnoringExceptions(writerBeforeCoref); IOUtils.CloseIgnoringExceptions(writerAfterCoref); if (HybridCorefProperties.CheckTime(props)) { System.Console.Error.Printf("END-TO-END COREF Elapsed time: %.3f seconds\n", (((new DateTime()).GetTime() - startTime.GetTime()) / 1000F)); } // System.err.printf("CORENLP PROCESS TIME TOTAL: %.3f seconds\n", cs.mentionExtractor.corenlpProcessTime); if (HybridCorefProperties.CheckMemory(props)) { CheckMemoryUsage(); } // scoring if (HybridCorefProperties.DoScore(props)) { string summary = CorefScorer.GetEvalSummary(CorefProperties.GetScorerPath(props), goldOutput, beforeCorefOutput); CorefScorer.PrintScoreSummary(summary, logger, false); summary = CorefScorer.GetEvalSummary(CorefProperties.GetScorerPath(props), goldOutput, afterCorefOutput); CorefScorer.PrintScoreSummary(summary, logger, true); CorefScorer.PrintFinalConllScore(summary); } }
/// <summary> /// Initializes a new instance of the <see cref="ResourceManager"/> class. /// </summary> public ResourceManager(RedwoodConfiguration configuration) { this.configuration = configuration; }
/// <summary> /// Initializes a new instance of the <see cref="RedwoodMiddleware"/> class. /// </summary> public RedwoodMiddleware(OwinMiddleware next, RedwoodConfiguration configuration) : base(next) { this.configuration = configuration; }
/// <summary> /// Initializes a new instance of the <see cref="RedwoodApp"/> class. /// </summary> public RedwoodApp(OwinMiddleware next, string applicationPhysicalPath, RedwoodConfiguration configuration) : base(next) { this.applicationPhysicalPath = applicationPhysicalPath; this.configuration = configuration; }
/// <summary> /// Initializes a new instance of the <see cref="DefaultViewModelSerializer"/> class. /// </summary> public DefaultViewModelSerializer(RedwoodConfiguration configuration) { this.viewModelProtector = configuration.ServiceLocator.GetService<IViewModelProtector>(); this.JsonFormatting = configuration.Debug ? Formatting.Indented : Formatting.None; }