private static void AnnotateFactories(IAnnotator annotator) { annotator.Annotate <LogManager>(type => { type.Annotate(_ => LogManager.CreateNullLogger() == NotNull <ILogger>()); type.Annotate(_ => LogManager.GetCurrentClassLogger() == NotNull <ILogger>()); type.Annotate(_ => LogManager.GetCurrentClassLogger(CanBeNull <Type>()) == NotNull <ILogger>()); type.Annotate(_ => LogManager.GetLogger(NotNull <string>()) == NotNull <ILogger>()); type.Annotate(_ => LogManager.GetLogger(NotNull <string>(), CanBeNull <Type>()) == NotNull <ILogger>()); type.Annotate(_ => LogManager.DefaultCultureInfo == NotNull <LogManager.GetCultureInfo>()); type.Annotate(_ => LogManager.Configuration == CanBeNull <LoggingConfiguration>()); type.Annotate(_ => LogManager.DisableLogging() == NotNull <IDisposable>()); type.Annotate(_ => LogManager.AddHiddenAssembly(NotNull <Assembly>())); type.Annotate(_ => LogManager.Flush(NotNull <AsyncContinuation>())); type.Annotate(_ => LogManager.Flush(NotNull <AsyncContinuation>(), Some <TimeSpan>())); type.Annotate(_ => LogManager.Flush(NotNull <AsyncContinuation>(), Some <int>())); }); annotator.Annotate <LogFactory>(type => { type.Annotate(x => x.CreateNullLogger() == NotNull <ILogger>()); type.Annotate(x => x.GetCurrentClassLogger() == NotNull <ILogger>()); type.Annotate(x => x.GetCurrentClassLogger(CanBeNull <Type>()) == NotNull <ILogger>()); type.Annotate(x => x.GetLogger(NotNull <string>()) == NotNull <ILogger>()); type.Annotate(x => x.GetLogger(NotNull <string>(), CanBeNull <Type>()) == NotNull <ILogger>()); type.Annotate(x => x.DefaultCultureInfo == CanBeNull <CultureInfo>()); type.Annotate(x => x.Configuration == CanBeNull <LoggingConfiguration>()); type.Annotate(x => x.DisableLogging() == NotNull <IDisposable>()); type.Annotate(x => x.SuspendLogging() == NotNull <IDisposable>()); type.Annotate(x => x.Flush(NotNull <AsyncContinuation>())); type.Annotate(x => x.Flush(NotNull <AsyncContinuation>(), Some <TimeSpan>())); type.Annotate(x => x.Flush(NotNull <AsyncContinuation>(), Some <int>())); }); }
public virtual void SetUp() { lock (typeof(CleanXmlAnnotatorTest)) { if (ptbInvertible == null) { ptbInvertible = new TokenizerAnnotator(false, "en", "invertible,ptb3Escaping=true"); } if (ptbNotInvertible == null) { ptbNotInvertible = new TokenizerAnnotator(false, "en", "invertible=false,ptb3Escaping=true"); } if (cleanXmlAllTags == null) { cleanXmlAllTags = new CleanXmlAnnotator(".*", string.Empty, string.Empty, false); } if (cleanXmlSomeTags == null) { cleanXmlSomeTags = new CleanXmlAnnotator("p", string.Empty, string.Empty, false); } if (cleanXmlEndSentences == null) { cleanXmlEndSentences = new CleanXmlAnnotator(".*", "p", string.Empty, false); } if (cleanXmlWithFlaws == null) { cleanXmlWithFlaws = new CleanXmlAnnotator(".*", string.Empty, string.Empty, true); } if (wtsSplitter == null) { wtsSplitter = new WordsToSentencesAnnotator(false); } } }
private IAnnotator GetParser() { if (parserProcessor == null) { IAnnotator parser = StanfordCoreNLP.GetExistingAnnotator("parse"); if (parser == null) { Properties emptyProperties = new Properties(); parser = new ParserAnnotator("coref.parse.md", emptyProperties); } if (parser == null) { // TODO: these assertions rule out the possibility of alternately named parse/pos annotators throw new AssertionError("Failed to get parser - this should not be possible"); } if (parser.Requires().Contains(typeof(CoreAnnotations.PartOfSpeechAnnotation))) { IAnnotator tagger = StanfordCoreNLP.GetExistingAnnotator("pos"); if (tagger == null) { throw new AssertionError("Parser required tagger, but failed to find the pos annotator"); } IList <IAnnotator> annotators = Generics.NewArrayList(); annotators.Add(tagger); annotators.Add(parser); parserProcessor = new AnnotationPipeline(annotators); } else { parserProcessor = parser; } } return(parserProcessor); }
private static void RunGeneration(IAnnotator annotator, NugetSpec nuspec, Args parsedArgs) { var version = parsedArgs.Version ?? new Version("1.0.0.0"); var dir = parsedArgs.Directory ?? new DirectoryInfo(Environment.CurrentDirectory); var fixedSpec = SpecWithVersion(nuspec, version); annotator.CreateNugetPackage(fixedSpec, dir); Console.WriteLine($"Generated version {version} in {dir.FullName}"); }
public virtual IAnnotator GetParser() { if (parserProcessor == null) { parserProcessor = StanfordCoreNLP.GetExistingAnnotator("parse"); System.Diagnostics.Debug.Assert((parserProcessor != null)); } return(parserProcessor); }
private static void AnnotateLogger(IAnnotator annotator) { annotator.Annotate <ILogger>(type => { AnnotateFatal(type); AnnotateError(type); AnnotateWarn(type); AnnotateInfo(type); AnnotateDebug(type); AnnotateTrace(type); }); }
public virtual Annotation Process(string sentence, string dateString, IAnnotator timeAnnotator) { log.Info("Processing text \"" + sentence + "\" with dateString = " + dateString); Annotation anno = new Annotation(sentence); if (dateString != null && !dateString.IsEmpty()) { anno.Set(typeof(CoreAnnotations.DocDateAnnotation), dateString); } pipeline.Annotate(anno); timeAnnotator.Annotate(anno); return(anno); }
private IAnnotator GetParser() { if (parserProcessor == null) { parserProcessor = StanfordCoreNLP.GetExistingAnnotator("parse"); if (parserProcessor == null) { Properties emptyProperties = new Properties(); parserProcessor = new ParserAnnotator("coref.parse.md", emptyProperties); } System.Diagnostics.Debug.Assert((parserProcessor != null)); } return(parserProcessor); }
/// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { Edu.Stanford.Nlp.Time.Suservlet.SUTimePipeline pipeline = new Edu.Stanford.Nlp.Time.Suservlet.SUTimePipeline(); IAnnotator timeAnnotator = pipeline.GetTimeAnnotator("sutime", new Properties()); BufferedReader @is = new BufferedReader(new InputStreamReader(Runtime.@in)); System.Console.Out.Write("> "); for (string line; (line = @is.ReadLine()) != null;) { Annotation ann = pipeline.Process(line, null, timeAnnotator); System.Console.Out.WriteLine(ann.Get(typeof(TimeAnnotations.TimexAnnotations))); System.Console.Out.Write("> "); } }
public static Annotation Annotate(string text, IAnnotator tokenizer, IAnnotator xmlRemover, IAnnotator splitter) { Annotation annotation = new Annotation(text); tokenizer.Annotate(annotation); if (xmlRemover != null) { xmlRemover.Annotate(annotation); } if (splitter != null) { splitter.Annotate(annotation); } return(annotation); }
private static int RunApp(string exeName, string[] args, IAnnotator annotator, NugetSpec nuspec) { try { var parsedArgs = ParseArgs(args); if (parsedArgs.ShowHelp) { ShowHelp(exeName, parsedArgs); return(parsedArgs.ParseError == null ? SuccessExitCode : InvalidArgumentsExitCode); } RunGeneration(annotator, nuspec, parsedArgs); return(SuccessExitCode); } catch (Exception e) { Console.WriteLine("Error: " + e.Message); return(ExceptionExitCode); } }
private static int RunApp(string exeName, string[] args, IAnnotator annotator, NugetSpec nuspec) { try { var parsedArgs = ParseArgs(args); if (parsedArgs.ShowHelp) { ShowHelp(exeName, parsedArgs); return parsedArgs.ParseError == null ? SuccessExitCode : InvalidArgumentsExitCode; } RunGeneration(annotator, nuspec, parsedArgs); return SuccessExitCode; } catch (Exception e) { Console.WriteLine("Error: " + e.Message); return ExceptionExitCode; } }
public GenericDataSetReader(StanfordCoreNLP processor, bool preProcessSentences, bool calculateHeadSpan, bool forceGenerationOfIndexSpans) { // import edu.stanford.nlp.util.logging.Redwood; // private static Redwood.RedwoodChannels log = Redwood.channels(GenericDataSetReader.class); this.logger = Logger.GetLogger(typeof(Edu.Stanford.Nlp.IE.Machinereading.GenericDataSetReader).FullName); this.logger.SetLevel(Level.Severe); if (processor != null) { SetProcessor(processor); } parserProcessor = null; /* old parser options * parser.setOptionFlags(new String[] { * "-outputFormat", "penn,typedDependenciesCollapsed", * "-maxLength", "100", * "-retainTmpSubcategories" * }); */ this.preProcessSentences = preProcessSentences; this.calculateHeadSpan = calculateHeadSpan; this.forceGenerationOfIndexSpans = forceGenerationOfIndexSpans; }
public virtual void TestSignature() { Properties props = new Properties(); props.SetProperty("sample.prop", "v1"); AnnotatorPool pool = new AnnotatorPool(); pool.Register("sample", props, new AnnotatorPoolTest.SampleAnnotatorFactory(props)); IAnnotator a1 = pool.Get("sample"); System.Console.Out.WriteLine("First annotator: " + a1); pool.Register("sample", props, new AnnotatorPoolTest.SampleAnnotatorFactory(props)); IAnnotator a2 = pool.Get("sample"); System.Console.Out.WriteLine("Second annotator: " + a2); NUnit.Framework.Assert.IsTrue(a1 == a2); props.SetProperty("sample.prop", "v2"); pool.Register("sample", props, new AnnotatorPoolTest.SampleAnnotatorFactory(props)); IAnnotator a3 = pool.Get("sample"); System.Console.Out.WriteLine("Third annotator: " + a3); NUnit.Framework.Assert.IsTrue(a1 != a3); }
/// <exception cref="System.IO.IOException"/> private void AddResults(IHttpServletRequest request, IHttpServletResponse response) { // if we can't handle UTF-8, need to do something like this... //String originalQuery = request.getParameter("q"); //String query = WebappUtil.convertString(originalQuery); string query = request.GetParameter("q"); string dateString = request.GetParameter("d"); // TODO: this always returns true... bool dateError = !pipeline.IsDateOkay(dateString); bool includeOffsets = ParseBoolean(request.GetParameter("includeOffsets")); PrintWriter @out = response.GetWriter(); if (dateError) { @out.Println("<br><br>Warning: unparseable date " + StringEscapeUtils.EscapeHtml4(dateString)); } if (!StringUtils.IsNullOrEmpty(query)) { Properties props = GetTimeAnnotatorProperties(request); string annotatorType = request.GetParameter("annotator"); if (annotatorType == null) { annotatorType = "sutime"; } IAnnotator timeAnnotator = pipeline.GetTimeAnnotator(annotatorType, props); if (timeAnnotator != null) { Annotation anno = pipeline.Process(query, dateString, timeAnnotator); @out.Println("<h3>Annotated Text</h3> <em>(tagged using " + annotatorType + "</em>)"); DisplayAnnotation(@out, query, anno, includeOffsets); } else { @out.Println("<br><br>Error creating annotator for " + StringEscapeUtils.EscapeHtml4(annotatorType)); } } }
private static void CheckResult(WordToSentenceProcessor <CoreLabel> wts, IAnnotator tokenizer, string testSentence, params string[] gold) { Annotation annotation = new Annotation(testSentence); ptbNL.Annotate(annotation); IList <CoreLabel> tokens = annotation.Get(typeof(CoreAnnotations.TokensAnnotation)); IList <IList <CoreLabel> > sentences = wts.Process(tokens); NUnit.Framework.Assert.AreEqual("Output number of sentences didn't match:\n" + Arrays.ToString(gold) + " vs. \n" + sentences + '\n', gold.Length, sentences.Count); Annotation[] goldAnnotations = new Annotation[gold.Length]; for (int i = 0; i < gold.Length; ++i) { goldAnnotations[i] = new Annotation(gold[i]); tokenizer.Annotate(goldAnnotations[i]); IList <CoreLabel> goldTokens = goldAnnotations[i].Get(typeof(CoreAnnotations.TokensAnnotation)); IList <CoreLabel> testTokens = sentences[i]; int goldTokensSize = goldTokens.Count; NUnit.Framework.Assert.AreEqual("Sentence lengths didn't match:\n" + goldTokens + " vs. \n" + testTokens + '\n', goldTokensSize, testTokens.Count); for (int j = 0; j < goldTokensSize; ++j) { NUnit.Framework.Assert.AreEqual(goldTokens[j].Word(), testTokens[j].Word()); } } }
public TokenizerAnnotator(bool verbose, Properties props, string options) { if (props == null) { props = new Properties(); } // check if segmenting must be done if (props.GetProperty("tokenize.language") != null && LanguageInfo.IsSegmenterLanguage(props.GetProperty("tokenize.language"))) { useSegmenter = true; if (LanguageInfo.GetLanguageFromString(props.GetProperty("tokenize.language")) == LanguageInfo.HumanLanguage.Arabic) { segmenterAnnotator = new ArabicSegmenterAnnotator("segment", props); } else { if (LanguageInfo.GetLanguageFromString(props.GetProperty("tokenize.language")) == LanguageInfo.HumanLanguage.Chinese) { segmenterAnnotator = new ChineseSegmenterAnnotator("segment", props); } else { segmenterAnnotator = null; throw new Exception("No segmenter implemented for: " + LanguageInfo.GetLanguageFromString(props.GetProperty("tokenize.language"))); } } } else { useSegmenter = false; segmenterAnnotator = null; } Verbose = PropertiesUtils.GetBool(props, "tokenize.verbose", verbose); TokenizerAnnotator.TokenizerType type = TokenizerAnnotator.TokenizerType.GetTokenizerType(props); factory = InitFactory(type, props, options); }
private static AnnotationFile GetFirstFile(IAnnotator annotator) { return CoreHelper.GetAnnotations(annotator).GenerateFiles().First(); }
private static void AnnotateLogger(IAnnotator annotator) { annotator.Annotate<ILogger>(type => { AnnotateFatal(type); AnnotateError(type); AnnotateWarn(type); AnnotateInfo(type); AnnotateDebug(type); AnnotateTrace(type); }); }
// It can't be a singletonList() since it isn't copied but is mutated. public virtual void AddAnnotator(IAnnotator annotator) { annotators.Add(annotator); accumulatedTime.Add(new MutableLong()); }
private static void AnnotateFactories(IAnnotator annotator) { annotator.Annotate<LogManager>(type => { type.Annotate(_ => LogManager.CreateNullLogger() == NotNull<ILogger>()); type.Annotate(_ => LogManager.GetCurrentClassLogger() == NotNull<ILogger>()); type.Annotate(_ => LogManager.GetCurrentClassLogger(CanBeNull<Type>()) == NotNull<ILogger>()); type.Annotate(_ => LogManager.GetLogger(NotNull<string>()) == NotNull<ILogger>()); type.Annotate(_ => LogManager.GetLogger(NotNull<string>(), CanBeNull<Type>()) == NotNull<ILogger>()); type.Annotate(_ => LogManager.DefaultCultureInfo == NotNull<LogManager.GetCultureInfo>()); type.Annotate(_ => LogManager.Configuration == CanBeNull<LoggingConfiguration>()); type.Annotate(_ => LogManager.DisableLogging() == NotNull<IDisposable>()); type.Annotate(_ => LogManager.AddHiddenAssembly(NotNull<Assembly>())); type.Annotate(_ => LogManager.Flush(NotNull<AsyncContinuation>())); type.Annotate(_ => LogManager.Flush(NotNull<AsyncContinuation>(), Some<TimeSpan>())); type.Annotate(_ => LogManager.Flush(NotNull<AsyncContinuation>(), Some<int>())); }); annotator.Annotate<LogFactory>(type => { type.Annotate(x => x.CreateNullLogger() == NotNull<ILogger>()); type.Annotate(x => x.GetCurrentClassLogger() == NotNull<ILogger>()); type.Annotate(x => x.GetCurrentClassLogger(CanBeNull<Type>()) == NotNull<ILogger>()); type.Annotate(x => x.GetLogger(NotNull<string>()) == NotNull<ILogger>()); type.Annotate(x => x.GetLogger(NotNull<string>(), CanBeNull<Type>()) == NotNull<ILogger>()); type.Annotate(x => x.DefaultCultureInfo == CanBeNull<CultureInfo>()); type.Annotate(x => x.Configuration == CanBeNull<LoggingConfiguration>()); type.Annotate(x => x.DisableLogging() == NotNull<IDisposable>()); type.Annotate(x => x.SuspendLogging() == NotNull<IDisposable>()); type.Annotate(x => x.Flush(NotNull<AsyncContinuation>())); type.Annotate(x => x.Flush(NotNull<AsyncContinuation>(), Some<TimeSpan>())); type.Annotate(x => x.Flush(NotNull<AsyncContinuation>(), Some<int>())); }); }