public DefaultControlBuilderFactory(RedwoodConfiguration configuration)
{
this.configuration = configuration;
ViewCompilerFactory = () => configuration.ServiceLocator.GetService<IViewCompiler>();
markupFileLoader = configuration.ServiceLocator.GetService<IMarkupFileLoader>();
}
public static void MinimalSetup()
{
Properties props = new Properties();
props.SetProperty("log.output", "stderr");
RedwoodConfiguration.Apply(props);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
public static void Main(string[] args)
{
RedwoodConfiguration.Standard().Apply();
// Disable SLF4J crap.
ArgumentParser.FillOptions(typeof(Edu.Stanford.Nlp.IE.KBPEnsembleExtractor), args);
object @object = IOUtils.ReadObjectFromURLOrClasspathOrFileSystem(StatisticalModel);
IKBPRelationExtractor statisticalExtractor;
if (@object is LinearClassifier)
{
//noinspection unchecked
statisticalExtractor = new KBPStatisticalExtractor((IClassifier <string, string>)@object);
}
else
{
if (@object is KBPStatisticalExtractor)
{
statisticalExtractor = (KBPStatisticalExtractor)@object;
}
else
{
throw new InvalidCastException(@object.GetType() + " cannot be cast into a " + typeof(KBPStatisticalExtractor));
}
}
logger.Info("Read statistical model from " + StatisticalModel);
IKBPRelationExtractor extractor = new Edu.Stanford.Nlp.IE.KBPEnsembleExtractor(new KBPTokensregexExtractor(TokensregexDir), new KBPSemgrexExtractor(SemgrexDir), statisticalExtractor);
IList <Pair <KBPRelationExtractor.KBPInput, string> > testExamples = IKBPRelationExtractor.ReadDataset(TestFile);
extractor.ComputeAccuracy(testExamples.Stream(), Predictions.Map(null));
}
public static void Demonstrate(string text, POSMode partOfSpeechMode, NERMode namedEntityRecognitionMode, bool disableLogging = true)
{
if (disableLogging)
{
RedwoodConfiguration.current().clear().apply();
}
//Input
//Console.WriteLine("Input: {0}\n\n\n", text);
////Tokenization
//Console.WriteLine("Tokenization:");
//Tokenisation.TokenizeText(text);
//Console.WriteLine("\n\n\n");
////POS
//Console.WriteLine("Part Of Speech:");
//PartOfSpeech.Tag(text, partOfSpeechMode);
//Console.WriteLine("\n\n\n");
////NER
//Console.WriteLine("Named Entity Recognition:");
//var ner = new NER(namedEntityRecognitionMode);
//Console.WriteLine(ner.classifyToString(text));
//Console.WriteLine("\n\n\n");
////Parser
//Console.WriteLine("Parsed Text:");
//Parser.ParseString(text);
//Console.WriteLine("\n\n\n");
//Find co-reference
CorefAnnotator.FindCoreferenceResolution(text);
}
/// <summary>
/// Setup extended tagger that includes POS, lemma and entity analysis
/// </summary>
private void SetupExtendedTagger()
{
PerformanceTester.StartMET("NLP");
// Get path to Stanford NLP models
var jarRoot = Path.Combine(Utility.GetResourcesFolder(), @"stanford-corenlp-3.9.2-models");
// Turn off logging
RedwoodConfiguration.current().clear().apply();
var props = new java.util.Properties();
props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner");
// Makes Named Entity Recognition work in the library
props.setProperty("ner.useSUTime", "0");
props.put("ner.applyFineGrained", "0");
props.put("ner.fine.regexner.mapping", jarRoot + @"\edu\stanford\nlp\models\kbp\english\");
// Set current directory
var curDir = Environment.CurrentDirectory;
var modelsDirectory = curDir + "\\" + jarRoot + @"\edu\stanford\nlp\models";
Directory.SetCurrentDirectory(jarRoot);
// Load Stanford NLP
Tagger = new StanfordCoreNLP(props);
PerformanceTester.StopMET("NLP");
}
// Logging to file facilities.
// The prefix is used to append stuff in front of the logging messages
/// <exception cref="System.IO.IOException"/>
public virtual void InitLog(File logFilePath)
{
RedwoodConfiguration.Empty().Handlers(RedwoodConfiguration.Handlers.Chain(RedwoodConfiguration.Handlers.ShowAllChannels(), RedwoodConfiguration.Handlers.stderr), RedwoodConfiguration.Handlers.File(logFilePath.ToString())).Apply();
// fh.setFormatter(new NewlineLogFormatter());
System.Console.Out.WriteLine("Starting Ssurgeon log, at " + logFilePath.GetAbsolutePath() + " date=" + DateFormat.GetDateInstance(DateFormat.Full).Format(new DateTime()));
log.Info("Starting Ssurgeon log, date=" + DateFormat.GetDateInstance(DateFormat.Full).Format(new DateTime()));
}
/// <summary>
/// Initializes a new instance of the <see cref="RedwoodPresenter"/> class.
/// </summary>
public RedwoodPresenter(RedwoodConfiguration configuration)
{
RedwoodViewBuilder = configuration.ServiceLocator.GetService<IRedwoodViewBuilder>();
ViewModelLoader = configuration.ServiceLocator.GetService<IViewModelLoader>();
ViewModelSerializer = configuration.ServiceLocator.GetService<IViewModelSerializer>();
OutputRenderer = configuration.ServiceLocator.GetService<IOutputRenderer>();
CsrfProtector = configuration.ServiceLocator.GetService<ICsrfProtector>();
}
public static void Execute(string option, string text, bool disableLogging = true)
{
if (disableLogging)
{
RedwoodConfiguration.current().clear().apply();
}
var jarRoot = @"../../../data/paket-files/stanford-corenlp-3.9.1-models/";
var props = new Properties();
props.setProperty("annotators", option);
props.setProperty("ner.useSUTime", "0");
// We should change current directory, so StanfordCoreNLP could find all the model files automatically
var curDir = Environment.CurrentDirectory;
Directory.SetCurrentDirectory(jarRoot);
var pipeline = new StanfordNLP.StanfordCoreNLP(props);
Directory.SetCurrentDirectory(curDir);
// Annotation
var annotation = new StanfordNLP.Annotation(text);
pipeline.annotate(annotation);
//get sentencesAnnotation to get sentences
var sentencesAnnotation = new CoreAnnotations.SentencesAnnotation().getClass();
//get tokensAnnotaion to get tokens in each sentence
var tokensAnnotaion = new CoreAnnotations.TokensAnnotation().getClass();
//get posAnnotation to get POS result of each token
var posAnnotation = new CoreAnnotations.PartOfSpeechAnnotation().getClass();
//get nerAnnotation to get NER result of each token
var nerAnnotaion = new CoreAnnotations.NamedEntityTagAnnotation().getClass();
var deparseAnnotation = new TreeCoreAnnotations.TreeAnnotation().getClass();
//deparseAnnotation = new TypedDependency().getClass();
var sentences = annotation.get(sentencesAnnotation) as ArrayList;
foreach (CoreMap sentence in sentences.toArray())
{
var tokens = (ArrayList)sentence.get(tokensAnnotaion);
Console.WriteLine("Token-POS-NER: ");
foreach (CoreLabel token in tokens)
{
Console.Write($"{token.value()}-{token.get(posAnnotation)}-{token.get(nerAnnotaion)} ");
}
Console.WriteLine("\n\n\n");
var parsedText = (Tree)sentence.get(deparseAnnotation);
if (parsedText != null)
{
Console.WriteLine("Parsed Text: ");
new TreePrint("penn,typedDependenciesCollapsed").printTree(parsedText);
}
}
}
/// <exception cref="System.IO.IOException"/>
public static void Main(string[] args)
{
RedwoodConfiguration.Standard().Apply();
// Disable SLF4J crap.
ArgumentParser.FillOptions(typeof(Edu.Stanford.Nlp.IE.KBPSemgrexExtractor), args);
Edu.Stanford.Nlp.IE.KBPSemgrexExtractor extractor = new Edu.Stanford.Nlp.IE.KBPSemgrexExtractor(Dir);
IList <Pair <KBPRelationExtractor.KBPInput, string> > testExamples = IKBPRelationExtractor.ReadDataset(TestFile);
extractor.ComputeAccuracy(testExamples.Stream(), Predictions.Map(null));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
public static void Main(string[] args)
{
RedwoodConfiguration.Standard().Apply();
// Disable SLF4J crap.
ArgumentParser.FillOptions(typeof(Edu.Stanford.Nlp.IE.KBPStatisticalExtractor), args);
// Fill command-line options
// Load the test (or dev) data
Redwood.Util.ForceTrack("Test data");
IList <Pair <KBPRelationExtractor.KBPInput, string> > testExamples = IKBPRelationExtractor.ReadDataset(TestFile);
log.Info("Read " + testExamples.Count + " examples");
Redwood.Util.EndTrack("Test data");
// If we can't find an existing model, train one
if (!IOUtils.ExistsInClasspathOrFileSystem(ModelFile))
{
Redwood.Util.ForceTrack("Training data");
IList <Pair <KBPRelationExtractor.KBPInput, string> > trainExamples = IKBPRelationExtractor.ReadDataset(TrainFile);
log.Info("Read " + trainExamples.Count + " examples");
log.Info(string.Empty + trainExamples.Stream().Map(null).Filter(null).Count() + " are " + KBPRelationExtractorConstants.NoRelation);
Redwood.Util.EndTrack("Training data");
// Featurize + create the dataset
Redwood.Util.ForceTrack("Creating dataset");
RVFDataset <string, string> dataset = new RVFDataset <string, string>();
AtomicInteger i = new AtomicInteger(0);
long beginTime = Runtime.CurrentTimeMillis();
trainExamples.Stream().Parallel().ForEach(null);
// This takes a while per example
trainExamples.Clear();
// Free up some memory
Redwood.Util.EndTrack("Creating dataset");
// Train the classifier
log.Info("Training classifier:");
IClassifier <string, string> classifier = TrainMultinomialClassifier(dataset, FeatureThreshold, Sigma);
dataset.Clear();
// Free up some memory
// Save the classifier
IOUtils.WriteObjectToFile(new Edu.Stanford.Nlp.IE.KBPStatisticalExtractor(classifier), ModelFile);
}
// Read either a newly-trained or pre-trained model
object model = IOUtils.ReadObjectFromURLOrClasspathOrFileSystem(ModelFile);
Edu.Stanford.Nlp.IE.KBPStatisticalExtractor classifier_1;
if (model is IClassifier)
{
//noinspection unchecked
classifier_1 = new Edu.Stanford.Nlp.IE.KBPStatisticalExtractor((IClassifier <string, string>)model);
}
else
{
classifier_1 = ((Edu.Stanford.Nlp.IE.KBPStatisticalExtractor)model);
}
// Evaluate the model
classifier_1.ComputeAccuracy(testExamples.Stream(), Predictions.Map(null));
}
/// <exception cref="System.IO.IOException"/>
public static void Main(string[] args)
{
RedwoodConfiguration.Standard().Apply();
Redwood.Util.StartTrack("main");
// Read the data
IStream <SimpleSentiment.SentimentDatum> data = IStream.Concat(IStream.Concat(IStream.Concat(Imdb("/users/gabor/tmp/aclImdb/train/pos", SentimentClass.Positive), Imdb("/users/gabor/tmp/aclImdb/train/neg", SentimentClass.Negative)), IStream.Concat
(Imdb("/users/gabor/tmp/aclImdb/test/pos", SentimentClass.Positive), Imdb("/users/gabor/tmp/aclImdb/test/neg", SentimentClass.Negative))), IStream.Concat(IStream.Concat(Stanford("/users/gabor/tmp/train.tsv"), Stanford("/users/gabor/tmp/test.tsv"
)), IStream.Concat(Twitter("/users/gabor/tmp/twitter.csv"), Unlabelled("/users/gabor/tmp/wikipedia"))));
// Train the model
OutputStream stream = IOUtils.GetFileOutputStream("/users/gabor/tmp/model.ser.gz");
SimpleSentiment classifier = SimpleSentiment.Train(data, Optional.Of(stream));
stream.Close();
log.Info(classifier.Classify("I think life is great"));
Redwood.Util.EndTrack("main");
}
/// <summary>
/// Configures the Redwood logger using a reasonable set of defaults,
/// which can be overruled by the supplied Properties file.
/// </summary>
/// <param name="props">The properties file to overrule or augment the default configuration</param>
public static void Apply(Properties props)
{
//--Tweak Properties
//(output to stderr)
if (props.GetProperty("log.output") == null)
{
props.SetProperty("log.output", "stderr");
}
//(capture system streams)
if (props.GetProperty("log.captureStderr") == null)
{
props.SetProperty("log.captureStderr", "true");
}
//(apply properties)
RedwoodConfiguration.Apply(props);
//--Strange Tweaks
//(adapt legacy logging systems)
JavaUtilLoggingAdaptor.Adapt();
}
//*************************************************/
// METHODS
//*************************************************/
#region Methods
/// <summary>
/// Setup tagger including POS
/// </summary>
private void SetupTagger()
{
PerformanceTester.StartMET("NLP");
// Get path to Stanford NLP models
var jarRoot = Path.Combine(Utility.GetResourcesFolder(), @"stanford-corenlp-3.9.2-models");
// Turn off logging
RedwoodConfiguration.current().clear().apply();
// Set properties
var props = new java.util.Properties();
props.setProperty("annotators", "tokenize, ssplit, pos");
// Set current directory
var curDir = Environment.CurrentDirectory;
var modelsDirectory = curDir + "\\" + jarRoot + @"\edu\stanford\nlp\models";
Directory.SetCurrentDirectory(jarRoot);
// Load Stanford NLP
Tagger = new StanfordCoreNLP(props);
PerformanceTester.StopMET("NLP");
}
/// <exception cref="System.Exception"/>
public static void RunCoref(Properties props)
{
/*
* property, environment setting
*/
Redwood.HideChannelsEverywhere("debug-cluster", "debug-mention", "debug-preprocessor", "debug-docreader", "debug-mergethres", "debug-featureselection", "debug-md");
int nThreads = HybridCorefProperties.GetThreadCounts(props);
string timeStamp = Calendar.GetInstance().GetTime().ToString().ReplaceAll("\\s", "-").ReplaceAll(":", "-");
Logger logger = Logger.GetLogger(typeof(Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem).FullName);
// set log file path
if (props.Contains(HybridCorefProperties.LogProp))
{
File logFile = new File(props.GetProperty(HybridCorefProperties.LogProp));
RedwoodConfiguration.Current().Handlers(RedwoodConfiguration.Handlers.File(logFile)).Apply();
Redwood.Log("Starting coref log");
}
log.Info(props.ToString());
if (HybridCorefProperties.CheckMemory(props))
{
CheckMemoryUsage();
}
Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem cs = new Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem(props);
/*
* output setting
*/
// prepare conll output
string goldOutput = null;
string beforeCorefOutput = null;
string afterCorefOutput = null;
PrintWriter writerGold = null;
PrintWriter writerBeforeCoref = null;
PrintWriter writerAfterCoref = null;
if (HybridCorefProperties.DoScore(props))
{
string pathOutput = CorefProperties.ConllOutputPath(props);
(new File(pathOutput)).Mkdir();
goldOutput = pathOutput + "output-" + timeStamp + ".gold.txt";
beforeCorefOutput = pathOutput + "output-" + timeStamp + ".predicted.txt";
afterCorefOutput = pathOutput + "output-" + timeStamp + ".coref.predicted.txt";
writerGold = new PrintWriter(new FileOutputStream(goldOutput));
writerBeforeCoref = new PrintWriter(new FileOutputStream(beforeCorefOutput));
writerAfterCoref = new PrintWriter(new FileOutputStream(afterCorefOutput));
}
// run coref
MulticoreWrapper <Pair <Document, Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem>, StringBuilder[]> wrapper = new MulticoreWrapper <Pair <Document, Edu.Stanford.Nlp.Coref.Hybrid.HybridCorefSystem>, StringBuilder[]>(nThreads, new _IThreadsafeProcessor_134
());
// conll output and logs
DateTime startTime = null;
if (HybridCorefProperties.CheckTime(props))
{
startTime = new DateTime();
System.Console.Error.Printf("END-TO-END COREF Start time: %s\n", startTime);
}
// run processes
int docCnt = 0;
while (true)
{
Document document = cs.docMaker.NextDoc();
if (document == null)
{
break;
}
wrapper.Put(Pair.MakePair(document, cs));
docCnt = LogOutput(wrapper, writerGold, writerBeforeCoref, writerAfterCoref, docCnt);
}
// Finished reading the input. Wait for jobs to finish
wrapper.Join();
docCnt = LogOutput(wrapper, writerGold, writerBeforeCoref, writerAfterCoref, docCnt);
IOUtils.CloseIgnoringExceptions(writerGold);
IOUtils.CloseIgnoringExceptions(writerBeforeCoref);
IOUtils.CloseIgnoringExceptions(writerAfterCoref);
if (HybridCorefProperties.CheckTime(props))
{
System.Console.Error.Printf("END-TO-END COREF Elapsed time: %.3f seconds\n", (((new DateTime()).GetTime() - startTime.GetTime()) / 1000F));
}
// System.err.printf("CORENLP PROCESS TIME TOTAL: %.3f seconds\n", cs.mentionExtractor.corenlpProcessTime);
if (HybridCorefProperties.CheckMemory(props))
{
CheckMemoryUsage();
}
// scoring
if (HybridCorefProperties.DoScore(props))
{
string summary = CorefScorer.GetEvalSummary(CorefProperties.GetScorerPath(props), goldOutput, beforeCorefOutput);
CorefScorer.PrintScoreSummary(summary, logger, false);
summary = CorefScorer.GetEvalSummary(CorefProperties.GetScorerPath(props), goldOutput, afterCorefOutput);
CorefScorer.PrintScoreSummary(summary, logger, true);
CorefScorer.PrintFinalConllScore(summary);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ResourceManager"/> class.
/// </summary>
public ResourceManager(RedwoodConfiguration configuration)
{
this.configuration = configuration;
}
/// <summary>
/// Initializes a new instance of the <see cref="RedwoodMiddleware"/> class.
/// </summary>
public RedwoodMiddleware(OwinMiddleware next, RedwoodConfiguration configuration)
: base(next)
{
this.configuration = configuration;
}
/// <summary>
/// Initializes a new instance of the <see cref="RedwoodApp"/> class.
/// </summary>
public RedwoodApp(OwinMiddleware next, string applicationPhysicalPath, RedwoodConfiguration configuration) : base(next)
{
this.applicationPhysicalPath = applicationPhysicalPath;
this.configuration = configuration;
}
/// <summary>
/// Initializes a new instance of the <see cref="DefaultViewModelSerializer"/> class.
/// </summary>
public DefaultViewModelSerializer(RedwoodConfiguration configuration)
{
this.viewModelProtector = configuration.ServiceLocator.GetService<IViewModelProtector>();
this.JsonFormatting = configuration.Debug ? Formatting.Indented : Formatting.None;
}
