/// <summary>
/// Load a batch of models from disk, while running the function "featurizer" on each of the models before adding it
/// to the batch.
/// </summary>
/// <remarks>
/// Load a batch of models from disk, while running the function "featurizer" on each of the models before adding it
/// to the batch. This gives the loader a chance to experiment with new featurization techniques.
/// </remarks>
/// <param name="inputStream">the input stream to load from</param>
/// <param name="featurizer">
/// a function that gets run on every GraphicalModel, and has a chance to edit them (eg by adding
/// or changing features)
/// </param>
/// <exception cref="System.IO.IOException"/>
private void ReadFrom(InputStream inputStream, IConsumer <GraphicalModel> featurizer)
{
GraphicalModel read;
while ((read = GraphicalModel.ReadFromStream(inputStream)) != null)
{
featurizer.Accept(read);
Add(read);
}
}
public virtual void Object(int indent, IConsumer <JSONOutputter.IWriter> callback)
{
writer.Write("{");
Pointer <bool> firstCall = new Pointer <bool>(true);
callback.Accept(null);
// First call overhead
// Write the key
// Write the value
Newline();
Indent(indent);
writer.Write("}");
}
/// <summary><inheritDoc/></summary>
public virtual VALUE Get <Value>(Type key)
{
for (int i = 0; i < size; i++)
{
if (key == keys[i])
{
if (listener != null)
{
listener.Accept(key);
}
// For tracking which entities were returned by the CoreMap
return((VALUE)values[i]);
}
}
return(null);
}
/// <summary>
/// Annotate a collection of input annotations IN PARALLEL, making use of
/// threads given in numThreads
/// </summary>
/// <param name="annotations">The input annotations to process</param>
/// <param name="numThreads">The number of threads to run on</param>
/// <param name="callback">
/// A function to be called when an annotation finishes.
/// The return value of the callback is ignored.
/// </param>
public virtual void Annotate(IEnumerable <Annotation> annotations, int numThreads, IConsumer <Annotation> callback)
{
// case: single thread (no point in spawning threads)
if (numThreads == 1)
{
foreach (Annotation ann in annotations)
{
Annotate(ann);
callback.Accept(ann);
}
}
// Java's equivalent to ".map{ lambda(annotation) => annotate(annotation) }
IEnumerable <IRunnable> threads = null;
//(logging)
//(annotate)
//(callback)
//(logging again)
// Thread
Redwood.Util.ThreadAndRun(this.GetType().GetSimpleName(), threads, numThreads);
}
/// <exception cref="System.IO.IOException"/>
private static void OutputByWriter(IConsumer <StringWriter> printer, PrintWriter @out)
{
StringWriter output = new StringWriter();
printer.Accept(output);
output.Flush();
string escapedXml = StringEscapeUtils.EscapeHtml4(output.ToString());
string[] lines = escapedXml.Split("\n");
@out.Print("<div><pre>");
foreach (string line in lines)
{
int numSpaces = 0;
while (numSpaces < line.Length && line[numSpaces] == ' ')
{
@out.Print(" ");
++numSpaces;
}
@out.Print(Sharpen.Runtime.Substring(line, numSpaces));
@out.Print("\n");
}
@out.Print("</pre></div>");
}
public static DelegatedConsumer <TIn> Map <TIn, TOut>(this IConsumer <TOut> consumer, Func <TIn, TOut> mapFunc) => new DelegatedConsumer <TIn>(input => consumer.Accept(mapFunc(input)), consumer.Priority);
/// <summary>Returns a collection of keyphrases, defined as relevant noun phrases and verbs in the sentence.</summary>
/// <remarks>
/// Returns a collection of keyphrases, defined as relevant noun phrases and verbs in the sentence.
/// Each token of the sentence is consumed at most once.
/// What counts as a keyphrase is in general quite subjective -- this method is just one possible interpretation
/// (in particular, Gabor's interpretation).
/// Please don't rely on this method to produce exactly your interpretation of what a keyphrase is.
/// </remarks>
/// <returns>A list of spans in the sentence, where each one corresponds to a keyphrase.</returns>
/// <author>Gabor Angeli</author>
public virtual IList <Span> KeyphraseSpans()
{
//
// Implementation note:
// This is implemented roughly as a finite state automata, looking for sequences of nouns, adjective+nouns, verbs,
// and a few special cases of prepositions.
// The code defines a transition matrix, based on POS tags and lemmas, where at each word we update the valid next
// tags/words based on the current tag/word we see.
// Note: The tag 'B' is used for the verb "to be", rather than the usual 'V' tag.
// Note: The tag 'X' is used for proper nouns, rather than the usual 'N' tag.
// Note: The tag 'Z' is used for possessives, rather than the usual 'P' tag.
//
// The output
IList <Span> spans = new List <Span>();
// The marker for where the last span began
int spanBegin = -1;
// The expected next states
ICollection <char> expectNextTag = new HashSet <char>();
ICollection <string> expectNextLemma = new HashSet <string>();
// A special marker for when we look-ahead and only accept the last word if
// the word after it is ok (e.g., PP attachments).
bool inLookahead = false;
// The transition matrix, over POS tags.
IConsumer <char> updateExpectation = null;
// 'water freezing' is fishy, but 'freezing water' is ok.
// Run the FSA:
for (int i = 0; i < sentence.Length(); ++i)
{
// Get some variables
string tag = sentence.PosTag(i);
char coarseTag = char.ToUpperCase(tag[0]);
string lemma = sentence.Lemma(i).ToLower();
// Tweak the tag
if (coarseTag == 'V' && lemma.Equals("be"))
{
coarseTag = 'B';
}
else
{
if (tag.StartsWith("NNP"))
{
coarseTag = 'X';
}
else
{
if (tag.StartsWith("POS"))
{
coarseTag = 'Z';
}
}
}
// (don't collapse 'ing' nouns)
if (coarseTag == 'N' && sentence.Word(i).EndsWith("ing"))
{
coarseTag = 'G';
}
// Transition
if (spanBegin < 0 && !sentence.Word(i).Equals("%") && (coarseTag == 'N' || coarseTag == 'V' || coarseTag == 'J' || coarseTag == 'X' || coarseTag == 'G'))
{
// Case: we were not in a span, but we hit a valid start tag.
spanBegin = i;
updateExpectation.Accept(coarseTag);
inLookahead = false;
}
else
{
if (spanBegin >= 0)
{
// Case: we're in a span
if (expectNextTag.Contains(coarseTag))
{
// Case: we hit a valid expected POS tag.
// update the transition matrix.
updateExpectation.Accept(coarseTag);
inLookahead = false;
}
else
{
if (expectNextLemma.Contains(lemma))
{
switch (lemma)
{
case "of":
{
// Case: we hit a valid word. Do something special.
// These prepositions are valid to subsume into a noun phrase.
// Update the transition matrix, and mark this as conditionally ok.
updateExpectation.Accept('I');
inLookahead = true;
break;
}
case "'s":
{
// Possessives often denote a longer compound phrase
updateExpectation.Accept('Z');
inLookahead = true;
break;
}
default:
{
throw new InvalidOperationException("Unknown special lemma: " + lemma);
}
}
}
else
{
// Case: We have transitioned to an 'invalid' state, and therefore the span should end.
if (inLookahead)
{
// If we were in a lookahead token, ignore the last token (as per the lookahead definition)
spans.Add(Span.FromValues(spanBegin, i - 1));
}
else
{
// Otherwise, add the span
spans.Add(Span.FromValues(spanBegin, i));
}
// We may also have started a new span.
// Check to see if we have started a new span.
if (coarseTag == 'N' || coarseTag == 'V' || coarseTag == 'J' || coarseTag == 'X' || coarseTag == 'G')
{
spanBegin = i;
updateExpectation.Accept(coarseTag);
}
else
{
spanBegin = -1;
}
inLookahead = false;
}
}
}
}
}
// Add a potential last span
if (spanBegin >= 0)
{
spans.Add(Span.FromValues(spanBegin, inLookahead ? sentence.Length() - 1 : sentence.Length()));
}
// Return
return(spans);
}
public override void ChannelReadComplete(IChannelHandlerContext context)
{
_channelReadCompleteConsumer.Accept(context);
}
