public virtual CoreMapAggregator GetDefaultTokensAggregator()
{
if (defaultTokensAggregator == null && (defaultTokensAggregators != null || aggregateToTokens))
{
CoreLabelTokenFactory tokenFactory = (aggregateToTokens) ? new CoreLabelTokenFactory() : null;
IDictionary <Type, CoreMapAttributeAggregator> aggregators = defaultTokensAggregators;
if (aggregators == null)
{
aggregators = CoreMapAttributeAggregator.DefaultNumericTokensAggregators;
}
defaultTokensAggregator = CoreMapAggregator.GetAggregator(aggregators, null, tokenFactory);
}
return(defaultTokensAggregator);
}
public CoreMapAggregator(IDictionary <Type, CoreMapAttributeAggregator> aggregators, Type mergedKey, CoreLabelTokenFactory tokenFactory)
{
this.aggregators = aggregators;
this.mergedKey = mergedKey;
this.tokenFactory = tokenFactory;
}
public static Edu.Stanford.Nlp.Pipeline.CoreMapAggregator GetAggregator(IDictionary <Type, CoreMapAttributeAggregator> aggregators, Type key, CoreLabelTokenFactory tokenFactory)
{
return(new Edu.Stanford.Nlp.Pipeline.CoreMapAggregator(aggregators, key, tokenFactory));
}
// TODO replace with GrammaticalStructure#readCoNLLGrammaticalStructureCollection
public static void LoadConllFile(string inFile, IList <ICoreMap> sents, IList <DependencyTree> trees, bool unlabeled, bool cPOS)
{
CoreLabelTokenFactory tf = new CoreLabelTokenFactory(false);
try
{
using (BufferedReader reader = IOUtils.ReaderFromString(inFile))
{
IList <CoreLabel> sentenceTokens = new List <CoreLabel>();
DependencyTree tree = new DependencyTree();
foreach (string line in IOUtils.GetLineIterable(reader, false))
{
string[] splits = line.Split("\t");
if (splits.Length < 10)
{
if (sentenceTokens.Count > 0)
{
trees.Add(tree);
ICoreMap sentence = new CoreLabel();
sentence.Set(typeof(CoreAnnotations.TokensAnnotation), sentenceTokens);
sents.Add(sentence);
tree = new DependencyTree();
sentenceTokens = new List <CoreLabel>();
}
}
else
{
string word = splits[1];
string pos = cPOS ? splits[3] : splits[4];
string depType = splits[7];
int head = -1;
try
{
head = System.Convert.ToInt32(splits[6]);
}
catch (NumberFormatException)
{
continue;
}
CoreLabel token = tf.MakeToken(word, 0, 0);
token.SetTag(pos);
token.Set(typeof(CoreAnnotations.CoNLLDepParentIndexAnnotation), head);
token.Set(typeof(CoreAnnotations.CoNLLDepTypeAnnotation), depType);
sentenceTokens.Add(token);
if (!unlabeled)
{
tree.Add(head, depType);
}
else
{
tree.Add(head, Config.Unknown);
}
}
}
}
}
catch (IOException e)
{
throw new RuntimeIOException(e);
}
}
