public static void AddSegmentPairs(this IWordAlignmentModel model, ParallelTextCorpus corpus,
Func <string, string> sourcePreprocessor = null, Func <string, string> targetPreprocessor = null,
int maxCount = int.MaxValue)
{
foreach (ParallelTextSegment segment in corpus.Segments.Where(s => !s.IsEmpty).Take(maxCount))
{
model.AddSegmentPair(segment, sourcePreprocessor, targetPreprocessor);
}
}
public void AddSegmentPair(IReadOnlyList <string> sourceSegment, IReadOnlyList <string> targetSegment,
WordAlignmentMatrix hintMatrix = null)
{
_directWordAlignmentModel.AddSegmentPair(sourceSegment, targetSegment, hintMatrix);
WordAlignmentMatrix invertedHintMatrix = hintMatrix?.Clone();
invertedHintMatrix?.Transpose();
_inverseWordAlignmentModel.AddSegmentPair(targetSegment, sourceSegment, invertedHintMatrix);
}
public static void AddSegmentPair(this IWordAlignmentModel model, ParallelTextSegment segment,
Func <string, string> sourcePreprocessor = null, Func <string, string> targetPreprocessor = null)
{
if (segment.IsEmpty)
{
return;
}
IReadOnlyList <string> sourceSegment = segment.SourceSegment.Preprocess(sourcePreprocessor);
IReadOnlyList <string> targetSegment = segment.TargetSegment.Preprocess(targetPreprocessor);
model.AddSegmentPair(sourceSegment, targetSegment);
}
public static void AddSegmentPair(this IWordAlignmentModel model, ParallelTextSegment segment,
Func <string, string> sourcePreprocessor = null, Func <string, string> targetPreprocessor = null,
bool isUnknown = true)
{
if (segment.IsEmpty)
{
return;
}
IReadOnlyList <string> sourceTokens = segment.SourceSegment.Preprocess(sourcePreprocessor);
IReadOnlyList <string> targetTokens = segment.TargetSegment.Preprocess(targetPreprocessor);
model.AddSegmentPair(sourceTokens, targetTokens, segment.CreateAlignmentMatrix(isUnknown));
}
public void AddSegmentPair(IReadOnlyList <string> sourceSegment, IReadOnlyList <string> targetSegment)
{
_directWordAlignmentModel.AddSegmentPair(sourceSegment, targetSegment);
_inverseWordAlignmentModel.AddSegmentPair(targetSegment, sourceSegment);
}