public void Process(VarietyPair varietyPair)
{
IWordAligner aligner = _project.WordAligners[_alignerID];
varietyPair.WordPairs.Clear();
var counts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >();
foreach (Meaning meaning in varietyPair.Variety1.Words.Meanings)
{
Word[] words1 = varietyPair.Variety1.Words[meaning].Where(w => w.Shape.Count > 0).ToArray();
Word[] words2 = varietyPair.Variety2.Words[meaning].Where(w => w.Shape.Count > 0).ToArray();
if (words1.Length == 1 && words2.Length == 1)
{
Word word1 = words1.Single();
Word word2 = words2.Single();
WordPair wp = varietyPair.WordPairs.Add(word1, word2);
Alignment <Word, ShapeNode> alignment = aligner.Compute(wp).GetAlignments().First();
wp.PhoneticSimilarityScore = alignment.NormalizedScore;
UpdateCounts(aligner, counts, alignment);
}
else if (words1.Length > 0 && words2.Length > 0)
{
WordPair bestWordPair = null;
Alignment <Word, ShapeNode> bestAlignment = null;
foreach (Word w1 in words1)
{
foreach (Word w2 in words2)
{
Alignment <Word, ShapeNode> alignment = aligner.Compute(w1, w2).GetAlignments().First();
double score = alignment.NormalizedScore;
if (bestWordPair == null || score > bestWordPair.PhoneticSimilarityScore)
{
bestWordPair = new WordPair(w1, w2)
{
PhoneticSimilarityScore = score
};
bestAlignment = alignment;
}
}
}
varietyPair.WordPairs.Add(bestWordPair);
UpdateCounts(aligner, counts, bestAlignment);
}
}
varietyPair.SoundChangeFrequencyDistribution = counts;
}
public WordPairViewModel(IWordAligner aligner, WordPair wordPair, bool areVarietiesInOrder)
{
_wordPair = wordPair;
_areVarietiesInOrder = areVarietiesInOrder;
_meaning = new MeaningViewModel(_wordPair.Word1.Meaning);
_variety1 = new VarietyViewModel(_wordPair.VarietyPair.Variety1);
_variety2 = new VarietyViewModel(_wordPair.VarietyPair.Variety2);
IWordAlignerResult results = aligner.Compute(_wordPair);
_alignment = results.GetAlignments().First();
_prefixNode = new AlignedNodeViewModel(_alignment.Prefixes[0], _alignment.Prefixes[1]);
var nodes = new List<AlignedNodeViewModel>();
int i = 0;
for (int column = 0; column < _alignment.ColumnCount; column++)
{
string note = null;
if (i < _wordPair.AlignmentNotes.Count)
note = _wordPair.AlignmentNotes[i];
nodes.Add(new AlignedNodeViewModel(column, _alignment[0, column], _alignment[1, column], note));
i++;
}
_suffixNode = new AlignedNodeViewModel(_alignment.Suffixes[0], _alignment.Suffixes[1]);
_alignedNodes = new ReadOnlyCollection<AlignedNodeViewModel>(nodes);
_showInMultipleWordAlignmentCommand = new RelayCommand(ShowInMultipleWordAlignment);
}
public WordPairViewModel(IWordAligner aligner, WordPair wordPair, bool areVarietiesInOrder)
{
_wordPair = wordPair;
_areVarietiesInOrder = areVarietiesInOrder;
_meaning = new MeaningViewModel(_wordPair.Word1.Meaning);
_variety1 = new VarietyViewModel(_wordPair.VarietyPair.Variety1);
_variety2 = new VarietyViewModel(_wordPair.VarietyPair.Variety2);
IWordAlignerResult results = aligner.Compute(_wordPair);
_alignment = results.GetAlignments().First();
_prefixNode = new AlignedNodeViewModel(_alignment.Prefixes[0], _alignment.Prefixes[1]);
var nodes = new List <AlignedNodeViewModel>();
int i = 0;
for (int column = 0; column < _alignment.ColumnCount; column++)
{
string note = null;
if (i < _wordPair.AlignmentNotes.Count)
{
note = _wordPair.AlignmentNotes[i];
}
nodes.Add(new AlignedNodeViewModel(column, _alignment[0, column], _alignment[1, column], note));
i++;
}
_suffixNode = new AlignedNodeViewModel(_alignment.Suffixes[0], _alignment.Suffixes[1]);
_alignedNodes = new ReadOnlyCollection <AlignedNodeViewModel>(nodes);
}
public void Process(VarietyPair data)
{
IWordAligner aligner = _project.WordAligners[_alignerId];
var correspondenceColls = new Dictionary <FeatureSymbol, SoundCorrespondenceCollection>
{
{ CogFeatureSystem.Onset, new SoundCorrespondenceCollection() },
{ CogFeatureSystem.Nucleus, new SoundCorrespondenceCollection() },
{ CogFeatureSystem.Coda, new SoundCorrespondenceCollection() }
};
foreach (WordPair wordPair in data.WordPairs.Where(wp => wp.Cognacy))
{
Alignment <Word, ShapeNode> alignment = aligner.Compute(wordPair).GetAlignments().First();
for (int i = 0; i < alignment.ColumnCount; i++)
{
AlignmentCell <ShapeNode> cell1 = alignment[0, i];
AlignmentCell <ShapeNode> cell2 = alignment[1, i];
if (!cell1.IsNull && !cell2.IsNull && cell1.Count == 1 && cell2.Count == 1)
{
SymbolicFeatureValue pos1, pos2;
if (cell1.First.Annotation.FeatureStruct.TryGetValue(CogFeatureSystem.SyllablePosition, out pos1) &&
cell2.First.Annotation.FeatureStruct.TryGetValue(CogFeatureSystem.SyllablePosition, out pos2) &&
(FeatureSymbol)pos1 == (FeatureSymbol)pos2)
{
Ngram <Segment> ngram1 = cell1.ToNgram(_segmentPool);
Ngram <Segment> ngram2 = cell2.ToNgram(_segmentPool);
Segment seg1 = ngram1.First;
Segment seg2 = ngram2.First;
if (!seg1.Equals(seg2))
{
SoundCorrespondenceCollection correspondences = correspondenceColls[(FeatureSymbol)pos1];
SoundCorrespondence corr;
if (!correspondences.TryGet(seg1, seg2, out corr))
{
corr = new SoundCorrespondence(seg1, seg2);
correspondences.Add(corr);
}
corr.Frequency++;
corr.WordPairs.Add(wordPair);
}
}
}
}
}
foreach (KeyValuePair <FeatureSymbol, SoundCorrespondenceCollection> kvp in correspondenceColls)
{
data.CognateSoundCorrespondencesByPosition[kvp.Key].ReplaceAll(kvp.Value);
}
}
protected override ReturnCode DoWork(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter)
{
ReturnCode retcode = ReturnCode.Okay;
if (!RawScores && !NormalizedScores)
{
Warnings.Add("Neither raw scores nor normalized scores were selected. Defaulting to normalized.");
RawScores = false;
NormalizedScores = true;
}
if (RawScores && NormalizedScores)
{
Warnings.Add("Please specify either raw or normalized scores, but not both. Defaulting to normalized.");
RawScores = false;
NormalizedScores = true;
}
SetupProject();
Meaning meaning = MeaningFactory.Create();
IWordAligner wordAligner = Project.WordAligners["primary"];
foreach (string line in ReadLines(inputReader))
{
string[] wordTexts = line.Split(' ');
if (wordTexts.Length != 2)
{
Errors.Add(line, "Each line should have two space-separated words in it.");
continue;
}
Word[] words = wordTexts.Select(wordText => ParseWordOnce(wordText, meaning, Project)).ToArray();
if (words.Length != 2 || words.Any(w => w == null))
{
Errors.Add(line, "One or more of this line's words failed to parse. Successfully parsed words: {0}",
string.Join(", ", words.Where(w => w != null).Select(w => w.StrRep)));
continue;
}
IWordAlignerResult result = wordAligner.Compute(words[0], words[1]);
Alignment <Word, ShapeNode> alignment = result.GetAlignments().First();
outputWriter.WriteLine("{0} {1} {2}", words[0].StrRep, words[1].StrRep,
RawScores ? alignment.RawScore : alignment.NormalizedScore);
if (Verbose)
{
outputWriter.Write(alignment.ToString(Enumerable.Empty <string>()));
outputWriter.WriteLine();
}
}
return(retcode);
}
private static void WriteWordPairs(StreamWriter writer, IWordAligner aligner, IEnumerable<WordPair> wordPairs)
{
bool first = true;
foreach (WordPair pair in wordPairs.OrderByDescending(wp => wp.PhoneticSimilarityScore))
{
if (!first)
writer.WriteLine();
IWordAlignerResult results = aligner.Compute(pair);
Alignment<Word, ShapeNode> alignment = results.GetAlignments().First();
writer.Write(pair.Word1.Meaning.Gloss);
if (!string.IsNullOrEmpty(pair.Word1.Meaning.Category))
writer.Write(" ({0})", pair.Word1.Meaning.Category);
writer.WriteLine();
writer.Write(alignment.ToString(pair.AlignmentNotes));
writer.WriteLine("Similarity: {0:p}", pair.PhoneticSimilarityScore);
first = false;
}
}
private void E(VarietyPair pair)
{
ICognateIdentifier cognateIdentifier = _project.CognateIdentifiers[CognateIdentifierId];
var cognateCorrCounts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >();
IWordAligner aligner = _project.WordAligners[AlignerId];
int cognateCount = 0;
double totalScore = 0;
foreach (WordPair wordPair in pair.WordPairs)
{
IWordAlignerResult alignerResult = aligner.Compute(wordPair);
cognateIdentifier.UpdatePredictedCognacy(wordPair, alignerResult);
Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First();
if (wordPair.Cognacy)
{
for (int column = 0; column < alignment.ColumnCount; column++)
{
SoundContext lhs = alignment.ToSoundContext(_segmentPool, 0, column, aligner.ContextualSoundClasses);
Ngram <Segment> corr = alignment[1, column].ToNgram(_segmentPool);
cognateCorrCounts[lhs].Increment(corr);
}
cognateCount++;
}
wordPair.PhoneticSimilarityScore = alignment.NormalizedScore;
totalScore += wordPair.PhoneticSimilarityScore;
}
pair.CognateCount = cognateCount;
pair.CognateSoundCorrespondenceFrequencyDistribution = cognateCorrCounts;
if (pair.WordPairs.Count == 0)
{
pair.LexicalSimilarityScore = 0;
pair.PhoneticSimilarityScore = 0;
}
else
{
pair.LexicalSimilarityScore = (double)cognateCount / pair.WordPairs.Count;
pair.PhoneticSimilarityScore = totalScore / pair.WordPairs.Count;
}
}
private static void WriteWordPairs(StreamWriter writer, IWordAligner aligner, IEnumerable <WordPair> wordPairs)
{
bool first = true;
foreach (WordPair pair in wordPairs.OrderByDescending(wp => wp.PhoneticSimilarityScore))
{
if (!first)
{
writer.WriteLine();
}
IWordAlignerResult results = aligner.Compute(pair);
Alignment <Word, ShapeNode> alignment = results.GetAlignments().First();
writer.Write(pair.Word1.Meaning.Gloss);
if (!string.IsNullOrEmpty(pair.Word1.Meaning.Category))
{
writer.Write(" ({0})", pair.Word1.Meaning.Category);
}
writer.WriteLine();
writer.Write(alignment.ToString(pair.AlignmentNotes));
writer.WriteLine("Similarity: {0:p}", pair.PhoneticSimilarityScore);
first = false;
}
}
public void Process(VarietyPair varietyPair)
{
IWordAligner aligner = _project.WordAligners[_alignerID];
var ambiguousMeanings = new List <Tuple <Meaning, IWordAlignerResult, IWordAlignerResult[]> >();
varietyPair.WordPairs.Clear();
var cognateCorrCounts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >();
int cognateCount = 0;
foreach (Meaning meaning in varietyPair.Variety1.Words.Meanings)
{
Word[] words1 = varietyPair.Variety1.Words[meaning].Where(w => w.Shape.Count > 0).ToArray();
Word[] words2 = varietyPair.Variety2.Words[meaning].Where(w => w.Shape.Count > 0).ToArray();
if (words1.Length == 1 && words2.Length == 1)
{
Word word1 = words1.Single();
Word word2 = words2.Single();
WordPair wp = varietyPair.WordPairs.Add(word1, word2);
_project.CognacyDecisions.UpdateActualCognacy(wp);
IWordAlignerResult alignerResult = aligner.Compute(wp);
_thresholdCognateIdentifier.UpdatePredictedCognacy(wp, alignerResult);
Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First();
if (wp.Cognacy)
{
UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, alignment);
cognateCount++;
}
wp.PhoneticSimilarityScore = alignment.NormalizedScore;
}
else if (words1.Length > 0 && words2.Length > 0)
{
IWordAlignerResult[] alignerResults = words1.SelectMany(w1 => words2.Select(w2 => aligner.Compute(w1, w2))).ToArray();
IWordAlignerResult maxAlignerResult = alignerResults.MaxBy(a => a.BestRawScore);
ambiguousMeanings.Add(Tuple.Create(meaning, maxAlignerResult, alignerResults));
WordPair wp = varietyPair.WordPairs.Add(maxAlignerResult.Words[0], maxAlignerResult.Words[1]);
_thresholdCognateIdentifier.UpdatePredictedCognacy(wp, maxAlignerResult);
}
}
ICognateIdentifier cognateIdentifier = _project.CognateIdentifiers[_cognateIdentifierID];
for (int i = 0; i < ambiguousMeanings.Count; i++)
{
ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> > newCognateCorrCounts = cognateCorrCounts.Clone();
int newCognateCount = cognateCount;
for (int j = i + 1; j < ambiguousMeanings.Count; j++)
{
if (varietyPair.WordPairs[ambiguousMeanings[j].Item1].Cognacy)
{
UpdateCognateCorrespondenceCounts(aligner, newCognateCorrCounts, ambiguousMeanings[j].Item2.GetAlignments().First());
newCognateCount++;
}
}
IWordAlignerResult bestAlignerResult = null;
WordPair bestWordPair = null;
foreach (IWordAlignerResult alignerResult in ambiguousMeanings[i].Item3)
{
ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> > alignmentCognateCorrCounts = newCognateCorrCounts.Clone();
int alignmentCognateCount = newCognateCount;
Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First();
varietyPair.WordPairs.Remove(ambiguousMeanings[i].Item1);
WordPair wordPair = varietyPair.WordPairs.Add(alignerResult.Words[0], alignerResult.Words[1]);
_thresholdCognateIdentifier.UpdatePredictedCognacy(wordPair, alignerResult);
if (wordPair.Cognacy)
{
UpdateCognateCorrespondenceCounts(aligner, alignmentCognateCorrCounts, alignment);
alignmentCognateCount++;
}
varietyPair.CognateCount = alignmentCognateCount;
varietyPair.CognateSoundCorrespondenceFrequencyDistribution = alignmentCognateCorrCounts;
cognateIdentifier.UpdatePredictedCognacy(wordPair, alignerResult);
wordPair.PhoneticSimilarityScore = alignment.NormalizedScore;
if (bestWordPair == null || Compare(wordPair, bestWordPair) > 0)
{
bestWordPair = wordPair;
bestAlignerResult = alignerResult;
}
}
Debug.Assert(bestWordPair != null);
varietyPair.WordPairs.Remove(ambiguousMeanings[i].Item1);
varietyPair.WordPairs.Add(bestWordPair);
_project.CognacyDecisions.UpdateActualCognacy(bestWordPair);
if (bestWordPair.Cognacy)
{
UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, bestAlignerResult.GetAlignments().First());
cognateCount++;
}
}
varietyPair.CognateCount = cognateCount;
varietyPair.CognateSoundCorrespondenceFrequencyDistribution = cognateCorrCounts;
}
private void AlignWords()
{
if (_selectedMeaning == null)
{
return;
}
_busyService.ShowBusyIndicatorUntilFinishDrawing();
var words = new HashSet <Word>();
foreach (VarietyPair vp in _projectService.Project.VarietyPairs)
{
WordPair wp;
if (vp.WordPairs.TryGetValue(_selectedMeaning.DomainMeaning, out wp))
{
words.Add(wp.Word1);
words.Add(wp.Word2);
}
}
if (words.Count == 0)
{
_words.Clear();
return;
}
IWordAligner aligner = _projectService.Project.WordAligners[ComponentIdentifiers.PrimaryWordAligner];
Alignment <Word, ShapeNode> alignment;
if (words.Count == 1)
{
Word word = words.First();
Annotation <ShapeNode> prefixAnn = word.Prefix;
var prefix = new AlignmentCell <ShapeNode>(prefixAnn != null ? word.Shape.GetNodes(prefixAnn.Span).Where(NodeFilter) : Enumerable.Empty <ShapeNode>());
IEnumerable <AlignmentCell <ShapeNode> > columns = word.Shape.GetNodes(word.Stem.Span).Where(NodeFilter).Select(n => new AlignmentCell <ShapeNode>(n));
Annotation <ShapeNode> suffixAnn = word.Suffix;
var suffix = new AlignmentCell <ShapeNode>(suffixAnn != null ? word.Shape.GetNodes(suffixAnn.Span).Where(NodeFilter) : Enumerable.Empty <ShapeNode>());
alignment = new Alignment <Word, ShapeNode>(0, 0, Tuple.Create(word, prefix, columns, suffix));
}
else
{
IWordAlignerResult result = aligner.Compute(words);
alignment = result.GetAlignments().First();
}
List <Cluster <Word> > cognateSets = _projectService.Project.GenerateCognateSets(_selectedMeaning.DomainMeaning).OrderBy(c => c.Noise).ThenByDescending(c => c.DataObjects.Count).ToList();
ColumnCount = alignment.ColumnCount;
using (_words.BulkUpdate())
{
_words.Clear();
for (int i = 0; i < alignment.SequenceCount; i++)
{
AlignmentCell <ShapeNode> prefix = alignment.Prefixes[i];
Word word = alignment.Sequences[i];
IEnumerable <AlignmentCell <ShapeNode> > columns = Enumerable.Range(0, alignment.ColumnCount).Select(col => alignment[i, col]);
AlignmentCell <ShapeNode> suffix = alignment.Suffixes[i];
int cognateSetIndex = cognateSets.FindIndex(set => set.DataObjects.Contains(word));
_words.Add(new MultipleWordAlignmentWordViewModel(word, prefix, columns, suffix, cognateSetIndex == cognateSets.Count - 1 ? int.MaxValue : cognateSetIndex + 1));
}
}
}
public void Process(VarietyPair varietyPair)
{
IWordAligner aligner = _project.WordAligners[_alignerID];
varietyPair.WordPairs.Clear();
var cognateCorrCounts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >();
int cognateCount = 0;
foreach (Meaning meaning in varietyPair.Variety1.Words.Meanings)
{
Word[] words1 = varietyPair.Variety1.Words[meaning].Where(w => w.Shape.Count > 0).ToArray();
Word[] words2 = varietyPair.Variety2.Words[meaning].Where(w => w.Shape.Count > 0).ToArray();
if (words1.Length == 1 && words2.Length == 1)
{
Word word1 = words1.Single();
Word word2 = words2.Single();
WordPair wp = varietyPair.WordPairs.Add(word1, word2);
_project.CognacyDecisions.UpdateActualCognacy(wp);
IWordAlignerResult alignerResult = aligner.Compute(wp);
_thresholdCognateIdentifier.UpdatePredictedCognacy(wp, alignerResult);
Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First();
if (wp.Cognacy)
{
UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, alignment);
cognateCount++;
}
wp.PhoneticSimilarityScore = alignment.NormalizedScore;
}
else if (words1.Length > 0 && words2.Length > 0)
{
WordPair bestWordPair = null;
IWordAlignerResult bestAlignerResult = null;
foreach (Word w1 in words1)
{
foreach (Word w2 in words2)
{
IWordAlignerResult alignerResult = aligner.Compute(w1, w2);
if (bestAlignerResult == null || alignerResult.BestRawScore > bestAlignerResult.BestRawScore)
{
bestWordPair = new WordPair(w1, w2);
bestAlignerResult = alignerResult;
}
}
}
Debug.Assert(bestWordPair != null);
varietyPair.WordPairs.Add(bestWordPair);
_project.CognacyDecisions.UpdateActualCognacy(bestWordPair);
_thresholdCognateIdentifier.UpdatePredictedCognacy(bestWordPair, bestAlignerResult);
Alignment <Word, ShapeNode> alignment = bestAlignerResult.GetAlignments().First();
if (bestWordPair.Cognacy)
{
UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, alignment);
cognateCount++;
}
bestWordPair.PhoneticSimilarityScore = alignment.NormalizedScore;
}
}
varietyPair.CognateCount = cognateCount;
varietyPair.CognateSoundCorrespondenceFrequencyDistribution = cognateCorrCounts;
}
