public void Process(VarietyPair varietyPair) { IWordAligner aligner = _project.WordAligners[_alignerID]; varietyPair.WordPairs.Clear(); var counts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >(); foreach (Meaning meaning in varietyPair.Variety1.Words.Meanings) { Word[] words1 = varietyPair.Variety1.Words[meaning].Where(w => w.Shape.Count > 0).ToArray(); Word[] words2 = varietyPair.Variety2.Words[meaning].Where(w => w.Shape.Count > 0).ToArray(); if (words1.Length == 1 && words2.Length == 1) { Word word1 = words1.Single(); Word word2 = words2.Single(); WordPair wp = varietyPair.WordPairs.Add(word1, word2); Alignment <Word, ShapeNode> alignment = aligner.Compute(wp).GetAlignments().First(); wp.PhoneticSimilarityScore = alignment.NormalizedScore; UpdateCounts(aligner, counts, alignment); } else if (words1.Length > 0 && words2.Length > 0) { WordPair bestWordPair = null; Alignment <Word, ShapeNode> bestAlignment = null; foreach (Word w1 in words1) { foreach (Word w2 in words2) { Alignment <Word, ShapeNode> alignment = aligner.Compute(w1, w2).GetAlignments().First(); double score = alignment.NormalizedScore; if (bestWordPair == null || score > bestWordPair.PhoneticSimilarityScore) { bestWordPair = new WordPair(w1, w2) { PhoneticSimilarityScore = score }; bestAlignment = alignment; } } } varietyPair.WordPairs.Add(bestWordPair); UpdateCounts(aligner, counts, bestAlignment); } } varietyPair.SoundChangeFrequencyDistribution = counts; }
public WordPairViewModel(IWordAligner aligner, WordPair wordPair, bool areVarietiesInOrder) { _wordPair = wordPair; _areVarietiesInOrder = areVarietiesInOrder; _meaning = new MeaningViewModel(_wordPair.Word1.Meaning); _variety1 = new VarietyViewModel(_wordPair.VarietyPair.Variety1); _variety2 = new VarietyViewModel(_wordPair.VarietyPair.Variety2); IWordAlignerResult results = aligner.Compute(_wordPair); _alignment = results.GetAlignments().First(); _prefixNode = new AlignedNodeViewModel(_alignment.Prefixes[0], _alignment.Prefixes[1]); var nodes = new List<AlignedNodeViewModel>(); int i = 0; for (int column = 0; column < _alignment.ColumnCount; column++) { string note = null; if (i < _wordPair.AlignmentNotes.Count) note = _wordPair.AlignmentNotes[i]; nodes.Add(new AlignedNodeViewModel(column, _alignment[0, column], _alignment[1, column], note)); i++; } _suffixNode = new AlignedNodeViewModel(_alignment.Suffixes[0], _alignment.Suffixes[1]); _alignedNodes = new ReadOnlyCollection<AlignedNodeViewModel>(nodes); _showInMultipleWordAlignmentCommand = new RelayCommand(ShowInMultipleWordAlignment); }
public WordPairViewModel(IWordAligner aligner, WordPair wordPair, bool areVarietiesInOrder) { _wordPair = wordPair; _areVarietiesInOrder = areVarietiesInOrder; _meaning = new MeaningViewModel(_wordPair.Word1.Meaning); _variety1 = new VarietyViewModel(_wordPair.VarietyPair.Variety1); _variety2 = new VarietyViewModel(_wordPair.VarietyPair.Variety2); IWordAlignerResult results = aligner.Compute(_wordPair); _alignment = results.GetAlignments().First(); _prefixNode = new AlignedNodeViewModel(_alignment.Prefixes[0], _alignment.Prefixes[1]); var nodes = new List <AlignedNodeViewModel>(); int i = 0; for (int column = 0; column < _alignment.ColumnCount; column++) { string note = null; if (i < _wordPair.AlignmentNotes.Count) { note = _wordPair.AlignmentNotes[i]; } nodes.Add(new AlignedNodeViewModel(column, _alignment[0, column], _alignment[1, column], note)); i++; } _suffixNode = new AlignedNodeViewModel(_alignment.Suffixes[0], _alignment.Suffixes[1]); _alignedNodes = new ReadOnlyCollection <AlignedNodeViewModel>(nodes); }
public void Process(VarietyPair data) { IWordAligner aligner = _project.WordAligners[_alignerId]; var correspondenceColls = new Dictionary <FeatureSymbol, SoundCorrespondenceCollection> { { CogFeatureSystem.Onset, new SoundCorrespondenceCollection() }, { CogFeatureSystem.Nucleus, new SoundCorrespondenceCollection() }, { CogFeatureSystem.Coda, new SoundCorrespondenceCollection() } }; foreach (WordPair wordPair in data.WordPairs.Where(wp => wp.Cognacy)) { Alignment <Word, ShapeNode> alignment = aligner.Compute(wordPair).GetAlignments().First(); for (int i = 0; i < alignment.ColumnCount; i++) { AlignmentCell <ShapeNode> cell1 = alignment[0, i]; AlignmentCell <ShapeNode> cell2 = alignment[1, i]; if (!cell1.IsNull && !cell2.IsNull && cell1.Count == 1 && cell2.Count == 1) { SymbolicFeatureValue pos1, pos2; if (cell1.First.Annotation.FeatureStruct.TryGetValue(CogFeatureSystem.SyllablePosition, out pos1) && cell2.First.Annotation.FeatureStruct.TryGetValue(CogFeatureSystem.SyllablePosition, out pos2) && (FeatureSymbol)pos1 == (FeatureSymbol)pos2) { Ngram <Segment> ngram1 = cell1.ToNgram(_segmentPool); Ngram <Segment> ngram2 = cell2.ToNgram(_segmentPool); Segment seg1 = ngram1.First; Segment seg2 = ngram2.First; if (!seg1.Equals(seg2)) { SoundCorrespondenceCollection correspondences = correspondenceColls[(FeatureSymbol)pos1]; SoundCorrespondence corr; if (!correspondences.TryGet(seg1, seg2, out corr)) { corr = new SoundCorrespondence(seg1, seg2); correspondences.Add(corr); } corr.Frequency++; corr.WordPairs.Add(wordPair); } } } } } foreach (KeyValuePair <FeatureSymbol, SoundCorrespondenceCollection> kvp in correspondenceColls) { data.CognateSoundCorrespondencesByPosition[kvp.Key].ReplaceAll(kvp.Value); } }
protected override ReturnCode DoWork(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { ReturnCode retcode = ReturnCode.Okay; if (!RawScores && !NormalizedScores) { Warnings.Add("Neither raw scores nor normalized scores were selected. Defaulting to normalized."); RawScores = false; NormalizedScores = true; } if (RawScores && NormalizedScores) { Warnings.Add("Please specify either raw or normalized scores, but not both. Defaulting to normalized."); RawScores = false; NormalizedScores = true; } SetupProject(); Meaning meaning = MeaningFactory.Create(); IWordAligner wordAligner = Project.WordAligners["primary"]; foreach (string line in ReadLines(inputReader)) { string[] wordTexts = line.Split(' '); if (wordTexts.Length != 2) { Errors.Add(line, "Each line should have two space-separated words in it."); continue; } Word[] words = wordTexts.Select(wordText => ParseWordOnce(wordText, meaning, Project)).ToArray(); if (words.Length != 2 || words.Any(w => w == null)) { Errors.Add(line, "One or more of this line's words failed to parse. Successfully parsed words: {0}", string.Join(", ", words.Where(w => w != null).Select(w => w.StrRep))); continue; } IWordAlignerResult result = wordAligner.Compute(words[0], words[1]); Alignment <Word, ShapeNode> alignment = result.GetAlignments().First(); outputWriter.WriteLine("{0} {1} {2}", words[0].StrRep, words[1].StrRep, RawScores ? alignment.RawScore : alignment.NormalizedScore); if (Verbose) { outputWriter.Write(alignment.ToString(Enumerable.Empty <string>())); outputWriter.WriteLine(); } } return(retcode); }
private static void WriteWordPairs(StreamWriter writer, IWordAligner aligner, IEnumerable<WordPair> wordPairs) { bool first = true; foreach (WordPair pair in wordPairs.OrderByDescending(wp => wp.PhoneticSimilarityScore)) { if (!first) writer.WriteLine(); IWordAlignerResult results = aligner.Compute(pair); Alignment<Word, ShapeNode> alignment = results.GetAlignments().First(); writer.Write(pair.Word1.Meaning.Gloss); if (!string.IsNullOrEmpty(pair.Word1.Meaning.Category)) writer.Write(" ({0})", pair.Word1.Meaning.Category); writer.WriteLine(); writer.Write(alignment.ToString(pair.AlignmentNotes)); writer.WriteLine("Similarity: {0:p}", pair.PhoneticSimilarityScore); first = false; } }
private void E(VarietyPair pair) { ICognateIdentifier cognateIdentifier = _project.CognateIdentifiers[CognateIdentifierId]; var cognateCorrCounts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >(); IWordAligner aligner = _project.WordAligners[AlignerId]; int cognateCount = 0; double totalScore = 0; foreach (WordPair wordPair in pair.WordPairs) { IWordAlignerResult alignerResult = aligner.Compute(wordPair); cognateIdentifier.UpdatePredictedCognacy(wordPair, alignerResult); Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First(); if (wordPair.Cognacy) { for (int column = 0; column < alignment.ColumnCount; column++) { SoundContext lhs = alignment.ToSoundContext(_segmentPool, 0, column, aligner.ContextualSoundClasses); Ngram <Segment> corr = alignment[1, column].ToNgram(_segmentPool); cognateCorrCounts[lhs].Increment(corr); } cognateCount++; } wordPair.PhoneticSimilarityScore = alignment.NormalizedScore; totalScore += wordPair.PhoneticSimilarityScore; } pair.CognateCount = cognateCount; pair.CognateSoundCorrespondenceFrequencyDistribution = cognateCorrCounts; if (pair.WordPairs.Count == 0) { pair.LexicalSimilarityScore = 0; pair.PhoneticSimilarityScore = 0; } else { pair.LexicalSimilarityScore = (double)cognateCount / pair.WordPairs.Count; pair.PhoneticSimilarityScore = totalScore / pair.WordPairs.Count; } }
private static void WriteWordPairs(StreamWriter writer, IWordAligner aligner, IEnumerable <WordPair> wordPairs) { bool first = true; foreach (WordPair pair in wordPairs.OrderByDescending(wp => wp.PhoneticSimilarityScore)) { if (!first) { writer.WriteLine(); } IWordAlignerResult results = aligner.Compute(pair); Alignment <Word, ShapeNode> alignment = results.GetAlignments().First(); writer.Write(pair.Word1.Meaning.Gloss); if (!string.IsNullOrEmpty(pair.Word1.Meaning.Category)) { writer.Write(" ({0})", pair.Word1.Meaning.Category); } writer.WriteLine(); writer.Write(alignment.ToString(pair.AlignmentNotes)); writer.WriteLine("Similarity: {0:p}", pair.PhoneticSimilarityScore); first = false; } }
public void Process(VarietyPair varietyPair) { IWordAligner aligner = _project.WordAligners[_alignerID]; var ambiguousMeanings = new List <Tuple <Meaning, IWordAlignerResult, IWordAlignerResult[]> >(); varietyPair.WordPairs.Clear(); var cognateCorrCounts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >(); int cognateCount = 0; foreach (Meaning meaning in varietyPair.Variety1.Words.Meanings) { Word[] words1 = varietyPair.Variety1.Words[meaning].Where(w => w.Shape.Count > 0).ToArray(); Word[] words2 = varietyPair.Variety2.Words[meaning].Where(w => w.Shape.Count > 0).ToArray(); if (words1.Length == 1 && words2.Length == 1) { Word word1 = words1.Single(); Word word2 = words2.Single(); WordPair wp = varietyPair.WordPairs.Add(word1, word2); _project.CognacyDecisions.UpdateActualCognacy(wp); IWordAlignerResult alignerResult = aligner.Compute(wp); _thresholdCognateIdentifier.UpdatePredictedCognacy(wp, alignerResult); Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First(); if (wp.Cognacy) { UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, alignment); cognateCount++; } wp.PhoneticSimilarityScore = alignment.NormalizedScore; } else if (words1.Length > 0 && words2.Length > 0) { IWordAlignerResult[] alignerResults = words1.SelectMany(w1 => words2.Select(w2 => aligner.Compute(w1, w2))).ToArray(); IWordAlignerResult maxAlignerResult = alignerResults.MaxBy(a => a.BestRawScore); ambiguousMeanings.Add(Tuple.Create(meaning, maxAlignerResult, alignerResults)); WordPair wp = varietyPair.WordPairs.Add(maxAlignerResult.Words[0], maxAlignerResult.Words[1]); _thresholdCognateIdentifier.UpdatePredictedCognacy(wp, maxAlignerResult); } } ICognateIdentifier cognateIdentifier = _project.CognateIdentifiers[_cognateIdentifierID]; for (int i = 0; i < ambiguousMeanings.Count; i++) { ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> > newCognateCorrCounts = cognateCorrCounts.Clone(); int newCognateCount = cognateCount; for (int j = i + 1; j < ambiguousMeanings.Count; j++) { if (varietyPair.WordPairs[ambiguousMeanings[j].Item1].Cognacy) { UpdateCognateCorrespondenceCounts(aligner, newCognateCorrCounts, ambiguousMeanings[j].Item2.GetAlignments().First()); newCognateCount++; } } IWordAlignerResult bestAlignerResult = null; WordPair bestWordPair = null; foreach (IWordAlignerResult alignerResult in ambiguousMeanings[i].Item3) { ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> > alignmentCognateCorrCounts = newCognateCorrCounts.Clone(); int alignmentCognateCount = newCognateCount; Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First(); varietyPair.WordPairs.Remove(ambiguousMeanings[i].Item1); WordPair wordPair = varietyPair.WordPairs.Add(alignerResult.Words[0], alignerResult.Words[1]); _thresholdCognateIdentifier.UpdatePredictedCognacy(wordPair, alignerResult); if (wordPair.Cognacy) { UpdateCognateCorrespondenceCounts(aligner, alignmentCognateCorrCounts, alignment); alignmentCognateCount++; } varietyPair.CognateCount = alignmentCognateCount; varietyPair.CognateSoundCorrespondenceFrequencyDistribution = alignmentCognateCorrCounts; cognateIdentifier.UpdatePredictedCognacy(wordPair, alignerResult); wordPair.PhoneticSimilarityScore = alignment.NormalizedScore; if (bestWordPair == null || Compare(wordPair, bestWordPair) > 0) { bestWordPair = wordPair; bestAlignerResult = alignerResult; } } Debug.Assert(bestWordPair != null); varietyPair.WordPairs.Remove(ambiguousMeanings[i].Item1); varietyPair.WordPairs.Add(bestWordPair); _project.CognacyDecisions.UpdateActualCognacy(bestWordPair); if (bestWordPair.Cognacy) { UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, bestAlignerResult.GetAlignments().First()); cognateCount++; } } varietyPair.CognateCount = cognateCount; varietyPair.CognateSoundCorrespondenceFrequencyDistribution = cognateCorrCounts; }
private void AlignWords() { if (_selectedMeaning == null) { return; } _busyService.ShowBusyIndicatorUntilFinishDrawing(); var words = new HashSet <Word>(); foreach (VarietyPair vp in _projectService.Project.VarietyPairs) { WordPair wp; if (vp.WordPairs.TryGetValue(_selectedMeaning.DomainMeaning, out wp)) { words.Add(wp.Word1); words.Add(wp.Word2); } } if (words.Count == 0) { _words.Clear(); return; } IWordAligner aligner = _projectService.Project.WordAligners[ComponentIdentifiers.PrimaryWordAligner]; Alignment <Word, ShapeNode> alignment; if (words.Count == 1) { Word word = words.First(); Annotation <ShapeNode> prefixAnn = word.Prefix; var prefix = new AlignmentCell <ShapeNode>(prefixAnn != null ? word.Shape.GetNodes(prefixAnn.Span).Where(NodeFilter) : Enumerable.Empty <ShapeNode>()); IEnumerable <AlignmentCell <ShapeNode> > columns = word.Shape.GetNodes(word.Stem.Span).Where(NodeFilter).Select(n => new AlignmentCell <ShapeNode>(n)); Annotation <ShapeNode> suffixAnn = word.Suffix; var suffix = new AlignmentCell <ShapeNode>(suffixAnn != null ? word.Shape.GetNodes(suffixAnn.Span).Where(NodeFilter) : Enumerable.Empty <ShapeNode>()); alignment = new Alignment <Word, ShapeNode>(0, 0, Tuple.Create(word, prefix, columns, suffix)); } else { IWordAlignerResult result = aligner.Compute(words); alignment = result.GetAlignments().First(); } List <Cluster <Word> > cognateSets = _projectService.Project.GenerateCognateSets(_selectedMeaning.DomainMeaning).OrderBy(c => c.Noise).ThenByDescending(c => c.DataObjects.Count).ToList(); ColumnCount = alignment.ColumnCount; using (_words.BulkUpdate()) { _words.Clear(); for (int i = 0; i < alignment.SequenceCount; i++) { AlignmentCell <ShapeNode> prefix = alignment.Prefixes[i]; Word word = alignment.Sequences[i]; IEnumerable <AlignmentCell <ShapeNode> > columns = Enumerable.Range(0, alignment.ColumnCount).Select(col => alignment[i, col]); AlignmentCell <ShapeNode> suffix = alignment.Suffixes[i]; int cognateSetIndex = cognateSets.FindIndex(set => set.DataObjects.Contains(word)); _words.Add(new MultipleWordAlignmentWordViewModel(word, prefix, columns, suffix, cognateSetIndex == cognateSets.Count - 1 ? int.MaxValue : cognateSetIndex + 1)); } } }
public void Process(VarietyPair varietyPair) { IWordAligner aligner = _project.WordAligners[_alignerID]; varietyPair.WordPairs.Clear(); var cognateCorrCounts = new ConditionalFrequencyDistribution <SoundContext, Ngram <Segment> >(); int cognateCount = 0; foreach (Meaning meaning in varietyPair.Variety1.Words.Meanings) { Word[] words1 = varietyPair.Variety1.Words[meaning].Where(w => w.Shape.Count > 0).ToArray(); Word[] words2 = varietyPair.Variety2.Words[meaning].Where(w => w.Shape.Count > 0).ToArray(); if (words1.Length == 1 && words2.Length == 1) { Word word1 = words1.Single(); Word word2 = words2.Single(); WordPair wp = varietyPair.WordPairs.Add(word1, word2); _project.CognacyDecisions.UpdateActualCognacy(wp); IWordAlignerResult alignerResult = aligner.Compute(wp); _thresholdCognateIdentifier.UpdatePredictedCognacy(wp, alignerResult); Alignment <Word, ShapeNode> alignment = alignerResult.GetAlignments().First(); if (wp.Cognacy) { UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, alignment); cognateCount++; } wp.PhoneticSimilarityScore = alignment.NormalizedScore; } else if (words1.Length > 0 && words2.Length > 0) { WordPair bestWordPair = null; IWordAlignerResult bestAlignerResult = null; foreach (Word w1 in words1) { foreach (Word w2 in words2) { IWordAlignerResult alignerResult = aligner.Compute(w1, w2); if (bestAlignerResult == null || alignerResult.BestRawScore > bestAlignerResult.BestRawScore) { bestWordPair = new WordPair(w1, w2); bestAlignerResult = alignerResult; } } } Debug.Assert(bestWordPair != null); varietyPair.WordPairs.Add(bestWordPair); _project.CognacyDecisions.UpdateActualCognacy(bestWordPair); _thresholdCognateIdentifier.UpdatePredictedCognacy(bestWordPair, bestAlignerResult); Alignment <Word, ShapeNode> alignment = bestAlignerResult.GetAlignments().First(); if (bestWordPair.Cognacy) { UpdateCognateCorrespondenceCounts(aligner, cognateCorrCounts, alignment); cognateCount++; } bestWordPair.PhoneticSimilarityScore = alignment.NormalizedScore; } } varietyPair.CognateCount = cognateCount; varietyPair.CognateSoundCorrespondenceFrequencyDistribution = cognateCorrCounts; }