private IEnumerable <NounPhrase> GetTopNounPhrases(IReifiedTextual source) => source.Phrases.AsParallel().WithDegreeOfParallelism(Concurrency.Max) .OfNounPhrase() .InSubjectRole() .InObjectRole() .Distinct(CompareNounPhrases) .OrderBy(nounPhrase => nounPhrase.Weight);
static void WeightSimilarVerbPhrases(IReifiedTextual source)
{
//Reify the query source so that it may be queried to form a full self join (Cartesian product with itself.
// in the two subsequent from clauses both query the reified collection in parallel.
var toConsider = source.Phrases.OfVerbPhrase().WithSubjectOrObject();
GroupAndWeight(toConsider, (x, y) => x.IsSimilarTo(y), scaleBy: 0.5);
}
private async Task <ParallelQuery <VerbPhrase> > GetTopVerbPhrasesAsync(IReifiedTextual source) =>
await Task.Run(() =>
from verbal in source.Phrases.AsParallel()
.WithDegreeOfParallelism(Concurrency.Max)
.OfVerbPhrase()
.WithSubject()
.WithObject()
.Distinct((x, y) => x.IsSimilarTo(y))
let weightOfAssociatedEntities = verbal.Subjects.Concat(verbal.DirectAndIndirectObjects).Sum(e => e.Weight)
orderby verbal.Weight + weightOfAssociatedEntities
select verbal);
static void NormalizeWeights(IReifiedTextual source)
{
if (source.Phrases.Any())
{
var maxWeight = source.Phrases.Max(p => p.Weight);
if ((int)maxWeight != 0)
{
foreach (var phrase in source.Phrases)
{
phrase.Weight = phrase.Weight / maxWeight * 100;
}
}
}
}
/// <summary>
/// Increase noun weights in a document by abstracting over synonyms
/// </summary>
/// <param name="source">the Document whose noun weights may be modified</param>
static void WeightSimilarNouns(IReifiedTextual source)
{
var toConsider = from e in source.Words
.AsParallel().WithDegreeOfParallelism(Concurrency.Max)
.OfEntity()
.InSubjectOrObjectRole() //Currently, include only those nouns which exist in relationships with some IVerbal or IReferencer.
select e.Match()
.When((IReferencer r) => r.RefersTo != null)
.Then((IReferencer r) => r.RefersTo)
.Case((IEntity x) => new AggregateEntity(x)).Result()
into y
where y != null
select y;
GroupAndWeight(toConsider, (x, y) => x.IsSimilarTo(y), scaleBy: 1);
}
/// <summary>
/// Returns the top results for the given document using a heuristic which emphasizes the occurrence of Entities above
/// other metrics.
/// </summary>
/// <param name="source">The Document from which to retrieve results.</param>
/// <returns>The top results for the given document using a heuristic which emphasizes the occurrence of Entities above
/// other metrics.</returns>
public static IEnumerable <(string, float)> GetTopResultsByEntity(IReifiedTextual source)
{
var results = from entity in source.Phrases.OfEntity()
.AsParallel().WithDegreeOfParallelism(Concurrency.Max)
let e = entity.Match()
.When((IReferencer r) => r.RefersTo?.Any() ?? false)
.Then((IReferencer r) => r.RefersTo)
.Result(entity)
where e != null
group new { Name = e.Text, Value = (float)Math.Round(e.Weight, 2) } by e.Text into byText
where byText.Any()
let top = byText.MaxBy(x => x.Value)
orderby top.Value descending
select(top.Name, top.Value);
return(results);
}
private static IEnumerable <SvoRelationship> GetVerbWiseRelationships(IReifiedTextual source)
{
var relationships =
from verbal in source.Phrases
.OfVerbPhrase()
.AsParallel()
.WithDegreeOfParallelism(Concurrency.Max)
.WithSubject(s => !(s is IReferencer) || (s as IReferencer).RefersTo != null)
.Distinct((x, y) => x.IsSimilarTo(y))
from entity in verbal.Subjects.AsParallel().WithDegreeOfParallelism(Concurrency.Max)
let subject = entity.Match()
.When((IReferencer r) => r.RefersTo?.Any() ?? false)
.Then((IReferencer r) => r.RefersTo)
.Result(entity)
from direct in verbal.DirectObjects.DefaultIfEmpty()
from indirect in verbal.IndirectObjects.DefaultIfEmpty()
where subject != null && (direct ?? indirect) != null /*&& (subject.Text != (direct ?? indirect).Text)*/
let relationship = new SvoRelationship(subject, verbal, direct, indirect)
orderby relationship.Weight descending
select relationship;
return(relationships.Distinct());
}
private static IEnumerable <(string relationship, float weight)> GetTopResultsByVerbal(IReifiedTextual source) =>
static void WeightSimilarVerbs(IReifiedTextual source)
{
var toConsider = source.Words.OfVerb().WithSubjectOrObject();
GroupAndWeight(toConsider, (x, y) => x.IsSimilarTo(y), scaleBy: 1);
}
static void HackSubjectPropernounImportance(IReifiedTextual source) => source.Phrases.AsParallel().WithDegreeOfParallelism(Concurrency.Max) .OfNounPhrase() .Where(nounPhrase => nounPhrase.Words.OfProperNoun().Any()) .ForAll(nounPhrase => nounPhrase.Weight *= 2);
