private IEnumerable <NounPhrase> GetTopNounPhrases(IReifiedTextual source) => source.Phrases.AsParallel().WithDegreeOfParallelism(Concurrency.Max) .OfNounPhrase() .InSubjectRole() .InObjectRole() .Distinct(CompareNounPhrases) .OrderBy(nounPhrase => nounPhrase.Weight);
static void WeightSimilarVerbPhrases(IReifiedTextual source) { //Reify the query source so that it may be queried to form a full self join (Cartesian product with itself. // in the two subsequent from clauses both query the reified collection in parallel. var toConsider = source.Phrases.OfVerbPhrase().WithSubjectOrObject(); GroupAndWeight(toConsider, (x, y) => x.IsSimilarTo(y), scaleBy: 0.5); }
private async Task <ParallelQuery <VerbPhrase> > GetTopVerbPhrasesAsync(IReifiedTextual source) => await Task.Run(() => from verbal in source.Phrases.AsParallel() .WithDegreeOfParallelism(Concurrency.Max) .OfVerbPhrase() .WithSubject() .WithObject() .Distinct((x, y) => x.IsSimilarTo(y)) let weightOfAssociatedEntities = verbal.Subjects.Concat(verbal.DirectAndIndirectObjects).Sum(e => e.Weight) orderby verbal.Weight + weightOfAssociatedEntities select verbal);
static void NormalizeWeights(IReifiedTextual source) { if (source.Phrases.Any()) { var maxWeight = source.Phrases.Max(p => p.Weight); if ((int)maxWeight != 0) { foreach (var phrase in source.Phrases) { phrase.Weight = phrase.Weight / maxWeight * 100; } } } }
/// <summary> /// Increase noun weights in a document by abstracting over synonyms /// </summary> /// <param name="source">the Document whose noun weights may be modified</param> static void WeightSimilarNouns(IReifiedTextual source) { var toConsider = from e in source.Words .AsParallel().WithDegreeOfParallelism(Concurrency.Max) .OfEntity() .InSubjectOrObjectRole() //Currently, include only those nouns which exist in relationships with some IVerbal or IReferencer. select e.Match() .When((IReferencer r) => r.RefersTo != null) .Then((IReferencer r) => r.RefersTo) .Case((IEntity x) => new AggregateEntity(x)).Result() into y where y != null select y; GroupAndWeight(toConsider, (x, y) => x.IsSimilarTo(y), scaleBy: 1); }
/// <summary> /// Returns the top results for the given document using a heuristic which emphasizes the occurrence of Entities above /// other metrics. /// </summary> /// <param name="source">The Document from which to retrieve results.</param> /// <returns>The top results for the given document using a heuristic which emphasizes the occurrence of Entities above /// other metrics.</returns> public static IEnumerable <(string, float)> GetTopResultsByEntity(IReifiedTextual source) { var results = from entity in source.Phrases.OfEntity() .AsParallel().WithDegreeOfParallelism(Concurrency.Max) let e = entity.Match() .When((IReferencer r) => r.RefersTo?.Any() ?? false) .Then((IReferencer r) => r.RefersTo) .Result(entity) where e != null group new { Name = e.Text, Value = (float)Math.Round(e.Weight, 2) } by e.Text into byText where byText.Any() let top = byText.MaxBy(x => x.Value) orderby top.Value descending select(top.Name, top.Value); return(results); }
private static IEnumerable <SvoRelationship> GetVerbWiseRelationships(IReifiedTextual source) { var relationships = from verbal in source.Phrases .OfVerbPhrase() .AsParallel() .WithDegreeOfParallelism(Concurrency.Max) .WithSubject(s => !(s is IReferencer) || (s as IReferencer).RefersTo != null) .Distinct((x, y) => x.IsSimilarTo(y)) from entity in verbal.Subjects.AsParallel().WithDegreeOfParallelism(Concurrency.Max) let subject = entity.Match() .When((IReferencer r) => r.RefersTo?.Any() ?? false) .Then((IReferencer r) => r.RefersTo) .Result(entity) from direct in verbal.DirectObjects.DefaultIfEmpty() from indirect in verbal.IndirectObjects.DefaultIfEmpty() where subject != null && (direct ?? indirect) != null /*&& (subject.Text != (direct ?? indirect).Text)*/ let relationship = new SvoRelationship(subject, verbal, direct, indirect) orderby relationship.Weight descending select relationship; return(relationships.Distinct()); }
private static IEnumerable <(string relationship, float weight)> GetTopResultsByVerbal(IReifiedTextual source) =>
static void WeightSimilarVerbs(IReifiedTextual source) { var toConsider = source.Words.OfVerb().WithSubjectOrObject(); GroupAndWeight(toConsider, (x, y) => x.IsSimilarTo(y), scaleBy: 1); }
static void HackSubjectPropernounImportance(IReifiedTextual source) => source.Phrases.AsParallel().WithDegreeOfParallelism(Concurrency.Max) .OfNounPhrase() .Where(nounPhrase => nounPhrase.Words.OfProperNoun().Any()) .ForAll(nounPhrase => nounPhrase.Weight *= 2);