public void GetTimeSeriesShouldProcessComments()
{
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, null, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
int userId = 0;
m_commentsProcessorMock.Setup(cp => cp.ConvertAnswersToQuestions(It.IsAny<string>(), It.IsAny<IEnumerable<CommentInfo>>()))
.Returns(new[] {
new CommentInfo { CreationDate = new DateTime(2010, 07, 01), Id = 9876, IsQuestion = true } ,
new CommentInfo { CreationDate = new DateTime(2010, 07, 02), Id = 9875, IsQuestion = true }
});
m_questionRepo.Setup(ar => ar.Get(It.IsAny<string>(), It.Is<IEnumerable<int>>(arr => arr.IsEqual(new[] { 9876, 9875 }))))
.Returns(new List<QuestionInfo> {
new QuestionInfo { CreationDate = new DateTime(2010, 06, 30), Id = 9876, Tags = new List<string> { "tag1", "tag3" } } ,
new QuestionInfo { CreationDate = new DateTime(2010, 06, 29), Id = 9875, Tags = new List<string> { "tag3", "tag4" } }
});
m_PersistQuestionsRepo.Setup(qpr => qpr.Get(It.IsAny<string>(), It.Is<IEnumerable<int>>(arr => arr.IsEqual(new[] { 9876, 9875 })))).Returns(new List<QuestionInfo> { });
var actual = target.RequestTimeSeries("", userId, null,null, Rollup.Weekly);
CollectionAssert.AreEquivalent(new[] { "tag1", "tag2", "tag3", "tag4" }, actual.Timeseries.Keys);
Assert.IsTrue(actual.Timeseries["tag1"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(1, 2, 0) }, { new DateTime(2010, 06, 27), new TimeseriesPeriod(0, 0, 1) } }));
Assert.IsTrue(actual.Timeseries["tag2"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(2, 1, 0) }}));
Assert.IsTrue(actual.Timeseries["tag3"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(1, 0, 0) }, { new DateTime(2010, 06, 27), new TimeseriesPeriod(0, 0, 2) } }));
Assert.IsTrue(actual.Timeseries["tag4"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(0, 1, 0) }, { new DateTime(2010, 06, 27), new TimeseriesPeriod(0, 0, 1) } }));
}
public void TestSummarizeExampleText()
{
string sample = @"Automatic text summarization is the technique which
automatically creates an abstract or summary of a text. The
technique has been developed for many years.
According to Hovy and Lin there are two ways to view
text summarization either as text extraction or text abstraction.
Text extraction means to extract pieces of an original text
on a statistical basis or with heuristic methods and put them
together into a shorter text with the same information content.
Sometimes, the extracted fragments are post-edited, for example
by deleting subordinate clauses or joining incomplete clauses
to form complete clauses. Text abstraction is to parse
the original text in a linguistic way, interpret the text and find
new concepts to describe the text and then generate a new
shorter text with the same information content. This is in many
aspects similar to what human abstractors do when writing
an abstract, using surface level information like headings, key
phrases, positions and so on.";
string summary = Summarizer.Summarize(sample);
Assert.NotNull(summary);
Assert.IsInstanceOf <string>(summary);
Assert.IsNotEmpty(summary);
Console.WriteLine(summary);
}
public override void Run()
{
// Detects input devices if any
InputDevice inputDevice = ExampleUtil.ChooseInputDeviceFromConsole();
if (inputDevice == null)
{
Console.WriteLine("No input devices, so can't run this example.");
ExampleUtil.PressAnyKeyToContinue();
return;
}
else
{
Console.WriteLine("Connected to: " + inputDevice.Name);
}
inputDevice.Open();
inputDevice.StartReceiving(null);
Summarizer summarizer = new Summarizer(inputDevice);
ExampleUtil.PressAnyKeyToContinue();
inputDevice.StopReceiving();
inputDevice.Close();
Summarizer.serialPort.Close();
inputDevice.RemoveAllEventHandlers();
}
public void Run()
{
var numbersToSum = new List <double> ();
CurrentConsole.WriteLine("Please enter numbers you want me to add. When you have finished just enter 'done'.");
while (true)
{
CurrentConsole.Write("Enter a number or type 'done': ");
var inputString = Console.ReadLine();
var inputNumber = 0.0;
if (double.TryParse(inputString, out inputNumber))
{
numbersToSum.Add(inputNumber);
}
else
{
if (inputString.ToLowerInvariant() == "done")
{
break;
}
CurrentConsole.WriteLine(string.Format("I'm sorry, '{0}' is not a number or the word done.", inputString));
}
}
var sum = Summarizer.Sum(numbersToSum.ToArray());
CurrentConsole.WriteLine(string.Format("The sum of the numbers {0} is {1}", string.Join(", ", numbersToSum.Select(n => n.ToString())), sum));
}
/// <summary>
/// Summarizes the specified text.
/// </summary>
/// <param name="text">The text.</param>
/// <param name="percentage">The percentage of sentences to have in the final text.</param>
/// <param name="language">The language.</param>
/// <returns>The summarized text</returns>
public Document Summarize(Document text, float percentage, TextSummarizationLanguage language)
{
if (!Summarizers.TryGetValue(language, out var Summarizer))
{
return(text);
}
return(Summarizer.Summarize(text, percentage));
}
protected override void UpdateSummary(User currentUser, WorkDay wDay)
{
summaryInfo = wDay == null ? SummaryInfo.Zero : Summarizer.GetSummaryFromYear(currentUser, wDay.WorkTime.DateOfWork);
RaisePropertyChanged(nameof(SummaryTotalHours));
RaisePropertyChanged(nameof(SummaryTotalRequired));
RaisePropertyChanged(nameof(SummaryTotalAccounted));
RaisePropertyChanged(nameof(SummaryTotalOvertime));
}
static void Main(string[] args)
{
//Summarizer.prepareForRouge();
//Summarizer.generateModels();
//Summarizer.ComparativeSummarization();
//Summarizer.TimeLineSummarization();
Summarizer.GeneralAndQuerySummarization();
}
/// <summary>
/// Summarizes the specified text.
/// </summary>
/// <param name="text">The text.</param>
/// <param name="sentenceCount">The number of sentences to have in the final text.</param>
/// <param name="language">The language.</param>
/// <returns>The summarized text</returns>
public Document Summarize(Document text, int sentenceCount, TextSummarizationLanguage language)
{
if (!Summarizers.TryGetValue(language, out var Summarizer))
{
return(text);
}
return(Summarizer.Summarize(text, sentenceCount));
}
public void Setup()
{
quantifierTrapezoidalFunction = new TrapezoidalFunction(0.12, 0.44, 0.36, 0.16);
summarizerTrapezoidalFunction = new TrapezoidalFunction(0.0, 40.0, 30.0, 0);
quantifierFuzzySet = new FuzzySet(quantifierTrapezoidalFunction);
summarizerFuzzySet = new FuzzySet(summarizerTrapezoidalFunction);
quantifier = new Quantifier("SOME", quantifierFuzzySet);
summarizer = new Summarizer("YOUNG", summarizerFuzzySet);
}
public FarmZilla(ConfigurationConstants configuration)
{
_configuration = configuration;
_dependencyFinder = new DependencyFinder();
_fileCacheFarmer = new FileCacheFarmer();
_serviceFinder = new ServiceFinder();
_summarizer = new Summarizer(configuration.SummaryOutputPath);
}
public static void problem()
{
Summarizer s1 = new Summarizer(2.2F);
Summarizer s2 = new Summarizer(3.0F);
Summarizer s3 = new Summarizer(5.0F, 1.0F);
Console.WriteLine("First answer: {0}", s1.computeAnswer());
Console.WriteLine("Second answer: {0}", s2.computeAnswer());
Console.WriteLine("Third answer: {0}", s3.computeAnswer());
}
static void OnPressThread()
{
try
{
inputDevice.Open();
inputDevice.StartReceiving(null);
Summarizer summarizer = new Summarizer(inputDevice);
}
catch (Exception)
{
}
}
public void AddToCollection()
{
if (FunctionSelectionVm.Function == null)
{
MessageBox.Show("Please setup function in configurator!");
return;
}
var temp = new Summarizer(LabelNameTB, new FuzzySet(FunctionSelectionVm.Function));
AttributeSelected.Summarizers.Add(temp);
Parent.SummaryVm.Qualifiers.Add(temp);
Draw();
}
private static void Main()
{
SummarizerArguments summarizerArguments = new SummarizerArguments
{
InputFile = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "TextualData", "AutomaticSummarization.txt")
};
SummarizedDocument summarizedDocument = Summarizer.Summarize(summarizerArguments);
string summery = string.Join(Environment.NewLine, summarizedDocument.Sentences);
Console.Write(summery);
Console.ReadLine();
}
private static void Main()
{
SummarizedDocument summarizedDocument = Summarizer.Summarize(
new FileContentProvider(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "TextualData\\AutomaticSummarization.txt")),
new SummarizerArguments
{
Language = "en",
MaxSummarySentences = 5
});
string summery = string.Join(Environment.NewLine, summarizedDocument.Sentences);
Console.Write(summery);
Console.ReadLine();
}
public void GetTimeSeriesShouldCallPersistRepoWithSameParams()
{
string site = string.Empty;
int userId = 123;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
var expected = new TimeseriesResult(Rollup.Daily);
Mock<IPersistTimeseriesRepository> persistTsRepo = new Mock<IPersistTimeseriesRepository>();
persistTsRepo.Setup(r => r.GetTimeSeries(site, userId, start, end, Rollup.Weekly)).Returns(expected);
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, persistTsRepo.Object, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
TimeseriesResult actual;
actual = target.GetTimeSeries(site, userId, start, end, Rollup.Weekly);
persistTsRepo.VerifyAll();
}
public void GetTimeSeriesReturnPersistRepoDataIfNotNull()
{
var expected = new TimeseriesResult(Rollup.Daily);
Mock<IPersistTimeseriesRepository> persistTsRepo = new Mock<IPersistTimeseriesRepository>();
persistTsRepo.Setup(r => r.GetTimeSeries(It.IsAny<string>(), It.IsAny<int>(), It.IsAny<DateTime?>(), It.IsAny<DateTime?>(), It.IsAny<Rollup>())).Returns(expected);
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, persistTsRepo.Object, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
string site = string.Empty;
int userId = 0;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
TimeseriesResult actual;
actual = target.GetTimeSeries(site, userId, start, end, Rollup.Weekly);
persistTsRepo.VerifyAll();
Assert.AreEqual(expected, actual);
}
public static string OpenText(string inputText, int maxSummSentences)
{
var sw = Stopwatch.StartNew();
var args = new SummarizerArguments
{
MaxSummarySentences = maxSummSentences,
MaxSummarySizeInPercent = 100
};
var summarizedDocument = Summarizer.Summarize(new DirectTextContentProvider(inputText), args);
var sb = new StringBuilder();
sb.AppendLine($"Summarised content in {sw.ElapsedMilliseconds} ms");
sb.AppendLine(" ===== Summary =============================== ");
summarizedDocument.Sentences.ForEach(s => sb.AppendLine(string.Format("{0}\r\n", s)));
sb.AppendLine(" ===== Keywords =============================== ");
summarizedDocument.Concepts.ForEach(c => sb.AppendLine(string.Format("\t{0}", c)));
return(sb.ToString());
}
private void SummarizeButton_Click(object sender, EventArgs e)
{
int sentCount = 1;
int.TryParse(numericUpDown1.Text, out sentCount);
SummarizerArguments sumargs = new SummarizerArguments
{
DictionaryLanguage = "en",
DisplayLines = sentCount,
DisplayPercent = 0,
InputFile = "",
InputString = OriginalTextBox.Text
};
SummarizedDocument doc = Summarizer.Summarize(sumargs);
string summary = string.Join("\r\n\r\n", doc.Sentences.ToArray());
SummaryTextBox.Text = summary;
}
/// <summary>
/// Summarizes an enumerable of double messages into summarized doubles.
/// </summary>
/// <typeparam name="T">The type of messages to summarize.</typeparam>
/// <param name="messages">Enumerable of double messages.</param>
/// <param name="interval">The time interval each summary value should cover.</param>
/// <returns>List of summarized doubles.</returns>
internal static List <IntervalData <T> > Summarizer <T>(IEnumerable <Message <T> > messages, TimeSpan interval)
{
return(messages
.OrderBy(msg => msg.OriginatingTime)
.GroupBy(msg => Summarizer <T, T> .GetIntervalStartTime(msg.OriginatingTime, interval))
.Select(
group =>
{
var firstMessage = group.First();
var lastMessage = group.Last();
return IntervalData.Create(
lastMessage.Data, // Take last value as representative value for plotting
group.Min(m => m.Data), // Minimum value
group.Max(m => m.Data), // Maximum value
firstMessage.OriginatingTime, // First message's OT
lastMessage.OriginatingTime - firstMessage.OriginatingTime); // Interval between first and last messages
}).ToList());
}
private string SummarizeText(string text)
{
// Set text summarize arguments
SummarizerArguments sumargs = new SummarizerArguments
{
DictionaryLanguage = "en",
DisplayLines = 5,
DisplayPercent = 0,
InputFile = "",
InputString = text
};
// Summarize text
SummarizedDocument doc = Summarizer.Summarize(sumargs);
// Return result
return(string.Join("\r\n\r\n", doc.Sentences.ToArray()));
}
public override void Run()
{
// Prompt user to choose an input device (or if there is only one, use that one).
InputDevice inputDevice = ExampleUtil.ChooseInputDeviceFromConsole();
if (inputDevice == null)
{
Console.WriteLine("No input devices, so can't run this example.");
ExampleUtil.PressAnyKeyToContinue();
return;
}
inputDevice.Open();
inputDevice.StartReceiving(null);
Summarizer summarizer = new Summarizer(inputDevice);
ExampleUtil.PressAnyKeyToContinue();
inputDevice.StopReceiving();
inputDevice.Close();
inputDevice.RemoveAllEventHandlers();
}
public void TestPipelineWithSourceFile()
{
// TODO: rewrite unit test to handle all the methods in a file
// For now, just pull the first method from the file and proceed as TestPipelineXMLSnipper().
var srcmlMethod = MethodExtractor.ExtractAllMethodsFromFile("../Sample Methods/sample methods.cpp").First().Item3;
// Verify the method definition
Assert.IsInstanceOf <MethodDefinition>(srcmlMethod, "MethodDefinition found.");
Console.WriteLine(srcmlMethod.ToString());
// Extract SUnit Statements from MethodDefinition
var statements = SUnitExtractor.ExtractAll(srcmlMethod).ToList();
// verify the statements selected
Assert.IsNotEmpty(statements, "statements selected from method definition");
Console.WriteLine(statements.ToString());
// Translate Statements into SUnits
List <SUnit> sunits = statements.ConvertAll(
new Converter <Statement, SUnit>(SUnitTranslator.Translate));
// verify sunits have been translated
Assert.That(sunits.TrueForAll(s => s.action != null), "All SUnits initialized.");
Console.WriteLine(sunits.ToString());
// Generate text from SUnits
List <string> sentences = sunits.ConvertAll(
new Converter <SUnit, string>(TextGenerator.GenerateText));
// verify string generated
Assert.That(sentences.TrueForAll(s => s.Length > 0));
Console.WriteLine(sentences);
// Collect text and summarize
var methodDocument = String.Join <string>(" ", sentences);
var summary = Summarizer.Summarize(methodDocument);
// verify summary
Assert.That(!summary.Equals(""));
Console.WriteLine(summary);
}
/// <summary>
/// Summarizes an enumerable of double messages into summarized doubles.
/// </summary>
/// <typeparam name="T">The type of messages to summarize.</typeparam>
/// <param name="messages">Enumerable of double messages.</param>
/// <param name="interval">The time interval each summary value should cover.</param>
/// <returns>List of summarized doubles.</returns>
internal static List <IntervalData <T> > Summarizer <T>(IEnumerable <Message <T> > messages, TimeSpan interval)
{
return(messages
.OrderBy(msg => msg.OriginatingTime)
.GroupBy(msg => Summarizer <T, T> .GetIntervalStartTime(msg.OriginatingTime, interval))
.Select(
group =>
{
var firstMessage = group.First();
var lastMessage = group.Last();
// Use the last value as representative for summarization, with the first message
// originating time.
return IntervalData.Create(
value: lastMessage.Data,
minimum: group.Min(m => m.Data),
maximum: group.Max(m => m.Data),
originatingTime: firstMessage.OriginatingTime,
interval: lastMessage.OriginatingTime - firstMessage.OriginatingTime);
}).ToList());
}
public override void Run()
{
// Prompt user to choose an input device (or if there is only one, use that one).
var inputDevice = ExampleUtil.ChooseInputDeviceFromConsole();
if (inputDevice == null)
{
Console.WriteLine("No input devices, so can't run this example.");
ExampleUtil.PressAnyKeyToContinue();
return;
}
inputDevice.Open();
inputDevice.StartReceiving(null);
var summarizer = new Summarizer(inputDevice);
ExampleUtil.PressAnyKeyToContinue();
inputDevice.StopReceiving();
inputDevice.Close();
inputDevice.RemoveAllEventHandlers();
}
/// <summary>
/// Constructor for the factory that generates the summarizer
/// </summary>
static SummarizerFactory()
{
_summarizer = new Summarizer();
}
public void GetTimeSeriesShouldSaveResultIfRequestingFromServer()
{
string site = string.Empty;
int userId = 123;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
Mock<IPersistTimeseriesRepository> persistTsRepo = new Mock<IPersistTimeseriesRepository>();
persistTsRepo.Setup(r => r.GetTimeSeries(site, userId, start, end, Rollup.Weekly)).Returns((TimeseriesResult)null);
persistTsRepo.Setup(r => r.SaveTimeseries(site, userId, It.IsAny<TimeseriesResult>()));
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, persistTsRepo.Object, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
TimeseriesResult actual;
actual = target.GetTimeSeries(site, userId, start, end, Rollup.Weekly);
persistTsRepo.VerifyAll();
}
public void TestPipelineXMLSnippet()
{
// SrcML sample method
string srcmlOutput = @"<function><type><name> bool </name></type> <name> findInFiles </name><parameter_list> () </parameter_list>
<block>{
<decl_stmt><decl><type><specifier>const</specifier> <name>TCHAR</name> <modifier>*</modifier></type><name>dir2Search</name> <init>= <expr><call><name><name>_findReplaceDlg</name><operator>.</operator><name>getDir2Search</name></name><argument_list>()</argument_list></call></expr></init></decl>;</decl_stmt>
<expr_stmt><expr><call><name>findFilesInOut</name><argument_list>()</argument_list></call></expr>;</expr_stmt>
<if>if <condition>(<expr><operator>!</operator><name><name>dir2Search</name><index>[<expr><literal type=""number"">0</literal></expr>]</index></name> <operator>||</operator> <operator>!</operator><call><name><operator>::</operator><name>PathFileExists</name></name><argument_list>(<argument><expr><name>dir2Search</name></expr></argument>)</argument_list></call></expr>)</condition><then>
<block>{
<return>return <expr><literal type = ""boolean"" > false </literal></expr>;</return>
}</block></then></if>
<decl_stmt><decl><type><name>string</name></type> <name>findString</name> <init>= <expr><literal type = ""string"" > """" </literal ></expr ></init ></decl >;</decl_stmt>
<expr_stmt><expr><call><name>gethurry</name><argument_list>()</argument_list></call></expr>;</expr_stmt>
<macro><name>findInOne</name><argument_list>(<argument>int a</argument>, <argument>findString</argument>)</argument_list></macro><empty_stmt>;</empty_stmt>
<decl_stmt><decl><type><name>bool</name></type> <name>isRecursive</name> <init>= <expr><call><name><name>_findReplaceDlg</name><operator >.</operator><name>isRecursive</name></name><argument_list>()</argument_list></call></expr></init></decl>;</decl_stmt>
<decl_stmt><decl><type><name>bool</name></type> <name>isInHiddenDir</name> <init>= <expr><call><name><name>_findReplaceDlg</name><operator >.</operator><name>isInHiddenDir</name></name><argument_list>()</argument_list></call></expr></init></decl>;</decl_stmt>
<if>if <condition>(<expr><call><name><name>a</name><operator >.</operator><name>size</name></name><argument_list>()</argument_list></call> <operator >==</operator> <literal type = ""number"" > 0 </literal></expr>)</condition><then>
<block>{
<expr_stmt><expr><call><name><name>a</name><operator >.</operator><name>setFindInFilesDirFilter</name></name><argument_list>(<argument><expr><literal type = ""string""> ""dddd"" </literal ></expr ></argument>, <argument><expr><call><name>TEXT</name><argument_list>(<argument><expr><literal type = ""string"" > ""*.*"" </literal ></expr></argument>)</argument_list></call></expr></argument>)</argument_list></call></expr>;</expr_stmt>
<expr_stmt><expr><call><name><name>a</name><operator >.</operator><name>getPatterns</name></name><argument_list>(<argument><expr><name>findString</name></expr></argument>)</argument_list></call></expr>;</expr_stmt>
}</block></then></if>
<return>return <expr><literal type = ""boolean"" > true </literal ></expr>;</return>
}</block></function>";
// Convert raw string to MethodDefinition
var fileSetup = new SrcMLFileUnitSetup(Language.CPlusPlus);
var parser = new CPlusPlusCodeParser();
var fileUnit = fileSetup.GetFileUnitForXmlSnippet(srcmlOutput, "sampletestmethods.cpp");
var scope = parser.ParseFileUnit(fileUnit);
var srcmlMethod = scope.GetDescendants <MethodDefinition>().First();
// Verify the method definition
Assert.IsInstanceOf <MethodDefinition>(srcmlMethod, "MethodDefinition found.");
Console.WriteLine(srcmlMethod.ToString());
// Extract SUnit Statements from MethodDefinition
var statements = new List <Statement>();
statements.AddRange(SUnitExtractor.ExtractEnding(srcmlMethod));
statements.AddRange(SUnitExtractor.ExtractSameAction(srcmlMethod));
statements.AddRange(SUnitExtractor.ExtractVoidReturn(srcmlMethod));
// verify the statements selected
Assert.IsNotEmpty(statements, "statements selected from method definition");
Console.WriteLine(statements.ToString());
// Translate Statements into SUnits
List <SUnit> sunits = statements.ConvertAll(
new Converter <Statement, SUnit> (SUnitTranslator.Translate));
// verify sunits have been translated
Assert.That(sunits.TrueForAll(s => s.action != null), "All SUnits initialized.");
Console.WriteLine(sunits.ToString());
// Generate text from SUnits
List <string> sentences = sunits.ConvertAll(
new Converter <SUnit, string> (TextGenerator.GenerateText));
// verify string generated
Assert.That(sentences.TrueForAll(s => s.Length > 0));
Console.WriteLine(sentences);
// Collect text and summarize
var methodDocument = String.Join <string>(" ", sentences);
var summary = Summarizer.Summarize(methodDocument);
// verify summary
Assert.That(!summary.Equals(""));
Console.WriteLine(summary);
}
private static IEnumerable <FullTextResult> ConvertSearchResultsToFullTextResults(ISearchResults SearchResults, Summarizer Summarizer, int PageNumber = 0)
{
IEnumerable <SearchResult> results;
results = SearchResults.AsEnumerable();
var numNodesInSet = 0;
var resultNumber = 0;
var allResults = new List <FullTextResult>();
var returnAllFieldsInXslt = Config.Instance.GetBooleanByKey("ReturnAllFieldsInXSLT");
foreach (var result in results)
{
//var resultNumber = toSkip + numNodesInSet + 1;
resultNumber++;
OnResultOutput(new ResultOutputEventArgs(result, PageNumber, resultNumber, numNodesInSet + 1));
var ftResult = new FullTextResult();
ftResult.ExamineResult = result;
ftResult.NodeId = result.Id;
ftResult.Score = result.Score;
ftResult.Number = resultNumber;
var nodeTypeAlias = result.Fields.ContainsKey("nodeTypeAlias") ? result.Fields["nodeTypeAlias"] : "[Need to add 'nodeTypeAlias' to <IndexAttributeFields>]";
ftResult.NodeTypeAlias = nodeTypeAlias;
//var node = new XElement("node",
// new XAttribute("id", ),
// new XAttribute("score", ),
// new XAttribute("number", )
//);
if (returnAllFieldsInXslt)
{
//Add all fields from index, you would think this would slow things
//down, but it doesn't (that much) really, could be useful
//foreach (var field in result.Fields)
//{
// ftResult.Fields.Add(field.Key, field.Value);
//}
ftResult.Fields = result.Fields;
}
//Add title (optionally highlighted)
string title;
Summarizer.GetTitle(result, out title);
ftResult.Title = title;
//Add Summary(optionally highlighted)
string summary;
Summarizer.GetSummary(result, out summary);
ftResult.Summary = summary;
allResults.Add(ftResult);
numNodesInSet++;
}
return(allResults);
}
/// <summary>
/// Take ISearchResults from examine, create title and body summary, and convert to an XML document
/// This is broadly based off the same function in the Examine codebase, the XML it returns should be
/// broadly compatible, that seems best...
/// </summary>
/// <returns>XPathNodeIterator to return to Umbraco XSLT foreach</returns>
static XPathNodeIterator ResultsAsXml(ISearchResults searchResults, Summarizer Summarizer, int pageNumber = 0, int pageLength = 0, Stopwatch stopwatch = null)
{
var output = new XDocument();
var numNodesInSet = 0;
var numResults = searchResults.TotalItemCount;
if (numResults < 1)
{
return(ReturnError("NoResults", "Your search returned no results"));
}
IEnumerable <SearchResult> results;
var toSkip = 0;
if (pageLength > 0)
{
if (pageNumber > 1)
{
toSkip = (pageNumber - 1) * pageLength;
}
results = searchResults.Skip(toSkip).Take(pageLength);
}
else
{
results = searchResults.AsEnumerable();
}
var rootNode = new XElement("results");
var nodesNode = new XElement("nodes");
var returnAllFieldsInXslt = Config.Instance.GetBooleanByKey("ReturnAllFieldsInXSLT");
foreach (var result in results)
{
var resultNumber = toSkip + numNodesInSet + 1;
OnResultOutput(new ResultOutputEventArgs(result, pageNumber, resultNumber, numNodesInSet + 1));
var node = new XElement("node",
new XAttribute("id", result.Id),
new XAttribute("score", result.Score),
new XAttribute("number", resultNumber)
);
if (returnAllFieldsInXslt)
{
//Add all fields from index, you would think this would slow things
//down, but it doesn't (that much) really, could be useful
foreach (var field in result.Fields)
{
node.Add(
new XElement("data",
new XAttribute("alias", field.Key),
new XCData(field.Value)
));
}
}
//Add title (optionally highlighted)
string title;
Summarizer.GetTitle(result, out title);
node.Add(
new XElement("data",
new XAttribute("alias", "FullTextTitle"),
new XCData(title)
));
//Add Summary(optionally highlighted)
string summary;
Summarizer.GetSummary(result, out summary);
node.Add(
new XElement("data",
new XAttribute("alias", "FullTextSummary"),
new XCData(summary)
));
nodesNode.Add(node);
numNodesInSet++;
}
if (numNodesInSet > 0)
{
rootNode.Add(nodesNode);
var summary = new XElement("summary");
summary.Add(new XAttribute("numResults", numResults));
var numPages = numResults % pageLength == 0 ? numResults / pageLength : numResults / pageLength + 1;
summary.Add(new XAttribute("numPages", numPages));
if (stopwatch != null)
{
stopwatch.Stop();
double millisecs = stopwatch.ElapsedMilliseconds;
var numSecs = Math.Round((millisecs / 1000), 3);
summary.Add(new XAttribute("timeTaken", numSecs));
}
summary.Add(new XAttribute("firstResult", toSkip + 1));
var lastResult = toSkip + pageLength;
if (lastResult > numResults)
{
lastResult = numResults;
}
summary.Add(new XAttribute("lastResult", lastResult));
rootNode.Add(summary);
output.Add(rootNode);
}
else
{
return(ReturnError("NoPage", "Pagination incorrectly set up, no results on page " + pageNumber));
}
return(output.CreateNavigator().Select("/"));
}
public void RequestTimeSeriesShouldReturnQuestionAndAnswers()
{
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, null, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
string site = string.Empty;
int userId = 0;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
TimeseriesResult actual;
actual = target.RequestTimeSeries(site, userId, start, end, Rollup.Weekly);
CollectionAssert.AreEqual(new []{"tag1", "tag2", "tag3", "tag4"}, actual.Timeseries.Keys);
Assert.IsTrue(actual.Timeseries["tag1"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(1, 2, 0) } }));
Assert.IsTrue(actual.Timeseries["tag2"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(2, 1, 0) } }));
Assert.IsTrue(actual.Timeseries["tag3"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(1, 0, 0) } }));
Assert.IsTrue(actual.Timeseries["tag4"].IsEqual(new Dictionary<DateTime, TimeseriesPeriod> { { new DateTime(2009, 06, 21), new TimeseriesPeriod(0, 1, 0) } }));
}
public void RequestTimeSeriesShouldSaveQuestions()
{
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, null, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
string site = string.Empty;
int userId = 0;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
TimeseriesResult actual;
actual = target.RequestTimeSeries(site, userId, start, end, Rollup.Weekly);
m_PersistQuestionsRepo.Verify(r => r.SaveQuestions(site, m_UserQuestionData), Times.Once());
}
public void Setup()
{
function = new TriangularFunction(0.0, 0.2);
fuzzySet = new FuzzySet(function);
summarizer = new Summarizer("YOUNG", fuzzySet);
}
public static void Main(string[] args)
{
if(args.Length==0)
{
Console.WriteLine(@"Usage: earlzplorer coin.conf rpcport command
Commands:
summarizeblocks begin-end
summarizeblocktransactions blocknum
getblockbynumber blocknum
dumpblockswithtx begin-end
dumpblockwithtx blocknum
richlist [untilblock]
richminers [begin-end] (defaults to last 10,000 blocks)
note: The best time can be had by redirecting output to a file, and then using a text editor to skim the file
");
return;
}
var config=File.ReadAllLines(args[0]);
string username=null;
string password=null;
int? port=null;
foreach(var eachline in config)
{
var line=eachline;
line=line.Trim();
if(line.StartsWith("rpcuser="******"rpcuser="******"rpcpassword="******"rpcpassword="******"rpcport="))
{
port=int.Parse(line.Substring("rpcport=".Length));
}
}
if(!port.HasValue)
{
port=int.Parse(args[1]);
}
var bc=new BitnetClient("http://127.0.0.1:"+port.Value, username, password);
var s=new Summarizer(bc);
Range r;
int num;
switch(args[2])
{
case "summarizeblocks":
r=ParseRange(args);
s.SummarizeBlocks(r.Begin,r.End);
break;
case "dumpblockswithtx":
r=ParseRange(args);
s.DumpBlocksWithTx(r.Begin, r.End);
break;
case "dumpblockwithtx":
num=int.Parse(args[3]);
s.DumpBlocksWithTx(num, num+1);
break;
default:
Console.WriteLine("Unknown command");
return;
}
}
private void button3_Click(object sender, EventArgs e)
{
string path = "E:\\Dropbox\\Masters\\myMSc\\PracticalPart\\Sematic_K-MEANSClustering\\MScDataSets\\Reuters21578\\00\\000.xml";
string path_data_docCollection = "E:\\Dropbox\\Masters\\myMSc\\PracticalPart\\Sematic_K-MEANSClustering\\MScDataSets\\Reuters21578\\data\\00_" + maxNoDoc + "_docCollection.xml";
string path_data_mainCLusterNodeList = "E:\\Dropbox\\Masters\\myMSc\\PracticalPart\\Sematic_K-MEANSClustering\\MScDataSets\\Reuters21578\\data\\00_" + maxNoDoc + "_mainCLusterNodeList.xml";
if (cboxDataSet.Text == "Reu_01")
{
path = "E:\\Dropbox\\Masters\\myMSc\\PracticalPart\\Sematic_K-MEANSClustering\\MScDataSets\\Reuters21578\\Reu_01\\reut2-000_small.xml";
path_data_docCollection = "E:\\Dropbox\\Masters\\myMSc\\PracticalPart\\Sematic_K-MEANSClustering\\MScDataSets\\Reuters21578\\data\\01_" + maxNoDoc + "_docCollection.xml";
path_data_mainCLusterNodeList = "E:\\Dropbox\\Masters\\myMSc\\PracticalPart\\Sematic_K-MEANSClustering\\MScDataSets\\Reuters21578\\data\\01_" + maxNoDoc + "_mainCLusterNodeList.xml";
}
else if (cboxDataSet.Text == "Re0")
{
}
else
{
//this is test ds so already assigned
}
if (File.Exists(path_data_docCollection) && File.Exists(path_data_mainCLusterNodeList))
{
docCollection.DocumentList = DeSerializeObject <List <string> >(path_data_docCollection);
mainCLusterNodeList = DeSerializeObject <List <ClusterNode> >(path_data_mainCLusterNodeList);
}
else
{
XmlDocument doc2 = new XmlDocument();
doc2.Load(path);
XmlNodeList nodelist = doc2.DocumentElement.SelectNodes("/main/REUTERS");
int n = 1;
int sentCount = 5;
foreach (XmlNode node in nodelist)
{
ClusterNode item = new ClusterNode();
item.cLusters = new List <string>();
XmlNode body = node.SelectSingleNode("TEXT/BODY");
if (body == null)
{
continue;
}
string text = body.InnerText;
if (body.InnerText.Length > 400)
{
SummarizerArguments sumargs = new SummarizerArguments
{
DictionaryLanguage = "en",
DisplayLines = sentCount,
DisplayPercent = 0,
InputFile = "",
InputString = body.InnerText
};
SummarizedDocument doc = Summarizer.Summarize(sumargs);
string summary = string.Join("\r\n\r\n", doc.Sentences.ToArray());
text = summary;
int len = summary.Length;
}
XmlNodeList nodelistPLACES = node.SelectNodes("PLACES/D");
foreach (XmlNode place in nodelistPLACES)
{
item.cLusters.Add(place.InnerText);
}
//XmlNodeList nodelistTOPICS = node.SelectNodes("TOPICS/D");
//foreach (XmlNode topic in nodelistTOPICS)
//{
// item.cLusters.Add(topic.InnerText);
//}
docCollection.DocumentList.Add(text);
item.id = n;
item.content = text;
//item.cLuster = cLuster.InnerXml;
mainCLusterNodeList.Add(item);
n++;
if (n > maxNoDoc)
{
break;
}
}
SerializeObject(docCollection.DocumentList, path_data_docCollection);
SerializeObject(mainCLusterNodeList, path_data_mainCLusterNodeList);
}
int totalDoc = 0;
if (int.TryParse(docCollection.DocumentList.Count.ToString(), out totalDoc))
{
lblTotalDoc.Text = totalDoc.ToString();
}
MessageBox.Show("Done");
}
public void RequestTimeSeriesShouldRequestUnknownQuestionIds()
{
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, null, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
string site = string.Empty;
int userId = 0;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
TimeseriesResult actual;
actual = target.RequestTimeSeries(site, userId, start, end, Rollup.Weekly);
m_questionRepo.Verify(q => q.Get(It.IsAny<string>(), It.Is<IEnumerable<int>>(v => v.IsEqual(new[] { 2345 }))), Times.Exactly(1));
}
/// <summary>
/// Take ISearchResults from examine, create title and body summary, and return a collection of SearchResultPages
/// </summary>
/// <param name="AltSearchResults">
/// Collection of search results.
/// </param>
/// <param name="AltSummarizer">
/// Summarizer to format results
/// </param>
/// <param name="PageNumber">
/// The current page Number
/// </param>
/// <param name="PageLength">
/// Num Results per page
/// </param>
private static AlternateSpellingsInfo UpdateAlternateResults(AlternateSpellingsInfo AltSpellingsInfo, ISearchResults AltSearchResults, Summarizer AltSummarizer, int PageNumber, int PageLength)
{
//Convert from Examine Results to FullText Results
var convertedResults = ConvertSearchResultsToFullTextResults(AltSearchResults, AltSummarizer, PageNumber).ToList();
AltSpellingsInfo.AllResults = convertedResults;
//Handle Paging
var allResults = convertedResults.ToList();
var totalResults = allResults.Count();
AltSpellingsInfo.TotalResults = totalResults;
var numPages = totalResults % PageLength == 0 ? totalResults / PageLength : totalResults / PageLength + 1;
AltSpellingsInfo.NumOfPages = numPages;
var allPages = new List <SearchResultPage>();
//divide results into pages
if (PageLength > 0)
{
var toSkip = 0;
for (int i = 0; i < numPages; i++)
{
var page = new SearchResultPage();
page.PageNum = i + 1;
toSkip = i * PageLength;
page.Results = allResults.Skip(toSkip).Take(PageLength);
page.ResultsOnPage = page.Results.Count();
page.FirstResult = toSkip + 1;
var lastResult = toSkip + PageLength;
if (lastResult > totalResults)
{
lastResult = totalResults;
}
page.LastResult = lastResult;
allPages.Add(page);
}
}
AltSpellingsInfo.Pages = allPages;
return(AltSpellingsInfo);
}
public void RequestTimeSeriesShouldNotCallQuestionRepoIfNoAnswers()
{
m_answerRepo = new Mock<IAnswersRepository>();
m_answerRepo.Setup(q => q.Get(It.IsAny<string>(), It.IsAny<int>())).Returns(new List<AnswerInfo>());
Summarizer target = new Summarizer(m_questionRepo.Object, m_answerRepo.Object, m_PersistQuestionsRepo.Object, null, m_commentsRepositoryMock.Object, m_commentsProcessorMock.Object);
string site = string.Empty;
int userId = 0;
Nullable<DateTime> start = new Nullable<DateTime>();
Nullable<DateTime> end = new Nullable<DateTime>();
TimeseriesResult actual;
actual = target.RequestTimeSeries(site, userId, start, end, Rollup.Weekly);
m_questionRepo.Verify(q => q.Get(It.IsAny<string>(), It.IsAny<IEnumerable<int>>()), Times.Never());
}
//TODO : Make summarizer send completion data , pass Bg worker as argument
private void BackgroundWorker_DoWork(object sender, DoWorkEventArgs e)
{
ThreadObjectHelper toh = (ThreadObjectHelper)e.Argument;
Summarizer.Summarize(toh._files.ToList(), toh._outputFolderPath, bgw, ColorTranslator.ToHtml(toh._identifierColour), ColorTranslator.ToHtml(toh._dataTypeColour));
}
/// <summary>
/// Summarizes an enumerable of double messages into summarized doubles.
/// </summary>
/// <typeparam name="TKey">The type of the series key.</typeparam>
/// <typeparam name="T">The type of messages to summarize.</typeparam>
/// <param name="messages">Enumerable of double messages.</param>
/// <param name="interval">The time interval each summary value should cover.</param>
/// <returns>List of summarized doubles.</returns>
internal static List <IntervalData <Dictionary <TKey, T> > > SeriesSummarizer <TKey, T>(IEnumerable <Message <Dictionary <TKey, T> > > messages, TimeSpan interval)
{
return(messages
.OrderBy(msg => msg.OriginatingTime)
.GroupBy(msg => Summarizer <T, T> .GetIntervalStartTime(msg.OriginatingTime, interval))
.Select(
group =>
{
var firstMessage = group.First();
var lastMessage = group.Last();
var min = new Dictionary <TKey, T>();
var max = new Dictionary <TKey, T>();
foreach (var message in group)
{
foreach (var kvp in message.Data)
{
// Update min
if (!min.ContainsKey(kvp.Key))
{
min.Add(kvp.Key, kvp.Value);
}
else if (kvp.Value is IComparable <T> comparable)
{
if (comparable.CompareTo(min[kvp.Key]) == -1)
{
min[kvp.Key] = kvp.Value;
}
}
else if (kvp.Value is IComparable untypedComparable)
{
if (untypedComparable.CompareTo(min[kvp.Key]) == -1)
{
min[kvp.Key] = kvp.Value;
}
}
else
{
throw new InvalidOperationException("Cannot summarize over values that are not comparable.");
}
// Update max
if (!max.ContainsKey(kvp.Key))
{
max.Add(kvp.Key, kvp.Value);
}
else if (kvp.Value is IComparable <T> comparable)
{
if (comparable.CompareTo(max[kvp.Key]) == 1)
{
max[kvp.Key] = kvp.Value;
}
}
else if (kvp.Value is IComparable untypedComparable)
{
if (untypedComparable.CompareTo(max[kvp.Key]) == 1)
{
max[kvp.Key] = kvp.Value;
}
}
else
{
throw new InvalidOperationException("Cannot summarize over values that are not comparable.");
}
}
}
// Use the last value as representative for summarization, with the first message
// originating time.
return IntervalData.Create(
value: lastMessage.Data,
minimum: min,
maximum: max,
originatingTime: firstMessage.OriginatingTime,
interval: lastMessage.OriginatingTime - firstMessage.OriginatingTime);
}).ToList());
}