public DataView GetTLView(int terminalID, string sortBy) { //Get a view of TLs for the specified terminal TLDataset tls = new TLDataset(); DataView view = tls.TLTable.DefaultView; TLViewerService2Client client = new TLViewerService2Client(); try { if (terminalID > 0) { DataSet ds = client.GetTLView2(terminalID); if (ds != null) { tls.Merge(ds); for (int i = 0; i < tls.TLTable.Rows.Count; i++) { tls.TLTable[i].TerminalID = terminalID; } if (sortBy.Trim().Length == 0) { sortBy = "TLNumber"; } view.Sort = sortBy; } } client.Close(); } catch (TimeoutException te) { client.Abort(); throw new ApplicationException(te.Message); } catch (FaultException <TLViewerFault> tle) { client.Abort(); throw new ApplicationException(tle.Detail.Message); } catch (FaultException fe) { client.Abort(); throw new ApplicationException(fe.Message); } catch (CommunicationException ce) { client.Abort(); throw new ApplicationException(ce.Message); } return(view); }
public DataView GetAgentSummary(int terminalID, string sortBy) { //Get an agent summary view for the specified terminal TLDataset tls = new TLDataset(); DataView view = tls.TLTable.DefaultView; TLViewerService2Client client = new TLViewerService2Client(); try { DataSet ds = client.GetAgentSummary2(terminalID); if (ds != null) { tls.Merge(ds); if (sortBy.Trim().Length == 0) { sortBy = "AgentNumber"; } view.Sort = sortBy; } client.Close(); } catch (TimeoutException te) { client.Abort(); throw new ApplicationException(te.Message); } catch (FaultException <TLViewerFault> tle) { client.Abort(); throw new ApplicationException(tle.Detail.Message); } catch (FaultException fe) { client.Abort(); throw new ApplicationException(fe.Message); } catch (CommunicationException ce) { client.Abort(); throw new ApplicationException(ce.Message); } return(view); }
public override void Compute() { TLDatasetsList listOfDatasets = (TLDatasetsList)Workspace.Load("listOfDatasets"); if (listOfDatasets == null) { throw new ComponentException("Received null listOfDatasets"); } if (listOfDatasets.Count == 0) { throw new ComponentException("Received empty listOfDatasets"); } int index = (int)Workspace.Load("indexOfDataset"); if (index < 0) { throw new ComponentException("Received negative indexOfDataset"); } if (index > listOfDatasets.Count) { throw new ComponentException("Received indexOfDataset is higher that number of datasets in the received list."); } TLDataset dataset = listOfDatasets[index]; Workspace.Store("sourceArtifacts", dataset.SourceArtifacts); Workspace.Store("targetArtifacts", dataset.TargetArtifacts); Workspace.Store("answerSet", dataset.AnswerSet); Workspace.Store("datasetName", dataset.Name); Logger.Info(String.Format("Loaded dataset '{0}' source artifacts collection (id:{1}), and target artifacts collection (id:{2})", dataset.Name, dataset.SourceArtifacts.CollectionId, dataset.TargetArtifacts.CollectionId)); }
public override void Compute() { // Loading artifacts & datasets from workspace TLArtifactsCollection sourceArtifacts = (TLArtifactsCollection)Workspace.Load("sourceArtifacts"); TLArtifactsCollection targetArtifacts = (TLArtifactsCollection)Workspace.Load("targetArtifacts"); TLSimilarityMatrix answerMatrix = (TLSimilarityMatrix)Workspace.Load("answerMatrix"); TLSimilarityMatrix similarityMatrix = (TLSimilarityMatrix)Workspace.Load("similarityMatrix"); // Checking for null arguments if (sourceArtifacts == null) { throw new ComponentException("The loaded source artifacts cannot be null!"); } if (targetArtifacts == null) { throw new ComponentException("The loaded target artifacts cannot be null!"); } if (answerMatrix == null) { throw new ComponentException("The loaded answer matrix cannot be null!"); } if (similarityMatrix == null) { throw new ComponentException("The loaded similarity matrix cannot be null!"); } // Results calculation TLDatasetsList datasets = new TLDatasetsList(); var dataset = new TLDataset("Experiment results"); dataset.AnswerSet = answerMatrix; dataset.SourceArtifacts = sourceArtifacts; dataset.TargetArtifacts = targetArtifacts; datasets.Add(dataset); TLSimilarityMatricesCollection similarityMatrices = new TLSimilarityMatricesCollection(); similarityMatrix.Name = "Experiment results"; similarityMatrices.Add(similarityMatrix); MetricComputationEngine engine = new MetricComputationEngine(datasets, Logger, m_config); //wrap result similarity matrix into TracingResult var tracingResults = GroupOfTracingResults <SingleTracingResults> .Adapt(similarityMatrices, "Experiment results"); engine.AddTracingResults(tracingResults); var results = engine.ComputeResults(); // Store the results in the workspace Workspace.Store("results", results); }
/// <summary> /// Computes the specified tracing results. /// </summary> /// <param name="tracingResults">The tracing results.</param> /// <param name="dataset">The dataset.</param> /// <returns></returns> public override Metric Compute(SingleTracingResults tracingResults, TLDataset dataset) { LineSeries precisionRecallCurve = new LineSeries(MetricName, MetricDescription); //only if tracing results are not null... if (tracingResults != null) { var resultMatrix = tracingResults.ResultMatrix; var answerSet = dataset.AnswerSet; var sourceArtifacts = dataset.SourceArtifacts; TLLinksList resultLinks = resultMatrix.AllLinks; resultLinks.Sort(); int numberOfRelevant = 0; foreach (TLArtifact sourceArtifact in sourceArtifacts.Values) { numberOfRelevant += answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); } //add point only if number of relevant and number of retrieved links are greater than 0 //basically don't allow division by 0 if (numberOfRelevant == 0) { m_logger.Warn("Number of relevant links is 0, thus the recall value cannot be computed for the Precision Recall Curve"); } else { int numberOfCorrectlyRetrieved = 0; int numberOfRetrieved = 0; foreach (TLSingleLink link in resultLinks) { numberOfRetrieved++; //check if this is a relevant link if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) { numberOfCorrectlyRetrieved++; } double recall = (double)numberOfCorrectlyRetrieved / numberOfRelevant; //don't need to check if number of retrieved is greater than 0 as it is always the case double precision = (double)numberOfCorrectlyRetrieved / numberOfRetrieved; precisionRecallCurve.AddPoint(new Point(recall, precision)); } } } return(precisionRecallCurve); }
public override void Compute() { var datasetsLocations = ReadDatasetsLocations(); TLDatasetsList listOfDatasets = new TLDatasetsList(); foreach (DatasetLocation locations in datasetsLocations) { TLDataset dataset = new TLDataset(locations.DatasetName); string error; //do validation if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Source Artifacts File", out error)) { dataset.SourceArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.SourceArtifactsLocation, m_config.TrimElementValues); Logger.Info(String.Format("Source artifacts imported from {0}.", locations.SourceArtifactsLocation)); } else { throw new ComponentException(error); } //do validation if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error)) { dataset.TargetArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.TargetArtifactsLocation, m_config.TrimElementValues); Logger.Info(String.Format("Target artifacts imported from {0}.", locations.TargetArtifactsLocation)); } else { throw new ComponentException(error); } //do validation if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error)) { dataset.AnswerSet = CoestDatasetImporterHelper.ImportAnswerSet(locations.AnswerSetLocation, dataset.SourceArtifacts, locations.SourceArtifactsLocation, dataset.TargetArtifacts, locations.TargetArtifactsLocation, Logger, m_config.TrimElementValues); Logger.Info(String.Format("Answer set imported from {0}.", locations.AnswerSetLocation)); } else { throw new ComponentException(error); } listOfDatasets.Add(dataset); } Workspace.Store("listOfDatasets", listOfDatasets); Workspace.Store("numberOfDatasets", listOfDatasets.Count); }
public TLDataset GetTLs(string terminal, DateTime startDate, DateTime endDate) { //Event handler for change in selected terminal TLDataset tls = new TLDataset(); try { DataSet ds = FillDataset(USP_TLS, TBL_TLS, new object[] { terminal, startDate.ToString("yyyy-MM-dd"), endDate.ToString("yyyy-MM-dd") }); if (ds.Tables[TBL_TLS] != null) { tls.Merge(ds); } } catch (ApplicationException ex) { throw ex; } catch (Exception ex) { throw new ApplicationException("Unexpected exception creating TL list.", ex); } return(tls); }
public BoxSummaryData Calculate(SingleTracingResults singleTechniqueResults, TLDataset dataset) { SortedDictionary <string, double> intermediateResults = m_metricComputation.Calculate(singleTechniqueResults.ResultMatrix, dataset); double[] dataPoints = intermediateResults.Values.ToArray(); if (dataPoints.Length == 0 && m_logger != null) { m_logger.Warn("Metric computation of '" + m_metricName + "' returned zero matching results for " + dataset.Name + " for one of the techniques. It may be valid results, but it may also mean that there is mismatch of ids in the answer matrix and corresponding artifacts."); } var summaryData = new BoxSummaryData(m_metricName, m_metricDescription); summaryData.AddPoint(new BoxPlotPoint(dataPoints)); return(summaryData); }
public TLDataset FindTL(string terminal, string TLNumber) { // TLDataset tls = new TLDataset(); try { DataSet ds = FillDataset(USP_TLS_FIND, TBL_TLS, new object[] { terminal, TLNumber }); if (ds.Tables[TBL_TLS] != null) { tls.Merge(ds); } } catch (ApplicationException ex) { throw ex; } catch (Exception ex) { throw new ApplicationException("Unexpected exception finding TL.", ex); } return(tls); }
public SortedDictionary <string, double> Calculate(TLSimilarityMatrix resultMatrix, TLDataset dataset) { var answerSet = dataset.AnswerSet; var sourceArtifacts = dataset.SourceArtifacts; SortedDictionary <string, double> metricValues = new SortedDictionary <string, double>(); resultMatrix.Threshold = m_threshold; foreach (TLArtifact sourceArtifact in sourceArtifacts.Values) { int numberOfRelevant = answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); double recall = 0.0; if (numberOfRelevant > 0) { TLLinksList resultsListForArtifact = resultMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); resultsListForArtifact.Sort(); int numberOfCorrectlyRetrieved = 0; foreach (TLSingleLink link in resultsListForArtifact) { //check if this is relevant link if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) { numberOfCorrectlyRetrieved++; } } recall = (double)numberOfCorrectlyRetrieved / numberOfRelevant; metricValues.Add(sourceArtifact.Id, recall); } } resultMatrix.Threshold = 0.0; return(metricValues); }
public double Compare(SingleTracingResults techniqueOneResults, SingleTracingResults techniqueTwoResults, TLDataset dataset) { return(s_rankTest.Compare(techniqueOneResults, techniqueTwoResults, dataset)); }
public override Metric Compute(SingleTracingResults tracingResults, TLDataset dataset) { SummaryStatistics precisionComputation = new SummaryStatistics(MetricName, MetricDescription, new PrecisionAtRecall100(), m_logger); return(precisionComputation.Calculate(tracingResults, dataset)); }
/// <summary> /// Computes the specified metric for given tracing results. /// It provided tracing results are equal default(T) - usually null, /// then Compute method still must return Metric that is not null. /// Metric needs to return either LineSeries or BoxSummaryData (possibly there are going to be more options in future) /// </summary> /// <param name="tracingResults">The tracing results.</param> /// <returns></returns> public abstract Metric Compute(T tracingResults, TLDataset dataset);
private List <double> PrepareDataForWilcox(SingleTracingResults techniqueOneResults, SingleTracingResults techniqueTwoResults, TLDataset dataset) { var avePrec1 = (SortedDictionary <string, double>)m_baseMetricComputation.Calculate(techniqueOneResults.ResultMatrix, dataset); var avePrec2 = (SortedDictionary <string, double>)m_baseMetricComputation.Calculate(techniqueTwoResults.ResultMatrix, dataset); var deltas = ScoreComputationHelper.Delta(avePrec1, avePrec2); return(deltas.Values.ToList()); }
public double Compare(SingleTracingResults techniqueOneResults, SingleTracingResults techniqueTwoResults, TLDataset dataset) { List <double> differencesResults = PrepareDataForWilcox(techniqueOneResults, techniqueTwoResults, dataset); return(Wilcox(differencesResults.ToArray())); }
public SortedDictionary <string, double> Calculate(TLSimilarityMatrix resultMatrix, TLDataset dataset) { var answerSet = dataset.AnswerSet; var sourceArtifacts = dataset.SourceArtifacts; SortedDictionary <string, double> metricValues = new SortedDictionary <string, double>(); foreach (TLArtifact sourceArtifact in sourceArtifacts.Values) { int numberOfRelevant = answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); //?? double averagePrecision = 0.0; //do calculation only if there are relevant links if (numberOfRelevant > 0) { TLLinksList resultsListForArtifact = resultMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); resultsListForArtifact.Sort(); int numRetrieved = 0; int numCorrectlyRetrieved = 0; double sumPrecision = 0; int numSameRankPosition = 1; int sumSameRankPosition = 0; bool hasCorrectlyRetrieved = false; double lastSimilarityScore = -1; foreach (TLSingleLink link in resultsListForArtifact) { numRetrieved++; if (link.Score != lastSimilarityScore) { if (hasCorrectlyRetrieved) { double averageRankPosition = (double)sumSameRankPosition / numSameRankPosition; sumPrecision += (double)numCorrectlyRetrieved / averageRankPosition; } numSameRankPosition = 1; sumSameRankPosition = numRetrieved; hasCorrectlyRetrieved = false; } else { numSameRankPosition++; sumSameRankPosition += numRetrieved; } if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) { numCorrectlyRetrieved++; hasCorrectlyRetrieved = true; } lastSimilarityScore = link.Score; } if (hasCorrectlyRetrieved) { double averageRankPosition = sumSameRankPosition / numSameRankPosition; sumPrecision += (double)numCorrectlyRetrieved / averageRankPosition; } averagePrecision = (double)sumPrecision / numberOfRelevant; metricValues.Add(sourceArtifact.Id, averagePrecision); } } return(metricValues); }
public SortedDictionary <string, double> Calculate(TLSimilarityMatrix resultMatrix, TLDataset dataset) { var answerSet = dataset.AnswerSet; var sourceArtifacts = dataset.SourceArtifacts; SortedDictionary <string, double> metricValues = new SortedDictionary <string, double>(); foreach (TLArtifact sourceArtifact in sourceArtifacts.Values) { int totalNumberOfCorrectLinks = answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); double precision = 0.0; resultMatrix.Threshold = 0.0; TLLinksList resultsListForArtifact = resultMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); resultsListForArtifact.Sort(); int numberOfCorrectlyRetrieved = 0; int numberOfRetrieved = 0; double scoreOfLastCorrectLink = 0; bool foundLastCorrectLink = false; foreach (TLSingleLink link in resultsListForArtifact) { numberOfRetrieved++; //if all correct links has not been found yet if (foundLastCorrectLink == false) { //check if this is relevant link if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId)) { numberOfCorrectlyRetrieved++; if (numberOfCorrectlyRetrieved == totalNumberOfCorrectLinks) { foundLastCorrectLink = true; scoreOfLastCorrectLink = answerSet.GetScoreForLink(link.SourceArtifactId, link.TargetArtifactId); } } } else if (foundLastCorrectLink) { //if all correct link were found // retrieve all the documents that have the same relevance score as the document with the last correct link double score = answerSet.GetScoreForLink(link.SourceArtifactId, link.TargetArtifactId); if (!score.Equals(scoreOfLastCorrectLink)) { break; } } } if (numberOfCorrectlyRetrieved != totalNumberOfCorrectLinks) { //if number of correctly retrieved links is not equal once results list was exhausted, //it means there are some links not retrieved with probability zero. the precision is calculated by taking all target documents count //because then also all documents with probability zero would have to be retrieved precision = (double)totalNumberOfCorrectLinks / dataset.TargetArtifacts.Count; metricValues.Add(sourceArtifact.Id, precision); } else if (numberOfRetrieved > 0) { precision = (double)numberOfCorrectlyRetrieved / numberOfRetrieved; metricValues.Add(sourceArtifact.Id, precision); } } return(metricValues); }
public override Metric Compute(SingleTracingResults tracingResults, TLDataset dataset) { SummaryStatistics recallComputation = new SummaryStatistics(MetricName, MetricDescription, new Recall(m_threshold), m_logger); return(recallComputation.Calculate(tracingResults, dataset)); }