public override void Compute()
        {
            var datasetsLocations = ReadDatasetsLocations();

            TLDatasetsList listOfDatasets = new TLDatasetsList();

            foreach (DatasetLocation locations in datasetsLocations)
            {
                TLDataset dataset = new TLDataset(locations.DatasetName);

                string error;

                //do validation
                if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Source Artifacts File", out error))
                {
                    dataset.SourceArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.SourceArtifactsLocation, m_config.TrimElementValues);
                    Logger.Info(String.Format("Source artifacts imported from {0}.", locations.SourceArtifactsLocation));
                }
                else
                {
                    throw new ComponentException(error);
                }

                //do validation
                if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error))
                {
                    dataset.TargetArtifacts = CoestDatasetImporterHelper.ImportArtifacts(locations.TargetArtifactsLocation, m_config.TrimElementValues);
                    Logger.Info(String.Format("Target artifacts imported from {0}.", locations.TargetArtifactsLocation));
                }
                else
                {
                    throw new ComponentException(error);
                }

                //do validation
                if (CoestDatasetImporterHelper.ValidatePath(locations.SourceArtifactsLocation, "Target Artifacts File", out error))
                {
                    dataset.AnswerSet = CoestDatasetImporterHelper.ImportAnswerSet(locations.AnswerSetLocation, dataset.SourceArtifacts, locations.SourceArtifactsLocation, dataset.TargetArtifacts, locations.TargetArtifactsLocation, Logger, m_config.TrimElementValues);
                    Logger.Info(String.Format("Answer set imported from {0}.", locations.AnswerSetLocation));
                }
                else
                {
                    throw new ComponentException(error);
                }

                listOfDatasets.Add(dataset);
            }

            Workspace.Store("listOfDatasets", listOfDatasets);
            Workspace.Store("numberOfDatasets", listOfDatasets.Count);
        }
        public BoxSummaryData Calculate(SingleTracingResults singleTechniqueResults, TLDataset dataset)
        {
            SortedDictionary<string, double> intermediateResults = m_metricComputation.Calculate(singleTechniqueResults.ResultMatrix, dataset);

            double[] dataPoints = intermediateResults.Values.ToArray();
            if (dataPoints.Length == 0 && m_logger != null)
            {
                m_logger.Warn("Metric computation of '" + m_metricName + "' returned zero matching results for " + dataset.Name + " for one of the techniques. It may be valid results, but it may also mean that there is mismatch of ids in the answer matrix and corresponding artifacts." );
            }

            var summaryData = new BoxSummaryData(m_metricName, m_metricDescription);
            summaryData.AddPoint(new BoxPlotPoint(dataPoints));

            return summaryData;
        }
        public override void Compute()
        {
            // Loading artifacts & datasets from workspace
            TLArtifactsCollection sourceArtifacts = (TLArtifactsCollection)Workspace.Load("sourceArtifacts");
            TLArtifactsCollection targetArtifacts = (TLArtifactsCollection)Workspace.Load("targetArtifacts");
            TLSimilarityMatrix answerMatrix = (TLSimilarityMatrix)Workspace.Load("answerMatrix");
            TLSimilarityMatrix similarityMatrix = (TLSimilarityMatrix)Workspace.Load("similarityMatrix");

            // Checking for null arguments
            if (sourceArtifacts == null)
            {
                throw new ComponentException("The loaded source artifacts cannot be null!");
            }
            if (targetArtifacts == null)
            {
                throw new ComponentException("The loaded target artifacts cannot be null!");
            }
            if (answerMatrix == null)
            {
                throw new ComponentException("The loaded answer matrix cannot be null!");
            }
            if (similarityMatrix == null)
            {
                throw new ComponentException("The loaded similarity matrix cannot be null!");
            }

            // Results calculation
            TLDatasetsList datasets = new TLDatasetsList();
            var dataset = new TLDataset("Experiment results");
            dataset.AnswerSet = answerMatrix;
            dataset.SourceArtifacts = sourceArtifacts;
            dataset.TargetArtifacts = targetArtifacts;
            datasets.Add(dataset);

            TLSimilarityMatricesCollection similarityMatrices = new TLSimilarityMatricesCollection();
            similarityMatrix.Name = "Experiment results";
            similarityMatrices.Add(similarityMatrix);

            MetricComputationEngine engine = new MetricComputationEngine(datasets, Logger, m_config);

            //wrap result similarity matrix into TracingResult
            var tracingResults = GroupOfTracingResults<SingleTracingResults>.Adapt(similarityMatrices, "Experiment results");
            engine.AddTracingResults(tracingResults);
            var results = engine.ComputeResults();

            // Store the results in the workspace
            Workspace.Store("results", results);
        }
        public SortedDictionary<string, double> Calculate(TLSimilarityMatrix resultMatrix, TLDataset dataset)
        {
            var answerSet = dataset.AnswerSet;
            var sourceArtifacts = dataset.SourceArtifacts;

            SortedDictionary<string, double> metricValues = new SortedDictionary<string, double>();

            resultMatrix.Threshold = m_threshold;

            foreach (TLArtifact sourceArtifact in sourceArtifacts.Values)
            {
                int numberOfRelevant = answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id);

                double recall = 0.0;
                if (numberOfRelevant > 0)
                {
                    TLLinksList resultsListForArtifact = resultMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact.Id);
                    resultsListForArtifact.Sort();

                    int numberOfCorrectlyRetrieved = 0;

                    foreach (TLSingleLink link in resultsListForArtifact)
                    {
                        //check if this is relevant link
                        if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
                        {
                            numberOfCorrectlyRetrieved++;
                        }
                    }

                    recall = (double)numberOfCorrectlyRetrieved / numberOfRelevant;
                    metricValues.Add(sourceArtifact.Id, recall);
                }
            }

            resultMatrix.Threshold = 0.0;

            return metricValues;
        }
        public SortedDictionary<string, double> Calculate(TLSimilarityMatrix resultMatrix, TLDataset dataset)
        {
            var answerSet = dataset.AnswerSet;
            var sourceArtifacts = dataset.SourceArtifacts;

            SortedDictionary<string, double> metricValues = new SortedDictionary<string, double>();
            
            foreach (TLArtifact sourceArtifact in sourceArtifacts.Values)
            {
                int totalNumberOfCorrectLinks = answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id);

                double precision = 0.0;
                resultMatrix.Threshold = 0.0;
                TLLinksList resultsListForArtifact = resultMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact.Id);
                resultsListForArtifact.Sort();

                int numberOfCorrectlyRetrieved = 0;
                int numberOfRetrieved = 0;
                double scoreOfLastCorrectLink = 0;
                bool foundLastCorrectLink = false;
                foreach (TLSingleLink link in resultsListForArtifact)
                {
                    numberOfRetrieved++;
                    
                    //if all correct links has not been found yet
                    if (foundLastCorrectLink == false)
                    {
                        //check if this is relevant link
                        if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
                        {
                            numberOfCorrectlyRetrieved++;
                            if (numberOfCorrectlyRetrieved == totalNumberOfCorrectLinks)
                            {
                                foundLastCorrectLink = true;
                                scoreOfLastCorrectLink = answerSet.GetScoreForLink(link.SourceArtifactId, link.TargetArtifactId);
                            }
                        }
                    } 
                    else if (foundLastCorrectLink)
                    {
                        //if all correct link were found
                        // retrieve all the documents that have the same relevance score as the document with the last correct link
                        double score = answerSet.GetScoreForLink(link.SourceArtifactId, link.TargetArtifactId);
                        if (!score.Equals(scoreOfLastCorrectLink)) break;
                    }
                }

                if (numberOfCorrectlyRetrieved != totalNumberOfCorrectLinks)
                {
                    //if number of correctly retrieved links is not equal once results list was exhausted,
                    //it means there are some links not retrieved with probability zero. the precision is calculated by taking all target documents count 
                    //because then also all documents with probability zero would have to be retrieved
                    precision = (double)totalNumberOfCorrectLinks / dataset.TargetArtifacts.Count;
                    metricValues.Add(sourceArtifact.Id, precision);

                } else if (numberOfRetrieved > 0)
                {
                    precision = (double)numberOfCorrectlyRetrieved / numberOfRetrieved;
                    metricValues.Add(sourceArtifact.Id, precision);
                }
            }
         
            return metricValues;
        }
        public SortedDictionary<string, double> Calculate(TLSimilarityMatrix resultMatrix, TLDataset dataset)
        {
            var answerSet = dataset.AnswerSet;
            var sourceArtifacts = dataset.SourceArtifacts;

            SortedDictionary<string, double> metricValues = new SortedDictionary<string, double>();

            foreach (TLArtifact sourceArtifact in sourceArtifacts.Values)
            {
                int numberOfRelevant = answerSet.GetCountOfLinksAboveThresholdForSourceArtifact(sourceArtifact.Id); //??
                
                double averagePrecision = 0.0;
                
                //do calculation only if there are relevant links
                if (numberOfRelevant > 0)
                {
                    TLLinksList resultsListForArtifact = resultMatrix.GetLinksAboveThresholdForSourceArtifact(sourceArtifact.Id);
                    resultsListForArtifact.Sort();

                    int numRetrieved = 0;
                    int numCorrectlyRetrieved = 0;
                    double sumPrecision = 0;
                    int numSameRankPosition = 1;
                    int sumSameRankPosition = 0;
                    bool hasCorrectlyRetrieved = false;
                    double lastSimilarityScore = -1;
                    foreach (TLSingleLink link in resultsListForArtifact)
                    {
                        numRetrieved++;
                        if (link.Score != lastSimilarityScore)
                        {
                            if (hasCorrectlyRetrieved)
                            {
                                double averageRankPosition = (double)sumSameRankPosition / numSameRankPosition;
                                sumPrecision += (double)numCorrectlyRetrieved / averageRankPosition;
                            }
                            numSameRankPosition = 1;
                            sumSameRankPosition = numRetrieved;
                            hasCorrectlyRetrieved = false;
                        }
                        else
                        {
                            numSameRankPosition++;
                            sumSameRankPosition += numRetrieved;
                        }
                        if (answerSet.IsLinkAboveThreshold(link.SourceArtifactId, link.TargetArtifactId))
                        {
                            numCorrectlyRetrieved++;
                            hasCorrectlyRetrieved = true;
                        }
                        lastSimilarityScore = link.Score;
                    }
                    if (hasCorrectlyRetrieved)
                    {
                        double averageRankPosition = sumSameRankPosition / numSameRankPosition;
                        sumPrecision += (double)numCorrectlyRetrieved / averageRankPosition;
                    }

                    averagePrecision = (double)sumPrecision / numberOfRelevant;
                    metricValues.Add(sourceArtifact.Id, averagePrecision);
                }
            }

            return metricValues;
        }