/// <summary>
/// Execute the FindRelated search, create and populate the tables
/// </summary>
/// <param name="odbcDsn">ODBC DSN to access the SQL server</param>
/// <param name="relatedTableName">Name of the FindRelated SQL table to create</param>
/// <param name="inputFileInfo">FileInfo object with information about the input CSV file</param>
/// <param name="resume">True if resuming a previous run</param>
/// <param name="outputFilename">Output filename</param>
public void Go(
string odbcDsn,
string relatedTableName,
FileInfo inputFileInfo,
bool resume,
string outputFilename)
{
if (NCBI.ApiKeyExists)
{
Trace.WriteLine("Using API key: " + NCBI.ApiKeyPath);
}
else
{
Trace.WriteLine("Performance is limited to under 3 requests per second.");
Trace.WriteLine("Consider pasting an API key into " + NCBI.ApiKeyPath);
Trace.WriteLine("Or set the NCBI_API_KEY_FILE environemnt variable to the API key file path");
Trace.WriteLine("For more information, see https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/");
}
Trace.WriteLine($"Requesting up to {ELINK_IDS_PER_REQUEST} PMIDs per NCBI eLink API request");
Database db = new Database(odbcDsn);
string queueTableName = relatedTableName + "_queue";
InputQueue inputQueue;
if (!resume)
{
if (!CreateOutputFile(outputFilename))
{
return;
}
CreateTables(db, relatedTableName, queueTableName);
inputQueue = new InputQueue(inputFileInfo, db, queueTableName);
}
else
{
inputQueue = new InputQueue(db, queueTableName);
}
while (inputQueue.Next(ELINK_IDS_PER_REQUEST) > 0)
{
BackgroundWorker.ReportProgress((100 * inputQueue.Progress) / inputQueue.TotalPmidsAdded);
Trace.WriteLine($"{DateTime.Now} - executing API query for related articles for {inputQueue.CurrentPmids.Count()} PMIDs");
// Do the linked publication search for the author's PMIDs and process the results.
// This returns a Dictionary that maps author publications (from the PeoplePublications table)
// to linked publications, so each key is one of the author publications read from the DB originally.
string xml = null;
bool success = false;
bool failed = false;
while (!success)
{
try
{
xml = ExecuteRelatedSearch(inputQueue.CurrentPmids);
success = true;
}
catch (Exception ex)
{
Trace.WriteLine(DateTime.Now + " - an error occurred while executing the related query, attempting to repeat the search");
Trace.WriteLine(ex.Message);
Trace.WriteLine(ex.StackTrace);
failed = true;
}
}
if (failed)
{
Trace.WriteLine(DateTime.Now + " - successfully recovered from the error, continuing execution");
}
else
{
Dictionary <int, List <int> > relatedSearchResults = GetIdsFromXml(xml, out Dictionary <int, Dictionary <int, RankAndScore> > relatedRanks);
bool completed;
completed = WriteRelatedRanksToOutputFileAndDatabase(db, relatedTableName, relatedSearchResults, relatedRanks, outputFilename, inputQueue);
if (!completed) // WriteRelatedRankToOutputFile() returns false if the user stopped the operation
{
break;
}
}
}
BackgroundWorker.ReportProgress(100);
}
/// <summary>
/// Go through all of the ranks and scores retrieved from the server for each PMID and write them to the output file and the database.
/// </summary>
/// <param name="db">Database to write to</param>
/// <param name="relatedTableName">Name of the related table</param>
/// <param name="relatedSearchResults">NCBI search results parsed into a dictionary that maps queried PMIDs to a list of related PMIDs</param>
/// <param name="relatedRanks">Dictionary parsed from NCBI search results that maps each queried PMID to a dictionary of related PMIDs and their ranks and scores</param>
/// <param name="outputFilename">Output file to append to</param>
/// <param name="inputQueue">Input queue for marking success or error</param>
/// <returns>True if a lines were successfully added to the file and table, false if an error occurred</returns>
private bool WriteRelatedRanksToOutputFileAndDatabase(Database db, string relatedTableName,
Dictionary <int, List <int> > relatedSearchResults, Dictionary <int, Dictionary <int, RankAndScore> > relatedRanks,
string outputFilename, InputQueue inputQueue)
{
if (BackgroundWorker != null && BackgroundWorker.CancellationPending)
{
Trace.WriteLine(DateTime.Now + " - stopped");
return(false);
}
foreach (int pmid in relatedSearchResults.Keys)
{
List <int> relatedPmids = relatedSearchResults[pmid];
if (relatedPmids == null)
{
Trace.WriteLine($"{DateTime.Now} - found empty related PMID list for PMID {pmid} ({++_pmidsProcessed} of {inputQueue.TotalPmidsAdded})");
}
else if (!relatedRanks.ContainsKey(pmid))
{
Trace.WriteLine($"{DateTime.Now} - no ranks or scores found for PMID {pmid} ({++_pmidsProcessed} of {inputQueue.TotalPmidsAdded})");
}
else
{
Trace.WriteLine($"{DateTime.Now} - found {relatedPmids.Count} results for PMID {pmid} ({++_pmidsProcessed} of {inputQueue.TotalPmidsAdded})");
Dictionary <int, RankAndScore> ranksAndScores = relatedRanks[pmid];
foreach (int relatedPmid in relatedPmids)
{
if (!ranksAndScores.ContainsKey(relatedPmid))
{
Trace.WriteLine(DateTime.Now + " - unable to find related ranks and scores for PMID " + pmid + ", related PMID " + relatedPmid);
}
else
{
RankAndScore rankAndScore = ranksAndScores[relatedPmid];
string line = String.Format("{0},{1},{2},{3}", pmid, relatedPmid, rankAndScore.Rank, rankAndScore.Score);
string output = line + Environment.NewLine;
try
{
File.AppendAllText(outputFilename, output);
}
catch (Exception ex)
{
Trace.WriteLine(DateTime.Now + " - unable to append '" + line + "' to the output file: " + ex.Message);
Trace.WriteLine(ex.StackTrace);
Trace.WriteLine(DateTime.Now + " - stopping the run, use the Resume button to resume");
inputQueue.MarkError(pmid);
return(false);
}
bool written = WriteRelatedRankToDatabase(db, relatedTableName, pmid, relatedPmid, rankAndScore.Rank, rankAndScore.Score);
if (!written)
{
return(false);
}
}
}
}
// Mark the PMID processed in the queue
inputQueue.MarkProcessed(pmid);
}
return(true);
}
