public void TestFileLoggerConstructor(string logFileNameBase, string expectedBaseName)
{
FileLogger.ResetLogFileName();
var logger = new FileLogger(logFileNameBase, BaseLogger.LogLevels.INFO);
logger.Info("Info message");
// This debug message won't appear in the log file because the LogLevel is INFO
logger.Debug("Debug message");
if (string.IsNullOrWhiteSpace(logFileNameBase))
{
ProgRunner.SleepMilliseconds(500);
FileLogger.FlushPendingMessages();
}
Console.WriteLine();
Console.WriteLine("Log file path: " + FileLogger.LogFilePath);
var expectedName = expectedBaseName + "_" + DateTime.Now.ToString("MM-dd-yyyy") + FileLogger.LOG_FILE_EXTENSION;
if (!FileLogger.LogFilePath.EndsWith(expectedName))
{
Assert.Fail("Log file name was not in the expected format of " + expectedName + "; see " + FileLogger.LogFilePath);
}
}
private static int ProcessSQLiteDB(FileInfo fiSourceFile, string sourceTableName)
{
try
{
var runner = new Runner
{
ShowProgressAtConsole = true
};
RegisterEvents(runner);
var success = runner.ProcessSQLite(fiSourceFile.DirectoryName, fiSourceFile.Name, sourceTableName);
if (success)
{
Console.WriteLine();
Console.WriteLine("Processing complete");
ProgRunner.SleepMilliseconds(750);
return(0);
}
ConsoleMsgUtils.ShowWarning("Error computing protein parsimony: RunAlgorithm reports false");
ProgRunner.SleepMilliseconds(1500);
return(-3);
}
catch (Exception ex)
{
ConsoleMsgUtils.ShowError("Error computing protein parsimony", ex);
ProgRunner.SleepMilliseconds(1500);
return(-2);
}
}
public void TestFileLogger(string logDirectory, string logFileNameBase, string message, logMsgType entryType, int logCount, int logDelayMilliseconds)
{
var logFilePath = Path.Combine(logDirectory, logFileNameBase);
var logger = new clsFileLogger(logFilePath);
var randGenerator = new Random();
string formatString;
if (logCount < 10)
{
formatString = "{0} {1}/{2}";
}
else
{
formatString = "{0} {1,2}/{2}";
}
for (var i = 0; i < logCount; i++)
{
logger.PostEntry(string.Format(formatString, message, i + 1, logCount), entryType, true);
ProgRunner.SleepMilliseconds(logDelayMilliseconds + randGenerator.Next(0, logDelayMilliseconds / 10));
}
var expectedName = logFileNameBase + "_" + DateTime.Now.ToString("MM-dd-yyyy");
if (!logger.CurrentLogFilePath.Contains(expectedName))
{
Assert.Fail("Log file name was not in the expected format of " + expectedName + "; see " + logger.CurrentLogFilePath);
}
Console.WriteLine("Log entries written to " + logger.CurrentLogFilePath);
}
public void TestFileLogger(
string logDirectory,
string logFileNameBase,
string message,
BaseLogger.LogLevels entryType,
int logCount,
int logDelayMilliseconds)
{
FileLogger.ResetLogFileName();
var logFilePath = Path.Combine(logDirectory, logFileNameBase);
var logger = new FileLogger(logFilePath);
var randGenerator = new Random();
for (var i = 0; i < logCount; i++)
{
var logMessage = message + " " + i;
switch (entryType)
{
case BaseLogger.LogLevels.DEBUG:
logger.Debug(logMessage);
break;
case BaseLogger.LogLevels.INFO:
logger.Info(logMessage);
break;
case BaseLogger.LogLevels.WARN:
logger.Warn(logMessage);
break;
case BaseLogger.LogLevels.ERROR:
logger.Error(logMessage);
break;
case BaseLogger.LogLevels.FATAL:
logger.Fatal(logMessage);
break;
default:
logger.Fatal("Unrecognized log type: " + entryType);
break;
}
ProgRunner.SleepMilliseconds(logDelayMilliseconds + randGenerator.Next(0, logDelayMilliseconds / 10));
}
var expectedName = logFileNameBase + "_" + DateTime.Now.ToString("MM-dd-yyyy") + FileLogger.LOG_FILE_EXTENSION;
if (!FileLogger.LogFilePath.EndsWith(expectedName))
{
Assert.Fail("Log file name was not in the expected format of " + expectedName + "; see " + FileLogger.LogFilePath);
}
Console.WriteLine("Log entries written to " + FileLogger.LogFilePath);
}
/// <summary>
/// Entry method
/// </summary>
private static void Main()
{
try
{
mLogger = new FileLogger(@"Logs\StatusMsgDBUpdater", BaseLogger.LogLevels.INFO);
var appVersion = ProcessFilesOrDirectoriesBase.GetEntryOrExecutingAssembly().GetName().Version.ToString();
mLogger.Info("=== Started StatusMessageDBUpdater V" + appVersion + " =====");
var restart = true;
var runFailureCount = 0;
var startTime = DateTime.UtcNow;
while (restart)
{
// Start the main program running
try
{
var mainProcess = new MainProgram();
mainProcess.DebugEvent += MainProcess_DebugEvent;
mainProcess.ErrorEvent += MainProcess_ErrorEvent;
mainProcess.WarningEvent += MainProcess_WarningEvent;
mainProcess.StatusEvent += MainProcess_StatusEvent;
if (!mainProcess.InitMgr(startTime))
{
ProgRunner.SleepMilliseconds(1500);
return;
}
// Start the main process
// If it receives the ReadConfig command, DoProcess will return true
restart = mainProcess.DoProcess();
runFailureCount = 0;
}
catch (Exception ex2)
{
ShowErrorMessage("Error running the main process", ex2);
runFailureCount++;
var sleepSeconds = 1.5 * runFailureCount;
if (sleepSeconds > 30)
{
sleepSeconds = 30;
}
ProgRunner.SleepMilliseconds((int)(sleepSeconds * 1000));
}
}
FileLogger.FlushPendingMessages();
}
catch (Exception ex)
{
ShowErrorMessage("Error starting application", ex);
}
ProgRunner.SleepMilliseconds(1500);
}
static int Main(string[] args)
{
if (args.Length < 1)
{
ShowSyntax();
return(0);
}
FileInfo fiSourceFile;
try
{
var inputFilePath = args[0];
fiSourceFile = new FileInfo(args[0]);
if (!fiSourceFile.Exists)
{
ShowSyntax("Input file not found: " + inputFilePath);
ProgRunner.SleepMilliseconds(1500);
return(-1);
}
}
catch (Exception ex)
{
ShowSyntax("Exception validating the input file path: " + ex.Message);
ProgRunner.SleepMilliseconds(1500);
return(-2);
}
var sqliteExtensions = new SortedSet <string>(StringComparer.OrdinalIgnoreCase)
{
".db",
".db3",
".sqlite",
".sqlite3"
};
if (sqliteExtensions.Contains(fiSourceFile.Extension))
{
var sourceTableName = args.Length > 1 ? args[1] : Runner.DEFAULT_SQLITE_TABLE;
var result = ProcessSQLiteDB(fiSourceFile, sourceTableName);
return(result);
}
else
{
var result = ProcessTextFile(args, fiSourceFile);
return(result);
}
}
public void TestQueueLogger(string logDirectory, string logFileNameBase, string message, logMsgType entryType, int logCount, int logDelayMilliseconds)
{
var logFilePath = Path.Combine(logDirectory, logFileNameBase);
var logger = new clsFileLogger(logFilePath);
var randGenerator = new Random();
var queueLogger = new clsQueLogger(logger);
string formatString;
if (logCount < 10)
{
formatString = "{0} {1}/{2}";
}
else
{
formatString = "{0} {1,2}/{2}";
}
for (var i = 0; i < logCount; i++)
{
queueLogger.PostEntry(string.Format(formatString, message, i + 1, logCount), entryType, true);
ProgRunner.SleepMilliseconds(logDelayMilliseconds + randGenerator.Next(0, logDelayMilliseconds / 10));
}
if (logCount > 5)
{
var messages = new List <clsLogEntry>();
for (var i = 0; i < logCount; i++)
{
messages.Add(new clsLogEntry(string.Format(formatString, "Bulk " + message, i + 1, logCount), entryType));
}
queueLogger.PostEntries(messages);
}
// Sleep to give the Queue logger time to log the log entries
ProgRunner.SleepMilliseconds(4000);
var expectedName = logFileNameBase + "_" + DateTime.Now.ToString(clsFileLogger.FILENAME_DATE_STAMP);
if (!logger.CurrentLogFilePath.Contains(expectedName))
{
Assert.Fail("Log file name was not in the expected format of " + expectedName + "; see " + logger.CurrentLogFilePath);
}
Console.WriteLine("Log entries written to " + logger.CurrentLogFilePath);
}
private bool ProcessFileThreaded(string inputFilePath, string outputDirectoryPath)
{
try
{
mInputFilePath = inputFilePath;
mOutputDirectoryPath = outputDirectoryPath;
var processingThread = new Thread(ProcessFileWork);
processingThread.Start();
var threadAborted = false;
// Loop until URL call finishes, or until timeoutSeconds elapses
while (processingThread.ThreadState != ThreadState.Stopped)
{
ProgRunner.SleepMilliseconds(25);
if (processingThread.ThreadState == ThreadState.Aborted)
{
threadAborted = true;
break;
}
Application.DoEvents();
}
if (threadAborted)
{
ShowErrorMessage("The processing thread was aborted");
return(false);
}
return(mProcessingSuccess);
}
catch (Exception ex)
{
ShowErrorMessage("Error in ProcessFileThreaded", ex);
return(false);
}
}
public void TestLogTools(string logDirectory, string logFileNameBase, BaseLogger.LogLevels entryType, BaseLogger.LogLevels logThresholdLevel)
{
var logFilePath = Path.Combine(logDirectory, logFileNameBase);
LogTools.CreateFileLogger(logFilePath, logThresholdLevel);
Console.WriteLine("Log file; " + LogTools.CurrentLogFilePath);
var message = "Test log " + entryType + " via LogTools (log threshold is " + logThresholdLevel + ")";
switch (entryType)
{
case BaseLogger.LogLevels.DEBUG:
LogTools.LogDebug(message);
break;
case BaseLogger.LogLevels.INFO:
LogTools.LogMessage(message);
break;
case BaseLogger.LogLevels.WARN:
LogTools.LogWarning(message);
break;
case BaseLogger.LogLevels.ERROR:
LogTools.LogError(message);
break;
case BaseLogger.LogLevels.FATAL:
LogTools.LogFatalError(message);
break;
default:
LogTools.LogError("Unrecognized log type: " + entryType);
break;
}
ProgRunner.SleepMilliseconds(100);
LogTools.FlushPendingMessages();
}
private void TestStaticLogging(
string message,
BaseLogger.LogLevels entryType,
int logCount,
int logDelayMilliseconds,
string expectedLogFileName)
{
var randGenerator = new Random();
string formatString;
if (logCount < 10)
{
formatString = "{0} {1}/{2}";
}
else
{
formatString = "{0} {1,2}/{2}";
}
for (var i = 0; i < logCount; i++)
{
FileLogger.WriteLog(entryType, string.Format(formatString, message, i + 1, logCount));
ProgRunner.SleepMilliseconds(logDelayMilliseconds + randGenerator.Next(0, logDelayMilliseconds / 10));
}
if (!FileLogger.LogFilePath.EndsWith(expectedLogFileName))
{
var errMsg = "Log file name was not in the expected format of " + expectedLogFileName + "; see " + FileLogger.LogFilePath;
Assert.Fail(errMsg);
}
FileLogger.FlushPendingMessages();
Console.WriteLine("Log entries written to " + FileLogger.LogFilePath);
}
/// <summary>
/// Call the Python script to create the plots
/// </summary>
/// <returns>True if success, otherwise false</returns>
/// <remarks>Call ErrorHistogramsToPng and ErrorScatterPlotsToPng prior to calling this method</remarks>
private bool GeneratePlotsWithPython(MetadataFileInfo metadataFileInfo)
{
if (!PythonInstalled)
{
NotifyPythonNotFound("Could not find the python executable");
return(false);
}
if (metadataFileInfo.BaseOutputFile.Directory == null)
{
OnErrorEvent("Unable to determine the parent directory of the base output file: " + metadataFileInfo.BaseOutputFile.FullName);
return(false);
}
if (!metadataFileInfo.BaseOutputFile.Directory.Exists)
{
metadataFileInfo.BaseOutputFile.Directory.Create();
}
var workDir = metadataFileInfo.BaseOutputFile.Directory.FullName;
var exeDirectory = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
if (exeDirectory == null)
{
OnErrorEvent("Unable to determine the path to the directory with the PPMErrorCharter executable");
return(false);
}
var pythonScriptFile = new FileInfo(Path.Combine(exeDirectory, "PPMErrorCharter_Plotter.py"));
if (!pythonScriptFile.Exists)
{
OnErrorEvent("Python plotting script not found: " + pythonScriptFile.FullName);
return(false);
}
var baseOutputName = metadataFileInfo.BaseOutputFile.Name;
var metadataFile = new FileInfo(Path.Combine(workDir, "MZRefinery_Plotting_Metadata.txt"));
OnDebugEvent("Creating " + metadataFile.FullName);
using (var writer = new StreamWriter(new FileStream(metadataFile.FullName, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)))
{
var plotFilesDefined = 0;
if (metadataFileInfo.HistogramPlotFile != null)
{
writer.WriteLine("HistogramPlotFilePath=" + metadataFileInfo.HistogramPlotFile.FullName);
plotFilesDefined++;
}
if (metadataFileInfo.MassErrorPlotFile != null)
{
writer.WriteLine("MassErrorPlotFilePath=" + metadataFileInfo.MassErrorPlotFile.FullName);
plotFilesDefined++;
}
if (plotFilesDefined < 2)
{
writer.WriteLine("BaseOutputName=" + baseOutputName);
metadataFileInfo.HistogramPlotFile = new FileInfo(Path.Combine(metadataFileInfo.BaseOutputFile.FullName, baseOutputName + "_Histograms.png"));
metadataFileInfo.MassErrorPlotFile = new FileInfo(Path.Combine(metadataFileInfo.BaseOutputFile.FullName, baseOutputName + "_MassErrors.png"));
}
writer.WriteLine("HistogramData=" + metadataFileInfo.ErrorHistogramsExportFileName);
writer.WriteLine("MassErrorVsTimeData=" + metadataFileInfo.MassErrorVsTimeExportFileName);
writer.WriteLine("MassErrorVsMassData=" + metadataFileInfo.MassErrorVsMassExportFileName);
}
var args = PathUtils.PossiblyQuotePath(pythonScriptFile.FullName) + " " + PathUtils.PossiblyQuotePath(metadataFile.FullName);
OnDebugEvent("{0} {1}", PythonPath, args);
var progRunner = new ProgRunner
{
Arguments = args,
CreateNoWindow = true,
MonitoringInterval = 2000,
Name = "PythonPlotter",
Program = PythonPath,
Repeat = false,
RepeatHoldOffTime = 0,
WorkDir = workDir
};
RegisterEvents(progRunner);
const int MAX_RUNTIME_SECONDS = 600;
const int MONITOR_INTERVAL_MILLISECONDS = 1000;
var runtimeExceeded = false;
try
{
// Start the program executing
progRunner.StartAndMonitorProgram();
var startTime = DateTime.UtcNow;
// Loop until program is complete, or until MAX_RUNTIME_SECONDS seconds elapses
while (progRunner.State != ProgRunner.States.NotMonitoring)
{
ProgRunner.SleepMilliseconds(MONITOR_INTERVAL_MILLISECONDS);
if (DateTime.UtcNow.Subtract(startTime).TotalSeconds < MAX_RUNTIME_SECONDS)
{
continue;
}
OnErrorEvent("Plot creation with Python has taken more than {0:F0} minutes; aborting", MAX_RUNTIME_SECONDS / 60.0);
progRunner.StopMonitoringProgram(kill: true);
runtimeExceeded = true;
break;
}
}
catch (Exception ex)
{
OnErrorEvent("Exception creating plots using Python", ex);
return(false);
}
if (runtimeExceeded)
{
return(false);
}
// Examine the exit code
if (progRunner.ExitCode == 0)
{
OnStatusEvent("Generated plots; see:\n " + metadataFileInfo.HistogramPlotFile?.FullName + "\nand\n " + metadataFileInfo.MassErrorPlotFile?.FullName);
if (DeleteTempFiles)
{
// Delete the temp export files
try
{
metadataFile.Delete();
File.Delete(Path.Combine(workDir, metadataFileInfo.ErrorHistogramsExportFileName));
File.Delete(Path.Combine(workDir, metadataFileInfo.MassErrorVsTimeExportFileName));
File.Delete(Path.Combine(workDir, metadataFileInfo.MassErrorVsMassExportFileName));
}
catch (Exception ex)
{
OnErrorEvent("Error deleting files: " + ex.Message);
}
}
else
{
ConsoleMsgUtils.ShowDebug("{0}\n {1}\n {2}\n {3}\n {4}",
"Not deleting the following temporary files since debug mode is enabled",
metadataFile.FullName,
Path.Combine(workDir, metadataFileInfo.ErrorHistogramsExportFileName),
Path.Combine(workDir, metadataFileInfo.MassErrorVsTimeExportFileName),
Path.Combine(workDir, metadataFileInfo.MassErrorVsMassExportFileName));
}
return(true);
}
OnErrorEvent("Python ExitCode = " + progRunner.ExitCode);
return(false);
}
private void ReadBinaryData(string dataDirectoryPath, DatasetFileInfo datasetFileInfo, double acquisitionLengthMinutes)
{
try
{
// Open the data directory using the ProteoWizardWrapper
var pWiz = new pwiz.ProteowizardWrapper.MSDataFileReader(dataDirectoryPath);
try
{
var runStartTime = Convert.ToDateTime(pWiz.RunStartTime);
// Update AcqTimeEnd if possible
if (runStartTime < datasetFileInfo.AcqTimeEnd)
{
if (datasetFileInfo.AcqTimeEnd.Subtract(runStartTime).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = runStartTime;
if (acquisitionLengthMinutes > 0)
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(acquisitionLengthMinutes);
}
}
}
}
catch (Exception)
{
// Leave the times unchanged
}
// Instantiate the ProteoWizard Data Parser class
var pWizParser = new clsProteoWizardDataParser(pWiz, mDatasetStatsSummarizer, mTICAndBPIPlot,
mLCMS2DPlot, mSaveLCMS2DPlots, mSaveTICAndBPI,
mCheckCentroidingStatus)
{
HighResMS1 = true,
HighResMS2 = true
};
RegisterEvents(pWizParser);
var ticStored = false;
double runtimeMinutes = 0;
if (pWiz.ChromatogramCount > 0)
{
// Process the chromatograms
pWizParser.StoreChromatogramInfo(datasetFileInfo, out ticStored, out _, out runtimeMinutes);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, runtimeMinutes);
}
if (pWiz.SpectrumCount > 0)
{
// Process the spectral data
var skipExistingScans = (pWiz.ChromatogramCount > 0);
pWizParser.StoreMSSpectraInfo(ticStored, ref runtimeMinutes,
skipExistingScans,
skipScansWithNoIons: true,
maxScansToTrackInDetail: MAX_SCANS_TO_TRACK_IN_DETAIL,
maxScansForTicAndBpi: MAX_SCANS_FOR_TIC_AND_BPI);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, runtimeMinutes);
}
pWiz.Dispose();
ProgRunner.GarbageCollectNow();
}
catch (Exception ex)
{
OnErrorEvent("Exception reading the Binary Data in the Agilent TOF .D directory using ProteoWizard: " + ex.Message, ex);
}
}
protected bool GeneratePlotsWithPython(FileInfo exportFile, DirectoryInfo workDir)
{
if (!PythonInstalled)
{
NotifyPythonNotFound("Could not find the python executable");
return(false);
}
var exeDirectory = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
if (exeDirectory == null)
{
OnErrorEvent("Unable to determine the path to the directory with the MSFileInfoScanner executable");
return(false);
}
var pythonScriptFile = new FileInfo(Path.Combine(exeDirectory, "MSFileInfoScanner_Plotter.py"));
if (!pythonScriptFile.Exists)
{
OnErrorEvent("Python plotting script not found: " + pythonScriptFile.FullName);
return(false);
}
var args = PathUtils.PossiblyQuotePath(pythonScriptFile.FullName) + " " + PathUtils.PossiblyQuotePath(exportFile.FullName);
OnDebugEvent(string.Format("{0} {1}", PythonPath, args));
var progRunner = new ProgRunner
{
Arguments = args,
CreateNoWindow = true,
MonitoringInterval = 2000,
Name = "PythonPlotter",
Program = PythonPath,
Repeat = false,
RepeatHoldOffTime = 0,
WorkDir = workDir.FullName
};
RegisterEvents(progRunner);
const int MAX_RUNTIME_SECONDS = 600;
const int MONITOR_INTERVAL_MILLISECONDS = 1000;
var runtimeExceeded = false;
try
{
// Start the program executing
progRunner.StartAndMonitorProgram();
var startTime = DateTime.UtcNow;
// Loop until program is complete, or until MAX_RUNTIME_SECONDS seconds elapses
while (progRunner.State != ProgRunner.States.NotMonitoring)
{
ProgRunner.SleepMilliseconds(MONITOR_INTERVAL_MILLISECONDS);
if (DateTime.UtcNow.Subtract(startTime).TotalSeconds < MAX_RUNTIME_SECONDS)
{
continue;
}
OnErrorEvent(string.Format("Plot creation with Python has taken more than {0:F0} minutes; aborting", MAX_RUNTIME_SECONDS / 60.0));
progRunner.StopMonitoringProgram(kill: true);
runtimeExceeded = true;
break;
}
}
catch (Exception ex)
{
OnErrorEvent("Exception creating plots using Python", ex);
return(false);
}
if (runtimeExceeded)
{
return(false);
}
// Examine the exit code
if (progRunner.ExitCode == 0)
{
var success = RenameTempPngFile(exportFile, workDir);
return(success);
}
OnErrorEvent("Python ExitCode = " + progRunner.ExitCode);
return(false);
}
public static int Main()
{
var commandLineParser = new clsParseCommandLine();
mInputFilePath = string.Empty;
mOutputDirectoryPath = string.Empty;
mParameterFilePath = string.Empty;
mRecurseDirectories = false;
mMaxLevelsToRecurse = 0;
mQuietMode = false;
mLogMessagesToFile = false;
mLogFilePath = string.Empty;
mLastSubtaskProgressTime = DateTime.UtcNow;
try
{
var proceed = false;
if (commandLineParser.ParseCommandLine())
{
if (SetOptionsUsingCommandLineParameters(commandLineParser))
{
proceed = true;
}
}
if (commandLineParser.ParameterCount + commandLineParser.NonSwitchParameterCount == 0 && !commandLineParser.NeedToShowHelp)
{
#if GUI
ShowGUI();
#else
ShowProgramHelp();
#endif
return(0);
}
if (!proceed || commandLineParser.NeedToShowHelp || mInputFilePath.Length == 0)
{
ShowProgramHelp();
return(-1);
}
mMASIC = new clsMASIC();
RegisterMasicEvents(mMASIC);
mMASIC.Options.DatasetLookupFilePath = mDatasetLookupFilePath;
mMASIC.Options.SICOptions.DatasetID = mDatasetID;
if (!string.IsNullOrEmpty(mMASICStatusFilename))
{
mMASIC.Options.MASICStatusFilename = mMASICStatusFilename;
}
mMASIC.LogMessagesToFile = mLogMessagesToFile;
mMASIC.LogFilePath = mLogFilePath;
mMASIC.LogDirectoryPath = mLogDirectoryPath;
if (!mQuietMode)
{
#if GUI
mProgressForm = new frmProgress();
mProgressForm.InitializeProgressForm("Parsing " + Path.GetFileName(mInputFilePath), 0, 100, false, true);
mProgressForm.InitializeSubtask(string.Empty, 0, 100, false);
mProgressForm.ResetKeyPressAbortProcess();
mProgressForm.Show();
Application.DoEvents();
#else
Console.WriteLine("Parsing " + Path.GetFileName(mInputFilePath));
#endif
}
int returnCode;
if (mRecurseDirectories)
{
if (mMASIC.ProcessFilesAndRecurseDirectories(mInputFilePath, mOutputDirectoryPath,
mOutputDirectoryAlternatePath, mRecreateDirectoryHierarchyInAlternatePath,
mParameterFilePath, mMaxLevelsToRecurse))
{
returnCode = 0;
}
else
{
returnCode = (int)mMASIC.ErrorCode;
}
}
else if (mMASIC.ProcessFilesWildcard(mInputFilePath, mOutputDirectoryPath, mParameterFilePath))
{
returnCode = 0;
}
else
{
returnCode = (int)mMASIC.ErrorCode;
if (returnCode != 0)
{
Console.WriteLine("Error while processing: " + mMASIC.GetErrorMessage());
}
}
if (returnCode != 0)
{
ProgRunner.SleepMilliseconds(1500);
}
return(returnCode);
}
catch (Exception ex)
{
ShowErrorMessage("Error occurred in Program->Main: " + Environment.NewLine + ex.Message);
ProgRunner.SleepMilliseconds(1500);
return(-1);
}
#if GUI
finally
{
if (mProgressForm != null)
{
mProgressForm.HideForm();
mProgressForm = null;
}
}
#endif
}
private static int ProcessTextFile(IReadOnlyList <string> args, FileInfo fiSourceFile)
{
string parsimonyResultsFilePath;
string proteinGroupMembersFilePath;
try
{
if (args.Count < 2)
{
Runner.GetDefaultOutputFileNames(fiSourceFile, out parsimonyResultsFilePath, out proteinGroupMembersFilePath);
}
else
{
Runner.GetDefaultOutputFileNames(fiSourceFile, out _, out proteinGroupMembersFilePath);
parsimonyResultsFilePath = args[1];
}
var outputFile = new FileInfo(parsimonyResultsFilePath);
// ReSharper disable once MergeIntoPattern
if (outputFile.Directory != null && !outputFile.Directory.Exists)
{
outputFile.Directory.Create();
}
}
catch (Exception ex)
{
ShowSyntax("Exception validating the output file path: " + ex.Message);
ProgRunner.SleepMilliseconds(1500);
return(-2);
}
try
{
var runner = new Runner
{
ShowProgressAtConsole = true
};
RegisterEvents(runner);
var success = runner.ProcessTextFile(fiSourceFile, parsimonyResultsFilePath, proteinGroupMembersFilePath);
if (success)
{
Console.WriteLine();
Console.WriteLine("Processing Complete");
ProgRunner.SleepMilliseconds(750);
return(0);
}
ConsoleMsgUtils.ShowWarning("Error computing protein parsimony: ProcessTextFile reports false");
ProgRunner.SleepMilliseconds(1500);
return(-3);
}
catch (Exception ex)
{
ConsoleMsgUtils.ShowError("Error computing protein parsimony", ex);
ProgRunner.SleepMilliseconds(1500);
return(-4);
}
}
/// <summary>
/// Program entry point
/// </summary>
/// <param name="args"></param>
/// <returns>0 if no error, error code if an error</returns>
private static int Main(string[] args)
{
var programName = System.Reflection.Assembly.GetEntryAssembly()?.GetName().Name;
var exeName = Path.GetFileName(PRISM.FileProcessor.ProcessFilesOrDirectoriesBase.GetAppPath());
var parser = new CommandLineParser <CommandLineOptions>(programName, GetAppVersion())
{
ProgramInfo = "This program compares two or more files (typically in separate directories) to check whether the " +
"start of the files match, the end of the files match, and selected sections inside the files also match. " +
"Useful for comparing large files without reading the entire file. " +
"Alternatively, you can provide two directory paths and the program will compare all of the files " +
"in the first directory to the identically named files in the second directory.",
ContactInfo = "Program written by Matthew Monroe for the Department of Energy (PNNL, Richland, WA)" +
Environment.NewLine + Environment.NewLine +
"E-mail: [email protected] or [email protected]" + Environment.NewLine +
"Website: https://github.com/PNNL-Comp-Mass-Spec/ or https://panomics.pnnl.gov/ or https://www.pnnl.gov/integrative-omics"
};
parser.UsageExamples.Add(
"Program syntax 1: compare two files; in this case the filenames cannot have wildcards" +
Environment.NewLine +
" " + exeName + " FilePath1 FilePath2" + Environment.NewLine +
" [/N:NumberOfSamples] [/Bytes:SampleSizeBytes]" + Environment.NewLine +
" [/KB:SizeKB] [/MB:SizeMB] [/GB:SizeGB]" + Environment.NewLine +
" [/L[:LogFilePath]] [/LogDirectory:LogDirectoryPath]");
parser.UsageExamples.Add("Program syntax 2: compare two directories (including all subdirectories)" + Environment.NewLine +
" " + exeName + " DirectoryPath1 DirectoryPath2" + Environment.NewLine +
" [/N:NumberOfSamples] [/Bytes:SampleSizeBytes]" + Environment.NewLine +
" [/L] [/LogDirectory]");
parser.UsageExamples.Add(ConsoleMsgUtils.WrapParagraph(
"Program syntax 3: compare a set of files in one directory to identically named files in a separate directory. " +
"Use wildcards in FileMatchSpec to specify the files to examine") + Environment.NewLine +
" " + exeName + " FileMatchSpec DirectoryPathToExamine" + Environment.NewLine +
" [/N:NumberOfSamples] [/Bytes:SampleSizeBytes]" + Environment.NewLine +
" [/L] [/LogDirectory]");
parser.UsageExamples.Add(ConsoleMsgUtils.WrapParagraph(
"Program syntax 4: compare a DMS dataset's files between the storage server and the archive. " +
"The first argument must be DMS; the second argument is the Dataset Name.") + Environment.NewLine +
" " + exeName + " DMS DatasetNameToCheck" + Environment.NewLine +
" [/N:NumberOfSamples] [/Bytes:SampleSizeBytes]" + Environment.NewLine +
" [/L] [/LogDirectory]");
var result = parser.ParseArgs(args);
var options = result.ParsedResults;
if (!result.Success || !options.Validate())
{
if (parser.CreateParamFileProvided)
{
return(0);
}
// Delay for 1000 msec in case the user double clicked this file from within Windows Explorer (or started the program via a shortcut)
ProgRunner.SleepMilliseconds(1000);
return(-1);
}
try
{
int returnCode;
mProcessingClass = new SampledFileComparer
{
NumberOfSamples = options.NumberOfSamples,
SampleSizeBytes = options.SampleSizeBytes,
IgnoreErrorsWhenUsingWildcardMatching = true,
LogMessagesToFile = options.LogMessagesToFile,
LogFilePath = options.LogFilePath,
LogDirectoryPath = options.LogDirectoryPath
};
mProcessingClass.ProgressUpdate += ProcessingClass_ProgressChanged;
mProcessingClass.ProgressReset += ProcessingClass_ProgressReset;
if (string.IsNullOrWhiteSpace(options.InputFileOrDirectoryPath))
{
ShowErrorMessage("Base file or directory to compare is empty");
return(-1);
}
bool success;
if (string.Equals(options.InputFileOrDirectoryPath, "DMS", StringComparison.OrdinalIgnoreCase) &&
options.ComparisonFileOrDirectoryPath.IndexOf("\\", StringComparison.Ordinal) < 0)
{
// InputFile is "DMS"
// Treat ComparisonFile as a dataset name and compare files on the storage server to files in the archive
// This feature does not yet support files in MyEMSL
success = mProcessingClass.ProcessDMSDataset(options.ComparisonFileOrDirectoryPath);
}
else
{
// Comparing two files or two directories
success = mProcessingClass.ProcessFilesWildcard(options.InputFileOrDirectoryPath, options.ComparisonFileOrDirectoryPath);
}
if (success)
{
returnCode = 0;
}
else
{
returnCode = (int)mProcessingClass.ErrorCode;
if (returnCode != 0)
{
ShowErrorMessage("Error while processing: " + mProcessingClass.GetErrorMessage());
}
}
return(returnCode);
}
catch (Exception ex)
{
ShowErrorMessage("Error occurred in modMain->Main: " + ex.Message);
return(-1);
}
}
private void TestRunProgram(string exeName, string cmdArgs, bool createNoWindow, bool writeConsoleOutput, int maxRuntimeSeconds, bool programAbortExpected)
{
const int MONITOR_INTERVAL_MSEC = 500;
var utilityExe = new FileInfo(Path.Combine(UTILITIES_DIRECTORY, exeName));
if (!utilityExe.Exists)
{
Assert.Fail("Exe not found: " + utilityExe.FullName);
}
var processStats = new PRISMWin.ProcessStats();
var workDir = @"C:\Temp";
var tempDir = new DirectoryInfo(workDir);
if (!tempDir.Exists)
{
tempDir.Create();
}
var coreCount = processStats.GetCoreCount();
Console.WriteLine("Machine has {0} cores", coreCount);
Assert.GreaterOrEqual(coreCount, 2, "Core count less than 2");
var progRunner = new ProgRunner
{
Arguments = cmdArgs,
CreateNoWindow = createNoWindow,
MonitoringInterval = MONITOR_INTERVAL_MSEC,
Name = "ProgRunnerUnitTest",
Program = utilityExe.FullName,
Repeat = false,
RepeatHoldOffTime = 0,
WorkDir = workDir,
CacheStandardOutput = false,
EchoOutputToConsole = false,
WriteConsoleOutputToFile = writeConsoleOutput,
ConsoleOutputFileIncludesCommandLine = true
};
progRunner.StartAndMonitorProgram();
var cachedProcessID = 0;
var dtStartTime = DateTime.UtcNow;
var abortProcess = false;
while (progRunner.State != ProgRunner.States.NotMonitoring)
{
if (cachedProcessID == 0)
{
cachedProcessID = progRunner.PID;
}
ProgRunner.SleepMilliseconds(MONITOR_INTERVAL_MSEC / 2);
try
{
if (cachedProcessID > 0)
{
var cpuUsage = processStats.GetCoreUsageByProcessID(cachedProcessID);
Console.WriteLine("CPU Usage: " + cpuUsage);
}
}
catch (Exception ex)
{
Console.WriteLine("Unable to get the core usage: {0}", ex.Message);
}
ProgRunner.SleepMilliseconds(MONITOR_INTERVAL_MSEC / 2);
if (maxRuntimeSeconds > 0)
{
if (DateTime.UtcNow.Subtract(dtStartTime).TotalSeconds > maxRuntimeSeconds)
{
Console.WriteLine("Aborting ProcessID {0} since runtime has exceeded {1} seconds", cachedProcessID, maxRuntimeSeconds);
abortProcess = true;
}
}
if (abortProcess)
{
progRunner.StopMonitoringProgram(kill: true);
}
}
processStats.ClearCachedPerformanceCounterForProcessID(cachedProcessID);
if (writeConsoleOutput)
{
ProgRunner.SleepMilliseconds(250);
var consoleOutputFilePath = progRunner.ConsoleOutputFilePath;
Assert.IsNotEmpty(consoleOutputFilePath, "Console output file path is empty");
var consoleOutputFile = new FileInfo(consoleOutputFilePath);
Assert.True(consoleOutputFile.Exists, "File not found: " + consoleOutputFilePath);
var secondsSinceMidnight = (int)(DateTime.Now.Subtract(DateTime.Today).TotalSeconds);
if (consoleOutputFile.DirectoryName == null)
{
Assert.Fail("Unable to determine the parent directory of " + consoleOutputFilePath);
}
var newFilePath = Path.Combine(consoleOutputFile.DirectoryName,
Path.GetFileNameWithoutExtension(consoleOutputFilePath) + "_" + secondsSinceMidnight + ".txt");
consoleOutputFile.MoveTo(newFilePath);
// Open the file and assure that the first line contains the .exe name
using (var reader = new StreamReader(new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
{
if (reader.EndOfStream)
{
Assert.Fail("The ConsoleOutput file is empty: " + newFilePath);
}
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
Assert.Fail("The first line of the ConsoleOutput file is empty: " + newFilePath);
}
if (!dataLine.ToLower().Contains(exeName))
{
Assert.Fail("The first line of the ConsoleOutput file does not contain " + exeName + ": " + newFilePath);
}
}
}
if (programAbortExpected)
{
Assert.True(abortProcess, "Process was expected to be aborted due to excessive runtime, but was not");
}
else
{
Assert.False(abortProcess, "Process was aborted due to excessive runtime; this is unexpected");
}
}
private void TestGarbageCollectionWork(int iterations, int gcEvents, int dictionaryCount, int dictionarySize)
{
// This method will create 2.5 million FileInfo objects (if iterations is 2500000) and store those in random locations in various dictionaries
// It will remove half of the dictionaries gcEvents times, calling GarbageCollectNow after each removal
var dictionaries = new List <Dictionary <int, List <FileInfo> > >();
var rand = new Random();
var keysRemoved = 0;
var valuesRemoved = 0;
var lastGC = DateTime.UtcNow;
var gcInterval = (int)Math.Floor(iterations / (double)gcEvents);
for (var i = 0; i < iterations; i++)
{
// Pick a dictionary at random (add new dictionaries if required)
var dictionaryIndex = rand.Next(0, dictionaryCount - 1);
while (dictionaries.Count < dictionaryIndex + 1)
{
dictionaries.Add(new Dictionary <int, List <FileInfo> >());
}
var currentDictionary = dictionaries[dictionaryIndex];
// Create a random filename, 100 characters long, composed of upper and lowercase letters
var randomName = new StringBuilder();
for (var charIndex = 0; charIndex < 100; charIndex++)
{
if (rand.NextDouble() > 0.85)
{
randomName.Append((char)rand.Next(65, 91));
}
else
{
randomName.Append((char)rand.Next(97, 123));
}
}
// Create a FileInfo object
var randomFile = new FileInfo(randomName + ".txt");
// Pick a random key for selecting a list to append the FileInfo object to
var key = rand.Next(0, dictionarySize);
if (currentDictionary.TryGetValue(key, out var values))
{
values.Add(randomFile);
}
else
{
var newValues = new List <FileInfo> {
randomFile
};
currentDictionary.Add(key, newValues);
}
if (i == 0 || i % gcInterval != 0)
{
continue;
}
// Remove half of the dictionaries, at random
var targetCount = dictionaries.Count / 2;
for (var j = 0; j < targetCount; j++)
{
var dictionaryIndexToRemove = rand.Next(0, dictionaries.Count - 1);
var dictionaryToRemove = dictionaries[dictionaryIndexToRemove];
keysRemoved = dictionaryToRemove.Keys.Count;
foreach (var keyToRemove in dictionaryToRemove.Keys)
{
valuesRemoved += dictionaryToRemove[keyToRemove].Count;
}
dictionaries.RemoveAt(dictionaryIndexToRemove);
}
Console.WriteLine("Garbage collect at {0:yyyy-MM-dd hh:mm:ss tt} ({1:F1} seconds elapsed)",
DateTime.Now, DateTime.UtcNow.Subtract(lastGC).TotalSeconds);
ProgRunner.GarbageCollectNow();
lastGC = DateTime.UtcNow;
}
Console.WriteLine("{0} dictionaries", dictionaries.Count);
var totalKeys = 0;
var totalValues = 0;
foreach (var dictionary in dictionaries)
{
totalKeys += dictionary.Keys.Count;
foreach (var key in dictionary.Keys)
{
totalValues += dictionary[key].Count;
}
}
Console.WriteLine("{0} total keys", totalKeys);
Console.WriteLine("{0} total values", totalValues);
Console.WriteLine("{0} keys removed", keysRemoved);
Console.WriteLine("{0} values removed", valuesRemoved);
}
