/// <summary>
/// Examine the files that exist below a dataset folder
/// Auto-determine the ones that should be purged
/// </summary>
/// <param name="udtDatasetInfo">Dataset info</param>
/// <param name="lstJobsToPurge">Jobs whose folders will be deleted</param>
/// <returns>List of files that are safe to delete</returns>
public SortedSet<string> FindDatasetFilesToAutoPurge(clsStorageOperations.udtDatasetInfoType udtDatasetInfo, out List<int> lstJobsToPurge)
{
var lstServerFilesToPurge = new SortedSet<string>();
lstJobsToPurge = new List<int>();
var reJobFolder = new Regex(@"_Auto(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
var diDatasetFolder = new DirectoryInfo(udtDatasetInfo.ServerFolderPath);
// Process files in the dataset folder
switch (udtDatasetInfo.RawDataType)
{
case "dot_raw_files":
AddFilesToPurge(diDatasetFolder, "*.raw", lstServerFilesToPurge);
break;
case "dot_wiff_files":
case "sciex_wiff_files":
AddFilesToPurge(diDatasetFolder, "*.wiff", lstServerFilesToPurge);
AddFilesToPurge(diDatasetFolder, "*.wiff.scan", lstServerFilesToPurge);
break;
case "dot_mzml_files":
AddFilesToPurge(diDatasetFolder, "*.mzML", lstServerFilesToPurge);
AddFilesToPurge(diDatasetFolder, "*.mzXML", lstServerFilesToPurge);
break;
case "dot_uimf_files":
AddFilesToPurge(diDatasetFolder, "*.uimf", lstServerFilesToPurge);
break;
case "bruker_maldi_imaging":
AddFilesToPurge(diDatasetFolder, "*.zip", lstServerFilesToPurge);
break;
case "zipped_s_folders":
AddFilesToPurge(diDatasetFolder, "*.zip", lstServerFilesToPurge);
break;
case "bruker_maldi_spot":
AddFilesToPurge(diDatasetFolder, "*.*", lstServerFilesToPurge);
break;
}
// Purge all files over 2 MB in size
AddFilesToPurge(diDatasetFolder, "*.*", 2048, false, lstServerFilesToPurge);
// Process the directories below the dataset folder
// Construct a list of the folders that exist at the dataset folder level
foreach (var diSubDir in diDatasetFolder.GetDirectories())
{
var subDirNameUpper = diSubDir.Name.ToUpper();
if (diSubDir.Name == "QC")
// Do not purge the QC folder
continue;
if (subDirNameUpper.EndsWith(".D"))
{
// Instrument data folder
AddFilesToPurge(diSubDir, "*.yep", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf
AddFilesToPurge(diSubDir, "*.baf", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf
AddFilesToPurge(diSubDir, "ser", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf
AddFilesToPurge(diSubDir, "fid", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf
AddFilesToPurge(diSubDir, "DATA.MS", 0, true, lstServerFilesToPurge); // Agilent_GC_MS_01
AddFilesToPurge(diSubDir, "MSProfile.bin", 0, true, lstServerFilesToPurge); // Agilent_QTof
AddFilesToPurge(diSubDir, "MSPeak.bin", 250, true, lstServerFilesToPurge); // Agilent_QQQ
AddFilesToPurge(diSubDir, "MSScan.bin", 1000, true, lstServerFilesToPurge); // Agilent_QQQ
// Purge all files over 2 MB in size
AddFilesToPurge(diSubDir, "*.*", 2048, true, lstServerFilesToPurge);
continue;
}
if (subDirNameUpper.EndsWith(".RAW"))
{
AddFilesToPurge(diSubDir, "*.raw", 0, true, lstServerFilesToPurge);
// Purge all files over 2 MB in size
AddFilesToPurge(diSubDir, "*.*", 2048, true, lstServerFilesToPurge);
continue;
}
if (subDirNameUpper.StartsWith("MSXML_GEN"))
{
AddFilesToPurgeDateThreshold(diSubDir, 90, lstServerFilesToPurge);
continue;
}
if (subDirNameUpper.StartsWith("DTA_GEN") || subDirNameUpper.StartsWith("DTA_REF"))
{
// Purge after 1.5 years
AddFilesToPurgeDateThreshold(diSubDir, 548, lstServerFilesToPurge);
continue;
}
var reMatch = reJobFolder.Match(diSubDir.Name);
if (reMatch.Success)
{
// This is an analysis job folder
if (diSubDir.Name.StartsWith("SIC"))
{
// This is a MASIC folder
AddFilesToPurge(diSubDir, "*.zip", lstServerFilesToPurge);
// Purge all files over 15 MB in size
AddFilesToPurge(diSubDir, "*.*", 15 * 1024, true, lstServerFilesToPurge);
}
else
{
// Other analysis job folders
// Purge the entire folder if all files are over 3 years old
var subDirPurged = AddFilesToPurgeDateThreshold(diSubDir, 3 * 365, lstServerFilesToPurge);
if (!subDirPurged)
{
// Files are not yet 3 years old
// If all of the files are 1 year old, then purge files over 50 MB
DateTime dtMostRecentUpdate;
var lstFiles = FindFilesAndNewestDate(diSubDir, out dtMostRecentUpdate);
if (DateTime.UtcNow.Subtract(dtMostRecentUpdate).TotalDays > 365)
{
// Purge all files over 50 MB in size
var iFilesMatched = AddFilesToPurge(diSubDir, "*.*", 50 * 1024, true, lstServerFilesToPurge);
if (iFilesMatched == lstFiles.Count)
subDirPurged = true;
}
}
if (subDirPurged)
{
if (reMatch.Groups.Count > 0)
{
int jobNum;
if (int.TryParse(reMatch.Groups[1].Value, out jobNum))
lstJobsToPurge.Add(jobNum);
}
}
}
continue;
}
// Use a threshold of 9 months for all other subfolders
AddFilesToPurgeDateThreshold(diSubDir, 270, lstServerFilesToPurge);
}
return lstServerFilesToPurge;
}
/// <summary>
/// Examine each file in serverFiles and decide which is safe to delete based on the purge policy
/// </summary>
/// <param name="diDatasetFolder">Dataset folder to process</param>
/// <param name="udtDatasetInfo">Dataset info</param>
/// <param name="lstJobsToPurge">Jobs whose folders will be deleted</param>
/// <returns>List of files that are safe to delete</returns>
public SortedSet<string> FindDatasetFilesToPurge(
DirectoryInfo diDatasetFolder,
clsStorageOperations.udtDatasetInfoType udtDatasetInfo,
out List<int> lstJobsToPurge)
{
var ePurgePolicyToApply = udtDatasetInfo.PurgePolicy;
if (ePurgePolicyToApply == clsStorageOperations.PurgePolicyConstants.PurgeAll)
{
var lstServerFilesToPurge = new SortedSet<string>();
AddFilesToPurge(diDatasetFolder, "*.*", 0, true, lstServerFilesToPurge);
lstJobsToPurge = new List<int>();
return lstServerFilesToPurge;
}
if (ePurgePolicyToApply == clsStorageOperations.PurgePolicyConstants.PurgeAllExceptQC)
{
var lstServerFilesToPurge = new SortedSet<string>();
AddFilesToPurge(diDatasetFolder, "*.*", 0, false, lstServerFilesToPurge);
foreach (var diSubFolder in diDatasetFolder.GetDirectories())
{
if (diSubFolder.Name != "QC")
AddFilesToPurge(diSubFolder, "*.*", 0, true, lstServerFilesToPurge);
}
lstJobsToPurge = new List<int>();
return lstServerFilesToPurge;
}
// Auto-purge files for this dataset
return FindDatasetFilesToAutoPurge(udtDatasetInfo, out lstJobsToPurge);
}
/// <summary>
/// Initializes the manager
/// </summary>
/// <returns>TRUE for success; FALSE otherwise</returns>
public bool InitMgr()
{
// Get the manager settings
try
{
m_MgrSettings = new clsMgrSettings();
}
catch
{
// Failures are logged by clsMgrSettings
return false;
}
// Update the cached manager name
m_MgrName = m_MgrSettings.GetParam("MgrName");
// Set up the loggers
var logFileName = m_MgrSettings.GetParam("logfilename");
// LogLevel is 1 to 5: 1 for Fatal errors only, 4 for Fatal, Error, Warning, and Info, and 5 for everything including Debug messages
var debugLevel = int.Parse(m_MgrSettings.GetParam("debuglevel"));
clsLogTools.CreateFileLogger(logFileName, debugLevel);
var logCnStr = m_MgrSettings.GetParam("connectionstring");
clsLogTools.CreateDbLogger(logCnStr, "SpaceManager: " + m_MgrName);
// Make initial log entry
var msg = "=== Started Space Manager V" + System.Windows.Forms.Application.ProductVersion + " ===== ";
clsLogTools.WriteLog(clsLogTools.LoggerTypes.LogFile, clsLogTools.LogLevels.INFO, msg);
// Setup the message queue
m_MsgQueueInitSuccess = false;
m_MsgHandler = new clsMessageHandler();
m_MsgHandler.BrokerUri = m_MsgHandler.BrokerUri = m_MgrSettings.GetParam("MessageQueueURI");
m_MsgHandler.CommandQueueName = m_MgrSettings.GetParam("ControlQueueName");
m_MsgHandler.BroadcastTopicName = m_MgrSettings.GetParam("BroadcastQueueTopic");
m_MsgHandler.StatusTopicName = m_MgrSettings.GetParam("MessageQueueTopicMgrStatus");
m_MsgHandler.MgrSettings = m_MgrSettings;
// Initialize the message queue
// Start this in a separate thread so that we can abort the initialization if necessary
InitializeMessageQueue();
if (m_MsgQueueInitSuccess)
{
//Connect message handler events
m_MsgHandler.CommandReceived += OnCommandReceived;
m_MsgHandler.BroadcastReceived += OnBroadcastReceived;
}
// Setup a file watcher for the config file
var fInfo = new FileInfo(System.Windows.Forms.Application.ExecutablePath);
m_FileWatcher = new FileSystemWatcher();
m_FileWatcher.BeginInit();
m_FileWatcher.Path = fInfo.DirectoryName;
m_FileWatcher.IncludeSubdirectories = false;
m_FileWatcher.Filter = m_MgrSettings.GetParam("configfilename");
m_FileWatcher.NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size;
m_FileWatcher.EndInit();
m_FileWatcher.EnableRaisingEvents = true;
// Subscribe to the file watcher Changed event
m_FileWatcher.Changed += FileWatcherChanged;
// Set up the tool for getting tasks
m_Task = new clsSpaceMgrTask(m_MgrSettings);
// Set up the status file class
var statusFileNameLoc = Path.Combine(fInfo.DirectoryName, "Status.xml");
m_StatusFile = new clsStatusFile(statusFileNameLoc)
{
//Note: Might want to put this back in someday
//MonitorUpdateRequired += new StatusMonitorUpdateReceived(OnStatusMonitorUpdateReceived);
LogToMsgQueue = bool.Parse(m_MgrSettings.GetParam("LogStatusToMessageQueue")),
MgrName = m_MgrName,
MgrStatus = EnumMgrStatus.Running
};
m_StatusFile.WriteStatusFile();
// Set up the status reporting time
m_StatusTimer = new System.Timers.Timer();
m_StatusTimer.BeginInit();
m_StatusTimer.Enabled = false;
m_StatusTimer.Interval = 60000; // 1 minute
m_StatusTimer.EndInit();
m_StatusTimer.Elapsed += m_StatusTimer_Elapsed;
// Get the most recent job history
var historyFile = Path.Combine(m_MgrSettings.GetParam("ApplicationPath"), "History.txt");
if (File.Exists(historyFile))
{
try
{
// Create an instance of StreamReader to read from a file.
// The using statement also closes the StreamReader.
using (var sr = new StreamReader(historyFile))
{
String line;
// Read and display lines from the file until the end of
// the file is reached.
while ((line = sr.ReadLine()) != null)
{
if (line.Contains("RecentJob: "))
{
var tmpStr = line.Replace("RecentJob: ", "");
m_StatusFile.MostRecentJobInfo = tmpStr;
break;
}
}
}
}
catch (Exception ex)
{
LogError("Exception reading status history file", ex);
}
}
// Set up the storage operations class
m_StorageOps = new clsStorageOperations(m_MgrSettings);
// Everything worked!
return true;
}
