/// <summary> /// Examine the files that exist below a dataset folder /// Auto-determine the ones that should be purged /// </summary> /// <param name="udtDatasetInfo">Dataset info</param> /// <param name="lstJobsToPurge">Jobs whose folders will be deleted</param> /// <returns>List of files that are safe to delete</returns> public SortedSet<string> FindDatasetFilesToAutoPurge(clsStorageOperations.udtDatasetInfoType udtDatasetInfo, out List<int> lstJobsToPurge) { var lstServerFilesToPurge = new SortedSet<string>(); lstJobsToPurge = new List<int>(); var reJobFolder = new Regex(@"_Auto(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); var diDatasetFolder = new DirectoryInfo(udtDatasetInfo.ServerFolderPath); // Process files in the dataset folder switch (udtDatasetInfo.RawDataType) { case "dot_raw_files": AddFilesToPurge(diDatasetFolder, "*.raw", lstServerFilesToPurge); break; case "dot_wiff_files": case "sciex_wiff_files": AddFilesToPurge(diDatasetFolder, "*.wiff", lstServerFilesToPurge); AddFilesToPurge(diDatasetFolder, "*.wiff.scan", lstServerFilesToPurge); break; case "dot_mzml_files": AddFilesToPurge(diDatasetFolder, "*.mzML", lstServerFilesToPurge); AddFilesToPurge(diDatasetFolder, "*.mzXML", lstServerFilesToPurge); break; case "dot_uimf_files": AddFilesToPurge(diDatasetFolder, "*.uimf", lstServerFilesToPurge); break; case "bruker_maldi_imaging": AddFilesToPurge(diDatasetFolder, "*.zip", lstServerFilesToPurge); break; case "zipped_s_folders": AddFilesToPurge(diDatasetFolder, "*.zip", lstServerFilesToPurge); break; case "bruker_maldi_spot": AddFilesToPurge(diDatasetFolder, "*.*", lstServerFilesToPurge); break; } // Purge all files over 2 MB in size AddFilesToPurge(diDatasetFolder, "*.*", 2048, false, lstServerFilesToPurge); // Process the directories below the dataset folder // Construct a list of the folders that exist at the dataset folder level foreach (var diSubDir in diDatasetFolder.GetDirectories()) { var subDirNameUpper = diSubDir.Name.ToUpper(); if (diSubDir.Name == "QC") // Do not purge the QC folder continue; if (subDirNameUpper.EndsWith(".D")) { // Instrument data folder AddFilesToPurge(diSubDir, "*.yep", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf AddFilesToPurge(diSubDir, "*.baf", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf AddFilesToPurge(diSubDir, "ser", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf AddFilesToPurge(diSubDir, "fid", 0, true, lstServerFilesToPurge); // bruker_ft and bruker_tof_baf AddFilesToPurge(diSubDir, "DATA.MS", 0, true, lstServerFilesToPurge); // Agilent_GC_MS_01 AddFilesToPurge(diSubDir, "MSProfile.bin", 0, true, lstServerFilesToPurge); // Agilent_QTof AddFilesToPurge(diSubDir, "MSPeak.bin", 250, true, lstServerFilesToPurge); // Agilent_QQQ AddFilesToPurge(diSubDir, "MSScan.bin", 1000, true, lstServerFilesToPurge); // Agilent_QQQ // Purge all files over 2 MB in size AddFilesToPurge(diSubDir, "*.*", 2048, true, lstServerFilesToPurge); continue; } if (subDirNameUpper.EndsWith(".RAW")) { AddFilesToPurge(diSubDir, "*.raw", 0, true, lstServerFilesToPurge); // Purge all files over 2 MB in size AddFilesToPurge(diSubDir, "*.*", 2048, true, lstServerFilesToPurge); continue; } if (subDirNameUpper.StartsWith("MSXML_GEN")) { AddFilesToPurgeDateThreshold(diSubDir, 90, lstServerFilesToPurge); continue; } if (subDirNameUpper.StartsWith("DTA_GEN") || subDirNameUpper.StartsWith("DTA_REF")) { // Purge after 1.5 years AddFilesToPurgeDateThreshold(diSubDir, 548, lstServerFilesToPurge); continue; } var reMatch = reJobFolder.Match(diSubDir.Name); if (reMatch.Success) { // This is an analysis job folder if (diSubDir.Name.StartsWith("SIC")) { // This is a MASIC folder AddFilesToPurge(diSubDir, "*.zip", lstServerFilesToPurge); // Purge all files over 15 MB in size AddFilesToPurge(diSubDir, "*.*", 15 * 1024, true, lstServerFilesToPurge); } else { // Other analysis job folders // Purge the entire folder if all files are over 3 years old var subDirPurged = AddFilesToPurgeDateThreshold(diSubDir, 3 * 365, lstServerFilesToPurge); if (!subDirPurged) { // Files are not yet 3 years old // If all of the files are 1 year old, then purge files over 50 MB DateTime dtMostRecentUpdate; var lstFiles = FindFilesAndNewestDate(diSubDir, out dtMostRecentUpdate); if (DateTime.UtcNow.Subtract(dtMostRecentUpdate).TotalDays > 365) { // Purge all files over 50 MB in size var iFilesMatched = AddFilesToPurge(diSubDir, "*.*", 50 * 1024, true, lstServerFilesToPurge); if (iFilesMatched == lstFiles.Count) subDirPurged = true; } } if (subDirPurged) { if (reMatch.Groups.Count > 0) { int jobNum; if (int.TryParse(reMatch.Groups[1].Value, out jobNum)) lstJobsToPurge.Add(jobNum); } } } continue; } // Use a threshold of 9 months for all other subfolders AddFilesToPurgeDateThreshold(diSubDir, 270, lstServerFilesToPurge); } return lstServerFilesToPurge; }
/// <summary> /// Examine each file in serverFiles and decide which is safe to delete based on the purge policy /// </summary> /// <param name="diDatasetFolder">Dataset folder to process</param> /// <param name="udtDatasetInfo">Dataset info</param> /// <param name="lstJobsToPurge">Jobs whose folders will be deleted</param> /// <returns>List of files that are safe to delete</returns> public SortedSet<string> FindDatasetFilesToPurge( DirectoryInfo diDatasetFolder, clsStorageOperations.udtDatasetInfoType udtDatasetInfo, out List<int> lstJobsToPurge) { var ePurgePolicyToApply = udtDatasetInfo.PurgePolicy; if (ePurgePolicyToApply == clsStorageOperations.PurgePolicyConstants.PurgeAll) { var lstServerFilesToPurge = new SortedSet<string>(); AddFilesToPurge(diDatasetFolder, "*.*", 0, true, lstServerFilesToPurge); lstJobsToPurge = new List<int>(); return lstServerFilesToPurge; } if (ePurgePolicyToApply == clsStorageOperations.PurgePolicyConstants.PurgeAllExceptQC) { var lstServerFilesToPurge = new SortedSet<string>(); AddFilesToPurge(diDatasetFolder, "*.*", 0, false, lstServerFilesToPurge); foreach (var diSubFolder in diDatasetFolder.GetDirectories()) { if (diSubFolder.Name != "QC") AddFilesToPurge(diSubFolder, "*.*", 0, true, lstServerFilesToPurge); } lstJobsToPurge = new List<int>(); return lstServerFilesToPurge; } // Auto-purge files for this dataset return FindDatasetFilesToAutoPurge(udtDatasetInfo, out lstJobsToPurge); }
/// <summary> /// Initializes the manager /// </summary> /// <returns>TRUE for success; FALSE otherwise</returns> public bool InitMgr() { // Get the manager settings try { m_MgrSettings = new clsMgrSettings(); } catch { // Failures are logged by clsMgrSettings return false; } // Update the cached manager name m_MgrName = m_MgrSettings.GetParam("MgrName"); // Set up the loggers var logFileName = m_MgrSettings.GetParam("logfilename"); // LogLevel is 1 to 5: 1 for Fatal errors only, 4 for Fatal, Error, Warning, and Info, and 5 for everything including Debug messages var debugLevel = int.Parse(m_MgrSettings.GetParam("debuglevel")); clsLogTools.CreateFileLogger(logFileName, debugLevel); var logCnStr = m_MgrSettings.GetParam("connectionstring"); clsLogTools.CreateDbLogger(logCnStr, "SpaceManager: " + m_MgrName); // Make initial log entry var msg = "=== Started Space Manager V" + System.Windows.Forms.Application.ProductVersion + " ===== "; clsLogTools.WriteLog(clsLogTools.LoggerTypes.LogFile, clsLogTools.LogLevels.INFO, msg); // Setup the message queue m_MsgQueueInitSuccess = false; m_MsgHandler = new clsMessageHandler(); m_MsgHandler.BrokerUri = m_MsgHandler.BrokerUri = m_MgrSettings.GetParam("MessageQueueURI"); m_MsgHandler.CommandQueueName = m_MgrSettings.GetParam("ControlQueueName"); m_MsgHandler.BroadcastTopicName = m_MgrSettings.GetParam("BroadcastQueueTopic"); m_MsgHandler.StatusTopicName = m_MgrSettings.GetParam("MessageQueueTopicMgrStatus"); m_MsgHandler.MgrSettings = m_MgrSettings; // Initialize the message queue // Start this in a separate thread so that we can abort the initialization if necessary InitializeMessageQueue(); if (m_MsgQueueInitSuccess) { //Connect message handler events m_MsgHandler.CommandReceived += OnCommandReceived; m_MsgHandler.BroadcastReceived += OnBroadcastReceived; } // Setup a file watcher for the config file var fInfo = new FileInfo(System.Windows.Forms.Application.ExecutablePath); m_FileWatcher = new FileSystemWatcher(); m_FileWatcher.BeginInit(); m_FileWatcher.Path = fInfo.DirectoryName; m_FileWatcher.IncludeSubdirectories = false; m_FileWatcher.Filter = m_MgrSettings.GetParam("configfilename"); m_FileWatcher.NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size; m_FileWatcher.EndInit(); m_FileWatcher.EnableRaisingEvents = true; // Subscribe to the file watcher Changed event m_FileWatcher.Changed += FileWatcherChanged; // Set up the tool for getting tasks m_Task = new clsSpaceMgrTask(m_MgrSettings); // Set up the status file class var statusFileNameLoc = Path.Combine(fInfo.DirectoryName, "Status.xml"); m_StatusFile = new clsStatusFile(statusFileNameLoc) { //Note: Might want to put this back in someday //MonitorUpdateRequired += new StatusMonitorUpdateReceived(OnStatusMonitorUpdateReceived); LogToMsgQueue = bool.Parse(m_MgrSettings.GetParam("LogStatusToMessageQueue")), MgrName = m_MgrName, MgrStatus = EnumMgrStatus.Running }; m_StatusFile.WriteStatusFile(); // Set up the status reporting time m_StatusTimer = new System.Timers.Timer(); m_StatusTimer.BeginInit(); m_StatusTimer.Enabled = false; m_StatusTimer.Interval = 60000; // 1 minute m_StatusTimer.EndInit(); m_StatusTimer.Elapsed += m_StatusTimer_Elapsed; // Get the most recent job history var historyFile = Path.Combine(m_MgrSettings.GetParam("ApplicationPath"), "History.txt"); if (File.Exists(historyFile)) { try { // Create an instance of StreamReader to read from a file. // The using statement also closes the StreamReader. using (var sr = new StreamReader(historyFile)) { String line; // Read and display lines from the file until the end of // the file is reached. while ((line = sr.ReadLine()) != null) { if (line.Contains("RecentJob: ")) { var tmpStr = line.Replace("RecentJob: ", ""); m_StatusFile.MostRecentJobInfo = tmpStr; break; } } } } catch (Exception ex) { LogError("Exception reading status history file", ex); } } // Set up the storage operations class m_StorageOps = new clsStorageOperations(m_MgrSettings); // Everything worked! return true; }