/// <summary>
/// Aggregate Data and return UnitInfo List
/// </summary>
public DataAggregatorResult AggregateData(FahLog fahLog, QueueData queueData, UnitInfoLogData unitInfo)
{
if (Logger.IsDebugEnabled)
{
foreach (var s in fahLog.Where(x => x.LineType == LogLineType.Error))
{
Logger.DebugFormat(Constants.ClientNameFormat, ClientName, String.Format("Failed to parse log line: {0}", s));
}
}
var currentClientRun = GetCurrentClientRun(fahLog);
if (currentClientRun == null)
{
return null;
}
var result = new DataAggregatorResult();
result.StartTime = currentClientRun.Data.StartTime;
result.Arguments = currentClientRun.Data.Arguments;
result.ClientVersion = currentClientRun.Data.ClientVersion;
result.UserID = currentClientRun.Data.UserID;
result.MachineID = currentClientRun.Data.MachineID;
result.Status = currentClientRun.SlotRuns[0].Data.Status;
// Decision Time: If Queue Read fails parse from logs only
if (queueData != null)
{
GenerateUnitInfoDataFromQueue(result, queueData, fahLog, unitInfo);
result.Queue = BuildClientQueue(queueData);
result.CurrentUnitIndex = result.Queue.CurrentIndex;
}
else
{
Logger.WarnFormat(Constants.ClientNameFormat, ClientName,
"Queue unavailable or failed read. Parsing logs without queue.");
GenerateUnitInfoDataFromLogs(result, fahLog, unitInfo);
// default Unit Index if only parsing logs
result.CurrentUnitIndex = 1;
}
if (result.UnitInfos == null || result.UnitInfos[result.CurrentUnitIndex] == null || result.UnitInfos[result.CurrentUnitIndex].LogLines == null)
{
result.CurrentLogLines = currentClientRun.ToList();
}
else
{
result.CurrentLogLines = result.UnitInfos[result.CurrentUnitIndex].LogLines;
}
return result;
}
/// <summary>
/// Reads the content from a unitinfo.txt file.
/// </summary>
/// <param name="path">Path to the log file.</param>
/// <exception cref="System.ArgumentException">if path is null or empty.</exception>
/// <exception cref="System.FormatException">if log data fails parsing.</exception>
public static UnitInfoLogData Read(string path)
{
if (String.IsNullOrEmpty(path))
{
throw new ArgumentException("Argument 'path' cannot be a null or empty string.");
}
string[] logLines = File.ReadAllLines(path);
var data = new UnitInfoLogData();
string line = null;
try
{
foreach (string s in logLines)
{
line = s;
/* Name (Only Read Here) */
if (line.StartsWith("Name: "))
{
data.ProteinName = line.Substring(6);
}
/* Tag (Could be read here or through the queue.dat) */
else if (line.StartsWith("Tag: "))
{
data.ProteinTag = line.Substring(5);
Match projectNumberFromTagMatch;
if ((projectNumberFromTagMatch = UnitInfoLogRegex.RegexProjectNumberFromTag.Match(data.ProteinTag)).Success)
{
data.ProjectID = Int32.Parse(projectNumberFromTagMatch.Groups["ProjectNumber"].Value);
data.ProjectRun = Int32.Parse(projectNumberFromTagMatch.Groups["Run"].Value);
data.ProjectClone = Int32.Parse(projectNumberFromTagMatch.Groups["Clone"].Value);
data.ProjectGen = Int32.Parse(projectNumberFromTagMatch.Groups["Gen"].Value);
}
}
/* DownloadTime (Could be read here or through the queue.dat) */
else if (line.StartsWith("Download time: "))
{
data.DownloadTime = DateTime.ParseExact(line.Substring(15), "MMMM d H:mm:ss",
DateTimeFormatInfo.InvariantInfo,
DateTimeParse.Styles);
}
/* DueTime (Could be read here or through the queue.dat) */
else if (line.StartsWith("Due time: "))
{
data.DueTime = DateTime.ParseExact(line.Substring(10), "MMMM d H:mm:ss",
DateTimeFormatInfo.InvariantInfo,
DateTimeParse.Styles);
}
/* Progress (Supplemental Read - if progress percentage cannot be determined through FAHlog.txt) */
else if (line.StartsWith("Progress: "))
{
data.Progress = Int32.Parse(line.Substring(10, line.IndexOf("%") - 10));
}
}
}
catch (Exception ex)
{
throw new FormatException(String.Format(CultureInfo.CurrentCulture, "Failed to parse line '{0}'", line), ex);
}
return data;
}
private UnitInfo[] GenerateUnitInfoDataFromQueue(QueueData q)
{
Debug.Assert(q != null);
var parsedUnits = new UnitInfo[10];
_unitLogLines = new IList<LogLine>[10];
for (int queueIndex = 0; queueIndex < parsedUnits.Length; queueIndex++)
{
// Get the Log Lines for this queue position from the reader
_unitLogLines[queueIndex] = _logInterpreterLegacy.GetLogLinesForQueueIndex(queueIndex);
// Get the FAH Log Data from the Log Lines
FahLogUnitData fahLogUnitData = LogReader.GetFahLogDataFromLogLines(_unitLogLines[queueIndex]);
UnitInfoLogData unitInfoLogData = null;
// On the Current Queue Index
if (queueIndex == q.CurrentIndex)
{
// Get the UnitInfo Log Data
unitInfoLogData = GetUnitInfoLogData();
}
parsedUnits[queueIndex] = BuildUnitInfo(q.GetQueueEntry((uint)queueIndex), fahLogUnitData, unitInfoLogData);
if (parsedUnits[queueIndex] == null)
{
if (queueIndex == q.CurrentIndex)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not verify log section for current queue entry {0}. Trying to parse with most recent log section.", queueIndex);
_logger.Warn(Constants.ClientNameFormat, ClientName, message);
_unitLogLines[queueIndex] = _logInterpreterLegacy.CurrentWorkUnitLogLines;
// If got no Work Unit Log Lines based on Current Work Unit Log Lines
// then take the entire Current Client Run Log Lines - likely the run
// was short and never contained any Work Unit Data.
if (_unitLogLines[queueIndex] == null)
{
_unitLogLines[queueIndex] = _logInterpreterLegacy.CurrentClientRunLogLines;
}
fahLogUnitData = LogReader.GetFahLogDataFromLogLines(_unitLogLines[queueIndex]);
if (_currentClientRun.Status.Equals(SlotStatus.GettingWorkPacket))
{
// Use either the current Work Unit log lines or current Client Run log lines
// as decided upon above... don't clear it here and show the user nothing - 10/9/10
//_unitLogLines[queueIndex] = null;
fahLogUnitData = new FahLogUnitData();
unitInfoLogData = new UnitInfoLogData();
}
parsedUnits[queueIndex] = BuildUnitInfo(q.GetQueueEntry((uint)queueIndex), fahLogUnitData, unitInfoLogData, true);
}
else
{
// Just skip this unit and continue
string message = String.Format(CultureInfo.CurrentCulture,
"Could not find or verify log section for queue entry {0} (this is not a problem).", queueIndex);
_logger.Debug(Constants.ClientNameFormat, ClientName, message);
}
}
}
return parsedUnits;
}
private UnitInfo BuildUnitInfo(QueueEntry queueEntry, FahLogUnitData fahLogUnitData, UnitInfoLogData unitInfoLogData, bool matchOverride)
{
// queueEntry can be null
Debug.Assert(fahLogUnitData != null);
// unitInfoLogData can be null
var unit = new UnitInfo();
unit.UnitStartTimeStamp = fahLogUnitData.UnitStartTimeStamp;
unit.FramesObserved = fahLogUnitData.FramesObserved;
unit.CoreVersion = fahLogUnitData.CoreVersion;
unit.UnitResult = fahLogUnitData.UnitResult;
if (queueEntry != null)
{
PopulateUnitInfoFromQueueEntry(queueEntry, unit);
SearchFahLogUnitDataProjects(unit, fahLogUnitData);
PopulateUnitInfoFromLogs(CurrentClientRun, fahLogUnitData, unitInfoLogData, unit);
if (ProjectsMatch(unit, fahLogUnitData) ||
ProjectsMatch(unit, unitInfoLogData) ||
matchOverride)
{
// continue parsing the frame data
ParseFrameData(fahLogUnitData.FrameDataList, unit);
}
else
{
return null;
}
}
else
{
PopulateUnitInfoFromLogs(CurrentClientRun, fahLogUnitData, unitInfoLogData, unit);
ParseFrameData(fahLogUnitData.FrameDataList, unit);
}
return unit;
}
private UnitInfo BuildUnitInfo(QueueEntry queueEntry, FahLogUnitData fahLogUnitData, UnitInfoLogData unitInfoLogData)
{
return BuildUnitInfo(queueEntry, fahLogUnitData, unitInfoLogData, false);
}
private static void PopulateUnitInfoFromLogs(ClientRun currentClientRun, FahLogUnitData fahLogUnitData,
UnitInfoLogData unitInfoLogData, UnitInfo unit)
{
Debug.Assert(currentClientRun != null);
Debug.Assert(fahLogUnitData != null);
// unitInfoLogData can be null
Debug.Assert(unit != null);
/* Project (R/C/G) (Could have already been read through Queue) */
if (unit.ProjectIsUnknown())
{
unit.ProjectID = fahLogUnitData.ProjectID;
unit.ProjectRun = fahLogUnitData.ProjectRun;
unit.ProjectClone = fahLogUnitData.ProjectClone;
unit.ProjectGen = fahLogUnitData.ProjectGen;
}
if (fahLogUnitData.Threads > 1)
{
unit.SlotType = SlotType.CPU;
}
if (unitInfoLogData != null)
{
unit.ProteinName = unitInfoLogData.ProteinName;
/* Tag (Could have already been read through Queue) */
if (unit.ProteinTag.Length == 0)
{
unit.ProteinTag = unitInfoLogData.ProteinTag;
}
/* DownloadTime (Could have already been read through Queue) */
if (unit.DownloadTime.IsUnknown())
{
unit.DownloadTime = unitInfoLogData.DownloadTime;
}
/* DueTime (Could have already been read through Queue) */
if (unit.DueTime.IsUnknown())
{
unit.DueTime = unitInfoLogData.DueTime;
}
/* FinishedTime (Not available in unitinfo log) */
/* Project (R/C/G) (Could have already been read through Queue) */
if (unit.ProjectIsUnknown())
{
unit.ProjectID = unitInfoLogData.ProjectID;
unit.ProjectRun = unitInfoLogData.ProjectRun;
unit.ProjectClone = unitInfoLogData.ProjectClone;
unit.ProjectGen = unitInfoLogData.ProjectGen;
}
}
/* FoldingID and Team from Last Client Run (Could have already been read through Queue) */
if (unit.FoldingID.Equals(Constants.DefaultFoldingID) &&
!String.IsNullOrEmpty(currentClientRun.FoldingID))
{
unit.FoldingID = currentClientRun.FoldingID;
}
if (unit.Team == Constants.DefaultTeam)
{
unit.Team = currentClientRun.Team;
}
// Possibly check the currentClientRun from the log file.
// The queue will have the ID and Team that was set when the work unit was received.
//if (unit.FoldingID.Equals(Default.FoldingID) ||
// !unit.FoldingID.Equals(currentClientRun.FoldingID))
//{
// unit.FoldingID = currentClientRun.FoldingID;
//}
//if (unit.Team == Default.Team ||
// unit.Team != currentClientRun.Team)
//{
// unit.Team = currentClientRun.Team;
//}
}
private static void UpdateUnitInfoFromLogData(UnitInfo unitInfo, ClientRunData clientRunData, UnitRunData unitRunData, UnitInfoLogData unitInfoLogData)
{
Debug.Assert(unitInfo != null);
Debug.Assert(clientRunData != null);
Debug.Assert(unitRunData != null);
// unitInfoLogData can be null
/* Project (R/C/G) (Could have already been read through Queue) */
if (unitInfo.ProjectIsUnknown())
{
unitInfo.ProjectID = unitRunData.ProjectID;
unitInfo.ProjectRun = unitRunData.ProjectRun;
unitInfo.ProjectClone = unitRunData.ProjectClone;
unitInfo.ProjectGen = unitRunData.ProjectGen;
}
if (unitRunData.Threads > 1)
{
unitInfo.SlotType = SlotType.CPU;
}
if (unitInfoLogData != null)
{
unitInfo.ProteinName = unitInfoLogData.ProteinName;
/* Tag (Could have already been read through Queue) */
if (unitInfo.ProteinTag.Length == 0)
{
unitInfo.ProteinTag = unitInfoLogData.ProteinTag;
}
/* DownloadTime (Could have already been read through Queue) */
if (unitInfo.DownloadTime.IsUnknown())
{
unitInfo.DownloadTime = unitInfoLogData.DownloadTime;
}
/* DueTime (Could have already been read through Queue) */
if (unitInfo.DueTime.IsUnknown())
{
unitInfo.DueTime = unitInfoLogData.DueTime;
}
/* FinishedTime (Not available in unitinfo log) */
/* Project (R/C/G) (Could have already been read through Queue) */
if (unitInfo.ProjectIsUnknown())
{
unitInfo.ProjectID = unitInfoLogData.ProjectID;
unitInfo.ProjectRun = unitInfoLogData.ProjectRun;
unitInfo.ProjectClone = unitInfoLogData.ProjectClone;
unitInfo.ProjectGen = unitInfoLogData.ProjectGen;
}
}
/* FoldingID and Team from last ClientRun (Could have already been read through Queue) */
if (unitInfo.FoldingID == Constants.DefaultFoldingID && unitInfo.Team == Constants.DefaultTeam)
{
if (!String.IsNullOrEmpty(clientRunData.FoldingID))
{
unitInfo.FoldingID = clientRunData.FoldingID;
unitInfo.Team = clientRunData.Team;
}
}
// The queue will have the FoldingID and Team that was set in the client when the work unit was assigned.
// If the user subsequently changed their FoldingID and Team before this unit was completed the
// FoldingID and Team read from the queue will NOT reflect that change.
//if (unitInfo.FoldingID != clientRunData.FoldingID || unitInfo.Team != clientRunData.Team)
//{
// if (!String.IsNullOrEmpty(clientRunData.FoldingID))
// {
// unitInfo.FoldingID = clientRunData.FoldingID;
// unitInfo.Team = clientRunData.Team;
// }
//}
}
private static UnitInfo BuildUnitInfo(QueueEntry queueEntry, ClientRun clientRun, UnitRun unitRun, UnitInfoLogData unitInfoLogData, bool matchOverride = false)
{
// queueEntry can be null
Debug.Assert(clientRun != null);
// unitInfoLogData can be null
var unit = new UnitInfo();
UnitRunData unitRunData;
if (unitRun == null)
{
if (matchOverride)
{
unitRunData = new UnitRunData();
}
else
{
return null;
}
}
else
{
unit.LogLines = unitRun.ToList();
unitRunData = unitRun.Data;
}
unit.UnitStartTimeStamp = unitRunData.UnitStartTimeStamp ?? TimeSpan.MinValue;
unit.FramesObserved = unitRunData.FramesObserved;
unit.CoreVersion = unitRunData.CoreVersion;
unit.UnitResult = unitRunData.WorkUnitResult;
if (queueEntry != null)
{
UpdateUnitInfoFromQueueData(unit, queueEntry);
SearchFahLogUnitDataProjects(unit, unitRunData);
UpdateUnitInfoFromLogData(unit, clientRun.Data, unitRunData, unitInfoLogData);
if (!ProjectsMatch(unit, unitRunData) && !ProjectsMatch(unit, unitInfoLogData) && !matchOverride)
{
return null;
}
}
else
{
UpdateUnitInfoFromLogData(unit, clientRun.Data, unitRunData, unitInfoLogData);
}
return unit;
}
private void GenerateUnitInfoDataFromQueue(DataAggregatorResult result, QueueData q, FahLog fahLog, UnitInfoLogData unitInfo)
{
Debug.Assert(q != null);
result.UnitInfos = new Dictionary<int, UnitInfo>(10);
for (int i = 0; i < 10; i++)
{
result.UnitInfos[i] = null;
}
var clientRun = GetCurrentClientRun(fahLog);
for (int queueIndex = 0; queueIndex < result.UnitInfos.Count; queueIndex++)
{
var unitRun = GetUnitRunForQueueIndex(fahLog, queueIndex);
UnitInfoLogData unitInfoLogData = null;
// On the Current Queue Index
if (queueIndex == q.CurrentIndex)
{
// Get the UnitInfo Log Data
unitInfoLogData = unitInfo;
}
var queueEntry = q.GetQueueEntry((uint)queueIndex);
result.UnitInfos[queueIndex] = BuildUnitInfo(queueEntry, clientRun, unitRun, unitInfoLogData);
if (result.UnitInfos[queueIndex] == null)
{
if (queueIndex == q.CurrentIndex)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not verify log section for current queue index {0} {1}.", queueIndex, queueEntry.ToProjectInfo());
Logger.WarnFormat(Constants.ClientNameFormat, ClientName, message);
unitRun = GetCurrentUnitRun(fahLog);
var slotRun = GetCurrentSlotRun(fahLog);
if (slotRun != null && slotRun.Data.Status == SlotStatus.GettingWorkPacket)
{
unitRun = null;
unitInfoLogData = new UnitInfoLogData();
}
result.UnitInfos[queueIndex] = BuildUnitInfo(queueEntry, clientRun, unitRun, unitInfoLogData, true);
}
else if (Logger.IsDebugEnabled)
{
// Write debug info and skip this unit
var projectInfo = queueEntry.ToProjectInfo();
if (!projectInfo.ProjectIsUnknown())
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not find log section for queue index {0} {1}.", queueIndex, queueEntry.ToProjectInfo());
Logger.DebugFormat(Constants.ClientNameFormat, ClientName, message);
}
}
}
}
}
private void GenerateUnitInfoDataFromLogs(DataAggregatorResult result, FahLog fahLog, UnitInfoLogData unitInfo)
{
result.UnitInfos = new Dictionary<int, UnitInfo>(2);
for (int i = 0; i < 2; i++)
{
result.UnitInfos[i] = null;
}
var currentClientRun = GetCurrentClientRun(fahLog);
var previousUnitRun = GetPreviousUnitRun(fahLog);
if (previousUnitRun != null)
{
result.UnitInfos[0] = BuildUnitInfo(null, currentClientRun, previousUnitRun, null);
}
var currentUnitRun = GetCurrentUnitRun(fahLog);
result.UnitInfos[1] = BuildUnitInfo(null, currentClientRun, currentUnitRun, unitInfo, currentUnitRun == null);
}
/// <summary>
/// Parse the content from the unitinfo.txt file.
/// </summary>
/// <param name="logFilePath">Path to the log file.</param>
/// <exception cref="System.ArgumentException">Throws if logFilePath is null or empty.</exception>
/// <exception cref="System.IO.IOException">Throws if file specified by logFilePath cannot be read.</exception>
/// <exception cref="System.FormatException">Throws if log data fails parsing.</exception>
public static UnitInfoLogData GetUnitInfoLogData(string logFilePath)
{
if (String.IsNullOrEmpty(logFilePath))
{
throw new ArgumentException("Argument 'logFilePath' cannot be a null or empty string.");
}
string[] logLines;
try
{
logLines = File.ReadAllLines(logFilePath);
}
catch (Exception ex)
{
throw new IOException(String.Format(CultureInfo.CurrentCulture, "Failed to read file '{0}'", logFilePath), ex);
}
var data = new UnitInfoLogData();
string line = null;
try
{
foreach (string s in logLines)
{
line = s;
/* Name (Only Read Here) */
if (line.StartsWith("Name: "))
{
data.ProteinName = line.Substring(6);
}
/* Tag (Could be read here or through the queue.dat) */
else if (line.StartsWith("Tag: "))
{
data.ProteinTag = line.Substring(5);
Match mProjectNumberFromTag;
if ((mProjectNumberFromTag = RegexProjectNumberFromTag.Match(data.ProteinTag)).Success)
{
data.ProjectID = Int32.Parse(mProjectNumberFromTag.Result("${ProjectNumber}"));
data.ProjectRun = Int32.Parse(mProjectNumberFromTag.Result("${Run}"));
data.ProjectClone = Int32.Parse(mProjectNumberFromTag.Result("${Clone}"));
data.ProjectGen = Int32.Parse(mProjectNumberFromTag.Result("${Gen}"));
}
}
/* DownloadTime (Could be read here or through the queue.dat) */
else if (line.StartsWith("Download time: "))
{
data.DownloadTime = DateTime.ParseExact(line.Substring(15), "MMMM d H:mm:ss",
DateTimeFormatInfo.InvariantInfo,
LogReaderExtensions.DateTimeStyle);
}
/* DueTime (Could be read here or through the queue.dat) */
else if (line.StartsWith("Due time: "))
{
data.DueTime = DateTime.ParseExact(line.Substring(10), "MMMM d H:mm:ss",
DateTimeFormatInfo.InvariantInfo,
LogReaderExtensions.DateTimeStyle);
}
/* Progress (Supplemental Read - if progress percentage cannot be determined through FAHlog.txt) */
else if (line.StartsWith("Progress: "))
{
data.Progress = Int32.Parse(line.Substring(10, line.IndexOf("%") - 10));
}
}
}
catch (Exception ex)
{
throw new FormatException(String.Format(CultureInfo.CurrentCulture, "Failed to parse line '{0}'", line), ex);
}
return(data);
}
