/// <summary>
/// Aggregate Data and return UnitInfo List
/// </summary>
public DataAggregatorResult AggregateData(FahLog fahLog, QueueData queueData, UnitInfoLogData unitInfo)
{
if (Logger.IsDebugEnabled)
{
foreach (var s in fahLog.Where(x => x.LineType == LogLineType.Error))
{
Logger.DebugFormat(Constants.ClientNameFormat, ClientName, String.Format("Failed to parse log line: {0}", s));
}
}
var currentClientRun = GetCurrentClientRun(fahLog);
if (currentClientRun == null)
{
return null;
}
var result = new DataAggregatorResult();
result.StartTime = currentClientRun.Data.StartTime;
result.Arguments = currentClientRun.Data.Arguments;
result.ClientVersion = currentClientRun.Data.ClientVersion;
result.UserID = currentClientRun.Data.UserID;
result.MachineID = currentClientRun.Data.MachineID;
result.Status = currentClientRun.SlotRuns[0].Data.Status;
// Decision Time: If Queue Read fails parse from logs only
if (queueData != null)
{
GenerateUnitInfoDataFromQueue(result, queueData, fahLog, unitInfo);
result.Queue = BuildClientQueue(queueData);
result.CurrentUnitIndex = result.Queue.CurrentIndex;
}
else
{
Logger.WarnFormat(Constants.ClientNameFormat, ClientName,
"Queue unavailable or failed read. Parsing logs without queue.");
GenerateUnitInfoDataFromLogs(result, fahLog, unitInfo);
// default Unit Index if only parsing logs
result.CurrentUnitIndex = 1;
}
if (result.UnitInfos == null || result.UnitInfos[result.CurrentUnitIndex] == null || result.UnitInfos[result.CurrentUnitIndex].LogLines == null)
{
result.CurrentLogLines = currentClientRun.ToList();
}
else
{
result.CurrentLogLines = result.UnitInfos[result.CurrentUnitIndex].LogLines;
}
return result;
}
/// <summary>
/// Process the cached log files that exist on this machine
/// </summary>
private void Process()
{
var sw = Stopwatch.StartNew();
// Set successful Last Retrieval Time
LastRetrievalTime = DateTime.Now;
// Re-Init Slot Level Members Before Processing
_slotModel.Initialize();
#region Setup Aggregator
var dataAggregator = new LegacyDataAggregator {
Logger = Logger
};
dataAggregator.ClientName = Settings.Name;
string queueFilePath = Path.Combine(Prefs.CacheDirectory, Settings.CachedQueueFileName());
string fahLogFilePath = Path.Combine(Prefs.CacheDirectory, Settings.CachedFahLogFileName());
string unitInfoLogFilePath = Path.Combine(Prefs.CacheDirectory, Settings.CachedUnitInfoFileName());
#endregion
#region Run the Aggregator
var queue = ReadQueueFile(queueFilePath);
var fahLog = FahLog.Read(File.ReadLines(fahLogFilePath), FahLogType.Legacy);
var unitInfo = ReadUnitInfoFile(unitInfoLogFilePath);
var result = dataAggregator.AggregateData(fahLog, queue, unitInfo);
// Issue 126 - Use the Folding ID, Team, User ID, and Machine ID from the FAHlog data.
// Use the Current Queue Entry as a backup data source.
PopulateRunLevelData(result, _slotModel);
if (result.Queue != null)
{
PopulateRunLevelData(result.Queue.Current, _slotModel);
}
_slotModel.Queue = result.Queue;
_slotModel.CurrentLogLines = result.CurrentLogLines;
_slotModel.UnitLogLines = result.UnitInfos.OrderBy(x => x.Key).Select(x => x.Value != null ? x.Value.LogLines : null).ToArray();
#endregion
var parsedUnits = new UnitInfoModel[result.UnitInfos.Count];
for (int i = 0; i < result.UnitInfos.Count; i++)
{
if (result.UnitInfos[i] != null)
{
parsedUnits[i] = BuildUnitInfoLogic(result.UnitInfos[i], true);
}
}
// *** THIS HAS TO BE DONE BEFORE UPDATING SlotModel.UnitInfoLogic ***
UpdateBenchmarkData(_slotModel.UnitInfoModel, parsedUnits, result.CurrentUnitIndex);
// Update the UnitInfoLogic if we have a Status
if (result.Status != SlotStatus.Unknown)
{
_slotModel.UnitInfoModel = parsedUnits[result.CurrentUnitIndex];
}
HandleReturnedStatus(result.Status, _slotModel);
_slotModel.UnitInfoModel.ShowPPDTrace(Logger, _slotModel.Name, _slotModel.Status,
Prefs.Get <PpdCalculationType>(Preference.PpdCalculation),
Prefs.Get <BonusCalculationType>(Preference.BonusCalculation));
string statusMessage = String.Format(CultureInfo.CurrentCulture, "Client Status: {0}", _slotModel.Status);
Logger.InfoFormat(Constants.ClientNameFormat, _slotModel.Name, statusMessage);
string message = String.Format(CultureInfo.CurrentCulture, "Retrieval finished: {0}", sw.GetExecTime());
Logger.InfoFormat(Constants.ClientNameFormat, Settings.Name, message);
}
private void GenerateUnitInfoDataFromQueue(DataAggregatorResult result, QueueData q, FahLog fahLog, UnitInfoLogData unitInfo)
{
Debug.Assert(q != null);
result.UnitInfos = new Dictionary<int, UnitInfo>(10);
for (int i = 0; i < 10; i++)
{
result.UnitInfos[i] = null;
}
var clientRun = GetCurrentClientRun(fahLog);
for (int queueIndex = 0; queueIndex < result.UnitInfos.Count; queueIndex++)
{
var unitRun = GetUnitRunForQueueIndex(fahLog, queueIndex);
UnitInfoLogData unitInfoLogData = null;
// On the Current Queue Index
if (queueIndex == q.CurrentIndex)
{
// Get the UnitInfo Log Data
unitInfoLogData = unitInfo;
}
var queueEntry = q.GetQueueEntry((uint)queueIndex);
result.UnitInfos[queueIndex] = BuildUnitInfo(queueEntry, clientRun, unitRun, unitInfoLogData);
if (result.UnitInfos[queueIndex] == null)
{
if (queueIndex == q.CurrentIndex)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not verify log section for current queue index {0} {1}.", queueIndex, queueEntry.ToProjectInfo());
Logger.WarnFormat(Constants.ClientNameFormat, ClientName, message);
unitRun = GetCurrentUnitRun(fahLog);
var slotRun = GetCurrentSlotRun(fahLog);
if (slotRun != null && slotRun.Data.Status == SlotStatus.GettingWorkPacket)
{
unitRun = null;
unitInfoLogData = new UnitInfoLogData();
}
result.UnitInfos[queueIndex] = BuildUnitInfo(queueEntry, clientRun, unitRun, unitInfoLogData, true);
}
else if (Logger.IsDebugEnabled)
{
// Write debug info and skip this unit
var projectInfo = queueEntry.ToProjectInfo();
if (!projectInfo.ProjectIsUnknown())
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not find log section for queue index {0} {1}.", queueIndex, queueEntry.ToProjectInfo());
Logger.DebugFormat(Constants.ClientNameFormat, ClientName, message);
}
}
}
}
}
private static UnitRun GetUnitRunForQueueIndex(FahLog fahLog, int queueIndex)
{
foreach (var clientRun in fahLog.ClientRuns)
{
var slotRun = clientRun.SlotRuns[0];
var unitRun = slotRun.UnitRuns.FirstOrDefault(x => x.QueueIndex == queueIndex);
if (unitRun != null)
{
return unitRun;
}
}
return null;
}
private static UnitRun GetPreviousUnitRun(FahLog fahLog)
{
Debug.Assert(fahLog != null);
var slotRun = GetCurrentSlotRun(fahLog);
return slotRun != null && slotRun.UnitRuns.Count > 1 ? slotRun.UnitRuns.ElementAt(1) : null;
}
private static UnitRun GetCurrentUnitRun(FahLog fahLog)
{
Debug.Assert(fahLog != null);
var slotRun = GetCurrentSlotRun(fahLog);
return slotRun != null ? slotRun.UnitRuns.FirstOrDefault() : null;
}
private static ClientRun GetCurrentClientRun(FahLog fahLog)
{
Debug.Assert(fahLog != null);
return fahLog.ClientRuns.FirstOrDefault();
}
private static SlotRun GetCurrentSlotRun(FahLog fahLog)
{
Debug.Assert(fahLog != null);
var clientRun = GetCurrentClientRun(fahLog);
return clientRun != null ? clientRun.SlotRuns[0] : null;
}
public void Client_v7_10_0()
{
const int slotId = 0;
_dataAggregator.ClientName = "Client_v7_10";
var fahLog = FahLog.Read(File.ReadLines("..\\..\\..\\TestFiles\\Client_v7_10\\log.txt"), FahLogType.FahClient);
string message = File.ReadAllText("..\\..\\..\\TestFiles\\Client_v7_10\\units.txt");
var unitCollection = new UnitCollection();
unitCollection.Fill(JsonMessageConnection.GetNextJsonMessage(ref message));
message = File.ReadAllText("..\\..\\..\\TestFiles\\Client_v7_10\\info.txt");
var info = new Info();
info.Fill(JsonMessageConnection.GetNextJsonMessage(ref message));
message = File.ReadAllText("..\\..\\..\\TestFiles\\Client_v7_10\\options.txt");
var options = new Options();
options.Fill(JsonMessageConnection.GetNextJsonMessage(ref message));
message = File.ReadAllText("..\\..\\..\\TestFiles\\Client_v7_10\\slot-options1.txt");
var slotOptions = new SlotOptions();
slotOptions.Fill(JsonMessageConnection.GetNextJsonMessage(ref message));
var result = _dataAggregator.AggregateData(fahLog.ClientRuns.First(), unitCollection, info, options, slotOptions, new UnitInfo(), slotId);
Assert.AreEqual(1, result.UnitInfos.Count);
Assert.IsFalse(result.UnitInfos.Any(x => x.Value == null));
#region Check Data Aggregator
Assert.IsNotNull(result.Queue);
Assert.AreEqual(1, result.CurrentUnitIndex);
Assert.AreEqual(new DateTime(2012, 1, 11, 3, 24, 22), result.StartTime);
Assert.AreEqual(null, result.Arguments);
Assert.AreEqual(null, result.ClientVersion);
Assert.AreEqual(null, result.UserID);
Assert.AreEqual(0, result.MachineID);
Assert.AreEqual(SlotStatus.Unknown, result.Status);
Assert.IsNotNull(result.CurrentLogLines);
Assert.IsFalse(result.UnitInfos.Any(x => x.Value.LogLines == null));
if (result.UnitInfos.ContainsKey(result.CurrentUnitIndex))
{
Assert.AreEqual(result.CurrentLogLines, result.UnitInfos[result.CurrentUnitIndex].LogLines);
}
#endregion
var unitInfoData = result.UnitInfos[result.CurrentUnitIndex];
#region Check Unit Info Data Values
Assert.AreEqual(null, unitInfoData.OwningSlotName);
Assert.AreEqual(null, unitInfoData.OwningClientName);
Assert.AreEqual(null, unitInfoData.OwningClientPath);
Assert.AreEqual(-1, unitInfoData.OwningSlotId);
Assert.AreEqual(DateTime.MinValue, unitInfoData.UnitRetrievalTime);
Assert.AreEqual("harlam357", unitInfoData.FoldingID);
Assert.AreEqual(32, unitInfoData.Team);
Assert.AreEqual(SlotType.CPU, unitInfoData.SlotType);
Assert.AreEqual(new DateTime(2012, 1, 10, 23, 20, 27), unitInfoData.DownloadTime);
Assert.AreEqual(new DateTime(2012, 1, 22, 16, 22, 51), unitInfoData.DueTime);
Assert.AreEqual(new TimeSpan(3, 25, 32), unitInfoData.UnitStartTimeStamp);
Assert.AreEqual(DateTime.MinValue, unitInfoData.FinishedTime);
Assert.AreEqual(2.27f, unitInfoData.CoreVersion);
Assert.AreEqual(7610, unitInfoData.ProjectID);
Assert.AreEqual(630, unitInfoData.ProjectRun);
Assert.AreEqual(0, unitInfoData.ProjectClone);
Assert.AreEqual(59, unitInfoData.ProjectGen);
Assert.AreEqual(String.Empty, unitInfoData.ProteinName);
Assert.AreEqual(String.Empty, unitInfoData.ProteinTag);
Assert.AreEqual(WorkUnitResult.Unknown, unitInfoData.UnitResult);
Assert.AreEqual(660000, unitInfoData.RawFramesComplete);
Assert.AreEqual(2000000, unitInfoData.RawFramesTotal);
Assert.AreEqual(10, unitInfoData.FramesObserved);
Assert.AreEqual(33, unitInfoData.CurrentFrame.FrameID);
Assert.AreEqual(new TimeSpan(4, 46, 8), unitInfoData.CurrentFrame.TimeOfFrame);
Assert.AreEqual(new TimeSpan(0, 8, 31), unitInfoData.CurrentFrame.FrameDuration);
Assert.AreEqual("A4", unitInfoData.CoreID);
#endregion
}
private void GenerateUnitInfoDataFromLogs(DataAggregatorResult result, FahLog fahLog, UnitInfoLogData unitInfo)
{
result.UnitInfos = new Dictionary<int, UnitInfo>(2);
for (int i = 0; i < 2; i++)
{
result.UnitInfos[i] = null;
}
var currentClientRun = GetCurrentClientRun(fahLog);
var previousUnitRun = GetPreviousUnitRun(fahLog);
if (previousUnitRun != null)
{
result.UnitInfos[0] = BuildUnitInfo(null, currentClientRun, previousUnitRun, null);
}
var currentUnitRun = GetCurrentUnitRun(fahLog);
result.UnitInfos[1] = BuildUnitInfo(null, currentClientRun, currentUnitRun, unitInfo, currentUnitRun == null);
}
private void PopulateClientRunsInTree(FahLog fahLog)
{
int i = 0;
foreach (var clientRun in fahLog.ClientRuns.Reverse())
{
treeView1.Nodes.Add(i.ToString(), "Run " + i);
foreach (var slotRun in clientRun.SlotRuns.Values)
{
treeView1.Nodes[i].Nodes.Add(slotRun.FoldingSlot.ToString(), String.Format(CultureInfo.InvariantCulture,
"Slot {0}", slotRun.FoldingSlot));
int j = 0;
foreach (var unitRun in slotRun.UnitRuns.Reverse())
{
treeView1.Nodes[i].Nodes[slotRun.FoldingSlot.ToString()].Nodes.Add(j.ToString(), String.Format(CultureInfo.InvariantCulture,
"Queue ({0}) Line ({1}) Index", unitRun.QueueIndex, unitRun.StartIndex));
j++;
}
}
i++;
}
}
private void btnParse_Click(object sender, EventArgs e)
{
if (txtLogPath.Text.Length == 0) return;
treeView1.Nodes.Clear();
if (File.Exists(txtLogPath.Text))
{
FahLogType fahLogType = GetLogFileType();
//var sw = Stopwatch.StartNew();
_fahLog = FahLog.Read(File.ReadLines(txtLogPath.Text), fahLogType);
//sw.Stop();
//Debug.WriteLine("FahLog.Read ET: {0}", sw.Elapsed);
_logLines = _fahLog.ToList();
PopulateClientRunsInTree(_fahLog);
richTextBox1.SetLogLines(_logLines, String.Empty, true);
}
else
{
MessageBox.Show(this, String.Format(CultureInfo.CurrentCulture,
"File '{0}' does not exist.", txtLogPath.Text), Text, MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
private void GenerateUnitInfoDataFromQueue(DataAggregatorResult result, QueueData q, FahLog fahLog, UnitInfoLogData unitInfo)
{
Debug.Assert(q != null);
result.UnitInfos = new Dictionary <int, UnitInfo>(10);
for (int i = 0; i < 10; i++)
{
result.UnitInfos[i] = null;
}
var clientRun = GetCurrentClientRun(fahLog);
for (int queueIndex = 0; queueIndex < result.UnitInfos.Count; queueIndex++)
{
var unitRun = GetUnitRunForQueueIndex(fahLog, queueIndex);
UnitInfoLogData unitInfoLogData = null;
// On the Current Queue Index
if (queueIndex == q.CurrentIndex)
{
// Get the UnitInfo Log Data
unitInfoLogData = unitInfo;
}
var queueEntry = q.GetQueueEntry((uint)queueIndex);
result.UnitInfos[queueIndex] = BuildUnitInfo(queueEntry, clientRun, unitRun, unitInfoLogData);
if (result.UnitInfos[queueIndex] == null)
{
if (queueIndex == q.CurrentIndex)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not verify log section for current queue index {0} {1}.", queueIndex, queueEntry.ToProjectInfo());
Logger.WarnFormat(Constants.ClientNameFormat, ClientName, message);
unitRun = GetCurrentUnitRun(fahLog);
var slotRun = GetCurrentSlotRun(fahLog);
if (slotRun != null && slotRun.Data.Status == SlotStatus.GettingWorkPacket)
{
unitRun = null;
unitInfoLogData = new UnitInfoLogData();
}
result.UnitInfos[queueIndex] = BuildUnitInfo(queueEntry, clientRun, unitRun, unitInfoLogData, true);
}
else if (Logger.IsDebugEnabled)
{
// Write debug info and skip this unit
var projectInfo = queueEntry.ToProjectInfo();
if (!projectInfo.ProjectIsUnknown())
{
string message = String.Format(CultureInfo.CurrentCulture,
"Could not find log section for queue index {0} {1}.", queueIndex, queueEntry.ToProjectInfo());
Logger.DebugFormat(Constants.ClientNameFormat, ClientName, message);
}
}
}
}
}
private static ClientRun GetCurrentClientRun(FahLog fahLog)
{
Debug.Assert(fahLog != null);
return(fahLog.ClientRuns.FirstOrDefault());
}
private static void GenerateUnitInfoDataFromLogs(DataAggregatorResult result, FahLog fahLog, UnitInfoLogData unitInfo)
{
result.UnitInfos = new Dictionary <int, UnitInfo>(2);
for (int i = 0; i < 2; i++)
{
result.UnitInfos[i] = null;
}
var currentClientRun = GetCurrentClientRun(fahLog);
var previousUnitRun = GetPreviousUnitRun(fahLog);
if (previousUnitRun != null)
{
result.UnitInfos[0] = BuildUnitInfo(null, currentClientRun, previousUnitRun, null);
}
var currentUnitRun = GetCurrentUnitRun(fahLog);
result.UnitInfos[1] = BuildUnitInfo(null, currentClientRun, currentUnitRun, unitInfo, currentUnitRun == null);
}
