private static void AddEventData(XElement xmlelement, logxGenericLogElement parent, logXContainer _ret)
{
var doc = new XDocument(new XElement(parent.EventName));
var root = doc.Root;
foreach (var a in xmlelement.Attributes())
{
if (a.Name.Namespace.NamespaceName == "")
{
root.Add(new XAttribute(a.Name.ToString(), a.Value));
}
}
parent.EventDataXML = doc.ToString();
int _logcounter = 0;
foreach (XElement x in xmlelement.Elements())
{
logxGenericLogElement _newparent = new logxGenericLogElement()
{
Item = parent.Item,
EventName = parent.EventName + "." + x.Name.LocalName,
PersonIdentifier = parent.PersonIdentifier,
TimeStamp = parent.TimeStamp,
EventID = _logcounter,
};
AddEventData(x, _newparent, _ret);
_ret.AddEvent(_newparent);
_logcounter++;
}
}
// TODO: Fix bug with EventID and ParentEventID
private static void ReadLogDataEEFromXMLString(string XML, logXContainer _ret)
{
XmlDocument doc = new XmlDocument();
doc.LoadXml(XML);
string PersonIdentifier = "Unknown";
string Element = "";
XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable);
nsmgr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
nsmgr.AddNamespace("cbaloggingmodel", "http://www.softcon.de/cba/cbaloggingmodel");
foreach (XmlNode row in doc.SelectNodes("//logEntry[@xsi:type='cbaloggingmodel:CBAItemLogEntry']", nsmgr))
{
PersonIdentifier = row.Attributes["user"].Value.ToString();
Element = row.Attributes["name"].Value.Replace("de.softcon.cba.runtime.", "").ToString();
}
int _logcounter = 0;
foreach (XmlNode row in doc.SelectNodes("//logEntry"))
{
if (row.ChildNodes.Count == 1)
{
XDocument _xmlElement = XDocument.Parse(row.ChildNodes[0].OuterXml);
if (row.ChildNodes[0].Attributes["xsi:type"] != null)
{
logxGenericLogElement _parament = new logxGenericLogElement()
{
EventName = row.ChildNodes[0].Attributes["xsi:type"].Value.Replace("cbaloggingmodel:", ""),
PersonIdentifier = PersonIdentifier,
TimeStamp = DateTime.Parse(row.Attributes["timeStamp"].Value),
Item = Element,
EventID = _logcounter
};
_logcounter++;
AddEventData(_xmlElement.Root, _parament, _ret);
_ret.AddEvent(_parament);
}
}
}
}
public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments)
{
try
{
bool _personIdentifierIsNumber = false;
if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER"))
{
_personIdentifierIsNumber = true;
}
string _personIdentifierColumnName = "PersonIdentifier";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier"))
{
_personIdentifierColumnName = ParsedCommandLineArguments.ParameterDictionary["personidentifier"];
}
string _language = "ENG";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language"))
{
_language = ParsedCommandLineArguments.ParameterDictionary["language"];
}
double _utcoffset = 0;
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("utcoffset"))
{
_utcoffset = double.Parse(ParsedCommandLineArguments.ParameterDictionary["utcoffset"]);
}
// Create logXContainer
logXContainer _ret = new LogFSM_LogX2019.logXContainer()
{
PersonIdentifierName = _personIdentifierColumnName,
PersonIdentifierIsNumber = _personIdentifierIsNumber
};
_ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary);
if (ParsedCommandLineArguments.Transform_ConcordanceTable.Trim() != "")
{
if (File.Exists(ParsedCommandLineArguments.Transform_ConcordanceTable))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Read Concordance Table.");
}
_ret.ReadConcordanceTable(ParsedCommandLineArguments.Transform_ConcordanceTable);
}
}
// Update from Server if requested
string _web = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_web))
{
_web = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_web];
}
string _username = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_user))
{
_username = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_user];
}
string _password = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_password))
{
_password = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_password];
}
string _key = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_key))
{
_key = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_key];
}
string _mask = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_mask))
{
_mask = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_mask];
}
bool _override = false;
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_overwrite))
{
bool.TryParse(ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_overwrite], out _override);
}
if (_web.Trim() != "")
{
var _sessions = RetrieveListOfSessesions(_web, _username, _password, _key);
UpdateFiles(_sessions, _web, _username, _password, _key, ParsedCommandLineArguments.Transform_InputFolders[0], _mask, _override);
}
if (ParsedCommandLineArguments.Transform_OutputStata.Trim() == "" && ParsedCommandLineArguments.Transform_OutputXLSX.Trim() == "" &&
ParsedCommandLineArguments.Transform_OutputZCSV.Trim() == "" && ParsedCommandLineArguments.Transform_OutputSPSS.Trim() == "")
{
return;
}
// Iterate over all input folders
foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders)
{
if (!Directory.Exists(inFolder))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'.");
}
continue;
}
string[] listOfZipFiles = Directory.GetFiles(inFolder, ParsedCommandLineArguments.Mask, SearchOption.AllDirectories);
foreach (string zfilename in listOfZipFiles)
{
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
if (ParsedCommandLineArguments.Verbose)
{
Console.Write("Info: Read Zip File '" + zfilename + "' ");
}
try
{
using (ZipFile zip = ZipFile.Read(zfilename))
{
foreach (var entry in zip)
{
string _sessionFileName = Path.GetFileNameWithoutExtension(zip.Name);
using (MemoryStream zipStream = new MemoryStream())
{
entry.ExtractWithPassword(zipStream, "");
zipStream.Position = 0;
try
{
if (entry.FileName.StartsWith("browser.log"))
{
// TODO: Process
}
else if (entry.FileName.StartsWith("server.log"))
{
// TODO: Process
}
else if (entry.FileName.StartsWith("Results.csv"))
{
// TODO: Process
}
else if (entry.FileName.StartsWith("Results.sav"))
{
// TODO: Process
}
else if (entry.FileName.StartsWith("Results.dta"))
{
// TODO: Process
}
else if (entry.FileName.StartsWith("Session.json"))
{
// TODO: Process
}
else
{
StreamReader sr = new StreamReader(zipStream);
string line;
int linecounter = 0;
while ((line = sr.ReadLine()) != null)
{
if (ParsedCommandLineArguments.Transform_LogVersion == "default")
{
if (entry.FileName.StartsWith("Trace.json"))
{
string _json = line;
if (_json.EndsWith(","))
{
_json = _json.Substring(0, _json.Length - 1);
}
try
{
List <LogDataTransformer_IB_REACT_8_12__8_13.Log_IB_8_12__8_13> _log = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseLogElements(_json, "IRTlibPlayer_V01");
// TODO: Add flag to extract full name (project.task) vs. short name (project)
foreach (var _l in _log)
{
if (_l.EventName == "")
{
_l.Element = "(Platform)";
}
var g = new logxGenericLogElement()
{
Item = _l.Element,
EventID = _l.EventID,
EventName = _l.EventName,
PersonIdentifier = _l.PersonIdentifier,
TimeStamp = _l.TimeStamp
};
try
{
g.EventDataXML = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.XmlSerializeToString(_l);
_ret.AddEvent(g);
}
catch (Exception _innerex)
{
Console.WriteLine("Error Processing xml: '" + g.EventDataXML + "' - Details: " + _innerex.Message);
}
}
}
catch (Exception _ex)
{
Console.WriteLine("Error processing file '" + entry.FileName + "' ('" + zfilename + "'): " + _ex.Message);
}
}
else if (entry.FileName.StartsWith("Keyboard.json"))
{
string _json = line;
if (_json.EndsWith(","))
{
_json = _json.Substring(0, _json.Length - 1);
}
List <KeyStroke> _keyStrokes = JsonConvert.DeserializeObject <List <KeyStroke> >(_json);
// TODO: Process
}
else if (entry.FileName.StartsWith("Mouse.json"))
{
string _json = line;
if (_json.EndsWith(","))
{
_json = _json.Substring(0, _json.Length - 1);
}
var _data = JsonConvert.DeserializeObject <MouseEvent[]>(_json);
// TODO: Process
}
else if (entry.FileName.StartsWith("ItemScore.json"))
{
string _json = line;
if (_json.EndsWith(","))
{
_json = _json.Substring(0, _json.Length - 1);
}
try
{
LogDataTransformer_IRTlibPlayer_V01.json_IRTlib_V01__ItemScore _itemScoreEvent =
JsonConvert.DeserializeObject <LogDataTransformer_IRTlibPlayer_V01.json_IRTlib_V01__ItemScore>(_json);
string task = _itemScoreEvent.Context.Task;
string item = _itemScoreEvent.Context.Item;
string personIdentifier = _itemScoreEvent.SessionId;
LogDataTransformer_IB_REACT_8_12__8_13.itemScore _itemScore =
LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseItemScore(_itemScoreEvent.ItemScore, task, item, personIdentifier);
logxGenericResultElement g = new logxGenericResultElement()
{
PersonIdentifier = personIdentifier
};
g.Results.Add(item + "." + task + "." + "hitsAccumulated", _itemScore.hitsAccumulated);
g.Results.Add(item + "." + task + "." + "hitsCount", _itemScore.hitsCount);
g.Results.Add(item + "." + task + "." + "missesAccumulated", _itemScore.missesAccumulated);
g.Results.Add(item + "." + task + "." + "missesCount", _itemScore.missesCount);
g.Results.Add(item + "." + task + "." + "classMaxWeighed", _itemScore.classMaxWeighed);
g.Results.Add(item + "." + task + "." + "classMaxName", _itemScore.classMaxName);
g.Results.Add(item + "." + task + "." + "totalResult", _itemScore.totalResult);
g.Results.Add(item + "." + task + "." + "nbUserInteractions", _itemScore.nbUserInteractions);
g.Results.Add(item + "." + task + "." + "nbUserInteractionsTotal", _itemScore.nbUserInteractionsTotal);
g.Results.Add(item + "." + task + "." + "firstReactionTimeTotal", _itemScore.firstReactionTimeTotal);
g.Results.Add(item + "." + task + "." + "taskExecutionTime", _itemScore.taskExecutionTime);
g.Results.Add(item + "." + task + "." + "taskExecutionTimeTotal", _itemScore.taskExecutionTimeTotal);
foreach (var hit in _itemScore.Hits)
{
if (!g.Results.ContainsKey(item + "." + task + "." + hit.Value.Name + ".Result"))
{
g.Results.Add(item + "." + task + "." + hit.Value.Name + ".Result", hit.Value.IsTrue);
}
else
{
g.Results[item + "." + task + "." + hit.Value.Name + ".Result"] = hit.Value.IsTrue;
}
if (!g.Results.ContainsKey(item + "." + task + "." + hit.Value.Name + ".Weight"))
{
g.Results.Add(item + "." + task + "." + hit.Value.Name + ".Weight", hit.Value.Weight);
}
else
{
g.Results[item + "." + task + "." + hit.Value.Name + ".Weight"] = hit.Value.Weight;
}
if (!g.Results.ContainsKey(item + "." + task + "." + hit.Value.Name + ".ResultText"))
{
g.Results.Add(item + "." + task + "." + hit.Value.Name + ".ResultText", hit.Value.ResultText);
}
else
{
g.Results[item + "." + task + "." + hit.Value.Name + ".ResultText"] = hit.Value.ResultText;
}
}
foreach (var cls in _itemScore.ClassResults)
{
if (cls.Value != null)
{
if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".Result"))
{
g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".Result", cls.Value.IsTrue);
}
else
{
g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".Result"] = cls.Value.IsTrue;
}
if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".HitMiss"))
{
g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".HitMiss", cls.Value.Name);
}
else
{
g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".HitMiss"] = cls.Value.Name;
}
if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".Weight"))
{
g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".Weight", cls.Value.Weight);
}
else
{
g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".Weight"] = cls.Value.Weight;
}
if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".ResultText"))
{
g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".ResultText", cls.Value.ResultText);
}
else
{
g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".ResultText"] = cls.Value.ResultText;
}
}
}
_ret.AddResults(g);
}
catch (Exception _ex)
{
Console.WriteLine("Error processing ItemScore: " + _ex.ToString());
}
}
else if (entry.FileName.StartsWith("Snapshot.json"))
{
string _json = line;
if (_json.EndsWith(","))
{
_json = _json.Substring(0, _json.Length - 1);
}
string personIdentifier = _sessionFileName;
logxGenericResultElement g = new logxGenericResultElement()
{
PersonIdentifier = personIdentifier
};
JsonDocument doc = JsonDocument.Parse(_json);
foreach (var jsonelement in doc.RootElement.EnumerateObject())
{
// Extract variable values from snapshots
try
{
if (jsonelement.Name == "variables")
{
foreach (var jsonchild in jsonelement.Value.EnumerateObject())
{
string element = jsonchild.Name;
string[] subelements = element.Split("/", StringSplitOptions.RemoveEmptyEntries);
string testname = subelements[0].Replace("test=", "");
string itemname = subelements[1].Replace("item=", "");
string taskname = subelements[2].Replace("task=", "");
foreach (var varelement in jsonchild.Value.EnumerateObject())
{
string varname = "";
string varvalue = "";
string vartype = "";
foreach (var part in varelement.Value.EnumerateObject())
{
if (part.Name == "name")
{
varname = part.Value.GetString();
}
else if (part.Name == "value")
{
varvalue = part.Value.ToString();
}
else if (part.Name == "type")
{
vartype = part.Value.ToString();
}
}
g.Results.Add(itemname + "." + taskname + ".var." + varname, varvalue);
}
}
}
}
catch { }
}
_ret.AddResults(g);
}
else if (entry.FileName.StartsWith("Log.json"))
{
string _json = line;
if (_json.EndsWith(","))
{
_json = _json.Substring(0, _json.Length - 1);
}
try
{
List <LogDataTransformer_IB_REACT_8_12__8_13.Log_IB_8_12__8_13> _log =
LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseTraceLogs(_json, _utcoffset);
foreach (var _l in _log)
{
_l.PersonIdentifier = _sessionFileName;
var g = new logxGenericLogElement()
{
Item = _l.Element,
EventID = _l.EventID,
EventName = _l.EventName,
PersonIdentifier = _l.PersonIdentifier,
TimeStamp = _l.TimeStamp
};
g.EventDataXML = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.XmlSerializeToString(_l);
_ret.AddEvent(g);
}
}
catch (Exception _ex)
{
Console.WriteLine("Error processing file '" + entry.FileName + "' ('" + zfilename + "'): " + _ex.Message);
}
}
else
{
Console.WriteLine("Unknown file type: " + entry.FileName);
}
}
else
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("failed.");
}
Console.WriteLine("Version '" + ParsedCommandLineArguments.Transform_LogVersion + "' not supported.");
return;
}
linecounter++;
}
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine(" ok ('" + linecounter + " lines).");
}
sr.Close();
}
}
catch (Exception _ex)
{
Console.WriteLine("Error processing file '" + entry.FileName + "': " + _ex.Message);
return;
}
}
}
}
}
catch (Exception _ex)
{
Console.Write("Info: Read Zip Error '" + zfilename + "' " + _ex.Message);
}
}
}
_ret.UpdateRelativeTimes();
_ret.CreateLookup();
// Export
if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with Stata file(s).");
}
_ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language);
}
if (ParsedCommandLineArguments.Transform_OutputSPSS.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with SPSS file(s).");
}
_ret.ExportSPSS(ParsedCommandLineArguments.Transform_OutputSPSS, _language);
}
if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create XLSX file.");
}
_ret.ExportXLSX(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_OutputXES.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create XES file.");
}
_ret.ExportXES(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with CSV file(s).");
}
_ret.ExportCSV(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create Codebook File.");
}
_ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language);
}
if (_ret.ExportErrors.Count > 0)
{
Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files.");
if (ParsedCommandLineArguments.Verbose)
{
for (int i = 0; i < _ret.ExportErrors.Count; i++)
{
Console.WriteLine(_ret.ExportErrors[i]);
}
}
}
}
catch (Exception _ex)
{
Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.ToString());
}
}
public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments)
{
try
{
bool _personIdentifierIsNumber = false;
if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER"))
{
_personIdentifierIsNumber = true;
}
string _personIdentifier = "PersonIdentifier";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier"))
{
_personIdentifier = ParsedCommandLineArguments.ParameterDictionary["personidentifier"];
}
string _language = "ENG";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language"))
{
_language = ParsedCommandLineArguments.ParameterDictionary["language"];
}
// Create logXContainer
logXContainer _ret = new LogFSM_LogX2019.logXContainer()
{
PersonIdentifierName = _personIdentifier,
PersonIdentifierIsNumber = _personIdentifierIsNumber
};
_ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary);
// Update from Server if requested
string _web = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_web))
{
_web = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_web];
}
string _username = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_user))
{
_username = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_user];
}
string _password = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_password))
{
_password = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_password];
}
string _key = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_key))
{
_key = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_key];
}
string _mask = "";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_mask))
{
_mask = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_mask];
}
if (_web.Trim() != "")
{
UpdateFiles(_web, _username, _password, ParsedCommandLineArguments.Transform_InputFolders[0], _mask);
}
if (ParsedCommandLineArguments.Transform_OutputStata.Trim() == "" && ParsedCommandLineArguments.Transform_OutputXLSX.Trim() == "" &&
ParsedCommandLineArguments.Transform_OutputZCSV.Trim() == "" && ParsedCommandLineArguments.Transform_OutputSPSS.Trim() == "")
{
return;
}
// Iterate over all input filters
foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders)
{
if (!Directory.Exists(inFolder))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'.");
}
continue;
}
string[] listOfFiles = Directory.GetFiles(inFolder, ParsedCommandLineArguments.Mask, SearchOption.AllDirectories);
foreach (string fileName in listOfFiles)
{
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
if (ParsedCommandLineArguments.Verbose)
{
Console.Write("Info: Read File '" + fileName + "' ");
}
try
{
string line;
var reader = new StreamReader(fileName);
int linecounter = 0;
while ((line = reader.ReadLine()) != null)
{
if (ParsedCommandLineArguments.Transform_LogVersion == "default")
{
List <LogDataTransformer_IB_REACT_8_12__8_13.Log_IB_8_12__8_13> _log =
LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseLogElements(line, "IBSD_V01");
foreach (var _l in _log)
{
var g = new logxGenericLogElement()
{
Item = _l.Element,
EventID = _l.EventID,
EventName = _l.EventName,
PersonIdentifier = _l.PersonIdentifier,
TimeStamp = _l.TimeStamp
};
g.EventDataXML = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.XmlSerializeToString(_l);
_ret.AddEvent(g);
}
}
else
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("failed.");
}
Console.WriteLine("Version '" + ParsedCommandLineArguments.Transform_LogVersion + "' not supported.");
return;
}
linecounter++;
}
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine(" ok ('" + linecounter + " lines).");
}
reader.Close();
}
catch (Exception exfile)
{
Console.WriteLine("Error processing file '" + fileName + "'. Details: " + Environment.NewLine + exfile.Message.ToString());
}
}
}
// TODO: Check!
//_ret.UpdateRelativeTimes();
_ret.CreateLookup();
if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with Stata file(s).");
}
_ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language);
}
if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create XLSX file.");
}
_ret.ExportXLSX(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_OutputSPSS.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with SPSS / PSPP file(s).");
}
_ret.ExportCSV(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with CSV file(s).");
}
_ret.ExportCSV(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create Codebook File.");
}
_ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language);
}
if (_ret.ExportErrors.Count > 0)
{
Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files.");
if (ParsedCommandLineArguments.Verbose)
{
for (int i = 0; i < _ret.ExportErrors.Count; i++)
{
Console.WriteLine(_ret.ExportErrors[i]);
}
}
}
}
catch (Exception _ex)
{
Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.Message.ToString());
}
}
private static void processPISA_BQ_single_XML(string[] Element, DateTime dt1970, logXContainer _ret, XmlSerializer logSerializer, string _PersonIdentifier, string _xml)
{
var _tr = new StringReader(_xml);
log _log = (log)logSerializer.Deserialize(_tr);
if (_log.itemGroup != null)
{
// update epoch information
foreach (var i in _log.itemGroup)
{
if (!i.epochSpecified && i.userEvents.Length > 0)
{
i.epoch = i.userEvents[0].epoch;
}
}
List <logItemGroup> _sortedItemGroupList = _log.itemGroup.OrderBy(o => o.epoch).ToList();
DateTime _MinAbsoluteTime = DateTime.MaxValue;
DateTime _PreviousEvent = DateTime.MaxValue;
if (_log.User != _PersonIdentifier)
{
throw new Exception("Person identifier miss-match.");
}
int _EventID = _ret.GetMaxID(_PersonIdentifier);
int _EventVisitCounter = 0;
Dictionary <string, int> _elementVisitCounterDict = new Dictionary <string, int>();
string _currentElement = "";
foreach (var p in _sortedItemGroupList)
{
string _Element = p.code;
if (_currentElement != _Element)
{
_EventVisitCounter = 0;
_currentElement = _Element;
if (!_elementVisitCounterDict.ContainsKey(_Element))
{
_elementVisitCounterDict.Add(_Element, 0);
}
else
{
_elementVisitCounterDict[_Element] += 1;
}
}
DateTime _ElementStart = dt1970.AddMilliseconds(p.epoch);
if (_PreviousEvent == DateTime.MaxValue)
{
_PreviousEvent = _ElementStart;
}
foreach (var _event in p.userEvents)
{
string _LogEventName = _event.type;
DateTime _AbsoluteTime = dt1970.AddMilliseconds(_event.epoch);
if (_AbsoluteTime < _MinAbsoluteTime)
{
_MinAbsoluteTime = _AbsoluteTime;
}
Dictionary <string, string> _EventValues = new Dictionary <string, string>();
for (int i = 0; i < _event.ItemsElementName.Length; i++)
{
if (_event.ItemsElementName[i].ToString() == "context")
{
_EventValues.Add("Context", _event.Items[i]);
}
else if (_event.ItemsElementName[i].ToString() == "value")
{
_EventValues.Add("Value", _event.Items[i]);
}
else if (_event.ItemsElementName[i].ToString() == "id")
{
_EventValues.Add("Id", _event.Items[i]);
}
else
{
throw new Exception("Element name not expected.");
}
}
_EventValues.Add("RelativeTimeFrame", (_AbsoluteTime - _ElementStart).TotalMilliseconds.ToString());
_EventValues.Add("RelativeTimePrevious", (_AbsoluteTime - _PreviousEvent).TotalMilliseconds.ToString());
if (Element.Length == 0 || Element.Contains <string>(_Element))
{
var doc = new XDocument(new XElement(_LogEventName));
var root = doc.Root;
foreach (string val in _EventValues.Keys)
{
root.Add(new XAttribute(val, _EventValues[val]));
}
logxGenericLogElement _parament = new logxGenericLogElement()
{
PersonIdentifier = _PersonIdentifier,
Item = _Element,
EventID = _EventID,
EventName = _LogEventName,
TimeStamp = _AbsoluteTime,
EventDataXML = doc.ToString()
};
_ret.AddEvent(_parament);
}
_EventID += 1;
_EventVisitCounter += 1;
_PreviousEvent = _AbsoluteTime;
}
}
// check for suspicious times
/* TODO
* _currentElement = "";
* List<string> framesWithSuspiciousData = new List<string>();
* foreach (var v in _inMemoryTempDataEvents)
* {
* if (_currentElement != v.EventName)
* _currentElement = v.EventName;
*
* if (v.TimeDifferencePrevious.TotalMinutes > 30)
* v.AddEventValue("Flag", "TimeToLong");
*
* if (v.TimeDifferencePrevious.TotalMilliseconds < 0)
* v.AddEventValue("Flag", "TimeNegative");
* }
*/
}
_tr.Close();
}
private static int piaac_R1_extract_IB_XML_process_child(int _logcounter, logXContainer _ret, Dictionary <string, string> _lookup, XmlNode node, string Path, int Level, string PersonIdentifier, string _rawSequenceID, string _rawBooklet, string _rawEventType, string _rawEventName, long _relativeTime, CommandLineArguments ParsedCommandLineArguments)
{
string _name = "";
foreach (XmlNode n in node.ChildNodes)
{
_name = n.LocalName;
if (Path.Trim() != "")
{
_name = Path + "." + _name;
}
string _itemID = _rawBooklet + "_" + _rawSequenceID;
if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID"))
{
if (_lookup.ContainsKey(_itemID))
{
_itemID = _lookup[_itemID];
}
}
XmlAttributeCollection atributos = n.Attributes;
logxGenericLogElement _parament = new logxGenericLogElement()
{
PersonIdentifier = PersonIdentifier,
EventName = _name,
Item = _itemID,
RelativeTime = _relativeTime,
EventID = _logcounter
};
_logcounter++;
var doc = new XDocument(new XElement(_parament.EventName));
var root = doc.Root;
foreach (XmlAttribute at in atributos)
{
if (at.LocalName == "cbaloggingmodel" || at.LocalName == "cbascoringresultmm" || at.LocalName == "snapshot" || at.LocalName == "xmi" || at.LocalName == "xsi")
{
// ignore attributes by default
if (ParsedCommandLineArguments.Flags.Contains("INCLUDE_XML_ATTRIBUTES"))
{
root.Add(new XAttribute(at.LocalName, at.Value));
}
}
else
{
root.Add(new XAttribute(at.LocalName, at.Value));
}
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION"))
{
root.Add(new XAttribute("PIAACBookelt", _rawBooklet));
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION"))
{
root.Add(new XAttribute("PIAACEventType", _rawEventType));
root.Add(new XAttribute("PIAACEventName", _rawEventName));
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_XML_NESTING_LEVEL"))
{
root.Add(new XAttribute("XMLNextingLevel", Level.ToString()));
}
_parament.EventDataXML = doc.ToString();
_ret.AddEvent(_parament);
}
foreach (XmlNode n in node.ChildNodes)
{
piaac_R1_extract_IB_XML_process_child(_logcounter, _ret, _lookup, n, _name, Level + 1, PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
return(_logcounter);
}
private static int ReadLogDataPIAACFromLDAExportString(int _logcounter, int _lineCounter, string line, logXContainer _ret, string _PersonIdentifier, CommandLineArguments ParsedCommandLineArguments, EventDataListExtension.ESortType sort, Dictionary <string, string> _lookup, string[] Element)
{
string[] _cols = line.Split('\t');
if (_cols.Length > 1 && _lineCounter > 0)
{
_PersonIdentifier = _cols[0] + "_" + _cols[1];
string _rawBooklet = _cols[2];
string _rawSequenceID = _cols[3];
string _rawEventName = _cols[4];
string _rawEventType = _cols[5];
string _relativeTimeString = _cols[6];
string _rawEventDescription = _cols[7];
string _itemID = _rawBooklet + "_" + _rawSequenceID;
long _relativeTime = long.Parse(_cols[6], CultureInfo.InvariantCulture);
if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID"))
{
if (_lookup.ContainsKey(_itemID))
{
_itemID = _lookup[_itemID];
}
}
bool _isFiltered = Element.Length == 0;
if (Element.Contains(_itemID))
{
_isFiltered = true;
}
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
return(_logcounter);
}
if (_isFiltered)
{
if (_rawEventDescription.StartsWith("<cbaloggingmodel", StringComparison.InvariantCulture))
{
// IB xml events
_logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
else if (_rawEventDescription.StartsWith("<cbascoringresultmm", StringComparison.InvariantCulture))
{
// IB item score
if (_rawEventType == "ItemScore")
{
if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES") || !ParsedCommandLineArguments.Flags.Contains("INCLUDE_NO_ITEMSCORES"))
{
_logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
}
else if (_rawEventType == "itemScoreResult")
{
// by default ignore second item score
if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES"))
{
_logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
}
else
{
throw new Exception("The value '" + _rawEventName + "' was not expected in line " + _lineCounter + ".");
}
}
else if (_rawEventDescription.StartsWith("http://localhost:8080", StringComparison.InvariantCulture))
{
// ignore snapshots
}
else
{
// Pre-process event values
logxGenericLogElement _parament = new logxGenericLogElement()
{
PersonIdentifier = _PersonIdentifier,
EventName = _rawEventType,
Item = _itemID,
RelativeTime = _relativeTime,
EventID = _logcounter
};
_logcounter++;
var doc = new XDocument(new XElement(_parament.EventName));
var root = doc.Root;
_rawEventDescription = _rawEventDescription.Replace("|$*", "|*$");
string[] _kvps = _rawEventDescription.Split("|*$");
foreach (string _kvp in _kvps)
{
string[] _kv = _kvp.Split('=');
if (_kv.Length == 2)
{
root.Add(new XAttribute(_kv[0], _kv[1]));
}
else if (_kv.Length == 1)
{
root.Add(new XAttribute("value", _kv[0]));
}
else
{
string _remainingvalue = _kv[1];
for (int i = 2; i < _kv.Length; i++)
{
_remainingvalue = _remainingvalue + "=" + _kv[i];
}
root.Add(new XAttribute(_kv[0], _remainingvalue));
}
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION"))
{
root.Add(new XAttribute("PIAACBookelt", _rawBooklet));
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION"))
{
root.Add(new XAttribute("PIAACSequenceID", _rawSequenceID));
root.Add(new XAttribute("PIAACEventType", _rawEventType));
root.Add(new XAttribute("PIAACEventName", _rawEventName));
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_TIME_INFORMATION"))
{
root.Add(new XAttribute("RelativeTimeString", _relativeTimeString));
}
_parament.EventDataXML = doc.ToString();
_ret.AddEvent(_parament);
}
}
}
return(_logcounter);
}
