private static void AddEventData(XElement xmlelement, logxGenericLogElement parent, logXContainer _ret) { var doc = new XDocument(new XElement(parent.EventName)); var root = doc.Root; foreach (var a in xmlelement.Attributes()) { if (a.Name.Namespace.NamespaceName == "") { root.Add(new XAttribute(a.Name.ToString(), a.Value)); } } parent.EventDataXML = doc.ToString(); int _logcounter = 0; foreach (XElement x in xmlelement.Elements()) { logxGenericLogElement _newparent = new logxGenericLogElement() { Item = parent.Item, EventName = parent.EventName + "." + x.Name.LocalName, PersonIdentifier = parent.PersonIdentifier, TimeStamp = parent.TimeStamp, EventID = _logcounter, }; AddEventData(x, _newparent, _ret); _ret.AddEvent(_newparent); _logcounter++; } }
// TODO: Fix bug with EventID and ParentEventID private static void ReadLogDataEEFromXMLString(string XML, logXContainer _ret) { XmlDocument doc = new XmlDocument(); doc.LoadXml(XML); string PersonIdentifier = "Unknown"; string Element = ""; XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable); nsmgr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance"); nsmgr.AddNamespace("cbaloggingmodel", "http://www.softcon.de/cba/cbaloggingmodel"); foreach (XmlNode row in doc.SelectNodes("//logEntry[@xsi:type='cbaloggingmodel:CBAItemLogEntry']", nsmgr)) { PersonIdentifier = row.Attributes["user"].Value.ToString(); Element = row.Attributes["name"].Value.Replace("de.softcon.cba.runtime.", "").ToString(); } int _logcounter = 0; foreach (XmlNode row in doc.SelectNodes("//logEntry")) { if (row.ChildNodes.Count == 1) { XDocument _xmlElement = XDocument.Parse(row.ChildNodes[0].OuterXml); if (row.ChildNodes[0].Attributes["xsi:type"] != null) { logxGenericLogElement _parament = new logxGenericLogElement() { EventName = row.ChildNodes[0].Attributes["xsi:type"].Value.Replace("cbaloggingmodel:", ""), PersonIdentifier = PersonIdentifier, TimeStamp = DateTime.Parse(row.Attributes["timeStamp"].Value), Item = Element, EventID = _logcounter }; _logcounter++; AddEventData(_xmlElement.Root, _parament, _ret); _ret.AddEvent(_parament); } } } }
public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments) { try { bool _personIdentifierIsNumber = false; if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER")) { _personIdentifierIsNumber = true; } string _personIdentifierColumnName = "PersonIdentifier"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier")) { _personIdentifierColumnName = ParsedCommandLineArguments.ParameterDictionary["personidentifier"]; } string _language = "ENG"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language")) { _language = ParsedCommandLineArguments.ParameterDictionary["language"]; } double _utcoffset = 0; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("utcoffset")) { _utcoffset = double.Parse(ParsedCommandLineArguments.ParameterDictionary["utcoffset"]); } // Create logXContainer logXContainer _ret = new LogFSM_LogX2019.logXContainer() { PersonIdentifierName = _personIdentifierColumnName, PersonIdentifierIsNumber = _personIdentifierIsNumber }; _ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary); if (ParsedCommandLineArguments.Transform_ConcordanceTable.Trim() != "") { if (File.Exists(ParsedCommandLineArguments.Transform_ConcordanceTable)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Read Concordance Table."); } _ret.ReadConcordanceTable(ParsedCommandLineArguments.Transform_ConcordanceTable); } } // Update from Server if requested string _web = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_web)) { _web = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_web]; } string _username = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_user)) { _username = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_user]; } string _password = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_password)) { _password = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_password]; } string _key = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_key)) { _key = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_key]; } string _mask = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_mask)) { _mask = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_mask]; } bool _override = false; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_overwrite)) { bool.TryParse(ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_overwrite], out _override); } if (_web.Trim() != "") { var _sessions = RetrieveListOfSessesions(_web, _username, _password, _key); UpdateFiles(_sessions, _web, _username, _password, _key, ParsedCommandLineArguments.Transform_InputFolders[0], _mask, _override); } if (ParsedCommandLineArguments.Transform_OutputStata.Trim() == "" && ParsedCommandLineArguments.Transform_OutputXLSX.Trim() == "" && ParsedCommandLineArguments.Transform_OutputZCSV.Trim() == "" && ParsedCommandLineArguments.Transform_OutputSPSS.Trim() == "") { return; } // Iterate over all input folders foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders) { if (!Directory.Exists(inFolder)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'."); } continue; } string[] listOfZipFiles = Directory.GetFiles(inFolder, ParsedCommandLineArguments.Mask, SearchOption.AllDirectories); foreach (string zfilename in listOfZipFiles) { if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Info: Max number of cases reached."); } break; } if (ParsedCommandLineArguments.Verbose) { Console.Write("Info: Read Zip File '" + zfilename + "' "); } try { using (ZipFile zip = ZipFile.Read(zfilename)) { foreach (var entry in zip) { string _sessionFileName = Path.GetFileNameWithoutExtension(zip.Name); using (MemoryStream zipStream = new MemoryStream()) { entry.ExtractWithPassword(zipStream, ""); zipStream.Position = 0; try { if (entry.FileName.StartsWith("browser.log")) { // TODO: Process } else if (entry.FileName.StartsWith("server.log")) { // TODO: Process } else if (entry.FileName.StartsWith("Results.csv")) { // TODO: Process } else if (entry.FileName.StartsWith("Results.sav")) { // TODO: Process } else if (entry.FileName.StartsWith("Results.dta")) { // TODO: Process } else if (entry.FileName.StartsWith("Session.json")) { // TODO: Process } else { StreamReader sr = new StreamReader(zipStream); string line; int linecounter = 0; while ((line = sr.ReadLine()) != null) { if (ParsedCommandLineArguments.Transform_LogVersion == "default") { if (entry.FileName.StartsWith("Trace.json")) { string _json = line; if (_json.EndsWith(",")) { _json = _json.Substring(0, _json.Length - 1); } try { List <LogDataTransformer_IB_REACT_8_12__8_13.Log_IB_8_12__8_13> _log = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseLogElements(_json, "IRTlibPlayer_V01"); // TODO: Add flag to extract full name (project.task) vs. short name (project) foreach (var _l in _log) { if (_l.EventName == "") { _l.Element = "(Platform)"; } var g = new logxGenericLogElement() { Item = _l.Element, EventID = _l.EventID, EventName = _l.EventName, PersonIdentifier = _l.PersonIdentifier, TimeStamp = _l.TimeStamp }; try { g.EventDataXML = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.XmlSerializeToString(_l); _ret.AddEvent(g); } catch (Exception _innerex) { Console.WriteLine("Error Processing xml: '" + g.EventDataXML + "' - Details: " + _innerex.Message); } } } catch (Exception _ex) { Console.WriteLine("Error processing file '" + entry.FileName + "' ('" + zfilename + "'): " + _ex.Message); } } else if (entry.FileName.StartsWith("Keyboard.json")) { string _json = line; if (_json.EndsWith(",")) { _json = _json.Substring(0, _json.Length - 1); } List <KeyStroke> _keyStrokes = JsonConvert.DeserializeObject <List <KeyStroke> >(_json); // TODO: Process } else if (entry.FileName.StartsWith("Mouse.json")) { string _json = line; if (_json.EndsWith(",")) { _json = _json.Substring(0, _json.Length - 1); } var _data = JsonConvert.DeserializeObject <MouseEvent[]>(_json); // TODO: Process } else if (entry.FileName.StartsWith("ItemScore.json")) { string _json = line; if (_json.EndsWith(",")) { _json = _json.Substring(0, _json.Length - 1); } try { LogDataTransformer_IRTlibPlayer_V01.json_IRTlib_V01__ItemScore _itemScoreEvent = JsonConvert.DeserializeObject <LogDataTransformer_IRTlibPlayer_V01.json_IRTlib_V01__ItemScore>(_json); string task = _itemScoreEvent.Context.Task; string item = _itemScoreEvent.Context.Item; string personIdentifier = _itemScoreEvent.SessionId; LogDataTransformer_IB_REACT_8_12__8_13.itemScore _itemScore = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseItemScore(_itemScoreEvent.ItemScore, task, item, personIdentifier); logxGenericResultElement g = new logxGenericResultElement() { PersonIdentifier = personIdentifier }; g.Results.Add(item + "." + task + "." + "hitsAccumulated", _itemScore.hitsAccumulated); g.Results.Add(item + "." + task + "." + "hitsCount", _itemScore.hitsCount); g.Results.Add(item + "." + task + "." + "missesAccumulated", _itemScore.missesAccumulated); g.Results.Add(item + "." + task + "." + "missesCount", _itemScore.missesCount); g.Results.Add(item + "." + task + "." + "classMaxWeighed", _itemScore.classMaxWeighed); g.Results.Add(item + "." + task + "." + "classMaxName", _itemScore.classMaxName); g.Results.Add(item + "." + task + "." + "totalResult", _itemScore.totalResult); g.Results.Add(item + "." + task + "." + "nbUserInteractions", _itemScore.nbUserInteractions); g.Results.Add(item + "." + task + "." + "nbUserInteractionsTotal", _itemScore.nbUserInteractionsTotal); g.Results.Add(item + "." + task + "." + "firstReactionTimeTotal", _itemScore.firstReactionTimeTotal); g.Results.Add(item + "." + task + "." + "taskExecutionTime", _itemScore.taskExecutionTime); g.Results.Add(item + "." + task + "." + "taskExecutionTimeTotal", _itemScore.taskExecutionTimeTotal); foreach (var hit in _itemScore.Hits) { if (!g.Results.ContainsKey(item + "." + task + "." + hit.Value.Name + ".Result")) { g.Results.Add(item + "." + task + "." + hit.Value.Name + ".Result", hit.Value.IsTrue); } else { g.Results[item + "." + task + "." + hit.Value.Name + ".Result"] = hit.Value.IsTrue; } if (!g.Results.ContainsKey(item + "." + task + "." + hit.Value.Name + ".Weight")) { g.Results.Add(item + "." + task + "." + hit.Value.Name + ".Weight", hit.Value.Weight); } else { g.Results[item + "." + task + "." + hit.Value.Name + ".Weight"] = hit.Value.Weight; } if (!g.Results.ContainsKey(item + "." + task + "." + hit.Value.Name + ".ResultText")) { g.Results.Add(item + "." + task + "." + hit.Value.Name + ".ResultText", hit.Value.ResultText); } else { g.Results[item + "." + task + "." + hit.Value.Name + ".ResultText"] = hit.Value.ResultText; } } foreach (var cls in _itemScore.ClassResults) { if (cls.Value != null) { if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".Result")) { g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".Result", cls.Value.IsTrue); } else { g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".Result"] = cls.Value.IsTrue; } if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".HitMiss")) { g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".HitMiss", cls.Value.Name); } else { g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".HitMiss"] = cls.Value.Name; } if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".Weight")) { g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".Weight", cls.Value.Weight); } else { g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".Weight"] = cls.Value.Weight; } if (!g.Results.ContainsKey(item + "." + task + "." + cls.Value.HitMissClass + ".ResultText")) { g.Results.Add(item + "." + task + "." + cls.Value.HitMissClass + ".ResultText", cls.Value.ResultText); } else { g.Results[item + "." + task + "." + cls.Value.HitMissClass + ".ResultText"] = cls.Value.ResultText; } } } _ret.AddResults(g); } catch (Exception _ex) { Console.WriteLine("Error processing ItemScore: " + _ex.ToString()); } } else if (entry.FileName.StartsWith("Snapshot.json")) { string _json = line; if (_json.EndsWith(",")) { _json = _json.Substring(0, _json.Length - 1); } string personIdentifier = _sessionFileName; logxGenericResultElement g = new logxGenericResultElement() { PersonIdentifier = personIdentifier }; JsonDocument doc = JsonDocument.Parse(_json); foreach (var jsonelement in doc.RootElement.EnumerateObject()) { // Extract variable values from snapshots try { if (jsonelement.Name == "variables") { foreach (var jsonchild in jsonelement.Value.EnumerateObject()) { string element = jsonchild.Name; string[] subelements = element.Split("/", StringSplitOptions.RemoveEmptyEntries); string testname = subelements[0].Replace("test=", ""); string itemname = subelements[1].Replace("item=", ""); string taskname = subelements[2].Replace("task=", ""); foreach (var varelement in jsonchild.Value.EnumerateObject()) { string varname = ""; string varvalue = ""; string vartype = ""; foreach (var part in varelement.Value.EnumerateObject()) { if (part.Name == "name") { varname = part.Value.GetString(); } else if (part.Name == "value") { varvalue = part.Value.ToString(); } else if (part.Name == "type") { vartype = part.Value.ToString(); } } g.Results.Add(itemname + "." + taskname + ".var." + varname, varvalue); } } } } catch { } } _ret.AddResults(g); } else if (entry.FileName.StartsWith("Log.json")) { string _json = line; if (_json.EndsWith(",")) { _json = _json.Substring(0, _json.Length - 1); } try { List <LogDataTransformer_IB_REACT_8_12__8_13.Log_IB_8_12__8_13> _log = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseTraceLogs(_json, _utcoffset); foreach (var _l in _log) { _l.PersonIdentifier = _sessionFileName; var g = new logxGenericLogElement() { Item = _l.Element, EventID = _l.EventID, EventName = _l.EventName, PersonIdentifier = _l.PersonIdentifier, TimeStamp = _l.TimeStamp }; g.EventDataXML = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.XmlSerializeToString(_l); _ret.AddEvent(g); } } catch (Exception _ex) { Console.WriteLine("Error processing file '" + entry.FileName + "' ('" + zfilename + "'): " + _ex.Message); } } else { Console.WriteLine("Unknown file type: " + entry.FileName); } } else { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("failed."); } Console.WriteLine("Version '" + ParsedCommandLineArguments.Transform_LogVersion + "' not supported."); return; } linecounter++; } if (ParsedCommandLineArguments.Verbose) { Console.WriteLine(" ok ('" + linecounter + " lines)."); } sr.Close(); } } catch (Exception _ex) { Console.WriteLine("Error processing file '" + entry.FileName + "': " + _ex.Message); return; } } } } } catch (Exception _ex) { Console.Write("Info: Read Zip Error '" + zfilename + "' " + _ex.Message); } } } _ret.UpdateRelativeTimes(); _ret.CreateLookup(); // Export if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with Stata file(s)."); } _ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language); } if (ParsedCommandLineArguments.Transform_OutputSPSS.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with SPSS file(s)."); } _ret.ExportSPSS(ParsedCommandLineArguments.Transform_OutputSPSS, _language); } if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create XLSX file."); } _ret.ExportXLSX(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_OutputXES.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create XES file."); } _ret.ExportXES(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with CSV file(s)."); } _ret.ExportCSV(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create Codebook File."); } _ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language); } if (_ret.ExportErrors.Count > 0) { Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files."); if (ParsedCommandLineArguments.Verbose) { for (int i = 0; i < _ret.ExportErrors.Count; i++) { Console.WriteLine(_ret.ExportErrors[i]); } } } } catch (Exception _ex) { Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.ToString()); } }
public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments) { try { bool _personIdentifierIsNumber = false; if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER")) { _personIdentifierIsNumber = true; } string _personIdentifier = "PersonIdentifier"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier")) { _personIdentifier = ParsedCommandLineArguments.ParameterDictionary["personidentifier"]; } string _language = "ENG"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language")) { _language = ParsedCommandLineArguments.ParameterDictionary["language"]; } // Create logXContainer logXContainer _ret = new LogFSM_LogX2019.logXContainer() { PersonIdentifierName = _personIdentifier, PersonIdentifierIsNumber = _personIdentifierIsNumber }; _ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary); // Update from Server if requested string _web = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_web)) { _web = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_web]; } string _username = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_user)) { _username = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_user]; } string _password = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_password)) { _password = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_password]; } string _key = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_JOB_TRANSFORM_key)) { _key = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_JOB_TRANSFORM_key]; } string _mask = ""; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey(CommandLineArguments._CMDA_mask)) { _mask = ParsedCommandLineArguments.ParameterDictionary[CommandLineArguments._CMDA_mask]; } if (_web.Trim() != "") { UpdateFiles(_web, _username, _password, ParsedCommandLineArguments.Transform_InputFolders[0], _mask); } if (ParsedCommandLineArguments.Transform_OutputStata.Trim() == "" && ParsedCommandLineArguments.Transform_OutputXLSX.Trim() == "" && ParsedCommandLineArguments.Transform_OutputZCSV.Trim() == "" && ParsedCommandLineArguments.Transform_OutputSPSS.Trim() == "") { return; } // Iterate over all input filters foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders) { if (!Directory.Exists(inFolder)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'."); } continue; } string[] listOfFiles = Directory.GetFiles(inFolder, ParsedCommandLineArguments.Mask, SearchOption.AllDirectories); foreach (string fileName in listOfFiles) { if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Info: Max number of cases reached."); } break; } if (ParsedCommandLineArguments.Verbose) { Console.Write("Info: Read File '" + fileName + "' "); } try { string line; var reader = new StreamReader(fileName); int linecounter = 0; while ((line = reader.ReadLine()) != null) { if (ParsedCommandLineArguments.Transform_LogVersion == "default") { List <LogDataTransformer_IB_REACT_8_12__8_13.Log_IB_8_12__8_13> _log = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.ParseLogElements(line, "IBSD_V01"); foreach (var _l in _log) { var g = new logxGenericLogElement() { Item = _l.Element, EventID = _l.EventID, EventName = _l.EventName, PersonIdentifier = _l.PersonIdentifier, TimeStamp = _l.TimeStamp }; g.EventDataXML = LogDataTransformer_IB_REACT_8_12__8_13.JSON_IB_8_12__8_13_helper.XmlSerializeToString(_l); _ret.AddEvent(g); } } else { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("failed."); } Console.WriteLine("Version '" + ParsedCommandLineArguments.Transform_LogVersion + "' not supported."); return; } linecounter++; } if (ParsedCommandLineArguments.Verbose) { Console.WriteLine(" ok ('" + linecounter + " lines)."); } reader.Close(); } catch (Exception exfile) { Console.WriteLine("Error processing file '" + fileName + "'. Details: " + Environment.NewLine + exfile.Message.ToString()); } } } // TODO: Check! //_ret.UpdateRelativeTimes(); _ret.CreateLookup(); if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with Stata file(s)."); } _ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language); } if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create XLSX file."); } _ret.ExportXLSX(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_OutputSPSS.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with SPSS / PSPP file(s)."); } _ret.ExportCSV(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with CSV file(s)."); } _ret.ExportCSV(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create Codebook File."); } _ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language); } if (_ret.ExportErrors.Count > 0) { Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files."); if (ParsedCommandLineArguments.Verbose) { for (int i = 0; i < _ret.ExportErrors.Count; i++) { Console.WriteLine(_ret.ExportErrors[i]); } } } } catch (Exception _ex) { Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.Message.ToString()); } }
private static void processPISA_BQ_single_XML(string[] Element, DateTime dt1970, logXContainer _ret, XmlSerializer logSerializer, string _PersonIdentifier, string _xml) { var _tr = new StringReader(_xml); log _log = (log)logSerializer.Deserialize(_tr); if (_log.itemGroup != null) { // update epoch information foreach (var i in _log.itemGroup) { if (!i.epochSpecified && i.userEvents.Length > 0) { i.epoch = i.userEvents[0].epoch; } } List <logItemGroup> _sortedItemGroupList = _log.itemGroup.OrderBy(o => o.epoch).ToList(); DateTime _MinAbsoluteTime = DateTime.MaxValue; DateTime _PreviousEvent = DateTime.MaxValue; if (_log.User != _PersonIdentifier) { throw new Exception("Person identifier miss-match."); } int _EventID = _ret.GetMaxID(_PersonIdentifier); int _EventVisitCounter = 0; Dictionary <string, int> _elementVisitCounterDict = new Dictionary <string, int>(); string _currentElement = ""; foreach (var p in _sortedItemGroupList) { string _Element = p.code; if (_currentElement != _Element) { _EventVisitCounter = 0; _currentElement = _Element; if (!_elementVisitCounterDict.ContainsKey(_Element)) { _elementVisitCounterDict.Add(_Element, 0); } else { _elementVisitCounterDict[_Element] += 1; } } DateTime _ElementStart = dt1970.AddMilliseconds(p.epoch); if (_PreviousEvent == DateTime.MaxValue) { _PreviousEvent = _ElementStart; } foreach (var _event in p.userEvents) { string _LogEventName = _event.type; DateTime _AbsoluteTime = dt1970.AddMilliseconds(_event.epoch); if (_AbsoluteTime < _MinAbsoluteTime) { _MinAbsoluteTime = _AbsoluteTime; } Dictionary <string, string> _EventValues = new Dictionary <string, string>(); for (int i = 0; i < _event.ItemsElementName.Length; i++) { if (_event.ItemsElementName[i].ToString() == "context") { _EventValues.Add("Context", _event.Items[i]); } else if (_event.ItemsElementName[i].ToString() == "value") { _EventValues.Add("Value", _event.Items[i]); } else if (_event.ItemsElementName[i].ToString() == "id") { _EventValues.Add("Id", _event.Items[i]); } else { throw new Exception("Element name not expected."); } } _EventValues.Add("RelativeTimeFrame", (_AbsoluteTime - _ElementStart).TotalMilliseconds.ToString()); _EventValues.Add("RelativeTimePrevious", (_AbsoluteTime - _PreviousEvent).TotalMilliseconds.ToString()); if (Element.Length == 0 || Element.Contains <string>(_Element)) { var doc = new XDocument(new XElement(_LogEventName)); var root = doc.Root; foreach (string val in _EventValues.Keys) { root.Add(new XAttribute(val, _EventValues[val])); } logxGenericLogElement _parament = new logxGenericLogElement() { PersonIdentifier = _PersonIdentifier, Item = _Element, EventID = _EventID, EventName = _LogEventName, TimeStamp = _AbsoluteTime, EventDataXML = doc.ToString() }; _ret.AddEvent(_parament); } _EventID += 1; _EventVisitCounter += 1; _PreviousEvent = _AbsoluteTime; } } // check for suspicious times /* TODO * _currentElement = ""; * List<string> framesWithSuspiciousData = new List<string>(); * foreach (var v in _inMemoryTempDataEvents) * { * if (_currentElement != v.EventName) * _currentElement = v.EventName; * * if (v.TimeDifferencePrevious.TotalMinutes > 30) * v.AddEventValue("Flag", "TimeToLong"); * * if (v.TimeDifferencePrevious.TotalMilliseconds < 0) * v.AddEventValue("Flag", "TimeNegative"); * } */ } _tr.Close(); }
private static int piaac_R1_extract_IB_XML_process_child(int _logcounter, logXContainer _ret, Dictionary <string, string> _lookup, XmlNode node, string Path, int Level, string PersonIdentifier, string _rawSequenceID, string _rawBooklet, string _rawEventType, string _rawEventName, long _relativeTime, CommandLineArguments ParsedCommandLineArguments) { string _name = ""; foreach (XmlNode n in node.ChildNodes) { _name = n.LocalName; if (Path.Trim() != "") { _name = Path + "." + _name; } string _itemID = _rawBooklet + "_" + _rawSequenceID; if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID")) { if (_lookup.ContainsKey(_itemID)) { _itemID = _lookup[_itemID]; } } XmlAttributeCollection atributos = n.Attributes; logxGenericLogElement _parament = new logxGenericLogElement() { PersonIdentifier = PersonIdentifier, EventName = _name, Item = _itemID, RelativeTime = _relativeTime, EventID = _logcounter }; _logcounter++; var doc = new XDocument(new XElement(_parament.EventName)); var root = doc.Root; foreach (XmlAttribute at in atributos) { if (at.LocalName == "cbaloggingmodel" || at.LocalName == "cbascoringresultmm" || at.LocalName == "snapshot" || at.LocalName == "xmi" || at.LocalName == "xsi") { // ignore attributes by default if (ParsedCommandLineArguments.Flags.Contains("INCLUDE_XML_ATTRIBUTES")) { root.Add(new XAttribute(at.LocalName, at.Value)); } } else { root.Add(new XAttribute(at.LocalName, at.Value)); } } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION")) { root.Add(new XAttribute("PIAACBookelt", _rawBooklet)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION")) { root.Add(new XAttribute("PIAACEventType", _rawEventType)); root.Add(new XAttribute("PIAACEventName", _rawEventName)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_XML_NESTING_LEVEL")) { root.Add(new XAttribute("XMLNextingLevel", Level.ToString())); } _parament.EventDataXML = doc.ToString(); _ret.AddEvent(_parament); } foreach (XmlNode n in node.ChildNodes) { piaac_R1_extract_IB_XML_process_child(_logcounter, _ret, _lookup, n, _name, Level + 1, PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } return(_logcounter); }
private static int ReadLogDataPIAACFromLDAExportString(int _logcounter, int _lineCounter, string line, logXContainer _ret, string _PersonIdentifier, CommandLineArguments ParsedCommandLineArguments, EventDataListExtension.ESortType sort, Dictionary <string, string> _lookup, string[] Element) { string[] _cols = line.Split('\t'); if (_cols.Length > 1 && _lineCounter > 0) { _PersonIdentifier = _cols[0] + "_" + _cols[1]; string _rawBooklet = _cols[2]; string _rawSequenceID = _cols[3]; string _rawEventName = _cols[4]; string _rawEventType = _cols[5]; string _relativeTimeString = _cols[6]; string _rawEventDescription = _cols[7]; string _itemID = _rawBooklet + "_" + _rawSequenceID; long _relativeTime = long.Parse(_cols[6], CultureInfo.InvariantCulture); if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID")) { if (_lookup.ContainsKey(_itemID)) { _itemID = _lookup[_itemID]; } } bool _isFiltered = Element.Length == 0; if (Element.Contains(_itemID)) { _isFiltered = true; } if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { return(_logcounter); } if (_isFiltered) { if (_rawEventDescription.StartsWith("<cbaloggingmodel", StringComparison.InvariantCulture)) { // IB xml events _logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } else if (_rawEventDescription.StartsWith("<cbascoringresultmm", StringComparison.InvariantCulture)) { // IB item score if (_rawEventType == "ItemScore") { if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES") || !ParsedCommandLineArguments.Flags.Contains("INCLUDE_NO_ITEMSCORES")) { _logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } } else if (_rawEventType == "itemScoreResult") { // by default ignore second item score if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES")) { _logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } } else { throw new Exception("The value '" + _rawEventName + "' was not expected in line " + _lineCounter + "."); } } else if (_rawEventDescription.StartsWith("http://localhost:8080", StringComparison.InvariantCulture)) { // ignore snapshots } else { // Pre-process event values logxGenericLogElement _parament = new logxGenericLogElement() { PersonIdentifier = _PersonIdentifier, EventName = _rawEventType, Item = _itemID, RelativeTime = _relativeTime, EventID = _logcounter }; _logcounter++; var doc = new XDocument(new XElement(_parament.EventName)); var root = doc.Root; _rawEventDescription = _rawEventDescription.Replace("|$*", "|*$"); string[] _kvps = _rawEventDescription.Split("|*$"); foreach (string _kvp in _kvps) { string[] _kv = _kvp.Split('='); if (_kv.Length == 2) { root.Add(new XAttribute(_kv[0], _kv[1])); } else if (_kv.Length == 1) { root.Add(new XAttribute("value", _kv[0])); } else { string _remainingvalue = _kv[1]; for (int i = 2; i < _kv.Length; i++) { _remainingvalue = _remainingvalue + "=" + _kv[i]; } root.Add(new XAttribute(_kv[0], _remainingvalue)); } } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION")) { root.Add(new XAttribute("PIAACBookelt", _rawBooklet)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION")) { root.Add(new XAttribute("PIAACSequenceID", _rawSequenceID)); root.Add(new XAttribute("PIAACEventType", _rawEventType)); root.Add(new XAttribute("PIAACEventName", _rawEventName)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_TIME_INFORMATION")) { root.Add(new XAttribute("RelativeTimeString", _relativeTimeString)); } _parament.EventDataXML = doc.ToString(); _ret.AddEvent(_parament); } } } return(_logcounter); }