private static void AddEventData(XElement xmlelement, logxGenericLogElement parent, logXContainer _ret) { var doc = new XDocument(new XElement(parent.EventName)); var root = doc.Root; foreach (var a in xmlelement.Attributes()) { if (a.Name.Namespace.NamespaceName == "") { root.Add(new XAttribute(a.Name.ToString(), a.Value)); } } parent.EventDataXML = doc.ToString(); int _logcounter = 0; foreach (XElement x in xmlelement.Elements()) { logxGenericLogElement _newparent = new logxGenericLogElement() { Item = parent.Item, EventName = parent.EventName + "." + x.Name.LocalName, PersonIdentifier = parent.PersonIdentifier, TimeStamp = parent.TimeStamp, EventID = _logcounter, }; AddEventData(x, _newparent, _ret); _ret.AddEvent(_newparent); _logcounter++; } }
// TODO: Fix bug with EventID and ParentEventID private static void ReadLogDataEEFromXMLString(string XML, logXContainer _ret) { XmlDocument doc = new XmlDocument(); doc.LoadXml(XML); string PersonIdentifier = "Unknown"; string Element = ""; XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable); nsmgr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance"); nsmgr.AddNamespace("cbaloggingmodel", "http://www.softcon.de/cba/cbaloggingmodel"); foreach (XmlNode row in doc.SelectNodes("//logEntry[@xsi:type='cbaloggingmodel:CBAItemLogEntry']", nsmgr)) { PersonIdentifier = row.Attributes["user"].Value.ToString(); Element = row.Attributes["name"].Value.Replace("de.softcon.cba.runtime.", "").ToString(); } int _logcounter = 0; foreach (XmlNode row in doc.SelectNodes("//logEntry")) { if (row.ChildNodes.Count == 1) { XDocument _xmlElement = XDocument.Parse(row.ChildNodes[0].OuterXml); if (row.ChildNodes[0].Attributes["xsi:type"] != null) { logxGenericLogElement _parament = new logxGenericLogElement() { EventName = row.ChildNodes[0].Attributes["xsi:type"].Value.Replace("cbaloggingmodel:", ""), PersonIdentifier = PersonIdentifier, TimeStamp = DateTime.Parse(row.Attributes["timeStamp"].Value), Item = Element, EventID = _logcounter }; _logcounter++; AddEventData(_xmlElement.Root, _parament, _ret); _ret.AddEvent(_parament); } } } }
public static logXContainer CreateGenericLogContainer(List <string> StataFiles, string PersonIdentifierName, bool PersonIdentifierIsNumber, string[] ExcludedElements) { logXContainer _ret = new logXContainer() { PersonIdentifierName = PersonIdentifierName, PersonIdentifierIsNumber = PersonIdentifierIsNumber }; try { int _progressCounter = 0; foreach (string StataLogInFileName in StataFiles) { Console.WriteLine(StataLogInFileName + " - " + _progressCounter + " / " + StataFiles.Count()); StataFileReader _stataLogFileReader = new StataFileReader(StataLogInFileName, true); // cache stataVariables and stataValueLabels for performance reasons var _stataVariables = _stataLogFileReader.Variables; var _stataValueLabes = _stataLogFileReader.ValueLabels; int _personIdentifierIndex = GetVariableIndex(_ret.PersonIdentifierName, _stataVariables); int _absolutTimeIndex = GetVariableIndex("AbsoluteTime", _stataVariables); int _lineCounterIndex = GetVariableIndex("LineCounter", _stataVariables); int _logTypeIndex = GetVariableIndex("LogType", _stataVariables); int _logCounterIndex = GetVariableIndex("LogCounter", _stataVariables); int _relativeTimeIndex = GetVariableIndex("RelativeTime", _stataVariables); int _absoluteTimeIndex = GetVariableIndex("AbsoluteTime", _stataVariables); int _elementIndex = GetVariableIndex("Element", _stataVariables); int _varNameIndex = GetVariableIndex("VariableName", _stataVariables); int _varValueIndex = GetVariableIndex("VariableValue", _stataVariables); int _varValueLabelIndex = GetVariableIndex("ValueLabel", _stataVariables); int _xmlIndex = GetVariableIndex("XML", _stataVariables); // find first RealTime for each TestTaker Dictionary <string, DateTime> _firstLoginForEachPerson = new Dictionary <string, DateTime>(); foreach (var _line in _stataLogFileReader) { string _personIdentifier = GetStringValue(_personIdentifierIndex, _line, _stataVariables, _stataValueLabes); DateTime _absolutetime = GetDateTimeValue(_absolutTimeIndex, _line, _stataVariables, _stataValueLabes); if (!_firstLoginForEachPerson.ContainsKey(_personIdentifier)) { _firstLoginForEachPerson.Add(_personIdentifier, DateTime.MaxValue); } if (_absolutetime.Year != 1) { if (_firstLoginForEachPerson[_personIdentifier] > _absolutetime) { _firstLoginForEachPerson[_personIdentifier] = _absolutetime; } } } // extract log data to genereric log element foreach (var _line in _stataLogFileReader) { string _personIdentifier = GetStringValue(_personIdentifierIndex, _line, _stataVariables, _stataValueLabes); int _linecounter = GetIntValue(_lineCounterIndex, _line, _stataVariables); string _logType = GetStringValue(_logTypeIndex, _line, _stataVariables, _stataValueLabes); int _logcounter = GetIntValue(_logCounterIndex, _line, _stataVariables); long _relativetime = GetLogValue(_relativeTimeIndex, _line, _stataVariables); DateTime _absolutetime = GetDateTimeValue(_absolutTimeIndex, _line, _stataVariables, _stataValueLabes); if (_absolutetime.Year == 1) { _absolutetime = _firstLoginForEachPerson[_personIdentifier].AddMilliseconds(-1 * _relativetime); } string _element = GetStringValue(_elementIndex, _line, _stataVariables, _stataValueLabes); string _varname = GetStringValue(_varNameIndex, _line, _stataVariables, _stataValueLabes); string _value = GetStringValue(_varValueIndex, _line, _stataVariables, _stataValueLabes); string _valueLabel = GetStringValue(_varValueLabelIndex, _line, _stataVariables, _stataValueLabes); string _xml = GetStringValue(_xmlIndex, _line, _stataVariables, _stataValueLabes); #region process log events if (!ExcludedElements.Contains <string>(_element)) { if (_logType == "TTLogVariableChanged") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsVariableChanged).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsVariableChanged() { Sender = _element, Variable = _varname, Value = _value, ValueLabel = _valueLabel }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else if (_logType == "TTLogRealTime") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsRealTime).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsRealTime() { RealTime = _absolutetime }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else if (_logType == "TTLogLoading") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsLoading).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsLoading() { Sender = _element }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else if (_logType == "TTLogLoaded") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsLoaded).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsLoaded() { Sender = _element }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else if (_logType == "TTLogUnloaded") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsUnloaded).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsUnloaded() { Sender = _element }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else if (_logType == "TTLogUnloading") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsUnloading).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsUnloading() { Sender = _element }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else if (_logType == "TTLogIBStopTask") { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, Item = _element, EventName = typeof(TBAToolsIBStopTask).Name, EventDataXML = TBAToolsLogReader.XmlSerializeToString(new TBAToolsIBStopTask() { Sender = _element }), PersonIdentifier = _personIdentifier, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } else { _ret.AddEvent(new logxGenericLogElement() { EventID = _logcounter, EventName = _logType, Item = _element, PersonIdentifier = _personIdentifier, EventDataXML = _xml, TimeStamp = _absolutetime, RelativeTime = _relativetime }); } } #endregion } _stataLogFileReader.Close(); _progressCounter++; } } catch (Exception _ex) { Console.WriteLine(_ex.ToString()); throw new Exception(); } return(_ret); }
private static void processPISA_BQ_single_XML(string[] Element, DateTime dt1970, logXContainer _ret, XmlSerializer logSerializer, string _PersonIdentifier, string _xml) { var _tr = new StringReader(_xml); log _log = (log)logSerializer.Deserialize(_tr); if (_log.itemGroup != null) { // update epoch information foreach (var i in _log.itemGroup) { if (!i.epochSpecified && i.userEvents.Length > 0) { i.epoch = i.userEvents[0].epoch; } } List <logItemGroup> _sortedItemGroupList = _log.itemGroup.OrderBy(o => o.epoch).ToList(); DateTime _MinAbsoluteTime = DateTime.MaxValue; DateTime _PreviousEvent = DateTime.MaxValue; if (_log.User != _PersonIdentifier) { throw new Exception("Person identifier miss-match."); } int _EventID = _ret.GetMaxID(_PersonIdentifier); int _EventVisitCounter = 0; Dictionary <string, int> _elementVisitCounterDict = new Dictionary <string, int>(); string _currentElement = ""; foreach (var p in _sortedItemGroupList) { string _Element = p.code; if (_currentElement != _Element) { _EventVisitCounter = 0; _currentElement = _Element; if (!_elementVisitCounterDict.ContainsKey(_Element)) { _elementVisitCounterDict.Add(_Element, 0); } else { _elementVisitCounterDict[_Element] += 1; } } DateTime _ElementStart = dt1970.AddMilliseconds(p.epoch); if (_PreviousEvent == DateTime.MaxValue) { _PreviousEvent = _ElementStart; } foreach (var _event in p.userEvents) { string _LogEventName = _event.type; DateTime _AbsoluteTime = dt1970.AddMilliseconds(_event.epoch); if (_AbsoluteTime < _MinAbsoluteTime) { _MinAbsoluteTime = _AbsoluteTime; } Dictionary <string, string> _EventValues = new Dictionary <string, string>(); for (int i = 0; i < _event.ItemsElementName.Length; i++) { if (_event.ItemsElementName[i].ToString() == "context") { _EventValues.Add("Context", _event.Items[i]); } else if (_event.ItemsElementName[i].ToString() == "value") { _EventValues.Add("Value", _event.Items[i]); } else if (_event.ItemsElementName[i].ToString() == "id") { _EventValues.Add("Id", _event.Items[i]); } else { throw new Exception("Element name not expected."); } } _EventValues.Add("RelativeTimeFrame", (_AbsoluteTime - _ElementStart).TotalMilliseconds.ToString()); _EventValues.Add("RelativeTimePrevious", (_AbsoluteTime - _PreviousEvent).TotalMilliseconds.ToString()); if (Element.Length == 0 || Element.Contains <string>(_Element)) { var doc = new XDocument(new XElement(_LogEventName)); var root = doc.Root; foreach (string val in _EventValues.Keys) { root.Add(new XAttribute(val, _EventValues[val])); } logxGenericLogElement _parament = new logxGenericLogElement() { PersonIdentifier = _PersonIdentifier, Item = _Element, EventID = _EventID, EventName = _LogEventName, TimeStamp = _AbsoluteTime, EventDataXML = doc.ToString() }; _ret.AddEvent(_parament); } _EventID += 1; _EventVisitCounter += 1; _PreviousEvent = _AbsoluteTime; } } // check for suspicious times /* TODO * _currentElement = ""; * List<string> framesWithSuspiciousData = new List<string>(); * foreach (var v in _inMemoryTempDataEvents) * { * if (_currentElement != v.EventName) * _currentElement = v.EventName; * * if (v.TimeDifferencePrevious.TotalMinutes > 30) * v.AddEventValue("Flag", "TimeToLong"); * * if (v.TimeDifferencePrevious.TotalMilliseconds < 0) * v.AddEventValue("Flag", "TimeNegative"); * } */ } _tr.Close(); }
private static int piaac_R1_extract_IB_XML_process_child(int _logcounter, logXContainer _ret, Dictionary <string, string> _lookup, XmlNode node, string Path, int Level, string PersonIdentifier, string _rawSequenceID, string _rawBooklet, string _rawEventType, string _rawEventName, long _relativeTime, CommandLineArguments ParsedCommandLineArguments) { string _name = ""; foreach (XmlNode n in node.ChildNodes) { _name = n.LocalName; if (Path.Trim() != "") { _name = Path + "." + _name; } string _itemID = _rawBooklet + "_" + _rawSequenceID; if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID")) { if (_lookup.ContainsKey(_itemID)) { _itemID = _lookup[_itemID]; } } XmlAttributeCollection atributos = n.Attributes; logxGenericLogElement _parament = new logxGenericLogElement() { PersonIdentifier = PersonIdentifier, EventName = _name, Item = _itemID, RelativeTime = _relativeTime, EventID = _logcounter }; _logcounter++; var doc = new XDocument(new XElement(_parament.EventName)); var root = doc.Root; foreach (XmlAttribute at in atributos) { if (at.LocalName == "cbaloggingmodel" || at.LocalName == "cbascoringresultmm" || at.LocalName == "snapshot" || at.LocalName == "xmi" || at.LocalName == "xsi") { // ignore attributes by default if (ParsedCommandLineArguments.Flags.Contains("INCLUDE_XML_ATTRIBUTES")) { root.Add(new XAttribute(at.LocalName, at.Value)); } } else { root.Add(new XAttribute(at.LocalName, at.Value)); } } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION")) { root.Add(new XAttribute("PIAACBookelt", _rawBooklet)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION")) { root.Add(new XAttribute("PIAACEventType", _rawEventType)); root.Add(new XAttribute("PIAACEventName", _rawEventName)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_XML_NESTING_LEVEL")) { root.Add(new XAttribute("XMLNextingLevel", Level.ToString())); } _parament.EventDataXML = doc.ToString(); _ret.AddEvent(_parament); } foreach (XmlNode n in node.ChildNodes) { piaac_R1_extract_IB_XML_process_child(_logcounter, _ret, _lookup, n, _name, Level + 1, PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } return(_logcounter); }
private static int ReadLogDataPIAACFromLDAExportString(int _logcounter, int _lineCounter, string line, logXContainer _ret, string _PersonIdentifier, CommandLineArguments ParsedCommandLineArguments, EventDataListExtension.ESortType sort, Dictionary <string, string> _lookup, string[] Element) { string[] _cols = line.Split('\t'); if (_cols.Length > 1 && _lineCounter > 0) { _PersonIdentifier = _cols[0] + "_" + _cols[1]; string _rawBooklet = _cols[2]; string _rawSequenceID = _cols[3]; string _rawEventName = _cols[4]; string _rawEventType = _cols[5]; string _relativeTimeString = _cols[6]; string _rawEventDescription = _cols[7]; string _itemID = _rawBooklet + "_" + _rawSequenceID; long _relativeTime = long.Parse(_cols[6], CultureInfo.InvariantCulture); if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID")) { if (_lookup.ContainsKey(_itemID)) { _itemID = _lookup[_itemID]; } } bool _isFiltered = Element.Length == 0; if (Element.Contains(_itemID)) { _isFiltered = true; } if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { return(_logcounter); } if (_isFiltered) { if (_rawEventDescription.StartsWith("<cbaloggingmodel", StringComparison.InvariantCulture)) { // IB xml events _logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } else if (_rawEventDescription.StartsWith("<cbascoringresultmm", StringComparison.InvariantCulture)) { // IB item score if (_rawEventType == "ItemScore") { if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES") || !ParsedCommandLineArguments.Flags.Contains("INCLUDE_NO_ITEMSCORES")) { _logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } } else if (_rawEventType == "itemScoreResult") { // by default ignore second item score if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES")) { _logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments); } } else { throw new Exception("The value '" + _rawEventName + "' was not expected in line " + _lineCounter + "."); } } else if (_rawEventDescription.StartsWith("http://localhost:8080", StringComparison.InvariantCulture)) { // ignore snapshots } else { // Pre-process event values logxGenericLogElement _parament = new logxGenericLogElement() { PersonIdentifier = _PersonIdentifier, EventName = _rawEventType, Item = _itemID, RelativeTime = _relativeTime, EventID = _logcounter }; _logcounter++; var doc = new XDocument(new XElement(_parament.EventName)); var root = doc.Root; _rawEventDescription = _rawEventDescription.Replace("|$*", "|*$"); string[] _kvps = _rawEventDescription.Split("|*$"); foreach (string _kvp in _kvps) { string[] _kv = _kvp.Split('='); if (_kv.Length == 2) { root.Add(new XAttribute(_kv[0], _kv[1])); } else if (_kv.Length == 1) { root.Add(new XAttribute("value", _kv[0])); } else { string _remainingvalue = _kv[1]; for (int i = 2; i < _kv.Length; i++) { _remainingvalue = _remainingvalue + "=" + _kv[i]; } root.Add(new XAttribute(_kv[0], _remainingvalue)); } } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION")) { root.Add(new XAttribute("PIAACBookelt", _rawBooklet)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION")) { root.Add(new XAttribute("PIAACSequenceID", _rawSequenceID)); root.Add(new XAttribute("PIAACEventType", _rawEventType)); root.Add(new XAttribute("PIAACEventName", _rawEventName)); } if (!ParsedCommandLineArguments.Flags.Contains("HIDE_TIME_INFORMATION")) { root.Add(new XAttribute("RelativeTimeString", _relativeTimeString)); } _parament.EventDataXML = doc.ToString(); _ret.AddEvent(_parament); } } } return(_logcounter); }