public static List <EventData> ReadLogDataLogFSMJsonString(string JSON, bool RelativeTime, EventDataListExtension.ESortType Sort)
{
List <EventData> _return = JsonConvert.DeserializeObject <List <EventData> >(JSON);
if (RelativeTime)
{
_return = EventDataListExtension.SortByRelativeTime(_return, Sort);
_return.ComputeTimedifferencePreviousWithRelativeTimes();
}
else
{
_return = EventDataListExtension.SortByTimeStamp(_return, Sort);
_return.ComputeTimedifferencePrevious();
}
return(_return);
}
public static List <EventData> ReadLogDataJsonLite(string FileName, bool RelativeTime, EventDataListExtension.ESortType Sort)
{
string _json = File.ReadAllText(FileName);
var _data = JsonConvert.DeserializeObject <List <Dictionary <string, string> > >(_json);
List <EventData> _return = new List <EventData>();
foreach (var _l in _data)
{
EventData e = new EventData();
foreach (var _k in _l.Keys)
{
if (_k == "PersonIdentifier")
{
e.PersonIdentifier = _l[_k];
}
else if (_k == "TimeStamp")
{
e.TimeStamp = DateTime.Parse(_l[_k]);
}
else if (_k == "EventName")
{
e.EventName = _l[_k];
}
else if (_k == "TimeDifferencePrevious")
{
e.TimeDifferencePrevious = TimeSpan.Parse(_l[_k]);
}
else
{
e.AddEventValue(_k, _l[_k]);
}
}
_return.Add(e);
}
if (RelativeTime)
{
_return = EventDataListExtension.SortByRelativeTime(_return, Sort);
_return.ComputeTimedifferencePreviousWithRelativeTimes();
}
else
{
_return = EventDataListExtension.SortByTimeStamp(_return, Sort);
_return.ComputeTimedifferencePrevious();
}
return(_return);
}
public static List <EventData> ReadLogDataLogFSMJson(string FileName, bool RelativeTime, EventDataListExtension.ESortType Sort)
{
string _json = File.ReadAllText(FileName);
List <EventData> _return = JsonConvert.DeserializeObject <List <EventData> >(_json);
if (RelativeTime)
{
_return = EventDataListExtension.SortByRelativeTime(_return, Sort);
_return.ComputeTimedifferencePreviousWithRelativeTimes();
}
else
{
_return = EventDataListExtension.SortByTimeStamp(_return, Sort);
_return.ComputeTimedifferencePrevious();
}
return(_return);
}
public static List <EventData> ReadLogUniversalLogFormat(string ZipFileName, string PersonIdentifier, bool RelativeTime, CommandLineArguments ParsedCommandLineArguments, string[] Elements, EventDataListExtension.ESortType Sort)
{
string _columnNamePersonIdentifier = "PersonIdentifier";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("columnnamepersonidentifier"))
{
_columnNamePersonIdentifier = ParsedCommandLineArguments.ParameterDictionary["columnnamepersonidentifier"];
}
string _columnNameEventName = "EventName";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("columnnameeventname"))
{
_columnNameEventName = ParsedCommandLineArguments.ParameterDictionary["columnnameeventname"];
}
string _columnNameElement = "Element";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("columnnameelement"))
{
_columnNameElement = ParsedCommandLineArguments.ParameterDictionary["columnnameelement"];
}
string _columnNameTimeStamp = "TimeStamp";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("columnnametimestamp"))
{
_columnNameTimeStamp = ParsedCommandLineArguments.ParameterDictionary["columnnametimestamp"];
}
string _columnDelimiter = ";";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("columndelimiter"))
{
_columnDelimiter = ParsedCommandLineArguments.ParameterDictionary["columndelimiter"];
}
string _ignoretables = "Log;Results";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("ignoretables"))
{
_ignoretables = ParsedCommandLineArguments.ParameterDictionary["ignoretables"];
}
List <string> _listOfIgnoreTables = new List <string>();
foreach (var t in _ignoretables.Split(";").ToArray <string>())
{
_listOfIgnoreTables.Add(t.Trim());
}
List <string> _notEventSpecificValues = new List <string>();
_notEventSpecificValues.Add(_columnNameElement);
_notEventSpecificValues.Add(_columnNameEventName);
_notEventSpecificValues.Add(_columnNamePersonIdentifier);
_notEventSpecificValues.Add(_columnNameTimeStamp);
if (!ParsedCommandLineArguments.Flags.Contains("INCLUDEREFERENCEVARIABLES"))
{
_notEventSpecificValues.Add("Path");
_notEventSpecificValues.Add("ReferenceCounter");
_notEventSpecificValues.Add("ParentReferenceCounter");
}
DateTime dt1960 = new DateTime(1960, 1, 1, 0, 0, 0, 0);
List <EventData> _return = new List <EventData>();;
using (ZipFile zip = ZipFile.Read(ZipFileName))
{
foreach (ZipEntry e in zip.Entries)
{
string _filenameWithoutExt = Path.GetFileNameWithoutExtension(e.FileName);
#region STATA
if (e.FileName.ToLower().EndsWith(".dta") && !_listOfIgnoreTables.Contains(_filenameWithoutExt))
{
using (MemoryStream zipStream = new MemoryStream())
{
e.Extract(zipStream);
zipStream.Position = 0;
var str = new StataFileReader(zipStream, true);
// cache variables
Dictionary <int, StataVariable> _eventDataKeys = new Dictionary <int, StataVariable>();
var vardict = new Dictionary <string, Tuple <int, StataVariable> >();
for (int i = 0; i < str.Variables.Count; i++)
{
vardict.Add(str.Variables[i].Name, new Tuple <int, StataVariable>(i, str.Variables[i]));
if (!_notEventSpecificValues.Contains(str.Variables[i].Name))
{
_eventDataKeys.Add(i, str.Variables[i]);
}
}
// cache value labels
Dictionary <string, Dictionary <int, string> > valuedict = new Dictionary <string, Dictionary <int, string> >();
foreach (var v in str.ValueLabels)
{
if (!valuedict.ContainsKey(v.Item1))
{
valuedict.Add(v.Item1, new Dictionary <int, string>());
}
valuedict[v.Item1].Add(v.Item2, v.Item3);
}
// cache element list
List <string> _selectedElements = new List <string>();
string _elementLabelSetName = vardict[_columnNameElement].Item2.ValueLabelName;
if (_elementLabelSetName != "")
{
var _elementLabelSet = valuedict[_elementLabelSetName];
foreach (var v in _elementLabelSet.Keys)
{
_selectedElements.Add(v.ToString());
}
}
int _elementColumnIndex = vardict[_columnNameElement].Item1;
Dictionary <int, string> _elementValueLabelDict = new Dictionary <int, string>();
if (vardict[_columnNameElement].Item2.ValueLabelName != "")
{
_elementValueLabelDict = valuedict[vardict[_columnNameElement].Item2.ValueLabelName];
}
int _personIdentifierColumnIndex = vardict[_columnNamePersonIdentifier].Item1;
Dictionary <int, string> _personIdentifierValueLabelDict = new Dictionary <int, string>();
if (vardict[_columnNamePersonIdentifier].Item2.ValueLabelName != "")
{
_personIdentifierValueLabelDict = valuedict[vardict[_columnNamePersonIdentifier].Item2.ValueLabelName];
}
int _eventNameColumnIndex = vardict[_columnNameEventName].Item1;
Dictionary <int, string> _eventNameValueLabelDict = new Dictionary <int, string>();
if (vardict[_columnNameEventName].Item2.ValueLabelName != "")
{
_eventNameValueLabelDict = valuedict[vardict[_columnNameEventName].Item2.ValueLabelName];
}
int _timeStampColumnIndex = vardict[_columnNameTimeStamp].Item1;
Dictionary <int, string> _timeStampLabelDict = new Dictionary <int, string>();
if (vardict[_columnNameTimeStamp].Item2.ValueLabelName != "")
{
_timeStampLabelDict = valuedict[vardict[_columnNameTimeStamp].Item2.ValueLabelName];
}
foreach (var _line in str)
{
if (_selectedElements.Count == 0 || _selectedElements.Contains(_line[_elementColumnIndex].ToString()))
{
string _eventName = _line[_eventNameColumnIndex].ToString();
if (_eventNameValueLabelDict.Count > 0)
{
_eventName = _eventNameValueLabelDict[int.Parse(_eventName)];
}
string _personIdentifier = _line[_personIdentifierColumnIndex].ToString();
if (_personIdentifierValueLabelDict.Count > 0)
{
_personIdentifier = _personIdentifierValueLabelDict[int.Parse(_personIdentifier)];
}
string _element = _line[_elementColumnIndex].ToString();
if (_elementValueLabelDict.Count > 0 && _elementValueLabelDict.ContainsKey(int.Parse(_element)))
{
_element = _elementValueLabelDict[int.Parse(_element)];
}
DateTime _timeStamp = DateTime.MinValue;
if (RelativeTime)
{
DateTime.TryParse(_line[_timeStampColumnIndex].ToString(), out _timeStamp);
}
else
{
try
{
_timeStamp = dt1960.AddMilliseconds(double.Parse(_line[_timeStampColumnIndex].ToString()));
}
catch
{
//TODO VERSION 0.3: Add a more proper check for invalid time stamps
}
}
var _eventValues = new Dictionary <string, string>();
foreach (var v in _eventDataKeys.Keys)
{
string _dictName = vardict[_eventDataKeys[v].Name].Item2.ValueLabelName;
if (valuedict.ContainsKey(_dictName))
{
var _dict = valuedict[_dictName];
string _value = _line[v].ToString();
if (_dict.ContainsKey(int.Parse(_value)))
{
_value = _dict[int.Parse(_value)].ToString();
}
_eventValues.Add(_eventDataKeys[v].Name, _value);
}
else
{
_eventValues.Add(_eventDataKeys[v].Name, _line[v].ToString());
}
}
bool _add = true;
if (_personIdentifier != PersonIdentifier)
{
_add = false;
}
else if (ParsedCommandLineArguments.Elements.Length != 0 && !ParsedCommandLineArguments.Elements.Contains <string>(_element))
{
_add = false;
}
else if (ParsedCommandLineArguments.Events.Length != 0 && !ParsedCommandLineArguments.Events.Contains <string>(_eventName))
{
_add = false;
}
else if (ParsedCommandLineArguments.ExcludedElements.Length != 0 && ParsedCommandLineArguments.ExcludedElements.Contains <string>(_element))
{
_add = false;
}
else if (ParsedCommandLineArguments.ExcludedEvents.Length != 0 && ParsedCommandLineArguments.ExcludedEvents.Contains <string>(_eventName))
{
_add = false;
}
if (_add)
{
_return.Add(new EventData()
{
Element = _element,
EventName = _eventName,
PersonIdentifier = _personIdentifier,
TimeStamp = _timeStamp,
EventValues = _eventValues
});
}
}
}
}
}
#endregion
#region CSV
if (e.FileName.ToLower().EndsWith(".csv") && !_listOfIgnoreTables.Contains(_filenameWithoutExt))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.Write("- read: " + e.FileName + " (" + e.UncompressedSize + ")");
}
// TODO: Compute exact limit for memory streams
if (e.UncompressedSize > 2000000000)
{
var fileName = Path.GetTempFileName();
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("--> temp file: " + fileName);
}
using (FileStream fileStream = File.OpenWrite(fileName))
{
e.Extract(fileStream);
}
using (FileStream fileStream = File.OpenRead(fileName))
{
extracCSVStream_ReadLogUniversalLogFormat(PersonIdentifier, RelativeTime, ParsedCommandLineArguments, _columnNamePersonIdentifier, _columnNameEventName, _columnNameElement, _columnNameTimeStamp, _columnDelimiter, _notEventSpecificValues, dt1960, _return, fileStream);
}
File.Delete(fileName);
}
else
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("");
}
using (MemoryStream zipStream = new MemoryStream())
{
e.Extract(zipStream);
zipStream.Position = 0;
extracCSVStream_ReadLogUniversalLogFormat(PersonIdentifier, RelativeTime, ParsedCommandLineArguments, _columnNamePersonIdentifier, _columnNameEventName, _columnNameElement, _columnNameTimeStamp, _columnDelimiter, _notEventSpecificValues, dt1960, _return, zipStream);
}
}
}
#endregion
#region SPSS
if (e.FileName.ToLower().EndsWith(".sav") && !_listOfIgnoreTables.Contains(_filenameWithoutExt))
{
using (MemoryStream zipStream = new MemoryStream())
{
e.Extract(zipStream);
zipStream.Position = 0;
SpssLib.DataReader.SpssReader spssDataset = new SpssLib.DataReader.SpssReader(zipStream);
foreach (var record in spssDataset.Records)
{
string _personIdentifier = "";
string _eventName = "";
string _element = "";
DateTime _timeStamp = DateTime.MinValue;
var _eventValues = new Dictionary <string, string>();
foreach (var variable in spssDataset.Variables)
{
if (variable.Name == _columnNamePersonIdentifier)
{
_personIdentifier = record.GetValue(variable).ToString();
if (variable.ValueLabels.Count > 0)
{
double _doubleValue = (double)record.GetValue(variable);
if (variable.ValueLabels.ContainsKey(_doubleValue))
{
_personIdentifier = variable.ValueLabels[_doubleValue];
}
}
}
else if (variable.Name == _columnNameEventName)
{
_eventName = record.GetValue(variable).ToString();
if (variable.ValueLabels.Count > 0)
{
double _doubleValue = (double)record.GetValue(variable);
if (variable.ValueLabels.ContainsKey(_doubleValue))
{
_eventName = variable.ValueLabels[_doubleValue];
}
}
}
else if (variable.Name == _columnNameElement)
{
_element = record.GetValue(variable).ToString();
if (variable.ValueLabels.Count > 0)
{
double _doubleValue = (double)record.GetValue(variable);
if (variable.ValueLabels.ContainsKey(_doubleValue))
{
_element = variable.ValueLabels[_doubleValue];
}
}
}
else if (variable.Name == _columnNameTimeStamp)
{
string _TimeStamp = record.GetValue(variable).ToString();
if (variable.ValueLabels.Count > 0)
{
double _doubleValue = (double)record.GetValue(variable);
if (variable.ValueLabels.ContainsKey(_doubleValue))
{
_TimeStamp = variable.ValueLabels[_doubleValue];
}
}
if (RelativeTime)
{
DateTime.TryParse(_TimeStamp, out _timeStamp);
}
else
{
try
{
_timeStamp = dt1960.AddMilliseconds(double.Parse(_TimeStamp));
}
catch
{
//TODO VERSION 0.3: Add a more proper check for invalid time stamps
}
}
}
else
{
if (!_notEventSpecificValues.Contains(variable.Name))
{
string _key = variable.Name;
string _value = record.GetValue(variable).ToString();
if (variable.ValueLabels.Count > 0)
{
double _doubleValue = (double)record.GetValue(variable);
if (variable.ValueLabels.ContainsKey(_doubleValue))
{
_value = variable.ValueLabels[_doubleValue];
}
}
_eventValues.Add(_key, _value);
}
}
}
bool _add = true;
if (_personIdentifier != PersonIdentifier)
{
_add = false;
}
else if (ParsedCommandLineArguments.Elements.Length != 0 && !ParsedCommandLineArguments.Elements.Contains <string>(_element))
{
_add = false;
}
else if (ParsedCommandLineArguments.Events.Length != 0 && !ParsedCommandLineArguments.Events.Contains <string>(_eventName))
{
_add = false;
}
else if (ParsedCommandLineArguments.ExcludedElements.Length != 0 && ParsedCommandLineArguments.ExcludedElements.Contains <string>(_element))
{
_add = false;
}
else if (ParsedCommandLineArguments.ExcludedEvents.Length != 0 && ParsedCommandLineArguments.ExcludedEvents.Contains <string>(_eventName))
{
_add = false;
}
if (_add)
{
_return.Add(new EventData()
{
Element = _element,
EventName = _eventName,
PersonIdentifier = _personIdentifier,
TimeStamp = _timeStamp,
EventValues = _eventValues
});
}
}
}
}
#endregion
}
}
if (RelativeTime)
{
_return = EventDataListExtension.SortByRelativeTime(_return, Sort);
_return.ComputeTimedifferencePreviousWithRelativeTimes();
}
else
{
_return = EventDataListExtension.SortByTimeStamp(_return, Sort);
_return.ComputeTimedifferencePrevious();
}
return(_return);
}
public static List <EventData> ReadLogDataEE(string FileName, bool RelativeTime, EventDataListExtension.ESortType Sort)
{
string _xml = File.ReadAllText(FileName);
var _return = ReadLogDataEEFromXMLString(_xml, RelativeTime, Sort);
return(_return);
}
public static List <EventData> ReadLogDataEEFromXMLString(string XML, bool RelativeTime, EventDataListExtension.ESortType Sort)
{
XmlDocument doc = new XmlDocument();
doc.LoadXml(XML);
string PersonIdentifier = "Unknown";
List <EventData> _return = new List <EventData>();
XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable);
nsmgr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
nsmgr.AddNamespace("cbaloggingmodel", "http://www.softcon.de/cba/cbaloggingmodel");
foreach (XmlNode row in doc.SelectNodes("//logEntry[@xsi:type='cbaloggingmodel:CBAItemLogEntry']", nsmgr))
{
PersonIdentifier = row.Attributes["user"].Value.ToString();
}
foreach (XmlNode row in doc.SelectNodes("//logEntry"))
{
if (row.ChildNodes.Count == 1)
{
XDocument _xmlElement = XDocument.Parse(row.ChildNodes[0].OuterXml);
if (row.ChildNodes[0].Attributes["xsi:type"] != null)
{
EventData _parament = new EventData()
{
EventName = row.ChildNodes[0].Attributes["xsi:type"].Value.Replace("cbaloggingmodel:", ""), PersonIdentifier = PersonIdentifier, TimeStamp = DateTime.Parse(row.Attributes["timeStamp"].Value)
};
_return.Add(_parament);
AddEventData(_xmlElement.Root, _parament, _return);
}
}
}
if (RelativeTime)
{
_return = EventDataListExtension.SortByRelativeTime(_return, Sort);
_return.ComputeTimedifferencePreviousWithRelativeTimes();
}
else
{
_return = EventDataListExtension.SortByTimeStamp(_return, Sort);
_return.ComputeTimedifferencePrevious();
}
return(_return);
}
private static int ReadLogDataPIAACFromLDAExportString(int _logcounter, int _lineCounter, string line, logXContainer _ret, string _PersonIdentifier, CommandLineArguments ParsedCommandLineArguments, EventDataListExtension.ESortType sort, Dictionary <string, string> _lookup, string[] Element)
{
string[] _cols = line.Split('\t');
if (_cols.Length > 1 && _lineCounter > 0)
{
_PersonIdentifier = _cols[0] + "_" + _cols[1];
string _rawBooklet = _cols[2];
string _rawSequenceID = _cols[3];
string _rawEventName = _cols[4];
string _rawEventType = _cols[5];
string _relativeTimeString = _cols[6];
string _rawEventDescription = _cols[7];
string _itemID = _rawBooklet + "_" + _rawSequenceID;
long _relativeTime = long.Parse(_cols[6], CultureInfo.InvariantCulture);
if (!ParsedCommandLineArguments.Flags.Contains("USE_BOOKLLET_SEQUENCE_ID"))
{
if (_lookup.ContainsKey(_itemID))
{
_itemID = _lookup[_itemID];
}
}
bool _isFiltered = Element.Length == 0;
if (Element.Contains(_itemID))
{
_isFiltered = true;
}
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
return(_logcounter);
}
if (_isFiltered)
{
if (_rawEventDescription.StartsWith("<cbaloggingmodel", StringComparison.InvariantCulture))
{
// IB xml events
_logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
else if (_rawEventDescription.StartsWith("<cbascoringresultmm", StringComparison.InvariantCulture))
{
// IB item score
if (_rawEventType == "ItemScore")
{
if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES") || !ParsedCommandLineArguments.Flags.Contains("INCLUDE_NO_ITEMSCORES"))
{
_logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
}
else if (_rawEventType == "itemScoreResult")
{
// by default ignore second item score
if (1 == 1 || ParsedCommandLineArguments.Flags.Contains("INCLUDE_ALL_ITEMSCORES"))
{
_logcounter = piaac_R1_extract_IB_XML(_logcounter, _ret, _lookup, _rawEventDescription, _PersonIdentifier, _rawSequenceID, _rawBooklet, _rawEventType, _rawEventName, _relativeTime, ParsedCommandLineArguments);
}
}
else
{
throw new Exception("The value '" + _rawEventName + "' was not expected in line " + _lineCounter + ".");
}
}
else if (_rawEventDescription.StartsWith("http://localhost:8080", StringComparison.InvariantCulture))
{
// ignore snapshots
}
else
{
// Pre-process event values
logxGenericLogElement _parament = new logxGenericLogElement()
{
PersonIdentifier = _PersonIdentifier,
EventName = _rawEventType,
Item = _itemID,
RelativeTime = _relativeTime,
EventID = _logcounter
};
_logcounter++;
var doc = new XDocument(new XElement(_parament.EventName));
var root = doc.Root;
_rawEventDescription = _rawEventDescription.Replace("|$*", "|*$");
string[] _kvps = _rawEventDescription.Split("|*$");
foreach (string _kvp in _kvps)
{
string[] _kv = _kvp.Split('=');
if (_kv.Length == 2)
{
root.Add(new XAttribute(_kv[0], _kv[1]));
}
else if (_kv.Length == 1)
{
root.Add(new XAttribute("value", _kv[0]));
}
else
{
string _remainingvalue = _kv[1];
for (int i = 2; i < _kv.Length; i++)
{
_remainingvalue = _remainingvalue + "=" + _kv[i];
}
root.Add(new XAttribute(_kv[0], _remainingvalue));
}
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_BOOKLET_INFORMATION"))
{
root.Add(new XAttribute("PIAACBookelt", _rawBooklet));
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_EVENT_INFORMATION"))
{
root.Add(new XAttribute("PIAACSequenceID", _rawSequenceID));
root.Add(new XAttribute("PIAACEventType", _rawEventType));
root.Add(new XAttribute("PIAACEventName", _rawEventName));
}
if (!ParsedCommandLineArguments.Flags.Contains("HIDE_TIME_INFORMATION"))
{
root.Add(new XAttribute("RelativeTimeString", _relativeTimeString));
}
_parament.EventDataXML = doc.ToString();
_ret.AddEvent(_parament);
}
}
}
return(_logcounter);
}
public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments)
{
try
{
bool _personIdentifierIsNumber = false;
if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER"))
{
_personIdentifierIsNumber = true;
}
string _personIdentifier = "PersonIdentifier";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier"))
{
_personIdentifier = ParsedCommandLineArguments.ParameterDictionary["personidentifier"];
}
string _language = "ENG";
if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language"))
{
_language = ParsedCommandLineArguments.ParameterDictionary["language"];
}
if (!ParsedCommandLineArguments.RelativeTime)
{
ParsedCommandLineArguments.RelativeTime = true;
Console.Write("Note: Changed to relative times. ");
}
EventDataListExtension.ESortType sort = EventDataListExtension.ESortType.ElementAndTime;
if (ParsedCommandLineArguments.Flags.Contains("DONT_ORDER_EVENTS"))
{
sort = EventDataListExtension.ESortType.None;
}
Dictionary <string, string> _lookup = LogDataTransformer_PIAACR1_Module_V01.GetPIAACR1LookupDictionary();
#region Search Source Files
List <string> _listOfTXTFiles = new List <string>();
List <string> _listOfZIPArchivesWithTXTFiles = new List <string>();
foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders)
{
if (File.Exists(inFolder))
{
if (inFolder.ToLower().EndsWith(".zip"))
{
_listOfZIPArchivesWithTXTFiles.Add(inFolder);
}
else if (inFolder.ToLower().EndsWith(".txt"))
{
_listOfTXTFiles.Add(inFolder);
}
}
else
{
if (!Directory.Exists(inFolder))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'.");
}
continue;
}
var _tmpXMLFileList = Directory.GetFiles(inFolder, "*.txt", SearchOption.AllDirectories);
foreach (string s in _tmpXMLFileList)
{
_listOfTXTFiles.Add(s);
}
var _tmpZIPFileList = Directory.GetFiles(inFolder, "*.zip", SearchOption.AllDirectories);
foreach (string s in _tmpZIPFileList)
{
_listOfZIPArchivesWithTXTFiles.Add(s);
}
}
}
#endregion
#region Process Source Files
logXContainer _ret = new logXContainer()
{
PersonIdentifierIsNumber = _personIdentifierIsNumber, PersonIdentifierName = _personIdentifier
};
_ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary);
int _logcounter = 0;
foreach (string zfilename in _listOfZIPArchivesWithTXTFiles)
{
using (ZipFile zip = ZipFile.Read(zfilename))
{
foreach (var entry in zip)
{
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
// TODO: Check FitsMask
if (1 == 1 || CommandLineArguments.FitsMask(entry.FileName, ParsedCommandLineArguments.Mask))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.Write("Info: Read File '" + entry.FileName + "' ");
}
using (MemoryStream zipStream = new MemoryStream())
{
entry.ExtractWithPassword(zipStream, "");
zipStream.Position = 0;
try
{
StreamReader sr = new StreamReader(zipStream);
string line = String.Empty;
int _lineCounter = 0;
while ((line = sr.ReadLine()) != null)
{
_logcounter = ReadLogDataPIAACFromLDAExportString(_logcounter, _lineCounter, line, _ret, _personIdentifier, ParsedCommandLineArguments, sort, _lookup, new string[] { });
_lineCounter++;
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
}
}
catch (Exception _ex)
{
Console.WriteLine("Error processing file '" + entry.FileName + "': " + _ex.Message);
return;
}
}
Console.WriteLine("ok.");
}
}
}
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
}
foreach (string txtFile in _listOfTXTFiles)
{
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
if (1 == 1 || CommandLineArguments.FitsMask(Path.GetFileName(txtFile), ParsedCommandLineArguments.Mask))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Read File '" + Path.GetFileName(txtFile) + "' ");
}
try
{
StreamReader sr = new StreamReader(txtFile);
string line = String.Empty;
int _lineCounter = 0;
while ((line = sr.ReadLine()) != null)
{
_logcounter = ReadLogDataPIAACFromLDAExportString(_logcounter, _lineCounter, line, _ret, _personIdentifier, ParsedCommandLineArguments, sort, _lookup, new string[] { });
_lineCounter++;
if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases)
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Info: Max number of cases reached.");
}
break;
}
}
}
catch (Exception _ex)
{
Console.WriteLine("Error processing file '" + txtFile + "': " + _ex.Message);
return;
}
Console.WriteLine("ok.");
}
}
#endregion
#region Export Universal Log Format
// TODO: Check!
//_ret.UpdateRelativeTimes();
_ret.CreateLookup();
if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with Stata file(s).");
}
_ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language);
}
if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create XLSX file.");
}
_ret.ExportXLSX(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create ZIP archive with CSV file(s).");
}
_ret.ExportCSV(ParsedCommandLineArguments);
}
if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "")
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create Codebook File.");
}
_ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language);
}
if (ParsedCommandLineArguments.Transform_ConcordanceTable.Trim() != "")
{
if (!File.Exists(ParsedCommandLineArguments.Transform_ConcordanceTable))
{
if (ParsedCommandLineArguments.Verbose)
{
Console.WriteLine("Create Concordance Table.");
}
_ret.CreateConcordanceTable(ParsedCommandLineArguments.Transform_ConcordanceTable);
}
}
if (_ret.ExportErrors.Count > 0)
{
Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files.");
if (ParsedCommandLineArguments.Verbose)
{
for (int i = 0; i < _ret.ExportErrors.Count; i++)
{
Console.WriteLine(_ret.ExportErrors[i]);
}
}
}
#endregion
}
catch (Exception _ex)
{
Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.Message.ToString());
}
}
