public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments) { try { bool _personIdentifierIsNumber = false; if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER")) { _personIdentifierIsNumber = true; } string _personIdentifier = "PersonIdentifier"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier")) { _personIdentifier = ParsedCommandLineArguments.ParameterDictionary["personidentifier"]; } string _language = "ENG"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language")) { _language = ParsedCommandLineArguments.ParameterDictionary["language"]; } List <string> _listOfXMLFiles = new List <string>(); List <string> _listOfZIPArchivesWithXMLFiles = new List <string>(); foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders) { if (File.Exists(inFolder)) { if (inFolder.ToLower().EndsWith(".zip")) { _listOfZIPArchivesWithXMLFiles.Add(inFolder); } else if (inFolder.ToLower().EndsWith(".xml")) { _listOfXMLFiles.Add(inFolder); } } else { if (!Directory.Exists(inFolder)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'."); } continue; } var _tmpXMLFileList = Directory.GetFiles(inFolder, "*.xml", SearchOption.AllDirectories); foreach (string s in _tmpXMLFileList) { _listOfXMLFiles.Add(s); } var _tmpZIPFileList = Directory.GetFiles(inFolder, "*.zip", SearchOption.AllDirectories); foreach (string s in _tmpZIPFileList) { _listOfZIPArchivesWithXMLFiles.Add(s); } } } logXContainer _ret = new logXContainer() { PersonIdentifierIsNumber = _personIdentifierIsNumber, PersonIdentifierName = _personIdentifier }; _ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary); if (ParsedCommandLineArguments.Transform_ConcordanceTable.Trim() != "") { if (File.Exists(ParsedCommandLineArguments.Transform_ConcordanceTable)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Read Concordance Table."); } _ret.ReadConcordanceTable(ParsedCommandLineArguments.Transform_ConcordanceTable); } } foreach (string zfilename in _listOfZIPArchivesWithXMLFiles) { using (ZipFile zip = ZipFile.Read(zfilename)) { foreach (var entry in zip) { if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { break; } // TODO: Check FitsMask if (1 == 1 || CommandLineArguments.FitsMask(entry.FileName, ParsedCommandLineArguments.Mask)) { if (ParsedCommandLineArguments.Verbose) { Console.Write("Info: Read File '" + entry.FileName + "' "); } using (MemoryStream zipStream = new MemoryStream()) { entry.ExtractWithPassword(zipStream, ""); zipStream.Position = 0; try { StreamReader sr = new StreamReader(zipStream); string _fileContentAsString = sr.ReadToEnd(); if (_fileContentAsString.Trim().Length > 0) { ReadLogDataEEFromXMLString(_fileContentAsString, _ret); } } catch (Exception _ex) { Console.WriteLine("Error processing file '" + entry.FileName + "': " + _ex.Message); return; } } Console.WriteLine("ok."); } } } if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Info: Max number of cases reached."); } break; } } foreach (string xfilename in _listOfXMLFiles) { if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Info: Max number of cases reached."); } break; } if (1 == 1 || CommandLineArguments.FitsMask(Path.GetFileName(xfilename), ParsedCommandLineArguments.Mask)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Info: Read File '" + Path.GetFileName(xfilename) + "' "); } try { StreamReader sr = new StreamReader(xfilename); string _fileContentAsString = sr.ReadToEnd(); if (_fileContentAsString.Trim().Length > 0) { ReadLogDataEEFromXMLString(_fileContentAsString, _ret); } } catch (Exception _ex) { Console.WriteLine("Error processing file '" + xfilename + "': " + _ex.Message); return; } Console.WriteLine("ok."); } } _ret.UpdateRelativeTimes(); _ret.CreateLookup(); if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with Stata file(s)."); } _ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language); } if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create XLSX file."); } _ret.ExportXLSX(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with CSV file(s)."); } _ret.ExportCSV(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create Codebook File."); } _ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language); } if (ParsedCommandLineArguments.Transform_ConcordanceTable.Trim() != "") { if (!File.Exists(ParsedCommandLineArguments.Transform_ConcordanceTable)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create Concordance Table."); } _ret.CreateConcordanceTable(ParsedCommandLineArguments.Transform_ConcordanceTable); } } if (_ret.ExportErrors.Count > 0) { Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files."); if (ParsedCommandLineArguments.Verbose) { for (int i = 0; i < _ret.ExportErrors.Count; i++) { Console.WriteLine(_ret.ExportErrors[i]); } } } } catch (Exception _ex) { Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.Message.ToString()); } }
public static void ProcessLogFilesOnly(Stopwatch Watch, CommandLineArguments ParsedCommandLineArguments) { try { bool _personIdentifierIsNumber = false; if (ParsedCommandLineArguments.Flags.Contains("NUMERICPERSONIDENTIFIER")) { _personIdentifierIsNumber = true; } string _personIdentifier = "PersonIdentifier"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("personidentifier")) { _personIdentifier = ParsedCommandLineArguments.ParameterDictionary["personidentifier"]; } string _language = "ENG"; if (ParsedCommandLineArguments.ParameterDictionary.ContainsKey("language")) { _language = ParsedCommandLineArguments.ParameterDictionary["language"]; } DateTime dt1970 = new DateTime(1970, 1, 1, 0, 0, 0, 0); XmlSerializer logSerializer = new XmlSerializer(typeof(log)); #region Search Source Files List <string> _listOfXMLFiles = new List <string>(); List <string> _listOfZIPArchivesWithXMLFiles = new List <string>(); foreach (string inFolder in ParsedCommandLineArguments.Transform_InputFolders) { // Input is file if (File.Exists(inFolder)) { if (inFolder.ToLower().EndsWith(".zip")) { // Single ZIP file _listOfZIPArchivesWithXMLFiles.Add(inFolder); } else if (inFolder.ToLower().EndsWith(".xml")) { // Single XML file _listOfXMLFiles.Add(inFolder); } } else { if (!Directory.Exists(inFolder)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Warning: Directory not exists: '" + inFolder + "'."); } continue; } var _tmpXMLFileList = Directory.GetFiles(inFolder, "*.xml", SearchOption.AllDirectories); foreach (string s in _tmpXMLFileList) { _listOfXMLFiles.Add(s); } var _tmpZIPFileList = Directory.GetFiles(inFolder, "*.zip", SearchOption.AllDirectories); foreach (string s in _tmpZIPFileList) { _listOfZIPArchivesWithXMLFiles.Add(s); } } } #endregion logXContainer _ret = new logXContainer() { PersonIdentifierIsNumber = _personIdentifierIsNumber, PersonIdentifierName = _personIdentifier }; _ret.LoadCodebookDictionary(ParsedCommandLineArguments.Transform_Dictionary); foreach (string xfilename in _listOfXMLFiles) { if (xfilename.EndsWith("-log.xml")) { StreamReader _sr = new StreamReader(xfilename); var _xml = CleanInvalidXmlChars(_sr.ReadToEnd()); string _PersonIdentifier = Path.GetFileName(xfilename).Replace("-log.xml", ""); processPISA_BQ_single_XML(ParsedCommandLineArguments.Elements, dt1970, _ret, logSerializer, _PersonIdentifier, _xml); _sr.Close(); } } foreach (string zfilename in _listOfZIPArchivesWithXMLFiles) { try { using (var outerInputZipFile = ZipFile.Read(zfilename)) { foreach (var outerInputZipEntry in outerInputZipFile.Entries) { if (outerInputZipEntry.FileName.EndsWith("-log.xml") && outerInputZipEntry.UncompressedSize != 0) { #region Single XML file string _PersonIdentifier = Path.GetFileName(outerInputZipEntry.FileName).Replace("-log.xml", ""); if (ParsedCommandLineArguments.Verbose) { Console.WriteLine(_PersonIdentifier + " -- " + _ret.GetNumberOfPersons); } using (MemoryStream innerZIPEntryMemoryStream = new MemoryStream()) { outerInputZipEntry.Password = ParsedCommandLineArguments.ZIPPassword; outerInputZipEntry.Extract(innerZIPEntryMemoryStream); innerZIPEntryMemoryStream.Position = 0; var _sr = new StreamReader(innerZIPEntryMemoryStream); var _xml = CleanInvalidXmlChars(_sr.ReadToEnd()); processPISA_BQ_single_XML(ParsedCommandLineArguments.Elements, dt1970, _ret, logSerializer, _PersonIdentifier, _xml); _sr.Close(); } #endregion } else if (outerInputZipEntry.FileName.EndsWith("Session2.zip") && outerInputZipEntry.UncompressedSize != 0) { #region ZIP archive with XML files string _PersonIdentifier = Path.GetFileName(outerInputZipEntry.FileName).Replace("-Session2.zip", ""); using (MemoryStream outerZIPEntryMemoryStream = new MemoryStream()) { if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { break; } if (ParsedCommandLineArguments.Verbose) { Console.WriteLine(_PersonIdentifier + " -- " + _ret.GetNumberOfPersons); } outerInputZipEntry.Password = ParsedCommandLineArguments.ZIPPassword; outerInputZipEntry.Extract(outerZIPEntryMemoryStream); outerZIPEntryMemoryStream.Position = 0; using (var innerZIP = ZipFile.Read(outerZIPEntryMemoryStream)) { foreach (var innerZIPEntry in innerZIP.Entries) { if (innerZIPEntry.FileName.EndsWith("_Data.zip") && innerZIPEntry.UncompressedSize != 0) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine(innerZIPEntry.FileName); } using (MemoryStream innerZIPEntryMemoryStream = new MemoryStream()) { innerZIPEntry.Password = ParsedCommandLineArguments.ZIPPassword; innerZIPEntry.Extract(innerZIPEntryMemoryStream); innerZIPEntryMemoryStream.Position = 0; using (var inner2Zip = ZipFile.Read(innerZIPEntryMemoryStream)) { foreach (var inner2ZIPEntry in inner2Zip.Entries) { if (inner2ZIPEntry.FileName.EndsWith("-log.xml") && inner2ZIPEntry.UncompressedSize != 0) { using (MemoryStream inner2ZIPEntryMemoryStream = new MemoryStream()) { //inner2ZIPEntry.Password = ParsedCommandLineArguments.ZIPPassword; inner2ZIPEntry.Extract(inner2ZIPEntryMemoryStream); inner2ZIPEntryMemoryStream.Position = 0; var _sr = new StreamReader(inner2ZIPEntryMemoryStream); var _xml = CleanInvalidXmlChars(_sr.ReadToEnd()); processPISA_BQ_single_XML(ParsedCommandLineArguments.Elements, dt1970, _ret, logSerializer, _PersonIdentifier, _xml); } } } } } } } } } #endregion } } } } catch (Exception _ex) { _ret.ExportErrors.Add("Error reading file '" + zfilename + "': " + _ex.Message); } } if (ParsedCommandLineArguments.MaxNumberOfCases > 0 && _ret.GetNumberOfPersons >= ParsedCommandLineArguments.MaxNumberOfCases) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Info: Max number of cases reached."); } } #region Export Universal Log Format _ret.UpdateRelativeTimes(); _ret.CreateLookup(); if (ParsedCommandLineArguments.Transform_OutputStata.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with Stata file(s)."); } _ret.ExportStata(ParsedCommandLineArguments.Transform_OutputStata, _language); } if (ParsedCommandLineArguments.Transform_OutputXLSX.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create XLSX file."); } _ret.ExportXLSX(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_OutputZCSV.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create ZIP archive with CSV file(s)."); } _ret.ExportCSV(ParsedCommandLineArguments); } if (ParsedCommandLineArguments.Transform_Codebook.Trim() != "") { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create Codebook File."); } _ret.CreateCodebook(ParsedCommandLineArguments.Transform_Codebook, _language); } if (ParsedCommandLineArguments.Transform_ConcordanceTable.Trim() != "") { if (!File.Exists(ParsedCommandLineArguments.Transform_ConcordanceTable)) { if (ParsedCommandLineArguments.Verbose) { Console.WriteLine("Create Concordance Table."); } _ret.CreateConcordanceTable(ParsedCommandLineArguments.Transform_ConcordanceTable); } } if (_ret.ExportErrors.Count > 0) { Console.WriteLine(_ret.ExportErrors.Count + " error(s) creating output files."); if (ParsedCommandLineArguments.Verbose) { for (int i = 0; i < _ret.ExportErrors.Count; i++) { Console.WriteLine(_ret.ExportErrors[i]); } } } #endregion } catch (Exception _ex) { Console.WriteLine("Error transforming log data. Details: " + Environment.NewLine + _ex.Message.ToString()); } }