public void ParseXmlTests()
{
try
{
var parser = new DataParser();
parser.LoadParserSettings("Sample/parser_settings.json");
var rows = GetSampleOrders();
var failedRecords = new Dictionary <string, string>();
var allData = new List <ParsedData>();
foreach (var row in rows)
{
var data = parser.ParseXML(row.OrderXML, out List <string> errors, false);
if (data != null)
{
allData.AddRange(data);
}
else
{
failedRecords.Add(row.OrderNumber, string.Join(",", errors));
}
}
Assert.IsTrue(rows.Count > 0);
Assert.AreEqual(0, failedRecords.Count);
// Create the test folder
FileHelper.CreateDirectory("temp");
// Flatten the data
var flattenData = allData.GroupBy(x => x.Name)
.Select(group => new ParsedData
{
Name = group.Key,
Data = group.SelectMany(item => item.Data).Distinct().ToList()
}).ToList();
foreach (var item in flattenData)
{
string path = "temp/ParseXmlTests_" + item.Name + ".csv";
// Delete previous test file, if any
FileHelper.DeleteFile(path);
parser.ExportAsCsv(item.Data, path);
}
}
catch (Exception ex)
{
Assert.Fail(ex.ToString());
}
}
static void RunProgram()
{
string jobId = DateTime.Now.ToString("yyyyMMddhhmmss");
try
{
Log.Debug("Processing in progress...");
// Basic validation for sftp config
if (string.IsNullOrEmpty(AppSettings.SftpHost) ||
string.IsNullOrEmpty(AppSettings.SftpUsername) ||
string.IsNullOrEmpty(AppSettings.SftpPassword) ||
string.IsNullOrEmpty(AppSettings.SftpDownloadDirectory) ||
string.IsNullOrEmpty(AppSettings.SftpDownloadFilename) ||
string.IsNullOrEmpty(AppSettings.SftpUploadDirectory))
{
Log.Error("Missing SFTP Configuration.");
SendSystemEmail(Constants.StatusEmailSubject + " - Configuration Error", "Missing SFTP Configuration.");
return;
}
if (!SftpHelper.VerifyConnection(out string error))
{
Log.Error("Unable to connect to SFTP, " + error);
SendSystemEmail(Constants.StatusEmailSubject + " - Failed to connect to SFTP", "Please ensure the SFTP Configuration is valid.<h3>Exception</h3>" + error);
return;
}
string downloadFolderPath = Path.Combine(AppSettings.DownloadPath, jobId);
string workingFolderPath = Path.Combine(AppSettings.WorkingPath, jobId);
var parser = new DataParser();
parser.LoadParserSettings();
var allData = new List <ParsedData>();
var failedRows = new List <Tuple <int, string, string> >();
// Download from sftp remote server
DownloadedFile downloadedFile = SftpHelper.DownloadFile(AppSettings.SftpDownloadDirectory, AppSettings.SftpDownloadFilename, downloadFolderPath);
// Terminate if no files available for download
if (downloadedFile == null)
{
Log.Error("No files available for download");
SendSystemEmail(Constants.StatusEmailSubject, "No files available for download.");
FileHelper.DeleteFile(downloadFolderPath);
return;
}
//Read from downloaded csv and parse into order list
var rows = CsvUtil.GetRecords <InOrderConfirmation>(downloadedFile.FullName);
var rowIndex = 1; //csv header consider as 1 row
foreach (var row in rows)
{
rowIndex++;
var data = parser.ParseXML(row.OrderXML, out List <string> errors, enableThrowException: false);
if (data != null && !errors.Any())
{
allData.AddRange(data);
}
else
{
failedRows.Add(new Tuple <int, string, string>(rowIndex, row.OrderNumber, string.Join(",", errors)));
}
}
// Terminate if any invalid rows
if (failedRows.Any())
{
Log.Error("Invalid Records Found");
SendParseErrorEmail(jobId, failedRows);
ArchiveFiles(jobId, true);
return;
}
// Process Export and Upload
var results = new Dictionary <string, UploadResult>();
if (allData.Any())
{
FileHelper.CreateDirectory(workingFolderPath);
// Flatten the data
var flattenData = allData.GroupBy(x => x.Name)
.Select(group => new ParsedData
{
Name = group.Key,
Data = group.SelectMany(item => item.Data).Distinct().ToList()
}).ToList();
// Export csv and upload to SFTP
foreach (var item in flattenData)
{
string filename = item.Name + "_" + jobId + ".csv";
string path = workingFolderPath + " /" + filename;
try
{
parser.ExportAsCsv(item.Data, path);
var result = UploadExportCsvToSftp(AppSettings.SftpUploadDirectory, path);
results.Add(filename, result);
}
catch (Exception ex)
{
results.Add(filename, new UploadResult
{
Success = false,
ErrorMessage = "Failed to export csv for upload. Exception: " + ex.ToString()
});
}
}
}
// Send status result email
SendStatusEmail(jobId, results);
}
catch (Exception ex)
{
Log.Error("RunProgram Error", ex);
SendSystemEmail(Constants.StatusEmailSubject + " - Task Error", ex.ToString());
}
finally
{
ArchiveFiles(jobId, true);
}
}