public void ParseXmlTests()
        {
            try
            {
                var parser = new DataParser();
                parser.LoadParserSettings("Sample/parser_settings.json");

                var rows          = GetSampleOrders();
                var failedRecords = new Dictionary <string, string>();

                var allData = new List <ParsedData>();

                foreach (var row in rows)
                {
                    var data = parser.ParseXML(row.OrderXML, out List <string> errors, false);
                    if (data != null)
                    {
                        allData.AddRange(data);
                    }
                    else
                    {
                        failedRecords.Add(row.OrderNumber, string.Join(",", errors));
                    }
                }

                Assert.IsTrue(rows.Count > 0);
                Assert.AreEqual(0, failedRecords.Count);

                // Create the test folder
                FileHelper.CreateDirectory("temp");

                // Flatten the data
                var flattenData = allData.GroupBy(x => x.Name)
                                  .Select(group => new ParsedData
                {
                    Name = group.Key,
                    Data = group.SelectMany(item => item.Data).Distinct().ToList()
                }).ToList();

                foreach (var item in flattenData)
                {
                    string path = "temp/ParseXmlTests_" + item.Name + ".csv";

                    // Delete previous test file, if any
                    FileHelper.DeleteFile(path);

                    parser.ExportAsCsv(item.Data, path);
                }
            }
            catch (Exception ex)
            {
                Assert.Fail(ex.ToString());
            }
        }
Example #2
0
        static void RunProgram()
        {
            string jobId = DateTime.Now.ToString("yyyyMMddhhmmss");

            try
            {
                Log.Debug("Processing in progress...");

                // Basic validation for sftp config
                if (string.IsNullOrEmpty(AppSettings.SftpHost) ||
                    string.IsNullOrEmpty(AppSettings.SftpUsername) ||
                    string.IsNullOrEmpty(AppSettings.SftpPassword) ||
                    string.IsNullOrEmpty(AppSettings.SftpDownloadDirectory) ||
                    string.IsNullOrEmpty(AppSettings.SftpDownloadFilename) ||
                    string.IsNullOrEmpty(AppSettings.SftpUploadDirectory))
                {
                    Log.Error("Missing SFTP Configuration.");
                    SendSystemEmail(Constants.StatusEmailSubject + " - Configuration Error", "Missing SFTP Configuration.");
                    return;
                }

                if (!SftpHelper.VerifyConnection(out string error))
                {
                    Log.Error("Unable to connect to SFTP, " + error);
                    SendSystemEmail(Constants.StatusEmailSubject + " - Failed to connect to SFTP", "Please ensure the SFTP Configuration is valid.<h3>Exception</h3>" + error);
                    return;
                }

                string downloadFolderPath = Path.Combine(AppSettings.DownloadPath, jobId);
                string workingFolderPath  = Path.Combine(AppSettings.WorkingPath, jobId);

                var parser = new DataParser();
                parser.LoadParserSettings();

                var allData    = new List <ParsedData>();
                var failedRows = new List <Tuple <int, string, string> >();

                // Download from sftp remote server
                DownloadedFile downloadedFile = SftpHelper.DownloadFile(AppSettings.SftpDownloadDirectory, AppSettings.SftpDownloadFilename, downloadFolderPath);

                // Terminate if no files available for download
                if (downloadedFile == null)
                {
                    Log.Error("No files available for download");
                    SendSystemEmail(Constants.StatusEmailSubject, "No files available for download.");
                    FileHelper.DeleteFile(downloadFolderPath);
                    return;
                }

                //Read from downloaded csv and parse into order list
                var rows = CsvUtil.GetRecords <InOrderConfirmation>(downloadedFile.FullName);

                var rowIndex = 1; //csv header consider as 1 row
                foreach (var row in rows)
                {
                    rowIndex++;
                    var data = parser.ParseXML(row.OrderXML, out List <string> errors, enableThrowException: false);
                    if (data != null && !errors.Any())
                    {
                        allData.AddRange(data);
                    }
                    else
                    {
                        failedRows.Add(new Tuple <int, string, string>(rowIndex, row.OrderNumber, string.Join(",", errors)));
                    }
                }

                // Terminate if any invalid rows
                if (failedRows.Any())
                {
                    Log.Error("Invalid Records Found");
                    SendParseErrorEmail(jobId, failedRows);
                    ArchiveFiles(jobId, true);
                    return;
                }

                // Process Export and Upload
                var results = new Dictionary <string, UploadResult>();

                if (allData.Any())
                {
                    FileHelper.CreateDirectory(workingFolderPath);

                    // Flatten the data
                    var flattenData = allData.GroupBy(x => x.Name)
                                      .Select(group => new ParsedData
                    {
                        Name = group.Key,
                        Data = group.SelectMany(item => item.Data).Distinct().ToList()
                    }).ToList();

                    // Export csv and upload to SFTP
                    foreach (var item in flattenData)
                    {
                        string filename = item.Name + "_" + jobId + ".csv";
                        string path     = workingFolderPath + " /" + filename;

                        try
                        {
                            parser.ExportAsCsv(item.Data, path);

                            var result = UploadExportCsvToSftp(AppSettings.SftpUploadDirectory, path);
                            results.Add(filename, result);
                        }
                        catch (Exception ex)
                        {
                            results.Add(filename, new UploadResult
                            {
                                Success      = false,
                                ErrorMessage = "Failed to export csv for upload. Exception: " + ex.ToString()
                            });
                        }
                    }
                }

                // Send status result email
                SendStatusEmail(jobId, results);
            }
            catch (Exception ex)
            {
                Log.Error("RunProgram Error", ex);
                SendSystemEmail(Constants.StatusEmailSubject + " - Task Error", ex.ToString());
            }
            finally
            {
                ArchiveFiles(jobId, true);
            }
        }