public void ParseDataSourceDetails(string xmlInputData, ref int dataSourceId, ref string dataSourceName, ref List <string> validationErrors, ref List <string> validationWarnings) { if (string.IsNullOrEmpty(xmlInputData)) { return; } ValidateXML(xmlInputData); validationErrors = _XMLValidationErrors; validationWarnings = _XMLValidationWarnings; //xml might be invalid, still lets try if we can get datasource id or name try { MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(xmlInputData)); XPathDocument docNav = new XPathDocument(ms); XPathNavigator nav = docNav.CreateNavigator(); XPathNodeIterator nodes = nav.Select("/SreRequest/DataSource/Id"); nodes.MoveNext(); int.TryParse(nodes.Current.Value, out dataSourceId); nodes = nav.Select("/SreRequest/DataSource/Name"); nodes.MoveNext(); dataSourceName = nodes.Current.Value; } catch (Exception ex) { string errorId = Guid.NewGuid().ToString(); string errorMessage = errorId + ex.ToString() + (ex.InnerException == null ? string.Empty : ex.InnerException.Message); DataSource.TraceError(errorMessage); validationErrors.Add(string.Format(__validationErrorMsg, errorId)); } }
void JobProcessor_Elapsed(object sender, ElapsedEventArgs e) { try { if (DataSourceIsDisabled()) { return; } if (DataSourceIsTemporarilyStopped()) { return; } if (!this.IsRunning) { this.IsRunning = true; DataSource.TraceInformation(" - Starting"); Job = JobProcessor.Instantiate(DataSource.Id, DataSource.Name, ProcessingBy); Job.IsTriggeredBySqlPuller = true; this.DataSource.BusinessRules.Execute(Job, null, RuleSetTypes.SqlPullInit); DataTable table = ExecuteQuery(); InvokeAfterExecutingSelect(new SqlWatcherEventArgs(table)); if (table.Rows.Count == 0) { Job.TraceInformation(" - No records found."); lock (_lock) { if (Registry.Instance.Entries.ContainsKey(Job.JobIdentifier)) { Job job = Registry.Instance.Entries[Job.JobIdentifier] as Job; Registry.Instance.Entries.Remove(Job.JobIdentifier); job.Dispose(); job = null; } } } else { if (this.ReturnType == "I") { DataSource.TraceInformation("{0} - found {1} records. Processing with interface {2}", DataSource.Name, table.Rows.Count, InterfaceName); string fileName = GenerateFile(table); InvokeFileDownloaded(new FileSystemWatcherEventArgs(DataSourceParameters, fileName)); DataSource.TraceInformation("{0} - records processed with interface.", DataSource.Name); } else { DataSource.TraceInformation("{0} - found {1} records. Processing...", DataSource.Name, table.Rows.Count); Job.Feed(table); JobProcessor jobProcessor = new JobProcessor(); StringBuilder result = jobProcessor.ProcessJob(Job); if (Job.IsErrored) { new SqlWatcherHelper(Job).ExecuteRecoveryScript(); } string outputFolder = DataSource.GetOutputFolder(DataSource.Id, DataSource.Keys); string actualOutputFolder = outputFolder; string outputFileName = DataSource.GetOutputFileName(DataSource.Id, DataSource.Keys, outputFolder, DataSource.Id.ToString()); if (File.Exists(outputFileName)) { string buName = Path.Combine(outputFolder, string.Format("{0}{1}_{2}", DataSource.Name, Guid.NewGuid().ToString(), Path.GetFileName(outputFileName))); new FileUtility().FileCopy(outputFileName, buName, true); } if (result.Length > 0) { using (StreamWriter tw = new StreamWriter(outputFileName)) { tw.Write(result); tw.Close(); } DataSource.TraceInformation("{0} - A data set from {1} was successfully processed. Output file was {2}", DataSource.Name, DataSource.Name, outputFileName); Registry.Instance.Pullers.InvokeFileProcessed(DataSource.Id, jobProcessor.JobId, DataSource.Keys, outputFileName, outputFolder, string.Empty); } else { DataSource.TraceInformation("{0} - Failed to process the SQL data set from '{1}', empty data came from output writer! Check log for more details.", DataSource.Name, DataSource.Name); } jobProcessor.Dispose(); } } this.IsRunning = false; timer.Enabled = true; } } catch (Exception exp) { this.IsRunning = false; DataSource.TraceError(exp.ToString()); timer.Enabled = true; } Trace.Flush(); }
public DataTable ParseData(string xmlInputData, ref List <string> csvRows, ref List <string> validationErrors, ref List <string> validationWarnings) { DataTable dataTable = new DataTable("SRERequest"); if (string.IsNullOrEmpty(xmlInputData)) { return(dataTable); } ValidateXML(xmlInputData); validationErrors = _XMLValidationErrors; validationWarnings = _XMLValidationWarnings; if (validationErrors.Count > 0) { return(dataTable); } try { MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(xmlInputData)); XPathDocument docNav = new XPathDocument(ms); XPathNavigator nav = docNav.CreateNavigator(); XPathNodeIterator nodes = nav.Select("/SRERequest/Rows/Row"); int rowCounter = 0; while (nodes.MoveNext()) { #region Attribute Traverse XPathNodeIterator attributeNodes = nodes.Current.SelectDescendants(XPathNodeType.Element, false); List <object> oneRow = new List <object>(); foreach (XPathNavigator attributeNode in attributeNodes) { if (!attributeNode.LocalName.Equals("Attribute", StringComparison.OrdinalIgnoreCase)) { continue; } XPathNodeIterator nodeAVPairs = attributeNode.SelectDescendants(XPathNodeType.Element, false); string columnName = string.Empty; foreach (XPathNavigator nodeAVPair in nodeAVPairs) { if (nodeAVPair.LocalName.Equals("Name", StringComparison.OrdinalIgnoreCase)) { columnName = nodeAVPair.Value; } else if (nodeAVPair.LocalName.Equals("Value", StringComparison.OrdinalIgnoreCase)) { oneRow.Add(nodeAVPair.Value); } } if (rowCounter == 0) { dataTable.Columns.Add(columnName); } } #endregion Attribute Traverse #region CSVRows string csvRow = string.Empty; if (rowCounter == 0) { foreach (DataColumn column in dataTable.Columns) { csvRow += "\"" + column.ColumnName + "\","; } csvRows.Add(csvRow); csvRow = string.Empty; } foreach (object cell in oneRow) { csvRow += "\"" + cell.ToString() + "\","; } csvRows.Add(csvRow); #endregion CSVRows dataTable.Rows.Add(oneRow.ToArray()); rowCounter++; } } catch (Exception ex) { string errorId = Guid.NewGuid().ToString(); string errorMessage = errorId + ex.ToString() + (ex.InnerException == null ? string.Empty : ex.InnerException.Message); DataSource.TraceError(errorMessage); validationErrors.Add(string.Format(__validationErrorMsg, errorId)); } return(dataTable); }
private DataTable ReadCSVFileUsingVB(ref List <string> csvRows, ref List <string> emptyRowWarnings, ref int columnCount) { string localFileName = Job.FileName; if (Job.IsHavingSpecialHeaderAndOrFooter) { localFileName = Job.FileNameWithoutHeaderAndOrFooter; } if (!File.Exists(localFileName)) { return(new DataTable()); } TextFieldParser parser = new TextFieldParser(localFileName); parser.TextFieldType = FieldType.Delimited; parser.SetDelimiters(string.IsNullOrEmpty(DataSource.Delimiter) ? "," : DataSource.Delimiter); bool columnInformationTaken = false; DataTable table = new DataTable(); bool failed = false; while (!parser.EndOfData) { string[] fields = parser.ReadFields(); if (!columnInformationTaken) { if (DataSource.IsFirstRowHeader) { foreach (string field in fields) { table.Columns.Add(field); } columnCount = table.Columns.Count; } else { foreach (IdpeAttribute attribute in DataSource.AcceptableAttributes) { table.Columns.Add(attribute.Name); } if (table.Columns.Count != fields.Length) { DataSource.TraceError("1.Data source attribute count and data fed is not equal, could not parse data! Acceptable attributes = {0}, input file columns = {1}", table.Columns.Count, fields.Length); failed = true; break; } table.Rows.Add(fields); columnCount = fields.Length; } columnInformationTaken = true; } else { if (table.Columns.Count != fields.Length) { DataSource.TraceError("2.Data source attribute count and data fed is not equal, could not parse data! Acceptable attributes = {0}, input file columns = {1}", table.Columns.Count, fields.Length); failed = true; break; } table.Rows.Add(fields); } string commaSeparatedList = fields.Aggregate((a, x) => a + "\",\"" + x).ToString(); csvRows.Add("\"" + commaSeparatedList + "\""); } parser.Close(); if (failed) { parser.Dispose(); parser = null; table.Dispose(); table = null; return(new DataTable()); } else { return(table); } }