public void ParseDataSourceDetails(string xmlInputData, ref int dataSourceId, ref string dataSourceName,
ref List <string> validationErrors, ref List <string> validationWarnings)
{
if (string.IsNullOrEmpty(xmlInputData))
{
return;
}
ValidateXML(xmlInputData);
validationErrors = _XMLValidationErrors;
validationWarnings = _XMLValidationWarnings;
//xml might be invalid, still lets try if we can get datasource id or name
try
{
MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(xmlInputData));
XPathDocument docNav = new XPathDocument(ms);
XPathNavigator nav = docNav.CreateNavigator();
XPathNodeIterator nodes = nav.Select("/SreRequest/DataSource/Id"); nodes.MoveNext();
int.TryParse(nodes.Current.Value, out dataSourceId);
nodes = nav.Select("/SreRequest/DataSource/Name"); nodes.MoveNext();
dataSourceName = nodes.Current.Value;
}
catch (Exception ex)
{
string errorId = Guid.NewGuid().ToString();
string errorMessage = errorId + ex.ToString() + (ex.InnerException == null ? string.Empty : ex.InnerException.Message);
DataSource.TraceError(errorMessage);
validationErrors.Add(string.Format(__validationErrorMsg, errorId));
}
}
void JobProcessor_Elapsed(object sender, ElapsedEventArgs e)
{
try
{
if (DataSourceIsDisabled())
{
return;
}
if (DataSourceIsTemporarilyStopped())
{
return;
}
if (!this.IsRunning)
{
this.IsRunning = true;
DataSource.TraceInformation(" - Starting");
Job = JobProcessor.Instantiate(DataSource.Id, DataSource.Name, ProcessingBy);
Job.IsTriggeredBySqlPuller = true;
this.DataSource.BusinessRules.Execute(Job, null, RuleSetTypes.SqlPullInit);
DataTable table = ExecuteQuery();
InvokeAfterExecutingSelect(new SqlWatcherEventArgs(table));
if (table.Rows.Count == 0)
{
Job.TraceInformation(" - No records found.");
lock (_lock)
{
if (Registry.Instance.Entries.ContainsKey(Job.JobIdentifier))
{
Job job = Registry.Instance.Entries[Job.JobIdentifier] as Job;
Registry.Instance.Entries.Remove(Job.JobIdentifier);
job.Dispose();
job = null;
}
}
}
else
{
if (this.ReturnType == "I")
{
DataSource.TraceInformation("{0} - found {1} records. Processing with interface {2}", DataSource.Name, table.Rows.Count, InterfaceName);
string fileName = GenerateFile(table);
InvokeFileDownloaded(new FileSystemWatcherEventArgs(DataSourceParameters, fileName));
DataSource.TraceInformation("{0} - records processed with interface.", DataSource.Name);
}
else
{
DataSource.TraceInformation("{0} - found {1} records. Processing...", DataSource.Name, table.Rows.Count);
Job.Feed(table);
JobProcessor jobProcessor = new JobProcessor();
StringBuilder result = jobProcessor.ProcessJob(Job);
if (Job.IsErrored)
{
new SqlWatcherHelper(Job).ExecuteRecoveryScript();
}
string outputFolder = DataSource.GetOutputFolder(DataSource.Id, DataSource.Keys);
string actualOutputFolder = outputFolder;
string outputFileName = DataSource.GetOutputFileName(DataSource.Id, DataSource.Keys, outputFolder, DataSource.Id.ToString());
if (File.Exists(outputFileName))
{
string buName = Path.Combine(outputFolder, string.Format("{0}{1}_{2}", DataSource.Name, Guid.NewGuid().ToString(), Path.GetFileName(outputFileName)));
new FileUtility().FileCopy(outputFileName, buName, true);
}
if (result.Length > 0)
{
using (StreamWriter tw = new StreamWriter(outputFileName))
{
tw.Write(result);
tw.Close();
}
DataSource.TraceInformation("{0} - A data set from {1} was successfully processed. Output file was {2}", DataSource.Name, DataSource.Name, outputFileName);
Registry.Instance.Pullers.InvokeFileProcessed(DataSource.Id, jobProcessor.JobId, DataSource.Keys, outputFileName, outputFolder, string.Empty);
}
else
{
DataSource.TraceInformation("{0} - Failed to process the SQL data set from '{1}', empty data came from output writer! Check log for more details.", DataSource.Name, DataSource.Name);
}
jobProcessor.Dispose();
}
}
this.IsRunning = false;
timer.Enabled = true;
}
}
catch (Exception exp)
{
this.IsRunning = false;
DataSource.TraceError(exp.ToString());
timer.Enabled = true;
}
Trace.Flush();
}
public DataTable ParseData(string xmlInputData, ref List <string> csvRows,
ref List <string> validationErrors, ref List <string> validationWarnings)
{
DataTable dataTable = new DataTable("SRERequest");
if (string.IsNullOrEmpty(xmlInputData))
{
return(dataTable);
}
ValidateXML(xmlInputData);
validationErrors = _XMLValidationErrors;
validationWarnings = _XMLValidationWarnings;
if (validationErrors.Count > 0)
{
return(dataTable);
}
try
{
MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(xmlInputData));
XPathDocument docNav = new XPathDocument(ms);
XPathNavigator nav = docNav.CreateNavigator();
XPathNodeIterator nodes = nav.Select("/SRERequest/Rows/Row");
int rowCounter = 0;
while (nodes.MoveNext())
{
#region Attribute Traverse
XPathNodeIterator attributeNodes = nodes.Current.SelectDescendants(XPathNodeType.Element, false);
List <object> oneRow = new List <object>();
foreach (XPathNavigator attributeNode in attributeNodes)
{
if (!attributeNode.LocalName.Equals("Attribute", StringComparison.OrdinalIgnoreCase))
{
continue;
}
XPathNodeIterator nodeAVPairs = attributeNode.SelectDescendants(XPathNodeType.Element, false);
string columnName = string.Empty;
foreach (XPathNavigator nodeAVPair in nodeAVPairs)
{
if (nodeAVPair.LocalName.Equals("Name", StringComparison.OrdinalIgnoreCase))
{
columnName = nodeAVPair.Value;
}
else if (nodeAVPair.LocalName.Equals("Value", StringComparison.OrdinalIgnoreCase))
{
oneRow.Add(nodeAVPair.Value);
}
}
if (rowCounter == 0)
{
dataTable.Columns.Add(columnName);
}
}
#endregion Attribute Traverse
#region CSVRows
string csvRow = string.Empty;
if (rowCounter == 0)
{
foreach (DataColumn column in dataTable.Columns)
{
csvRow += "\"" + column.ColumnName + "\",";
}
csvRows.Add(csvRow);
csvRow = string.Empty;
}
foreach (object cell in oneRow)
{
csvRow += "\"" + cell.ToString() + "\",";
}
csvRows.Add(csvRow);
#endregion CSVRows
dataTable.Rows.Add(oneRow.ToArray());
rowCounter++;
}
}
catch (Exception ex)
{
string errorId = Guid.NewGuid().ToString();
string errorMessage = errorId + ex.ToString() + (ex.InnerException == null ? string.Empty : ex.InnerException.Message);
DataSource.TraceError(errorMessage);
validationErrors.Add(string.Format(__validationErrorMsg, errorId));
}
return(dataTable);
}
private DataTable ReadCSVFileUsingVB(ref List <string> csvRows, ref List <string> emptyRowWarnings, ref int columnCount)
{
string localFileName = Job.FileName;
if (Job.IsHavingSpecialHeaderAndOrFooter)
{
localFileName = Job.FileNameWithoutHeaderAndOrFooter;
}
if (!File.Exists(localFileName))
{
return(new DataTable());
}
TextFieldParser parser = new TextFieldParser(localFileName);
parser.TextFieldType = FieldType.Delimited;
parser.SetDelimiters(string.IsNullOrEmpty(DataSource.Delimiter) ? "," : DataSource.Delimiter);
bool columnInformationTaken = false;
DataTable table = new DataTable();
bool failed = false;
while (!parser.EndOfData)
{
string[] fields = parser.ReadFields();
if (!columnInformationTaken)
{
if (DataSource.IsFirstRowHeader)
{
foreach (string field in fields)
{
table.Columns.Add(field);
}
columnCount = table.Columns.Count;
}
else
{
foreach (IdpeAttribute attribute in DataSource.AcceptableAttributes)
{
table.Columns.Add(attribute.Name);
}
if (table.Columns.Count != fields.Length)
{
DataSource.TraceError("1.Data source attribute count and data fed is not equal, could not parse data! Acceptable attributes = {0}, input file columns = {1}",
table.Columns.Count, fields.Length);
failed = true;
break;
}
table.Rows.Add(fields);
columnCount = fields.Length;
}
columnInformationTaken = true;
}
else
{
if (table.Columns.Count != fields.Length)
{
DataSource.TraceError("2.Data source attribute count and data fed is not equal, could not parse data! Acceptable attributes = {0}, input file columns = {1}",
table.Columns.Count, fields.Length);
failed = true;
break;
}
table.Rows.Add(fields);
}
string commaSeparatedList = fields.Aggregate((a, x) => a + "\",\"" + x).ToString();
csvRows.Add("\"" + commaSeparatedList + "\"");
}
parser.Close();
if (failed)
{
parser.Dispose();
parser = null;
table.Dispose();
table = null;
return(new DataTable());
}
else
{
return(table);
}
}
