/// <summary>
/// Cycle through the results and copy valid and non valid rows to separate tables.
/// </summary>
/// <param name="results"></param>
/// <param name="validResults"></param>
/// <param name="invalidResults"></param>
private void ValidateAndExtractData(DataTable results, DataTable validResults,
DataTable invalidResults, string refMapperPath)
{
Validator validator = new Validator();
// split up the work load
DataTable dtOdd = results.Clone();
DataTable dtEven = results.Clone();
for (int i = 0; i < results.Rows.Count; i++)
{
DataRow dr = results.Rows[i];
if (i % 2 == 0)
dtEven.ImportRow(dr);
else
dtOdd.ImportRow(dr);
}
// setup 2 worker delegates
Microsoft.Scripting.Utils.Action oddWorker = delegate
{
List<DataRow> dataRows = dtOdd.Select().ToList();
for (int i = 0; i < dataRows.Count; i++)
{
ValidateDataRow(validator, refMapperPath, validResults, invalidResults, dataRows[i]);
}
};
DataTable ValidResults2 = results.Clone();
DataTable InvalidResults2 = results.Clone();
Microsoft.Scripting.Utils.Action evenWorker = delegate
{
List<DataRow> dataRows = dtEven.Select().ToList();
for (int i = 0; i < dataRows.Count; i++)
{
ValidateDataRow(validator, refMapperPath, ValidResults2, InvalidResults2, dataRows[i]);
}
};
// run 2 delegates asynchronously
IAsyncResult evenHandler = evenWorker.BeginInvoke(null, null);
IAsyncResult oddHandler = oddWorker.BeginInvoke(null, null);
// wait for both to finish
evenWorker.EndInvoke(evenHandler);
oddWorker.EndInvoke(oddHandler);
// merge the results
validResults.Merge(ValidResults2);
invalidResults.Merge(InvalidResults2);
// remove the invalidReason column for the validResults table
validResults.Columns.Remove("InvalidReason");
}
/// <summary>
/// Command line exit Code Summary:
/// 0 – Application completed its task with no errors
/// 1 – Configuration.xml error
/// 2 – Missing plugin dll
/// 3 - Missing refmapper xml
/// 4 - Datasource directory error
/// 5 - XML output directory error
/// 6 - Data Transfer failed
/// 7 - Log directory error
/// 8 - Failed to move sent file to the "completed" subdirectory
/// 9 - No data to send. There was no valid data to send, double check source files.
/// </summary>
public static void Main(string[] args)
{
// path of where app is running from
_path = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location);
// init _conf and check files
if (CheckFiles(_path))
{
// log to catch errors
Log log = new Log(_conf.LogDirectory);
// load plugin
_plugin = PluginLoader.LoadExtractor("plugins\\" + _conf.Plugin);
// DataTable to store all the extracted data from files
DataTable results = new DataTable();
string[] files = System.IO.Directory.GetFiles(_conf.DataSourceDirectory);
// scans folder for file
foreach (string file in files)
{
// extract data
DataTable dt = _plugin.GetData(_conf, file);
// valid datatable of all valid rows
DataTable validDt = dt.Clone();
// validate the data
foreach (DataRow dr in dt.Rows)
{
Validator validator = new Validator();
bool valid = true;
if (!validator.ValidateDataRow(dr))
valid = false;
if (!_plugin.Validate(dr, _path + "\\plugins\\" + _conf.RefMapper))
valid = false;
// if valid import to validDt
if (valid)
{
validDt.ImportRow(dr);
}
else
{
log.write("Error validating row in " + file + ": ");
// print datarow contents to log file
foreach (DataColumn dc in dr.Table.Columns)
{
log.write(dc.ColumnName + " : " + dr[dc].ToString());
}
}
}
// merged data to Result
results.Merge(validDt);
}
// timestamp for use in the xml
DateTime date = DateTime.Now;
string timestamp = String.Format("{0:yyyyMMddHHmmss}", date);
// if results table is empty then don't generate an empty xml
if (results.Rows.Count != 0)
{
// generate xml
XmlOutput output = new XmlOutput(null);
string xmlFileName = timestamp + ".xml";
// need to parse an emtpy genelist
IList<SiteConf.Gene.Object> geneList = new List<SiteConf.Gene.Object>();
output.Generate("", _conf.XmlOutputDirectory + "\\" + xmlFileName, results, null, null, _conf.OrgHashCode, geneList);
// call VariantExporterCmdLn to send data
if (SendData())
{
// check if completed directory exist
if (!System.IO.Directory.Exists(_conf.DataSourceDirectory + "\\completed"))
{
// create "completed" sub directory inside the xmlOutputDirectory
System.IO.Directory.CreateDirectory(_conf.DataSourceDirectory + "\\completed");
}
// if send successful move files to completed sub directory
foreach (string file in files)
{
// rename file by appending the date to the end of file
// from this "file.tsv" to this "file_20140716143423.tsv"
string renamedFile = System.IO.Path.GetFileNameWithoutExtension(file) + "(" +
DateTime.Now.ToString("yyyyMMddHHmmss") + ")" + System.IO.Path.GetExtension(file);
try
{
System.IO.File.Move(file, _conf.DataSourceDirectory + "\\completed\\" + renamedFile);
log.write("Data from file: " + file + " sent.");
}
catch
{
log.write("Data from file: " + file + " was sent, but due to an error the file could not be moved to the completed sub directory.");
}
}
}
}
else
{
log.write("No valid data to send. Check source that all mandatory fields are provided and correct.");
System.Environment.ExitCode = 9;
}
}
}
private void ValidateDataRow(Validator validator, string refMapperPath,
DataTable validResults, DataTable invalidResults, DataRow dr)
{
bool valid = true;
// standard generic validation
if (!validator.ValidateDataRow(dr))
valid = false;
// site/lab specific validation
if (!plugin.Validate(dr, refMapperPath))
valid = false;
// depending on valid result the datarow is imported to its equivalent table
if (valid)
validResults.ImportRow(dr);
else
invalidResults.ImportRow(dr);
}