/// <summary>
/// Command line exit Code Summary:
/// 0 – Application completed its task with no errors
/// 1 – Configuration.xml error
/// 2 – Missing plugin dll
/// 3 - Missing refmapper xml
/// 4 - Datasource directory error
/// 5 - XML output directory error
/// 6 - Data Transfer failed
/// 7 - Log directory error
/// 8 - Failed to move sent file to the "completed" subdirectory
/// 9 - No data to send. There was no valid data to send, double check source files.
/// </summary>
public static void Main(string[] args)
{
// path of where app is running from
_path = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location);
// init _conf and check files
if (CheckFiles(_path))
{
// log to catch errors
Log log = new Log(_conf.LogDirectory);
// load plugin
_plugin = PluginLoader.LoadExtractor("plugins\\" + _conf.Plugin);
// DataTable to store all the extracted data from files
DataTable results = new DataTable();
string[] files = System.IO.Directory.GetFiles(_conf.DataSourceDirectory);
// scans folder for file
foreach (string file in files)
{
// extract data
DataTable dt = _plugin.GetData(_conf, file);
// valid datatable of all valid rows
DataTable validDt = dt.Clone();
// validate the data
foreach (DataRow dr in dt.Rows)
{
Validator validator = new Validator();
bool valid = true;
if (!validator.ValidateDataRow(dr))
valid = false;
if (!_plugin.Validate(dr, _path + "\\plugins\\" + _conf.RefMapper))
valid = false;
// if valid import to validDt
if (valid)
{
validDt.ImportRow(dr);
}
else
{
log.write("Error validating row in " + file + ": ");
// print datarow contents to log file
foreach (DataColumn dc in dr.Table.Columns)
{
log.write(dc.ColumnName + " : " + dr[dc].ToString());
}
}
}
// merged data to Result
results.Merge(validDt);
}
// timestamp for use in the xml
DateTime date = DateTime.Now;
string timestamp = String.Format("{0:yyyyMMddHHmmss}", date);
// if results table is empty then don't generate an empty xml
if (results.Rows.Count != 0)
{
// generate xml
XmlOutput output = new XmlOutput(null);
string xmlFileName = timestamp + ".xml";
// need to parse an emtpy genelist
IList<SiteConf.Gene.Object> geneList = new List<SiteConf.Gene.Object>();
output.Generate("", _conf.XmlOutputDirectory + "\\" + xmlFileName, results, null, null, _conf.OrgHashCode, geneList);
// call VariantExporterCmdLn to send data
if (SendData())
{
// check if completed directory exist
if (!System.IO.Directory.Exists(_conf.DataSourceDirectory + "\\completed"))
{
// create "completed" sub directory inside the xmlOutputDirectory
System.IO.Directory.CreateDirectory(_conf.DataSourceDirectory + "\\completed");
}
// if send successful move files to completed sub directory
foreach (string file in files)
{
// rename file by appending the date to the end of file
// from this "file.tsv" to this "file_20140716143423.tsv"
string renamedFile = System.IO.Path.GetFileNameWithoutExtension(file) + "(" +
DateTime.Now.ToString("yyyyMMddHHmmss") + ")" + System.IO.Path.GetExtension(file);
try
{
System.IO.File.Move(file, _conf.DataSourceDirectory + "\\completed\\" + renamedFile);
log.write("Data from file: " + file + " sent.");
}
catch
{
log.write("Data from file: " + file + " was sent, but due to an error the file could not be moved to the completed sub directory.");
}
}
}
}
else
{
log.write("No valid data to send. Check source that all mandatory fields are provided and correct.");
System.Environment.ExitCode = 9;
}
}
}
public void TestGenerateXmlAllFields()
{
DataTable dt = HVP_Standard.NewVariantInstanceTable();
// === Make dummy data ===
DataRow row = dt.NewRow();
row[Organisation.HashCode] = "XYZ123";
row[Patient.HashCode] = "XYZ123";
row[Gene.GeneName] = "BRCA1";
row[VariantInstance.Status] = "New";
row[VariantInstance.HashCode] = "XYZ123";
row[VariantInstance.VariantClass] = "cDNA";
row[VariantInstance.cDNA] = "c.1G>T";
row[VariantInstance.mRNA] = "r.1G>U";
row[VariantInstance.Genomic] = "g.100G>T";
row[VariantInstance.Protein] = "p.33Ser>Cys";
row[VariantInstance.InstanceDate] = new DateTime(2011, 4, 14, 17, 22, 0);
row[VariantInstance.Pathogenicity] = "1";
row[VariantInstance.PatientAge] = 36;
row[VariantInstance.TestMethod] = "1";
row[VariantInstance.SampleTissue] = "1";
row[VariantInstance.SampleSource] = "1";
row[VariantInstance.Justification] = "Justification";
row[VariantInstance.PubMed] = "1234";
row[VariantInstance.RecordedInDatabase] = true;
row[VariantInstance.SampleStored] = true;
row[VariantInstance.PedigreeAvailable] = true;
row[VariantInstance.VariantSegregatesWithDisease] = true;
row[VariantInstance.HistologyStored] = true;
dt.Rows.Add(row);
// === Output the data ===
XmlOutput output = new XmlOutput();
//output.Generate("testAll.xml", dt);
// === Test read-back ===
XmlDocument doc = ReadXml("testAll.xml");
Assert.IsTrue(doc != null);
Assert.IsTrue(doc.HasChildNodes);
XmlNodeList nodes = doc.GetElementsByTagName("HVP_Transaction");
Assert.IsTrue(nodes.Count == 1);
XmlNode node = nodes[0];
Console.WriteLine(node.Attributes["destination"].Value);
Assert.AreEqual("https://portal.hvpaustralia.org.au", node.Attributes["destination"].Value);
Assert.NotNull(node.Attributes["dateCreated"].Value);
Assert.IsTrue(node.HasChildNodes);
Assert.IsTrue(node.ChildNodes.Count == 1);
node = node.ChildNodes[0];
Assert.AreEqual("XYZ123", node.ChildNodes[0].InnerText);
Assert.AreEqual("XYZ123", node.ChildNodes[1].InnerText);
Assert.AreEqual("BRCA1", node.ChildNodes[2].InnerText);
Assert.AreEqual("NM001732", node.ChildNodes[3].InnerText);
Assert.AreEqual("1", node.ChildNodes[4].InnerText);
Assert.AreEqual("New", node.ChildNodes[5].InnerText);
Assert.AreEqual("XYZ123", node.ChildNodes[6].InnerText);
Assert.AreEqual("cDNA", node.ChildNodes[7].InnerText);
Assert.AreEqual("c.1G>T", node.ChildNodes[8].InnerText);
Assert.AreEqual("r.1G>U", node.ChildNodes[9].InnerText);
Assert.AreEqual("g.100G>T", node.ChildNodes[10].InnerText);
Assert.AreEqual("p.33Ser>Cys", node.ChildNodes[11].InnerText);
Assert.AreEqual("2011-04-14-17:22", node.ChildNodes[12].InnerText);
Assert.AreEqual("1", node.ChildNodes[13].InnerText);
Assert.AreEqual("Male", node.ChildNodes[14].InnerText);
Assert.AreEqual("36", node.ChildNodes[15].InnerText);
Assert.AreEqual("1", node.ChildNodes[16].InnerText);
Assert.AreEqual("1", node.ChildNodes[17].InnerText);
Assert.AreEqual("1", node.ChildNodes[18].InnerText);
Assert.AreEqual("Justification", node.ChildNodes[19].InnerText);
Assert.AreEqual("1234", node.ChildNodes[20].InnerText);
Assert.AreEqual("true", node.ChildNodes[21].InnerText);
Assert.AreEqual("true", node.ChildNodes[22].InnerText);
Assert.AreEqual("true", node.ChildNodes[23].InnerText);
Assert.AreEqual("true", node.ChildNodes[24].InnerText);
Assert.AreEqual("true", node.ChildNodes[25].InnerText);
}
/// <summary>
/// Process data from datagrid which includes encrypting and zipping data and
/// sending data to the specified destination
/// </summary>
/// <param name="serverAddress">Destination server data is sent to</param>
/// <param name="audit">Log data sent in the internal audit database</param>
private void ProcessAndSendData(string serverAddress, bool audit, bool isTest)
{
bool DataSuccessfullySent = false;
int sentVariants = 0;
Cursor.Current = Cursors.WaitCursor;
SplashScreen ss = new SplashScreen();
ss.Show();
DisableAllControls();
// if data is read from a data source file check if file is in use by another
// process, otherwise file can not be moved to the "sent" subdirectory.
if (_spreadsheetPath != string.Empty)
{
try
{
using (System.IO.Stream stream = new System.IO.FileStream(_spreadsheetPath, System.IO.FileMode.Open))
{
// Nothing to see here. Just testing to see if file is not opened or in use by another process.
}
}
catch (Exception ex)
{
ss.Close();
EnableAllControls();
// alert user file is being used by another process
MessageBox.Show("The current file: " + _spreadsheetPath +
" is being used by another process. " +
"\n" +
"\nIf you have the file open in an application, please close the application before continuing.",
"File in use: " + _spreadsheetPath,
MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
// log error exception
log.write("The file: " + _spreadsheetPath + " is being used by another process.");
log.write(ex.ToString());
log.write("No data was sent!");
// end this function
return;
}
}
if (dgVariant.Rows.Count != 0)
{
DataTable original = (DataTable)dgVariant.DataSource;
DataTable yesOnly = original.Clone();
SiteConf.Upload.Object upload = ExporterCommon.DataLoader.GetUpload(_uploadID);
// get all the genes with the disease tags
IList<SiteConf.Gene.Object> geneList = ExporterCommon.DataLoader.GetGeneList(upload.ID);
// sorting data to send
log.write("======= Begin Data Transfer =======");
log.write("Sorting data for sending....");
foreach (DataGridViewRow row in dgVariant.Rows)
{
if (Convert.ToBoolean(row.Cells[0].Value))
{
yesOnly.ImportRow(((DataRowView)row.DataBoundItem).Row);
}
}
sentVariants = yesOnly.Rows.Count;
if (yesOnly.Rows.Count > 0)
{
// timestamp for use in the xml and zip file names
DateTime dt = DateTime.Now;
string timestamp = String.Format("{0:yyyyMMddHHmmss}", dt);
XmlOutput output = new XmlOutput();
string xmlFileName = timestamp + ".xml";
CommonAppPath.CreateDirectory("raw");
string xmlFilePath = _commonAppPath + "\\raw\\" + xmlFileName;
// hash table to store the refseq names and versions
Hashtable refSeqNameHT = new Hashtable();
Hashtable refSeqVerHT = new Hashtable();
// hash tables to store patient grhanite hashes
getHashTableRefSeq(refSeqNameHT, refSeqVerHT, upload);
// generate xml file
log.write("Writing to xml...");
output.Generate(conf.PortalWebSite, xmlFilePath, yesOnly, refSeqNameHT,
refSeqVerHT, _OrgHashCode, geneList);
// send xml to server
try
{
string encryptedFile = timestamp + ".txt";
CommonAppPath.CreateDirectory("temp");
string encryptedFilePath = _commonAppPath + "\\temp\\" + encryptedFile;
// start the encryption executable
log.write("Encrypting data....");
// location of HVP_Encryption executable
string appPath = System.IO.Path.GetDirectoryName(
System.Reflection.Assembly.GetEntryAssembly().Location)
+ "\\Executables\\HVP_Encryption.exe";
// Encrypt file
Encryption.HVP_EncryptFile(appPath, xmlFilePath, encryptedFilePath, _commonAppPath + "\\keys\\",
conf.PrivateKey, conf.PublicKey, log);
// zip file name is combination of org hash code and date
string zippedFileName = _orgSite.OrgHashCode + "_" + DateTime.Now.ToString("yyyyMMddHHmmss");
string zippedFilePath = _commonAppPath + "\\temp\\" + zippedFileName;
// zip files
Decompression.ZippUpTransferFiles(zippedFileName, encryptedFilePath, _commonAppPath + "\\temp\\", conf.PrivateKey.Replace(".private", ""), log);
// send the zip file to destination server
log.write("Sending data to: " + serverAddress);
// webclient for uploading data
WebClient client = new WebClient();
// set proxy to webclient if any
client.Proxy = _proxy;
// send zipped file
string sent = Transmit.SendData(zippedFilePath + ".zip", serverAddress, client, log);
if (_CancelledAuth)
{
log.write("User cancelled proxy auth login");
return;
}
// sent ok we should get a post
if (sent == "<HTML>POST OK.</HTML>")
{
log.write("Data successfully sent!!! " + sentVariants + " variants sent.");
// only log data sent to db if sending to live server and not the test server
if (audit)
{
// save the transaction of each instance to the auditdb
log.write("Saving transaction to db...");
SaveAuditTransaction(yesOnly, xmlFilePath);
}
// clear the Invalid results table and disable the invalid data button
invalidResults = null;
btnErrors.Enabled = false;
// rename the sent xls and move to folder called "sent", creates folder if not exist
if (rbnSpreadsheet.Checked == true && isTest != true)
{
if (!System.IO.File.Exists(upload.DataSourceName + "\\sent"))
{
// create new directory called sent in the source location of xls
System.IO.Directory.CreateDirectory(upload.DataSourceName + "\\sent");
}
// get the source file extension
string sourceFileExt = System.IO.Path.GetExtension(_spreadsheetPath);
// set the name of file to the datee/time sent
string renamedFile = "(" + timestamp + ")" + sourceFileExt;
// move spreadsheet to directory called sent
bool fileMoved = false;
// loop to check if file is in use by another process, will continue to loop
// until file can be moved to the sent folder or user acknowledges it can't
// be moved by the program and user should move it themselves manually to
// avoid sending data from the file again.
while (!fileMoved)
{
try
{
System.IO.File.Move(_spreadsheetPath, upload.DataSourceName +
"\\sent\\" + System.IO.Path.GetFileName(_spreadsheetPath).Replace(sourceFileExt, "") + renamedFile);
fileMoved = true;
}
catch (System.IO.IOException ioEx)
{
// if unable to rename/move file
var result = MessageBox.Show("Unable to access file: " + _spreadsheetPath +
"\n" +
"\nThe exporter has successfully uploaded the data, however it is unable to move the file from datasource folder to the sent sub folder." +
"\n" +
"\nIf you have the file open in an application, please close the application and click Retry." +
"\n" +
"\nOr click Cancel and the file will remain in your datasource folder, you will need to manually move that file out of the datasource folder otherwise you may end up resending the data again.",
"Can not access file: " + _spreadsheetPath,
MessageBoxButtons.RetryCancel, MessageBoxIcon.Asterisk);
if (result == DialogResult.Cancel)
{
// if cancel exit loop with error
fileMoved = true;
// log error
log.write("Unable to access the file: " + _spreadsheetPath);
log.write(ioEx.ToString());
}
}
catch (Exception ex)
{
ss.Close();
EnableAllControls();
// log error
log.write("The data has been successfully uploaded however the Variant Exporter has encountered an error.");
log.write(ex.ToString());
// set to true to exit out of the loop
fileMoved = true;
}
}
}
// Data has been successfully sent
DataSuccessfullySent = true;
// clear the datagrid
dgVariant.DataSource = null;
// clear the spreadsheet path now that it has been sent
// keep the path if it was only a test send
if (!isTest)
_spreadsheetPath = "";
}
else
{
ss.Close();
EnableAllControls();
log.write("Error connecting to server!!!!");
MessageBox.Show("Error connecting to server, please try again.", "Connection Error", MessageBoxButtons.OK, MessageBoxIcon.Error); ;
}
}
catch (WebException ex)
{
ss.Close();
EnableAllControls();
log.write("Unexpected error!!!");
log.write(ex.ToString());
MessageBox.Show("Error connecting to server, please try again.", "Connection Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
Cursor.Current = Cursors.Default;
ss.Close();
EnableAllControls();
if (ClearCache)
{
// always clear the contents in the temp
log.write("Clearing contents from temp folder....");
System.IO.DirectoryInfo temp = new System.IO.DirectoryInfo(_commonAppPath + "\\temp");
foreach (System.IO.FileInfo file in temp.GetFiles())
{
file.Delete();
}
// remove the raw xml files
log.write("Clearing the contents from raw folder....");
System.IO.DirectoryInfo raw = new System.IO.DirectoryInfo(_commonAppPath + " \\raw");
foreach (System.IO.FileInfo file in raw.GetFiles())
{
file.Delete();
}
}
// clear the filename from title
lblTitle.Text = upload.Name;
}
}
else
{
ss.Close();
EnableAllControls();
MessageBox.Show("You have not selected any rows!!!", "Export Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
else
{
ss.Close();
EnableAllControls();
MessageBox.Show("There is nothing to send", "No Data selected");
}
log.write("======= Data Transfer Complete =======");
if (DataSuccessfullySent)
MessageBox.Show("Data Successfully sent. " + sentVariants + " variants sent.", "Data sent", MessageBoxButtons.OK);
}