Exemplo n.º 1
0
        /// <summary>
        /// Cycle through the results and copy valid and non valid rows to separate tables.
        /// </summary>
        /// <param name="results"></param>
        /// <param name="validResults"></param>
        /// <param name="invalidResults"></param>
        private void ValidateAndExtractData(DataTable results, DataTable validResults, 
            DataTable invalidResults, string refMapperPath)
        {
            Validator validator = new Validator();

            // split up the work load
            DataTable dtOdd = results.Clone();
            DataTable dtEven = results.Clone();
            for (int i = 0; i < results.Rows.Count; i++)
            {
                DataRow dr = results.Rows[i];
                if (i % 2 == 0)
                    dtEven.ImportRow(dr);
                else
                    dtOdd.ImportRow(dr);
            }

            // setup 2 worker delegates
            Microsoft.Scripting.Utils.Action oddWorker = delegate 
            {
                List<DataRow> dataRows = dtOdd.Select().ToList();
                for (int i = 0; i < dataRows.Count; i++)
                {
                    ValidateDataRow(validator, refMapperPath, validResults, invalidResults, dataRows[i]);
                }
            };

            DataTable ValidResults2 = results.Clone();
            DataTable InvalidResults2 = results.Clone();
            Microsoft.Scripting.Utils.Action evenWorker = delegate 
            {
                List<DataRow> dataRows = dtEven.Select().ToList();
                for (int i = 0; i < dataRows.Count; i++)
                {
                    ValidateDataRow(validator, refMapperPath, ValidResults2, InvalidResults2, dataRows[i]);
                }
            };

            // run 2 delegates asynchronously
            IAsyncResult evenHandler = evenWorker.BeginInvoke(null, null);
            IAsyncResult oddHandler = oddWorker.BeginInvoke(null, null);

            // wait for both to finish
            evenWorker.EndInvoke(evenHandler);
            oddWorker.EndInvoke(oddHandler);

            // merge the results
            validResults.Merge(ValidResults2);
            invalidResults.Merge(InvalidResults2);

            // remove the invalidReason column for the validResults table
            validResults.Columns.Remove("InvalidReason");
        }
Exemplo n.º 2
0
        /// <summary>
        /// Command line exit Code Summary:
        /// 0 – Application completed its task with no errors
        /// 1 – Configuration.xml error
        /// 2 – Missing plugin dll
        /// 3 - Missing refmapper xml
        /// 4 - Datasource directory error
        /// 5 - XML output directory error
        /// 6 - Data Transfer failed
        /// 7 - Log directory error
        /// 8 - Failed to move sent file to the "completed" subdirectory
        /// 9 - No data to send. There was no valid data to send, double check source files.
        /// </summary>

        public static void Main(string[] args)
        {
            // path of where app is running from
            _path = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location);

            // init _conf and check files
            if (CheckFiles(_path))
            {
                // log to catch errors
                Log log = new Log(_conf.LogDirectory);
                
                // load plugin
                _plugin = PluginLoader.LoadExtractor("plugins\\" + _conf.Plugin);

                // DataTable to store all the extracted data from files
                DataTable results = new DataTable();
                
                string[] files = System.IO.Directory.GetFiles(_conf.DataSourceDirectory);
                // scans folder for file
                foreach (string file in files)
                {
                    // extract data
                    DataTable dt = _plugin.GetData(_conf, file);

                    // valid datatable of all valid rows
                    DataTable validDt = dt.Clone();
                    
                    // validate the data
                    foreach (DataRow dr in dt.Rows)
                    {
                        Validator validator = new Validator();
                        bool valid = true;

                        if (!validator.ValidateDataRow(dr))
                            valid = false;

                        if (!_plugin.Validate(dr, _path + "\\plugins\\" + _conf.RefMapper))
                            valid =  false;

                        // if valid import to validDt
                        if (valid)
                        {
                            validDt.ImportRow(dr);
                        }
                        else
                        {
                            log.write("Error validating row in " + file + ": ");
                            // print datarow contents to log file
                            foreach (DataColumn dc in dr.Table.Columns)
                            {
                                log.write(dc.ColumnName + " : " + dr[dc].ToString());
                            }
                        }
                    }

                    // merged data to Result
                    results.Merge(validDt);
                }
                
                // timestamp for use in the xml
                DateTime date = DateTime.Now;
                string timestamp = String.Format("{0:yyyyMMddHHmmss}", date);

                // if results table is empty then don't generate an empty xml
                if (results.Rows.Count != 0)
                {
                    // generate xml
                    XmlOutput output = new XmlOutput(null);
                    string xmlFileName = timestamp + ".xml";
                    // need to parse an emtpy genelist
                    IList<SiteConf.Gene.Object> geneList = new List<SiteConf.Gene.Object>();
                    output.Generate("", _conf.XmlOutputDirectory + "\\" + xmlFileName, results, null, null, _conf.OrgHashCode, geneList);

                    // call VariantExporterCmdLn to send data
                    if (SendData())
                    {
                        // check if completed directory exist
                        if (!System.IO.Directory.Exists(_conf.DataSourceDirectory + "\\completed"))
                        {
                            // create "completed" sub directory inside the xmlOutputDirectory
                            System.IO.Directory.CreateDirectory(_conf.DataSourceDirectory + "\\completed");
                        }
                        
                        // if send successful move files to completed sub directory
                        foreach (string file in files)
                        {
                            // rename file by appending  the date to the end of file
                            // from this "file.tsv" to this "file_20140716143423.tsv"
                            string renamedFile = System.IO.Path.GetFileNameWithoutExtension(file) + "(" + 
                                DateTime.Now.ToString("yyyyMMddHHmmss") + ")" + System.IO.Path.GetExtension(file);

                            try
                            {
                                System.IO.File.Move(file, _conf.DataSourceDirectory + "\\completed\\" + renamedFile);
                                log.write("Data from file: " + file + " sent.");
                            }
                            catch
                            {
                                log.write("Data from file: " + file + " was sent, but due to an error the file could not be moved to the completed sub directory.");
                            }
                        }
                    }
                }
                else
                {
                    log.write("No valid data to send. Check source that all mandatory fields are provided and correct.");
                    System.Environment.ExitCode = 9;
                }
            }
        }
Exemplo n.º 3
0
        private void ValidateDataRow(Validator validator, string refMapperPath, 
            DataTable validResults, DataTable invalidResults, DataRow dr)
        {
            bool valid = true;

            // standard generic validation
            if (!validator.ValidateDataRow(dr))
                valid = false;

            // site/lab specific validation
            if (!plugin.Validate(dr, refMapperPath))
                valid = false;

            // depending on valid result the datarow is imported to its equivalent table
            if (valid)
                validResults.ImportRow(dr);
            else
                invalidResults.ImportRow(dr);
        }