Пример #1
0
        /// <summary>
        /// Serializes remaining file components onto disk after task has completed.
        /// </summary>
        public void SerializeRemainingFqComponents()
        {
            ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();

            while (toSerialize.Count != 0)
            {
                int threadId;
                FqFile_Component fqFileComponent = toSerialize.Dequeue();
                Boolean          result          = protoBuf.ProtobufSerializeFqFile(fqFileComponent, fqFileComponent.getFileName(), out threadId);
            }
        }
Пример #2
0
        /// <summary>
        /// Priming the component queue, used prior to the work of the tasks starting, this method ensures that there are
        /// some components deserialized into memory for use.
        /// </summary>
        public void PrimeFqFileComponentQueue()
        {
            int threadId;
            int count = 0;

            while (toDeserialize.Count != 0 && count < 2)
            {
                ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
                String           componentName        = toDeserialize.Dequeue();
                FqFile_Component component            = protoBuf.ProtobufDerializeFqFile(componentName, out threadId);
                toPerform.Enqueue(component);
                count++;
            }
        }
Пример #3
0
        /// <summary>
        /// Method asynchronously serializes components to memory.
        /// </summary>
        private void SerializeFqComponentToMemory()
        {
            while (toSerialize.Count != 0)
            {
                int threadId;
                FqFile_Component             component = toSerialize.Dequeue();
                ProtocolBuffersSerialization protoBuf  = new ProtocolBuffersSerialization();
                ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
                    = new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);

                IAsyncResult result = caller.BeginInvoke(component, component.getFileName(), out threadId, null, null);

                Boolean returnValue = caller.EndInvoke(out threadId, result);
            }
        }
        /// <summary>
        /// Parses file into component chunks through the parseFastq class before handing component details to FastqController
        /// for processing.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void loadWorker_DoWork(object sender, DoWorkEventArgs e)
        {
            BackgroundWorker worker = sender as BackgroundWorker;
            InputFq input = (InputFq)e.Argument;
            FileStream fileStream = input.fileStream;
            String fileName = input.fileName;

            ParseFastq parseFq;

            if ((worker.CancellationPending == true))
            {
                e.Cancel = true;
            }
            else
            {
                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();
                try
                {
                    worker.ReportProgress(3, "[PARSING FILE]");
                    parseFq = new ParseFastq(fileStream, fileName);

                    if (parseFq.getFastqFileCheck() == true && FastqController.CONTROLLER_STATE == FastqController.FastqControllerState.STATE_READY)
                    {
                        //Create new fqFileMap in controller and prime state for load
                        FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.PARSING;
                        FastqController.getInstance().CreateNewFastqFile(fileName, parseFq.GetFastqFileLength());
                        FastqController.getInstance().GetFqFileMap().InitializeReadMap();

                        int fqFileComponentNumber = 1;
                        ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
                        GenericFastqInputs processInputs = new GenericFastqInputs();
                        processInputs.TaskAction = Task_LoadTask.statement;
                        ITaskStrategy task = TaskDiscrimination.getTask(processInputs.TaskAction);

                        // Uses IEnummerable yield return to parse components back to this class via this foreach loop
                        foreach (FqFile_Component fqFileComponent in parseFq.ParseComponents())
                        {
                            Double progressPercent = (Double)(((fqFileComponentNumber-1) * FqFileMap.FQ_BLOCK_LIMIT) / (Double)(parseFq.GetLineCount() / 4));
                            worker.ReportProgress((int)(progressPercent*100), ParseFastq.REPORT_STATEMENT);

                            if (fqFileComponent.getFastqArraySize() >= 1)
                            {
                                int threadId;
                                String componentFileName = FastqController.getInstance().GetFqFileMap().FileGUID + "_" + Path.GetFileNameWithoutExtension(fileName) + "_" + fqFileComponentNumber + ProtocolBuffersSerialization.PROTOBUF_FILE_EXTENSION;
                                fqFileComponent.setFastqFileName(componentFileName);
                                fqFileComponent.setComponentNumber(fqFileComponentNumber);
                                fqFileComponent.setFqHashMap(FastqController.getInstance().GetFqFileMap().FqReadMap);

                                processInputs.FastqFile = fqFileComponent;
                                processInputs = task.perform(processInputs);
                                FastqController.getInstance().BuildFqFileMap(processInputs.FastqFile);

                                ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
                                    = new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);

                                IAsyncResult result = caller.BeginInvoke((processInputs.FastqFile as FqFile_Component), componentFileName, out threadId, null, null);

                                Boolean returnValue = caller.EndInvoke(out threadId, result);

                                if (returnValue == false)
                                {
                                    UserResponse.ErrorResponse("File serialization methods failed, please check you have hard disk space and restart the application", "File Error");
                                    Console.WriteLine("Serialization failed");
                                    loadWorker.CancelAsync();
                                }
                                else
                                {
                                    FastqController.getInstance().addFqFileComponentDirectory(componentFileName);
                                }
                                fqFileComponentNumber++;
                            }
                        }
                        parseFq.CloseReader();

                        task.confirmTaskEnd();
                        Console.WriteLine("\n*********\n");
                        FastqController.getInstance().GetFqFileMap().CalculateGlobalFileScores();
                        FastqController.getInstance().GetFqFileMap().GlobalDetails.OutputToConsole();
                        loadWorker.ReportProgress(100, task.getReportStatement());

                        stopwatch.Stop();
                        Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), stopwatch.Elapsed);

                        FastqController.getInstance().GetFqFileMap().LastTask = processInputs.TaskAction;
                        FastqController.getInstance().GetFqFileMap().TimeTaken = stopwatch.Elapsed.ToString();
                        this.UpdateGUIThread(processInputs);
                    }
                    else
                    {
                        worker.ReportProgress(0, "");
                        UserResponse.InformationResponse("File does not conform to standard format.", "File Error");
                        parseFq.CloseReader();
                    }
                }
                catch (IOException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }
                catch (InsufficientMemoryException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }
                catch (OutOfMemoryException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }
                catch (ArithmeticException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }

                FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.STATE_READY;
                stopwatch.Stop();
            }
        }
Пример #5
0
        /// <summary>
        /// Parses file into component chunks through the parseFastq class before handing component details to FastqController
        /// for processing.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void loadWorker_DoWork(object sender, DoWorkEventArgs e)
        {
            BackgroundWorker worker     = sender as BackgroundWorker;
            InputFq          input      = (InputFq)e.Argument;
            FileStream       fileStream = input.fileStream;
            String           fileName   = input.fileName;

            ParseFastq parseFq;

            if ((worker.CancellationPending == true))
            {
                e.Cancel = true;
            }
            else
            {
                Stopwatch stopwatch = new Stopwatch();
                stopwatch.Start();
                try
                {
                    worker.ReportProgress(3, "[PARSING FILE]");
                    parseFq = new ParseFastq(fileStream, fileName);

                    if (parseFq.getFastqFileCheck() == true && FastqController.CONTROLLER_STATE == FastqController.FastqControllerState.STATE_READY)
                    {
                        //Create new fqFileMap in controller and prime state for load
                        FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.PARSING;
                        FastqController.getInstance().CreateNewFastqFile(fileName, parseFq.GetFastqFileLength());
                        FastqController.getInstance().GetFqFileMap().InitializeReadMap();

                        int fqFileComponentNumber                  = 1;
                        ProtocolBuffersSerialization protoBuf      = new ProtocolBuffersSerialization();
                        GenericFastqInputs           processInputs = new GenericFastqInputs();
                        processInputs.TaskAction = Task_LoadTask.statement;
                        ITaskStrategy task = TaskDiscrimination.getTask(processInputs.TaskAction);

                        // Uses IEnummerable yield return to parse components back to this class via this foreach loop
                        foreach (FqFile_Component fqFileComponent in parseFq.ParseComponents())
                        {
                            Double progressPercent = (Double)(((fqFileComponentNumber - 1) * FqFileMap.FQ_BLOCK_LIMIT) / (Double)(parseFq.GetLineCount() / 4));
                            worker.ReportProgress((int)(progressPercent * 100), ParseFastq.REPORT_STATEMENT);

                            if (fqFileComponent.getFastqArraySize() >= 1)
                            {
                                int    threadId;
                                String componentFileName = FastqController.getInstance().GetFqFileMap().FileGUID + "_" + Path.GetFileNameWithoutExtension(fileName) + "_" + fqFileComponentNumber + ProtocolBuffersSerialization.PROTOBUF_FILE_EXTENSION;
                                fqFileComponent.setFastqFileName(componentFileName);
                                fqFileComponent.setComponentNumber(fqFileComponentNumber);
                                fqFileComponent.setFqHashMap(FastqController.getInstance().GetFqFileMap().FqReadMap);

                                processInputs.FastqFile = fqFileComponent;
                                processInputs           = task.perform(processInputs);
                                FastqController.getInstance().BuildFqFileMap(processInputs.FastqFile);


                                ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
                                    = new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);

                                IAsyncResult result = caller.BeginInvoke((processInputs.FastqFile as FqFile_Component), componentFileName, out threadId, null, null);

                                Boolean returnValue = caller.EndInvoke(out threadId, result);

                                if (returnValue == false)
                                {
                                    UserResponse.ErrorResponse("File serialization methods failed, please check you have hard disk space and restart the application", "File Error");
                                    Console.WriteLine("Serialization failed");
                                    loadWorker.CancelAsync();
                                }
                                else
                                {
                                    FastqController.getInstance().addFqFileComponentDirectory(componentFileName);
                                }
                                fqFileComponentNumber++;
                            }
                        }
                        parseFq.CloseReader();

                        task.confirmTaskEnd();
                        Console.WriteLine("\n*********\n");
                        FastqController.getInstance().GetFqFileMap().CalculateGlobalFileScores();
                        FastqController.getInstance().GetFqFileMap().GlobalDetails.OutputToConsole();
                        loadWorker.ReportProgress(100, task.getReportStatement());

                        stopwatch.Stop();
                        Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), stopwatch.Elapsed);

                        FastqController.getInstance().GetFqFileMap().LastTask  = processInputs.TaskAction;
                        FastqController.getInstance().GetFqFileMap().TimeTaken = stopwatch.Elapsed.ToString();
                        this.UpdateGUIThread(processInputs);
                    }
                    else
                    {
                        worker.ReportProgress(0, "");
                        UserResponse.InformationResponse("File does not conform to standard format.", "File Error");
                        parseFq.CloseReader();
                    }
                }
                catch (IOException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }
                catch (InsufficientMemoryException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }
                catch (OutOfMemoryException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }
                catch (ArithmeticException exception)
                {
                    Console.Write(exception.StackTrace);
                    UserResponse.ErrorResponse(exception.ToString());
                }

                FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.STATE_READY;
                stopwatch.Stop();
            }
        }
Пример #6
0
        /// <summary>
        /// The key method in the controller class, where fastqFile_components are serialized and deserialized in and out of
        /// the classes queues for processing.  The tasks are constructed through an interface - ITaskStrategy - which
        /// is designated through the task abstract factory class (TaskDiscriminator.cs).  After processing, files are deserialized and a details class
        /// for each component is populated, as are the global scores.  The use of Abstract/interface classes here allows multiple components
        /// to be processed with multiple task types within this basic code structure.
        /// </summary>
        /// <param name="worker">The backgroundworker thread</param>
        /// <param name="input">Generic inputs, including taskname and any further details necessary to complete a task such as nucleotide scores etc.</param>
        public void PerformAction(BackgroundWorker worker, GenericFastqInputs input)
        {
            BackgroundWorker loadWorker = worker;

            if (fqFileMap != null && (CONTROLLER_STATE == FastqControllerState.STATE_READY || CONTROLLER_STATE == FastqControllerState.PARSING))
            {
                sw = new Stopwatch();
                sw.Start();
                try
                {
                    toDeserialize = new Queue <String>(fqFileMap.getFileComponentDirectories());
                    PrimeFqFileComponentQueue();
                    protobufSerialization = new ProtocolBuffersSerialization();

                    ITaskStrategy task = TaskDiscrimination.getTask(input.TaskAction);
                    Console.WriteLine("Performing {0}", task.getStatement());

                    int count = 0;
                    while (toPerform.Count != 0)
                    {
                        Double progressPercent = (Double)(count / (Double)fqFileMap.getFileComponentDirectories().Count);
                        loadWorker.ReportProgress((int)(progressPercent * 100), task.getReportStatement());

                        FqFile_Component activeComponent = toPerform.Dequeue();
                        activeComponent.setFqHashMap(fqFileMap.FqReadMap);

                        if (toDeserialize.Count != 0)
                        {
                            int threadId;
                            ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller caller
                                = new ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller(protobufSerialization.ProtobufDerializeFqFile);
                            String       componentFileName = toDeserialize.Dequeue();
                            IAsyncResult result            = caller.BeginInvoke(componentFileName, out threadId, null, null);

                            Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
                            input.FastqFile = activeComponent;
                            input           = task.perform(input);
                            activeComponent = (FqFile_Component)input.FastqFile;
                            BuildFqFileMap(activeComponent);

                            FqFile_Component returnValue = caller.EndInvoke(out threadId, result);
                            toPerform.Enqueue(returnValue);
                        }
                        else if (toDeserialize.Count == 0)
                        {
                            Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
                            input.FastqFile = activeComponent;
                            input           = task.perform(input);
                            activeComponent = (FqFile_Component)input.FastqFile;
                            BuildFqFileMap(activeComponent);
                        }
                        toSerialize.Enqueue(activeComponent);
                        SerializeFqComponentToMemory();
                        count++;
                    }

                    SerializeRemainingFqComponents();
                    task.confirmTaskEnd();
                    Console.WriteLine("\n*********\n");
                    fqFileMap.CalculateGlobalFileScores();
                    fqFileMap.GlobalDetails.OutputToConsole();
                    loadWorker.ReportProgress(100, task.getReportStatement());

                    sw.Stop();
                    Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), sw.Elapsed);

                    fqFileMap.LastTask  = input.TaskAction;
                    fqFileMap.TimeTaken = sw.Elapsed.ToString();
                    observer.UpdateGUIThread(input);
                }
                catch (IOException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                catch (InsufficientMemoryException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                catch (OutOfMemoryException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                catch (ArithmeticException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                sw.Stop();
            }
        }
        /// <summary>
        /// Method asynchronously serializes components to memory.
        /// </summary>
        private void SerializeFqComponentToMemory()
        {
            while (toSerialize.Count != 0)
            {
                int threadId;
                FqFile_Component component = toSerialize.Dequeue();
                ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
                ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
                                    = new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);

                IAsyncResult result = caller.BeginInvoke(component, component.getFileName(), out threadId, null, null);

                Boolean returnValue = caller.EndInvoke(out threadId, result);
            }
        }
 /// <summary>
 /// Serializes remaining file components onto disk after task has completed.
 /// </summary>
 public void SerializeRemainingFqComponents()
 {
     ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
     while (toSerialize.Count != 0)
     {
         int threadId;
         FqFile_Component fqFileComponent = toSerialize.Dequeue();
         Boolean result = protoBuf.ProtobufSerializeFqFile(fqFileComponent, fqFileComponent.getFileName(), out threadId);
     }
 }
 /// <summary>
 /// Priming the component queue, used prior to the work of the tasks starting, this method ensures that there are 
 /// some components deserialized into memory for use.
 /// </summary>
 public void PrimeFqFileComponentQueue()
 {
     int threadId;
     int count = 0;
     while (toDeserialize.Count != 0 && count < 2)
     {
         ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
         String componentName = toDeserialize.Dequeue();
         FqFile_Component component = protoBuf.ProtobufDerializeFqFile(componentName, out threadId);
         toPerform.Enqueue(component);
         count++;
     }
 }
        /// <summary>
        /// The key method in the controller class, where fastqFile_components are serialized and deserialized in and out of 
        /// the classes queues for processing.  The tasks are constructed through an interface - ITaskStrategy - which 
        /// is designated through the task abstract factory class (TaskDiscriminator.cs).  After processing, files are deserialized and a details class
        /// for each component is populated, as are the global scores.  The use of Abstract/interface classes here allows multiple components 
        /// to be processed with multiple task types within this basic code structure.
        /// </summary>
        /// <param name="worker">The backgroundworker thread</param>
        /// <param name="input">Generic inputs, including taskname and any further details necessary to complete a task such as nucleotide scores etc.</param>
        public void PerformAction(BackgroundWorker worker, GenericFastqInputs input)
        {
            BackgroundWorker loadWorker = worker;

            if (fqFileMap != null && (CONTROLLER_STATE == FastqControllerState.STATE_READY || CONTROLLER_STATE == FastqControllerState.PARSING))
            {
                sw = new Stopwatch();
                sw.Start();
                try
                {
                    toDeserialize = new Queue<String>(fqFileMap.getFileComponentDirectories());
                    PrimeFqFileComponentQueue();
                    protobufSerialization = new ProtocolBuffersSerialization();

                    ITaskStrategy task = TaskDiscrimination.getTask(input.TaskAction);
                    Console.WriteLine("Performing {0}", task.getStatement());

                    int count = 0;
                    while (toPerform.Count != 0)
                    {
                        Double progressPercent = (Double)(count / (Double)fqFileMap.getFileComponentDirectories().Count);
                        loadWorker.ReportProgress((int)(progressPercent * 100), task.getReportStatement());

                        FqFile_Component activeComponent = toPerform.Dequeue();
                        activeComponent.setFqHashMap(fqFileMap.FqReadMap);

                        if (toDeserialize.Count != 0)
                        {
                            int threadId;
                            ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller caller
                                            = new ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller(protobufSerialization.ProtobufDerializeFqFile);
                            String componentFileName = toDeserialize.Dequeue();
                            IAsyncResult result = caller.BeginInvoke(componentFileName, out threadId, null, null);

                            Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
                            input.FastqFile = activeComponent;
                            input = task.perform(input);
                            activeComponent = (FqFile_Component)input.FastqFile;
                            BuildFqFileMap(activeComponent);

                            FqFile_Component returnValue = caller.EndInvoke(out threadId, result);
                            toPerform.Enqueue(returnValue);
                        }
                        else if (toDeserialize.Count == 0)
                        {
                            Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
                            input.FastqFile = activeComponent;
                            input = task.perform(input);
                            activeComponent = (FqFile_Component)input.FastqFile;
                            BuildFqFileMap(activeComponent);
                        }
                        toSerialize.Enqueue(activeComponent);
                        SerializeFqComponentToMemory();
                        count++;
                    }

                    SerializeRemainingFqComponents();
                    task.confirmTaskEnd();
                    Console.WriteLine("\n*********\n");
                    fqFileMap.CalculateGlobalFileScores();
                    fqFileMap.GlobalDetails.OutputToConsole();
                    loadWorker.ReportProgress(100, task.getReportStatement());

                    sw.Stop();
                    Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), sw.Elapsed);

                    fqFileMap.LastTask = input.TaskAction;
                    fqFileMap.TimeTaken = sw.Elapsed.ToString();
                    observer.UpdateGUIThread(input);
                }
                catch (IOException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                catch (InsufficientMemoryException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                catch (OutOfMemoryException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                catch (ArithmeticException exception)
                {
                    ControllerStateFailureResponse(exception.ToString(), "Error");
                }
                sw.Stop();
            }
        }