/// <summary>
/// Serializes remaining file components onto disk after task has completed.
/// </summary>
public void SerializeRemainingFqComponents()
{
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
while (toSerialize.Count != 0)
{
int threadId;
FqFile_Component fqFileComponent = toSerialize.Dequeue();
Boolean result = protoBuf.ProtobufSerializeFqFile(fqFileComponent, fqFileComponent.getFileName(), out threadId);
}
}
/// <summary>
/// Priming the component queue, used prior to the work of the tasks starting, this method ensures that there are
/// some components deserialized into memory for use.
/// </summary>
public void PrimeFqFileComponentQueue()
{
int threadId;
int count = 0;
while (toDeserialize.Count != 0 && count < 2)
{
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
String componentName = toDeserialize.Dequeue();
FqFile_Component component = protoBuf.ProtobufDerializeFqFile(componentName, out threadId);
toPerform.Enqueue(component);
count++;
}
}
/// <summary>
/// Method asynchronously serializes components to memory.
/// </summary>
private void SerializeFqComponentToMemory()
{
while (toSerialize.Count != 0)
{
int threadId;
FqFile_Component component = toSerialize.Dequeue();
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);
IAsyncResult result = caller.BeginInvoke(component, component.getFileName(), out threadId, null, null);
Boolean returnValue = caller.EndInvoke(out threadId, result);
}
}
/// <summary>
/// Parses file into component chunks through the parseFastq class before handing component details to FastqController
/// for processing.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void loadWorker_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
InputFq input = (InputFq)e.Argument;
FileStream fileStream = input.fileStream;
String fileName = input.fileName;
ParseFastq parseFq;
if ((worker.CancellationPending == true))
{
e.Cancel = true;
}
else
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
try
{
worker.ReportProgress(3, "[PARSING FILE]");
parseFq = new ParseFastq(fileStream, fileName);
if (parseFq.getFastqFileCheck() == true && FastqController.CONTROLLER_STATE == FastqController.FastqControllerState.STATE_READY)
{
//Create new fqFileMap in controller and prime state for load
FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.PARSING;
FastqController.getInstance().CreateNewFastqFile(fileName, parseFq.GetFastqFileLength());
FastqController.getInstance().GetFqFileMap().InitializeReadMap();
int fqFileComponentNumber = 1;
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
GenericFastqInputs processInputs = new GenericFastqInputs();
processInputs.TaskAction = Task_LoadTask.statement;
ITaskStrategy task = TaskDiscrimination.getTask(processInputs.TaskAction);
// Uses IEnummerable yield return to parse components back to this class via this foreach loop
foreach (FqFile_Component fqFileComponent in parseFq.ParseComponents())
{
Double progressPercent = (Double)(((fqFileComponentNumber-1) * FqFileMap.FQ_BLOCK_LIMIT) / (Double)(parseFq.GetLineCount() / 4));
worker.ReportProgress((int)(progressPercent*100), ParseFastq.REPORT_STATEMENT);
if (fqFileComponent.getFastqArraySize() >= 1)
{
int threadId;
String componentFileName = FastqController.getInstance().GetFqFileMap().FileGUID + "_" + Path.GetFileNameWithoutExtension(fileName) + "_" + fqFileComponentNumber + ProtocolBuffersSerialization.PROTOBUF_FILE_EXTENSION;
fqFileComponent.setFastqFileName(componentFileName);
fqFileComponent.setComponentNumber(fqFileComponentNumber);
fqFileComponent.setFqHashMap(FastqController.getInstance().GetFqFileMap().FqReadMap);
processInputs.FastqFile = fqFileComponent;
processInputs = task.perform(processInputs);
FastqController.getInstance().BuildFqFileMap(processInputs.FastqFile);
ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);
IAsyncResult result = caller.BeginInvoke((processInputs.FastqFile as FqFile_Component), componentFileName, out threadId, null, null);
Boolean returnValue = caller.EndInvoke(out threadId, result);
if (returnValue == false)
{
UserResponse.ErrorResponse("File serialization methods failed, please check you have hard disk space and restart the application", "File Error");
Console.WriteLine("Serialization failed");
loadWorker.CancelAsync();
}
else
{
FastqController.getInstance().addFqFileComponentDirectory(componentFileName);
}
fqFileComponentNumber++;
}
}
parseFq.CloseReader();
task.confirmTaskEnd();
Console.WriteLine("\n*********\n");
FastqController.getInstance().GetFqFileMap().CalculateGlobalFileScores();
FastqController.getInstance().GetFqFileMap().GlobalDetails.OutputToConsole();
loadWorker.ReportProgress(100, task.getReportStatement());
stopwatch.Stop();
Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), stopwatch.Elapsed);
FastqController.getInstance().GetFqFileMap().LastTask = processInputs.TaskAction;
FastqController.getInstance().GetFqFileMap().TimeTaken = stopwatch.Elapsed.ToString();
this.UpdateGUIThread(processInputs);
}
else
{
worker.ReportProgress(0, "");
UserResponse.InformationResponse("File does not conform to standard format.", "File Error");
parseFq.CloseReader();
}
}
catch (IOException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
catch (InsufficientMemoryException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
catch (OutOfMemoryException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
catch (ArithmeticException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.STATE_READY;
stopwatch.Stop();
}
}
/// <summary>
/// Parses file into component chunks through the parseFastq class before handing component details to FastqController
/// for processing.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void loadWorker_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
InputFq input = (InputFq)e.Argument;
FileStream fileStream = input.fileStream;
String fileName = input.fileName;
ParseFastq parseFq;
if ((worker.CancellationPending == true))
{
e.Cancel = true;
}
else
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
try
{
worker.ReportProgress(3, "[PARSING FILE]");
parseFq = new ParseFastq(fileStream, fileName);
if (parseFq.getFastqFileCheck() == true && FastqController.CONTROLLER_STATE == FastqController.FastqControllerState.STATE_READY)
{
//Create new fqFileMap in controller and prime state for load
FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.PARSING;
FastqController.getInstance().CreateNewFastqFile(fileName, parseFq.GetFastqFileLength());
FastqController.getInstance().GetFqFileMap().InitializeReadMap();
int fqFileComponentNumber = 1;
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
GenericFastqInputs processInputs = new GenericFastqInputs();
processInputs.TaskAction = Task_LoadTask.statement;
ITaskStrategy task = TaskDiscrimination.getTask(processInputs.TaskAction);
// Uses IEnummerable yield return to parse components back to this class via this foreach loop
foreach (FqFile_Component fqFileComponent in parseFq.ParseComponents())
{
Double progressPercent = (Double)(((fqFileComponentNumber - 1) * FqFileMap.FQ_BLOCK_LIMIT) / (Double)(parseFq.GetLineCount() / 4));
worker.ReportProgress((int)(progressPercent * 100), ParseFastq.REPORT_STATEMENT);
if (fqFileComponent.getFastqArraySize() >= 1)
{
int threadId;
String componentFileName = FastqController.getInstance().GetFqFileMap().FileGUID + "_" + Path.GetFileNameWithoutExtension(fileName) + "_" + fqFileComponentNumber + ProtocolBuffersSerialization.PROTOBUF_FILE_EXTENSION;
fqFileComponent.setFastqFileName(componentFileName);
fqFileComponent.setComponentNumber(fqFileComponentNumber);
fqFileComponent.setFqHashMap(FastqController.getInstance().GetFqFileMap().FqReadMap);
processInputs.FastqFile = fqFileComponent;
processInputs = task.perform(processInputs);
FastqController.getInstance().BuildFqFileMap(processInputs.FastqFile);
ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);
IAsyncResult result = caller.BeginInvoke((processInputs.FastqFile as FqFile_Component), componentFileName, out threadId, null, null);
Boolean returnValue = caller.EndInvoke(out threadId, result);
if (returnValue == false)
{
UserResponse.ErrorResponse("File serialization methods failed, please check you have hard disk space and restart the application", "File Error");
Console.WriteLine("Serialization failed");
loadWorker.CancelAsync();
}
else
{
FastqController.getInstance().addFqFileComponentDirectory(componentFileName);
}
fqFileComponentNumber++;
}
}
parseFq.CloseReader();
task.confirmTaskEnd();
Console.WriteLine("\n*********\n");
FastqController.getInstance().GetFqFileMap().CalculateGlobalFileScores();
FastqController.getInstance().GetFqFileMap().GlobalDetails.OutputToConsole();
loadWorker.ReportProgress(100, task.getReportStatement());
stopwatch.Stop();
Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), stopwatch.Elapsed);
FastqController.getInstance().GetFqFileMap().LastTask = processInputs.TaskAction;
FastqController.getInstance().GetFqFileMap().TimeTaken = stopwatch.Elapsed.ToString();
this.UpdateGUIThread(processInputs);
}
else
{
worker.ReportProgress(0, "");
UserResponse.InformationResponse("File does not conform to standard format.", "File Error");
parseFq.CloseReader();
}
}
catch (IOException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
catch (InsufficientMemoryException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
catch (OutOfMemoryException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
catch (ArithmeticException exception)
{
Console.Write(exception.StackTrace);
UserResponse.ErrorResponse(exception.ToString());
}
FastqController.CONTROLLER_STATE = FastqController.FastqControllerState.STATE_READY;
stopwatch.Stop();
}
}
/// <summary>
/// The key method in the controller class, where fastqFile_components are serialized and deserialized in and out of
/// the classes queues for processing. The tasks are constructed through an interface - ITaskStrategy - which
/// is designated through the task abstract factory class (TaskDiscriminator.cs). After processing, files are deserialized and a details class
/// for each component is populated, as are the global scores. The use of Abstract/interface classes here allows multiple components
/// to be processed with multiple task types within this basic code structure.
/// </summary>
/// <param name="worker">The backgroundworker thread</param>
/// <param name="input">Generic inputs, including taskname and any further details necessary to complete a task such as nucleotide scores etc.</param>
public void PerformAction(BackgroundWorker worker, GenericFastqInputs input)
{
BackgroundWorker loadWorker = worker;
if (fqFileMap != null && (CONTROLLER_STATE == FastqControllerState.STATE_READY || CONTROLLER_STATE == FastqControllerState.PARSING))
{
sw = new Stopwatch();
sw.Start();
try
{
toDeserialize = new Queue <String>(fqFileMap.getFileComponentDirectories());
PrimeFqFileComponentQueue();
protobufSerialization = new ProtocolBuffersSerialization();
ITaskStrategy task = TaskDiscrimination.getTask(input.TaskAction);
Console.WriteLine("Performing {0}", task.getStatement());
int count = 0;
while (toPerform.Count != 0)
{
Double progressPercent = (Double)(count / (Double)fqFileMap.getFileComponentDirectories().Count);
loadWorker.ReportProgress((int)(progressPercent * 100), task.getReportStatement());
FqFile_Component activeComponent = toPerform.Dequeue();
activeComponent.setFqHashMap(fqFileMap.FqReadMap);
if (toDeserialize.Count != 0)
{
int threadId;
ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller(protobufSerialization.ProtobufDerializeFqFile);
String componentFileName = toDeserialize.Dequeue();
IAsyncResult result = caller.BeginInvoke(componentFileName, out threadId, null, null);
Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
input.FastqFile = activeComponent;
input = task.perform(input);
activeComponent = (FqFile_Component)input.FastqFile;
BuildFqFileMap(activeComponent);
FqFile_Component returnValue = caller.EndInvoke(out threadId, result);
toPerform.Enqueue(returnValue);
}
else if (toDeserialize.Count == 0)
{
Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
input.FastqFile = activeComponent;
input = task.perform(input);
activeComponent = (FqFile_Component)input.FastqFile;
BuildFqFileMap(activeComponent);
}
toSerialize.Enqueue(activeComponent);
SerializeFqComponentToMemory();
count++;
}
SerializeRemainingFqComponents();
task.confirmTaskEnd();
Console.WriteLine("\n*********\n");
fqFileMap.CalculateGlobalFileScores();
fqFileMap.GlobalDetails.OutputToConsole();
loadWorker.ReportProgress(100, task.getReportStatement());
sw.Stop();
Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), sw.Elapsed);
fqFileMap.LastTask = input.TaskAction;
fqFileMap.TimeTaken = sw.Elapsed.ToString();
observer.UpdateGUIThread(input);
}
catch (IOException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (InsufficientMemoryException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (OutOfMemoryException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (ArithmeticException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
sw.Stop();
}
}
/// <summary>
/// Method asynchronously serializes components to memory.
/// </summary>
private void SerializeFqComponentToMemory()
{
while (toSerialize.Count != 0)
{
int threadId;
FqFile_Component component = toSerialize.Dequeue();
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);
IAsyncResult result = caller.BeginInvoke(component, component.getFileName(), out threadId, null, null);
Boolean returnValue = caller.EndInvoke(out threadId, result);
}
}
/// <summary>
/// Serializes remaining file components onto disk after task has completed.
/// </summary>
public void SerializeRemainingFqComponents()
{
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
while (toSerialize.Count != 0)
{
int threadId;
FqFile_Component fqFileComponent = toSerialize.Dequeue();
Boolean result = protoBuf.ProtobufSerializeFqFile(fqFileComponent, fqFileComponent.getFileName(), out threadId);
}
}
/// <summary>
/// Priming the component queue, used prior to the work of the tasks starting, this method ensures that there are
/// some components deserialized into memory for use.
/// </summary>
public void PrimeFqFileComponentQueue()
{
int threadId;
int count = 0;
while (toDeserialize.Count != 0 && count < 2)
{
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
String componentName = toDeserialize.Dequeue();
FqFile_Component component = protoBuf.ProtobufDerializeFqFile(componentName, out threadId);
toPerform.Enqueue(component);
count++;
}
}
/// <summary>
/// The key method in the controller class, where fastqFile_components are serialized and deserialized in and out of
/// the classes queues for processing. The tasks are constructed through an interface - ITaskStrategy - which
/// is designated through the task abstract factory class (TaskDiscriminator.cs). After processing, files are deserialized and a details class
/// for each component is populated, as are the global scores. The use of Abstract/interface classes here allows multiple components
/// to be processed with multiple task types within this basic code structure.
/// </summary>
/// <param name="worker">The backgroundworker thread</param>
/// <param name="input">Generic inputs, including taskname and any further details necessary to complete a task such as nucleotide scores etc.</param>
public void PerformAction(BackgroundWorker worker, GenericFastqInputs input)
{
BackgroundWorker loadWorker = worker;
if (fqFileMap != null && (CONTROLLER_STATE == FastqControllerState.STATE_READY || CONTROLLER_STATE == FastqControllerState.PARSING))
{
sw = new Stopwatch();
sw.Start();
try
{
toDeserialize = new Queue<String>(fqFileMap.getFileComponentDirectories());
PrimeFqFileComponentQueue();
protobufSerialization = new ProtocolBuffersSerialization();
ITaskStrategy task = TaskDiscrimination.getTask(input.TaskAction);
Console.WriteLine("Performing {0}", task.getStatement());
int count = 0;
while (toPerform.Count != 0)
{
Double progressPercent = (Double)(count / (Double)fqFileMap.getFileComponentDirectories().Count);
loadWorker.ReportProgress((int)(progressPercent * 100), task.getReportStatement());
FqFile_Component activeComponent = toPerform.Dequeue();
activeComponent.setFqHashMap(fqFileMap.FqReadMap);
if (toDeserialize.Count != 0)
{
int threadId;
ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller(protobufSerialization.ProtobufDerializeFqFile);
String componentFileName = toDeserialize.Dequeue();
IAsyncResult result = caller.BeginInvoke(componentFileName, out threadId, null, null);
Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
input.FastqFile = activeComponent;
input = task.perform(input);
activeComponent = (FqFile_Component)input.FastqFile;
BuildFqFileMap(activeComponent);
FqFile_Component returnValue = caller.EndInvoke(out threadId, result);
toPerform.Enqueue(returnValue);
}
else if (toDeserialize.Count == 0)
{
Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
input.FastqFile = activeComponent;
input = task.perform(input);
activeComponent = (FqFile_Component)input.FastqFile;
BuildFqFileMap(activeComponent);
}
toSerialize.Enqueue(activeComponent);
SerializeFqComponentToMemory();
count++;
}
SerializeRemainingFqComponents();
task.confirmTaskEnd();
Console.WriteLine("\n*********\n");
fqFileMap.CalculateGlobalFileScores();
fqFileMap.GlobalDetails.OutputToConsole();
loadWorker.ReportProgress(100, task.getReportStatement());
sw.Stop();
Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), sw.Elapsed);
fqFileMap.LastTask = input.TaskAction;
fqFileMap.TimeTaken = sw.Elapsed.ToString();
observer.UpdateGUIThread(input);
}
catch (IOException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (InsufficientMemoryException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (OutOfMemoryException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (ArithmeticException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
sw.Stop();
}
}