public Boolean ProtobufSerializeFqFile(FqFile_Component fq, String fileName, out int threadId)
{
Stopwatch sw = new Stopwatch();
threadId = Thread.CurrentThread.ManagedThreadId;
sw.Start();
try
{
using (var file = File.Create(fileName))
{
Serializer.Serialize <FqFile_Component>(file, fq);
sw.Stop();
Console.WriteLine("Protobuf Serilization Time: {0} of File: {1} On Thread: {2}", sw.Elapsed, fileName, threadId);
return(true);
}
}
catch (IOException exception)
{
Console.WriteLine("Protobuf Serialization Failed: {0}", exception.StackTrace);
}
catch (System.Runtime.Serialization.SerializationException exception)
{
Console.WriteLine("Protobuf Serialization Failed: {0}", exception.StackTrace);
}
return(false);
}
/// <summary>
/// Serializes remaining file components onto disk after task has completed.
/// </summary>
public void SerializeRemainingFqComponents()
{
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
while (toSerialize.Count != 0)
{
int threadId;
FqFile_Component fqFileComponent = toSerialize.Dequeue();
Boolean result = protoBuf.ProtobufSerializeFqFile(fqFileComponent, fqFileComponent.getFileName(), out threadId);
}
}
public override GenericFastqInputs perform(GenericFastqInputs inputs)
{
writer = inputs.SaveStreamWriter;
FqFile_Component component = (FqFile_Component)inputs.FastqFile;
for (int i = 0; i < component.getFastqArraySize(); i++)
{
writer.Write(component.getFastqSequenceByPosition(i).createFastqBlock(component.getMap()));
}
return(inputs);
}
/// <summary>
/// Priming the component queue, used prior to the work of the tasks starting, this method ensures that there are
/// some components deserialized into memory for use.
/// </summary>
public void PrimeFqFileComponentQueue()
{
int threadId;
int count = 0;
while (toDeserialize.Count != 0 && count < 2)
{
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
String componentName = toDeserialize.Dequeue();
FqFile_Component component = protoBuf.ProtobufDerializeFqFile(componentName, out threadId);
toPerform.Enqueue(component);
count++;
}
}
/// <summary>
/// Method asynchronously serializes components to memory.
/// </summary>
private void SerializeFqComponentToMemory()
{
while (toSerialize.Count != 0)
{
int threadId;
FqFile_Component component = toSerialize.Dequeue();
ProtocolBuffersSerialization protoBuf = new ProtocolBuffersSerialization();
ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufSerializeFqFile_AsyncMethodCaller(protoBuf.ProtobufSerializeFqFile);
IAsyncResult result = caller.BeginInvoke(component, component.getFileName(), out threadId, null, null);
Boolean returnValue = caller.EndInvoke(out threadId, result);
}
}
public FqSequence_IO(String sequencerType, String taskType, FqFile_Component fqFile)
{
if (taskType == SEQUENCE_TESTS_TASK)
{
distributes = new List<int>(40);
for (int j = 0; j <= SequencerDiscriminator.getSequencerSpecifier(sequencerType).getDistributionSpread(); j++)
{
distributes.Add(0);
}
perSeqQuals = new int[fqFile.getMaxSeqSize()];
subZeroOffset = SequencerDiscriminator.getSequencerSpecifier(sequencerType).getSubZeroQualities();
}
else if (taskType == ADAPTER_TASK)
{
adapters = Adapters.getInstance().getAdaptersList();
removedAdapters = null;
}
}
public FqSequence_IO(String sequencerType, String taskType, FqFile_Component fqFile)
{
if (taskType == SEQUENCE_TESTS_TASK)
{
distributes = new List <int>(40);
for (int j = 0; j <= SequencerDiscriminator.getSequencerSpecifier(sequencerType).getDistributionSpread(); j++)
{
distributes.Add(0);
}
perSeqQuals = new int[fqFile.getMaxSeqSize()];
subZeroOffset = SequencerDiscriminator.getSequencerSpecifier(sequencerType).getSubZeroQualities();
}
else if (taskType == ADAPTER_TASK)
{
adapters = Adapters.getInstance().getAdaptersList();
removedAdapters = null;
}
}
public IEnumerable<FqFile_Component> ParseStandardFormatComponent()
{
sw = new Stopwatch();
sw.Start();
int seqIndex = 0;
int blockNumber = 0;
BufferedStream bs;
StreamReader reader;
try
{
bs = new BufferedStream(fileReader);
reader = new StreamReader(bs, System.Text.Encoding.ASCII);
if (IsFastqFile == true)
{
FastqController.getInstance().GetFqFileMap().FastqFileFormatType = FILE_FORMAT_TYPE;
FqNucleotideRead fqRead = new FqNucleotideRead(' ', ' ');
FqFile_Component fastqFileComponent = new FqFile_Component();
long nLine = -1L;
while ((fastqHeader = reader.ReadLine()) != null)
{
nLine++;
if (nLine % 4 != 0) continue;
String seqlist = reader.ReadLine();
String infoHeader = reader.ReadLine();
String qscore = reader.ReadLine();
fqSeq = new FqSequence(seqIndex, fastqHeader, infoHeader, seqlist.Length);
seqIndex++;
for (int i = 0; i < (seqlist.Length); i++)
{
fqRead.resetFqNucleotideRead(seqlist[i], qscore[i]);
int hashcode = fqRead.getProxyCode();
fqSeq.addNucleotideRead(hashcode);
}
fastqFileComponent.addFastqSequence(fqSeq);
nLine += 3;
blockNumber ++;
if (blockNumber == FqFileMap.FQ_BLOCK_LIMIT)
{
yield return fastqFileComponent;
blockNumber = 0;
fastqFileComponent = new FqFile_Component();
}
}
yield return fastqFileComponent;
sw.Stop();
Console.WriteLine("Time to Parse File: " + sw.Elapsed + "s");
}
}
finally
{
fileReader.FlushAsync();
fileReader.Close();
}
}
/// <summary>
/// The key method in the controller class, where fastqFile_components are serialized and deserialized in and out of
/// the classes queues for processing. The tasks are constructed through an interface - ITaskStrategy - which
/// is designated through the task abstract factory class (TaskDiscriminator.cs). After processing, files are deserialized and a details class
/// for each component is populated, as are the global scores. The use of Abstract/interface classes here allows multiple components
/// to be processed with multiple task types within this basic code structure.
/// </summary>
/// <param name="worker">The backgroundworker thread</param>
/// <param name="input">Generic inputs, including taskname and any further details necessary to complete a task such as nucleotide scores etc.</param>
public void PerformAction(BackgroundWorker worker, GenericFastqInputs input)
{
BackgroundWorker loadWorker = worker;
if (fqFileMap != null && (CONTROLLER_STATE == FastqControllerState.STATE_READY || CONTROLLER_STATE == FastqControllerState.PARSING))
{
sw = new Stopwatch();
sw.Start();
try
{
toDeserialize = new Queue <String>(fqFileMap.getFileComponentDirectories());
PrimeFqFileComponentQueue();
protobufSerialization = new ProtocolBuffersSerialization();
ITaskStrategy task = TaskDiscrimination.getTask(input.TaskAction);
Console.WriteLine("Performing {0}", task.getStatement());
int count = 0;
while (toPerform.Count != 0)
{
Double progressPercent = (Double)(count / (Double)fqFileMap.getFileComponentDirectories().Count);
loadWorker.ReportProgress((int)(progressPercent * 100), task.getReportStatement());
FqFile_Component activeComponent = toPerform.Dequeue();
activeComponent.setFqHashMap(fqFileMap.FqReadMap);
if (toDeserialize.Count != 0)
{
int threadId;
ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller caller
= new ProtocolBuffersSerialization.ProbufDeserializeFqFile_AsyncMethodCaller(protobufSerialization.ProtobufDerializeFqFile);
String componentFileName = toDeserialize.Dequeue();
IAsyncResult result = caller.BeginInvoke(componentFileName, out threadId, null, null);
Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
input.FastqFile = activeComponent;
input = task.perform(input);
activeComponent = (FqFile_Component)input.FastqFile;
BuildFqFileMap(activeComponent);
FqFile_Component returnValue = caller.EndInvoke(out threadId, result);
toPerform.Enqueue(returnValue);
}
else if (toDeserialize.Count == 0)
{
Console.WriteLine("\n*** Processing: {0} ***\n", activeComponent.getFileName());
input.FastqFile = activeComponent;
input = task.perform(input);
activeComponent = (FqFile_Component)input.FastqFile;
BuildFqFileMap(activeComponent);
}
toSerialize.Enqueue(activeComponent);
SerializeFqComponentToMemory();
count++;
}
SerializeRemainingFqComponents();
task.confirmTaskEnd();
Console.WriteLine("\n*********\n");
fqFileMap.CalculateGlobalFileScores();
fqFileMap.GlobalDetails.OutputToConsole();
loadWorker.ReportProgress(100, task.getReportStatement());
sw.Stop();
Console.WriteLine("Task: {0} Completed in Time: {1}", task.getStatement(), sw.Elapsed);
fqFileMap.LastTask = input.TaskAction;
fqFileMap.TimeTaken = sw.Elapsed.ToString();
observer.UpdateGUIThread(input);
}
catch (IOException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (InsufficientMemoryException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (OutOfMemoryException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
catch (ArithmeticException exception)
{
ControllerStateFailureResponse(exception.ToString(), "Error");
}
sw.Stop();
}
}
public Boolean ProtobufSerializeFqFile(FqFile_Component fq, String fileName, out int threadId)
{
Stopwatch sw = new Stopwatch();
threadId = Thread.CurrentThread.ManagedThreadId;
sw.Start();
try
{
using (var file = File.Create(fileName))
{
Serializer.Serialize<FqFile_Component>(file, fq);
sw.Stop();
Console.WriteLine("Protobuf Serilization Time: {0} of File: {1} On Thread: {2}", sw.Elapsed, fileName, threadId);
return true;
}
}
catch (IOException exception)
{
Console.WriteLine("Protobuf Serialization Failed: {0}", exception.StackTrace);
}
catch (System.Runtime.Serialization.SerializationException exception)
{
Console.WriteLine("Protobuf Serialization Failed: {0}", exception.StackTrace);
}
return false;
}
public IEnumerable <FqFile_Component> ParseMultiLineFormatComponent()
{
sw = new Stopwatch();
sw.Start();
int seqIndex = 0;
int blockNumber = 0;
BufferedStream bs;
StreamReader reader;
try
{
bs = new BufferedStream(fileReader);
reader = new StreamReader(bs, System.Text.Encoding.ASCII);
if (IsFastqFile == true)
{
FastqController.getInstance().GetFqFileMap().FastqFileFormatType = FILE_FORMAT_TYPE;
FqNucleotideRead fqRead = new FqNucleotideRead(' ', ' ');
FqFile_Component fastqFileComponent = new FqFile_Component();
long nLine = -1L;
while ((fastqHeader = reader.ReadLine()) != null)
{
nLine++;
if (nLine % 6 != 0)
{
continue;
}
String seqlist = reader.ReadLine();
String seqlist2 = reader.ReadLine();
String infoHeader = reader.ReadLine();
String qscore = reader.ReadLine();
String qscore2 = reader.ReadLine();
fqSeq = new FqSequence(seqIndex, fastqHeader, infoHeader, (seqlist.Length + seqlist2.Length));
seqIndex++;
for (int i = 0; i < (seqlist.Length); i++)
{
fqRead.resetFqNucleotideRead(seqlist[i], qscore[i]);
int hashcode = fqRead.getProxyCode();
fqSeq.addNucleotideRead(hashcode);
}
if (seqlist2.Length > 0 && (seqlist2.Length == qscore2.Length))
{
for (int i = 0; i < (seqlist.Length); i++)
{
fqRead.resetFqNucleotideRead(seqlist2[i], qscore2[i]);
int hashcode = fqRead.getProxyCode();
fqSeq.addNucleotideRead(hashcode);
}
}
fastqFile.addFastqSequence(fqSeq);
nLine += 3;
blockNumber++;
if (blockNumber == FqFileMap.FQ_BLOCK_LIMIT)
{
yield return(fastqFileComponent);
blockNumber = 0;
fastqFileComponent = new FqFile_Component();
}
}
yield return(fastqFileComponent);
}
}
finally
{
fileReader.Close();
}
sw.Stop();
Console.WriteLine("Time to Parse File: " + sw.Elapsed + "s");
}