/// <summary>
/// Build contigs from graph. For contigs whose coverage is less than
/// the specified threshold, remove graph nodes belonging to them.
/// </summary>
/// <param name="deBruijnGraph">DeBruijn Graph.</param>
/// <param name="coverageThresholdForContigs">Coverage Threshold for contigs.</param>
/// <returns>Number of nodes removed.</returns>
public long RemoveLowCoverageContigs(DeBruijnGraph deBruijnGraph, double coverageThresholdForContigs)
{
if (deBruijnGraph == null)
{
throw new ArgumentNullException("deBruijnGraph");
}
if (coverageThresholdForContigs <= 0)
{
throw new ArgumentException("For removing low coverage contigs, coverage threshold should be a positive number");
}
_coverageThreshold = coverageThresholdForContigs;
_graph = deBruijnGraph;
DeBruijnGraph.ValidateGraph(deBruijnGraph);
ExcludeAmbiguousExtensions();
Parallel.ForEach(deBruijnGraph.GetNodes(), n => n.ComputeValidExtensions());
GetSimplePaths(false);
Parallel.ForEach(deBruijnGraph.GetNodes(),n=>n.UndoAmbiguousExtensions());
return deBruijnGraph.RemoveMarkedNodes();
}
/// <summary>
/// Build contig sequences from the graph.
/// </summary>
/// <param name="deBruijnGraph">De Bruijn graph.</param>
/// <returns>List of contig data.</returns>
public IEnumerable<ISequence> Build(DeBruijnGraph deBruijnGraph)
{
if (deBruijnGraph == null)
{
throw new ArgumentNullException("deBruijnGraph");
}
_graph = deBruijnGraph;
_coverageThreshold = -1;
DeBruijnGraph.ValidateGraph(deBruijnGraph);
ExcludeAmbiguousExtensions();
Parallel.ForEach(_graph.GetNodes(), n => n.PurgeInvalidExtensions());
return GetSimplePaths(true);
}
/// <summary>
/// Validate the graph nodes sequence, left edges and right edges
/// </summary>
/// <param name="graph">graph object</param>
/// <param name="nodeName">xml node name used for different testcases</param>
internal void ValidateGraph(DeBruijnGraph graph, string nodeName)
{
string nodesSequence = utilityObj.xmlUtil.GetTextValue(nodeName,
Constants.NodesSequenceNode);
string nodesLeftEdges = utilityObj.xmlUtil.GetTextValue(nodeName,
Constants.NodesLeftEdgesCountNode);
string nodesRightEdges = utilityObj.xmlUtil.GetTextValue(nodeName,
Constants.NodeRightEdgesCountNode);
string[] leftEdgesCount = ReadStringFromFile(nodesLeftEdges).Replace("\r\n", "").Split(',');
string[] rightEdgesCount = ReadStringFromFile(nodesRightEdges).Replace("\r\n", "").Split(',');
string[] nodesSequences = ReadStringFromFile(nodesSequence).Replace("\r\n", "").Split(',');
// Validate the nodes
for (int iseq = 0; iseq < nodesSequences.Length; iseq++)
{
DeBruijnNode dbnodes = graph.GetNodes().First(n => graph.GetNodeSequence(n).ConvertToString() == nodesSequences[iseq]
|| graph.GetNodeSequence(n).GetReverseComplementedSequence().ConvertToString() == nodesSequences[iseq]);
//Due to parallelization the left edges and right edges count
//can be swapped while processing. if actual left edges count
//is either equal to expected left edges count or right edges count and vice versa.
Assert.IsTrue(
dbnodes.LeftExtensionNodesCount.ToString((IFormatProvider)null) == leftEdgesCount[iseq] ||
dbnodes.LeftExtensionNodesCount.ToString((IFormatProvider)null) == rightEdgesCount[iseq]);
Assert.IsTrue(
dbnodes.RightExtensionNodesCount.ToString((IFormatProvider)null) == leftEdgesCount[iseq] ||
dbnodes.RightExtensionNodesCount.ToString((IFormatProvider)null) == rightEdgesCount[iseq]);
}
}
/// <summary>
/// Erode ends of graph that have coverage less than given erodeThreshold.
/// As optimization, we also check for dangling links and keeps track of the
/// lengths of the links found. No removal is done at this step.
/// This is done to get an idea of the different lengths at
/// which to run the dangling links purger step.
/// This method returns the lengths of dangling links found.
/// Locks: Method only does reads. No locking necessary here.
/// </summary>
/// <param name="graph">Input graph.</param>
/// <param name="erosionThreshold">Threshold for erosion.</param>
/// <returns>List of lengths of dangling links detected.</returns>
public IEnumerable<int> ErodeGraphEnds(DeBruijnGraph graph, int erosionThreshold = -1)
{
if (graph == null)
{
throw new ArgumentNullException("graph");
}
this.erodeThreshold = erosionThreshold;
this.danglingLinkLengths = new SortedSet<int>();
this.danglingLinkExtensionTasks = new List<Task<int>>();
IEnumerable<DeBruijnNode> graphNodes = graph.GetNodes();
BlockingCollection<int> linkLengths = new BlockingCollection<int>();
// Start consumer task
// TODO: This length list should contain elements from ~1-19, decide if
// having a separate thread keep those numbers sorted is worthwhile
Task collectionTask = Task.Run(() =>
{
while (!linkLengths.IsCompleted)
{
int length;
if (linkLengths.TryTake(out length))
{
this.danglingLinkLengths.Add(length);
}
}
});
//and now the producer
bool continueSearching = true;
while (continueSearching)
{
continueSearching = false;
int threshold = erosionThreshold;
Parallel.ForEach(graphNodes, node =>
{
continueSearching = true;
if (node.ExtensionsCount == 0)
{
if (threshold != -1 && node.KmerCount < threshold)
{
// Mark node for erosion
node.MarkNodeForDelete();
}
else
{
// Single node island.
linkLengths.Add(1);
}
}
else if (node.RightExtensionNodesCount == 0)
{
// End of possible dangling link
// Trace back to see if it is part of a dangling link.
DeBruijnPath link = TraceDanglingExtensionLink(false, new DeBruijnPath(), node, true);
if (link != null && link.PathNodes.Count > 0)
{
linkLengths.Add(link.PathNodes.Count);
}
}
else if (node.LeftExtensionNodesCount == 0)
{
// End of possible dangling link
// Trace back to see if it is part of a dangling link.
DeBruijnPath link = TraceDanglingExtensionLink(true, new DeBruijnPath(), node, true);
if (link != null && link.PathNodes.Count > 0)
{
linkLengths.Add(link.PathNodes.Count);
}
}
});
// Remove eroded nodes. In the out parameter, get the list of new
// end-points that was created by removing eroded nodes.
IList<DeBruijnNode> nodes = RemoveErodedNodes(graph);
if (nodes.Count == 0)
{
break;
}
graphNodes = nodes;
}
linkLengths.CompleteAdding();
Task.WaitAll(collectionTask);
ExtendDanglingLinks();
return this.danglingLinkLengths;
}
/// <summary>
/// Delete nodes marked for erosion. Update adjacent nodes to update their extension tables.
/// After nodes are deleted, some new end-points might be created. We need to check for
/// dangling links at these new points. This list is returned in the out parameter.
///
/// TODO: Perhaps refactor code so that the graph is only manipulated by itself?
/// Might make it easier to implement future performance improvements, or cost performance
/// </summary>
/// <param name="graph">De Bruijn Graph.</param>
private static IList<DeBruijnNode> RemoveErodedNodes(DeBruijnGraph graph)
{
bool eroded = graph.RemoveMarkedNodes()>0;
IList<DeBruijnNode> graphNodes;
if (eroded)
{
graphNodes = graph.GetNodes().AsParallel().Where(n =>
{
bool wasEndPoint = (n.LeftExtensionNodesCount == 0 || n.RightExtensionNodesCount == 0);
n.RemoveMarkedExtensions();
// Check if this is a new end point.
return (wasEndPoint || (n.LeftExtensionNodesCount == 0 || n.RightExtensionNodesCount == 0));
}).ToList();
}
else
{
graphNodes = new List<DeBruijnNode>();
}
return graphNodes;
}
/// <summary>
/// Detect nodes that are part of dangling links.
/// Locks: Method only does reads. No locking necessary here or its callees.
/// </summary>
/// <param name="deBruijnGraph">Input graph.</param>
/// <returns>List of nodes in dangling links.</returns>
public DeBruijnPathList DetectErroneousNodes(DeBruijnGraph deBruijnGraph)
{
if (deBruijnGraph == null)
throw new ArgumentNullException("deBruijnGraph");
var debruijnPaths = new ConcurrentBag<DeBruijnPath>();
Parallel.ForEach(deBruijnGraph.GetNodes(), node =>
{
if (node.ExtensionsCount == 0)
{
// Single node island
debruijnPaths.Add(new DeBruijnPath(node));
}
else if (node.RightExtensionNodesCount == 0)
{
// End of possible dangling link
// Trace back to see if it is part of a dangling link
var link = TraceDanglingExtensionLink(false, new DeBruijnPath(), node, true);
if (link != null && link.PathNodes.Count>0)
{
debruijnPaths.Add(link);
}
}
else if (node.LeftExtensionNodesCount == 0)
{
// End of possible dangling link
// Trace back to see if it is part of a dangling link
var link = TraceDanglingExtensionLink(true, new DeBruijnPath(), node, true);
if (link != null && link.PathNodes.Count>0)//if the first node is below the threshold, it is not added, leaving a link with no nodes, so check is needed
{
debruijnPaths.Add(link);
}
}
}
);
return new DeBruijnPathList(debruijnPaths);
}
/// <summary>
/// Detect nodes that are on redundant paths.
/// Start from any node that has ambiguous (more than one) extensions.
/// From this node, trace path for each extension until either they
/// converge to a single node or threshold length is exceeded.
/// In case they converge, we have a set of redundant paths.
/// We pick the best path based on the kmer counts of the path nodes.
/// All paths other than the best one are returned for removal.
/// Locks: Method only does reads. No locking necessary here or its callees.
/// </summary>
/// <param name="deBruijnGraph">De Bruijn Graph.</param>
/// <returns>List of path nodes to be deleted.</returns>
public DeBruijnPathList DetectErroneousNodes(DeBruijnGraph deBruijnGraph)
{
if (deBruijnGraph == null)
{
throw new ArgumentNullException("deBruijnGraph");
}
DeBruijnGraph.ValidateGraph(deBruijnGraph);
this.graph = deBruijnGraph;
List<DeBruijnPathList> redundantPaths = new List<DeBruijnPathList>();
Parallel.ForEach(
deBruijnGraph.GetNodes(),
node =>
{
// Need to check for both left and right extensions for ambiguity.
if (node.RightExtensionNodesCount > 1)
{
TraceDivergingExtensionPaths(node, node.GetRightExtensionNodesWithOrientation(), true, redundantPaths);
}
if (node.LeftExtensionNodesCount > 1)
{
TraceDivergingExtensionPaths(node, node.GetLeftExtensionNodesWithOrientation(), false, redundantPaths);
}
});
redundantPaths = RemoveDuplicates(redundantPaths);
return DetachBestPath(redundantPaths);
}