} //checked
public static ArrayList ExtractHand(float[,] recording)
{
int x = recording.GetLength(0);
int y = recording.GetLength(1);
ArrayList depthSequence = new ArrayList();
ArrayList depthSequenceOriginal = new ArrayList();
//[frame num, frame]
ArrayList threshFrames = new ArrayList();
//transposed frames of recording
for (int i = 0; i < x; i++)
{
float[] recordFrame = new float[217088];
for (int j = 0; j < 217088; j++)
{
recordFrame[j] = recording[i, j];
}
DenseMatrix frame = new DenseMatrix(424, 512, recordFrame);
threshFrames.Add(frame.Transpose());
}
int numFrames = x;
//Getting the Background Frame
DenseMatrix backFrame = (DenseMatrix)threshFrames[0];
//Variables for thresholding
int numPlast = 0;
int startThresh = 400;
int startUpThresh = 200;
int upThresh = 0;
int downThresh = 0;
int maxThresh = 0;
int minThresh = 0;
BWConnReturn maxClusterFinal = new BWConnReturn();
//Thresholding each frame one by one
for (int frameNum = 1; frameNum < numFrames; frameNum++)
{
bool bad = false;
//Background subtraction
DenseMatrix depthFrame = (DenseMatrix)threshFrames[frameNum];
depthSequenceOriginal.Add(depthFrame);
depthFrame = backFrame - depthFrame; //Can't find Absolute value function
DenseMatrix depthFrameTest = Threshold(depthFrame, 3000, 100);
ArrayList CC = BWConnComp(depthFrameTest, 512, 424);
//ArrayList is a list of objects that each contain clusters with the indeces and number of pixels
//Goal 1: Find the cluster with the most pixels and store that object
BWConnReturn maxCluster = new BWConnReturn();
for (int i = 0; i < CC.Count; i++)
{
if (((BWConnReturn)CC[i]).size > maxCluster.size)
{
maxCluster = (BWConnReturn)CC[i];
}
}
//Goal 2: set the indices to 1 and everything else to 0
depthFrameTest.Clear();
for (int i = 0; i < maxCluster.indices.Count; i++)
{
//get row and column from index
int row = (int)((int)maxCluster.indices[i] / depthFrame.ColumnCount);
int column = (int)((int)maxCluster.indices[i] % depthFrame.ColumnCount);
//set to 1
depthFrameTest[row, column] = 1;
}
//Now layer the original depthFrame over the new silhouette
depthFrameTest.PointwiseMultiply((DenseMatrix)depthSequenceOriginal[frameNum], depthFrame);
//Feedback loop that applies thresholds until satisfied with the result
bool goodClump = false;
int iterations = 0;
while (goodClump == false)
{
int threshold = 0;
//Get Hand from First Frame
if (frameNum == 1)
{
ArrayList goodIndices = findIndices(depthFrame, startThresh);
threshold = goodIndices.OfType <int>().Min();
downThresh = threshold - 200;
upThresh = threshold + startUpThresh;
}
else if (bad) //Re-Threshold Frames if first try doesn't work
{
if (startThresh > 1500)
{
ArrayList goodIndices = findIndices(depthFrame, (int)(maxThresh + minThresh) / 2);
threshold = goodIndices.OfType <int>().Min();
}
else
{
ArrayList goodIndices = findIndices(depthFrame, startThresh);
threshold = goodIndices.OfType <int>().Min();
startUpThresh = 200;
goodClump = true;
}
downThresh = threshold - 200;
}
//Threshold
depthFrameTest = Threshold(depthFrame, upThresh, downThresh);
//Connect Components
CC = BWConnComp(depthFrameTest, 512, 424);
//Find Max+IDs
maxClusterFinal = new BWConnReturn();
for (int i = 0; i < CC.Count; i++)
{
if (((BWConnReturn)CC[i]).size > maxClusterFinal.size)
{
maxClusterFinal = (BWConnReturn)CC[i];
}
}
//get Difference
int numP = maxClusterFinal.size;
int difference = numPlast - numP;
//Feedback steps for the first frame
if (frameNum == 1)
{
if ((numP > 4500) && (numP < 5500))
{
goodClump = true; //Decent sized clump
numPlast = numP;
}
else if (numP > 5500)
{
startUpThresh -= 10;
}
else if (numP < 4500)
{
startThresh += 20;
}
}
//Feedback for other frames based on difference in size of frame before it, also not too small
else if ((Math.Abs(difference) > 600) || (numP < 1600))
{
bad = true;
//Check to see if anything significant was caught
if ((difference > 0) || (numP < 1600))
{
maxThresh += 2;
upThresh = maxThresh;
}
else
{
maxThresh -= 2;
upThresh = maxThresh;
}
//Gotta stop sometime
iterations++;
if (iterations == 3000)
{
goodClump = true;
}
}
else
{
goodClump = true;
numPlast = numP;
}
}
//Set indices to 1
depthFrame.Clear();
for (int i = 0; i < maxClusterFinal.indices.Count; i++)
{
//get row and column from index
int row = (int)((int)maxClusterFinal.indices[i] / depthFrame.ColumnCount);
int column = (int)((int)maxClusterFinal.indices[i] % depthFrame.ColumnCount);
//set to 1
depthFrame[row, column] = 1;
}
//Todo: Fill the Holes
//
//
//Layer Depths onto silhouette to extract the max and min depths
DenseMatrix depthFrameLayered = new DenseMatrix(424, 512);
((DenseMatrix)depthSequenceOriginal[frameNum]).PointwiseMultiply(depthFrame, depthFrameLayered);
ArrayList greaterThanZeros = findIndices(depthFrameLayered, 0);
maxThresh = greaterThanZeros.OfType <int>().Max();
minThresh = greaterThanZeros.OfType <int>().Min();
startThresh = (int)((maxThresh + minThresh) / 2);
upThresh = maxThresh;
downThresh = minThresh - 300;
depthSequence.Add(depthFrame);
}
return(depthSequence);
}
/* Desired Functions:
* Connect Components
* Extract Hand
* Get Shape Descriptors
* Get Covariance Matrix (in Math.Net Possibly?)
* reshape
* getTfeatures
*/
/// <summary>
/// Returns a 2d int array with number of connect components and indices of connected components
/// </summary>
/// <param name="array"></param>
/// <param name="width"></param>
/// <param name="height"></param>
/// <returns></returns>
public static ArrayList BWConnComp(DenseMatrix array, int width, int height)
{
ArrayList nodes = new ArrayList();
ArrayList roots = new ArrayList();
ArrayList returnArray = new ArrayList();
//Make each pixel a separate node
for (int i = 0; i < array.RowCount; i++)
{
for (int j = 0; j < array.ColumnCount; j++)
{
ConnNode pixNode = new ConnNode(i * array.ColumnCount + j);
if (array[i, j] == 1)
{
pixNode.setWhite();
}
nodes.Add(pixNode);
}
}
//iterate through the nodes
foreach (ConnNode node in nodes)
{
//check if node's white
if (node.white == 1)
{
//first check if root
if (node.parent == null)
{
roots.Add(node);
}
//now check up,down,left,right
//
//up
//first check if in top row
if (!(node.id < width))
{
ConnNode newNode = (ConnNode)nodes[node.id - width];
//check if parent or inspected already
if (!(newNode.inspected))
{
//check if white
if (newNode.white == 1)
{
node.AddChild(newNode);
newNode.inspected = true;
}
}
}
//down
//first check if in bottom row
if (!(node.id >= (width * (height - 1))))
{
ConnNode newNode = (ConnNode)nodes[node.id + width];
//check if parent or inspected already
if (!(newNode.inspected))
{
//check if white
if (newNode.white == 1)
{
node.AddChild(newNode);
newNode.inspected = true;
}
}
}
//left
//first check if in left column
if (node.id % width != 0)
{
ConnNode newNode = (ConnNode)nodes[node.id - 1];
//check if parent or inspected already
if (!(newNode.inspected))
{
//check if white
if (newNode.white == 1)
{
node.AddChild(newNode);
newNode.inspected = true;
}
}
}
//right
//first check if in right column
if ((node.id + 1) % width != 0)
{
ConnNode newNode = (ConnNode)nodes[node.id + 1];
//check if parent or inspected already
if (!(newNode.inspected))
{
//check if white
if (newNode.white == 1)
{
node.AddChild(newNode);
newNode.inspected = true;
}
}
}
}
node.inspected = true;
}
//Go through each root and find size and indices
foreach (ConnNode newnode in roots)
{
//make new BWConnReturn
BWConnReturn values = new BWConnReturn();
//Get size of tree
values.size = newnode.GetSize();
//now get indices
values.indices = newnode.GetIndices();
//now add returntype to array
returnArray.Add(values);
}
return(returnArray);
} //checked
/* Desired Functions:
* Connect Components
* Extract Hand
* Get Shape Descriptors
* Get Covariance Matrix (in Math.Net Possibly?)
* reshape
* getTfeatures
*/
/// <summary>
/// Returns a 2d int array with number of connect components and indices of connected components
/// </summary>
/// <param name="array"></param>
/// <param name="width"></param>
/// <param name="height"></param>
/// <returns></returns>
public static List <BWConnReturn> BWConnComp(DenseMatrix array, int width, int height)
{
List <ConnNode> nodes = new List <ConnNode>();
List <ConnNode> roots = new List <ConnNode>();
string outstring = "";
for (int i = 0; i < array.ColumnCount; i++)
{
for (int j = 0; j < array.RowCount; j++)
{
outstring += array[j, i].ToString() + ",";
}
}
List <BWConnReturn> returnArray = new List <BWConnReturn>();
//Make each pixel a separate node
for (int i = 0; i < array.ColumnCount; i++)
{
for (int j = 0; j < array.RowCount; j++)
{
ConnNode pixNode = new ConnNode(i * array.RowCount + j);
if (array[j, i] == 1)
{
pixNode.setWhite();
}
nodes.Add(pixNode);
}
}
//iterate through the nodes
foreach (ConnNode node in nodes)
{
//check if node's white
if ((node.white == 1))
{
//first check if root
if (node.parent == null)
{
roots.Add(node);
}
//now check up,down,left,right
//
//up
//first check if in top row
if (!(node.id % height == 0))
{
ConnNode newNode = (ConnNode)nodes[node.id - 1];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//down
//first check if in bottom row
if (!((node.id + 1) % height == 0))
{
ConnNode newNode = (ConnNode)nodes[node.id + 1];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//left
//first check if in left column
if (node.id >= height)
{
ConnNode newNode = (ConnNode)nodes[node.id - height];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//right
//first check if in right column
if (node.id < (height * (width - 1)))
{
ConnNode newNode = (ConnNode)nodes[node.id + height];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//NW
if ((node.id >= height) && (!(node.id % height == 0)))
{
ConnNode newNode = (ConnNode)nodes[node.id - height - 1];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//NE
if ((node.id < (height * (width - 1))) && (!(node.id % height == 0)))
{
ConnNode newNode = (ConnNode)nodes[node.id + height - 1];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//SE
if ((node.id < (height * (width - 1))) && (!((node.id + 1) % height == 0)))
{
ConnNode newNode = (ConnNode)nodes[node.id + height + 1];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
//SW
if ((node.id >= height) && (!((node.id + 1) % height == 0)))
{
ConnNode newNode = (ConnNode)nodes[node.id - height + 1];
//check if white
if (newNode.white == 1)
{
//inspected already
if (!(newNode.inspected))
{
node.AddChild(newNode);
newNode.inspected = true;
}
else
{
//compare roots. if Different roots, combine trees
ConnNode root1 = node;
ConnNode root2 = newNode;
while (root1.parent != null)
{
root1 = root1.parent;
}
while (root2.parent != null)
{
root2 = root2.parent;
}
if (root1.id != root2.id)
{
root1.AddChild(root2);
roots.Remove(root2);
}
}
}
}
}
node.inspected = true;
}
//Go through each root and find size and indices
foreach (ConnNode newnode in roots)
{
//make new BWConnReturn
BWConnReturn values = new BWConnReturn();
//Get size of tree
values.size = newnode.GetSize();
//now get indices
values.indices = newnode.GetIndices();
//now add returntype to array
returnArray.Add(values);
}
return(returnArray);
} //checked
