private void flandmark_detect(ref Mat img, int[] bbox, ref FlandmarkModel model, out double[] landmarks, int[] bw_margin = null)
{
landmarks = new double[model.Data.Options.M * 2];
if (bw_margin != null)
{
model.Data.Options.bw_margin[0] = bw_margin[0];
model.Data.Options.bw_margin[1] = bw_margin[1];
}
if (!flandmark_get_normalized_image_frame(ref img, bbox, ref model.bb, ref model.NormalizedImageFrame, ref model))
{
landmarks = null;
return;
}
flandmark_detect_base(model.NormalizedImageFrame, model, landmarks);
model.sf[0] = (float)(model.bb[2] - model.bb[0]) / model.Data.Options.bw[0];
model.sf[1] = (float)(model.bb[3] - model.bb[1]) / model.Data.Options.bw[1];
for (int i = 0; i < 2 * model.Data.Options.M; i += 2)
{
landmarks[i] = landmarks[i] * model.sf[0] + model.bb[0];
landmarks[i + 1] = landmarks[i + 1] * model.sf[1] + model.bb[1];
}
}
private void flandmark_get_psi_mat_sparse(ref FlandmarkPSISparse Psi, ref FlandmarkModel model, int lbpidx)
{
//uint[] Features;
byte[] Images = model.NormalizedImageFrame;
uint im_H = (uint)model.Data.ImSize[0];
uint im_W = (uint)model.Data.ImSize[1];
uint[] Wins = model.Data.LBP[lbpidx].Wins;
UInt16 win_H = (UInt16)model.Data.LBP[lbpidx].WinSize[0];
UInt16 win_W = (UInt16)model.Data.LBP[lbpidx].WinSize[1];
UInt16 nPyramids = model.Data.LBP[lbpidx].HOP;
uint nDim = LibLBP.PyrGetDim(win_H, win_W, nPyramids) / 256;
uint nData = model.Data.LBP[lbpidx].WinsCols;
uint cnt0, mirror, x, x1, y, y1, idx;
uint[] win;
if (Psi.Idxs == null) // Psi.idxs.Length != nDim * nData)
{
Psi.Idxs = new uint[nDim * nData];
}
win = new uint[win_H * win_W];
for (uint i = 0; i < nData; ++i)
{
idx = Wins[GetIndex(0, (int)i, 4)] - 1;
x1 = Wins[GetIndex(1, (int)i, 4)] - 1;
y1 = Wins[GetIndex(2, (int)i, 4)] - 1;
mirror = Wins[GetIndex(3, (int)i, 4)];
int img_ptr = (int)(idx * im_H * im_W);
cnt0 = 0;
if (mirror == 0)
{
for (x = x1; x < x1 + win_W; x++)
{
for (y = y1; y < y1 + win_H; y++)
{
win[cnt0++] = Images[img_ptr + GetIndex((int)y, (int)x, (int)im_H)];
}
}
}
else
{
for (x = x1 + win_W - 1; x >= x1; x--)
{
for (y = y1; y < y1 + win_H; y++)
{
win[cnt0++] = Images[img_ptr + GetIndex((int)y, (int)x, (int)im_H)];
}
}
}
LibLBP.PyrFeaturesSparse(ref Psi.Idxs, nDim, win, win_H, win_W, nDim * i);
}
Psi.PsiCols = nData;
Psi.PsiRows = nDim;
//Psi.idxs = Features;
}
private void flandmark_detect_base(byte[] face_image, FlandmarkModel model, double[] landmarks)
{
Profiler.Start("flandmark_detect_base");
int M = model.Data.Options.M;
double[] W = model.W;
int tsize = -1, cols = -1, rows = -1;
int[] mapTable = model.Data.MapTable;
if (model.NormalizedImageFrame == null)
{
model.NormalizedImageFrame = face_image;
}
if (Cached_Psi_sparse == null)
{
Cached_Psi_sparse = new FlandmarkPSISparse[M];
}
Profiler.Start("flandmark_get_psi_mat_sparse");
Parallel.For(0, M, new ParallelOptions()
{
MaxDegreeOfParallelism = M
}, (idx) =>
{
if (Cached_Psi_sparse[idx] == null)
{
Cached_Psi_sparse[idx] = new FlandmarkPSISparse();
}
flandmark_get_psi_mat_sparse(ref Cached_Psi_sparse[idx], ref model, idx);
});
Profiler.End("flandmark_get_psi_mat_sparse");
//for (int idx = 0; idx < M; idx++)
//{
// Psi_sparse[idx] = new FLANDMARK_PSI_SPARSE();
// flandmark_get_psi_mat_sparse(ref Psi_sparse[idx], ref model, idx);
//}
List <double[]> q = new List <double[]>(M);
for (int i = 0; i < M; i++)
{
q.Add(null);
}
List <double[]> g = new List <double[]>(M - 1);
for (int i = 0; i < M - 1; i++)
{
g.Add(null);
}
int idx_qtemp = 0;
for (int idx = 0; idx < M; ++idx)
{
tsize = mapTable[GetIndex(idx, 1, M)] - mapTable[GetIndex(idx, 0, M)] + 1;
if (q_temp == null)
{
q_temp = new double[tsize];
}
else
{
if (q_temp.Length < tsize)
{
q_temp = new double[tsize];
}
}
Array.Copy(W, mapTable[GetIndex(idx, 0, M)] - 1, q_temp, 0, tsize);
// sparse dot product <W_q, PSI_q>
cols = (int)Cached_Psi_sparse[idx].PsiCols;
rows = (int)Cached_Psi_sparse[idx].PsiRows;
uint[] psi_temp = Cached_Psi_sparse[idx].Idxs;
//q[idx] = new double[cols];
double[] qind = new double[cols];
for (int i = 0; i < cols; ++i)
{
double dotprod = 0.0f;
for (int j = 0; j < rows; ++j)
{
idx_qtemp = (int)psi_temp[(rows * i) + j];
dotprod += q_temp[idx_qtemp];
}
qind[i] = dotprod;
}
q[idx] = qind;
if (idx > 0)
{
tsize = mapTable[GetIndex(idx, 3, M)] - mapTable[GetIndex(idx, 2, M)] + 1;
g[idx - 1] = new double[tsize];
Array.Copy(W, mapTable[GetIndex(idx, 2, M)] - 1, g[idx - 1], 0, tsize);
}
}
Profiler.Start("flandmark_argmax");
flandmark_argmax(landmarks, ref model.Data.Options, mapTable, Cached_Psi_sparse, q, g);
Profiler.End("flandmark_argmax");
g.Clear();
q.Clear();
Profiler.End("flandmark_detect_base");
}
private bool flandmark_get_normalized_image_frame(ref Mat input, int[] bbox, ref double[] bb, ref byte[] face_img, ref FlandmarkModel model)
{
bool flag;
int[] d = new int[2];
double[] c = new double[2], nd = new double[2];
// extend bbox by bw_margin
d[0] = bbox[2] - bbox[0] + 1; d[1] = bbox[3] - bbox[1] + 1;
c[0] = (bbox[2] + bbox[0]) / 2.0f; c[1] = (bbox[3] + bbox[1]) / 2.0f;
nd[0] = d[0] * model.Data.Options.bw_margin[0] / 100.0f + d[0];
nd[1] = d[1] * model.Data.Options.bw_margin[1] / 100.0f + d[1];
bb[0] = (c[0] - nd[0] / 2.0f);
bb[1] = (c[1] - nd[1] / 2.0f);
bb[2] = (c[0] + nd[0] / 2.0f);
bb[3] = (c[1] + nd[1] / 2.0f);
//flag = bb[0] > 0 && bb[1] > 0 && bb[2] < input.Width && bb[3] < input.Height
// && bbox[0] > 0 && bbox[1] > 0 && bbox[2] < input.Width && bbox[3] < input.Height;
//if (!flag)
// return false;
Rect region = new Rect((int)bb[0], (int)bb[1], (int)bb[2] - (int)bb[0] + 1, (int)bb[3] - (int)bb[1] + 1);
flag = input.Width <= 0 || input.Height <= 0 || region.Width <= 0 || region.Height <= 0;
if (flag)
{
return(false);
}
Rect clipRegion = new Rect(
Util.Clamp(region.X, 0, input.Width - 1), Util.Clamp(region.Y, 0, input.Height - 1),
Util.Clamp(region.Width, 0, input.Width), Util.Clamp(region.Height, 0, input.Height));
using (var resizedImage = new Mat(input, clipRegion.ToCvRect()))
{
double scalefactor = model.Data.Options.bw[0] / region.Width;
//resizedImage.Resize(new Size(model.Data.Options.bw[0], model.Data.Options.bw[1]), 0, 0, Inter);
resizedImage.Resize(new Size(clipRegion.Width * scalefactor, clipRegion.Height * scalefactor), 0, 0, Inter);
resizedImage.EqualizeHistogram();
// TODO: parallized
int step = model.Data.Options.bw[1];
byte[] face_img_tmp = face_img;
double regX = region.X, regY = region.Y, inpW = input.Width, inpH = input.Height;
Parallel.For(0, model.Data.Options.bw[0] * model.Data.Options.bw[1], new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
}, (ind) =>
{
int x = ind / step;
int y = ind % step;
double preX = x / scalefactor + regX;
double preY = y / scalefactor + regY;
byte value;
// TODO: padding
if (preX < 0 || preX >= inpW || preY < 0 || preY >= inpH)
{
value = 1;
}
else
{
int newY = (int)((Util.Clamp(preY, 0, inpH) - clipRegion.Y) * scalefactor);
int newX = (int)((Util.Clamp(preX, 0, inpW) - clipRegion.X) * scalefactor);
value = resizedImage.At <byte>(newY, newX);
}
face_img_tmp[x * step + y] = value;
});
face_img = face_img_tmp;
//for (int x = 0; x < model.Data.Options.bw[0]; ++x)
//{
// for (int y = 0; y < model.Data.Options.bw[1]; ++y)
// {
// face_img[GetIndex(y, x, model.Data.Options.bw[1])] = resizedImage.At<byte>(y, x);
// }
//}
}
return(true);
}
public Flandmark(FileNode filename)
{
int[] p_int;
int tsize = -1, tmp_tsize = -1;
Stream fin = filename.Open();
StreamReader reader = new StreamReader(fin);
FlandmarkModel tst = new FlandmarkModel();
fin.Position = 0;
ReadUntilSpace(reader);
tst.Data.Options.M = (byte)ReadUntilSpace(reader)[0];
ReadUntilSpace(reader);
tst.Data.Options.bw[0] = Convert.ToInt32(ReadUntilSpace(reader));
tst.Data.Options.bw[1] = Convert.ToInt32(ReadUntilSpace(reader));
ReadUntilSpace(reader);
tst.Data.Options.bw_margin[0] = Convert.ToInt32(ReadUntilSpace(reader));
tst.Data.Options.bw_margin[1] = Convert.ToInt32(ReadUntilSpace(reader));
ReadUntilSpace(reader);
tst.WRows = Convert.ToInt32(ReadUntilSpace(reader));
tst.WCols = Convert.ToInt32(ReadUntilSpace(reader));
ReadUntilSpace(reader);
tst.Data.ImSize[0] = Convert.ToInt32(ReadUntilSpace(reader));
tst.Data.ImSize[1] = Convert.ToInt32(ReadUntilSpace(reader));
int M = tst.Data.Options.M;
tst.Data.LBP = new FlandmarkLBP[M];
for (int i = 0; i < M; i++)
{
ReadUntilSpace(reader);
var lbp = new FlandmarkLBP();
lbp.WinsRows = (uint)Convert.ToInt32(ReadUntilSpace(reader));
lbp.WinsCols = (uint)Convert.ToInt32(ReadUntilSpace(reader));
tst.Data.LBP[i] = lbp;
}
for (int i = 0; i < 3; i++)
{
ReadUntilSpace(reader);
tst.Data.Options.PsiGRows[i] = Convert.ToInt32(ReadUntilSpace(reader));
tst.Data.Options.PsiGCols[i] = Convert.ToInt32(ReadUntilSpace(reader));
}
fin.Dispose();
reader.Dispose();
fin = filename.Open();
//TODO: fix position
fin.Position = 111;
BinaryReader breader = new BinaryReader(fin);
// load model.W
tst.W = new double[tst.WRows];
for (int i = 0; i < tst.WRows; i++)
{
tst.W[i] = breader.ReadDouble();
}
// load model.data.mapTable
p_int = new int[M * 4];
tst.Data.MapTable = new int[M * 4];
for (int i = 0; i < M * 4; i++)
{
p_int[i] = breader.ReadInt32();
}
for (int i = 0; i < M * 4; i++)
{
tst.Data.MapTable[i] = p_int[i];
}
p_int = null;
// load model.data.lbp
for (int i = 0; i < M; i++)
{
// lbp{idx}.winSize
p_int = new int[2];
p_int[0] = breader.ReadInt32();
p_int[1] = breader.ReadInt32();
tst.Data.LBP[i].WinSize[0] = p_int[0];
tst.Data.LBP[i].WinSize[1] = p_int[1];
p_int = null;
// lbp{idx}.hop
tst.Data.LBP[i].HOP = breader.ReadByte();
// lbp{idx}.wins
tsize = (int)(tst.Data.LBP[i].WinsRows * tst.Data.LBP[i].WinsCols);
tst.Data.LBP[i].Wins = new uint[tsize];
for (int r = 0; r < tsize; r++)
{
tst.Data.LBP[i].Wins[r] = breader.ReadUInt32();
}
}
// load model.options.S
tst.Data.Options.S = new int[4 * M];
for (int i = 0; i < 4 * M; i++)
{
tst.Data.Options.S[i] = breader.ReadInt32();
}
p_int = null;
// load model.options.PsiG
FlandmarkPsiG[] PsiGi = null;
for (int psigs_ind = 0; psigs_ind < 3; psigs_ind++)
{
tsize = tst.Data.Options.PsiGRows[psigs_ind] * tst.Data.Options.PsiGCols[psigs_ind];
switch (psigs_ind)
{
case 0:
tst.Data.Options.PsiGS0 = new FlandmarkPsiG[tsize];
PsiGi = tst.Data.Options.PsiGS0;
break;
case 1:
tst.Data.Options.PsiGS1 = new FlandmarkPsiG[tsize];
PsiGi = tst.Data.Options.PsiGS1;
break;
case 2:
tst.Data.Options.PsiGS2 = new FlandmarkPsiG[tsize];
PsiGi = tst.Data.Options.PsiGS2;
break;
}
int temp = 0;
for (int i = 0; i < tsize; i++)
{
PsiGi[i] = new FlandmarkPsiG();
// disp ROWS
temp = breader.ReadInt32();
PsiGi[i].Rows = temp;
// disp COLS
temp = breader.ReadInt32();
PsiGi[i].Cols = temp;
// disp
tmp_tsize = PsiGi[i].Rows * PsiGi[i].Cols;
PsiGi[i].Disp = new int[tmp_tsize];
for (int r = 0; r < tmp_tsize; r++)
{
PsiGi[i].Disp[r] = breader.ReadInt32();
}
}
}
fin.Dispose();
breader.Dispose();
tst.NormalizedImageFrame = new byte[tst.Data.Options.bw[0] * tst.Data.Options.bw[1]];
tst.bb = new double[4];
tst.sf = new float[2];
Model = tst;
}
