private static void SqlAddImage(Img img)
{
lock (_sqllock) {
using (var sqlCommand = _sqlConnection.CreateCommand()) {
sqlCommand.Connection = _sqlConnection;
var sb = new StringBuilder();
sb.Append($"INSERT INTO {AppConsts.TableImages} (");
sb.Append($"{AppConsts.AttrId}, ");
sb.Append($"{AppConsts.AttrName}, ");
sb.Append($"{AppConsts.AttrHash}, ");
sb.Append($"{AppConsts.AttrVector}, ");
sb.Append($"{AppConsts.AttrYear}, ");
sb.Append($"{AppConsts.AttrCounter}, ");
sb.Append($"{AppConsts.AttrBestId}, ");
sb.Append($"{AppConsts.AttrSig}, ");
sb.Append($"{AppConsts.AttrBestVDistance}, ");
sb.Append($"{AppConsts.AttrLastView}, ");
sb.Append($"{AppConsts.AttrLastCheck}");
sb.Append(") VALUES (");
sb.Append($"@{AppConsts.AttrId}, ");
sb.Append($"@{AppConsts.AttrName}, ");
sb.Append($"@{AppConsts.AttrHash}, ");
sb.Append($"@{AppConsts.AttrVector}, ");
sb.Append($"@{AppConsts.AttrYear}, ");
sb.Append($"@{AppConsts.AttrCounter}, ");
sb.Append($"@{AppConsts.AttrBestId}, ");
sb.Append($"@{AppConsts.AttrSig}, ");
sb.Append($"@{AppConsts.AttrBestVDistance}, ");
sb.Append($"@{AppConsts.AttrLastView}, ");
sb.Append($"@{AppConsts.AttrLastCheck}");
sb.Append(')');
sqlCommand.CommandText = sb.ToString();
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrId}", img.Id);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrName}", img.Name);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrHash}", img.Hash);
var array = Helper.ArrayFrom16(img.Vector);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrVector}", array);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrYear}", img.Year);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrCounter}", img.Counter);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrBestId}", img.BestId);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrSig}", img.Sig);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrBestVDistance}", img.BestVDistance);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrLastView}", img.LastView);
sqlCommand.Parameters.AddWithValue($"@{AppConsts.AttrLastCheck}", img.LastCheck);
sqlCommand.ExecuteNonQuery();
}
}
}
public ImgPanel(Img img, long size, Bitmap bitmap)
{
Img = img;
Size = size;
Bitmap = bitmap;
}
private static void ComputeInternal(BackgroundWorker backgroundworker)
{
Img img1 = null;
Img[] shadowcopy = null;
int activescope = 0;
int sig = 0;
int prevsig = 0;
lock (_imglock) {
if (_imgList.Count < 2)
{
backgroundworker.ReportProgress(0, "no images");
return;
}
shadowcopy = _imgList.Select(e => e.Value).OrderBy(e => e.LastCheck).ToArray();
}
activescope = shadowcopy.Count(e => e.Vector.Length != 0);
sig = _clusters.Rows;
foreach (var img in shadowcopy)
{
if (img.Sig == 0 && img.BestId == 0 || img.BestId == img.Id || !_imgList.ContainsKey(img.BestId))
{
img1 = img;
break;
}
if (img1 == null || img.LastCheck < img1.LastCheck)
{
img1 = img;
}
}
prevsig = img1.Sig;
if (img1.Sig != sig || img1.Vector.Length == 0)
{
var filename = FileHelper.NameToFileName(img1.Name);
var imagedata = FileHelper.ReadData(filename);
if (imagedata == null)
{
Delete(img1.Id);
return;
}
using (var bitmap = BitmapHelper.ImageDataToBitmap(imagedata)) {
if (bitmap == null)
{
Delete(img1.Id);
return;
}
var matrix = BitmapHelper.GetMatrix(imagedata);
var descriptors = SiftHelper.GetDescriptors(matrix);
var vector = ComputeVector(descriptors, backgroundworker);
img1.Vector = vector;
}
img1.Sig = sig;
}
var candidates = shadowcopy.Where(e => e.Id != img1.Id && e.Vector.Length != 0).ToArray();
if (candidates.Length > 0)
{
var bestid = img1.Id;
var bestvdistance = 100f;
for (var i = 0; i < candidates.Length; i++)
{
var img2 = candidates[i];
var vdistance = GetDistance(img1.Vector, img2.Vector);
if (vdistance < bestvdistance)
{
bestid = img2.Id;
bestvdistance = vdistance;
}
}
if (bestid != img1.BestId)
{
_sb.Clear();
_sb.Append($"(n:{candidates.Length}/c:{_clusters.Rows}) a:{_added}/f:{_found}/b:{_bad} [{img1.Id}:{prevsig}] {img1.BestVDistance:F2} \u2192 {bestvdistance:F2}");
backgroundworker.ReportProgress(0, _sb.ToString());
img1.BestId = bestid;
img1.Counter = 0;
}
if (img1.BestVDistance != bestvdistance)
{
img1.BestVDistance = bestvdistance;
}
}
img1.LastCheck = DateTime.Now;
}
private static void ImportInternal(BackgroundWorker backgroundworker)
{
FileInfo fileinfo;
lock (_rwlock) {
if (_rwList.Count == 0)
{
//ComputeInternal(backgroundworker);
return;
}
fileinfo = _rwList.ElementAt(0);
_rwList.RemoveAt(0);
}
/*
* _sb.Clear();
* _sb.Append($"a:{_added}/f:{_found}/b:{_bad}");
* backgroundworker.ReportProgress(0, _sb.ToString());
*/
var orgfilename = fileinfo.FullName;
if (!File.Exists(orgfilename))
{
return;
}
var imagedata = File.ReadAllBytes(orgfilename);
if (imagedata == null || imagedata.Length < 256)
{
FileHelper.MoveCorruptedFile(orgfilename);
_bad++;
return;
}
int year = DateTime.Now.Year;
var orgextension = Path.GetExtension(orgfilename);
if (orgextension.Equals(AppConsts.DatExtension, StringComparison.OrdinalIgnoreCase) ||
orgextension.Equals(AppConsts.MzxExtension, StringComparison.OrdinalIgnoreCase))
{
year = 0;
var password = Path.GetFileNameWithoutExtension(orgfilename);
var decrypteddata = orgextension.Equals(AppConsts.DatExtension, StringComparison.OrdinalIgnoreCase) ?
EncryptionHelper.DecryptDat(imagedata, password) :
EncryptionHelper.Decrypt(imagedata, password);
if (decrypteddata != null)
{
imagedata = decrypteddata;
}
}
var hash = MD5HashHelper.Compute(imagedata);
bool found;
Img imgfound;
lock (_imglock) {
found = _hashList.TryGetValue(hash, out imgfound);
}
if (found)
{
// we found the same image in a database
var filenamefound = FileHelper.NameToFileName(imgfound.Name);
if (File.Exists(filenamefound))
{
// no reason to add the same image from a heap; we have one
FileHelper.DeleteToRecycleBin(orgfilename);
if (imgfound.Year == 0 && year != 0)
{
imgfound.Year = year;
}
_found++;
return;
}
// found image is gone; delete it
Delete(imgfound.Id);
}
var bitmap = BitmapHelper.ImageDataToBitmap(imagedata);
if (bitmap == null)
{
var badname = Path.GetFileName(orgfilename);
var badfilename = $"{AppConsts.PathGb}\\{badname}{AppConsts.CorruptedExtension}";
if (File.Exists(badfilename))
{
FileHelper.DeleteToRecycleBin(badfilename);
}
File.WriteAllBytes(badfilename, imagedata);
FileHelper.DeleteToRecycleBin(orgfilename);
_bad++;
return;
}
bitmap.Dispose();
// we have to create unique name and a location in Hp folder
string newname;
string newfilename;
var iteration = -1;
do
{
iteration++;
newname = FileHelper.HashToName(hash, iteration);
newfilename = FileHelper.NameToFileName(newname);
} while (File.Exists(newfilename));
//var lv = GetMinLastView();
var lc = GetMinLastCheck();
var id = AllocateId();
var nimg = new Img(
id: id,
name: newname,
hash: hash,
vector: Array.Empty <ushort>(),
year: year,
counter: 0,
bestid: 0,
sig: 0,
bestvdistance: 100f,
lastview: new DateTime(2020, 1, 1),
lastcheck: lc);
Add(nimg);
var lastmodified = File.GetLastWriteTime(orgfilename);
if (lastmodified > DateTime.Now)
{
lastmodified = DateTime.Now;
}
if (!orgfilename.Equals(newfilename, StringComparison.OrdinalIgnoreCase))
{
FileHelper.WriteData(newfilename, imagedata);
File.SetLastWriteTime(newfilename, lastmodified);
FileHelper.DeleteToRecycleBin(orgfilename);
}
_added++;
}
public static void Find(int idX, IProgress <string> progress)
{
Img imgX = null;
do
{
lock (_imglock) {
if (_imgList.Count < 2)
{
progress.Report("No images to view");
return;
}
}
if (idX == 0)
{
imgX = null;
Img[] valid;
lock (_imglock)
{
valid = _imgList
.Where(
e => e.Value.BestId != 0 &&
e.Value.BestId != e.Value.Id &&
_imgList.ContainsKey(e.Value.BestId))
.Select(e => e.Value)
.ToArray();
}
if (valid.Length == 0)
{
progress.Report("No images to view");
return;
}
var mincounter = valid.Min(e => e.Counter);
var scope = valid.Where(e => e.Counter == mincounter).ToArray();
var scopelast = scope.Where(e => e.LastView.Year == 2020).ToArray();
if (scopelast.Length > 0)
{
scope = scopelast;
}
var rscope = scope.OrderBy(e => e.BestVDistance).Take(1000).ToArray();
var rindex = _random.Next(0, rscope.Length - 1);
imgX = rscope[rindex];
//imgX = valid.Where(e => e.Counter == mincounter).OrderBy(e => e.BestVDistance).FirstOrDefault();
//imgX = scope[rindex];
idX = imgX.Id;
if (!_imgList.TryGetValue(imgX.BestId, out var imgBest))
{
Delete(imgX.BestId);
progress.Report($"{imgX.BestId} deleted");
idX = 0;
continue;
}
}
AppVars.ImgPanel[0] = GetImgPanel(idX);
if (AppVars.ImgPanel[0] == null)
{
Delete(idX);
progress.Report($"{idX} deleted");
idX = 0;
continue;
}
imgX = AppVars.ImgPanel[0].Img;
var idY = imgX.BestId;
AppVars.ImgPanel[1] = GetImgPanel(idY);
if (AppVars.ImgPanel[1] == null)
{
Delete(idY);
progress.Report($"{idY} deleted");
idX = 0;
continue;
}
break;
}while (true);
lock (_imglock) {
var imgcount = _imgList.Count;
var diff = imgcount - _importLimit;
var g0 = _imgList.Count(e => e.Value.Counter == 0);
var g1 = _imgList.Count(e => e.Value.Counter == 1);
progress.Report($"0:{g0} 1:{g1} images:{imgcount}({diff}) distance: {imgX.BestVDistance:F2}");
}
}
public static void LoadImages(IProgress <string> progress)
{
lock (_imglock) {
_imgList.Clear();
var sb = new StringBuilder();
sb.Append("SELECT ");
sb.Append($"{AppConsts.AttrId}, "); // 0
sb.Append($"{AppConsts.AttrName}, "); // 1
sb.Append($"{AppConsts.AttrHash}, "); // 2
sb.Append($"{AppConsts.AttrVector}, "); // 3
sb.Append($"{AppConsts.AttrYear}, "); // 4
sb.Append($"{AppConsts.AttrCounter}, "); // 5
sb.Append($"{AppConsts.AttrBestId}, "); // 6
sb.Append($"{AppConsts.AttrSig}, "); // 7
sb.Append($"{AppConsts.AttrBestVDistance}, "); // 8
sb.Append($"{AppConsts.AttrLastView}, "); // 9
sb.Append($"{AppConsts.AttrLastCheck} "); // 10
sb.Append($"FROM {AppConsts.TableImages}");
var sqltext = sb.ToString();
lock (_sqllock) {
using (var sqlCommand = _sqlConnection.CreateCommand()) {
sqlCommand.Connection = _sqlConnection;
sqlCommand.CommandText = sqltext;
using (var reader = sqlCommand.ExecuteReader()) {
var dtn = DateTime.Now;
while (reader.Read())
{
var id = reader.GetInt32(0);
var name = reader.GetString(1);
var hash = reader.GetString(2);
var array = (byte[])reader[3];
var vector = Helper.ArrayTo16(array);
var year = reader.GetInt32(4);
var counter = reader.GetInt32(5);
var bestid = reader.GetInt32(6);
var sig = reader.GetInt32(7);
var bestvdistance = reader.GetFloat(8);
var lastview = reader.GetDateTime(9);
var lastcheck = reader.GetDateTime(10);
var img = new Img(
id: id,
name: name,
hash: hash,
vector: vector,
year: year,
counter: counter,
bestid: bestid,
sig: sig,
bestvdistance: bestvdistance,
lastview: lastview,
lastcheck: lastcheck
);
AddToMemory(img);
if (DateTime.Now.Subtract(dtn).TotalMilliseconds > AppConsts.TimeLapse)
{
dtn = DateTime.Now;
if (progress != null)
{
progress.Report($"Loading images ({_imgList.Count})...");
}
}
}
}
}
if (progress != null)
{
progress.Report("Loading vars...");
}
_id = 0;
sb.Length = 0;
sb.Append("SELECT ");
sb.Append($"{AppConsts.AttrId}, "); // 0
sb.Append($"{AppConsts.AttrImportLimit}, "); // 1
sb.Append($"{AppConsts.AttrClusters} "); // 2
sb.Append($"FROM {AppConsts.TableVars}");
sqltext = sb.ToString();
lock (_sqllock) {
using (var sqlCommand = new SqlCommand(sqltext, _sqlConnection)) {
using (var reader = sqlCommand.ExecuteReader()) {
while (reader.Read())
{
_id = reader.GetInt32(0);
_importLimit = reader.GetInt32(1);
var clusters = (byte[])reader[2];
var floatclusters = Helper.ArrayToFloat(clusters);
_clusters = new Mat(floatclusters.Length / 128, 128, MatType.CV_32F);
_clusters.SetArray(floatclusters);
break;
}
}
}
}
if (progress != null)
{
progress.Report("Database loaded");
}
}
}
}
