public virtual void Build(MetricDB db, ANNISetup setup)
{
// num_build_processors = 1;
this.DB = db;
var rows = new List<ANNI> ();
var pivsel = new PivotSelectorRandom(db.Count, RandomSets.GetRandom());
this.leader = new NANNI();
var ilc = new ANNI();
var cost = ilc.InternalBuild (setup, 0, 1.0, db, 2);
this.leader.Build (ilc);
int m = this.leader.clusters.Count;
double review_prob = cost.SingleCost - m; review_prob /= this.DB.Count;
var min_prob = Math.Sqrt (this.DB.Count) / this.DB.Count;
while (review_prob > min_prob) {
var row = new ANNI ();
rows.Add (row);
var _cost = row.InternalBuild (setup, m, review_prob, db, 2);
var _m = row.ACT.Count;
review_prob *= (_cost.SingleCost - _m) / this.DB.Count;
}
this.rows = rows.ToArray ();
}
public virtual void Build(MetricDB db, ANNISetup setup, int num_indexes, int num_tasks = -1)
{
// num_build_processors = 1;
this.DB = db;
--num_indexes;
this.rows = new ANNI[num_indexes];
var pivsel = new PivotSelectorRandom(db.Count, RandomSets.GetRandom());
this.leader = new NANNI();
var ilc = new ANNI();
var cost = ilc.InternalBuild (setup, 0, 1.0, db, 2);
this.leader.Build (ilc);
int m = this.leader.clusters.Count;
double review_prob = cost.SingleCost - m; review_prob /= this.DB.Count;
// ParallelOptions ops = new ParallelOptions ();
// ops.MaxDegreeOfParallelism = num_tasks;
// Parallel.For (0, num_indexes, ops, (int i) => {
// this.rows [i] = new ILC ();
// this.rows [i].InternalBuild (m, review_prob, db, num_indexes, pivsel);
// });
Console.WriteLine ("====> num_indexes: {0}", num_indexes);
LongParallel.For (0, num_indexes, (int i) => {
this.rows [i] = new ANNI ();
this.rows [i].InternalBuild (setup, m, review_prob, db, num_indexes);
}, num_tasks);
}
public void Build(MetricDB db, ANNISetup setup, int num_indexes, int num_tasks = -1)
{
// num_build_processors = 1;
this.DB = db;
var _rows = new ANNI[num_indexes];
LongParallel.For (0, num_indexes, (int i) => {
_rows [i] = new ANNI ();
_rows [i].InternalBuild (setup, 0, 1.0, db, num_indexes);
}, num_tasks);
// ParallelOptions ops = new ParallelOptions ();
// ops.MaxDegreeOfParallelism = num_processors;
// Parallel.For (0, num_indexes, ops, (int i) => {
// _rows [i] = new ILC ();
// _rows [i].Build (db, num_indexes, pivsel);
// });
this.leader = new NANNI();
this.leader.Build(_rows [0]);
this.rows = new ANNI[num_indexes - 1];
for (int i = 1; i < num_indexes; ++i) {
this.rows[i - 1] = _rows[i];
}
}
public static ANNI CreateANNI(string dbname, int expected_k)
{
MetricDB db = SpaceGenericIO.Load (dbname);
ANNI ilc = new ANNI ();
var annisetup = new ANNISetup (db.Count, expected_k);
ilc.Build (db, annisetup);
return ilc;
}
public static string ExecuteDMANNI(IndexArgumentSetup setup, string nick, int num_indexes)
{
var idxname = String.Format ("{0}/Index.DMANNI.{1}", nick, num_indexes);
return Execute (setup, nick, idxname, (db) => {
var milc = new XMANNI ();
// var annisetup = new ANNISetup(new PivotSelectorRandom(db.Count), (int)Math.Abs(setup.QARG), 0.001, 512, 64);
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
milc.Build (db, annisetup, num_indexes, true);
return milc;
});
}
public void Build(MetricDB db, ANNISetup setup, int num_indexes, bool optimizeDistances)
{
this.DB = db;
var k = 1 + setup.ExpectedK; // since we use items from the database as training queries
// select the queries to test the construction
var qlist = RandomSets.GetRandomSubSet(setup.NumberQueries, this.DB.Count);
this.leader = new XNANNI ();
this.leader.PartialBuild (db, setup.Selector);
this.rows = new TANNI[num_indexes - 1];
for (int i = 0; i < this.rows.Length; ++i) {
this.rows [i] = new TANNI ();
this.rows [i].PartialBuild (db, setup.Selector);
}
//int step_width = 128;
double currT = long.MaxValue;
double prevT = 0;
double currD = this.DB.Count;
double prevD = 0;
int iter = 0;
Console.WriteLine("xxxxxxxx BEGIN> db: {0}, indexes: {1}, setup: {2}",
Path.GetFileName(this.DB.Name), num_indexes, setup);
do {
this.PromotePivots(setup);
prevT = currT;
prevD = currD;
currT = DateTime.Now.Ticks;
foreach (var qID in qlist) {
var q = this.DB[qID];
var res = new Result(k);
currD += this.InternalSearchKNN(q, k, res);
}
currT = DateTime.Now.Ticks - currT;
currT /= qlist.Length;
currD /= qlist.Length;
++iter;
Console.WriteLine ("======> iter: {0}, timestamp: {1}, setup: {2}", iter, DateTime.Now, setup);
Console.WriteLine ("------> prevT: {0}, currT: {1}, prevT / currT: {2}", prevT, currT, prevT / currT);
Console.WriteLine ("------> prevD: {0}, currD: {1}, prevD / currD: {2}", prevD, currD, prevD / currD);
if (optimizeDistances) {
if (prevD < currD * (1 + setup.AlphaStop)) {
break;
}
} else {
if (prevT < currT * (1 + setup.AlphaStop)) {
break;
}
}
} while (true);
}
void PromotePivots(ANNISetup setup)
{
int step_width = setup.StepWidth / (this.rows.Length + 1);
LongParallel.For (0, this.rows.Length + 1, (i) => {
if (i == 0) {
for (int s = 0; s < step_width; ++s) {
int nextPivot = setup.Selector.NextPivot();
this.leader.PromoteObjectToPivot(nextPivot);
}
} else {
var idx = this.rows [i-1];
for (int s = 0; s < step_width; ++s) {
int nextPivot = setup.Selector.NextPivot();
idx.PromoteObjectToPivot(nextPivot);
}
}
});
}
public BuildSearchCost InternalBuild(ANNISetup setup, int leader_num_centers, double leader_review_prob, MetricDB db, int num_indexes)
{
this.DB = db;
int n = this.DB.Count;
var k = 1 + setup.ExpectedK; // since we use items from the database as training queries
var step_width = setup.StepWidth / num_indexes + 1;
this.ACT = new List<int>(256); // just a starting size
this.CT = new int[n];
this.DT = new double[n];
var pivID = 0;
this.ACT.Add (setup.Selector.NextPivot());
var piv = this.DB [this.ACT[pivID]];
for (int docID = 0; docID < n; ++docID) {
var d = this.DB.Dist (this.DB [docID], piv);
this.CT [docID] = pivID;
this.DT [docID] = d;
}
var cache = new Dictionary<int, double> (256);
var qlist = RandomSets.GetRandomSubSet(setup.NumberQueries, this.DB.Count);
Console.WriteLine("xxxxxxxx BEGIN> db: {0}, setup: {1}, indexes: {2}, timestamp: {3}",
Path.GetFileName(this.DB.Name), setup, num_indexes, DateTime.Now);
int iterID = 0;
var cost = new BuildSearchCost ();
long curr = n;
long prev = n;
do {
for (int s = 0; s < step_width; ++s, ++iterID) {
cache.Clear ();
pivID = this.ACT.Count;
var nextPivot = setup.Selector.NextPivot ();
this.ACT.Add (nextPivot);
this.CT [nextPivot] = -1;
this.DT [nextPivot] = 0.0;
piv = this.DB [this.ACT[pivID]];
for (int docID = 0; docID < n; ++docID) {
var c = this.CT [docID];
if (c == -1) {
continue;
}
double dcc;
if (!cache.TryGetValue (c, out dcc)) {
dcc = this.DB.Dist (this.DB [this.ACT[c]], piv); // <- incorrecto? deberia ser ACT[c]?, porque entonces funcionan algunas cosas?
cache [c] = dcc;
}
if (dcc <= 2 * this.DT [docID]) {
//var dmin = this.DT [docID];
//if (Math.Abs(dmin - dcc) <= dmin) {
var d = this.DB.Dist (this.DB [docID], piv);
if (d < this.DT [docID]) {
this.CT [docID] = pivID;
this.DT [docID] = d;
}
}
}
}
cost.SingleCost = 0;
foreach (var qID in qlist) {
cost.SingleCost += this.InternalSearchKNN(this.DB[qID], new Result(k));
}
cost.SingleCost /= qlist.Length;
double _prob = (Math.Max(cost.SingleCost - iterID, 1.0)) / n;
_prob = Math.Pow (_prob, num_indexes) * leader_review_prob;
var internal_cost = iterID * num_indexes + leader_num_centers;
var external_cost = n * _prob;
cost.CompositeCost = internal_cost + external_cost;
prev = curr;
curr = (long)cost.CompositeCost;
Console.WriteLine("---- {0}/{1}> #pivots: {2}, prev-cost: {3}, curr-cost: {4}, #idx: {5}, timestamp: {6}",
this, Path.GetFileName(this.DB.Name), this.ACT.Count, prev, curr, num_indexes, DateTime.Now);
} while (prev > curr * (1.0 + setup.AlphaStop));
return cost;
}
public void Build(MetricDB db, ANNISetup setup)
{
this.InternalBuild(setup, 0, 1.0, db, 1);
}
public static string ExecuteDNANNI(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.DNANNI", nick);
var resname = Execute (setup, nick, idxname, (db) => {
var nilc = new XNANNI ();
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
// var annisetup = new ANNISetup(new PivotSelectorRandom(db.Count), (int)Math.Abs(setup.QARG), 0.001, 128, 64);
nilc.Build (db, annisetup, true);
return nilc;
});
return resname;
}
public static string ExecuteMANNIv3(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.MANNIv3", nick);
return Execute (setup, nick, idxname, (db) => {
var milc = new FANNI ();
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
milc.Build (db, annisetup);
return milc;
});
}
public static string ExecuteMANNIv2(IndexArgumentSetup setup, string nick, int num_indexes)
{
var idxname = String.Format ("{0}/Index.MANNIv2.{1}", nick, num_indexes);
return Execute (setup, nick, idxname, (db) => {
var milc = new MANNIv2 ();
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
milc.Build (db, annisetup, num_indexes, setup.CORES);
return milc;
});
}