public static string ExecuteBNCInc(IndexArgumentSetup setup, string nick, int numPivs)
{
var idxname = String.Format ("{0}/Index.BNCInc.{1}", nick, numPivs);
return Execute (setup, nick, idxname, (db) => {
BNCInc bncinc = new BNCInc ();
bncinc.Build (db, numPivs);
return bncinc;
});
}
public static string ExecuteDMANNI(IndexArgumentSetup setup, string nick, int num_indexes)
{
var idxname = String.Format ("{0}/Index.DMANNI.{1}", nick, num_indexes);
return Execute (setup, nick, idxname, (db) => {
var milc = new XMANNI ();
// var annisetup = new ANNISetup(new PivotSelectorRandom(db.Count), (int)Math.Abs(setup.QARG), 0.001, 512, 64);
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
milc.Build (db, annisetup, num_indexes, true);
return milc;
});
}
public static string Execute(IndexArgumentSetup setup, string nick, string idxname, Func<MetricDB, Index> create)
{
var resname = GetResultName (nick, idxname, setup.QUERIES, setup.QARG, "");
if (File.Exists (resname)) {
return resname;
}
MetricDB db = SpaceGenericIO.Load (setup.BINARY_DATABASE);
Index idx;
if (!File.Exists (idxname)) {
Console.WriteLine ("*** creating index {0}", idxname);
var s = DateTime.Now.Ticks;
var c = db.NumberDistances;
idx = create (db);
SaveConstructionTime (idxname, DateTime.Now.Ticks - s, db.NumberDistances - c);
IndexGenericIO.Save (idxname, idx);
} else {
idx = IndexGenericIO.Load (idxname);
}
PerformSearch (resname, idx, idxname, setup);
return resname;
}
public static string ExecuteSeq(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.Seq", nick);
return Execute (setup, nick, idxname, (db) => {
var seq = new Sequential ();
seq.Build (db);
return seq;
});
}
public static string ExecuteSATRandom(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.SAT-Random", nick);
return Execute (setup, nick, idxname, (db) => {
SAT_Random sat = new SAT_Random ();
sat.Build (db, RandomSets.GetRandom());
return sat;
});
}
public static List<string> ExecuteNANNI(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.NANNI", nick);
var reslist = new List<string> ();
var resname = Execute (setup, nick, idxname, (db) => {
var nilc = new NANNI ();
nilc.Build (CreateANNI (setup.BINARY_DATABASE, (int)Math.Abs (setup.QARG)));
return nilc;
});
reslist.Add (resname);
// resname = Execute (setup, nick, idxname + "S", (db) => {
// var nilc = IndexGenericIO.Load(idxname);
// return nilc;
// });
// reslist.Add (resname);
return reslist;
}
public static void MainCOPHIR(IndexArgumentSetup setup)
{
var basename = Path.GetFileName (setup.DATABASE);
var nick = String.Format("{0}{1}", setup.PREFIX, basename);
if (!Directory.Exists (nick)) {
Directory.CreateDirectory (nick);
}
ExecuteMain (nick, setup, () => {
if (!File.Exists(setup.BINARY_DATABASE)) {
throw new Exception("The CoPhIR space doesn't support an automatic compilation");
}
});
}
public static string ExecuteVPTX(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.VPTX", nick);
return Execute (setup, nick, idxname, (db) => {
var vpt = new VPTX ();
vpt.Build (db, RandomSets.GetRandom());
return vpt;
});
}
public static string ExecuteKVP(IndexArgumentSetup setup, string nick, int k, int available_pivs)
{
MetricDB db = SpaceGenericIO.Load (setup.BINARY_DATABASE);
if (available_pivs == 0) {
available_pivs = (int)Math.Sqrt (db.Count);
}
var idxname = String.Format ("{0}/Index.KVP.{1}-{2}", nick, k, available_pivs);
return Execute (setup, nick, idxname, (_db) => {
KVP kvp = new KVP ();
kvp.Build (db, k, available_pivs);
return kvp;
});
}
public static string ExecuteEPTB(IndexArgumentSetup setup, string nick, int numgroups)
{
var idxname = String.Format ("{0}/Index.EPTB.{1}", nick, numgroups);
return Execute (setup, nick, idxname, (db) => {
EPTable eptable = new EPTable ();
eptable.Build (db, numgroups, (_db, rand) => new EPListOptimizedB (db, numgroups, rand), setup.CORES);
return eptable;
});
}
public static void MainWIKTIONARY(IndexArgumentSetup setup)
{
var basename = Path.GetFileName (setup.DATABASE);
var nick = String.Format("{0}{1}", setup.PREFIX, basename);
if (!Directory.Exists (nick)) {
Directory.CreateDirectory (nick);
}
ExecuteMain (nick, setup, () => {
if (!File.Exists(setup.BINARY_DATABASE)) {
Wiktionary sp = new Wiktionary();
sp.Build(setup.DATABASE);
SpaceGenericIO.Save(setup.BINARY_DATABASE, sp);
}
});
}
public static void ExecuteMain(string nick, IndexArgumentSetup setup, Action prepare_db)
{
var dbname = String.Format ("DB.{0}", Path.GetFileName(setup.DATABASE));
setup.BINARY_DATABASE = dbname;
prepare_db ();
SpaceGenericIO.Load (setup.BINARY_DATABASE);
var arglist = new System.Collections.Concurrent.ConcurrentQueue<String> ();
arglist.Enqueue ("--save");
arglist.Enqueue (String.Format ("Tab.ExactIndexes.{0}.{1}.qarg={2}.json", nick, Path.GetFileName (setup.QUERIES), setup.QARG));
arglist.Enqueue (Indexes.ExecuteSeq (setup, nick));
var actionlist = new List<Action> ();
if (setup.ExecuteNANNI) {
actionlist.Add (() => {
foreach (var resname in Indexes.ExecuteNANNI(setup, nick)) {
arglist.Enqueue(resname);
}
});
}
if (setup.ExecuteTNANNI) {
actionlist.Add (() => {
arglist.Enqueue(Indexes.ExecuteTNANNI(setup, nick));
});
}
if (setup.ExecuteDNANNI) {
actionlist.Add (() => {
arglist.Enqueue(Indexes.ExecuteDNANNI(setup, nick));
});
}
if (setup.ExecuteSAT) {
actionlist.Add (() => {
var resname = Indexes.ExecuteSAT (setup, nick);
arglist.Enqueue (resname);
});
}
if (setup.ExecuteSATDistal) {
actionlist.Add (() => {
var resname = Indexes.ExecuteSATDistal (setup, nick);
arglist.Enqueue (resname);
});
}
if (setup.ExecuteSATRandom) {
actionlist.Add (() => {
var resname = Indexes.ExecuteSATRandom (setup, nick);
arglist.Enqueue(resname);
});
}
if (setup.ExecuteVPT) {
actionlist.Add (() => {
var resname = Indexes.ExecuteVPT (setup, nick);
arglist.Enqueue (resname);
});
}
// arglist.Add (Indexes.ExecuteVPTX (setup, nick));
//arglist.Add (Indexes.ExecuteMILCv3 (setup, nick));
foreach (var bsize in setup.LC) {
var _bsize = bsize;
actionlist.Add (() => {
var resname = Indexes.ExecuteLC (setup, nick, _bsize);
arglist.Enqueue(resname);
});
}
foreach (var numGroups in setup.MANNI) {
var _numGroups = numGroups;
actionlist.Add (() => {
var resname = Indexes.ExecuteMANNI (setup, nick, _numGroups);
arglist.Enqueue(resname);
});
actionlist.Add (() => {
var resname = Indexes.ExecuteTMANNI (setup, nick, _numGroups);
arglist.Enqueue(resname);
});
actionlist.Add (() => {
var resname = Indexes.ExecuteDMANNI (setup, nick, _numGroups);
arglist.Enqueue(resname);
});
//arglist.Add (Indexes.ExecuteMILCv2 (setup, nick, numGroups));
}
foreach (var numGroups in setup.EPT) {
var _numGroups = numGroups;
actionlist.Add (() => {
var resname = Indexes.ExecuteEPTA (setup, nick, _numGroups);
arglist.Enqueue (resname);
});
// arglist.Add (Indexes.ExecuteEPT (setup, nick, numGroups));
}
foreach (var numPivs in setup.BNC) {
var _numPivs = numPivs;
actionlist.Add (() => {
var resname = Indexes.ExecuteBNCInc (setup, nick, _numPivs);
arglist.Enqueue (resname);
});
}
foreach (var numPivs in setup.KVP) {
var _numPivs = numPivs;
actionlist.Add (() => {
var resname = Indexes.ExecuteKVP (setup, nick, _numPivs, setup.KVP_Available);
arglist.Enqueue (resname);
});
}
foreach (var numPivs in setup.SPA) {
var _numPivs = numPivs;
actionlist.Add (() => {
var resname = Indexes.ExecuteSpaghetti (setup, nick, _numPivs);
arglist.Enqueue (resname);
});
}
foreach (var numPivs in setup.LAESA) {
var _numPivs = numPivs;
actionlist.Add (() => {
var resname = Indexes.ExecuteLAESA (setup, nick, _numPivs);
arglist.Enqueue (resname);
});
}
foreach (var alpha in setup.SSS) {
var _alpha = alpha;
actionlist.Add (() => {
var resname = Indexes.ExecuteSSS (setup, nick, _alpha, setup.SSS_max);
arglist.Enqueue (resname);
});
}
if (setup.SPAWN == 1) {
foreach (var action in actionlist) {
action.Invoke ();
}
} else {
LongParallel.ForEach (actionlist, (a) => a.Invoke (), setup.SPAWN);
}
if (setup.ExecuteSearch) {
Commands.Check (arglist);
}
}
public static void MainVEC(IndexArgumentSetup setup, string stype)
{
var basename = Path.GetFileName (setup.DATABASE);
var nick = String.Format("{0}{1}", setup.PREFIX, basename);
if (!Directory.Exists (nick)) {
Directory.CreateDirectory (nick);
}
switch (stype) {
case "VEC":
ExecuteMain (nick, setup, () => {
if (!File.Exists (setup.BINARY_DATABASE)) {
MemMinkowskiVectorDB<float> db = new MemMinkowskiVectorDB<float> ();
db.Build (setup.DATABASE);
SpaceGenericIO.Save (setup.BINARY_DATABASE, db);
}
});
break;
case "VEC_UInt8":
ExecuteMain (nick, setup, () => {
if (!File.Exists (setup.BINARY_DATABASE)) {
MemMinkowskiVectorDB<byte> db = new MemMinkowskiVectorDB<byte> ();
db.Build (setup.DATABASE);
SpaceGenericIO.Save (setup.BINARY_DATABASE, db);
}
});
break;
case "VEC_Int8":
ExecuteMain (nick, setup, () => {
if (!File.Exists (setup.BINARY_DATABASE)) {
MemMinkowskiVectorDB<sbyte> db = new MemMinkowskiVectorDB<sbyte> ();
db.Build (setup.DATABASE);
SpaceGenericIO.Save (setup.BINARY_DATABASE, db);
}
});
break;
case "VEC_Int16":
ExecuteMain (nick, setup, () => {
if (!File.Exists (setup.BINARY_DATABASE)) {
MemMinkowskiVectorDB<short> db = new MemMinkowskiVectorDB<short> ();
db.Build (setup.DATABASE);
SpaceGenericIO.Save (setup.BINARY_DATABASE, db);
}
});
break;
case "VEC_UInt16":
ExecuteMain (nick, setup, () => {
if (!File.Exists (setup.BINARY_DATABASE)) {
MemMinkowskiVectorDB<ushort> db = new MemMinkowskiVectorDB<ushort> ();
db.Build (setup.DATABASE);
SpaceGenericIO.Save (setup.BINARY_DATABASE, db);
}
});
break;
default:
throw new ArgumentException ("Error unknown vector subtype " + stype);
}
}
public static void MainSEQED(IndexArgumentSetup setup)
{
var basename = Path.GetFileName (setup.DATABASE);
var nick = String.Format("{0}{1}", setup.PREFIX, basename);
if (!Directory.Exists (nick)) {
Directory.CreateDirectory (nick);
}
ExecuteMain (nick, setup, () => {
SeqLevenshteinSpace<int> sp;
if (!File.Exists(setup.BINARY_DATABASE)) {
sp = new SeqLevenshteinSpace<int>();
sp.Build(setup.DATABASE);
SpaceGenericIO.Save(setup.BINARY_DATABASE, sp);
}
});
}
public static void MainDOC(IndexArgumentSetup setup)
{
var basename = Path.GetFileName (setup.DATABASE);
var nick = String.Format("{0}{1}", setup.PREFIX, basename);
if (!Directory.Exists (nick)) {
Directory.CreateDirectory (nick);
}
ExecuteMain (nick, setup, () => {
DocumentDB sp;
if (!File.Exists(setup.BINARY_DATABASE)) {
sp = new DocumentDB();
var list = new List<string>(Directory.EnumerateFiles(setup.DATABASE));
list.Sort();
sp.Build(setup.DATABASE, list);
SpaceGenericIO.Save(setup.BINARY_DATABASE, sp);
}
});
}
public static string ExecuteSpaghetti(IndexArgumentSetup setup, string nick, int numpivs)
{
var idxname = String.Format ("{0}/Index.Spaghetti.{1}", nick, numpivs);
return Execute (setup, nick, idxname, (db) => {
Spaghetti spa = new Spaghetti ();
spa.Build (db, numpivs);
return spa;
});
}
public static string ExecuteSSS(IndexArgumentSetup setup, string nick, double alpha, int maxPivs)
{
var idxname = String.Format ("{0}/Index.SSS.{1}", nick, alpha);
return Execute (setup, nick, idxname, (db) => {
SSS sss = new SSS ();
sss.Build (db, alpha, maxPivs, setup.CORES);
return sss;
});
}
public static string ExecuteLAESA(IndexArgumentSetup setup, string nick, int numpivs)
{
var idxname = String.Format ("{0}/Index.LAESA.{1}", nick, numpivs);
return Execute (setup, nick, idxname, (db) => {
LAESA laesa = new LAESA ();
laesa.Build (db, numpivs, setup.CORES);
return laesa;
});
}
public static void PerformSearch(string resname, Index idx, string idxname, IndexArgumentSetup setup)
{
if (setup.ExecuteSearch) {
Console.WriteLine ("======= Searching {0}", resname);
var ops = new ShellSearchOptions (setup.QUERIES, idxname, resname);
var qstream = new QueryStream (setup.QUERIES, setup.QARG);
Commands.Search (idx, qstream.Iterate (), ops);
GC.Collect ();
}
}
public static string ExecuteLC(IndexArgumentSetup setup, string nick, int bsize)
{
var idxname = String.Format ("{0}/Index.LC-{1}", nick, bsize);
return Execute (setup, nick, idxname, (db) => {
var lc = new LC ();
lc.Build (db, bsize, RandomSets.GetRandom());
return lc;
});
}
public static string ExecuteMANNIv2(IndexArgumentSetup setup, string nick, int num_indexes)
{
var idxname = String.Format ("{0}/Index.MANNIv2.{1}", nick, num_indexes);
return Execute (setup, nick, idxname, (db) => {
var milc = new MANNIv2 ();
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
milc.Build (db, annisetup, num_indexes, setup.CORES);
return milc;
});
}
public static string ExecuteMANNIv3(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.MANNIv3", nick);
return Execute (setup, nick, idxname, (db) => {
var milc = new FANNI ();
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
milc.Build (db, annisetup);
return milc;
});
}
public static string ExecuteDNANNI(IndexArgumentSetup setup, string nick)
{
var idxname = String.Format ("{0}/Index.DNANNI", nick);
var resname = Execute (setup, nick, idxname, (db) => {
var nilc = new XNANNI ();
var annisetup = new ANNISetup(db.Count, (int)Math.Abs(setup.QARG));
// var annisetup = new ANNISetup(new PivotSelectorRandom(db.Count), (int)Math.Abs(setup.QARG), 0.001, 128, 64);
nilc.Build (db, annisetup, true);
return nilc;
});
return resname;
}
public static void Main(string[] args)
{
OptionSet ops = null;
var setup = new IndexArgumentSetup ();
string stype = null;
ops = new OptionSet() {
{"database=", "Database in its ascii format. It will create a file DB.dbname in the current directory", v => setup.DATABASE = v },
{"queries=", "Queries in its ascii format", v => setup.QUERIES = v },
{"stype=", "The type of the metric space. Valid names VEC, VEC_Int16, VEC_UInt16, VEC_Int8, VEC_UInt8, DOC, SEQ-ED, STR-ED, WIKTIONARY, COPHIR282", v => stype = v},
{"qtype=", "Type of query, negative values should be integers and means for near neighbor search, positive values means for range search. Defaults to -30 (30NN)", v => setup.QARG = Double.Parse(v)},
{"prefix=", "Experiment's prefix", v => setup.PREFIX = v},
{"laesa=", "Run LAESA with the given pivot sizes (comma sep.)", v => LoadList(v, setup.LAESA)},
{"spa=", "Run Spaghetti with the given pivot sizes (comma sep.)", v => LoadList(v, setup.SPA)},
{"bnc=", "Run BNC-Inc with the given pivot sizes (comma sep.)", v => LoadList(v, setup.BNC)},
{"kvp=", "Run KVP with the given pivot sizes (comma sep.)", v => LoadList(v, setup.KVP)},
{"kvp-available=", "Run KVP with the given available pivots (defaults to 1024; zero means sqrt(n))", v => setup.KVP_Available = int.Parse(v)},
{"ept=", "Run EPT with the given group sizes (comma sep.)", v => LoadList(v, setup.EPT)},
{"manni=", "Run *MANNI* with the given number of indexes (comma sep.)", v => LoadList(v, setup.MANNI)},
{"lc=", "Run LC with the given block sizes (comma sep.)", v => LoadList(v, setup.LC)},
{"sss=", "Run SSS with the given alpha list (comma sep.)", v => LoadList(v, setup.SSS)},
{"sss-max=", "Run SSS with the given alpha list (comma sep.)", v => setup.SSS_max = int.Parse(v)},
{"vpt", "Run VPT", v => setup.ExecuteVPT = true},
{"sat", "Run Legacy SAT", v => setup.ExecuteSAT = true},
{"satrandom", "Run Random SAT", v => setup.ExecuteSATRandom = true},
{"satdistal", "Run Distal SAT", v => setup.ExecuteSATDistal = true},
{"fanni", "Run FANNI", v => setup.ExecuteFANNI = true},
{"nanni", "Run NANNI", v => setup.ExecuteNANNI = true},
{"tnanni", "Run TNANNI", v => setup.ExecuteTNANNI = true},
{"dnanni", "Run DNANNI", v => setup.ExecuteDNANNI = true},
{"help|h", "Shows this help", v => {
ops.WriteOptionDescriptions(Console.Out);
System.Environment.Exit(0);
}
},
{"skip-search", v => setup.ExecuteSearch = false},
{"cores=", "Maximum number of threads/cores to be used by independent indexes", v => setup.CORES = int.Parse(v)},
{"spawn=", "Number of running concurrent tests (indexes), notice that a spawn != 1 could produce weird values on performance variables sensed by self tunning indexes", v => setup.SPAWN = int.Parse(v)},
};
ops.Parse(args);
if (setup.DATABASE == null) {
ops.WriteOptionDescriptions (Console.Out);
throw new ArgumentNullException ("The database argument is mandatory");
}
if (setup.QUERIES == null) {
ops.WriteOptionDescriptions (Console.Out);
throw new ArgumentNullException ("The queries argument is mandatory");
}
if (stype == null) {
ops.WriteOptionDescriptions (Console.Out);
throw new ArgumentNullException ("The stype argument is mandatory");
}
if (stype.StartsWith("VEC")) {
MainVEC (setup, stype);
} else if (stype == "DOC") {
MainDOC (setup);
} else if (stype == "SEQ-ED") {
MainSEQED (setup);
} else if (stype == "STR-ED") {
MainSTRED (setup);
} else if (stype == "WIKTIONARY") {
MainWIKTIONARY (setup);
} else if (stype == "COPHIR282") {
MainCOPHIR(setup);
} else {
throw new ArgumentException (String.Format("Unknown space type {0}", stype));
}
}