Query Reader
Inheritance: IQueryStream
Esempio n. 1
0
File: Main.cs Progetto: vfaby/natix
        public void Test(string nick, MetricDB db, string queries, int num_centers, int num_perms, int num_refs)
        {
            var qstream = new QueryStream (queries);
            var reslist = new List<string> ();
            // Exhaustive search
            {
                Sequential seq = new Sequential ();
                seq.Build (db);
                var idxname = "Index.Sequential." + nick;
                IndexGenericIO.Save (idxname, seq);
                var resname = "Res." + idxname + "." + queries;
                if (!File.Exists (resname)) {
                    Commands.Search (seq, qstream.Iterate (), new ShellSearchOptions (queries, idxname, resname));
                }
                reslist.Add (resname);
            }

            ///
            /// The List of Clusters and variants
            ///

            // LC_RNN
            reslist.Add (this.TestLC ("Index.LC_RNN." + nick, db, num_centers, new LC_RNN (), queries, qstream));
            // LC
            reslist.Add (this.TestLC ("Index.LC." + nick, db, num_centers, new LC (), queries, qstream));
            // LC_IRNN
            reslist.Add (this.TestLC ("Index.LC_IRNN." + nick, db, num_centers, new LC_IRNN (), queries, qstream));
            // LC_PRNN
            reslist.Add (this.TestLC ("Index.LC_PRNN." + nick, db, num_centers, new LC_PRNN (), queries, qstream));
            // LC_ParallelBuild
            reslist.Add (this.TestLC ("Index.LC_ParallelBuild." + nick, db, num_centers, new LC_ParallelBuild (), queries, qstream));

            ///
            /// Permutation Based Indexes
            ///

            // Permutations
            reslist.Add (this.TestPI ("Index.Perms." + nick, db, num_perms, new Perms (), queries, qstream));
            // Brief Index
            reslist.Add (this.TestPI ("Index.BinPerms." + nick, db, num_perms, new BinPerms (), queries, qstream));
            // BinPermsTwoBits
            reslist.Add (this.TestPI ("Index.BinPermsTwoBits." + nick, db, num_perms, new BinPermsTwoBit (), queries, qstream));
            ///
            /// KNR
            ///

            {
                KnrSeqSearch idx;
                var idxname = "Index.KnrSeqSearch." + nick;
                if (File.Exists (idxname)) {
                    idx = (KnrSeqSearch)IndexGenericIO.Load (idxname);
                } else {
                    Console.WriteLine ("** Starting construction of '{0}'", idxname);
                    var knr = new KnrSeqSearch ();
                    var sample = RandomSets.GetRandomSubSet (num_refs, db.Count);
                    var refsdb = new SampleSpace ("", db, sample);
                    var refsidx = new LC ();
                    refsidx.Build (refsdb, refsdb.Count / 10);
                    knr.Build (db, refsidx, 7);
                    IndexGenericIO.Save (idxname, knr);
                    idx = knr;
                }
                idx.MAXCAND = 1024;
                this.TestKNR(idx, idxname, queries, num_refs, reslist, (I) => I);
                Console.WriteLine ("==== Working on a permuted space");
                idxname = idxname + ".proximity-sorted";
                if (!File.Exists(idxname)) {
                    idx = idx.GetSortedByPrefix();
                    idx.MAXCAND = 1024;
                    IndexGenericIO.Save(idxname, idx);
                } else {
                    idx = (KnrSeqSearch)IndexGenericIO.Load(idxname);
                }
                this.TestKNR(idx, idxname, queries, num_refs, reslist, (I) => new PermutedIndex(I));
            }
            reslist.Add("--horizontal");
            Commands.Check(reslist);
        }
Esempio n. 2
0
File: Main.cs Progetto: vfaby/natix
 public string TestPI(string idxname, MetricDB db, int num_perms, dynamic pi, string queries, QueryStream qstream)
 {
     var dbperms = RandomSets.GetRandomSubSet (num_perms, db.Count);
     dynamic idx;
     if (File.Exists(idxname)) {
         idx = IndexGenericIO.Load(idxname);
     } else {
         Console.WriteLine ("** Starting construction of '{0}'", idxname);
         pi.Build (db, new SampleSpace ("", db, dbperms));
         IndexGenericIO.Save (idxname, pi);
         idx = pi;
     }
     idx.MAXCAND = 1024;
     var resname = "Res." + idxname + "." + queries;
     if (!File.Exists (resname)) {
         Commands.Search (idx, qstream.Iterate (), new ShellSearchOptions (queries, idxname, resname));
     }
     return resname;
 }
Esempio n. 3
0
File: Main.cs Progetto: vfaby/natix
 string _Test(string idxname, MetricDB db, Func<Index> new_index, string queries)
 {
     QueryStream qstream = new QueryStream(queries);
     Index idx;
     if (File.Exists(idxname)) {
         idx = IndexGenericIO.Load(idxname);
     } else {
         Console.WriteLine ("** Starting construction of '{0}'", idxname);
         idx = new_index();
         IndexGenericIO.Save (idxname, idx);
     }
     var resname = "Res." + idxname + "." + queries;
     if (!File.Exists (resname)) {
         Commands.Search (idx, qstream.Iterate (), new ShellSearchOptions (queries, idxname, resname));
     }
     return resname;
 }
Esempio n. 4
0
File: Main.cs Progetto: vfaby/natix
 public string TestLC(string idxname, MetricDB db, int num_centers, dynamic lc, string queries, QueryStream qstream)
 {
     Index idx;
     if (File.Exists(idxname)) {
         idx = IndexGenericIO.Load(idxname);
     } else {
         Console.WriteLine ("** Starting construction of '{0}'", idxname);
         lc.Build (db, num_centers, SequenceBuilders.GetSeqXLB_SArray64(16));
         IndexGenericIO.Save (idxname, lc);
         idx = lc;
     }
     var resname = "Res." + idxname + "." + queries;
     if (!File.Exists (resname)) {
         Commands.Search (idx, qstream.Iterate (), new ShellSearchOptions (queries, idxname, resname));
     }
     return resname;
 }
Esempio n. 5
0
File: Main.cs Progetto: vfaby/natix
        public void TestKNR(KnrSeqSearch idx, string idxname, string queries, int num_refs, IList<string> reslist, Func<Index,Index> map)
        {
            // KnrSeqSearch
            var qstream = new QueryStream(queries);
            // PP-Index
            var resname = "Res." + idxname + "." + queries + ".PPIndex";
            var searchops = new ShellSearchOptions (queries, idxname, resname);
            if (!File.Exists (resname)) {
                Commands.Search (map(idx), qstream.Iterate (), searchops);
            }
            reslist.Add (resname);

            // Spearman Footrule
            resname = "Res." + idxname + "." + queries + ".SF";
            if (!File.Exists (resname)) {
                searchops = new ShellSearchOptions (queries, idxname, resname);
                Commands.Search (map(new KnrSeqSearchFootrule(idx)), qstream.Iterate (), searchops);
            }
            reslist.Add (resname);

            // Spearman Rho
            resname = "Res." + idxname + "." + queries + ".SR";
            if (!File.Exists (resname)) {
                searchops = new ShellSearchOptions (queries, idxname, resname);
                Commands.Search (map(new KnrSeqSearchSpearmanRho(idx)), qstream.Iterate (), searchops);
            }
            reslist.Add (resname);

            // Jaccard
            resname = "Res." + idxname + "." + queries + ".Jaccard";
            if (!File.Exists (resname)) {
                searchops = new ShellSearchOptions (queries, idxname, resname);
                Commands.Search (map(new KnrSeqSearchJaccard(idx)), qstream.Iterate (), searchops);
            }
            reslist.Add (resname);

            // RelMatches
            resname = "Res." + idxname + "." + queries + ".RelMatches";
                if (!File.Exists (resname)) {
                searchops = new ShellSearchOptions (queries, idxname, resname);
                Commands.Search (map(new KnrSeqSearchRelMatches(idx)), qstream.Iterate (), searchops);
            }
            reslist.Add (resname);

            // CNAPP
            reslist.Add(_Test("Index.CNAPP." + idxname, idx.DB, () => {
                var cnapp = new CNAPP();
                // cnapp.Build(idx, idx.K-2);
                cnapp.Build(idx, 1);
                return map(cnapp);
            }, queries));
        }
Esempio n. 6
0
        // <summary>
        /// Searches using a (command line) user interface arguments
        /// </summary>
        public static void Search(Index indexObject, IEnumerable<string> args, SearchFilter handler)
        {
            string index = null;
            string queries = null;
            string result = null;
            string names = null;
            bool help = false;
            bool disthist = true;
            bool force = false;
            int showmaxres = 30;
            string indexclass = null;
            var config = new Dictionary<string, object> ();

            OptionSet ops = new OptionSet () {
                { "i|index=",   v => index = v },
                { "q|queries=",  v => queries = v },
                { "r|result=", v => result = v },
                { "force|f", v => force = true},
                { "hidehist", v => disthist = false },
                { "names=", v => names = v},
                { "showmaxres=", v => showmaxres = int.Parse (v) },
                { "indexclass=", v => indexclass = v},
                { "h|?|help",   v => help = true },
                { "config=", delegate(string v) {
                var split = v.Split (':');
                if (split.Length != 2) {
                    throw new ArgumentNullException ("config command options should be in format --config key:value ");
                }
                config.Add (split [0], split [1]);
            }
                }
            };
            List<string> extraArgs = ops.Parse (args);

            if (help) {
                Console.WriteLine ("Usage: ");
                Console.WriteLine ("{0} search --index indexname --queries queriesfile [--result resname] [index args] [environ args]", Environment.GetCommandLineArgs () [0]);
                return;
            }
            if (result == null) {
                force = true;
            }
            if ((indexObject == null && index == null) || queries == null) {
                Console.WriteLine ("Usage: ");
                Console.WriteLine ("{0} search --index indexname --queries queriesfile [--result resname] [index args] [environ args]", Environment.GetCommandLineArgs () [0]);
                throw new ArgumentException (String.Format ("Some required arguments wasn't specified index: {0}, queries: {0}", index, queries));
            }
            if (force || !File.Exists (result)) {
                Console.WriteLine ("XXXXXXXXXX index: {0}, indexclass: {1}", index, indexclass);
                if (indexObject == null) {
                    indexObject = IndexLoader.Load (index, indexclass, config);
                } else {
                    index = String.Format ("<memory:{0}>", indexObject.ToString ());
                }
                indexObject.Configure (extraArgs);
                var searchOps = new ShellSearchOptions (queries, index, names, result, showmaxres, disthist, handler);
                var qstream = new QueryStream (queries);
                Search (indexObject, qstream.Iterate (), searchOps, extraArgs);
            } else {
                Console.WriteLine ("skipping search command since result file already exists. File: {0}", result);
            }
        }
Esempio n. 7
0
 public static void PerformSearch(string resname, Index idx, string idxname, IndexArgumentSetup setup)
 {
     if (setup.ExecuteSearch) {
         var ops = new ShellSearchOptions (setup.QUERIES, idxname, resname);
         var qstream = new QueryStream (setup.QUERIES, setup.QARG);
         Commands.Search (idx, qstream.Iterate (), ops);
         GC.Collect ();
     }
 }
Esempio n. 8
0
File: Main.cs Progetto: sadit/natix
        public static void Main(string[] args)
        {
            // Uso: VPForest db_file queries_file db_name dim tau
            string db_file="DB.colors";
            //string db_file="/home/memo/Descargas/db/colors/DB-colors.save";
            string queries_file="colors.queries";
            double querie_arg=.07;
            string query_type="Range";
            string dbname="colors";
            int dim=112;
            double tau=.07;
            //IList<float[]> queries=new List<float[]>();

            if (args.Length!=0 )
            {
                db_file=args[0];
                queries_file=args[1];
                querie_arg = Convert.ToDouble(args[2]);
                query_type = args [3];
                dbname = args [4];
                dim = Convert.ToInt32 (args [5]);
                if (args.Length == 7)
                    tau = Convert.ToDouble (args [6]);

            }

            // Leer DB
            if (!File.Exists (db_file)) {
                MemMinkowskiVectorDB<float> _db = new MemMinkowskiVectorDB<float> ();
                _db.Build (dbname+".ascii.header");
                SpaceGenericIO.Save (db_file, _db);
            }
            MetricDB DB;
            DB=SpaceGenericIO.Load(db_file,true);
            Console.WriteLine("DB Loaded size:{0}",DB.Count);

            int[] J={1,2,4,8,16}; // groups
            int [] I={1}; // not used
            foreach (int i in I)
            {
                foreach (int j in J)
                {
                    int pivspergrp=0;

                    // Crear índice VP-forest
                    //Console.WriteLine("Building Forest m:{0}",i/10d);
                    string VPF_file = "VP-Forest-"+dbname+"-Tau-" + tau + ".idx";
                    VP_Forest VPF_Search;
                    if (!File.Exists (VPF_file)) {
                        Chronos chr_time = new Chronos ();
                        chr_time.Start ();
                        VPF_Search = new VP_Forest (DB, _tau: tau);
                        chr_time.End ();
                        File.AppendAllText("index-"+dbname+"-construction-speed-VP-Forest.csv", string.Format("{0} {1}{2}",tau,chr_time.AccTime,Environment.NewLine));
                        VPF_Search.Save (new BinaryWriter (File.OpenWrite (VPF_file)));
                    } else {
                        VPF_Search = new VP_Forest ();
                        VPF_Search.Load (new BinaryReader(File.OpenRead(VPF_file)));
                    }

                    // indice secuencial
                    Sequential Seq=new Sequential();
                    Seq.Build(DB);

                    // índices EPT
                    EPTable eptable_rnd400=new EPTable();	// 400 pivots / group
                    EPTable eptable_rnd100=new EPTable(); 	// 100 pivots / group
                    EPTable eptable_rnd8=new EPTable();		// 8 pivots / group
                    EPTable eptable_rnd32=new EPTable();	// 32 pivots / group
                    EPTable eptable_opt=new EPTable();

                    // Construye los índices EPT
                    Chronos chr_ept;
                    string ept_file = "ept-opt-" + dbname + "-grps-" + j + ".idx";
                    if (!File.Exists (ept_file)) {
                        chr_ept = new Chronos ();
                        chr_ept.Start ();
                        eptable_opt.Build (DB, j, (MetricDB _db, Random seed) => new EPListOptimized (DB, j,seed, 1000, .8), 1);
                        chr_ept.End ();
                        File.AppendAllText ("index-" + dbname + "-construction-speed-ept.csv", string.Format ("EPT-opt {0} {1}{2}", j, chr_ept.AccTime, Environment.NewLine));
                        eptable_opt.Save (new BinaryWriter (File.OpenWrite (ept_file)));
                    } else {
                        eptable_opt.Load (new BinaryReader (File.OpenRead (ept_file)));
                    }

                    ept_file = "ept-rnd100-" + dbname + "-grps-" + j + ".idx";
                    if (!File.Exists (ept_file)) {
                        chr_ept = new Chronos ();
                        chr_ept.Start ();
                        eptable_rnd100.Build (DB, j);
                        chr_ept.End ();
                        File.AppendAllText ("index-" + dbname + "-construction-speed-ept.csv", string.Format ("EPT-rnd100 {0} {1}{2}", j, chr_ept.AccTime, Environment.NewLine));
                        eptable_rnd100.Save (new BinaryWriter (File.OpenWrite (ept_file)));
                    } else {
                        eptable_rnd100.Load (new BinaryReader (File.OpenRead (ept_file)));
                    }

                    ept_file = "ept-rnd8-" + dbname + "-grps-" + j + ".idx";
                    if (!File.Exists (ept_file)) {
                        chr_ept = new Chronos ();
                        chr_ept.Start ();
                        eptable_rnd8.Build (DB, j, (MetricDB _db, Random seed) => new EPListRandomPivots (DB, 8,seed), 1);
                        chr_ept.End ();
                        File.AppendAllText ("index-" + dbname + "-construction-speed-ept.csv", string.Format ("EPT-rnd8 {0} {1}{2}", j, chr_ept.AccTime, Environment.NewLine));
                        eptable_rnd8.Save (new BinaryWriter (File.OpenWrite (ept_file)));
                    } else {
                        eptable_rnd8.Load (new BinaryReader (File.OpenRead (ept_file)));
                    }

                    ept_file = "ept-rnd32-" + dbname + "-grps-" + j + ".idx";
                    if (!File.Exists (ept_file)) {
                        chr_ept = new Chronos ();
                        chr_ept.Start ();
                        eptable_rnd32.Build (DB, j, (MetricDB _db, Random seed) => new EPListRandomPivots (DB,32, seed), 1);
                        chr_ept.End ();
                        File.AppendAllText ("index-" + dbname + "-construction-speed-ept.csv", string.Format ("EPT-rnd32 {0} {1}{2}", j, chr_ept.AccTime, Environment.NewLine));
                        eptable_rnd32.Save (new BinaryWriter (File.OpenWrite (ept_file)));
                    } else {
                        eptable_rnd32.Load (new BinaryReader (File.OpenRead (ept_file)));
                    }

                    ept_file = "ept-rnd400-" + dbname + "-grps-" + j + ".idx";
                    if (!File.Exists (ept_file)) {
                        chr_ept = new Chronos ();
                        chr_ept.Start ();
                        eptable_rnd400.Build (DB, j, (MetricDB _db, Random seed) => new EPListRandomPivots (DB,400, seed), 1);
                        chr_ept.End ();
                        File.AppendAllText ("index-" + dbname + "-construction-speed-ept.csv", string.Format ("EPT-rnd400 {0} {1}{2}", j, chr_ept.AccTime, Environment.NewLine));
                        eptable_rnd400.Save (new BinaryWriter (File.OpenWrite (ept_file)));
                    } else {
                        eptable_rnd400.Load (new BinaryReader (File.OpenRead (ept_file)));
                    }

                    // generar queries
                    var qstream=new QueryStream(queries_file,querie_arg);
                    List<string> reslist=new List<string>();

                    // ======================= Búsquedas ===============================0000

                    string out_file=string.Format("res-{0}-dim[{2}]-dbsize[{1}]-{3}-",dbname,DB.Count,dim,query_type);
                    string complete_out_file;
                    // Sequential
                    complete_out_file=out_file+"Seq.dat";
                    Commands.Search(Seq,qstream.Iterate(),new ShellSearchOptions(queries_file,"Sequential",complete_out_file));
                    reslist.Add(complete_out_file);
                    // VPForest
                    complete_out_file=out_file+string.Format("tau[{0}]-VPForest.dat",VPF_Search.Tau);
                    Commands.Search(VPF_Search,qstream.Iterate(),new ShellSearchOptions(queries_file,"VP-Forest",complete_out_file));
                    reslist.Add(complete_out_file);
                    // EPTable_rnd-8
                    complete_out_file=out_file+"EPTable_rnd-numgroups["+j+"]-pivspergrp[8].dat";
                    Commands.Search(eptable_rnd8,qstream.Iterate(),new ShellSearchOptions(queries_file,"EPTable-rnd-8",complete_out_file));
                    reslist.Add(complete_out_file);
                    // EPTable_rnd-32
                    complete_out_file=out_file+"EPTable_rnd-numgroups["+j+"]-pivspergrp[32].dat";
                    Commands.Search(eptable_rnd32,qstream.Iterate(),new ShellSearchOptions(queries_file,"EPTable-rnd-32",complete_out_file));
                    reslist.Add(complete_out_file);
                    // EPTable_rnd-100
                    complete_out_file=out_file+"EPTable_rnd-numgroups["+j+"]-pivspergrp[100].dat";
                    Commands.Search(eptable_rnd100,qstream.Iterate(),new ShellSearchOptions(queries_file,"EPTable-rnd-100",complete_out_file));
                    reslist.Add(complete_out_file);
                    // EPTable_rnd-400
                    complete_out_file=out_file+"EPTable_rnd-numgroups["+j+"]-pivspergrp[400].dat";
                    Commands.Search(eptable_rnd400,qstream.Iterate(),new ShellSearchOptions(queries_file,"EPTable-rnd-400",complete_out_file));
                    reslist.Add(complete_out_file);
                    // EPTable_Opt
                    complete_out_file=out_file+"EPTable_Opt-numgroups["+j+"].dat";
                    Commands.Search(eptable_opt,qstream.Iterate(),new ShellSearchOptions(queries_file,"EPTable_Opt",complete_out_file));
                    reslist.Add(complete_out_file);
                    /**/

                    // Parámetros para guardar los resultados
                    reslist.Add("--horizontal");

                    reslist.Add(string.Format("--save=res-{0}-check-out-dim[{3}]-dbsize[{1}]-{5}-VPF-Tau[{2}]-EPT-gps[{4}]",
                                              dbname,DB.Count,tau,dim,j,query_type) );
                    Commands.Check(reslist);

                }
            }
        }

        #endregion Methods

        #region Other

        /*
        public static void LoadQueries<T>(out IList<T[]> queries,string filename) where T: struct
        {
            BinaryReader br=new BinaryReader(File.OpenRead(filename));
            queries=new List<T[]>();
            int	count=br.ReadInt32();
            int dim=br.ReadInt32();
            for (int i=0;i<count;i++)
            {
                queries.Add(new T[dim]);
                PrimitiveIO<T>.LoadVector(br,dim,queries[i]);
            }
        }

        public static void LoadQueries<T>(out IList<T[]> queries,string filename,int count,int dim) where T: struct
        {
            StreamReader br=new StreamReader(filename);
            queries=new List<T[]>();
            List<T> q=new List<T>(dim);
            string line="";
            for (int i=0;i<count;i++)
            {
                line=br.ReadLine();
                Console.WriteLine("Read: {0}",line);
                queries.Add(new T[dim]);

                PrimitiveIO<T>.LoadVector(line,q);
                queries[i]=q.ToArray();
            }
        }
        */

        #endregion Other
    }
}