/// <summary> /// Search shell (not interactive at this level) /// </summary> public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps) { BinaryWriter ResultOutput = null; if (searchOps.ResultName != null) { ResultOutput = new BinaryWriter (File.Create (searchOps.ResultName + ".tmp")); } int qid = 0; long totaltime = 0; SearchCost totalCost = new SearchCost (0, 0); if (ResultOutput != null) { var reslist = new ResultList (searchOps.IndexName, searchOps.QueryName); // Dirty.SerializeBinary (Output, reslist); reslist.Save (ResultOutput); } foreach (CommandQuery qItem in qReader) { long tstart = DateTime.Now.Ticks; SearchCost startCost = index.Cost; IResult res; var qobj = qItem.QObj; if (qobj == null) { qobj = index.DB.Parse (qItem.QRaw, true); } if (qItem.QTypeIsRange) { res = index.SearchRange (qobj, qItem.QArg); } else { res = index.SearchKNN (qobj, (int)qItem.QArg); } var qraw = qItem.QRaw; if (qraw.Length > 1024) { qraw = "<very-large-qraw>"; } SearchCost finalCost = index.Cost; finalCost.Internal -= startCost.Internal; finalCost.Total -= startCost.Total; totalCost.Internal += finalCost.Internal; totalCost.Total += finalCost.Total; long time = DateTime.Now.Ticks - tstart; totaltime += time; ResultInfo info = new ResultInfo (qid, qItem.EncodeQTypeQArgInSign(), qraw, finalCost, new TimeSpan (time), res); Console.WriteLine ("== qid: {0}", qid); Console.WriteLine ("== index: {0}, db: {1}, result: {2}", index, index.DB.Name, searchOps.ResultName); if (ResultOutput != null) { // Dirty.SerializeBinary (ResultOutput, info); info.Save (ResultOutput); } Console.WriteLine (info.ToString (searchOps.ShowMaxResult, null)); qid++; } if (ResultOutput != null) { ResultOutput.Close (); if (File.Exists (searchOps.ResultName)) { File.Delete (searchOps.ResultName); } File.Move (searchOps.ResultName + ".tmp", searchOps.ResultName); } Console.WriteLine ("Number queries: {0}", qid); Console.WriteLine ("Average total-numdists: {0}", (totalCost.Total + 0.0) / qid); Console.WriteLine ("Average internal-distances: {0}", (totalCost.Internal + 0.0) / qid); Console.WriteLine ("Average external-distances: {0}", (totalCost.Total - totalCost.Internal + 0.0) / qid); Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds); Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds); }
/// <summary> /// Search shell (not interactive at this level) /// </summary> public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps) { var summary = new ResultSummary () { ResultName = searchOps.ResultName, IndexName = searchOps.IndexName, QueriesName = searchOps.QueryName }; int qid = 0; long totaltime = 0; SearchCost totalCost = new SearchCost (0, 0); foreach (CommandQuery qItem in qReader) { long tstart = DateTime.Now.Ticks; SearchCost startCost = index.Cost; IResult res; var qobj = qItem.QObj; if (qobj == null) { qobj = index.DB.Parse (qItem.QRaw); } if (qItem.QTypeIsRange) { res = index.SearchRange (qobj, qItem.QArg); } else { res = index.SearchKNN (qobj, (int)qItem.QArg); } var qraw = qItem.QRaw; SearchCost finalCost = index.Cost; finalCost.Internal -= startCost.Internal; finalCost.Total -= startCost.Total; totalCost.Internal += finalCost.Internal; totalCost.Total += finalCost.Total; long time = DateTime.Now.Ticks - tstart; totaltime += time; var query = new Query () { QueryID = qid, QueryType = qItem.EncodeQTypeQArgInSign(), QueryRaw = qraw, SearchCostTotal = finalCost.Total, SearchCostInternal = finalCost.Internal, SearchTime = (new TimeSpan(time)).TotalSeconds, Result = new List<ItemPair>(res) }; Console.WriteLine ("----- QueryID: {0}, QueryType: {1} -----", query.QueryID, query.QueryType); if (res.Count == 0) { Console.WriteLine ("- results> empty result set"); } else { Console.WriteLine ("- results> count: {0}, first-dist: {1}, last-dist: {2}", res.Count, res.First.Dist, res.CoveringRadius); } Console.WriteLine ("- search-time: {0}, cost-internal-distances: {1}, cost-total-distances: {2}", query.SearchTime, query.SearchCostInternal, query.SearchCostTotal); Console.WriteLine ("- index: {0}, db: {1}, result: {2}", index, Path.GetFileName(index.DB.Name), Path.GetFileName(searchOps.ResultName)); summary.Add (query); qid++; } summary.ComputeSummary (); if (searchOps.ResultName != null) { File.WriteAllText (searchOps.ResultName, JsonConvert.SerializeObject (summary, Formatting.Indented)); } Console.WriteLine ("Number queries: {0}", qid); Console.WriteLine ("Average total-numdists: {0}", (totalCost.Total + 0.0) / qid); Console.WriteLine ("Average internal-distances: {0}", (totalCost.Internal + 0.0) / qid); Console.WriteLine ("Average external-distances: {0}", (totalCost.Total - totalCost.Internal + 0.0) / qid); Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds); Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds); }
public void TestKNR(KnrSeqSearch idx, string idxname, string queries, int num_refs, IList<string> reslist, Func<Index,Index> map) { // KnrSeqSearch var qstream = new QueryStream(queries); // PP-Index var resname = "Res." + idxname + "." + queries + ".PPIndex"; var searchops = new ShellSearchOptions (queries, idxname, resname); if (!File.Exists (resname)) { Commands.Search (map(idx), qstream.Iterate (), searchops); } reslist.Add (resname); // Spearman Footrule resname = "Res." + idxname + "." + queries + ".SF"; if (!File.Exists (resname)) { searchops = new ShellSearchOptions (queries, idxname, resname); Commands.Search (map(new KnrSeqSearchFootrule(idx)), qstream.Iterate (), searchops); } reslist.Add (resname); // Spearman Rho resname = "Res." + idxname + "." + queries + ".SR"; if (!File.Exists (resname)) { searchops = new ShellSearchOptions (queries, idxname, resname); Commands.Search (map(new KnrSeqSearchSpearmanRho(idx)), qstream.Iterate (), searchops); } reslist.Add (resname); // Jaccard resname = "Res." + idxname + "." + queries + ".Jaccard"; if (!File.Exists (resname)) { searchops = new ShellSearchOptions (queries, idxname, resname); Commands.Search (map(new KnrSeqSearchJaccard(idx)), qstream.Iterate (), searchops); } reslist.Add (resname); // RelMatches resname = "Res." + idxname + "." + queries + ".RelMatches"; if (!File.Exists (resname)) { searchops = new ShellSearchOptions (queries, idxname, resname); Commands.Search (map(new KnrSeqSearchRelMatches(idx)), qstream.Iterate (), searchops); } reslist.Add (resname); // CNAPP reslist.Add(_Test("Index.CNAPP." + idxname, idx.DB, () => { var cnapp = new CNAPP(); // cnapp.Build(idx, idx.K-2); cnapp.Build(idx, 1); return map(cnapp); }, queries)); }
/// <summary> /// Search shell (not interactive at this level) /// </summary> public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps, IEnumerable<string> args) { string names = null; string[] dbnames = null; if (searchOps.Names != null) { dbnames = File.ReadAllLines (names); } BinaryWriter ResultOutput = null; if (searchOps.ResultName != null) { ResultOutput = new BinaryWriter (File.Create (searchOps.ResultName + ".tmp")); } SortedDictionary<double, int> avg_hist = new SortedDictionary<double, int> (); int qid = 0; long totaltime = 0; SearchCost totalCost = new SearchCost (0, 0); if (ResultOutput != null) { var reslist = new ResultList (searchOps.IndexName, searchOps.QueryName); // Dirty.SerializeBinary (Output, reslist); reslist.Save (ResultOutput); } foreach (CommandQuery qItem in qReader) { long tstart = DateTime.Now.Ticks; SearchCost startCost = index.Cost; IResult res; if (qItem.QTypeIsRange) { res = index.ParseSearch (qItem.QRaw, qItem.QArg); } else { res = index.ParseKNNSearch (qItem.QRaw, (int)qItem.QArg); } SearchCost finalCost = index.Cost; finalCost.Internal -= startCost.Internal; finalCost.External -= startCost.External; totalCost.Internal += finalCost.Internal; totalCost.External += finalCost.External; long time = DateTime.Now.Ticks - tstart; totaltime += time; if (searchOps.Filter != null) { res = searchOps.Filter (qItem.QRaw, qItem.QArg, res, index); } SortedDictionary<double, int> hist = new SortedDictionary<double, int> (); if (searchOps.ShowHist) { foreach (ResultPair p in res) { if (hist.ContainsKey (p.dist)) { hist [p.dist]++; } else { hist [p.dist] = 1; } } foreach (var p in hist) { if (avg_hist.ContainsKey (p.Key)) { avg_hist [p.Key] += p.Value; } else { avg_hist [p.Key] = p.Value; } } if (avg_hist.Count > 1000) { searchOps.ShowHist = false; Console.WriteLine ("WARNING: Histogram of distances was disabled because there are too many bins"); } } ResultInfo info = new ResultInfo (qid, qItem.EncodeQTypeQArgInSign(), qItem.QRaw, finalCost, new TimeSpan (time), res); if (ResultOutput != null) { // Dirty.SerializeBinary (ResultOutput, info); info.Save (ResultOutput); } Console.WriteLine (info.ToString (searchOps.ShowMaxResult, dbnames)); if (searchOps.ShowHist) { Console.WriteLine ("Distance histogram (dist => counter)"); foreach (KeyValuePair<double, int> xp in hist) { Console.Write ("({0} => {1}), ", xp.Key, xp.Value); } Console.WriteLine ("<TheEnd>"); } Console.WriteLine ("Number Results: {0}", res.Count); qid++; } if (searchOps.ShowHist) { Console.WriteLine ("Average Distance histogram (dist => counter)"); foreach (KeyValuePair<double, int> xp in avg_hist) { Console.Write ("({0} => {1}), ", xp.Key, ((double)xp.Value) / qid); } Console.WriteLine ("<TheEnd>"); } if (ResultOutput != null) { ResultOutput.Close (); if (File.Exists (searchOps.ResultName)) { File.Delete (searchOps.ResultName); } File.Move (searchOps.ResultName + ".tmp", searchOps.ResultName); } Console.WriteLine ("Number queries: {0}", qid); Console.WriteLine ("Average numdists: {0}", (totalCost.Internal + totalCost.External + 0.0) / qid); Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds); Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds); }
// <summary> /// Searches using a (command line) user interface arguments /// </summary> public static void Search(Index indexObject, IEnumerable<string> args, SearchFilter handler) { string index = null; string queries = null; string result = null; string names = null; bool help = false; bool disthist = true; bool force = false; int showmaxres = 30; string indexclass = null; var config = new Dictionary<string, object> (); OptionSet ops = new OptionSet () { { "i|index=", v => index = v }, { "q|queries=", v => queries = v }, { "r|result=", v => result = v }, { "force|f", v => force = true}, { "hidehist", v => disthist = false }, { "names=", v => names = v}, { "showmaxres=", v => showmaxres = int.Parse (v) }, { "indexclass=", v => indexclass = v}, { "h|?|help", v => help = true }, { "config=", delegate(string v) { var split = v.Split (':'); if (split.Length != 2) { throw new ArgumentNullException ("config command options should be in format --config key:value "); } config.Add (split [0], split [1]); } } }; List<string> extraArgs = ops.Parse (args); if (help) { Console.WriteLine ("Usage: "); Console.WriteLine ("{0} search --index indexname --queries queriesfile [--result resname] [index args] [environ args]", Environment.GetCommandLineArgs () [0]); return; } if (result == null) { force = true; } if ((indexObject == null && index == null) || queries == null) { Console.WriteLine ("Usage: "); Console.WriteLine ("{0} search --index indexname --queries queriesfile [--result resname] [index args] [environ args]", Environment.GetCommandLineArgs () [0]); throw new ArgumentException (String.Format ("Some required arguments wasn't specified index: {0}, queries: {0}", index, queries)); } if (force || !File.Exists (result)) { Console.WriteLine ("XXXXXXXXXX index: {0}, indexclass: {1}", index, indexclass); if (indexObject == null) { indexObject = IndexLoader.Load (index, indexclass, config); } else { index = String.Format ("<memory:{0}>", indexObject.ToString ()); } indexObject.Configure (extraArgs); var searchOps = new ShellSearchOptions (queries, index, names, result, showmaxres, disthist, handler); var qstream = new QueryStream (queries); Search (indexObject, qstream.Iterate (), searchOps, extraArgs); } else { Console.WriteLine ("skipping search command since result file already exists. File: {0}", result); } }
public static void PerformSearch(string resname, Index idx, string idxname, IndexArgumentSetup setup) { if (setup.ExecuteSearch) { var ops = new ShellSearchOptions (setup.QUERIES, idxname, resname); var qstream = new QueryStream (setup.QUERIES, setup.QARG); Commands.Search (idx, qstream.Iterate (), ops); GC.Collect (); } }