/// <summary>
/// Search shell (not interactive at this level)
/// </summary>
public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps)
{
BinaryWriter ResultOutput = null;
if (searchOps.ResultName != null) {
ResultOutput = new BinaryWriter (File.Create (searchOps.ResultName + ".tmp"));
}
int qid = 0;
long totaltime = 0;
SearchCost totalCost = new SearchCost (0, 0);
if (ResultOutput != null) {
var reslist = new ResultList (searchOps.IndexName, searchOps.QueryName);
// Dirty.SerializeBinary (Output, reslist);
reslist.Save (ResultOutput);
}
foreach (CommandQuery qItem in qReader) {
long tstart = DateTime.Now.Ticks;
SearchCost startCost = index.Cost;
IResult res;
var qobj = qItem.QObj;
if (qobj == null) {
qobj = index.DB.Parse (qItem.QRaw, true);
}
if (qItem.QTypeIsRange) {
res = index.SearchRange (qobj, qItem.QArg);
} else {
res = index.SearchKNN (qobj, (int)qItem.QArg);
}
var qraw = qItem.QRaw;
if (qraw.Length > 1024) {
qraw = "<very-large-qraw>";
}
SearchCost finalCost = index.Cost;
finalCost.Internal -= startCost.Internal;
finalCost.Total -= startCost.Total;
totalCost.Internal += finalCost.Internal;
totalCost.Total += finalCost.Total;
long time = DateTime.Now.Ticks - tstart;
totaltime += time;
ResultInfo info = new ResultInfo (qid, qItem.EncodeQTypeQArgInSign(), qraw, finalCost, new TimeSpan (time), res);
Console.WriteLine ("== qid: {0}", qid);
Console.WriteLine ("== index: {0}, db: {1}, result: {2}", index, index.DB.Name, searchOps.ResultName);
if (ResultOutput != null) {
// Dirty.SerializeBinary (ResultOutput, info);
info.Save (ResultOutput);
}
Console.WriteLine (info.ToString (searchOps.ShowMaxResult, null));
qid++;
}
if (ResultOutput != null) {
ResultOutput.Close ();
if (File.Exists (searchOps.ResultName)) {
File.Delete (searchOps.ResultName);
}
File.Move (searchOps.ResultName + ".tmp", searchOps.ResultName);
}
Console.WriteLine ("Number queries: {0}", qid);
Console.WriteLine ("Average total-numdists: {0}", (totalCost.Total + 0.0) / qid);
Console.WriteLine ("Average internal-distances: {0}", (totalCost.Internal + 0.0) / qid);
Console.WriteLine ("Average external-distances: {0}", (totalCost.Total - totalCost.Internal + 0.0) / qid);
Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds);
Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds);
}
/// <summary>
/// Search shell (not interactive at this level)
/// </summary>
public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps)
{
var summary = new ResultSummary () {
ResultName = searchOps.ResultName,
IndexName = searchOps.IndexName,
QueriesName = searchOps.QueryName
};
int qid = 0;
long totaltime = 0;
SearchCost totalCost = new SearchCost (0, 0);
foreach (CommandQuery qItem in qReader) {
long tstart = DateTime.Now.Ticks;
SearchCost startCost = index.Cost;
IResult res;
var qobj = qItem.QObj;
if (qobj == null) {
qobj = index.DB.Parse (qItem.QRaw);
}
if (qItem.QTypeIsRange) {
res = index.SearchRange (qobj, qItem.QArg);
} else {
res = index.SearchKNN (qobj, (int)qItem.QArg);
}
var qraw = qItem.QRaw;
SearchCost finalCost = index.Cost;
finalCost.Internal -= startCost.Internal;
finalCost.Total -= startCost.Total;
totalCost.Internal += finalCost.Internal;
totalCost.Total += finalCost.Total;
long time = DateTime.Now.Ticks - tstart;
totaltime += time;
var query = new Query () {
QueryID = qid,
QueryType = qItem.EncodeQTypeQArgInSign(),
QueryRaw = qraw,
SearchCostTotal = finalCost.Total,
SearchCostInternal = finalCost.Internal,
SearchTime = (new TimeSpan(time)).TotalSeconds,
Result = new List<ItemPair>(res)
};
Console.WriteLine ("----- QueryID: {0}, QueryType: {1} -----", query.QueryID, query.QueryType);
if (res.Count == 0) {
Console.WriteLine ("- results> empty result set");
} else {
Console.WriteLine ("- results> count: {0}, first-dist: {1}, last-dist: {2}", res.Count, res.First.Dist, res.CoveringRadius);
}
Console.WriteLine ("- search-time: {0}, cost-internal-distances: {1}, cost-total-distances: {2}", query.SearchTime, query.SearchCostInternal, query.SearchCostTotal);
Console.WriteLine ("- index: {0}, db: {1}, result: {2}", index,
Path.GetFileName(index.DB.Name),
Path.GetFileName(searchOps.ResultName));
summary.Add (query);
qid++;
}
summary.ComputeSummary ();
if (searchOps.ResultName != null) {
File.WriteAllText (searchOps.ResultName, JsonConvert.SerializeObject (summary, Formatting.Indented));
}
Console.WriteLine ("Number queries: {0}", qid);
Console.WriteLine ("Average total-numdists: {0}", (totalCost.Total + 0.0) / qid);
Console.WriteLine ("Average internal-distances: {0}", (totalCost.Internal + 0.0) / qid);
Console.WriteLine ("Average external-distances: {0}", (totalCost.Total - totalCost.Internal + 0.0) / qid);
Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds);
Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds);
}
public void TestKNR(KnrSeqSearch idx, string idxname, string queries, int num_refs, IList<string> reslist, Func<Index,Index> map)
{
// KnrSeqSearch
var qstream = new QueryStream(queries);
// PP-Index
var resname = "Res." + idxname + "." + queries + ".PPIndex";
var searchops = new ShellSearchOptions (queries, idxname, resname);
if (!File.Exists (resname)) {
Commands.Search (map(idx), qstream.Iterate (), searchops);
}
reslist.Add (resname);
// Spearman Footrule
resname = "Res." + idxname + "." + queries + ".SF";
if (!File.Exists (resname)) {
searchops = new ShellSearchOptions (queries, idxname, resname);
Commands.Search (map(new KnrSeqSearchFootrule(idx)), qstream.Iterate (), searchops);
}
reslist.Add (resname);
// Spearman Rho
resname = "Res." + idxname + "." + queries + ".SR";
if (!File.Exists (resname)) {
searchops = new ShellSearchOptions (queries, idxname, resname);
Commands.Search (map(new KnrSeqSearchSpearmanRho(idx)), qstream.Iterate (), searchops);
}
reslist.Add (resname);
// Jaccard
resname = "Res." + idxname + "." + queries + ".Jaccard";
if (!File.Exists (resname)) {
searchops = new ShellSearchOptions (queries, idxname, resname);
Commands.Search (map(new KnrSeqSearchJaccard(idx)), qstream.Iterate (), searchops);
}
reslist.Add (resname);
// RelMatches
resname = "Res." + idxname + "." + queries + ".RelMatches";
if (!File.Exists (resname)) {
searchops = new ShellSearchOptions (queries, idxname, resname);
Commands.Search (map(new KnrSeqSearchRelMatches(idx)), qstream.Iterate (), searchops);
}
reslist.Add (resname);
// CNAPP
reslist.Add(_Test("Index.CNAPP." + idxname, idx.DB, () => {
var cnapp = new CNAPP();
// cnapp.Build(idx, idx.K-2);
cnapp.Build(idx, 1);
return map(cnapp);
}, queries));
}
/// <summary>
/// Search shell (not interactive at this level)
/// </summary>
public static void Search(Index index, IEnumerable<CommandQuery> qReader, ShellSearchOptions searchOps, IEnumerable<string> args)
{
string names = null;
string[] dbnames = null;
if (searchOps.Names != null) {
dbnames = File.ReadAllLines (names);
}
BinaryWriter ResultOutput = null;
if (searchOps.ResultName != null) {
ResultOutput = new BinaryWriter (File.Create (searchOps.ResultName + ".tmp"));
}
SortedDictionary<double, int> avg_hist = new SortedDictionary<double, int> ();
int qid = 0;
long totaltime = 0;
SearchCost totalCost = new SearchCost (0, 0);
if (ResultOutput != null) {
var reslist = new ResultList (searchOps.IndexName, searchOps.QueryName);
// Dirty.SerializeBinary (Output, reslist);
reslist.Save (ResultOutput);
}
foreach (CommandQuery qItem in qReader) {
long tstart = DateTime.Now.Ticks;
SearchCost startCost = index.Cost;
IResult res;
if (qItem.QTypeIsRange) {
res = index.ParseSearch (qItem.QRaw, qItem.QArg);
} else {
res = index.ParseKNNSearch (qItem.QRaw, (int)qItem.QArg);
}
SearchCost finalCost = index.Cost;
finalCost.Internal -= startCost.Internal;
finalCost.External -= startCost.External;
totalCost.Internal += finalCost.Internal;
totalCost.External += finalCost.External;
long time = DateTime.Now.Ticks - tstart;
totaltime += time;
if (searchOps.Filter != null) {
res = searchOps.Filter (qItem.QRaw, qItem.QArg, res, index);
}
SortedDictionary<double, int> hist = new SortedDictionary<double, int> ();
if (searchOps.ShowHist) {
foreach (ResultPair p in res) {
if (hist.ContainsKey (p.dist)) {
hist [p.dist]++;
} else {
hist [p.dist] = 1;
}
}
foreach (var p in hist) {
if (avg_hist.ContainsKey (p.Key)) {
avg_hist [p.Key] += p.Value;
} else {
avg_hist [p.Key] = p.Value;
}
}
if (avg_hist.Count > 1000) {
searchOps.ShowHist = false;
Console.WriteLine ("WARNING: Histogram of distances was disabled because there are too many bins");
}
}
ResultInfo info = new ResultInfo (qid, qItem.EncodeQTypeQArgInSign(), qItem.QRaw, finalCost, new TimeSpan (time), res);
if (ResultOutput != null) {
// Dirty.SerializeBinary (ResultOutput, info);
info.Save (ResultOutput);
}
Console.WriteLine (info.ToString (searchOps.ShowMaxResult, dbnames));
if (searchOps.ShowHist) {
Console.WriteLine ("Distance histogram (dist => counter)");
foreach (KeyValuePair<double, int> xp in hist) {
Console.Write ("({0} => {1}), ", xp.Key, xp.Value);
}
Console.WriteLine ("<TheEnd>");
}
Console.WriteLine ("Number Results: {0}", res.Count);
qid++;
}
if (searchOps.ShowHist) {
Console.WriteLine ("Average Distance histogram (dist => counter)");
foreach (KeyValuePair<double, int> xp in avg_hist) {
Console.Write ("({0} => {1}), ", xp.Key, ((double)xp.Value) / qid);
}
Console.WriteLine ("<TheEnd>");
}
if (ResultOutput != null) {
ResultOutput.Close ();
if (File.Exists (searchOps.ResultName)) {
File.Delete (searchOps.ResultName);
}
File.Move (searchOps.ResultName + ".tmp", searchOps.ResultName);
}
Console.WriteLine ("Number queries: {0}", qid);
Console.WriteLine ("Average numdists: {0}", (totalCost.Internal + totalCost.External + 0.0) / qid);
Console.WriteLine ("Total search time: {0}", (new TimeSpan (totaltime)).TotalSeconds);
Console.WriteLine ("Average search time: {0}", (new TimeSpan (totaltime / qid)).TotalSeconds);
}
// <summary>
/// Searches using a (command line) user interface arguments
/// </summary>
public static void Search(Index indexObject, IEnumerable<string> args, SearchFilter handler)
{
string index = null;
string queries = null;
string result = null;
string names = null;
bool help = false;
bool disthist = true;
bool force = false;
int showmaxres = 30;
string indexclass = null;
var config = new Dictionary<string, object> ();
OptionSet ops = new OptionSet () {
{ "i|index=", v => index = v },
{ "q|queries=", v => queries = v },
{ "r|result=", v => result = v },
{ "force|f", v => force = true},
{ "hidehist", v => disthist = false },
{ "names=", v => names = v},
{ "showmaxres=", v => showmaxres = int.Parse (v) },
{ "indexclass=", v => indexclass = v},
{ "h|?|help", v => help = true },
{ "config=", delegate(string v) {
var split = v.Split (':');
if (split.Length != 2) {
throw new ArgumentNullException ("config command options should be in format --config key:value ");
}
config.Add (split [0], split [1]);
}
}
};
List<string> extraArgs = ops.Parse (args);
if (help) {
Console.WriteLine ("Usage: ");
Console.WriteLine ("{0} search --index indexname --queries queriesfile [--result resname] [index args] [environ args]", Environment.GetCommandLineArgs () [0]);
return;
}
if (result == null) {
force = true;
}
if ((indexObject == null && index == null) || queries == null) {
Console.WriteLine ("Usage: ");
Console.WriteLine ("{0} search --index indexname --queries queriesfile [--result resname] [index args] [environ args]", Environment.GetCommandLineArgs () [0]);
throw new ArgumentException (String.Format ("Some required arguments wasn't specified index: {0}, queries: {0}", index, queries));
}
if (force || !File.Exists (result)) {
Console.WriteLine ("XXXXXXXXXX index: {0}, indexclass: {1}", index, indexclass);
if (indexObject == null) {
indexObject = IndexLoader.Load (index, indexclass, config);
} else {
index = String.Format ("<memory:{0}>", indexObject.ToString ());
}
indexObject.Configure (extraArgs);
var searchOps = new ShellSearchOptions (queries, index, names, result, showmaxres, disthist, handler);
var qstream = new QueryStream (queries);
Search (indexObject, qstream.Iterate (), searchOps, extraArgs);
} else {
Console.WriteLine ("skipping search command since result file already exists. File: {0}", result);
}
}
public static void PerformSearch(string resname, Index idx, string idxname, IndexArgumentSetup setup)
{
if (setup.ExecuteSearch) {
var ops = new ShellSearchOptions (setup.QUERIES, idxname, resname);
var qstream = new QueryStream (setup.QUERIES, setup.QARG);
Commands.Search (idx, qstream.Iterate (), ops);
GC.Collect ();
}
}