public void TestWAHBitArray4()
{
var b = new WAHBitArray();
int count = 25;
for (int i = 0; i < 5; i++)
{
b.Set(i, true);
}
for (int i = 5; i < count + 5; i++)
{
b.Set(i, false);
}
for (int i = 30; i < 64; i++)
{
b.Set(i, i != 35);
}
for (int i = 64; i < 100; i++)
{
b.Set(i, true);
}
var expected = b.GetBitIndexes().ToArray();
var b2 = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
var actual = b2.GetBitIndexes().ToArray();
expected.Should(Be.EqualTo(actual), "expected.Should(Be.EqualTo(actual))");
}
public void TestWAHBitArray1()
{
var b = new WAHBitArray();
int count = 31;
for (int i = 0; i < count; i++)
{
b.Set(i, true);
}
var expected = b.GetBitIndexes().ToArray();
var b2 = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
var actual = b2.GetBitIndexes().ToArray();
expected.Should(Be.EqualTo(actual), "expected.Should(Be.EqualTo(actual))");
}
public void SetBit(int index, bool val)
{
if (_bits != null)
{
_bits.Set(index, val);
}
else
{
_bits = new WAHBitArray();
_bits.Set(index, val);
}
isDirty = true;
}
public void SaveDocument(Document doc, bool deleteold)
{
while (_internalOP)
{
Thread.Sleep(50);
}
DateTime dt = FastDateTime.Now;
if (deleteold && doc.DocNumber > -1)
{
_deleted.Set(doc.DocNumber, true);
}
if (deleteold || doc.DocNumber == -1)
{
doc.DocNumber = _lastDocNum++;
}
_log(string.Format("indexing document: #{0} for Entity #{1}", doc.DocNumber, doc.FileName));
string dstr = JSON.Instance.ToJSON(doc, false);
_docs.WriteData(Helper.GetBytes(doc.DocNumber, false), Helper.GetBytes(dstr));
_hash.Set(Helper.GetBytes(doc.FileName), doc.DocNumber);
_log(string.Format("writing document to disk (ms) = {0}", FastDateTime.Now.Subtract(dt).TotalMilliseconds));
}
public WAHBitArray ExecutionPlan(string filter, bool freeCache = false)
{
_log(string.Format("Hoot::ExecutionPlan start freeCache is {0}", freeCache));
_log(string.Format("query : {0}", filter));
DateTime now = FastDateTime.Now;
string[] words = filter.Split(' ').OrderByDescending(w => w.Length).ToArray();
WAHBitArray bits = null;
var wildcardMatchers = new List <WildcardMatcher>();
var cacheToFree = new List <Tuple <string, Cache> >();
for (int i = 0; i < words.Length; i++)
{
string originWord = words[i];
string preparedWord = words[i];
var op = Cache.OPERATION.OR;
if (originWord.StartsWith("+"))
{
op = Cache.OPERATION.AND;
preparedWord = originWord.Replace("+", "");
}
else if (originWord.StartsWith("-"))
{
op = Cache.OPERATION.ANDNOT;
preparedWord = originWord.Replace("-", "");
}
if (originWord.Contains("*") || originWord.Contains("?"))
{
wildcardMatchers.Add(new WildcardMatcher(originWord.ToLower()));
}
else
{
Cache c;
var lowerWord = preparedWord.ToLowerInvariant();
if (_index.TryGetValue(lowerWord, out c))
{
LoadCacheIfNotLoaded(c);
cacheToFree.Add(Tuple.Create(lowerWord, c));
bits = DoBitOperation(bits, c, op);
}
else if (op == Cache.OPERATION.AND)
{
var cache = new Cache {
isLoaded = true
};
cache.SetBit(0, false);
bits = DoBitOperation(bits, cache, op);
}
}
if (i == words.Length - 1)
{
//asc: brutal hack - only for wildcards
op = Cache.OPERATION.AND;
WAHBitArray wildbits = null;
var foundMatcherWords = wildcardMatchers.ToDictionary(w => w, _ => new ConcurrentQueue <string>());
Parallel.ForEach(_index.Keys, w =>
{
foreach (var matcher in wildcardMatchers)
{
if (matcher.IsMatch(w))
{
foundMatcherWords[matcher].Enqueue(w);
break;
}
}
});
var loadWatch = Stopwatch.StartNew();
using (var bmp = CreateBitmapStream())
{
foreach (string key in foundMatcherWords.Values.SelectMany(_ => _))
{
var c = _index[key];
LoadCacheIfNotLoaded(c, bmp);
cacheToFree.Add(Tuple.Create(key, c));
}
}
_log(string.Format("Hoot wildcard load operation: {0} ms", loadWatch.Elapsed.TotalMilliseconds));
var bitWatch = Stopwatch.StartNew();
var matcherPlans = foundMatcherWords.Select(p =>
{
if (p.Value.Count == 0)
{
var falsePlan = new WAHBitArray();
falsePlan.Set(0, false);
return(falsePlan);
}
WAHBitArray matcherPlan = null;
foreach (string word in p.Value)
{
matcherPlan = DoBitOperation(matcherPlan, _index[word], Cache.OPERATION.OR);
}
return(matcherPlan);
}).Where(p => p != null).ToList();
wildbits = matcherPlans.Aggregate(wildbits, (acc, matcherPlan) => acc != null ? acc.And(matcherPlan) : matcherPlan);
_log(string.Format("Hoot wildcard bit operation: {0} ms", bitWatch.Elapsed.TotalMilliseconds));
if (wildbits != null)
{
bits = bits == null ? wildbits : (op == Cache.OPERATION.AND ? bits.And(wildbits) : bits.Or(wildbits));
}
}
}
if (bits == null)
{
return(new WAHBitArray());
}
// remove deleted docs
if (bits.Length > _deleted.Length)
{
_deleted.Length = bits.Length;
}
else if (bits.Length < _deleted.Length)
{
bits.Length = _deleted.Length;
}
WAHBitArray result = bits.And(_deleted.Not());
_log(string.Format("Hoot::ExecutionPlan freeCache is {0}", freeCache));
if (freeCache)
{
foreach (var c in cacheToFree)
{
_log(string.Format("Free cache from ExecutionPlan::ExecutionPlan for {0}", c.Item1));
c.Item2.FreeMemory(unload: true, freeUncompressedMemory: false);
}
//asc: clean cache buckets cache
_hash.Commit();
}
_log(string.Format("query time (ms) = {0}", FastDateTime.Now.Subtract(now).TotalMilliseconds));
return(result);
}
public void SetBit(int index, bool val)
{
if (_bits != null)
_bits.Set(index, val);
else
{
_bits = new WAHBitArray();
_bits.Set(index, val);
}
isDirty = true;
}