public List <int> FindPagedRows(string filter, int page, int pageSize, int skip, out int total)
{
var result = new List <int>();
while (_internalOP)
{
Thread.Sleep(50);
}
WAHBitArray bits = ExecutionPlan(filter);
var indexes = bits.GetBitIndexes().ToList();
total = indexes.Count;
int currentPos = 0, pageStartPos = page * pageSize + (skip > 0 ? skip : 0), pageEndPos = pageStartPos + pageSize;
bool isPageable = pageStartPos < pageEndPos;
if (!isPageable)
{
return(indexes);
}
foreach (int i in indexes)
{
currentPos++;
if (currentPos <= pageStartPos || currentPos > pageEndPos)
{
continue;
}
result.Add(i);
if (result.Count == pageSize)
{
break;
}
}
return(result);
}
public void SetCompressedBits(uint[] bits)
{
_bits = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, bits);
LastBitSaveLength = bits.Length;
isLoaded = true;
isDirty = false;
}
public void TestWAHBitArray4()
{
var b = new WAHBitArray();
int count = 25;
for (int i = 0; i < 5; i++)
{
b.Set(i, true);
}
for (int i = 5; i < count + 5; i++)
{
b.Set(i, false);
}
for (int i = 30; i < 64; i++)
{
b.Set(i, i != 35);
}
for (int i = 64; i < 100; i++)
{
b.Set(i, true);
}
var expected = b.GetBitIndexes().ToArray();
var b2 = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
var actual = b2.GetBitIndexes().ToArray();
expected.Should(Be.EqualTo(actual), "expected.Should(Be.EqualTo(actual))");
}
public List <TDocument> FindPagedDocuments <TDocument>(WAHBitArray plan, int page, int pageSize, int skip, out int total)
where TDocument : Document
{
var result = new List <TDocument>();
int pageStartPos = page * pageSize + (skip > 0 ? skip : 0), pageEndPos = pageStartPos + pageSize;
int[] indexes = plan.GetBitIndexes().ToArray();
total = indexes.Length;
for (int k = total - pageStartPos - 1; k > total - pageEndPos - 1; k--)
{
if (k < 0)
{
break;
}
var index = indexes[k];
if (index > _lastDocNum - 1)
{
continue;
}
byte[] b = _docs.ReadData(index);
var d = JSON.Instance.ToObject <TDocument>(Helper.GetString(b));
result.Add(d);
}
return(result);
}
public WAHBitArray Or(WAHBitArray op)
{
CheckBitArray(op);
BitArray b = (BitArray)_ba.Clone();
return(op.Or(b));
}
public void FreeMemory(bool unload, bool freeUncompressedMemory = true)
{
if (freeUncompressedMemory && _bits != null)
{
_bits.FreeMemory();
}
if (unload)
{
_bits = null;
isLoaded = false;
}
}
public void TestWAHBitArray1()
{
var b = new WAHBitArray();
int count = 31;
for (int i = 0; i < count; i++)
{
b.Set(i, true);
}
var expected = b.GetBitIndexes().ToArray();
var b2 = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
var actual = b2.GetBitIndexes().ToArray();
expected.Should(Be.EqualTo(actual), "expected.Should(Be.EqualTo(actual))");
}
private static WAHBitArray DoBitOperation(WAHBitArray bits, Cache c, Cache.OPERATION op)
{
if (bits != null)
{
bits = c.Op(bits, op);
}
else
{
bits = c.GetBitmap();
bits = op == Cache.OPERATION.ANDNOT ? bits.Not() : bits;
}
return(bits);
}
public void SetBit(int index, bool val)
{
if (_bits != null)
{
_bits.Set(index, val);
}
else
{
_bits = new WAHBitArray();
_bits.Set(index, val);
}
isDirty = true;
}
public WAHBitArray Xor(WAHBitArray op)
{
lock (_lock)
{
uint[] left;
uint[] right;
prelogic(op, out left, out right);
for (int i = 0; i < left.Length; i++)
{
left[i] ^= right[i];
}
return(new WAHBitArray(TYPE.Uncompressed_WAH, left));
}
}
public WAHBitArray Op(WAHBitArray bits, OPERATION op)
{
if (bits == null)
{
throw new InvalidOperationException("Bits is null");
}
if (op == OPERATION.AND)
{
return(_bits.And(bits));
}
if (op == OPERATION.OR)
{
return(_bits.Or(bits));
}
return(bits.And(_bits.Not()));
}
private void CheckBitArray(WAHBitArray op)
{
CheckBitArray();
if (op != null)
{
int L1 = _ba.Length;
int L2 = op.Length;
if (L1 != L2)
{
if (L1 > L2)
{
op.Length = L1;
}
else
{
_ba.Length = L2;
}
}
_size = _ba.Length;
}
}
private void ReadDeleted()
{
if (File.Exists(_path + _fileName + ".deleted") == false)
{
_deleted = new WAHBitArray();
return;
}
using (
FileStream del = new FileStream(_path + _fileName + ".deleted", FileMode.Open, FileAccess.ReadWrite,
FileShare.ReadWrite))
{
List <uint> ar = new List <uint>();
byte[] b = new byte[4];
while (del.Read(b, 0, 4) > 0)
{
ar.Add((uint)Helper.ToInt32(b, 0));
}
_deleted = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, ar.ToArray());
del.Close();
}
}
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right)
{
this.CheckBitArray();
left = this.GetUncompressed();
right = op.GetUncompressed();
int ic = left.Length;
int uc = right.Length;
if (ic > uc)
{
uint[] ar = new uint[ic];
right.CopyTo(ar, 0);
right = ar;
}
else if (ic < uc)
{
uint[] ar = new uint[uc];
left.CopyTo(ar, 0);
left = ar;
}
//FixLengths(ints, uncomp);
}
public WAHBitArray Xor(WAHBitArray op)
{
CheckBitArray(op);
return(op.Xor(_ba));
}
public WAHBitArray And(WAHBitArray op)
{
CheckBitArray(op);
return(op.And(_ba));
}
public WAHBitArray ExecutionPlan(string filter, bool freeCache = false)
{
_log(string.Format("Hoot::ExecutionPlan start freeCache is {0}", freeCache));
_log(string.Format("query : {0}", filter));
DateTime now = FastDateTime.Now;
string[] words = filter.Split(' ').OrderByDescending(w => w.Length).ToArray();
WAHBitArray bits = null;
var wildcardMatchers = new List <WildcardMatcher>();
var cacheToFree = new List <Tuple <string, Cache> >();
for (int i = 0; i < words.Length; i++)
{
string originWord = words[i];
string preparedWord = words[i];
var op = Cache.OPERATION.OR;
if (originWord.StartsWith("+"))
{
op = Cache.OPERATION.AND;
preparedWord = originWord.Replace("+", "");
}
else if (originWord.StartsWith("-"))
{
op = Cache.OPERATION.ANDNOT;
preparedWord = originWord.Replace("-", "");
}
if (originWord.Contains("*") || originWord.Contains("?"))
{
wildcardMatchers.Add(new WildcardMatcher(originWord.ToLower()));
}
else
{
Cache c;
var lowerWord = preparedWord.ToLowerInvariant();
if (_index.TryGetValue(lowerWord, out c))
{
LoadCacheIfNotLoaded(c);
cacheToFree.Add(Tuple.Create(lowerWord, c));
bits = DoBitOperation(bits, c, op);
}
else if (op == Cache.OPERATION.AND)
{
var cache = new Cache {
isLoaded = true
};
cache.SetBit(0, false);
bits = DoBitOperation(bits, cache, op);
}
}
if (i == words.Length - 1)
{
//asc: brutal hack - only for wildcards
op = Cache.OPERATION.AND;
WAHBitArray wildbits = null;
var foundMatcherWords = wildcardMatchers.ToDictionary(w => w, _ => new ConcurrentQueue <string>());
Parallel.ForEach(_index.Keys, w =>
{
foreach (var matcher in wildcardMatchers)
{
if (matcher.IsMatch(w))
{
foundMatcherWords[matcher].Enqueue(w);
break;
}
}
});
var loadWatch = Stopwatch.StartNew();
using (var bmp = CreateBitmapStream())
{
foreach (string key in foundMatcherWords.Values.SelectMany(_ => _))
{
var c = _index[key];
LoadCacheIfNotLoaded(c, bmp);
cacheToFree.Add(Tuple.Create(key, c));
}
}
_log(string.Format("Hoot wildcard load operation: {0} ms", loadWatch.Elapsed.TotalMilliseconds));
var bitWatch = Stopwatch.StartNew();
var matcherPlans = foundMatcherWords.Select(p =>
{
if (p.Value.Count == 0)
{
var falsePlan = new WAHBitArray();
falsePlan.Set(0, false);
return(falsePlan);
}
WAHBitArray matcherPlan = null;
foreach (string word in p.Value)
{
matcherPlan = DoBitOperation(matcherPlan, _index[word], Cache.OPERATION.OR);
}
return(matcherPlan);
}).Where(p => p != null).ToList();
wildbits = matcherPlans.Aggregate(wildbits, (acc, matcherPlan) => acc != null ? acc.And(matcherPlan) : matcherPlan);
_log(string.Format("Hoot wildcard bit operation: {0} ms", bitWatch.Elapsed.TotalMilliseconds));
if (wildbits != null)
{
bits = bits == null ? wildbits : (op == Cache.OPERATION.AND ? bits.And(wildbits) : bits.Or(wildbits));
}
}
}
if (bits == null)
{
return(new WAHBitArray());
}
// remove deleted docs
if (bits.Length > _deleted.Length)
{
_deleted.Length = bits.Length;
}
else if (bits.Length < _deleted.Length)
{
bits.Length = _deleted.Length;
}
WAHBitArray result = bits.And(_deleted.Not());
_log(string.Format("Hoot::ExecutionPlan freeCache is {0}", freeCache));
if (freeCache)
{
foreach (var c in cacheToFree)
{
_log(string.Format("Free cache from ExecutionPlan::ExecutionPlan for {0}", c.Item1));
c.Item2.FreeMemory(unload: true, freeUncompressedMemory: false);
}
//asc: clean cache buckets cache
_hash.Commit();
}
_log(string.Format("query time (ms) = {0}", FastDateTime.Now.Subtract(now).TotalMilliseconds));
return(result);
}
public WAHBitArray Op(WAHBitArray bits, OPERATION op)
{
if (bits == null)
{
throw new InvalidOperationException("Bits is null");
}
if (op == OPERATION.AND)
return _bits.And(bits);
if (op == OPERATION.OR)
return _bits.Or(bits);
return bits.And(_bits.Not());
}
public void SetBit(int index, bool val)
{
if (_bits != null)
_bits.Set(index, val);
else
{
_bits = new WAHBitArray();
_bits.Set(index, val);
}
isDirty = true;
}
public void FreeMemory(bool unload, bool freeUncompressedMemory = true)
{
if (freeUncompressedMemory && _bits != null)
_bits.FreeMemory();
if (unload)
{
_bits = null;
isLoaded = false;
}
}
public void SetCompressedBits(uint[] bits)
{
_bits = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, bits);
LastBitSaveLength = bits.Length;
isLoaded = true;
isDirty = false;
}
public WAHBitArray Or(WAHBitArray op)
{
CheckBitArray(op);
BitArray b = (BitArray)_ba.Clone();
return op.Or(b);
}
public QueryPlanResult(WAHBitArray bitArray, IEnumerable<KeyValuePair<DocumentIndexTypeToken,int>> versions)
{
_bitArray = bitArray;
_versions = versions.ToDictionary(p => p.Key, p => p.Value);
}
public WAHBitArray And(WAHBitArray op)
{
CheckBitArray(op);
return op.And(_ba);
}
private void CheckBitArray(WAHBitArray op)
{
CheckBitArray();
if (op != null)
{
int L1 = _ba.Length;
int L2 = op.Length;
if (L1 != L2)
{
if (L1 > L2)
op.Length = L1;
else
_ba.Length = L2;
}
_size = _ba.Length;
}
}
public WAHBitArray Xor(WAHBitArray op)
{
CheckBitArray(op);
return op.Xor(_ba);
}
public WAHBitArray Xor(WAHBitArray op)
{
lock (_lock)
{
uint[] left;
uint[] right;
prelogic(op, out left, out right);
for (int i = 0; i < left.Length; i++)
left[i] ^= right[i];
return new WAHBitArray(TYPE.Uncompressed_WAH, left);
}
}
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right)
{
this.CheckBitArray();
left = this.GetUncompressed();
right = op.GetUncompressed();
int ic = left.Length;
int uc = right.Length;
if (ic > uc)
{
uint[] ar = new uint[ic];
right.CopyTo(ar, 0);
right = ar;
}
else if (ic < uc)
{
uint[] ar = new uint[uc];
left.CopyTo(ar, 0);
left = ar;
}
//FixLengths(ints, uncomp);
}