public void TestSparseIds()
{
int maxId = 100000;
int[] count = new int[maxId];
var loader = new BigNestedIntArray.BufferedLoader(maxId);
for (int id = 0; id < maxId; id += ((id >> 2) + 1))
{
for (int val = 0; val < 3000; val += (id + 1))
{
if (loader.Add(id, val)) count[id]++;
}
}
var nestedArray = new BigNestedIntArray();
nestedArray.Load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++)
{
int cnt = nestedArray.GetData(id, buf);
Assert.AreEqual(count[id], cnt, "item count");
if (cnt > 0)
{
int val = 0;
for (int i = 0; i < cnt; i++)
{
Assert.AreEqual(val, buf[i], "item[" + i + "]");
val += (id + 1);
}
}
}
}
public MultiValueORFacetFilter(MultiValueFacetDataCache dataCache, int[] index)
{
_dataCache = dataCache;
_nestedArray = dataCache._nestedArray;
_index = index;
_bitset = new OpenBitSet(_dataCache.valArray.Count);
foreach (int i in _index)
{
_bitset.FastSet(i);
}
}
public void TestMemoryReuse()
{
int maxId = 4096;
int[] maxNumItems = { 1, 1, 2, 2, 3, 3, 3, 3, 1, 1 };
int[] minNumItems = { 1, 1, 0, 1, 0, 0, 2, 3, 1, 0 };
int[] count = new int[maxId];
BigIntBuffer buffer = new BigIntBuffer();
BigNestedIntArray.BufferedLoader loader = null;
BigNestedIntArray nestedArray = new BigNestedIntArray();
Random rand = new Random();
for (int i = 0; i < maxNumItems.Length; i++)
{
loader = new BigNestedIntArray.BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
for (int id = 0; id < maxId; id++)
{
count[id] = 0;
int cnt = Math.Max(rand.Next(maxNumItems[i] + 1), minNumItems[i]);
for (int val = 0; val < cnt; val++)
{
if (loader.Add(id, val))
{
count[id]++;
}
}
}
nestedArray.Load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++)
{
int cnt = nestedArray.GetData(id, buf);
Assert.AreEqual(count[id], cnt, "count[" + i + "," + id + "]");
if (cnt > 0)
{
for (int val = 0; val < cnt; val++)
{
Assert.AreEqual(val, buf[val], "item[" + i + "," + id + "," + val + "]");
}
}
}
if (i == 0)
{
maxId = maxId * 2;
count = new int[maxId];
}
}
}
public void TestAllocThenAddData()
{
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 1024, 2 };
int[,] count = new int[maxNumItems.Length, maxId];
AllocOnlyTestLoader loader = new AllocOnlyTestLoader(maxId);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.Length];
for (int i = 0; i < maxNumItems.Length; i++)
{
for (int id = 0; id < maxId; id++)
{
int cnt = id % (maxNumItems[i] + 1);
loader.AddSize(id, cnt);
count[i, id] = cnt;
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].Load(maxId, loader);
loader.Reset();
for (int id = 0; id < maxId; id++)
{
for (int data = 0; data < count[i, id]; data++)
{
nestedArray[i].AddData(id, data);
}
}
}
for (int i = 0; i < maxNumItems.Length; i++)
{
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++)
{
int cnt = nestedArray[i].GetData(id, buf);
Assert.AreEqual(count[i, id], cnt, "count[" + i + "," + id + "]");
if (cnt > 0)
{
for (int val = 0; val < cnt; val++)
{
Assert.AreEqual(val, buf[val], "item[" + i + "," + id + "," + val + "]");
}
}
}
}
}
public void TestBufferedLoaderReuse()
{
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 2000, 2 };
int[,] count = new int[maxNumItems.Length, maxId];
var buffer = new BigIntBuffer();
var loader = new BigNestedIntArray.BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
var nestedArray = new BigNestedIntArray[maxNumItems.Length];
for (int i = 0; i < maxNumItems.Length; i++)
{
for (int id = 0; id < maxId; id++)
{
int cnt = id % (maxNumItems[i] + 1);
for (int val = 0; val < cnt; val++)
{
if (loader.Add(id, val))
{
count[i, id]++;
}
}
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].Load(maxId, loader);
loader.Reset(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
}
for (int i = 0; i < maxNumItems.Length; i++)
{
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++)
{
int cnt = nestedArray[i].GetData(id, buf);
Assert.AreEqual(count[i, id], cnt, "count[" + i + "," + id + "]");
if (cnt > 0)
{
for (int val = 0; val < cnt; val++)
{
Assert.AreEqual(val, buf[val], "item[" + i + "," + id + "," + val + "]");
}
}
}
}
}
public void TestMaxItems()
{
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 1024, 2 };
int[,] count = new int[maxNumItems.Length, maxId];
AllocOnlyTestLoader loader = new AllocOnlyTestLoader(maxId);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.Length];
for (int i = 0; i < maxNumItems.Length; i++)
{
for (int id = 0; id < maxId; id++)
{
int cnt = id % 2000;
loader.AddSize(id, cnt);
count[i, id] = cnt;
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].MaxItems = maxNumItems[i];
nestedArray[i].Load(maxId, loader);
loader.Reset();
for (int id = 0; id < maxId; id++)
{
bool failed = false;
for (int data = 0; data < count[i, id]; data++)
{
if (nestedArray[i].AddData(id, data))
{
if (!failed && (data + 1 > maxNumItems[i]))
{
failed = true;
Assert.AreEqual(data, maxNumItems[i], "maxItems");
}
}
}
}
}
}
public void TestSparseIds()
{
int maxId = 100000;
int[] count = new int[maxId];
var loader = new BigNestedIntArray.BufferedLoader(maxId);
for (int id = 0; id < maxId; id += ((id >> 2) + 1))
{
for (int val = 0; val < 3000; val += (id + 1))
{
if (loader.Add(id, val))
{
count[id]++;
}
}
}
var nestedArray = new BigNestedIntArray();
nestedArray.Load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++)
{
int cnt = nestedArray.GetData(id, buf);
Assert.AreEqual(count[id], cnt, "item count");
if (cnt > 0)
{
int val = 0;
for (int i = 0; i < cnt; i++)
{
Assert.AreEqual(val, buf[i], "item[" + i + "]");
val += (id + 1);
}
}
}
}
public void TestMemoryReuse()
{
int maxId = 4096;
int[] maxNumItems = { 1, 1, 2, 2, 3, 3, 3, 3, 1, 1 };
int[] minNumItems = { 1, 1, 0, 1, 0, 0, 2, 3, 1, 0 };
int[] count = new int[maxId];
BigIntBuffer buffer = new BigIntBuffer();
BigNestedIntArray.BufferedLoader loader = null;
BigNestedIntArray nestedArray = new BigNestedIntArray();
Random rand = new Random();
for (int i = 0; i < maxNumItems.Length; i++)
{
loader = new BigNestedIntArray.BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
for (int id = 0; id < maxId; id++)
{
count[id] = 0;
int cnt = Math.Max(rand.Next(maxNumItems[i] + 1), minNumItems[i]);
for (int val = 0; val < cnt; val++)
{
if (loader.Add(id, val)) count[id]++;
}
}
nestedArray.Load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++)
{
int cnt = nestedArray.GetData(id, buf);
Assert.AreEqual(count[id], cnt, "count[" + i + "," + id + "]");
if (cnt > 0)
{
for (int val = 0; val < cnt; val++)
{
Assert.AreEqual(val, buf[val], "item[" + i + "," + id + "," + val + "]");
}
}
}
if (i == 0)
{
maxId = maxId * 2;
count = new int[maxId];
}
}
}
public MultiValueFacetDocIdSetIterator(MultiValueFacetDataCache dataCache, int index)
: base(dataCache, index)
{
_nestedArray = dataCache._nestedArray;
}
public MultiValueFacetFilter(MultiValueFacetDataCache dataCache, int index)
{
_dataCache = dataCache;
_nestedArray = dataCache._nestedArray;
_index = index;
}
public MultiValueOrFacetDocIdSetIterator(MultiValueFacetDataCache dataCache, OpenBitSet bs)
: base(dataCache, bs)
{
_nestedArray = dataCache.NestedArray;
}
public MultiRandomAccessDocIdSet(MultiValueFacetDataCache dataCache, OpenBitSet bitset)
{
this.dataCache = dataCache;
this.bitset = bitset;
this.nestedArray = dataCache.NestedArray;
}
internal MultiFacetRangeDocIdSetIterator(int start, int end, MultiValueFacetDataCache dataCache)
{
_start = start;
_end = end;
for (int i = start; i <= end; ++i)
{
_minID = Math.Min(_minID, dataCache.MinIDs[i]);
_maxID = Math.Max(_maxID, dataCache.MaxIDs[i]);
}
_doc = Math.Max(-1, _minID - 1);
nestedArray = dataCache.NestedArray;
}
internal MultiValueFacetCountCollector(BrowseSelection sel, FacetDataCache dataCache, string name, FacetSpec ospec)
: base(sel, dataCache, name, ospec)
{
_array = ((MultiValueFacetDataCache)(_dataCache))._nestedArray;
}
public MultiValuedPathFacetCountCollector(string name, string sep,
BrowseSelection sel, FacetSpec ospec, FacetDataCache dataCache)
: base(name, sep, sel, ospec, dataCache)
{
_array = ((MultiValueFacetDataCache)(dataCache)).NestedArray;
}
public MultiValueRandomAccessDocIdSet(MultiValueFacetDataCache dataCache, int index)
{
_dataCache = dataCache;
_index = index;
_nestedArray = dataCache.NestedArray;
}
public MultiValueWithWeightFacetDataCache()
{
_weightArray = new BigNestedIntArray();
}
public MultiValueFacetDataCache()
{
_nestedArray = new BigNestedIntArray();
}
public MultiValueFacetDocIdSetIterator(MultiValueFacetDataCache dataCache, int[] index, OpenBitSet bs)
: base(dataCache, index, bs)
{
_nestedArray = dataCache._nestedArray;
}
public void TestAllocThenAddData()
{
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 1024, 2 };
int[,] count = new int[maxNumItems.Length, maxId];
AllocOnlyTestLoader loader = new AllocOnlyTestLoader(maxId);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.Length];
for(int i = 0 ; i < maxNumItems.Length; i++)
{
for(int id = 0; id < maxId; id++)
{
int cnt = id % (maxNumItems[i] + 1);
loader.AddSize(id, cnt);
count[i, id] = cnt;
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].Load(maxId, loader);
loader.Reset();
for(int id = 0; id < maxId; id++)
{
for(int data = 0; data < count[i, id]; data++)
{
nestedArray[i].AddData(id, data);
}
}
}
for(int i = 0 ; i < maxNumItems.Length; i++)
{
int[] buf = new int[1024];
for(int id = 0; id < maxId; id++)
{
int cnt = nestedArray[i].GetData(id, buf);
Assert.AreEqual(count[i, id], cnt, "count[" + i + "," + id + "]");
if(cnt > 0)
{
for(int val = 0; val < cnt; val++)
{
Assert.AreEqual(val, buf[val], "item[" + i + "," + id + "," + val + "]");
}
}
}
}
}
public void TestMaxItems()
{
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 1024, 2 };
int[,] count = new int[maxNumItems.Length, maxId];
AllocOnlyTestLoader loader = new AllocOnlyTestLoader(maxId);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.Length];
for(int i = 0 ; i < maxNumItems.Length; i++)
{
for(int id = 0; id < maxId; id++)
{
int cnt = id % 2000;
loader.AddSize(id, cnt);
count[i, id] = cnt;
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].MaxItems = maxNumItems[i];
nestedArray[i].Load(maxId, loader);
loader.Reset();
for(int id = 0; id < maxId; id++)
{
bool failed = false;
for(int data = 0; data < count[i, id]; data++)
{
if(nestedArray[i].AddData(id, data))
{
if(!failed && (data + 1 > maxNumItems[i]))
{
failed = true;
Assert.AreEqual(data, maxNumItems[i], "maxItems");
}
}
}
}
}
}
public RangeRandomAccessDocIdSet(int start, int end, FacetDataCache dataCache, BigNestedIntArray nestedArray, bool multi)
{
_start = start;
_end = end;
_dataCache = dataCache;
_nestedArray = nestedArray;
_multi = multi;
}
public void TestBufferedLoaderReuse()
{
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 2000, 2 };
int[,] count = new int[maxNumItems.Length, maxId];
var buffer = new BigIntBuffer();
var loader = new BigNestedIntArray.BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
var nestedArray = new BigNestedIntArray[maxNumItems.Length];
for (int i = 0; i < maxNumItems.Length; i++)
{
for(int id = 0; id < maxId; id++)
{
int cnt = id % (maxNumItems[i] + 1);
for(int val = 0; val < cnt; val++)
{
if(loader.Add(id, val)) count[i, id]++;
}
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].Load(maxId, loader);
loader.Reset(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
}
for (int i = 0; i < maxNumItems.Length; i++)
{
int[] buf = new int[1024];
for(int id = 0; id < maxId; id++)
{
int cnt = nestedArray[i].GetData(id, buf);
Assert.AreEqual(count[i, id], cnt, "count[" + i + "," + id + "]");
if(cnt > 0)
{
for(int val = 0; val < cnt; val++)
{
Assert.AreEqual(val, buf[val], "item[" + i + "," + id + "," + val + "]");
}
}
}
}
}
internal MultiValueDocScorer(MultiValueFacetDataCache dataCache, IFacetTermScoringFunctionFactory scoreFunctionFactory, float[] boostList)
: base(scoreFunctionFactory.GetFacetTermScoringFunction(dataCache.valArray.Count, dataCache._nestedArray.size()), boostList)
{
_dataCache = dataCache;
_array = _dataCache._nestedArray;
}
