public void TestBigIntArray()
{
int count = 5000000;
var test = new BigIntArray(count);
var test2 = new int[count];
for (int i = 0; i < count; i++)
{
test.Add(i, i);
test2[i] = i;
}
for (int i = 0; i < count; i++)
{
Assert.AreEqual(0, test.Get(0));
}
int k = 0;
long start = System.Environment.TickCount;
for (int i = 0; i < count; i++)
{
k = test.Get(i);
}
long end = System.Environment.TickCount;
Console.WriteLine("Big array took: " + (end - start));
start = System.Environment.TickCount;
for (int i = 0; i < count; i++)
{
k = test2[i];
}
end = System.Environment.TickCount;
Console.WriteLine("int[] took: " + (end - start));
}
public void TestBigIntArray()
{
int count = 5000000;
var test = new BigIntArray(count);
var test2 = new int[count];
for (int i = 0; i < count; i++)
{
test.Add(i, i);
test2[i] = i;
}
for (int i = 0; i < count; i++)
{
Assert.AreEqual(0, test.Get(0));
}
int k = 0;
var sw = new Stopwatch();
sw.Start();
for (int i = 0; i < count; i++)
{
k = test.Get(i);
}
sw.Stop();
Console.WriteLine("Big array took: " + sw.ElapsedMilliseconds.ToString());
sw.Reset();
sw.Start();
for (int i = 0; i < count; i++)
{
k = test2[i];
}
sw.Stop();
Console.WriteLine("int[] took: " + sw.ElapsedMilliseconds.ToString());
}
/// <summary>
/// adds a pair of id and value to the buffer
/// </summary>
/// <param name="id"></param>
/// <param name="val"></param>
/// <returns></returns>
public bool Add(int id, int val)
{
int ptr = _info.Get(id << 1);
if (ptr == EOD)
{
// 1st insert
_info.Add(id << 1, val);
return(true);
}
int cnt = _info.Get((id << 1) + 1);
if (cnt == EOD)
{
// 2nd insert
_info.Add((id << 1) + 1, val);
return(true);
}
if (ptr >= 0)
{
// this id has two values stored in-line.
int firstVal = ptr;
int secondVal = cnt;
ptr = _buffer.Alloc(SEGSIZE);
_buffer.Set(ptr++, EOD);
_buffer.Set(ptr++, firstVal);
_buffer.Set(ptr++, secondVal);
_buffer.Set(ptr++, val);
cnt = 3;
}
else
{
ptr = (-ptr);
if (cnt >= _maxItems) // exceeded the limit
{
return(false);
}
if ((ptr % SEGSIZE) == 0)
{
int oldPtr = ptr;
ptr = _buffer.Alloc(SEGSIZE);
_buffer.Set(ptr++, (-oldPtr));
}
_buffer.Set(ptr++, val);
cnt++;
}
_info.Add(id << 1, (-ptr));
_info.Add((id << 1) + 1, cnt);
return(true);
}
public void TestBigIntArray()
{
int count = 5000000;
var test = new BigIntArray(count);
var test2 = new int[count];
for (int i = 0; i < count; i++)
{
test.Add(i, i);
test2[i] = i;
}
for (int i = 0; i < count; i++)
{
Assert.AreEqual(0, test.Get(0));
}
int k = 0;
long start = System.Environment.TickCount;
for (int i = 0; i < count; i++)
{
k = test.Get(i);
}
long end = System.Environment.TickCount;
Console.WriteLine("Big array took: " + (end - start));
start = System.Environment.TickCount;
for (int i = 0; i < count; i++)
{
k = test2[i];
}
end = System.Environment.TickCount;
Console.WriteLine("int[] took: " + (end - start));
}
public override void Load(BoboIndexReader reader)
{
int maxDoc = reader.MaxDoc;
BigIntArray order = new BigIntArray(maxDoc);
ITermValueList mterms = _termListFactory == null ? new TermStringList() : _termListFactory.CreateTermList();
List<int> minIDList = new List<int>();
List<int> maxIDList = new List<int>();
List<int> freqList = new List<int>();
TermDocs termDocs = null;
TermEnum termEnum = null;
int t = 0; // current term number
mterms.Add(null);
minIDList.Add(-1);
maxIDList.Add(-1);
freqList.Add(0);
t++;
try
{
termDocs = reader.TermDocs();
termEnum = reader.Terms(new Term(_indexFieldName, ""));
do
{
if (termEnum == null)
break;
Term term = termEnum.Term;
if (term == null || !_indexFieldName.Equals(term.Field))
break;
// store term text
// we expect that there is at most one term per document
if (t > MAX_VAL_COUNT)
{
throw new IOException("maximum number of value cannot exceed: " + MAX_VAL_COUNT);
}
string val = term.Text;
mterms.Add(val);
int bit = (0x00000001 << (t - 1));
termDocs.Seek(termEnum);
//freqList.add(termEnum.docFreq()); // removed because the df doesn't take into account the num of deletedDocs
int df = 0;
int minID = -1;
int maxID = -1;
if (termDocs.Next())
{
df++;
int docid = termDocs.Doc;
order.Add(docid, order.Get(docid) | bit);
minID = docid;
while (termDocs.Next())
{
df++;
docid = termDocs.Doc;
order.Add(docid, order.Get(docid) | bit);
}
maxID = docid;
}
freqList.Add(df);
minIDList.Add(minID);
maxIDList.Add(maxID);
t++;
} while (termEnum.Next());
}
finally
{
try
{
if (termDocs != null)
{
termDocs.Dispose();
}
}
finally
{
if (termEnum != null)
{
termEnum.Dispose();
}
}
}
mterms.Seal();
_dataCache = new FacetDataCache(order, mterms, freqList.ToArray(), minIDList.ToArray(), maxIDList.ToArray(), TermCountSize.Large);
}
