public DoubleArrayTrieBuilder(int thread_num)
{
array = null;
thread_num_ = thread_num;
#if NO_SUPPORT_PARALLEL_LIB
#else
lcts = new LimitedConcurrencyLevelTaskScheduler(thread_num_ * 2);
parallelOption = new ParallelOptions();
parallelOption.TaskScheduler = lcts;
#endif
lastQPS = 0.0;
lastQPSDelta = 0.0;
}
public bool build(IList<string> keyList, IList<int> valList, double max_slot_usage_rate_threshold = 0.95)
{
if (keyList == null)
{
Console.WriteLine("Key list is empty");
return false;
}
if (valList == null)
{
Console.WriteLine("Value list is empty");
return false;
}
if (keyList.Count != valList.Count)
{
Console.WriteLine("The size of key list and value list is not equal");
return false;
}
for (int i = 0; i < valList.Count; i++)
{
if (valList[i] <= -1)
{
Console.WriteLine("Invalidated value {0} at index {1}", valList[i], i);
return false;
}
}
MAX_SLOT_USAGE_RATE_THRESHOLD = max_slot_usage_rate_threshold;
slot_usage_rate_threshold_ = max_slot_usage_rate_threshold;
progress_ = 0;
key_ = keyList;
val_ = valList;
startDT = DateTime.Now;
array = new VarBigArray<unit_t>(key_.Count * 5);
used = new VarBigArray<int>(key_.Count * 5);
array[0] = new unit_t();
array[0].base1 = 1;
used[0] = 1;
next_chk_pos_ = 0;
Node root_node = new Node();
root_node.left = 0;
root_node.right = key_.Count;
root_node.depth = 0;
List<Node> siblings = new List<Node>();
fetch(root_node, siblings);
insert(siblings);
return true;
}
/// <summary>
/// Loads ArrayTrie from an arbitrary <see cref="Stream"/>.
/// <paramref name="sourceStream"/> is closed and
/// disposed once this method completes.
/// </summary>
/// <param name="sourceStream">
/// A <see cref="Stream"/> containing the model.
/// </param>
/// <param name="numberOfElementsInChunk">
/// Number of elements (2 int32) in read buffer.
/// Default is 2048 (16K buffer size)</param>
public void Load(Stream sourceStream, int numberOfElementsInChunk = 2048)
{
const int int32Size = sizeof(int);
const int elementSize = int32Size * 2;
var fileSizeInBytes = sourceStream.Length;
var numberOfElements = fileSizeInBytes / elementSize;
#if NO_SUPPORT_VERY_BIG_OBJECT
array = new VarBigArray<sunit_t>(numberOfElements);
#else
array = new sunit_t[numberOfElements];
#endif
using (var sr = new StreamReader(sourceStream))
using (var br = new BinaryReader(sr.BaseStream))
{
var buffersize = elementSize * numberOfElementsInChunk;
var buffer = new byte[elementSize * numberOfElementsInChunk];
var index = 0;
for (long j = 0; j <= numberOfElements / numberOfElementsInChunk; j++)
{
var numberOfReadBytes = br.Read(buffer, 0, buffersize);
if (numberOfReadBytes == buffersize)
{
for (int i = 0; i < numberOfElementsInChunk; i++, index++)
{
var base1 = BitConverter.ToInt32(buffer, elementSize * i);
var check = BitConverter.ToInt32(buffer, (elementSize * i) + int32Size);
array[index] = new sunit_t(base1, check);
}
}
else
{
for (int i = 0; i < numberOfReadBytes / elementSize; i++)
{
var base1 = BitConverter.ToInt32(buffer, elementSize * i);
var check = BitConverter.ToInt32(buffer, (elementSize * i) + int32Size);
array[index++] = new sunit_t(base1, check);
}
break;
}
}
}
}
public VectorQuantization()
{
dataSet = new VarBigArray <double>(1024 * 1024);
dataSetSize = 0;
}
void clear()
{
array = null;
}