//Generate feature id by NGram rules
public long RegenerateFeatureId(BTreeDictionary <long, long> old2new, long ysize)
{
var md5 = new AdvUtils.Security.Cryptography.MD5();
long maxid_ = 0;
Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
{
//Generate new feature id
var addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
var oldValue = maxid_;
while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
{
oldValue = maxid_;
}
//Create existed and new feature ids mapping
lock (thisLock)
{
old2new.Add(
GetId(arrayFeatureFreq[i].strFeature),
oldValue);
}
arrayFeatureFreq[i].value = oldValue;
});
return(maxid_);
}
//Generate feature id by NGram rules
public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
{
AdvUtils.Security.Cryptography.MD5 md5 = new AdvUtils.Security.Cryptography.MD5();
long maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
for (long i = 0;i < arrayFeatureFreqSize;i++)
#else
Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
#endif
{
//Generate new feature id
long addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
long oldValue = maxid_;
while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
{
oldValue = maxid_;
}
//Create existed and new feature ids mapping
lock (thisLock)
{
old2new.Add(
GetId(arrayFeatureFreq[i].strFeature),
oldValue);
}
arrayFeatureFreq[i].value = oldValue;
}
#if NO_SUPPORT_PARALLEL_LIB
#else
);
#endif
return maxid_;
}
public HugeFeatureLexicalDict(int thread_num, uint shrinkMemLoad)
{
SHRINK_AVALI_MEM_LOAD = shrinkMemLoad;
arrayFeatureFreq = new VarBigArray <FeatureFreq>(1024 * 1024);
arrayFeatureFreqSize = 0;
md5 = new AdvUtils.Security.Cryptography.MD5();
parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = thread_num;
}
