//Generate feature id by NGram rules
public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
{
AdvUtils.Security.Cryptography.MD5 md5 = new AdvUtils.Security.Cryptography.MD5();
long maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
for (long i = 0;i < arrayFeatureFreqSize;i++)
#else
Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
#endif
{
//Generate new feature id
long addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
long oldValue = maxid_;
while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
{
oldValue = maxid_;
}
//Create existed and new feature ids mapping
lock (thisLock)
{
old2new.Add(
GetId(arrayFeatureFreq[i].strFeature),
oldValue);
}
arrayFeatureFreq[i].value = oldValue;
}
#if NO_SUPPORT_PARALLEL_LIB
#else
);
#endif
return maxid_;
}
//Regenerate feature id and shrink features with lower frequency
public void Shrink(EncoderTagger[] xList, int freq)
{
BTreeDictionary<long, long> old2new = new BTreeDictionary<long, long>();
featureLexicalDict.Shrink(freq);
maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
int feature_count = xList.Length;
//Update feature ids
#if NO_SUPPORT_PARALLEL_LIB
for (int i = 0;i < feature_cache_.Count;i++)
#else
Parallel.For(0, feature_count, parallelOption, i =>
#endif
{
for (int j = 0; j < xList[i].feature_cache_.Count; j++)
{
List<long> newfs = new List<long>();
long rstValue = 0;
foreach (long v in xList[i].feature_cache_[j])
{
if (old2new.TryGetValue(v, out rstValue) == true)
{
newfs.Add(rstValue);
}
}
xList[i].feature_cache_[j] = newfs.ToArray();
}
}
#if NO_SUPPORT_PARALLEL_LIB
#else
);
#endif
Console.WriteLine("Feature size in total : {0}", maxid_);
}
public DefaultFeatureLexicalDict(int thread_num)
{
featureset_dict_ = new BTreeDictionary <string, FeatureIdPair>(StringComparer.Ordinal, 128);
maxid_ = 0;
parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = thread_num;
}
public void CBTree_AddItem()
{
var btree = new BTreeDictionary <string, int>();
btree.Add("3", 3);
Assert.AreEqual(1, btree.Count);
}
public void CBTree_SetNonExisting()
{
var btree = new BTreeDictionary <string, int>();
btree["3"] = 3;
Assert.AreEqual(3, btree["3"]);
}
public void TestGetOrAdd()
{
BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer);
Assert.AreEqual("a", data.GetOrAdd(1, "a"));
Assert.AreEqual("a", data.GetOrAdd(1, "b"));
}
public void CBTree_Get()
{
var btree = new BTreeDictionary <string, int>();
btree.Add("3", 3);
Assert.AreEqual(3, btree["3"]);
}
//Generate feature id by NGram rules
public long RegenerateFeatureId(BTreeDictionary <long, long> old2new, long ysize)
{
var md5 = new AdvUtils.Security.Cryptography.MD5();
long maxid_ = 0;
Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
{
//Generate new feature id
var addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
var oldValue = maxid_;
while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
{
oldValue = maxid_;
}
//Create existed and new feature ids mapping
lock (thisLock)
{
old2new.Add(
GetId(arrayFeatureFreq[i].strFeature),
oldValue);
}
arrayFeatureFreq[i].value = oldValue;
});
return(maxid_);
}
static void Main(string[] args)
{
BTreeDictionary <string, string> WordLookup = new BTreeDictionary <string, string>();
WordLookup.Add("Hello", "Cool World!");
WordLookup.Add("Hello", "Cooler World!");
WordLookup.Add("Brown", "Fox!");
WordLookup.Add("Foo", "Bar");
string v;
if (WordLookup.TryGetValue("Foo", out v))
{
Console.WriteLine("Foo was found! Value = {0}", v);
}
if (WordLookup.MoveFirst())
{
Console.WriteLine();
Console.WriteLine("Words in Lookup:");
do
{
Console.WriteLine("Key:{0}, Value={1} ", WordLookup.CurrentKey, WordLookup.CurrentValue);
} while (WordLookup.MoveNext());
}
Console.ReadLine();
}
public Scanner(
Program program,
IDynamicLinker dynamicLinker,
IServiceProvider services)
: base(program, services.RequireService <DecompilerEventListener>())
{
this.segmentMap = program.SegmentMap;
this.dynamicLinker = dynamicLinker;
this.Services = services;
this.eventListener = services.RequireService <DecompilerEventListener>();
this.cancelSvc = services.GetService <CancellationTokenSource>();
if (segmentMap == null)
{
throw new InvalidOperationException("Program must have an segment map.");
}
if (program.ImageMap == null)
{
program.ImageMap = segmentMap.CreateImageMap();
}
this.imageMap = program.ImageMap;
this.procQueue = new PriorityQueue <WorkItem>();
this.blocks = new BTreeDictionary <Address, BlockRange>();
this.blockStarts = new Dictionary <Block, Address>();
this.importReferences = program.ImportReferences;
this.visitedProcs = new HashSet <Procedure>();
this.cinj = new CommentInjector(program.User.Annotations);
this.sr = new ScanResults
{
KnownProcedures = program.User.Procedures.Keys.ToHashSet(),
KnownAddresses = program.ImageSymbols.ToDictionary(de => de.Key, de => de.Value),
ICFG = new DiGraph <RtlBlock>(),
};
}
//Regenerate feature id and shrink features with lower frequency
public void Shrink(EncoderTagger[] xList, int freq)
{
var old2new = new BTreeDictionary <long, long>();
featureLexicalDict.Shrink(freq);
maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
var feature_count = xList.Length;
//Update feature ids
Parallel.For(0, feature_count, parallelOption, i =>
{
for (var j = 0; j < xList[i].feature_cache_.Count; j++)
{
var newfs = new List <long>();
long rstValue = 0;
for (int index = 0; index < xList[i].feature_cache_[j].Length; index++)
{
var v = xList[i].feature_cache_[j][index];
if (old2new.TryGetValue(v, out rstValue) == true)
{
newfs.Add(rstValue);
}
}
xList[i].feature_cache_[j] = newfs.ToArray();
}
});
Logger.WriteLine("Feature size in total : {0}", maxid_);
}
public void TestRangeEnumerate()
{
BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer);
for (int i = 0; i < 100; i++)
{
Assert.IsTrue(data.TryAdd(i, i.ToString()));
}
int ix = 0;
foreach (KeyValuePair <int, string> kv in data.EnumerateRange(-500, 5000))
{
Assert.AreEqual(ix++, kv.Key);
}
Assert.AreEqual(100, ix);
foreach (
KeyValuePair <int, int> range in
new Dictionary <int, int> {
{ 6, 25 }, { 7, 25 }, { 8, 25 }, { 9, 25 }, { 22, 25 }, { 28, 28 }
})
{
ix = range.Key;
foreach (KeyValuePair <int, string> kv in data.EnumerateRange(ix, range.Value))
{
Assert.AreEqual(ix++, kv.Key);
}
Assert.AreEqual(range.Value, ix - 1);
}
}
public void CBTree_IndexOf_Empty()
{
var btree = new BTreeDictionary <string, int>();
int i = btree.Keys.IndexOf("3");
Assert.AreEqual(-1, i);
}
public void BTree_AddTwoItems()
{
var btree = new BTreeDictionary <string, int>();
btree.Add("3", 3);
btree.Add("2", 2);
Assert.AreEqual(2, btree.Count);
}
public void BTree_SetExisting()
{
var btree = new BTreeDictionary <string, int>();
btree["3"] = 3;
btree["3"] = 2;
Assert.AreEqual(2, btree["3"]);
}
public void CBTree_IndexOf_existing_leaf_item()
{
var btree = new BTreeDictionary <string, int> {
{ "3", 3 }
};
int i = btree.Keys.IndexOf("3");
Assert.AreEqual(0, i);
}
private BTreeDictionary <string, int> Given_Dictionary(IEnumerable <int> items)
{
var btree = new BTreeDictionary <string, int>();
foreach (var item in items)
{
btree.Add(item.ToString(), item);
}
return(btree);
}
public DefaultFeatureLexicalDict(int thread_num)
{
featureset_dict_ = new BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128);
maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
#else
parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = thread_num;
#endif
}
public DefaultFeatureLexicalDict(int thread_num)
{
featureset_dict_ = new BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128);
maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
#else
parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = thread_num;
#endif
}
public void CBTree_IndexOf_nonexisting_small_leafitem()
{
var btree = new BTreeDictionary <string, int> {
{ "3", 3 },
{ "2", 2 }
};
int i = btree.Keys.IndexOf("1");
Assert.AreEqual(~0, i);
}
public void CBTree_IndexOf_nonexisting_large_leafitem()
{
var btree = new BTreeDictionary <string, int> {
{ "4", 4 },
{ "2", 2 }
};
int i = btree.Keys.IndexOf("5");
Assert.AreEqual(~2, i);
}
public void CBTree_ForceInternalNode()
{
var btree = new BTreeDictionary <string, int>();
foreach (var i in Enumerable.Range(0, 256))
{
btree.Add(i.ToString(), i);
}
btree.Add("256", 256);
}
public C64Basic(IServiceProvider services, string archId) : base(services, archId)
{
this.Description = "Commodore 64 Basic";
this.Endianness = EndianServices.Little;
this.PointerType = PrimitiveType.Ptr16;
this.InstructionBitSize = 8;
this.StackRegister = stackRegister;
this.FramePointerType = PrimitiveType.Ptr16;
program = new SortedList <ushort, C64BasicInstruction>();
mpAddrToInstr = new BTreeDictionary <Address, C64BasicInstruction>();
}
public void TestIndexer()
{
BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer, GetSample());
BTreeDictionary <int, string> copy = new BTreeDictionary <int, string>(Comparer);
copy.AddRange(data);
Assert.AreEqual(copy.Count, data.Count);
foreach (int key in data.Keys)
{
Assert.AreEqual(data[key], copy[key]);
}
}
public void CBTree_GetFromDeepTree()
{
var btree = new BTreeDictionary <string, int>();
foreach (var i in Enumerable.Range(0, 1000))
{
btree.Add(i.ToString(), i);
}
btree.Dump();
Assert.AreEqual(0, btree["0"]);
Assert.AreEqual(500, btree["500"]);
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("signDict.exe [raw dictionary] [binary dictionary]");
return;
}
DoubleArrayTrieBuilder daBuilder = new DoubleArrayTrieBuilder(4);
BTreeDictionary <string, int> dict = new BTreeDictionary <string, int>(StringComparer.Ordinal, 128);
string strLine = null;
StreamReader sr = new StreamReader(args[0], Encoding.UTF8);
StreamWriter sw = new StreamWriter(args[1] + ".prob", false, Encoding.UTF8);
BinaryWriter bw = new BinaryWriter(sw.BaseStream);
int index = 0;
while ((strLine = sr.ReadLine()) != null)
{
string[] items = strLine.Split('\t');
string strNGram = items[0].Trim();
if (dict.ContainsKey(strNGram) == true)
{
Console.WriteLine("duplicated line: {0}", strLine);
continue;
}
if (strNGram.Length == 0)
{
continue;
}
string[] vals = items[1].Split();
float prob = float.Parse(vals[0]);
float backoff = float.Parse(vals[1]);
//Write item into file
bw.Write(prob);
bw.Write(backoff);
dict.Add(strNGram, index);
index++;
}
sr.Close();
daBuilder.build(dict);
daBuilder.save(args[1] + ".da");
bw.Close();
}
public void TestFirstAndLast()
{
BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer);
data.Add(1, "a");
data.Add(2, "b");
data.Add(3, "c");
data.Add(4, "d");
data.Add(5, "e");
Assert.AreEqual(1, data.First().Key);
Assert.AreEqual("a", data.First().Value);
data.Remove(1);
Assert.AreEqual(2, data.First().Key);
Assert.AreEqual("b", data.First().Value);
Assert.AreEqual(5, data.Last().Key);
Assert.AreEqual("e", data.Last().Value);
data.Remove(5);
Assert.AreEqual(4, data.Last().Key);
Assert.AreEqual("d", data.Last().Value);
data.Remove(4);
data.Remove(3);
KeyValuePair <int, string> kv;
Assert.IsTrue(data.TryGetLast(out kv));
Assert.IsTrue(data.TryGetFirst(out kv));
data.Remove(2);
Assert.IsFalse(data.TryGetLast(out kv));
Assert.IsFalse(data.TryGetFirst(out kv));
try
{
data.First();
Assert.Fail("Should raise InvalidOperationException");
}
catch (InvalidOperationException)
{
}
try
{
data.Last();
Assert.Fail("Should raise InvalidOperationException");
}
catch (InvalidOperationException)
{
}
}
public void TestArray()
{
List <KeyValuePair <int, string> > sample = new List <KeyValuePair <int, string> >(GetSample());
BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer, sample);
sample.Sort((a, b) => data.Comparer.Compare(a.Key, b.Key));
KeyValuePair <int, string>[] array = data.ToArray();
Assert.AreEqual(sample.Count, array.Length);
for (int i = 0; i < sample.Count; i++)
{
Assert.AreEqual(sample[i].Key, array[i].Key);
Assert.AreEqual(sample[i].Value, array[i].Value);
}
}
internal static BTreeDictionary <DateTime, int> LoadStatsFromDisk(string statsPath)
{
BTreeDictionary <DateTime, int> stats = new BTreeDictionary <DateTime, int>();
using (StreamReader reader = new StreamReader(statsPath))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
string[] kvp = line.Split('#');
stats.Add(DateTime.Parse(kvp[0]), Int32.Parse(kvp[1]));
}
}
return(stats);
}
public void CBTree_Enumerate()
{
var btree = new BTreeDictionary <string, int>();
btree.Add("3", 3);
btree.Add("2", 2);
var e = btree.GetEnumerator();
Assert.IsTrue(e.MoveNext());
Assert.AreEqual("2", e.Current.Key);
Assert.AreEqual(2, e.Current.Value);
Assert.IsTrue(e.MoveNext());
Assert.AreEqual("3", e.Current.Key);
Assert.AreEqual(3, e.Current.Value);
Assert.IsFalse(e.MoveNext());
}
public void TestClone()
{
BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer, GetSample());
BTreeDictionary <int, string> copy = (BTreeDictionary <int, string>)((ICloneable)data).Clone();
using (IEnumerator <KeyValuePair <int, string> > e1 = data.GetEnumerator())
using (IEnumerator <KeyValuePair <int, string> > e2 = copy.GetEnumerator())
{
while (e1.MoveNext() && e2.MoveNext())
{
Assert.AreEqual(e1.Current.Key, e2.Current.Key);
Assert.AreEqual(e1.Current.Value, e2.Current.Value);
}
Assert.IsFalse(e1.MoveNext() || e2.MoveNext());
}
}
static void Main(string[] args)
{
//Test BTreeDictionary for very large data set
BTreeDictionary<long, long> sdict = new BTreeDictionary<long, long>();
for (long i = 10000; i >= 0; i-=2)
{
sdict.Add(i, i);
}
long val = sdict.KeyList[123];
long n = sdict.ValueList[123];
sdict.RemoveAt(123);
long cnt = sdict.ValueList.Count;
Console.WriteLine("Done.");
Console.ReadLine();
}
//Regenerate feature id and shrink features with lower frequency
public void Shrink(EncoderTagger[] xList, int freq)
{
var old2new = new BTreeDictionary<long, long>();
featureLexicalDict.Shrink(freq);
maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
var feature_count = xList.Length;
//Update feature ids
#if NO_SUPPORT_PARALLEL_LIB
for (int i = 0;i < feature_cache_.Count;i++)
#else
Parallel.For(0, feature_count, parallelOption, i =>
#endif
{
for (var j = 0; j < xList[i].feature_cache_.Count; j++)
{
var newfs = new List<long>();
long rstValue = 0;
for (int index = 0; index < xList[i].feature_cache_[j].Length; index++)
{
var v = xList[i].feature_cache_[j][index];
if (old2new.TryGetValue(v, out rstValue) == true)
{
newfs.Add(rstValue);
}
}
xList[i].feature_cache_[j] = newfs.ToArray();
}
}
#if NO_SUPPORT_PARALLEL_LIB
#else
);
#endif
Console.WriteLine("Feature size in total : {0}", maxid_);
}
public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
{
long new_maxid = 0;
//Regenerate new feature id and create feature ids mapping
foreach (var it in featureset_dict_)
{
var strFeature = it.Key;
//Regenerate new feature id
old2new.Add(it.Value.Key, new_maxid);
it.Value.Key = new_maxid;
var addValue = (strFeature[0] == 'U' ? ysize : ysize * ysize);
new_maxid += addValue;
}
return new_maxid;
}
public void Clear()
{
featureset_dict_.Clear();
featureset_dict_ = null;
}
//加载model文件
//返回值<0 为出错，=0为正常
public bool LoadModel(string filename)
{
var sr = new StreamReader(filename);
string strLine;
//读入版本号
strLine = sr.ReadLine();
version = uint.Parse(strLine.Split(':')[1].Trim());
//读入cost_factor
strLine = sr.ReadLine();
cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());
//读入maxid
strLine = sr.ReadLine();
maxid_ = long.Parse(strLine.Split(':')[1].Trim());
//读入xsize
strLine = sr.ReadLine();
xsize_ = uint.Parse(strLine.Split(':')[1].Trim());
//读入空行
strLine = sr.ReadLine();
//读入待标注的标签
y_ = new List<string>();
while (true)
{
strLine = sr.ReadLine();
if (strLine.Length == 0)
{
break;
}
y_.Add(strLine);
}
//读入unigram和bigram模板
unigram_templs_ = new List<string>();
bigram_templs_ = new List<string>();
while (sr.EndOfStream == false)
{
strLine = sr.ReadLine();
if (strLine.Length == 0)
{
break;
}
if (strLine[0] == 'U')
{
unigram_templs_.Add(strLine);
}
if (strLine[0] == 'B')
{
bigram_templs_.Add(strLine);
}
}
sr.Close();
//Load all feature set data
var filename_feature = filename + ".feature";
da = new DoubleArrayTrieSearch();
da.Load(filename_feature);
//Load all features alpha data
var filename_alpha = filename + ".alpha";
var sr_alpha = new StreamReader(filename_alpha);
var br_alpha = new BinaryReader(sr_alpha.BaseStream);
if (version == Utils.MODEL_TYPE_NORM)
{
//feature weight array
alpha_two_tuples = null;
alpha_ = new double[maxid_ + 1];
for (long i = 0; i < maxid_; i++)
{
alpha_[i] = br_alpha.ReadSingle();
}
}
else if (version == Utils.MODEL_TYPE_SHRINKED)
{
alpha_ = null;
alpha_two_tuples = new BTreeDictionary<long, double>();
for (long i = 0; i < maxid_; i++)
{
var key = br_alpha.ReadInt64();
double weight = br_alpha.ReadSingle();
alpha_two_tuples.Add(key, weight);
}
}
else
{
Console.WriteLine("This model is not supported.");
return false;
}
br_alpha.Close();
return true;
}
