public void BTree_GetFromDeepTree()
{
var btree = new BTreeDictionary <string, int>();
foreach (var i in Enumerable.Range(0, 1000))
{
btree.Add(i.ToString(), i);
}
btree.Dump();
Assert.AreEqual(0, btree["0"]);
Assert.AreEqual(500, btree["500"]);
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("signDict.exe [raw dictionary] [binary dictionary]");
return;
}
DoubleArrayTrieBuilder daBuilder = new DoubleArrayTrieBuilder(4);
BTreeDictionary <string, int> dict = new BTreeDictionary <string, int>(StringComparer.Ordinal, 128);
string strLine = null;
StreamReader sr = new StreamReader(args[0], Encoding.UTF8);
StreamWriter sw = new StreamWriter(args[1] + ".prob", false, Encoding.UTF8);
BinaryWriter bw = new BinaryWriter(sw.BaseStream);
int index = 0;
while ((strLine = sr.ReadLine()) != null)
{
string[] items = strLine.Split('\t');
string strNGram = items[0].Trim();
if (dict.ContainsKey(strNGram) == true)
{
Console.WriteLine("duplicated line: {0}", strLine);
continue;
}
if (strNGram.Length == 0)
{
continue;
}
string[] vals = items[1].Split();
float prob = float.Parse(vals[0]);
float backoff = float.Parse(vals[1]);
//Write item into file
bw.Write(prob);
bw.Write(backoff);
dict.Add(strNGram, index);
index++;
}
sr.Close();
daBuilder.build(dict);
daBuilder.save(args[1] + ".da");
bw.Close();
}
/// <summary>
/// Adds a new basic block to the procedure <paramref name="proc"/>.
/// </summary>
/// <param name="addr"></param>
/// <param name="proc"></param>
/// <param name="blockName"></param>
/// <returns></returns>
public Block AddBlock(Address addr, Procedure proc, string blockName)
{
Block b = new Block(proc, addr, blockName);
if (!blocks.TryGetUpperBound(addr, out var br))
{
var lastMem = segmentMap.Segments.Values.Last().MemoryArea;
blocks.Add(addr, new BlockRange(b, addr.ToLinear(), lastMem.BaseAddress.ToLinear() + (uint)lastMem.Length));
}
else
{
blocks.Add(addr, new BlockRange(b, addr.ToLinear(), br.Start));
}
blockStarts.Add(b, addr);
proc.ControlGraph.Blocks.Add(b);
imageMap.AddItem(addr, new ImageMapBlock(addr)
{
Block = b
});
return(b);
}
/// <summary>
/// Inserts blanks entries before the current location.
/// The current location becomes the first inserted item.
/// </summary>
/// <param name="quantity">Number of entries to insert.</param>
public virtual void Insert(int quantity = 1)
{
// Shifts current indices
if (_fonts.MoveLast())
{
do
{
if (_fonts.CurrentKey >= _index)
{
Font currentFont = _fonts.CurrentValue;
int newKey = _fonts.CurrentKey + quantity;
_fonts.Remove();
_fonts.Add(newKey, currentFont);
}
} while (_fonts.MovePrevious());
}
// Adds blank spaces
for (int i = _index; i < _index + quantity; i++)
{
_fonts.Add(_index, null);
}
}
internal static BTreeDictionary <DateTime, int> LoadStatsFromDisk(string statsPath)
{
BTreeDictionary <DateTime, int> stats = new BTreeDictionary <DateTime, int>();
using (StreamReader reader = new StreamReader(statsPath))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
string[] kvp = line.Split('#');
stats.Add(DateTime.Parse(kvp[0]), Int32.Parse(kvp[1]));
}
}
return(stats);
}
private static void AddDefaultTagset()
{
Index = new BTreeDictionary <string, List <string> >();
Index.Add("default", new List <string> {
"JobSearch", "CodeHelp", "FCCBug", "SoftwareSupport", "OSSelection", "Meeting"
});
ColorIndex = new BTreeDictionary <string, Dictionary <string, Color> >();
ColorIndex.Add("default", new Dictionary <string, Color>());
Color[] colors = ColorGenerator.GenerateHSLuvColors(6);
for (int i = 0; i < Index["default"].Count; i++)
{
ColorIndex["default"].Add(Index["default"][i], colors[i]);
}
}
public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
{
long new_maxid = 0;
//Regenerate new feature id and create feature ids mapping
foreach (KeyValuePair<string, FeatureIdPair> it in featureset_dict_)
{
string strFeature = it.Key;
//Regenerate new feature id
old2new.Add(it.Value.Key, new_maxid);
it.Value.Key = new_maxid;
long addValue = (strFeature[0] == 'U' ? ysize : ysize * ysize);
new_maxid += addValue;
}
return new_maxid;
}
public List <BTreeDictionary <string, int> > GetReadableResultsForTerm(string term)
{
BTreeDictionary <string, int> bi = new BTreeDictionary <string, int>();
BTreeDictionary <string, int> tri = new BTreeDictionary <string, int>();
BTreeDictionary <string, int> four = new BTreeDictionary <string, int>();
BTreeDictionary <string, int> five = new BTreeDictionary <string, int>();
foreach (var kvp in FullIndex)
{
var arr = kvp.Key.Split(' ');
if (arr.Contains(term))
{
switch (arr.Length)
{
case 2:
bi.Add(kvp);
break;
case 3:
tri.Add(kvp);
break;
case 4:
four.Add(kvp);
break;
case 5:
five.Add(kvp);
break;
default:
Console.WriteLine("Whoops");
break;
}
}
}
var ret = new List <BTreeDictionary <string, int> >();
ret.Add(bi);
ret.Add(tri);
ret.Add(four);
ret.Add(five);
return(ret);
}
static void Main(string[] args)
{
//Test BTreeDictionary for very large data set
BTreeDictionary<long, long> sdict = new BTreeDictionary<long, long>();
for (long i = 10000; i >= 0; i-=2)
{
sdict.Add(i, i);
}
long val = sdict.KeyList[123];
long n = sdict.ValueList[123];
sdict.RemoveAt(123);
long cnt = sdict.ValueList.Count;
Console.WriteLine("Done.");
Console.ReadLine();
}
static void Main(string[] args)
{
//Test BTreeDictionary for very large data set
BTreeDictionary <long, long> sdict = new BTreeDictionary <long, long>();
for (long i = 10000; i >= 0; i -= 2)
{
sdict.Add(i, i);
}
long val = sdict.KeyList[123];
long n = sdict.ValueList[123];
sdict.RemoveAt(123);
long cnt = sdict.ValueList.Count;
Console.WriteLine("Done.");
Console.ReadLine();
}
public static void ReadIndexFromDisk(string file, string colorfile)
{
var jsonString = File.ReadAllText(file);
var jsonString2 = File.ReadAllText(colorfile);
Index = JsonSerializer.Deserialize <BTreeDictionary <string, List <string> > >(jsonString);
//ColorIndex = JsonSerializer.Deserialize<BTreeDictionary<string,Dictionary<string, Color>>>(jsonString2);
ColorIndex = new BTreeDictionary <string, Dictionary <string, Color> >();
using (StreamReader sr = new StreamReader(colorfile))
{
ColorConverter colorConverter = new ColorConverter();
Dictionary <string, Color> val = new Dictionary <string, Color>();
string tagset = sr.ReadLine().Split(' ')[1];
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
var arr = line.Split(' ');
if (!arr[0].StartsWith("Tagset") && !arr[0].StartsWith("End"))
{
int argb = Int32.Parse(arr[1], NumberStyles.HexNumber);
Color clr = Color.FromArgb(argb);
val.Add(arr[0], clr);
}
if (arr[0].StartsWith("Tagset"))
{
tagset = arr[1];
}
if (arr[0].StartsWith("End"))
{
ColorIndex.Add(tagset, val);
val = new Dictionary <string, Color>();
}
}
}
}
//加载model文件
//返回值<0 为出错，=0为正常
public bool LoadModel(string filename)
{
StreamReader sr = new StreamReader(filename);
string strLine;
//读入版本号
strLine = sr.ReadLine();
version = uint.Parse(strLine.Split(':')[1].Trim());
//读入cost_factor
strLine = sr.ReadLine();
cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());
//读入maxid
strLine = sr.ReadLine();
maxid_ = long.Parse(strLine.Split(':')[1].Trim());
//读入xsize
strLine = sr.ReadLine();
xsize_ = uint.Parse(strLine.Split(':')[1].Trim());
//读入空行
strLine = sr.ReadLine();
//读入待标注的标签
y_ = new List <string>();
while (true)
{
strLine = sr.ReadLine();
if (strLine.Length == 0)
{
break;
}
y_.Add(strLine);
}
//读入unigram和bigram模板
unigram_templs_ = new List <string>();
bigram_templs_ = new List <string>();
while (sr.EndOfStream == false)
{
strLine = sr.ReadLine();
if (strLine.Length == 0)
{
break;
}
if (strLine[0] == 'U')
{
unigram_templs_.Add(strLine);
}
if (strLine[0] == 'B')
{
bigram_templs_.Add(strLine);
}
}
sr.Close();
//Load all feature set data
string filename_feature = filename + ".feature";
da = new DoubleArrayTrieSearch();
da.Load(filename_feature);
//Load all features alpha data
string filename_alpha = filename + ".alpha";
StreamReader sr_alpha = new StreamReader(filename_alpha);
BinaryReader br_alpha = new BinaryReader(sr_alpha.BaseStream);
if (version == Utils.MODEL_TYPE_NORM)
{
//feature weight array
alpha_two_tuples = null;
alpha_ = new double[maxid_ + 1];
for (long i = 0; i < maxid_; i++)
{
alpha_[i] = br_alpha.ReadSingle();
}
}
else if (version == Utils.MODEL_TYPE_SHRINKED)
{
alpha_ = null;
alpha_two_tuples = new BTreeDictionary <long, double>();
for (long i = 0; i < maxid_; i++)
{
long key = br_alpha.ReadInt64();
double weight = br_alpha.ReadSingle();
alpha_two_tuples.Add(key, weight);
}
}
else
{
Console.WriteLine("This model is not supported.");
return(false);
}
br_alpha.Close();
return(true);
}
//加载model文件
//返回值<0 为出错，=0为正常
public bool LoadModel(string filename)
{
var sr = new StreamReader(filename);
string strLine;
//读入版本号
strLine = sr.ReadLine();
version = uint.Parse(strLine.Split(':')[1].Trim());
//读入cost_factor
strLine = sr.ReadLine();
cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());
//读入maxid
strLine = sr.ReadLine();
maxid_ = long.Parse(strLine.Split(':')[1].Trim());
//读入xsize
strLine = sr.ReadLine();
xsize_ = uint.Parse(strLine.Split(':')[1].Trim());
//读入空行
strLine = sr.ReadLine();
//读入待标注的标签
y_ = new List<string>();
while (true)
{
strLine = sr.ReadLine();
if (strLine.Length == 0)
{
break;
}
y_.Add(strLine);
}
//读入unigram和bigram模板
unigram_templs_ = new List<string>();
bigram_templs_ = new List<string>();
while (sr.EndOfStream == false)
{
strLine = sr.ReadLine();
if (strLine.Length == 0)
{
break;
}
if (strLine[0] == 'U')
{
unigram_templs_.Add(strLine);
}
if (strLine[0] == 'B')
{
bigram_templs_.Add(strLine);
}
}
sr.Close();
//Load all feature set data
var filename_feature = filename + ".feature";
da = new DoubleArrayTrieSearch();
da.Load(filename_feature);
//Load all features alpha data
var filename_alpha = filename + ".alpha";
var sr_alpha = new StreamReader(filename_alpha);
var br_alpha = new BinaryReader(sr_alpha.BaseStream);
if (version == Utils.MODEL_TYPE_NORM)
{
//feature weight array
alpha_two_tuples = null;
alpha_ = new double[maxid_ + 1];
for (long i = 0; i < maxid_; i++)
{
alpha_[i] = br_alpha.ReadSingle();
}
}
else if (version == Utils.MODEL_TYPE_SHRINKED)
{
alpha_ = null;
alpha_two_tuples = new BTreeDictionary<long, double>();
for (long i = 0; i < maxid_; i++)
{
var key = br_alpha.ReadInt64();
double weight = br_alpha.ReadSingle();
alpha_two_tuples.Add(key, weight);
}
}
else
{
Console.WriteLine("This model is not supported.");
return false;
}
br_alpha.Close();
return true;
}
public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
{
long new_maxid = 0;
//Regenerate new feature id and create feature ids mapping
foreach (var it in featureset_dict_)
{
var strFeature = it.Key;
//Regenerate new feature id
old2new.Add(it.Value.Key, new_maxid);
it.Value.Key = new_maxid;
var addValue = (strFeature[0] == 'U' ? ysize : ysize * ysize);
new_maxid += addValue;
}
return new_maxid;
}
