예제 #1
0
        //Generate feature id by NGram rules
        public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
        {
            AdvUtils.Security.Cryptography.MD5 md5 = new AdvUtils.Security.Cryptography.MD5();
            long maxid_ = 0;

#if NO_SUPPORT_PARALLEL_LIB
            for (long i = 0;i < arrayFeatureFreqSize;i++)
#else
            Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
#endif
            {
                //Generate new feature id
                long addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
                long oldValue = maxid_;
                while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
                {
                    oldValue = maxid_;
                }

                //Create existed and new feature ids mapping
                lock (thisLock)
                {
                    old2new.Add(
                        GetId(arrayFeatureFreq[i].strFeature),
                        oldValue);
                }

                arrayFeatureFreq[i].value = oldValue;
            }
#if NO_SUPPORT_PARALLEL_LIB
#else
            );
#endif
            return maxid_;
        }
예제 #2
0
        //Regenerate feature id and shrink features with lower frequency
        public void Shrink(EncoderTagger[] xList, int freq)
        {
            BTreeDictionary<long, long> old2new = new BTreeDictionary<long, long>();
            featureLexicalDict.Shrink(freq);
            maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
            int feature_count = xList.Length;

            //Update feature ids
#if NO_SUPPORT_PARALLEL_LIB
            for (int i = 0;i < feature_cache_.Count;i++)
#else
            Parallel.For(0, feature_count, parallelOption, i =>
#endif
            {
                for (int j = 0; j < xList[i].feature_cache_.Count; j++)
                {
                    List<long> newfs = new List<long>();
                    long rstValue = 0;
                    foreach (long v in xList[i].feature_cache_[j])
                    {
                        if (old2new.TryGetValue(v, out rstValue) == true)
                        {
                            newfs.Add(rstValue);
                        }
                    }
                    xList[i].feature_cache_[j] = newfs.ToArray();
                }
            }
#if NO_SUPPORT_PARALLEL_LIB
#else
            );
#endif

            Console.WriteLine("Feature size in total : {0}", maxid_);
        }
 public DefaultFeatureLexicalDict(int thread_num)
 {
     featureset_dict_ = new BTreeDictionary <string, FeatureIdPair>(StringComparer.Ordinal, 128);
     maxid_           = 0;
     parallelOption   = new ParallelOptions();
     parallelOption.MaxDegreeOfParallelism = thread_num;
 }
예제 #4
0
        public void CBTree_AddItem()
        {
            var btree = new BTreeDictionary <string, int>();

            btree.Add("3", 3);
            Assert.AreEqual(1, btree.Count);
        }
예제 #5
0
        public void CBTree_SetNonExisting()
        {
            var btree = new BTreeDictionary <string, int>();

            btree["3"] = 3;
            Assert.AreEqual(3, btree["3"]);
        }
예제 #6
0
        public void TestGetOrAdd()
        {
            BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer);

            Assert.AreEqual("a", data.GetOrAdd(1, "a"));
            Assert.AreEqual("a", data.GetOrAdd(1, "b"));
        }
예제 #7
0
        public void CBTree_Get()
        {
            var btree = new BTreeDictionary <string, int>();

            btree.Add("3", 3);
            Assert.AreEqual(3, btree["3"]);
        }
예제 #8
0
        //Generate feature id by NGram rules
        public long RegenerateFeatureId(BTreeDictionary <long, long> old2new, long ysize)
        {
            var  md5    = new AdvUtils.Security.Cryptography.MD5();
            long maxid_ = 0;

            Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
            {
                //Generate new feature id
                var addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
                var oldValue = maxid_;
                while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
                {
                    oldValue = maxid_;
                }

                //Create existed and new feature ids mapping
                lock (thisLock)
                {
                    old2new.Add(
                        GetId(arrayFeatureFreq[i].strFeature),
                        oldValue);
                }

                arrayFeatureFreq[i].value = oldValue;
            });
            return(maxid_);
        }
예제 #9
0
파일: Program.cs 프로젝트: linhabc/20191
        static void Main(string[] args)
        {
            BTreeDictionary <string, string> WordLookup = new BTreeDictionary <string, string>();

            WordLookup.Add("Hello", "Cool World!");
            WordLookup.Add("Hello", "Cooler World!");
            WordLookup.Add("Brown", "Fox!");
            WordLookup.Add("Foo", "Bar");
            string v;

            if (WordLookup.TryGetValue("Foo", out v))
            {
                Console.WriteLine("Foo was found! Value = {0}", v);
            }

            if (WordLookup.MoveFirst())
            {
                Console.WriteLine();
                Console.WriteLine("Words in Lookup:");
                do
                {
                    Console.WriteLine("Key:{0}, Value={1} ", WordLookup.CurrentKey, WordLookup.CurrentValue);
                } while (WordLookup.MoveNext());
            }

            Console.ReadLine();
        }
예제 #10
0
파일: Scanner.cs 프로젝트: ghmole/reko
 public Scanner(
     Program program,
     IDynamicLinker dynamicLinker,
     IServiceProvider services)
     : base(program, services.RequireService <DecompilerEventListener>())
 {
     this.segmentMap    = program.SegmentMap;
     this.dynamicLinker = dynamicLinker;
     this.Services      = services;
     this.eventListener = services.RequireService <DecompilerEventListener>();
     this.cancelSvc     = services.GetService <CancellationTokenSource>();
     if (segmentMap == null)
     {
         throw new InvalidOperationException("Program must have an segment map.");
     }
     if (program.ImageMap == null)
     {
         program.ImageMap = segmentMap.CreateImageMap();
     }
     this.imageMap         = program.ImageMap;
     this.procQueue        = new PriorityQueue <WorkItem>();
     this.blocks           = new BTreeDictionary <Address, BlockRange>();
     this.blockStarts      = new Dictionary <Block, Address>();
     this.importReferences = program.ImportReferences;
     this.visitedProcs     = new HashSet <Procedure>();
     this.cinj             = new CommentInjector(program.User.Annotations);
     this.sr = new ScanResults
     {
         KnownProcedures = program.User.Procedures.Keys.ToHashSet(),
         KnownAddresses  = program.ImageSymbols.ToDictionary(de => de.Key, de => de.Value),
         ICFG            = new DiGraph <RtlBlock>(),
     };
 }
예제 #11
0
        //Regenerate feature id and shrink features with lower frequency
        public void Shrink(EncoderTagger[] xList, int freq)
        {
            var old2new = new BTreeDictionary <long, long>();

            featureLexicalDict.Shrink(freq);
            maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
            var feature_count = xList.Length;

            //Update feature ids
            Parallel.For(0, feature_count, parallelOption, i =>
            {
                for (var j = 0; j < xList[i].feature_cache_.Count; j++)
                {
                    var newfs     = new List <long>();
                    long rstValue = 0;
                    for (int index = 0; index < xList[i].feature_cache_[j].Length; index++)
                    {
                        var v = xList[i].feature_cache_[j][index];
                        if (old2new.TryGetValue(v, out rstValue) == true)
                        {
                            newfs.Add(rstValue);
                        }
                    }
                    xList[i].feature_cache_[j] = newfs.ToArray();
                }
            });

            Logger.WriteLine("Feature size in total : {0}", maxid_);
        }
예제 #12
0
        public void TestRangeEnumerate()
        {
            BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer);

            for (int i = 0; i < 100; i++)
            {
                Assert.IsTrue(data.TryAdd(i, i.ToString()));
            }

            int ix = 0;

            foreach (KeyValuePair <int, string> kv in data.EnumerateRange(-500, 5000))
            {
                Assert.AreEqual(ix++, kv.Key);
            }
            Assert.AreEqual(100, ix);

            foreach (
                KeyValuePair <int, int> range in
                new Dictionary <int, int> {
                { 6, 25 }, { 7, 25 }, { 8, 25 }, { 9, 25 }, { 22, 25 }, { 28, 28 }
            })
            {
                ix = range.Key;
                foreach (KeyValuePair <int, string> kv in data.EnumerateRange(ix, range.Value))
                {
                    Assert.AreEqual(ix++, kv.Key);
                }
                Assert.AreEqual(range.Value, ix - 1);
            }
        }
예제 #13
0
        public void CBTree_IndexOf_Empty()
        {
            var btree = new BTreeDictionary <string, int>();
            int i     = btree.Keys.IndexOf("3");

            Assert.AreEqual(-1, i);
        }
예제 #14
0
        public void BTree_AddTwoItems()
        {
            var btree = new BTreeDictionary <string, int>();

            btree.Add("3", 3);
            btree.Add("2", 2);
            Assert.AreEqual(2, btree.Count);
        }
예제 #15
0
        public void BTree_SetExisting()
        {
            var btree = new BTreeDictionary <string, int>();

            btree["3"] = 3;
            btree["3"] = 2;
            Assert.AreEqual(2, btree["3"]);
        }
예제 #16
0
        public void CBTree_IndexOf_existing_leaf_item()
        {
            var btree = new BTreeDictionary <string, int> {
                { "3", 3 }
            };
            int i = btree.Keys.IndexOf("3");

            Assert.AreEqual(0, i);
        }
예제 #17
0
        private BTreeDictionary <string, int> Given_Dictionary(IEnumerable <int> items)
        {
            var btree = new BTreeDictionary <string, int>();

            foreach (var item in items)
            {
                btree.Add(item.ToString(), item);
            }
            return(btree);
        }
예제 #18
0
        public DefaultFeatureLexicalDict(int thread_num)
        {
            featureset_dict_ = new BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128);
            maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
#else
            parallelOption = new ParallelOptions();
            parallelOption.MaxDegreeOfParallelism = thread_num;
#endif
        }
예제 #19
0
        public DefaultFeatureLexicalDict(int thread_num)
        {
            featureset_dict_ = new BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128);
            maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
#else
            parallelOption = new ParallelOptions();
            parallelOption.MaxDegreeOfParallelism = thread_num;
#endif
        }
예제 #20
0
        public void CBTree_IndexOf_nonexisting_small_leafitem()
        {
            var btree = new BTreeDictionary <string, int> {
                { "3", 3 },
                { "2", 2 }
            };
            int i = btree.Keys.IndexOf("1");

            Assert.AreEqual(~0, i);
        }
예제 #21
0
        public void CBTree_IndexOf_nonexisting_large_leafitem()
        {
            var btree = new BTreeDictionary <string, int> {
                { "4", 4 },
                { "2", 2 }
            };
            int i = btree.Keys.IndexOf("5");

            Assert.AreEqual(~2, i);
        }
예제 #22
0
        public void CBTree_ForceInternalNode()
        {
            var btree = new BTreeDictionary <string, int>();

            foreach (var i in Enumerable.Range(0, 256))
            {
                btree.Add(i.ToString(), i);
            }
            btree.Add("256", 256);
        }
예제 #23
0
파일: C64Basic.cs 프로젝트: qcyb/reko
 public C64Basic(IServiceProvider services, string archId) : base(services, archId)
 {
     this.Description        = "Commodore 64 Basic";
     this.Endianness         = EndianServices.Little;
     this.PointerType        = PrimitiveType.Ptr16;
     this.InstructionBitSize = 8;
     this.StackRegister      = stackRegister;
     this.FramePointerType   = PrimitiveType.Ptr16;
     program       = new SortedList <ushort, C64BasicInstruction>();
     mpAddrToInstr = new BTreeDictionary <Address, C64BasicInstruction>();
 }
예제 #24
0
        public void TestIndexer()
        {
            BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer, GetSample());
            BTreeDictionary <int, string> copy = new BTreeDictionary <int, string>(Comparer);

            copy.AddRange(data);
            Assert.AreEqual(copy.Count, data.Count);
            foreach (int key in data.Keys)
            {
                Assert.AreEqual(data[key], copy[key]);
            }
        }
예제 #25
0
        public void CBTree_GetFromDeepTree()
        {
            var btree = new BTreeDictionary <string, int>();

            foreach (var i in Enumerable.Range(0, 1000))
            {
                btree.Add(i.ToString(), i);
            }
            btree.Dump();
            Assert.AreEqual(0, btree["0"]);
            Assert.AreEqual(500, btree["500"]);
        }
예제 #26
0
        static void Main(string[] args)
        {
            if (args.Length != 2)
            {
                Console.WriteLine("signDict.exe [raw dictionary] [binary dictionary]");
                return;
            }

            DoubleArrayTrieBuilder daBuilder = new DoubleArrayTrieBuilder(4);


            BTreeDictionary <string, int> dict = new BTreeDictionary <string, int>(StringComparer.Ordinal, 128);
            string       strLine = null;
            StreamReader sr      = new StreamReader(args[0], Encoding.UTF8);
            StreamWriter sw      = new StreamWriter(args[1] + ".prob", false, Encoding.UTF8);
            BinaryWriter bw      = new BinaryWriter(sw.BaseStream);

            int index = 0;

            while ((strLine = sr.ReadLine()) != null)
            {
                string[] items    = strLine.Split('\t');
                string   strNGram = items[0].Trim();

                if (dict.ContainsKey(strNGram) == true)
                {
                    Console.WriteLine("duplicated line: {0}", strLine);
                    continue;
                }

                if (strNGram.Length == 0)
                {
                    continue;
                }

                string[] vals    = items[1].Split();
                float    prob    = float.Parse(vals[0]);
                float    backoff = float.Parse(vals[1]);

                //Write item into file
                bw.Write(prob);
                bw.Write(backoff);
                dict.Add(strNGram, index);
                index++;
            }
            sr.Close();

            daBuilder.build(dict);
            daBuilder.save(args[1] + ".da");

            bw.Close();
        }
예제 #27
0
        public void TestFirstAndLast()
        {
            BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer);

            data.Add(1, "a");
            data.Add(2, "b");
            data.Add(3, "c");
            data.Add(4, "d");
            data.Add(5, "e");

            Assert.AreEqual(1, data.First().Key);
            Assert.AreEqual("a", data.First().Value);
            data.Remove(1);
            Assert.AreEqual(2, data.First().Key);
            Assert.AreEqual("b", data.First().Value);

            Assert.AreEqual(5, data.Last().Key);
            Assert.AreEqual("e", data.Last().Value);
            data.Remove(5);
            Assert.AreEqual(4, data.Last().Key);
            Assert.AreEqual("d", data.Last().Value);

            data.Remove(4);
            data.Remove(3);

            KeyValuePair <int, string> kv;

            Assert.IsTrue(data.TryGetLast(out kv));
            Assert.IsTrue(data.TryGetFirst(out kv));
            data.Remove(2);
            Assert.IsFalse(data.TryGetLast(out kv));
            Assert.IsFalse(data.TryGetFirst(out kv));

            try
            {
                data.First();
                Assert.Fail("Should raise InvalidOperationException");
            }
            catch (InvalidOperationException)
            {
            }
            try
            {
                data.Last();
                Assert.Fail("Should raise InvalidOperationException");
            }
            catch (InvalidOperationException)
            {
            }
        }
예제 #28
0
        public void TestArray()
        {
            List <KeyValuePair <int, string> > sample = new List <KeyValuePair <int, string> >(GetSample());
            BTreeDictionary <int, string>      data   = new BTreeDictionary <int, string>(Comparer, sample);

            sample.Sort((a, b) => data.Comparer.Compare(a.Key, b.Key));
            KeyValuePair <int, string>[] array = data.ToArray();

            Assert.AreEqual(sample.Count, array.Length);
            for (int i = 0; i < sample.Count; i++)
            {
                Assert.AreEqual(sample[i].Key, array[i].Key);
                Assert.AreEqual(sample[i].Value, array[i].Value);
            }
        }
예제 #29
0
        internal static BTreeDictionary <DateTime, int> LoadStatsFromDisk(string statsPath)
        {
            BTreeDictionary <DateTime, int> stats = new BTreeDictionary <DateTime, int>();

            using (StreamReader reader = new StreamReader(statsPath))
            {
                while (!reader.EndOfStream)
                {
                    var      line = reader.ReadLine();
                    string[] kvp  = line.Split('#');
                    stats.Add(DateTime.Parse(kvp[0]), Int32.Parse(kvp[1]));
                }
            }
            return(stats);
        }
예제 #30
0
        public void CBTree_Enumerate()
        {
            var btree = new BTreeDictionary <string, int>();

            btree.Add("3", 3);
            btree.Add("2", 2);
            var e = btree.GetEnumerator();

            Assert.IsTrue(e.MoveNext());
            Assert.AreEqual("2", e.Current.Key);
            Assert.AreEqual(2, e.Current.Value);
            Assert.IsTrue(e.MoveNext());
            Assert.AreEqual("3", e.Current.Key);
            Assert.AreEqual(3, e.Current.Value);
            Assert.IsFalse(e.MoveNext());
        }
예제 #31
0
        public void TestClone()
        {
            BTreeDictionary <int, string> data = new BTreeDictionary <int, string>(Comparer, GetSample());
            BTreeDictionary <int, string> copy = (BTreeDictionary <int, string>)((ICloneable)data).Clone();

            using (IEnumerator <KeyValuePair <int, string> > e1 = data.GetEnumerator())
                using (IEnumerator <KeyValuePair <int, string> > e2 = copy.GetEnumerator())
                {
                    while (e1.MoveNext() && e2.MoveNext())
                    {
                        Assert.AreEqual(e1.Current.Key, e2.Current.Key);
                        Assert.AreEqual(e1.Current.Value, e2.Current.Value);
                    }
                    Assert.IsFalse(e1.MoveNext() || e2.MoveNext());
                }
        }
예제 #32
0
        static void Main(string[] args)
        {
            //Test BTreeDictionary for very large data set
            BTreeDictionary<long, long> sdict = new BTreeDictionary<long, long>();

            for (long i = 10000; i >= 0; i-=2)
            {
                sdict.Add(i, i);
            }

            long val = sdict.KeyList[123];
            long n = sdict.ValueList[123];
            sdict.RemoveAt(123);
            long cnt = sdict.ValueList.Count;


            Console.WriteLine("Done.");
            Console.ReadLine();
        }
예제 #33
0
        //Regenerate feature id and shrink features with lower frequency
        public void Shrink(EncoderTagger[] xList, int freq)
        {
            var old2new = new BTreeDictionary<long, long>();
            featureLexicalDict.Shrink(freq);
            maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
            var feature_count = xList.Length;

            //Update feature ids
#if NO_SUPPORT_PARALLEL_LIB
            for (int i = 0;i < feature_cache_.Count;i++)
#else
            Parallel.For(0, feature_count, parallelOption, i =>
#endif
            {
                for (var j = 0; j < xList[i].feature_cache_.Count; j++)
                {
                    var newfs = new List<long>();
                    long rstValue = 0;
                    for (int index = 0; index < xList[i].feature_cache_[j].Length; index++)
                    {
                        var v = xList[i].feature_cache_[j][index];
                        if (old2new.TryGetValue(v, out rstValue) == true)
                        {
                            newfs.Add(rstValue);
                        }
                    }
                    xList[i].feature_cache_[j] = newfs.ToArray();
                }
            }
#if NO_SUPPORT_PARALLEL_LIB
#else
);
#endif

            Console.WriteLine("Feature size in total : {0}", maxid_);
        }
예제 #34
0
        public long RegenerateFeatureId(BTreeDictionary<long, long> old2new, long ysize)
        {
            long new_maxid = 0;
            //Regenerate new feature id and create feature ids mapping
            foreach (var it in featureset_dict_)
            {
                var strFeature = it.Key;
                //Regenerate new feature id
                old2new.Add(it.Value.Key, new_maxid);
                it.Value.Key = new_maxid;

                var addValue = (strFeature[0] == 'U' ? ysize : ysize * ysize);
                new_maxid += addValue;
            }

            return new_maxid;
        }
예제 #35
0
 public void Clear()
 {
     featureset_dict_.Clear();
     featureset_dict_ = null;
 }
예제 #36
0
        //加载model文件
        //返回值<0 为出错,=0为正常
        public bool LoadModel(string filename)
        {
            var sr = new StreamReader(filename);
            string strLine;


            //读入版本号
            strLine = sr.ReadLine();
            version = uint.Parse(strLine.Split(':')[1].Trim());

            //读入cost_factor
            strLine = sr.ReadLine();
            cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());

            //读入maxid
            strLine = sr.ReadLine();
            maxid_ = long.Parse(strLine.Split(':')[1].Trim());

            //读入xsize
            strLine = sr.ReadLine();
            xsize_ = uint.Parse(strLine.Split(':')[1].Trim());

            //读入空行
            strLine = sr.ReadLine();

            //读入待标注的标签
            y_ = new List<string>();
            while (true)
            {
                strLine = sr.ReadLine();
                if (strLine.Length == 0)
                {
                    break;
                }
                y_.Add(strLine);
            }

            //读入unigram和bigram模板
            unigram_templs_ = new List<string>();
            bigram_templs_ = new List<string>();
            while (sr.EndOfStream == false)
            {
                strLine = sr.ReadLine();
                if (strLine.Length == 0)
                {
                    break;
                }
                if (strLine[0] == 'U')
                {
                    unigram_templs_.Add(strLine);
                }
                if (strLine[0] == 'B')
                {
                    bigram_templs_.Add(strLine);
                }
            }
            sr.Close();

            //Load all feature set data
            var filename_feature = filename + ".feature";
            da = new DoubleArrayTrieSearch();
            da.Load(filename_feature);


            //Load all features alpha data
            var filename_alpha = filename + ".alpha";
            var sr_alpha = new StreamReader(filename_alpha);
            var br_alpha = new BinaryReader(sr_alpha.BaseStream);

            if (version == Utils.MODEL_TYPE_NORM)
            {
                //feature weight array
                alpha_two_tuples = null;
                alpha_ = new double[maxid_ + 1];
                for (long i = 0; i < maxid_; i++)
                {
                    alpha_[i] = br_alpha.ReadSingle();
                }
            }
            else if (version == Utils.MODEL_TYPE_SHRINKED)
            {
                alpha_ = null;
                alpha_two_tuples = new BTreeDictionary<long, double>();
                for (long i = 0; i < maxid_; i++)
                {
                    var key = br_alpha.ReadInt64();
                    double weight = br_alpha.ReadSingle();
                    alpha_two_tuples.Add(key, weight);
                }
            }
            else
            {
                Console.WriteLine("This model is not supported.");
                return false;
            }
            br_alpha.Close();
            return true;
        }