示例#1
0
        public void TestReadAndLookUpTokenInfo()
        {
            List <short> tokenInfo = new List <short>();
            List <int>   features  = new List <int>();

            short[] tokenInfos = new short[3];
            tokenInfos[0] = 1;
            tokenInfos[1] = 2;
            tokenInfos[2] = 3;

            int[] featureInfos = new int[2];
            featureInfos[0] = 73;
            featureInfos[1] = 99;

            tokenInfo.Add((short)1);
            tokenInfo.Add((short)2);
            tokenInfo.Add((short)3);

            features.Add(73);
            features.Add(99);

            BufferEntry entry = new BufferEntry();

            entry.TokenInfo = tokenInfo;
            entry.Features  = features;

            entry.TokenInfos   = tokenInfos;
            entry.FeatureInfos = featureInfos;

            List <BufferEntry> bufferEntries = new List <BufferEntry>();

            bufferEntries.Add(entry);

            using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries))
            {
                using (var outputStream = File.Create(tokenInfoFile))
                {
                    compiler.Compile(outputStream);
                }
            }

            using (var inputStream = File.OpenRead(tokenInfoFile))
            {
                using (TokenInfoBuffer tokenInfoBuffer2 = new TokenInfoBuffer(inputStream))
                {
                    Assert.AreEqual(99, tokenInfoBuffer2.LookupFeature(0, 1));
                    Assert.AreEqual(73, tokenInfoBuffer2.LookupFeature(0, 0));
                }
            }
        }
        public void TestReadAndLookUpTokenInfo()
        {
            var tokenInfo = new List <short>();
            var features  = new List <int>();

            var tokenInfos = new short[] { 1, 2, 3 };

            var featureInfos = new int[] { 73, 99 };

            tokenInfo.Add(1);
            tokenInfo.Add(2);
            tokenInfo.Add(3);

            features.Add(73);
            features.Add(99);

            var entry = new BufferEntry();

            entry.TokenInfo = tokenInfo;
            entry.Features  = features;

            entry.TokenInfos   = tokenInfos;
            entry.FeatureInfos = featureInfos;

            var bufferEntries = new List <BufferEntry>();

            bufferEntries.Add(entry);

            using (var ms = new MemoryStream())
            {
                var compiler = new TokenInfoBufferCompiler(ms, bufferEntries);
                compiler.Compile();

                ms.Seek(0, SeekOrigin.Begin);

                var tokenInfoBuffer2 = new TokenInfoBuffer(ms);

                tokenInfoBuffer2.LookupFeature(0, 1).Is(99);
                tokenInfoBuffer2.LookupFeature(0, 0).Is(73);
            }
        }
示例#3
0
 string ExtractSingleFeature(int wordId, int field)
 {
     if (TokenInfoBuffer.isPartOfSpeechFeature(field))
     {
         var featureId = TokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field);
         return(PosValues[featureId]);
     }
     else
     {
         var featureId = TokenInfoBuffer.LookupFeature(wordId, field);
         return(StringValues[featureId]);
     }
 }
示例#4
0
        private string ExtractSingleFeature(int wordId, int field)
        {
            int featureId;

            if (tokenInfoBuffer.IsPartOfSpeechFeature(field))
            {
                featureId = tokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field);
                return(posValues.Get(featureId));
            }

            featureId = tokenInfoBuffer.LookupFeature(wordId, field);
            return(stringValues.Get(featureId));
        }