Пример #1
0
        public ReadSession(string collectionName,
                           ulong collectionId,
                           SessionFactory sessionFactory,
                           IConfigurationProvider config,
                           IStringModel tokenizer)
            : base(collectionName, collectionId, sessionFactory)
        {
            ValueStream      = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId)));
            KeyStream        = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId)));
            DocStream        = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId)));
            ValueIndexStream = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId)));
            KeyIndexStream   = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId)));
            DocIndexStream   = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId)));

            _docIx     = new DocIndexReader(DocIndexStream);
            _docs      = new DocMapReader(DocStream);
            _keyIx     = new ValueIndexReader(KeyIndexStream);
            _valIx     = new ValueIndexReader(ValueIndexStream);
            _keyReader = new ValueReader(KeyStream);
            _valReader = new ValueReader(ValueStream);
            _config    = config;
            _tokenizer = tokenizer;

            var posFileName = Path.Combine(SessionFactory.Dir, $"{CollectionId}.pos");

            _postingsStream = SessionFactory.CreateReadStream(posFileName);
        }
Пример #2
0
        public IEnumerable <IDictionary> Read(Query query)
        {
            var docIx          = new DocIndexReader(DocIndexStream);
            var docs           = new DocReader(DocStream);
            var keyIx          = new ValueIndexReader(KeyIndexStream);
            var valIx          = new ValueIndexReader(ValueIndexStream);
            var keyReader      = new ValueReader(KeyStream);
            var valReader      = new ValueReader(ValueStream);
            var postingsReader = new PostingsReader(PostingsStream);

            var keyHash = query.Term.Key.ToString().ToHash();
            var ix      = GetIndex(keyHash);
            var match   = ix.ClosestMatch(query.Term.Value.ToString());
            var docIds  = postingsReader.Read(match.PostingsOffset, match.PostingsSize);

            foreach (var docId in docIds)
            {
                var docInfo = docIx.Read(docId);
                var docMap  = docs.Read(docInfo.offset, docInfo.length);
                var doc     = new Dictionary <IComparable, IComparable>();

                for (int i = 0; i < docMap.Count; i++)
                {
                    var kvp   = docMap[i];
                    var kInfo = keyIx.Read(kvp.keyId);
                    var vInfo = valIx.Read(kvp.valId);
                    var key   = keyReader.Read(kInfo.offset, kInfo.len, kInfo.dataType);
                    var val   = valReader.Read(vInfo.offset, vInfo.len, vInfo.dataType);

                    doc[key] = val;
                }

                yield return(doc);
            }
        }
Пример #3
0
        public ReadSession(string collectionId,
                           SessionFactory sessionFactory,
                           IConfigurationService config)
            : base(collectionId, sessionFactory)
        {
            var collection = collectionId.ToHash();

            ValueStream      = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", collection)));
            KeyStream        = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", collection)));
            DocStream        = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", collection)));
            ValueIndexStream = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", collection)));
            KeyIndexStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", collection)));
            DocIndexStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", collection)));
            Index            = sessionFactory.GetCollectionIndex(collectionId.ToHash()) ?? new SortedList <long, VectorNode>();

            _docIx          = new DocIndexReader(DocIndexStream);
            _docs           = new DocReader(DocStream);
            _keyIx          = new ValueIndexReader(KeyIndexStream);
            _valIx          = new ValueIndexReader(ValueIndexStream);
            _keyReader      = new ValueReader(KeyStream);
            _valReader      = new ValueReader(ValueStream);
            _postingsReader = new RemotePostingsReader(config);

            _log = Logging.CreateWriter("readsession");
        }
Пример #4
0
        public DocumentStreamSession(string collectionName, ulong collectionId, SessionFactory sessionFactory)
            : base(collectionName, collectionId, sessionFactory)
        {
            ValueStream      = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId)));
            KeyStream        = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId)));
            DocStream        = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId)));
            ValueIndexStream = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId)));
            KeyIndexStream   = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId)));
            DocIndexStream   = sessionFactory.CreateReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId)));

            _docIx     = new DocIndexReader(DocIndexStream);
            _docs      = new DocMapReader(DocStream);
            _keyIx     = new ValueIndexReader(KeyIndexStream);
            _valIx     = new ValueIndexReader(ValueIndexStream);
            _keyReader = new ValueReader(KeyStream);
            _valReader = new ValueReader(ValueStream);
        }
Пример #5
0
        public DocumentStreamSession(string collectionId, SessionFactory sessionFactory)
            : base(collectionId, sessionFactory)
        {
            var collection = collectionId.ToHash();

            ValueStream      = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", collection)));
            KeyStream        = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", collection)));
            DocStream        = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", collection)));
            ValueIndexStream = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", collection)));
            KeyIndexStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", collection)));
            DocIndexStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", collection)));

            _docIx     = new DocIndexReader(DocIndexStream);
            _docs      = new DocReader(DocStream);
            _keyIx     = new ValueIndexReader(KeyIndexStream);
            _valIx     = new ValueIndexReader(ValueIndexStream);
            _keyReader = new ValueReader(KeyStream);
            _valReader = new ValueReader(ValueStream);
        }
Пример #6
0
        public ReadSession(ulong collectionId, LocalStorageSessionFactory sessionFactory)
            : base(collectionId, sessionFactory)
        {
            ValueStream      = sessionFactory.ValueStream;
            KeyStream        = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", collectionId)));
            DocStream        = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", collectionId)));
            ValueIndexStream = sessionFactory.ValueIndexStream;
            KeyIndexStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", collectionId)));
            DocIndexStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", collectionId)));
            PostingsStream   = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.pos", collectionId)));
            VectorStream     = sessionFactory.CreateReadWriteStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vec", collectionId)));
            Index            = sessionFactory.GetIndex(collectionId);

            _docIx          = new DocIndexReader(DocIndexStream);
            _docs           = new DocReader(DocStream);
            _keyIx          = new ValueIndexReader(KeyIndexStream);
            _valIx          = new ValueIndexReader(ValueIndexStream);
            _keyReader      = new ValueReader(KeyStream);
            _valReader      = new ValueReader(ValueStream);
            _postingsReader = new PagedPostingsReader(PostingsStream);
        }
Пример #7
0
        public BOWReadSession(string collectionName,
                              ulong collectionId,
                              SessionFactory sessionFactory,
                              IConfigurationProvider config)
            : base(collectionName, collectionId, sessionFactory)
        {
            ValueStream      = sessionFactory.CreateAsyncReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId)));
            KeyStream        = sessionFactory.CreateAsyncReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId)));
            DocStream        = sessionFactory.CreateAsyncReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId)));
            ValueIndexStream = sessionFactory.CreateAsyncReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId)));
            KeyIndexStream   = sessionFactory.CreateAsyncReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId)));
            DocIndexStream   = sessionFactory.CreateAsyncReadStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId)));

            _docIx          = new DocIndexReader(DocIndexStream);
            _docs           = new DocMapReader(DocStream);
            _keyIx          = new ValueIndexReader(KeyIndexStream);
            _valIx          = new ValueIndexReader(ValueIndexStream);
            _keyReader      = new ValueReader(KeyStream);
            _valReader      = new ValueReader(ValueStream);
            _postingsReader = new RemotePostingsReader(config, collectionName);
            _indexReaders   = new ConcurrentDictionary <long, NodeReader>();
            _config         = config;
        }