Example #1
0
        public IList <IDictionary <string, object> > ReadDocs(IEnumerable <KeyValuePair <long, float> > docs)
        {
            var result = new List <IDictionary <string, object> >();

            foreach (var d in docs)
            {
                var docInfo = _docIx.Read(d.Key);

                if (docInfo.offset < 0)
                {
                    continue;
                }

                var docMap = _docs.Read(docInfo.offset, docInfo.length);
                var doc    = new Dictionary <string, object>();

                for (int i = 0; i < docMap.Count; i++)
                {
                    var kvp   = docMap[i];
                    var kInfo = _keyIx.Read(kvp.keyId);
                    var vInfo = _valIx.Read(kvp.valId);
                    var key   = _keyReader.Read(kInfo.offset, kInfo.len, kInfo.dataType);
                    var val   = _valReader.Read(vInfo.offset, vInfo.len, vInfo.dataType);

                    doc[key.ToString()] = val;
                }

                doc["___docid"] = d.Key;
                doc["___score"] = d.Value;

                result.Add(doc);
            }

            return(result);
        }
Example #2
0
        public IEnumerable <IDictionary> Read(Query query)
        {
            var docIx          = new DocIndexReader(DocIndexStream);
            var docs           = new DocReader(DocStream);
            var keyIx          = new ValueIndexReader(KeyIndexStream);
            var valIx          = new ValueIndexReader(ValueIndexStream);
            var keyReader      = new ValueReader(KeyStream);
            var valReader      = new ValueReader(ValueStream);
            var postingsReader = new PostingsReader(PostingsStream);

            var keyHash = query.Term.Key.ToString().ToHash();
            var ix      = GetIndex(keyHash);
            var match   = ix.ClosestMatch(query.Term.Value.ToString());
            var docIds  = postingsReader.Read(match.PostingsOffset, match.PostingsSize);

            foreach (var docId in docIds)
            {
                var docInfo = docIx.Read(docId);
                var docMap  = docs.Read(docInfo.offset, docInfo.length);
                var doc     = new Dictionary <IComparable, IComparable>();

                for (int i = 0; i < docMap.Count; i++)
                {
                    var kvp   = docMap[i];
                    var kInfo = keyIx.Read(kvp.keyId);
                    var vInfo = valIx.Read(kvp.valId);
                    var key   = keyReader.Read(kInfo.offset, kInfo.len, kInfo.dataType);
                    var val   = valReader.Read(vInfo.offset, vInfo.len, vInfo.dataType);

                    doc[key] = val;
                }

                yield return(doc);
            }
        }
Example #3
0
        public IEnumerable <IDictionary> ReadDocs(IDictionary <long, float> docs)
        {
            foreach (var d in docs)
            {
                var docInfo = _docIx.Read(d.Key);

                if (docInfo.offset < 0)
                {
                    continue;
                }

                var docMap = _docs.Read(docInfo.offset, docInfo.length);
                var doc    = new Dictionary <IComparable, IComparable>();

                for (int i = 0; i < docMap.Count; i++)
                {
                    var kvp   = docMap[i];
                    var kInfo = _keyIx.Read(kvp.keyId);
                    var vInfo = _valIx.Read(kvp.valId);
                    var key   = _keyReader.Read(kInfo.offset, kInfo.len, kInfo.dataType);
                    var val   = _valReader.Read(vInfo.offset, vInfo.len, vInfo.dataType);

                    doc[key] = val;
                }

                doc["__docid"] = d.Key;
                doc["__score"] = d.Value;

                yield return(doc);
            }
        }
Example #4
0
        public IList <IDictionary> ReadDocs(IEnumerable <KeyValuePair <long, float> > docs)
        {
            var timer = Stopwatch.StartNew();

            var result = new List <IDictionary>();

            foreach (var d in docs)
            {
                var docInfo = _docIx.Read(d.Key);

                if (docInfo.offset < 0)
                {
                    continue;
                }

                var docMap = _docs.Read(docInfo.offset, docInfo.length);
                var doc    = new Dictionary <IComparable, IComparable>();

                for (int i = 0; i < docMap.Count; i++)
                {
                    var kvp   = docMap[i];
                    var kInfo = _keyIx.Read(kvp.keyId);
                    var vInfo = _valIx.Read(kvp.valId);
                    var key   = _keyReader.Read(kInfo.offset, kInfo.len, kInfo.dataType);
                    var val   = _valReader.Read(vInfo.offset, vInfo.len, vInfo.dataType);

                    doc[key] = val;
                }

                doc["__docid"] = d.Key;
                doc["__score"] = d.Value;

                result.Add(doc);
            }

            this.Log("read {0} docs in {1}", result.Count, timer.Elapsed);

            return(result
                   .GroupBy(x => (long)x["__docid"])
                   .SelectMany(g => g.OrderByDescending(x => (long)x["_created"]).Take(1))
                   .ToList());
        }