public IEnumerable <MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData, int take)
 {
     return(storage.MappedResults["ByViewAndEtagDesc"]
            // the index is sorted view ascending and then etag descending
            // we index before this index, then backward toward the last one.
            .SkipBefore(new RavenJObject {
         { "view", indexName }, { "etag", lastReducedEtag.ToByteArray() }
     })
            .TakeWhile(x => StringComparer.InvariantCultureIgnoreCase.Equals(x.Value <string>("view"), indexName))
            .Select(key =>
     {
         var mappedResultInfo = new MappedResultInfo
         {
             ReduceKey = key.Value <string>("reduceKey"),
             Etag = new Guid(key.Value <byte[]>("etag")),
             Timestamp = key.Value <DateTime>("timestamp"),
         };
         if (loadData)
         {
             var readResult = storage.MappedResults.Read(key);
             if (readResult != null)
             {
                 mappedResultInfo.Data = readResult.Data().ToJObject();
             }
         }
         return mappedResultInfo;
     }).Take(take));
 }
Beispiel #2
0
        public IEnumerable <MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData)
        {
            Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_etag");
            Api.MakeKey(session, MappedResults, indexName, Encoding.Unicode, MakeKeyGrbit.NewKey);
            Api.MakeKey(session, MappedResults, lastReducedEtag, MakeKeyGrbit.None);
            if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekLE) == false)
            {
                yield break;
            }

            while (Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"]) == indexName)
            {
                var result = new MappedResultInfo
                {
                    ReduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]),
                    Etag      = new Guid(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])),
                    Timestamp = Api.RetrieveColumnAsDateTime(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).Value,
                };
                if (loadData)
                {
                    result.Data = Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]).ToJObject();
                }
                yield return(result);

                // the index is view ascending and etag descending
                // that means that we are going backward to go up
                if (Api.TryMovePrevious(session, MappedResults) == false)
                {
                    yield break;
                }
            }
        }
        private IEnumerable <MappedResultInfo> GetReducedResultsForBucket(string index, string reduceKey, int level, int bucket, bool loadData)
        {
            var results = storage.ReduceResults["ByViewReduceKeyLevelAndBucket"]
                          .SkipTo(new RavenJObject
            {
                { "view", index },
                { "reduceKey", reduceKey },
                { "level", level },
                { "bucket", bucket }
            })
                          .TakeWhile(x => string.Equals(index, x.Value <string>("view"), StringComparison.InvariantCultureIgnoreCase) &&
                                     string.Equals(reduceKey, x.Value <string>("reduceKey"), StringComparison.InvariantCultureIgnoreCase) &&
                                     level == x.Value <int>("level") &&
                                     bucket == x.Value <int>("bucket"));

            bool hasResults = false;

            foreach (var result in results)
            {
                hasResults = true;
                var readResult = storage.ReduceResults.Read(result);

                var mappedResultInfo = new MappedResultInfo
                {
                    ReduceKey = readResult.Key.Value <string>("reduceKey"),
                    Etag      = new Guid(readResult.Key.Value <byte[]>("etag")),
                    Timestamp = readResult.Key.Value <DateTime>("timestamp"),
                    Bucket    = readResult.Key.Value <int>("bucket"),
                    Source    = readResult.Key.Value <int>("sourceBucket").ToString(),
                    Size      = readResult.Size,
                    Data      = loadData ? LoadMappedResult(readResult) : null
                };

                yield return(mappedResultInfo);
            }

            if (hasResults)
            {
                yield break;
            }

            yield return(new MappedResultInfo
            {
                Bucket = bucket,
                ReduceKey = reduceKey
            });
        }
Beispiel #4
0
        public IEnumerable <MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData, int take)
        {
            Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_etag");
            Api.MakeKey(session, MappedResults, indexName, Encoding.Unicode, MakeKeyGrbit.NewKey);
            Api.MakeKey(session, MappedResults, lastReducedEtag, MakeKeyGrbit.None);
            if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekLE) == false)
            {
                return(Enumerable.Empty <MappedResultInfo>());
            }

            var results = new Dictionary <string, MappedResultInfo>();

            while (
                results.Count < take &&
                Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex) == indexName)
            {
                var key = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]);
                var mappedResultInfo = new MappedResultInfo
                {
                    ReduceKey =
                        key,
                    Etag      = new Guid(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])),
                    Timestamp =
                        Api.RetrieveColumnAsDateTime(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).
                        Value,
                    Data = loadData
                                                        ? LoadMappedResults(key)
                                                        : null,
                    Size = Api.RetrieveColumnSize(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]) ?? 0
                };

                results[mappedResultInfo.ReduceKey] = mappedResultInfo;

                // the index is view ascending and etag descending
                // that means that we are going backward to go up
                if (Api.TryMovePrevious(session, MappedResults) == false)
                {
                    break;
                }
            }

            return(results.Values);
        }
Beispiel #5
0
        public List<MappedResultInfo> GetMappedResults(int view, HashSet<string> keysLeftToReduce, bool loadData, int take, HashSet<string> keysReturned, CancellationToken cancellationToken, List<MappedResultInfo> outputCollection = null)
        {
            if (outputCollection == null)
                outputCollection = new List<MappedResultInfo>();

            Api.JetSetCurrentIndex(session, MappedResults, "by_view_hashed_reduce_key_and_bucket");
            var keysToReduce = new HashSet<string>(keysLeftToReduce);
            foreach (var reduceKey in keysToReduce)
            {
                cancellationToken.ThrowIfCancellationRequested();

                keysLeftToReduce.Remove(reduceKey);

                Api.MakeKey(session, MappedResults, view, MakeKeyGrbit.NewKey);
                var hashReduceKey = HashReduceKey(reduceKey);

                keysReturned.Add(reduceKey);

                Api.MakeKey(session, MappedResults, hashReduceKey, MakeKeyGrbit.None);
                if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekGE) == false)
                    continue;

                do
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    var indexFromDb = Api.RetrieveColumnAsInt32(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"]);
                    var hashKeyFromDb = Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["hashed_reduce_key"]);

                    if (indexFromDb != view || hashReduceKey.SequenceEqual(hashKeyFromDb) == false)
                        break;
                    
                    var timestamp = Api.RetrieveColumnAsInt64(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).Value;
                    var keyFromDb = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]);

                    take--; // We have worked with this reduce key, so we consider it an output even if we don't add it. 

                    RavenJObject data = null;
                    if ( loadData )
                    {
                        data = LoadMappedResults(keyFromDb);
                        if ( data == null )
                            continue;
                    }
                    
                    
                    var result = new MappedResultInfo
                    {
                        Bucket = Api.RetrieveColumnAsInt32(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"]).Value,
                        ReduceKey = keyFromDb,
                        Etag = Etag.Parse(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])),
                        Timestamp = DateTime.FromBinary(timestamp),
                        Data = data,
                        Size = Api.RetrieveColumnSize(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]) ?? 0
                    };

                    outputCollection.Add(result);
                } 
                while (Api.TryMoveNext(session, MappedResults));

                if (take < 0)
                    return outputCollection;
            }

            return outputCollection;
        }
        public List<MappedResultInfo> GetMappedResults(int view, HashSet<string> keysLeftToReduce, bool loadData, int take, HashSet<string> keysReturned, CancellationToken cancellationToken, List<MappedResultInfo> outputCollection = null)
        {
            if (outputCollection == null)
                outputCollection = new List<MappedResultInfo>();

            var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey);
            var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data);
            
            var keysToReduce = new HashSet<string>(keysLeftToReduce);
            foreach (var reduceKey in keysToReduce)
            {
                cancellationToken.ThrowIfCancellationRequested();

                keysLeftToReduce.Remove(reduceKey);

                var reduceKeyHash = HashKey(reduceKey);
                var viewAndReduceKey = (Slice)CreateKey(view, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
                using (var iterator = mappedResultsByViewAndReduceKey.MultiRead(Snapshot, viewAndReduceKey))
                {
                    keysReturned.Add(reduceKey);

                    if (!iterator.Seek(Slice.BeforeAllKeys))
                        continue;

                    do
                    {
                        cancellationToken.ThrowIfCancellationRequested();

                        ushort version;
                        var value = LoadStruct(tableStorage.MappedResults, iterator.CurrentKey, writeBatch.Value, out version);
                        if (value == null)
                            continue;
                        var size = tableStorage.MappedResults.GetDataSize(Snapshot, iterator.CurrentKey);

                        var readReduceKey = value.ReadString(MappedResultFields.ReduceKey);

                        take--; // We have worked with this reduce key, so we consider it an output even if we don't add it. 

                        RavenJObject data = null;
                        if ( loadData )
                        {
                            data = LoadMappedResult(iterator.CurrentKey, readReduceKey, mappedResultsData);
                            if (data == null)
                                continue; // If we request to load data and it is not there, we ignore it
                        }

                        var mappedResult = new MappedResultInfo
                        {
                            Bucket = value.ReadInt(MappedResultFields.Bucket),
                            ReduceKey = readReduceKey,
                            Etag = Etag.Parse(value.ReadBytes(MappedResultFields.Etag)),
                            Timestamp = DateTime.FromBinary(value.ReadLong(MappedResultFields.Timestamp)),
                            Data = data,
                            Size = size
                        };

                        outputCollection.Add(mappedResult);
                    }
                    while (iterator.MoveNext());
                }

                if (take < 0)
                    return outputCollection;
            }

            return outputCollection;
        }