示例#1
0
        public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
        {
            Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");

            var viewReductionColumn = tableColumnsCache.ScheduledReductionColumns["view"];
            var levelReductionColumn = tableColumnsCache.ScheduledReductionColumns["level"];
            var reduceReductionColumn = tableColumnsCache.ScheduledReductionColumns["reduce_key"];
            var bucketReductionColumn = tableColumnsCache.ScheduledReductionColumns["bucket"];

            var keysToRemove = new List<string>();
            var output = new List<MappedResultInfo>();
            var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
            
            try
            {
                var first = true;
                foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    int initialBucket = 0;
                    bool needToMoveNext = false;
                    if (first)
                    {
                        first = false;
                        if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
                        {
                            if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
                            {
                                needToMoveNext = true;
                                initialBucket = getItemsToReduceParams.LastReduceKeyAndBucket.Bucket;
                            }
                        }
                    }

                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, initialBucket, MakeKeyGrbit.None);
                    
                    if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false ||
                        (needToMoveNext && Api.TryMoveNext(session, ScheduledReductions) == false) )
                    {
                        keysToRemove.Add(reduceKey);
                        continue;
                    }


                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);

                    if (Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
                    {
                        keysToRemove.Add(reduceKey);
                        continue;
                    }

                    // this isn't used for optimized reading, but to make it easier to delete records later on
                    OptimizedDeleter reader;
                    if (getItemsToReduceParams.ItemsToDelete.Count == 0)
                    {
                        getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
                    }
                    else
                    {
                        reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
                    }

                    reader.IndexId = getItemsToReduceParams.Index;

                    do
                    {
                        cancellationToken.ThrowIfCancellationRequested();

                        if (getItemsToReduceParams.Take <= 0)
                            return output;

                        var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, viewReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex);
                        var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, levelReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex).Value;
                        var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, reduceReductionColumn);

                        if (getItemsToReduceParams.Index != indexFromDb)
                            break;

                        if (levelFromDb != getItemsToReduceParams.Level)
                            break;

                        if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
                            break;

                        var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, bucketReductionColumn).Value;

                        var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb); 
                        var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);


                        if (thisIsNewScheduledReductionRow)
                        {
                            if (seenLocally.Add(rowKey))
                            {
                                getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
                                foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
                                {
                                    getItemsToReduceParams.Take--;

                                    output.Add(mappedResultInfo);
                                }
                            }
                        }
                    } 
                    while (Api.TryMoveNext(session, ScheduledReductions));

                    keysToRemove.Add(reduceKey);

                    if (getItemsToReduceParams.Take <= 0)
                        break;
                }

                return output;
            }
            finally
            {
                // In whatever condition we would have to return, we must signal the removal of the reduce keys.
                foreach (var keyToRemove in keysToRemove)
                    getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
            }
        }
示例#2
0
		public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams)
		{
			Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");
			var seenLocally = new HashSet<Tuple<string, int>>();
			foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray())
			{
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey);
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, 0, MakeKeyGrbit.None);
				if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
					continue;

				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey);
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);

				if(Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
					continue;

				// this isn't used for optimized reading, but to make it easier to delete records later on
				OptimizedDeleter reader;
				if (getItemsToReduceParams.ItemsToDelete.Count == 0)
				{
					getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
				}
				else
				{
					reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete[0];
				}
				do
				{
                    if (getItemsToReduceParams.Take <= 0)
                        break;
					var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex);
					var levelFromDb =
						Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
							Value;
					var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
												   tableColumnsCache.ScheduledReductionColumns["reduce_key"]);

					if (string.Equals(getItemsToReduceParams.Index, indexFromDb, StringComparison.OrdinalIgnoreCase) == false)
						continue;
					if (levelFromDb != getItemsToReduceParams.Level)
						continue;
					if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
						continue;

					var bucket =
							Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;

					var rowKey = Tuple.Create(reduceKeyFromDb, bucket);
					var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions);
					var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
					if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
					{
						if (seenLocally.Add(rowKey))
						{
							foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData))
							{
								getItemsToReduceParams.Take--;
								yield return mappedResultInfo;
							}
						}
					}

					if (getItemsToReduceParams.Take <= 0)
						yield break;
				} while (Api.TryMoveNext(session, ScheduledReductions));

				getItemsToReduceParams.ReduceKeys.Remove(reduceKey);

				if (getItemsToReduceParams.Take <= 0)
					break;
			}
		}
示例#3
0
		public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
		{
			Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");

            var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
            var keysToRemove = new List<string>();
			foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
			{
				cancellationToken.ThrowIfCancellationRequested();

				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, 0, MakeKeyGrbit.None);
				if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
					continue;

				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);

				if(Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
					continue;

				// this isn't used for optimized reading, but to make it easier to delete records later on
				OptimizedDeleter reader;
				if (getItemsToReduceParams.ItemsToDelete.Count == 0)
				{
					getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
				}
				else
				{
					reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
				}

                reader.IndexId = getItemsToReduceParams.Index;

				do
				{
					cancellationToken.ThrowIfCancellationRequested();

                    if (getItemsToReduceParams.Take <= 0)
                        break;
					var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], RetrieveColumnGrbit.RetrieveFromIndex);
					var levelFromDb =
						Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
							Value;
					var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
												   tableColumnsCache.ScheduledReductionColumns["reduce_key"]);

					if (getItemsToReduceParams.Index != indexFromDb)
						continue;
					if (levelFromDb != getItemsToReduceParams.Level)
						continue;
					if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
						continue;

					var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;

                    var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb); 
					var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);

					var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
					if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
					{
						if (seenLocally.Add(rowKey))
						{
							foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
							{
								getItemsToReduceParams.Take--;
								yield return mappedResultInfo;
							}
						}
					}

					if (getItemsToReduceParams.Take <= 0)
						yield break;
				} 
                while (Api.TryMoveNext(session, ScheduledReductions));

                keysToRemove.Add(reduceKey);

				if (getItemsToReduceParams.Take <= 0)
					break;
			}

            foreach (var keyToRemove in keysToRemove)
                getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
		}