public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary <ReduceKeyAndBucket, int> removed)
        {
            var viewAndDocumentId = CreateKey(view, documentId);

            var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);

            using (var iterator = mappedResultsByViewAndDocumentId.MultiRead(Snapshot, viewAndDocumentId))
            {
                if (!iterator.Seek(Slice.BeforeAllKeys))
                {
                    return;
                }

                do
                {
                    var id = iterator.CurrentKey.Clone();

                    ushort version;
                    var    value     = LoadJson(tableStorage.MappedResults, id, writeBatch.Value, out version);
                    var    reduceKey = value.Value <string>("reduceKey");
                    var    bucket    = value.Value <int>("bucket");

                    DeleteMappedResult(id, view, documentId, reduceKey, bucket.ToString(CultureInfo.InvariantCulture));

                    var reduceKeyAndBucket = new ReduceKeyAndBucket(bucket, reduceKey);
                    removed[reduceKeyAndBucket] = removed.GetOrDefault(reduceKeyAndBucket) + 1;
                }while (iterator.MoveNext());
            }
        }
        public void ScheduleReductions(string view, int level, ReduceKeyAndBucket reduceKeysAndBucket)
        {
            var etag = generator.CreateSequentialUuid(UuidType.ScheduledReductions).ToByteArray();

            storage.ScheduleReductions.UpdateKey(new RavenJObject
            {
                { "view", view },
                { "reduceKey", reduceKeysAndBucket.ReduceKey },
                { "bucket", reduceKeysAndBucket.Bucket },
                { "level", level },
                { "etag", etag },
                { "timestamp", SystemTime.UtcNow }
            });
        }
        public void DeleteMappedResultsForDocumentId(string documentId, string view, Dictionary <ReduceKeyAndBucket, int> removed)
        {
            foreach (var key in storage.MappedResults["ByViewAndDocumentId"].SkipTo(new RavenJObject
            {
                { "view", view },
                { "docId", documentId }
            }).TakeWhile(x => StringComparer.OrdinalIgnoreCase.Equals(x.Value <string>("view"), view) &&
                         StringComparer.OrdinalIgnoreCase.Equals(x.Value <string>("docId"), documentId)))
            {
                storage.MappedResults.Remove(key);

                var reduceKey = key.Value <string>("reduceKey");
                var bucket    = new ReduceKeyAndBucket(key.Value <int>("bucket"), reduceKey);
                removed[bucket] = removed.GetOrDefault(bucket) + 1;
            }
        }
        public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
        {
            var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
            var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);

            var id            = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
            var idAsString    = id.ToString();
            var reduceHashKey = HashKey(reduceKeysAndBuckets.ReduceKey);

            tableStorage.ScheduledReductions.Add(writeBatch.Value, idAsString, new RavenJObject
            {
                { "view", view },
                { "reduceKey", reduceKeysAndBuckets.ReduceKey },
                { "bucket", reduceKeysAndBuckets.Bucket },
                { "level", level },
                { "etag", id.ToByteArray() },
                { "timestamp", SystemTime.UtcNow }
            });

            scheduledReductionsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString);
            scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, level, reduceKeysAndBuckets.ReduceKey, reduceHashKey), idAsString);
        }
Ejemplo n.º 5
0
        public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary<ReduceKeyAndBucket, int> removed)
        {
            Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_doc_key");
            Api.MakeKey(session, MappedResults, view, MakeKeyGrbit.NewKey);
            Api.MakeKey(session, MappedResults, documentId, Encoding.Unicode, MakeKeyGrbit.None);
            if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekEQ) == false)
                return;

            Api.MakeKey(session, MappedResults, view, MakeKeyGrbit.NewKey);
            Api.MakeKey(session, MappedResults, documentId, Encoding.Unicode, MakeKeyGrbit.None);
            Api.JetSetIndexRange(session, MappedResults, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive);
            do
            {
                // esent index ranges are approximate, and we need to check them ourselves as well
                var documentIdFromDb = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"]);
                if (StringComparer.OrdinalIgnoreCase.Equals(documentIdFromDb, documentId) == false)
                    continue;
                var reduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"],
                                                           Encoding.Unicode);
                var bucket = Api.RetrieveColumnAsInt32(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"]).Value;

                var key = new ReduceKeyAndBucket(bucket, reduceKey);
                removed[key] = removed.GetOrDefault(key) + 1;
                Api.JetDelete(session, MappedResults);
            } while (Api.TryMoveNext(session, MappedResults));
        }
Ejemplo n.º 6
0
        public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
        {
            Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");

            var viewReductionColumn = tableColumnsCache.ScheduledReductionColumns["view"];
            var levelReductionColumn = tableColumnsCache.ScheduledReductionColumns["level"];
            var reduceReductionColumn = tableColumnsCache.ScheduledReductionColumns["reduce_key"];
            var bucketReductionColumn = tableColumnsCache.ScheduledReductionColumns["bucket"];

            var keysToRemove = new List<string>();
            var output = new List<MappedResultInfo>();
            var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
            
            try
            {
                var first = true;
                foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    int initialBucket = 0;
                    bool needToMoveNext = false;
                    if (first)
                    {
                        first = false;
                        if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
                        {
                            if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
                            {
                                needToMoveNext = true;
                                initialBucket = getItemsToReduceParams.LastReduceKeyAndBucket.Bucket;
                            }
                        }
                    }

                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, initialBucket, MakeKeyGrbit.None);
                    
                    if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false ||
                        (needToMoveNext && Api.TryMoveNext(session, ScheduledReductions) == false) )
                    {
                        keysToRemove.Add(reduceKey);
                        continue;
                    }


                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
                    Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
                    Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);

                    if (Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
                    {
                        keysToRemove.Add(reduceKey);
                        continue;
                    }

                    // this isn't used for optimized reading, but to make it easier to delete records later on
                    OptimizedDeleter reader;
                    if (getItemsToReduceParams.ItemsToDelete.Count == 0)
                    {
                        getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
                    }
                    else
                    {
                        reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
                    }

                    reader.IndexId = getItemsToReduceParams.Index;

                    do
                    {
                        cancellationToken.ThrowIfCancellationRequested();

                        if (getItemsToReduceParams.Take <= 0)
                            return output;

                        var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, viewReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex);
                        var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, levelReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex).Value;
                        var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, reduceReductionColumn);

                        if (getItemsToReduceParams.Index != indexFromDb)
                            break;

                        if (levelFromDb != getItemsToReduceParams.Level)
                            break;

                        if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
                            break;

                        var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, bucketReductionColumn).Value;

                        var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb); 
                        var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);


                        if (thisIsNewScheduledReductionRow)
                        {
                            if (seenLocally.Add(rowKey))
                            {
                                getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
                                foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
                                {
                                    getItemsToReduceParams.Take--;

                                    output.Add(mappedResultInfo);
                                }
                            }
                        }
                    } 
                    while (Api.TryMoveNext(session, ScheduledReductions));

                    keysToRemove.Add(reduceKey);

                    if (getItemsToReduceParams.Take <= 0)
                        break;
                }

                return output;
            }
            finally
            {
                // In whatever condition we would have to return, we must signal the removal of the reduce keys.
                foreach (var keyToRemove in keysToRemove)
                    getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
            }
        }
Ejemplo n.º 7
0
        public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBucket)
        {
            var bucket = reduceKeysAndBucket.Bucket;

            using (var map = new Update(session, ScheduledReductions, JET_prep.Insert))
            {
                Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], view);
                Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"],
                              reduceKeysAndBucket.ReduceKey, Encoding.Unicode);
                Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["hashed_reduce_key"],
                              HashReduceKey(reduceKeysAndBucket.ReduceKey));

                    Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["etag"], uuidGenerator.CreateSequentialUuid(UuidType.ScheduledReductions).ToByteArray());

                Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["timestamp"],
                              SystemTime.UtcNow.ToBinary());


                Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"],
                              bucket);

                Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], level);
                map.Save();
            }
            if (scheduledReductionsPerViewAndLevel != null)
                scheduledReductionsPerViewAndLevel.AddOrUpdate(view, new RemainingReductionPerLevel(level), (key, oldvalue) => oldvalue.IncrementPerLevelCounters(level));
        }
		public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
		{
			var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
			var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);

			var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
            var idSlice = (Slice)id.ToString();
		    var reduceHashKey = HashKey(reduceKeysAndBuckets.ReduceKey);

			var scheduledReduction = new Structure<ScheduledReductionFields>(tableStorage.ScheduledReductions.Schema);

			scheduledReduction.Set(ScheduledReductionFields.IndexId, view)
				.Set(ScheduledReductionFields.ReduceKey, reduceKeysAndBuckets.ReduceKey)
				.Set(ScheduledReductionFields.Bucket, reduceKeysAndBuckets.Bucket)
				.Set(ScheduledReductionFields.Level, level)
				.Set(ScheduledReductionFields.Etag, id.ToByteArray())
				.Set(ScheduledReductionFields.Timestamp, SystemTime.UtcNow.ToBinary());

			tableStorage.ScheduledReductions.AddStruct(writeBatch.Value, idSlice, scheduledReduction);

            var viewKey = CreateKey(view);
            var viewKeySlice = (Slice)viewKey;


            scheduledReductionsByView.MultiAdd(writeBatch.Value, viewKeySlice, idSlice);
            scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, (Slice)AppendToKey(viewKey, level, ReduceKeySizeLimited(reduceKeysAndBuckets.ReduceKey), reduceHashKey), idSlice);
		}
		public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary<ReduceKeyAndBucket, int> removed)
		{
            var viewKey = CreateKey(view);
            var viewKeySlice = new Slice(viewKey);
			var viewAndDocumentId = new Slice(AppendToKey(viewKey, documentId));

            var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
            var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView);
            var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey);
            var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket);
            var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data);

			using (var iterator = mappedResultsByViewAndDocumentId.MultiRead(Snapshot, viewAndDocumentId))
			{
				if (!iterator.Seek(Slice.BeforeAllKeys))
					return;

				do
				{
					// TODO: Check if we can relax the clone.
                    var id = iterator.CurrentKey.Clone();

					ushort version;
					var value = LoadStruct(tableStorage.MappedResults, id, writeBatch.Value, out version);
					var reduceKey = value.ReadString(MappedResultFields.ReduceKey);
					var bucket = value.ReadInt(MappedResultFields.Bucket);

                    var reduceKeyHash = HashKey(reduceKey);                  
                    var viewAndReduceKey = AppendToKey(viewKey, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
                    var viewAndReduceKeyAndSourceBucket = AppendToKey(viewAndReduceKey, bucket);

                    tableStorage.MappedResults.Delete(writeBatch.Value, id);

                    mappedResultsByViewAndDocumentId.MultiDelete(writeBatch.Value, viewAndDocumentId, id);
                    mappedResultsByView.MultiDelete(writeBatch.Value, viewKeySlice, id);
                    mappedResultsByViewAndReduceKey.MultiDelete(writeBatch.Value, (Slice)viewAndReduceKey, id);
                    mappedResultsByViewAndReduceKeyAndSourceBucket.MultiDelete(writeBatch.Value, (Slice)viewAndReduceKeyAndSourceBucket, id);
                    mappedResultsData.Delete(writeBatch.Value, id);

					var reduceKeyAndBucket = new ReduceKeyAndBucket(bucket, reduceKey);
					removed[reduceKeyAndBucket] = removed.GetOrDefault(reduceKeyAndBucket) + 1;
				}
				while (iterator.MoveNext());
			}
		}
Ejemplo n.º 10
0
		public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBucket)
		{
			var bucket = reduceKeysAndBucket.Bucket;

			using (var map = new Update(session, ScheduledReductions, JET_prep.Insert))
			{
			    Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], view);
				Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"],
							  reduceKeysAndBucket.ReduceKey, Encoding.Unicode);
				Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["hashed_reduce_key"],
							  HashReduceKey(reduceKeysAndBucket.ReduceKey));

					Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["etag"], uuidGenerator.CreateSequentialUuid(UuidType.ScheduledReductions).ToByteArray());

				Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["timestamp"],
							  SystemTime.UtcNow.ToBinary());


				Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"],
							  bucket);

				Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], level);
				map.Save();
			}
		}
Ejemplo n.º 11
0
		public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
		{
			var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
            var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
            {
                var etag = o as Etag;
                if (etag == null) 
                    return null;

                return (Slice)etag.ToString();
            });

            var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);

            var keysToRemove = new List<string>();
			foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
			{
				cancellationToken.ThrowIfCancellationRequested();

			    var reduceKeyHash = HashKey(reduceKey);
                var viewAndLevelAndReduceKey = (Slice) CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
				using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
				{
					if (!iterator.Seek(Slice.BeforeAllKeys))
						continue;

					do
					{
						cancellationToken.ThrowIfCancellationRequested();

						if (getItemsToReduceParams.Take <= 0)
							break;

						ushort version;
					    var value = LoadStruct(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version);
                        if (value == null) // TODO: Check if this is correct. 
                            continue;

						var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);                        

						var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
                        var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);

					    var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));
						var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);

						if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
						{
							if (seenLocally.Add(rowKey))
							{
								foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
								{
									getItemsToReduceParams.Take--;
									yield return mappedResultInfo;
								}
							}
						}

						if (getItemsToReduceParams.Take <= 0)
							yield break;
					}
					while (iterator.MoveNext());
				}

                keysToRemove.Add(reduceKey);

				if (getItemsToReduceParams.Take <= 0)
                    yield break;
			}

            foreach (var keyToRemove in keysToRemove)
                getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
		}
Ejemplo n.º 12
0
		public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
		{
			var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
			var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);

			var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
			var idAsString = id.ToString();
		    var reduceHashKey = HashKey(reduceKeysAndBuckets.ReduceKey);

			tableStorage.ScheduledReductions.Add(writeBatch.Value, idAsString, new RavenJObject
			{
				{"view", view},
				{"reduceKey", reduceKeysAndBuckets.ReduceKey},
				{"bucket", reduceKeysAndBuckets.Bucket},
				{"level", level},
				{"etag", id.ToByteArray()},
				{"timestamp", SystemTime.UtcNow}
			});

			scheduledReductionsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString);
			scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, level, reduceKeysAndBuckets.ReduceKey, reduceHashKey), idAsString);
		}
Ejemplo n.º 13
0
		public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary<ReduceKeyAndBucket, int> removed)
		{
			var viewAndDocumentId = CreateKey(view, documentId);

			var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
			using (var iterator = mappedResultsByViewAndDocumentId.MultiRead(Snapshot, viewAndDocumentId))
			{
				if (!iterator.Seek(Slice.BeforeAllKeys))
					return;

				do
				{
					var id = iterator.CurrentKey.Clone();

					ushort version;
					var value = LoadJson(tableStorage.MappedResults, id, writeBatch.Value, out version);
					var reduceKey = value.Value<string>("reduceKey");
					var bucket = value.Value<int>("bucket");

					DeleteMappedResult(id, view, documentId, reduceKey, bucket.ToString(CultureInfo.InvariantCulture));

					var reduceKeyAndBucket = new ReduceKeyAndBucket(bucket, reduceKey);
					removed[reduceKeyAndBucket] = removed.GetOrDefault(reduceKeyAndBucket) + 1;
				}
				while (iterator.MoveNext());
			}
		}
Ejemplo n.º 14
0
	    public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
		{
			var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
            var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
            {
                var etag = o as Etag;
                if (etag == null) 
                    return null;

                return (Slice)etag.ToString();
            });

            var keysToRemove = new List<string>();

            try
            {
                var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
                var mappedResults = new List<MappedResultInfo>();

                var first = true;
                foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    Slice start = Slice.BeforeAllKeys;
                    bool needToMoveNext = false;
                    if (first)
                    {
                        first = false;
                        if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
                        {
                            if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
                            {
                                throw new InvalidOperationException("Mismatches last reduce key with the remaining reduce keys in the params");
                            }
                            needToMoveNext = true;
                            start = CreateBucketAndEtagKey(getItemsToReduceParams.LastReduceKeyAndBucket.Bucket, Etag.Empty);
                        }
                    }
                    var viewAndLevelAndReduceKey = CreateScheduleReductionKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey);
                    using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
                    {
                        if (!iterator.Seek(start))
                            continue;

                        if(needToMoveNext && iterator.MoveNext() ==false)
                            continue;

                        do
                        {
                            cancellationToken.ThrowIfCancellationRequested();

                            if (getItemsToReduceParams.Take <= 0)
                                break;

                            var idValueReader = iterator.CurrentKey.CreateReader();
                            idValueReader.ReadBigEndianInt32(); // bucket
                            int _;
                            var id = new Slice(Etag.Parse(idValueReader.ReadBytes(16, out _)));


                            ushort version;
                            var value = LoadStruct(tableStorage.ScheduledReductions, id, writeBatch.Value, out version);
                            if (value == null) // TODO: Check if this is correct. 
                                continue;

                            var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);

                            var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
                            var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);

                            var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));

                            if (thisIsNewScheduledReductionRow)
                            {
                                if (seenLocally.Add(rowKey))
                                {
                                    getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
                                    foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
                                    {
                                        getItemsToReduceParams.Take--;

                                        mappedResults.Add(mappedResultInfo);
                                    }
                                }
                            }

                            if (getItemsToReduceParams.Take <= 0)
                                return mappedResults;
                        }
                        while (iterator.MoveNext());
                    }

                    keysToRemove.Add(reduceKey);

                    if (getItemsToReduceParams.Take <= 0)
                        break;
                }

                return mappedResults;
            }
            finally
            {
                foreach (var keyToRemove in keysToRemove)
                    getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
            }
		}
Ejemplo n.º 15
0
		public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
		{
			var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
			var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);

			var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
            var idSlice = (Slice)id.ToString();

			var scheduledReduction = new Structure<ScheduledReductionFields>(tableStorage.ScheduledReductions.Schema);

			scheduledReduction.Set(ScheduledReductionFields.IndexId, view)
				.Set(ScheduledReductionFields.ReduceKey, reduceKeysAndBuckets.ReduceKey)
				.Set(ScheduledReductionFields.Bucket, reduceKeysAndBuckets.Bucket)
				.Set(ScheduledReductionFields.Level, level)
				.Set(ScheduledReductionFields.Etag, id.ToByteArray())
				.Set(ScheduledReductionFields.Timestamp, SystemTime.UtcNow.ToBinary());

			tableStorage.ScheduledReductions.AddStruct(writeBatch.Value, idSlice, scheduledReduction);

            var viewKey = CreateViewKey(view);

            scheduledReductionsByView.MultiAdd(writeBatch.Value, viewKey, idSlice);
		    var scheduleReductionKey = CreateScheduleReductionKey(view, level, reduceKeysAndBuckets.ReduceKey);
		    scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, scheduleReductionKey, CreateBucketAndEtagKey(reduceKeysAndBuckets.Bucket, id));
			if (scheduledReductionsPerViewAndLevel != null)
				scheduledReductionsPerViewAndLevel.AddOrUpdate(view, new RemainingReductionPerLevel(level), (key, oldvalue) => oldvalue.IncrementPerLevelCounters(level));
		}
Ejemplo n.º 16
0
		public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
		{
			Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");

            var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
            var keysToRemove = new List<string>();
			foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
			{
				cancellationToken.ThrowIfCancellationRequested();

				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, 0, MakeKeyGrbit.None);
				if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
					continue;

				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
				Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
				Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);

				if(Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
					continue;

				// this isn't used for optimized reading, but to make it easier to delete records later on
				OptimizedDeleter reader;
				if (getItemsToReduceParams.ItemsToDelete.Count == 0)
				{
					getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
				}
				else
				{
					reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
				}

                reader.IndexId = getItemsToReduceParams.Index;

				do
				{
					cancellationToken.ThrowIfCancellationRequested();

                    if (getItemsToReduceParams.Take <= 0)
                        break;
					var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], RetrieveColumnGrbit.RetrieveFromIndex);
					var levelFromDb =
						Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
							Value;
					var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
												   tableColumnsCache.ScheduledReductionColumns["reduce_key"]);

					if (getItemsToReduceParams.Index != indexFromDb)
						continue;
					if (levelFromDb != getItemsToReduceParams.Level)
						continue;
					if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
						continue;

					var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;

                    var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb); 
					var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);

					var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
					if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
					{
						if (seenLocally.Add(rowKey))
						{
							foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
							{
								getItemsToReduceParams.Take--;
								yield return mappedResultInfo;
							}
						}
					}

					if (getItemsToReduceParams.Take <= 0)
						yield break;
				} 
                while (Api.TryMoveNext(session, ScheduledReductions));

                keysToRemove.Add(reduceKey);

				if (getItemsToReduceParams.Take <= 0)
					break;
			}

            foreach (var keyToRemove in keysToRemove)
                getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
		}