Example #1
0
        /// <summary>
        /// Applies the specified action for all shard sessions.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="commands">The shard sessions.</param>
        /// <param name="operation">The operation.</param>
        /// <returns></returns>
        public T[] Apply <T>(IList <IDatabaseCommands> commands, ShardRequestData request, Func <IDatabaseCommands, int, T> operation)
        {
            var list   = new List <T>();
            var errors = new List <Exception>();

            for (int i = 0; i < commands.Count; i++)
            {
                try
                {
                    list.Add(operation(commands[i], i));
                }
                catch (Exception e)
                {
                    var error = OnError;
                    if (error == null)
                    {
                        throw;
                    }
                    if (error(commands[i], request, e) == false)
                    {
                        throw;
                    }
                    errors.Add(e);
                }
            }

            // if ALL nodes failed, we still throw
            if (errors.Count == commands.Count)
            {
                throw new AggregateException(errors);
            }

            return(list.ToArray());
        }
        public async Task RefreshAsync <T>(T entity, CancellationToken token = default(CancellationToken))
        {
            DocumentMetadata value;

            if (entitiesAndMetadata.TryGetValue(entity, out value) == false)
            {
                throw new InvalidOperationException("Cannot refresh a transient instance");
            }
            IncrementRequestCount();

            var shardRequestData = new ShardRequestData
            {
                EntityType = typeof(T),
                Keys       = { value.Key }
            };
            var dbCommands = GetCommandsToOperateOn(shardRequestData);

            var results = await shardStrategy.ShardAccessStrategy.ApplyAsync(dbCommands, shardRequestData, async (dbCmd, i) =>
            {
                var jsonDocument = await dbCmd.GetAsync(value.Key, token).ConfigureAwait(false);
                if (jsonDocument == null)
                {
                    return(false);
                }

                RefreshInternal(entity, jsonDocument, value);
                return(true);
            }).WithCancellation(token).ConfigureAwait(false);

            if (results.All(x => x == false))
            {
                throw new InvalidOperationException("Document '" + value.Key + "' no longer exists and was probably deleted");
            }
        }
        private IList <Tuple <string, IDatabaseCommands> > GetShardsToOperateOn(ShardRequestData resultionData)
        {
            var shardIds = shardStrategy.ShardResolutionStrategy.PotentialShardsFor(resultionData);

            IEnumerable <KeyValuePair <string, IDatabaseCommands> > cmds = shardDbCommands;

            if (shardIds == null)
            {
                return(cmds.Select(x => Tuple.Create(x.Key, x.Value)).ToList());
            }

            var list = new List <Tuple <string, IDatabaseCommands> >();

            foreach (var shardId in shardIds)
            {
                IDatabaseCommands value;
                if (shardDbCommands.TryGetValue(shardId, out value) == false)
                {
                    throw new InvalidOperationException("Could not find shard id: " + shardId);
                }

                list.Add(Tuple.Create(shardId, value));
            }
            return(list);
        }
        public Task <T> LoadAsync <T>(string id)
        {
            object existingEntity;

            if (entitiesByKey.TryGetValue(id, out existingEntity))
            {
                return(CompletedTask.With((T)existingEntity));
            }

            IncrementRequestCount();
            var shardRequestData = new ShardRequestData
            {
                EntityType = typeof(T),
                Keys       = { id }
            };

            var dbCommands = GetCommandsToOperateOn(shardRequestData);
            var results    = shardStrategy.ShardAccessStrategy.ApplyAsync(dbCommands, shardRequestData, (commands, i) =>
            {
                var loadOperation = new LoadOperation(this, commands.DisableAllCaching, id);

                Func <Task> executer = null;
                executer             = () =>
                {
                    loadOperation.LogOperation();

                    var loadContext = loadOperation.EnterLoadContext();
                    return(commands.GetAsync(id).ContinueWith(task =>
                    {
                        if (loadContext != null)
                        {
                            loadContext.Dispose();
                        }

                        if (loadOperation.SetResult(task.Result))
                        {
                            return executer();
                        }
                        return new CompletedTask();
                    }).Unwrap());
                };
                return(executer().ContinueWith(_ =>
                {
                    _.AssertNotFailed();
                    return loadOperation.Complete <T>();
                }));
            });

            return(results.ContinueWith(task =>
            {
                var shardsContainThisDocument = task.Result.Where(x => !Equals(x, default(T))).ToArray();
                if (shardsContainThisDocument.Count() > 1)
                {
                    throw new InvalidOperationException("Found document with id: " + id +
                                                        " on more than a single shard, which is not allowed. Document keys have to be unique cluster-wide.");
                }

                return shardsContainThisDocument.FirstOrDefault();
            }));
        }
        public T Load <T>(string id)
        {
            if (IsDeleted(id))
            {
                return(default(T));
            }

            object existingEntity;

            if (entitiesByKey.TryGetValue(id, out existingEntity))
            {
                return((T)existingEntity);
            }
            JsonDocument value;

            if (includedDocumentsByKey.TryGetValue(id, out value))
            {
                includedDocumentsByKey.Remove(id);
                return(TrackEntity <T>(value));
            }
            IncrementRequestCount();
            var shardRequestData = new ShardRequestData
            {
                EntityType = typeof(T),
                Keys       = { id }
            };
            var dbCommands = GetCommandsToOperateOn(shardRequestData);
            var results    = shardStrategy.ShardAccessStrategy.Apply(dbCommands, shardRequestData, (commands, i) =>
            {
                var loadOperation = new ShardLoadOperation(this, commands.DisableAllCaching, id);
                bool retry;
                do
                {
                    loadOperation.LogOperation();
                    using (loadOperation.EnterLoadContext())
                    {
                        retry = loadOperation.SetResult(commands.Get(id));
                    }
                } while (retry);
                return(loadOperation.Complete <T>());
            });

            var shardsContainThisDocument = results.Where(x => !Equals(x, default(T))).ToArray();

            if (shardsContainThisDocument.Count() > 1)
            {
                throw new InvalidOperationException("Found document with id: " + id +
                                                    " on more than a single shard, which is not allowed. Document keys have to be unique cluster-wide.");
            }

            if (shardsContainThisDocument.Count() == 0)
            {
                RegisterMissing(id);
                return(default(T));
            }

            return(shardsContainThisDocument.First());
        }
        /// <summary>
        ///  Selects the shard ids appropriate for the specified data.
        ///  </summary>
        /// <returns>Return a list of shards ids that will be search. Returning null means search all shards.</returns>
        public virtual IList <string> PotentialShardsFor(ShardRequestData requestData)
        {
            if (requestData.Query != null)
            {
                //The issue was introduced when we added escaping for forward facing dashes so they won't be counted as comments
                //using Lucene unescaping may cause backward compatibility issues since people may have based their sharded strategy on
                //the query format and we can't break change that.
                var   unescapedQuery = requestData.Query.Query.Replace("\\/", "/");
                Regex regex;
                if (regexToCaptureShardIdFromQueriesByType.TryGetValue(requestData.EntityType, out regex) == false)
                {
                    return(PotentialShardsFor(requestData, null)); // we have no special knowledge, let us just query everything
                }
                var collection = regex.Matches(unescapedQuery);
                if (collection.Count == 0)
                {
                    return(PotentialShardsFor(requestData, null)); // we don't have the sharding field, we have to query over everything
                }
                var translateQueryValueToShardId = queryResultToStringByType[requestData.EntityType];

                var potentialShardsFor = collection.Cast <Match>().Select(match => translateQueryValueToShardId(match.Groups["shardId"].Value)).ToList();

                if (potentialShardsFor.Any(queryShardId => ShardIds.Contains(queryShardId, StringComparer.OrdinalIgnoreCase)) == false)
                {
                    return(PotentialShardsFor(requestData, null)); // we couldn't find the shard ids here, maybe there is something wrong in the query, sending to all shards
                }
                return(PotentialShardsFor(requestData, potentialShardsFor));
            }

            if (requestData.Keys.Count == 0) // we are only optimized for keys
            {
                return(PotentialShardsFor(requestData, null));
            }

            // we are looking for search by key, let us see if we can narrow it down by using the
            // embedded shard id.
            var list = new List <string>();

            foreach (var key in requestData.Keys)
            {
                var start = key.IndexOf(shardStrategy.Conventions.IdentityPartsSeparator, StringComparison.OrdinalIgnoreCase);
                if (start == -1)
                {
                    return(PotentialShardsFor(requestData, null)); // if we couldn't figure it out, select from all
                }
                var maybeShardId = key.Substring(0, start);

                if (ShardIds.Any(x => string.Equals(maybeShardId, x, StringComparison.OrdinalIgnoreCase)))
                {
                    list.Add(maybeShardId);
                }
                else
                {
                    return(PotentialShardsFor(requestData, null)); // we couldn't find it there, select from all
                }
            }
            return(PotentialShardsFor(requestData, list));
        }
        /// <summary>
        ///  Selects the shard ids appropriate for the specified data.
        ///  </summary><returns>Return a list of shards ids that will be search. Returning null means search all shards.</returns>
        public virtual IList <string> PotentialShardsFor(ShardRequestData requestData)
        {
            if (requestData.Query != null)
            {
                Regex regex;
                if (regexToCaptureShardIdFromQueriesByType.TryGetValue(requestData.EntityType, out regex) == false)
                {
                    return(null);                    // we have no special knowledge, let us just query everything
                }
                var collection = regex.Matches(requestData.Query.Query);
                if (collection.Count == 0)
                {
                    return(null);                    // we don't have the sharding field, we have to query over everything
                }
                var translateQueryValueToShardId = queryResultToStringByType[requestData.EntityType];

                var potentialShardsFor = collection.Cast <Match>().Select(match => translateQueryValueToShardId(match.Groups["shardId"].Value)).ToList();

                if (potentialShardsFor.Any(queryShardId => ShardIds.Contains(queryShardId, StringComparer.OrdinalIgnoreCase)) == false)
                {
                    return(null);                    // we couldn't find the shard ids here, maybe there is something wrong in the query, sending to all shards
                }
                return(potentialShardsFor);
            }

            if (requestData.Keys.Count == 0)             // we are only optimized for keys
            {
                return(null);
            }


            // we are looking for search by key, let us see if we can narrow it down by using the
            // embedded shard id.
            var list = new List <string>();

            foreach (var key in requestData.Keys)
            {
                var start = key.IndexOf(shardStrategy.Conventions.IdentityPartsSeparator, StringComparison.OrdinalIgnoreCase);
                if (start == -1)
                {
                    return(null);                    // if we couldn't figure it out, select from all
                }
                var maybeShardId = key.Substring(0, start);

                if (ShardIds.Any(x => string.Equals(maybeShardId, x, StringComparison.OrdinalIgnoreCase)))
                {
                    list.Add(maybeShardId);
                }
                else
                {
                    return(null);                    // we couldn't find it there, select from all
                }
            }
            return(list.ToArray());
        }
Example #8
0
        public Task <T[]> ApplyAsync <T>(IList <IAsyncDatabaseCommands> commands, ShardRequestData request, Func <IAsyncDatabaseCommands, int, Task <T> > operation)
        {
            var resultsTask = new TaskCompletionSource <List <T> >();
            var results     = new List <T>();
            var errors      = new List <Exception>();

            Action <int> executer = null;

            executer = index =>
            {
                if (index >= commands.Count)
                {
                    if (errors.Count == commands.Count)
                    {
                        throw new AggregateException(errors);
                    }
                    // finished all commands successfully
                    resultsTask.SetResult(results);
                    return;
                }

                operation(commands[index], index).ContinueWith(task =>
                {
                    if (task.IsFaulted)
                    {
                        var error = OnAsyncError;
                        if (error == null)
                        {
                            resultsTask.SetException(task.Exception);
                            return;
                        }
                        if (error(commands[index], request, task.Exception) == false)
                        {
                            resultsTask.SetException(task.Exception);
                            return;
                        }
                        errors.Add(task.Exception);
                    }
                    else
                    {
                        results.Add(task.Result);
                    }

                    // After we've dealt with one result, we call the operation on the next shard
                    executer(index + 1);
                });
            };

            executer(0);
            return(resultsTask.Task.ContinueWith(task => task.Result.ToArray()));
        }
		public IList<string> PotentialShardsFor(ShardRequestData requestData)
		{
			if (requestData.EntityType == typeof(User))
				return new[] { "Users" };
			if (requestData.EntityType == typeof(Blog))
				return new[] { "Blogs" };
			if (requestData.EntityType == typeof (Post) 
				|| requestData.EntityType == typeof (TotalVotesUp.ReduceResult)
				|| requestData.EntityType == typeof (TotalPostsPerDay.ReduceResult)
				)
				return Enumerable.Range(0, numberOfShardsForPosts).Select(i => "Posts" + (i + 1).ToString("D2")).ToArray();

			throw new ArgumentException("Cannot get shard id for '" + requestData.EntityType + "' because it is not a User, Blog or Post");
		}
Example #10
0
        void ISyncAdvancedSessionOperation.Refresh <T>(T entity)
        {
            DocumentMetadata value;

            if (entitiesAndMetadata.TryGetValue(entity, out value) == false)
            {
                throw new InvalidOperationException("Cannot refresh a transient instance");
            }
            IncrementRequestCount();


            var shardRequestData = new ShardRequestData
            {
                EntityType = typeof(T),
                Keys       = { value.Key }
            };
            var dbCommands = GetCommandsToOperateOn(shardRequestData);

            var results = shardStrategy.ShardAccessStrategy.Apply(dbCommands, shardRequestData, (dbCmd, i) =>
            {
                var jsonDocument = dbCmd.Get(value.Key);
                if (jsonDocument == null)
                {
                    return(false);
                }

                value.Metadata         = jsonDocument.Metadata;
                value.OriginalMetadata = (RavenJObject)jsonDocument.Metadata.CloneToken();
                value.ETag             = jsonDocument.Etag;
                value.OriginalValue    = jsonDocument.DataAsJson;
                var newEntity          = ConvertToEntity <T>(value.Key, jsonDocument.DataAsJson, jsonDocument.Metadata);
                foreach (
                    var property in
                    entity.GetType().GetProperties().Where(
                        property => property.CanWrite && property.CanRead && property.GetIndexParameters().Length == 0))
                {
                    property.SetValue(entity, property.GetValue(newEntity, null), null);
                }
                return(true);
            });

            if (results.All(x => x == false))
            {
                throw new InvalidOperationException("Document '" + value.Key + "' no longer exists and was probably deleted");
            }
        }
        /// <summary>
        /// Applies the specified action to all shard sessions in parallel
        /// </summary>
        public Task <T[]> ApplyAsync <T>(IList <IAsyncDatabaseCommands> commands, ShardRequestData request, Func <IAsyncDatabaseCommands, int, Task <T> > operation)
        {
            return(Task.Factory.ContinueWhenAll(commands.Select(operation).ToArray(), tasks =>
            {
                var results = new List <T>(tasks.Length);
                int index = 0;
                var handledExceptions = new List <Exception>();
                var unhandledExceptions = new List <Exception>();
                foreach (var task in tasks)
                {
                    try
                    {
                        results.Add(task.Result);
                    }
                    catch (Exception e)
                    {
                        var error = OnAsyncError;
                        if (error == null)
                        {
                            unhandledExceptions.Add(e);
                        }
                        else if (error(commands[index], request, e) == false)
                        {
                            unhandledExceptions.Add(e);
                        }
                        else
                        {
                            handledExceptions.Add(e);
                        }
                    }
                    index++;
                }

                if (unhandledExceptions.Any())
                {
                    throw new AggregateException(unhandledExceptions);
                }

                if (handledExceptions.Count == tasks.Length)
                {
                    throw new AggregateException(handledExceptions);
                }

                return results.ToArray();
            }));
        }
        /// <summary>
        /// Get the json document by key from the store
        /// </summary>
        private async Task<JsonDocument> GetJsonDocumentAsync(string documentKey)
        {
             var shardRequestData = new ShardRequestData
            {
                EntityType = typeof(object),
                Keys = { documentKey }
            };
            var dbCommands = GetCommandsToOperateOn(shardRequestData);

            var documents = await shardStrategy.ShardAccessStrategy.ApplyAsync(dbCommands,
                shardRequestData,
                (commands, i) => commands.GetAsync(documentKey)).ConfigureAwait(false);

            var document = documents.FirstOrDefault(x => x != null);
            if (document != null)
                return document;

            throw new InvalidOperationException("Document '" + documentKey + "' no longer exists and was probably deleted");
             
        }
        /// <summary>
        /// Applies the specified action to all shard sessions in parallel
        /// </summary>
        public T[] Apply <T>(IList <IDatabaseCommands> commands, ShardRequestData request, Func <IDatabaseCommands, int, T> operation)
        {
            var returnedLists = new T[commands.Count];
            var valueSet      = new bool[commands.Count];
            var errors        = new Exception[commands.Count];

            commands
            .Select((cmd, i) =>
                    Task.Factory.StartNew(() => operation(cmd, i))
                    .ContinueWith(task =>
            {
                try
                {
                    returnedLists[i] = task.Result;
                    valueSet[i]      = true;
                }
                catch (Exception e)
                {
                    var error = OnError;
                    if (error == null)
                    {
                        throw;
                    }
                    if (error(commands[i], request, e) == false)
                    {
                        throw;
                    }
                    errors[i] = e;
                }
            })
                    )
            .WaitAll();

            // if ALL nodes failed, we still throw
            if (errors.All(x => x != null))
            {
                throw new AggregateException(errors);
            }

            return(returnedLists.Where((t, i) => valueSet[i]).ToArray());
        }
        protected override JsonDocument GetJsonDocument(string documentKey)
        {
            var shardRequestData = new ShardRequestData
            {
                EntityType = typeof(object), Keys = { documentKey }
            };
            var dbCommands = GetCommandsToOperateOn(shardRequestData);

            var documents = shardStrategy.ShardAccessStrategy.Apply(dbCommands,
                                                                    shardRequestData,
                                                                    (commands, i) => commands.Get(documentKey));

            var document = documents.FirstOrDefault(x => x != null);

            if (document != null)
            {
                return(document);
            }

            throw new InvalidOperationException("Document '" + documentKey + "' no longer exists and was probably deleted");
        }
        /// <summary>
        /// Applies the specified action for all shard sessions.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="commands">The shard sessions.</param>
        /// <param name="operation">The operation.</param>
        /// <returns></returns>
        public T[] Apply <T>(IList <IDatabaseCommands> commands, ShardRequestData request, Func <IDatabaseCommands, int, T> operation)
        {
            var list   = new List <T>();
            var errors = new List <Exception>();

            for (int i = 0; i < commands.Count; i++)
            {
                try
                {
                    list.Add(operation(commands[i], i));
                }
                catch (Exception e)
                {
                    var error = OnError;
                    if (error == null)
                    {
                        throw;
                    }
                    if (error(commands[i], request, e) == false)
                    {
                        throw;
                    }
                    errors.Add(e);
                }
            }

            // if ALL nodes failed, we still throw
            if (errors.Count == commands.Count)
#if !NET35
            { throw new AggregateException(errors); }
#else
            { throw new InvalidOperationException("Got an error from all servers", errors.First())
                    {
                        Data = { { "Errors", errors } }
                    } };
#endif

            return(list.ToArray());
        }
 /// <summary>
 ///     Selects the shard ids appropriate for the specified data.
 /// </summary>
 /// <returns>
 ///     Return a list of shards ids that will be search. Returning null means search all shards.
 /// </returns>
 public IList<string> PotentialShardsFor(ShardRequestData requestData)
 {
     if (requestData.Query != null)
     {
         var shardIdRxpr =
             new Regex(
                 string.Format(
                     "\r\n{0}: \\s* (?<Open>\")(?<shardId>[^\"]+)(?<Close-Open>\") |\r\n{0}: \\s* (?<shardId>[^\"][^\\s]*)",
                     Regex.Escape("SegmentId")), RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
         Match match = shardIdRxpr.Match(requestData.Query.Query);
         if (match.Success)
         {
             string shardId = match.Groups["shardId"].ToString();
             var res = new List<string>();
             if (_shards.Keys.Any(k => k.StartsWith(shardId)))
             {
                 res.Add(shardId);
             }
             if (res.Count > 0)
                 return res;
         }
     }
     return null;
 }
 private IList <IDatabaseCommands> GetCommandsToOperateOn(ShardRequestData resultionData)
 {
     return(GetShardsToOperateOn(resultionData).Select(x => x.Item2).ToList());
 }
Example #18
0
 public IList<string> PotentialShardsFor(ShardRequestData requestData)
 {
     if (requestData.EntityType == typeof (Company))
     {
         // You can try to limit the potential shards based on the query
     }
     return null;
 }
Example #19
0
 /// <summary>
 /// Selects the shard ids appropriate for the specified data.
 /// </summary>
 /// <returns>Return a list of shards ids that will be search. Returning null means search all shards.</returns>
 public virtual IList <string> PotentialShardsFor(ShardRequestData requestData, List <string> potentialShardIds)
 {
     return(potentialShardIds);
 }