/// <summary>
/// Applies the specified action for all shard sessions.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="commands">The shard sessions.</param>
/// <param name="operation">The operation.</param>
/// <returns></returns>
public T[] Apply <T>(IList <IDatabaseCommands> commands, ShardRequestData request, Func <IDatabaseCommands, int, T> operation)
{
var list = new List <T>();
var errors = new List <Exception>();
for (int i = 0; i < commands.Count; i++)
{
try
{
list.Add(operation(commands[i], i));
}
catch (Exception e)
{
var error = OnError;
if (error == null)
{
throw;
}
if (error(commands[i], request, e) == false)
{
throw;
}
errors.Add(e);
}
}
// if ALL nodes failed, we still throw
if (errors.Count == commands.Count)
{
throw new AggregateException(errors);
}
return(list.ToArray());
}
public async Task RefreshAsync <T>(T entity, CancellationToken token = default(CancellationToken))
{
DocumentMetadata value;
if (entitiesAndMetadata.TryGetValue(entity, out value) == false)
{
throw new InvalidOperationException("Cannot refresh a transient instance");
}
IncrementRequestCount();
var shardRequestData = new ShardRequestData
{
EntityType = typeof(T),
Keys = { value.Key }
};
var dbCommands = GetCommandsToOperateOn(shardRequestData);
var results = await shardStrategy.ShardAccessStrategy.ApplyAsync(dbCommands, shardRequestData, async (dbCmd, i) =>
{
var jsonDocument = await dbCmd.GetAsync(value.Key, token).ConfigureAwait(false);
if (jsonDocument == null)
{
return(false);
}
RefreshInternal(entity, jsonDocument, value);
return(true);
}).WithCancellation(token).ConfigureAwait(false);
if (results.All(x => x == false))
{
throw new InvalidOperationException("Document '" + value.Key + "' no longer exists and was probably deleted");
}
}
private IList <Tuple <string, IDatabaseCommands> > GetShardsToOperateOn(ShardRequestData resultionData)
{
var shardIds = shardStrategy.ShardResolutionStrategy.PotentialShardsFor(resultionData);
IEnumerable <KeyValuePair <string, IDatabaseCommands> > cmds = shardDbCommands;
if (shardIds == null)
{
return(cmds.Select(x => Tuple.Create(x.Key, x.Value)).ToList());
}
var list = new List <Tuple <string, IDatabaseCommands> >();
foreach (var shardId in shardIds)
{
IDatabaseCommands value;
if (shardDbCommands.TryGetValue(shardId, out value) == false)
{
throw new InvalidOperationException("Could not find shard id: " + shardId);
}
list.Add(Tuple.Create(shardId, value));
}
return(list);
}
public Task <T> LoadAsync <T>(string id)
{
object existingEntity;
if (entitiesByKey.TryGetValue(id, out existingEntity))
{
return(CompletedTask.With((T)existingEntity));
}
IncrementRequestCount();
var shardRequestData = new ShardRequestData
{
EntityType = typeof(T),
Keys = { id }
};
var dbCommands = GetCommandsToOperateOn(shardRequestData);
var results = shardStrategy.ShardAccessStrategy.ApplyAsync(dbCommands, shardRequestData, (commands, i) =>
{
var loadOperation = new LoadOperation(this, commands.DisableAllCaching, id);
Func <Task> executer = null;
executer = () =>
{
loadOperation.LogOperation();
var loadContext = loadOperation.EnterLoadContext();
return(commands.GetAsync(id).ContinueWith(task =>
{
if (loadContext != null)
{
loadContext.Dispose();
}
if (loadOperation.SetResult(task.Result))
{
return executer();
}
return new CompletedTask();
}).Unwrap());
};
return(executer().ContinueWith(_ =>
{
_.AssertNotFailed();
return loadOperation.Complete <T>();
}));
});
return(results.ContinueWith(task =>
{
var shardsContainThisDocument = task.Result.Where(x => !Equals(x, default(T))).ToArray();
if (shardsContainThisDocument.Count() > 1)
{
throw new InvalidOperationException("Found document with id: " + id +
" on more than a single shard, which is not allowed. Document keys have to be unique cluster-wide.");
}
return shardsContainThisDocument.FirstOrDefault();
}));
}
public T Load <T>(string id)
{
if (IsDeleted(id))
{
return(default(T));
}
object existingEntity;
if (entitiesByKey.TryGetValue(id, out existingEntity))
{
return((T)existingEntity);
}
JsonDocument value;
if (includedDocumentsByKey.TryGetValue(id, out value))
{
includedDocumentsByKey.Remove(id);
return(TrackEntity <T>(value));
}
IncrementRequestCount();
var shardRequestData = new ShardRequestData
{
EntityType = typeof(T),
Keys = { id }
};
var dbCommands = GetCommandsToOperateOn(shardRequestData);
var results = shardStrategy.ShardAccessStrategy.Apply(dbCommands, shardRequestData, (commands, i) =>
{
var loadOperation = new ShardLoadOperation(this, commands.DisableAllCaching, id);
bool retry;
do
{
loadOperation.LogOperation();
using (loadOperation.EnterLoadContext())
{
retry = loadOperation.SetResult(commands.Get(id));
}
} while (retry);
return(loadOperation.Complete <T>());
});
var shardsContainThisDocument = results.Where(x => !Equals(x, default(T))).ToArray();
if (shardsContainThisDocument.Count() > 1)
{
throw new InvalidOperationException("Found document with id: " + id +
" on more than a single shard, which is not allowed. Document keys have to be unique cluster-wide.");
}
if (shardsContainThisDocument.Count() == 0)
{
RegisterMissing(id);
return(default(T));
}
return(shardsContainThisDocument.First());
}
/// <summary>
/// Selects the shard ids appropriate for the specified data.
/// </summary>
/// <returns>Return a list of shards ids that will be search. Returning null means search all shards.</returns>
public virtual IList <string> PotentialShardsFor(ShardRequestData requestData)
{
if (requestData.Query != null)
{
//The issue was introduced when we added escaping for forward facing dashes so they won't be counted as comments
//using Lucene unescaping may cause backward compatibility issues since people may have based their sharded strategy on
//the query format and we can't break change that.
var unescapedQuery = requestData.Query.Query.Replace("\\/", "/");
Regex regex;
if (regexToCaptureShardIdFromQueriesByType.TryGetValue(requestData.EntityType, out regex) == false)
{
return(PotentialShardsFor(requestData, null)); // we have no special knowledge, let us just query everything
}
var collection = regex.Matches(unescapedQuery);
if (collection.Count == 0)
{
return(PotentialShardsFor(requestData, null)); // we don't have the sharding field, we have to query over everything
}
var translateQueryValueToShardId = queryResultToStringByType[requestData.EntityType];
var potentialShardsFor = collection.Cast <Match>().Select(match => translateQueryValueToShardId(match.Groups["shardId"].Value)).ToList();
if (potentialShardsFor.Any(queryShardId => ShardIds.Contains(queryShardId, StringComparer.OrdinalIgnoreCase)) == false)
{
return(PotentialShardsFor(requestData, null)); // we couldn't find the shard ids here, maybe there is something wrong in the query, sending to all shards
}
return(PotentialShardsFor(requestData, potentialShardsFor));
}
if (requestData.Keys.Count == 0) // we are only optimized for keys
{
return(PotentialShardsFor(requestData, null));
}
// we are looking for search by key, let us see if we can narrow it down by using the
// embedded shard id.
var list = new List <string>();
foreach (var key in requestData.Keys)
{
var start = key.IndexOf(shardStrategy.Conventions.IdentityPartsSeparator, StringComparison.OrdinalIgnoreCase);
if (start == -1)
{
return(PotentialShardsFor(requestData, null)); // if we couldn't figure it out, select from all
}
var maybeShardId = key.Substring(0, start);
if (ShardIds.Any(x => string.Equals(maybeShardId, x, StringComparison.OrdinalIgnoreCase)))
{
list.Add(maybeShardId);
}
else
{
return(PotentialShardsFor(requestData, null)); // we couldn't find it there, select from all
}
}
return(PotentialShardsFor(requestData, list));
}
/// <summary>
/// Selects the shard ids appropriate for the specified data.
/// </summary><returns>Return a list of shards ids that will be search. Returning null means search all shards.</returns>
public virtual IList <string> PotentialShardsFor(ShardRequestData requestData)
{
if (requestData.Query != null)
{
Regex regex;
if (regexToCaptureShardIdFromQueriesByType.TryGetValue(requestData.EntityType, out regex) == false)
{
return(null); // we have no special knowledge, let us just query everything
}
var collection = regex.Matches(requestData.Query.Query);
if (collection.Count == 0)
{
return(null); // we don't have the sharding field, we have to query over everything
}
var translateQueryValueToShardId = queryResultToStringByType[requestData.EntityType];
var potentialShardsFor = collection.Cast <Match>().Select(match => translateQueryValueToShardId(match.Groups["shardId"].Value)).ToList();
if (potentialShardsFor.Any(queryShardId => ShardIds.Contains(queryShardId, StringComparer.OrdinalIgnoreCase)) == false)
{
return(null); // we couldn't find the shard ids here, maybe there is something wrong in the query, sending to all shards
}
return(potentialShardsFor);
}
if (requestData.Keys.Count == 0) // we are only optimized for keys
{
return(null);
}
// we are looking for search by key, let us see if we can narrow it down by using the
// embedded shard id.
var list = new List <string>();
foreach (var key in requestData.Keys)
{
var start = key.IndexOf(shardStrategy.Conventions.IdentityPartsSeparator, StringComparison.OrdinalIgnoreCase);
if (start == -1)
{
return(null); // if we couldn't figure it out, select from all
}
var maybeShardId = key.Substring(0, start);
if (ShardIds.Any(x => string.Equals(maybeShardId, x, StringComparison.OrdinalIgnoreCase)))
{
list.Add(maybeShardId);
}
else
{
return(null); // we couldn't find it there, select from all
}
}
return(list.ToArray());
}
public Task <T[]> ApplyAsync <T>(IList <IAsyncDatabaseCommands> commands, ShardRequestData request, Func <IAsyncDatabaseCommands, int, Task <T> > operation)
{
var resultsTask = new TaskCompletionSource <List <T> >();
var results = new List <T>();
var errors = new List <Exception>();
Action <int> executer = null;
executer = index =>
{
if (index >= commands.Count)
{
if (errors.Count == commands.Count)
{
throw new AggregateException(errors);
}
// finished all commands successfully
resultsTask.SetResult(results);
return;
}
operation(commands[index], index).ContinueWith(task =>
{
if (task.IsFaulted)
{
var error = OnAsyncError;
if (error == null)
{
resultsTask.SetException(task.Exception);
return;
}
if (error(commands[index], request, task.Exception) == false)
{
resultsTask.SetException(task.Exception);
return;
}
errors.Add(task.Exception);
}
else
{
results.Add(task.Result);
}
// After we've dealt with one result, we call the operation on the next shard
executer(index + 1);
});
};
executer(0);
return(resultsTask.Task.ContinueWith(task => task.Result.ToArray()));
}
public IList<string> PotentialShardsFor(ShardRequestData requestData)
{
if (requestData.EntityType == typeof(User))
return new[] { "Users" };
if (requestData.EntityType == typeof(Blog))
return new[] { "Blogs" };
if (requestData.EntityType == typeof (Post)
|| requestData.EntityType == typeof (TotalVotesUp.ReduceResult)
|| requestData.EntityType == typeof (TotalPostsPerDay.ReduceResult)
)
return Enumerable.Range(0, numberOfShardsForPosts).Select(i => "Posts" + (i + 1).ToString("D2")).ToArray();
throw new ArgumentException("Cannot get shard id for '" + requestData.EntityType + "' because it is not a User, Blog or Post");
}
void ISyncAdvancedSessionOperation.Refresh <T>(T entity)
{
DocumentMetadata value;
if (entitiesAndMetadata.TryGetValue(entity, out value) == false)
{
throw new InvalidOperationException("Cannot refresh a transient instance");
}
IncrementRequestCount();
var shardRequestData = new ShardRequestData
{
EntityType = typeof(T),
Keys = { value.Key }
};
var dbCommands = GetCommandsToOperateOn(shardRequestData);
var results = shardStrategy.ShardAccessStrategy.Apply(dbCommands, shardRequestData, (dbCmd, i) =>
{
var jsonDocument = dbCmd.Get(value.Key);
if (jsonDocument == null)
{
return(false);
}
value.Metadata = jsonDocument.Metadata;
value.OriginalMetadata = (RavenJObject)jsonDocument.Metadata.CloneToken();
value.ETag = jsonDocument.Etag;
value.OriginalValue = jsonDocument.DataAsJson;
var newEntity = ConvertToEntity <T>(value.Key, jsonDocument.DataAsJson, jsonDocument.Metadata);
foreach (
var property in
entity.GetType().GetProperties().Where(
property => property.CanWrite && property.CanRead && property.GetIndexParameters().Length == 0))
{
property.SetValue(entity, property.GetValue(newEntity, null), null);
}
return(true);
});
if (results.All(x => x == false))
{
throw new InvalidOperationException("Document '" + value.Key + "' no longer exists and was probably deleted");
}
}
/// <summary>
/// Applies the specified action to all shard sessions in parallel
/// </summary>
public Task <T[]> ApplyAsync <T>(IList <IAsyncDatabaseCommands> commands, ShardRequestData request, Func <IAsyncDatabaseCommands, int, Task <T> > operation)
{
return(Task.Factory.ContinueWhenAll(commands.Select(operation).ToArray(), tasks =>
{
var results = new List <T>(tasks.Length);
int index = 0;
var handledExceptions = new List <Exception>();
var unhandledExceptions = new List <Exception>();
foreach (var task in tasks)
{
try
{
results.Add(task.Result);
}
catch (Exception e)
{
var error = OnAsyncError;
if (error == null)
{
unhandledExceptions.Add(e);
}
else if (error(commands[index], request, e) == false)
{
unhandledExceptions.Add(e);
}
else
{
handledExceptions.Add(e);
}
}
index++;
}
if (unhandledExceptions.Any())
{
throw new AggregateException(unhandledExceptions);
}
if (handledExceptions.Count == tasks.Length)
{
throw new AggregateException(handledExceptions);
}
return results.ToArray();
}));
}
/// <summary>
/// Get the json document by key from the store
/// </summary>
private async Task<JsonDocument> GetJsonDocumentAsync(string documentKey)
{
var shardRequestData = new ShardRequestData
{
EntityType = typeof(object),
Keys = { documentKey }
};
var dbCommands = GetCommandsToOperateOn(shardRequestData);
var documents = await shardStrategy.ShardAccessStrategy.ApplyAsync(dbCommands,
shardRequestData,
(commands, i) => commands.GetAsync(documentKey)).ConfigureAwait(false);
var document = documents.FirstOrDefault(x => x != null);
if (document != null)
return document;
throw new InvalidOperationException("Document '" + documentKey + "' no longer exists and was probably deleted");
}
/// <summary>
/// Applies the specified action to all shard sessions in parallel
/// </summary>
public T[] Apply <T>(IList <IDatabaseCommands> commands, ShardRequestData request, Func <IDatabaseCommands, int, T> operation)
{
var returnedLists = new T[commands.Count];
var valueSet = new bool[commands.Count];
var errors = new Exception[commands.Count];
commands
.Select((cmd, i) =>
Task.Factory.StartNew(() => operation(cmd, i))
.ContinueWith(task =>
{
try
{
returnedLists[i] = task.Result;
valueSet[i] = true;
}
catch (Exception e)
{
var error = OnError;
if (error == null)
{
throw;
}
if (error(commands[i], request, e) == false)
{
throw;
}
errors[i] = e;
}
})
)
.WaitAll();
// if ALL nodes failed, we still throw
if (errors.All(x => x != null))
{
throw new AggregateException(errors);
}
return(returnedLists.Where((t, i) => valueSet[i]).ToArray());
}
protected override JsonDocument GetJsonDocument(string documentKey)
{
var shardRequestData = new ShardRequestData
{
EntityType = typeof(object), Keys = { documentKey }
};
var dbCommands = GetCommandsToOperateOn(shardRequestData);
var documents = shardStrategy.ShardAccessStrategy.Apply(dbCommands,
shardRequestData,
(commands, i) => commands.Get(documentKey));
var document = documents.FirstOrDefault(x => x != null);
if (document != null)
{
return(document);
}
throw new InvalidOperationException("Document '" + documentKey + "' no longer exists and was probably deleted");
}
/// <summary>
/// Applies the specified action for all shard sessions.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="commands">The shard sessions.</param>
/// <param name="operation">The operation.</param>
/// <returns></returns>
public T[] Apply <T>(IList <IDatabaseCommands> commands, ShardRequestData request, Func <IDatabaseCommands, int, T> operation)
{
var list = new List <T>();
var errors = new List <Exception>();
for (int i = 0; i < commands.Count; i++)
{
try
{
list.Add(operation(commands[i], i));
}
catch (Exception e)
{
var error = OnError;
if (error == null)
{
throw;
}
if (error(commands[i], request, e) == false)
{
throw;
}
errors.Add(e);
}
}
// if ALL nodes failed, we still throw
if (errors.Count == commands.Count)
#if !NET35
{ throw new AggregateException(errors); }
#else
{ throw new InvalidOperationException("Got an error from all servers", errors.First())
{
Data = { { "Errors", errors } }
} };
#endif
return(list.ToArray());
}
/// <summary>
/// Selects the shard ids appropriate for the specified data.
/// </summary>
/// <returns>
/// Return a list of shards ids that will be search. Returning null means search all shards.
/// </returns>
public IList<string> PotentialShardsFor(ShardRequestData requestData)
{
if (requestData.Query != null)
{
var shardIdRxpr =
new Regex(
string.Format(
"\r\n{0}: \\s* (?<Open>\")(?<shardId>[^\"]+)(?<Close-Open>\") |\r\n{0}: \\s* (?<shardId>[^\"][^\\s]*)",
Regex.Escape("SegmentId")), RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
Match match = shardIdRxpr.Match(requestData.Query.Query);
if (match.Success)
{
string shardId = match.Groups["shardId"].ToString();
var res = new List<string>();
if (_shards.Keys.Any(k => k.StartsWith(shardId)))
{
res.Add(shardId);
}
if (res.Count > 0)
return res;
}
}
return null;
}
private IList <IDatabaseCommands> GetCommandsToOperateOn(ShardRequestData resultionData)
{
return(GetShardsToOperateOn(resultionData).Select(x => x.Item2).ToList());
}
public IList<string> PotentialShardsFor(ShardRequestData requestData)
{
if (requestData.EntityType == typeof (Company))
{
// You can try to limit the potential shards based on the query
}
return null;
}
/// <summary>
/// Selects the shard ids appropriate for the specified data.
/// </summary>
/// <returns>Return a list of shards ids that will be search. Returning null means search all shards.</returns>
public virtual IList <string> PotentialShardsFor(ShardRequestData requestData, List <string> potentialShardIds)
{
return(potentialShardIds);
}