public void TestShardConnectionOpenException()
{
try
{
Action executeOnOpen = () =>
{
throw new InsufficientMemoryException();
};
var shardConnections = CreateConnections(10, executeOnOpen);
var mockCmd = new MockSqlCommand();
mockCmd.CommandText = "Select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 100))
{
cmd.CommandText = "select 1";
cmd.ExecuteReader();
}
}
}
catch (Exception ex)
{
if (ex is MultiShardAggregateException)
{
var maex = (MultiShardAggregateException)ex;
Logger.Log("Exception message: {0}.\n Exception tostring: {1}",
ex.Message, ex.ToString());
throw (maex.InnerException).InnerException;
}
throw;
}
}
public void TestShardCommandSucceedHandler()
{
var shardConnections = CreateConnections(10, () => { });
ConcurrentDictionary <ShardLocation, bool> passedLocations = new ConcurrentDictionary <ShardLocation, bool>();
using (var conn = new MultiShardConnection(shardConnections))
{
Func <CancellationToken, MockSqlCommand, DbDataReader> executeReaderFunc = (token, cmd) =>
{
Thread.Sleep(TimeSpan.FromSeconds(2));
return(new MockSqlDataReader());
};
MockSqlCommand mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = executeReaderFunc;
mockCmd.CommandText = "Select 1";
using (MultiShardCommand cmd = MultiShardCommand.Create(conn, mockCmd, 10))
{
cmd.ShardExecutionSucceeded += new EventHandler <ShardExecutionEventArgs>((obj, eventArgs) =>
{
Assert.IsTrue(shardConnections.Select(x => x.Item1).Contains(eventArgs.ShardLocation), "The ShardLocation passed to the event handler does not exist in the set of passed in ShardLocations");
passedLocations[eventArgs.ShardLocation] = true;
});
CommandBehavior behavior = CommandBehavior.Default;
cmd.ExecuteReader(behavior);
}
}
Assert.AreEqual(shardConnections.Count, passedLocations.Count, "Not every ShardLocation had its corresponding event handler invoked.");
}
public void TestShardCommandCancellationHandler()
{
// Create connections to a few shards
var shardConnections = CreateConnections(10, () => { });
var mockCmd = new MockSqlCommand();
var cmdStartEvent = new ManualResetEvent(false);
mockCmd.ExecuteReaderFunc = (token, cmd) =>
{
while (true)
{
if (token == null)
{
break;
}
token.ThrowIfCancellationRequested();
Thread.Sleep(500);
cmdStartEvent.Set();
}
return(new MockSqlDataReader());
};
mockCmd.CommandText = "select 1";
ConcurrentDictionary <ShardLocation, bool> passedLocations = new ConcurrentDictionary <ShardLocation, bool>();
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.ShardExecutionCanceled += new EventHandler <ShardExecutionEventArgs>((obj, eventArgs) =>
{
Assert.IsTrue(shardConnections.Select(x => x.Item1).Contains(eventArgs.ShardLocation),
"The ShardLocation passed to the event handler does not exist in the set of passed in ShardLocations");
passedLocations[eventArgs.ShardLocation] = true;
});
try
{
// start the Cancel on a separate thread
Task executeTask = Task.Run(() =>
{
cmdStartEvent.WaitOne();
cmd.Cancel();
});
cmd.ExecuteReader();
executeTask.Wait();
Assert.Fail("We should always be throwing an exception.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is OperationCanceledException, "OperationCanceledException expected. Found {0}!", ex.ToString());
}
}
}
Assert.AreEqual(shardConnections.Count, passedLocations.Count, "Not every ShardLocation had its corresponding event handler invoked.");
}
public MockSqlCommand Clone()
{
var clone = new MockSqlCommand(CommandTimeout);
clone.CommandText = this.CommandText;
clone.CommandType = this.CommandType;
clone.Connection = this.Connection;
clone.ExecuteReaderFunc = this.ExecuteReaderFunc;
return(clone);
}
public void TestShardCommandRetryExhaustion()
{
var retryPolicy = new RetryPolicy(2, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100));
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
// Create ten mocked connections, half of them will throw exceptions on Open
for (int i = 0; i < 10; i++)
{
string database = string.Format("Shard{0}", i);
int j = i;
Action executeOnOpen = () =>
{
if (j < 5)
{
throw new TimeoutException();
}
};
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon));
}
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = (t, c) => new MockSqlDataReader();
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.ExecutionOptions = MultiShardExecutionOptions.None;
cmd.RetryPolicy = retryPolicy;
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
MultiShardDataReader rdr = cmd.ExecuteReader(CommandBehavior.Default);
// Validate the right exception is re-thrown
Assert.IsTrue(rdr.MultiShardExceptions.Count == 5, "Expected MultiShardExceptions!");
foreach (MultiShardException ex in rdr.MultiShardExceptions)
{
Assert.IsTrue(ex.InnerException is TimeoutException, "Expected TimeoutException!");
}
// Validate that the connections for the faulted readers are closed
for (int i = 0; i < 5; i++)
{
Assert.IsTrue(shardConnections[i].Item2.State == ConnectionState.Closed,
"Expected Connection to be Closed!");
}
}
}
}
public void TestShardCommandRetryBasic()
{
var retryPolicy = new RetryPolicy(4, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100));
var openRetryCounts = new int[10];
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
// Create ten mocked connections, each will retry retryPolicy.RetryCount - 1 times,
// and keep it's own actual retry count in one of the elements of openRetryCounts
for (int i = 0; i < 10; i++)
{
string database = string.Format("Shard{0}", i);
// We want to close on the value of i
int j = i;
Action executeOnOpen = () =>
{
if (openRetryCounts[j] < (retryPolicy.RetryCount - 1))
{
Logger.Log("Current retry count for database: {0} is {1}", database, openRetryCounts[j]);
openRetryCounts[j]++;
throw new TimeoutException();
}
};
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon));
}
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = (t, c) => new MockSqlDataReader();
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.ExecutionOptions = MultiShardExecutionOptions.None;
cmd.RetryPolicy = retryPolicy;
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
cmd.ExecuteReader(CommandBehavior.Default);
}
}
for (int i = 0; i < openRetryCounts.Length; i++)
{
Assert.AreEqual(retryPolicy.RetryCount - 1, openRetryCounts[i]);
}
}
public void TestShardCommandCancellation()
{
// Create connections to a few shards
var shardConnections = CreateConnections(10, () => { });
var mockCmd = new MockSqlCommand();
var cmdStartEvent = new ManualResetEvent(false);
mockCmd.ExecuteReaderFunc = (token, cmd) =>
{
while (true)
{
if (token == null)
{
break;
}
token.ThrowIfCancellationRequested();
Thread.Sleep(500);
cmdStartEvent.Set();
}
return(new MockSqlDataReader());
};
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
try
{
// start the Cancel on a separate thread
Task executeTask = Task.Run(() =>
{
cmdStartEvent.WaitOne();
cmd.Cancel();
});
cmd.ExecuteReader();
executeTask.Wait();
Assert.Fail("We should always be throwing an exception.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is OperationCanceledException, "OperationCanceledException expected. Found {0}!", ex.ToString());
}
}
}
}
public void TestShardCommandTimeoutException()
{
var shardConnections = CreateConnections(10, () => { });
Func <CancellationToken, MockSqlCommand, DbDataReader> executeReaderFunc = (token, cmd) =>
{
Thread.Sleep(TimeSpan.FromSeconds(2));
return(new MockSqlDataReader());
};
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = executeReaderFunc;
mockCmd.CommandText = "Select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 1))
{
cmd.ExecuteReader();
}
}
}
public void TestShardCommandFaultHandler()
{
var shardConnections = CreateConnections(10, () => { });
Func <CancellationToken, MockSqlCommand, DbDataReader> executeReaderFunc = (token, cmd) =>
{
throw new InsufficientMemoryException();
};
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = executeReaderFunc;
mockCmd.CommandText = "Select 1";
ConcurrentDictionary <ShardLocation, bool> passedLocations = new ConcurrentDictionary <ShardLocation, bool>();
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 1))
{
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
cmd.CommandTimeout = 300;
cmd.ShardExecutionFaulted += new EventHandler <ShardExecutionEventArgs>((obj, eventArgs) =>
{
Assert.IsTrue(shardConnections.Select(x => x.Item1).Contains(eventArgs.ShardLocation), "The ShardLocation passed to the event handler does not exist in the set of passed in ShardLocations");
passedLocations[eventArgs.ShardLocation] = true;
Assert.IsInstanceOfType(eventArgs.Exception, typeof(InsufficientMemoryException), "An incorrect exception type was passed to the event handler.");
});
try
{
// We want to execute to completion so we get to the validation at the end of the function.
cmd.ExecuteReader();
}
catch { }
}
}
Assert.AreEqual(shardConnections.Count, passedLocations.Count, "Not every ShardLocation had its corresponding event handler invoked.");
}
public void TestShardCommandRetryConnectionReopen()
{
var retryPolicy = new RetryPolicy(4, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100));
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
// Callback to execute when the MockCommand is invoked
Func <CancellationToken, MockSqlCommand, DbDataReader> ExecuteReaderFunc = null;
// Number of times each command has been retried
var commandRetryCounts = new int[10];
// Create ten mocked connections,
// a few of them will throw an exception on Open
// and the rest will throw an exception on command execution upto 2 retries
// At the end, all commands should complete successfully.
for (int i = 0; i < 10; i++)
{
string database = string.Format("{0}", i);
int j = i;
int retryCount = 0;
Action executeOnOpen = () =>
{
if (j < 5)
{
if (retryCount < 3)
{
retryCount++;
throw new TimeoutException();
}
}
};
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("Shard", database), mockCon));
}
ExecuteReaderFunc = (t, r) =>
{
int index = Int32.Parse(((MockSqlConnection)(r.Connection)).ConnectionString);
if (r.Connection.State == ConnectionState.Closed)
{
throw new InvalidOperationException("Command shouldn't be executed on a closed connection!");
}
if (index > 5 && commandRetryCounts[index] < 3)
{
commandRetryCounts[index]++;
r.RetryCount++;
r.Connection.Close();
throw new TimeoutException();
}
else
{
var mockRdr = new MockSqlDataReader();
mockRdr.ExecuteOnReadAsync = (rdr) =>
{
return(Task.Run <bool>(() =>
{
bool isClosed = rdr.IsClosed;
rdr.Close();
return !isClosed;
}));
};
return(mockRdr);
}
};
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = ExecuteReaderFunc;
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.RetryPolicy = retryPolicy;
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
using (var reader = cmd.ExecuteReaderAsync().Result)
{
// Validate that we successfully received a reader
// from each one of the shards
int readerCount = 0;
while (reader.Read())
{
readerCount++;
}
Assert.AreEqual(10, readerCount, "Expected 10 readers!");
}
}
}
}
public MockSqlCommand Clone()
{
var clone = new MockSqlCommand(CommandTimeout);
clone.CommandText = this.CommandText;
clone.CommandType = this.CommandType;
clone.Connection = this.Connection;
clone.ExecuteReaderFunc = this.ExecuteReaderFunc;
return clone;
}