public void TestShardCommandRetryExhaustion()
{
var retryPolicy = new RetryPolicy(2, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100));
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
// Create ten mocked connections, half of them will throw exceptions on Open
for (int i = 0; i < 10; i++)
{
string database = string.Format("Shard{0}", i);
int j = i;
Action executeOnOpen = () =>
{
if (j < 5)
{
throw new TimeoutException();
}
};
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon));
}
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = (t, c) => new MockSqlDataReader();
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.ExecutionOptions = MultiShardExecutionOptions.None;
cmd.RetryPolicy = retryPolicy;
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
MultiShardDataReader rdr = cmd.ExecuteReader(CommandBehavior.Default);
// Validate the right exception is re-thrown
Assert.IsTrue(rdr.MultiShardExceptions.Count == 5, "Expected MultiShardExceptions!");
foreach (MultiShardException ex in rdr.MultiShardExceptions)
{
Assert.IsTrue(ex.InnerException is TimeoutException, "Expected TimeoutException!");
}
// Validate that the connections for the faulted readers are closed
for (int i = 0; i < 5; i++)
{
Assert.IsTrue(shardConnections[i].Item2.State == ConnectionState.Closed,
"Expected Connection to be Closed!");
}
}
}
}
private List <Tuple <ShardLocation, DbConnection> > CreateConnections(int count, Action executeOnOpen)
{
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
for (int i = 0; i < count; i++)
{
string database = string.Format("Shard{0}", i);
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon));
}
return(shardConnections);
}
public void TestShardCommandRetryBasic()
{
var retryPolicy = new RetryPolicy(4, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100));
var openRetryCounts = new int[10];
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
// Create ten mocked connections, each will retry retryPolicy.RetryCount - 1 times,
// and keep it's own actual retry count in one of the elements of openRetryCounts
for (int i = 0; i < 10; i++)
{
string database = string.Format("Shard{0}", i);
// We want to close on the value of i
int j = i;
Action executeOnOpen = () =>
{
if (openRetryCounts[j] < (retryPolicy.RetryCount - 1))
{
Logger.Log("Current retry count for database: {0} is {1}", database, openRetryCounts[j]);
openRetryCounts[j]++;
throw new TimeoutException();
}
};
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon));
}
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = (t, c) => new MockSqlDataReader();
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.ExecutionOptions = MultiShardExecutionOptions.None;
cmd.RetryPolicy = retryPolicy;
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
cmd.ExecuteReader(CommandBehavior.Default);
}
}
for (int i = 0; i < openRetryCounts.Length; i++)
{
Assert.AreEqual(retryPolicy.RetryCount - 1, openRetryCounts[i]);
}
}
public void TestShardCommandRetryConnectionReopen()
{
var retryPolicy = new RetryPolicy(4, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100));
var shardConnections = new List <Tuple <ShardLocation, DbConnection> >();
// Callback to execute when the MockCommand is invoked
Func <CancellationToken, MockSqlCommand, DbDataReader> ExecuteReaderFunc = null;
// Number of times each command has been retried
var commandRetryCounts = new int[10];
// Create ten mocked connections,
// a few of them will throw an exception on Open
// and the rest will throw an exception on command execution upto 2 retries
// At the end, all commands should complete successfully.
for (int i = 0; i < 10; i++)
{
string database = string.Format("{0}", i);
int j = i;
int retryCount = 0;
Action executeOnOpen = () =>
{
if (j < 5)
{
if (retryCount < 3)
{
retryCount++;
throw new TimeoutException();
}
}
};
var mockCon = new MockSqlConnection(database, executeOnOpen);
shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("Shard", database), mockCon));
}
ExecuteReaderFunc = (t, r) =>
{
int index = Int32.Parse(((MockSqlConnection)(r.Connection)).ConnectionString);
if (r.Connection.State == ConnectionState.Closed)
{
throw new InvalidOperationException("Command shouldn't be executed on a closed connection!");
}
if (index > 5 && commandRetryCounts[index] < 3)
{
commandRetryCounts[index]++;
r.RetryCount++;
r.Connection.Close();
throw new TimeoutException();
}
else
{
var mockRdr = new MockSqlDataReader();
mockRdr.ExecuteOnReadAsync = (rdr) =>
{
return(Task.Run <bool>(() =>
{
bool isClosed = rdr.IsClosed;
rdr.Close();
return !isClosed;
}));
};
return(mockRdr);
}
};
var mockCmd = new MockSqlCommand();
mockCmd.ExecuteReaderFunc = ExecuteReaderFunc;
mockCmd.CommandText = "select 1";
using (var conn = new MultiShardConnection(shardConnections))
{
using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300))
{
cmd.RetryPolicy = retryPolicy;
cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults;
using (var reader = cmd.ExecuteReaderAsync().Result)
{
// Validate that we successfully received a reader
// from each one of the shards
int readerCount = 0;
while (reader.Read())
{
readerCount++;
}
Assert.AreEqual(10, readerCount, "Expected 10 readers!");
}
}
}
}
