public void TestShardCommandRetryExhaustion() { var retryPolicy = new RetryPolicy(2, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100)); var shardConnections = new List <Tuple <ShardLocation, DbConnection> >(); // Create ten mocked connections, half of them will throw exceptions on Open for (int i = 0; i < 10; i++) { string database = string.Format("Shard{0}", i); int j = i; Action executeOnOpen = () => { if (j < 5) { throw new TimeoutException(); } }; var mockCon = new MockSqlConnection(database, executeOnOpen); shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon)); } var mockCmd = new MockSqlCommand(); mockCmd.ExecuteReaderFunc = (t, c) => new MockSqlDataReader(); mockCmd.CommandText = "select 1"; using (var conn = new MultiShardConnection(shardConnections)) { using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300)) { cmd.ExecutionOptions = MultiShardExecutionOptions.None; cmd.RetryPolicy = retryPolicy; cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults; MultiShardDataReader rdr = cmd.ExecuteReader(CommandBehavior.Default); // Validate the right exception is re-thrown Assert.IsTrue(rdr.MultiShardExceptions.Count == 5, "Expected MultiShardExceptions!"); foreach (MultiShardException ex in rdr.MultiShardExceptions) { Assert.IsTrue(ex.InnerException is TimeoutException, "Expected TimeoutException!"); } // Validate that the connections for the faulted readers are closed for (int i = 0; i < 5; i++) { Assert.IsTrue(shardConnections[i].Item2.State == ConnectionState.Closed, "Expected Connection to be Closed!"); } } } }
private List <Tuple <ShardLocation, DbConnection> > CreateConnections(int count, Action executeOnOpen) { var shardConnections = new List <Tuple <ShardLocation, DbConnection> >(); for (int i = 0; i < count; i++) { string database = string.Format("Shard{0}", i); var mockCon = new MockSqlConnection(database, executeOnOpen); shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon)); } return(shardConnections); }
public void TestShardCommandRetryBasic() { var retryPolicy = new RetryPolicy(4, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100)); var openRetryCounts = new int[10]; var shardConnections = new List <Tuple <ShardLocation, DbConnection> >(); // Create ten mocked connections, each will retry retryPolicy.RetryCount - 1 times, // and keep it's own actual retry count in one of the elements of openRetryCounts for (int i = 0; i < 10; i++) { string database = string.Format("Shard{0}", i); // We want to close on the value of i int j = i; Action executeOnOpen = () => { if (openRetryCounts[j] < (retryPolicy.RetryCount - 1)) { Logger.Log("Current retry count for database: {0} is {1}", database, openRetryCounts[j]); openRetryCounts[j]++; throw new TimeoutException(); } }; var mockCon = new MockSqlConnection(database, executeOnOpen); shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("test", database), mockCon)); } var mockCmd = new MockSqlCommand(); mockCmd.ExecuteReaderFunc = (t, c) => new MockSqlDataReader(); mockCmd.CommandText = "select 1"; using (var conn = new MultiShardConnection(shardConnections)) { using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300)) { cmd.ExecutionOptions = MultiShardExecutionOptions.None; cmd.RetryPolicy = retryPolicy; cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults; cmd.ExecuteReader(CommandBehavior.Default); } } for (int i = 0; i < openRetryCounts.Length; i++) { Assert.AreEqual(retryPolicy.RetryCount - 1, openRetryCounts[i]); } }
public void TestShardCommandRetryConnectionReopen() { var retryPolicy = new RetryPolicy(4, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(100)); var shardConnections = new List <Tuple <ShardLocation, DbConnection> >(); // Callback to execute when the MockCommand is invoked Func <CancellationToken, MockSqlCommand, DbDataReader> ExecuteReaderFunc = null; // Number of times each command has been retried var commandRetryCounts = new int[10]; // Create ten mocked connections, // a few of them will throw an exception on Open // and the rest will throw an exception on command execution upto 2 retries // At the end, all commands should complete successfully. for (int i = 0; i < 10; i++) { string database = string.Format("{0}", i); int j = i; int retryCount = 0; Action executeOnOpen = () => { if (j < 5) { if (retryCount < 3) { retryCount++; throw new TimeoutException(); } } }; var mockCon = new MockSqlConnection(database, executeOnOpen); shardConnections.Add(new Tuple <ShardLocation, DbConnection>(new ShardLocation("Shard", database), mockCon)); } ExecuteReaderFunc = (t, r) => { int index = Int32.Parse(((MockSqlConnection)(r.Connection)).ConnectionString); if (r.Connection.State == ConnectionState.Closed) { throw new InvalidOperationException("Command shouldn't be executed on a closed connection!"); } if (index > 5 && commandRetryCounts[index] < 3) { commandRetryCounts[index]++; r.RetryCount++; r.Connection.Close(); throw new TimeoutException(); } else { var mockRdr = new MockSqlDataReader(); mockRdr.ExecuteOnReadAsync = (rdr) => { return(Task.Run <bool>(() => { bool isClosed = rdr.IsClosed; rdr.Close(); return !isClosed; })); }; return(mockRdr); } }; var mockCmd = new MockSqlCommand(); mockCmd.ExecuteReaderFunc = ExecuteReaderFunc; mockCmd.CommandText = "select 1"; using (var conn = new MultiShardConnection(shardConnections)) { using (var cmd = MultiShardCommand.Create(conn, mockCmd, 300)) { cmd.RetryPolicy = retryPolicy; cmd.ExecutionPolicy = MultiShardExecutionPolicy.PartialResults; using (var reader = cmd.ExecuteReaderAsync().Result) { // Validate that we successfully received a reader // from each one of the shards int readerCount = 0; while (reader.Read()) { readerCount++; } Assert.AreEqual(10, readerCount, "Expected 10 readers!"); } } } }