/// <summary>
/// Gets or creates connection to the cluster.
/// </summary>
/// <param name="partitionKey"> </param>
/// <returns> </returns>
public Connection GetOrCreateConnection(PartitionKey partitionKey)
{
//Sort the nodes by load (used first)
Node leastUsedNode =
_nodes.Where(n => n.IsUp).SmallestOrDefault(n => n.ConnectionCount > 0 ? n.Load : _config.NewConnectionTreshold - 1);
//try get a connection from it
Connection connection = leastUsedNode.GetConnection();
//smallest connection from smallest node
if (connection != null && connection.Load < _config.NewConnectionTreshold)
return connection;
if (_config.MaxConnections <= 0 || _connectionCount < _config.MaxConnections)
{
//try to get a new connection from this smallest node
Connection newConnection = leastUsedNode.CreateConnection();
if (newConnection != null)
{
Interlocked.Increment(ref _connectionCount);
newConnection.OnConnectionChange +=
(c, ev) => { if (ev.Connected == false) Interlocked.Decrement(ref _connectionCount); };
return newConnection;
}
}
return connection;
}
public Connection GetOrCreateConnection(PartitionKey partitionKey)
{
Connection connection = null;
//try pick an unused connection
while (_connections.TryPop(out connection))
{
if (connection.IsConnected)
return connection;
}
//check if we may create another connection
if (_config.MaxConnections <= 0 || _nodes.Sum(n => n.ConnectionCount) < _config.MaxConnections)
{
//all connections in use, or non available, go and create one at random node
int count = _nodes.Count;
int offset = _rndGen.Next(count);
for (int i = 0; i < count; i++)
{
connection = _nodes[(offset + i) % count].CreateConnection();
if (connection != null)
return connection;
}
}
//yikes nothing available
return null;
}
/// <summary>
/// Gets or creates connection to the cluster.
/// </summary>
/// <param name="partitionKey"> </param>
/// <returns> </returns>
/// <exception cref="CqlException">Can not connect to any node of the cluster! All connectivity to the cluster seems to be lost</exception>
public Connection GetOrCreateConnection(PartitionKey partitionKey)
{
Connection connection;
int count = _nodes.Count;
int offset = _rnd.Next(count);
//try to get an unused connection from a random node
for (int i = 0; i < count; i++)
{
try
{
connection = _nodes[(offset + i) % count].GetConnection();
if (connection != null && connection.Load < _config.NewConnectionTreshold)
return connection;
}
catch
{
//ignore, try another node
}
}
//check if we may create another connection
if (_config.MaxConnections <= 0 || _nodes.Sum(n => n.ConnectionCount) < _config.MaxConnections)
{
//iterate over nodes and try to create a new one
for (int i = 0; i < count; i++)
{
try
{
connection = _nodes[(offset + i) % count].CreateConnection();
if (connection != null)
return connection;
}
catch
{
//ignore, try another node
}
}
}
//iterate over nodes and get an existing one
for (int i = 0; i < count; i++)
{
connection = _nodes[(offset + i) % count].GetConnection();
if (connection != null)
return connection;
}
return null;
}
/// <summary>
/// Gets or creates connection to the cluster.
/// </summary>
/// <param name="scope"> The scope. </param>
/// <param name="partitionKey"> The partition key. </param>
/// <returns> </returns>
public Connection GetOrCreateConnection(ConnectionScope scope, PartitionKey partitionKey)
{
//provide connections on command level only
if(scope == ConnectionScope.Connection)
return null;
int count = _nodes.Count;
int offset = _rnd.Next(count);
Connection connection = null;
//try to get an unused connection from a random node
for(int i = 0; i < count; i++)
{
Node randomNode = _nodes[(offset + i)%count];
//skip if node is down
if(!randomNode.IsUp) continue;
//try get a connection from it
connection = randomNode.GetConnection();
//connection is not used to the max, ok to use
if(connection != null && connection.Load < _config.NewConnectionTreshold)
break;
//get a new connection to the node if possible
if(_config.MaxConnections <= 0 || _connectionCount < _config.MaxConnections)
{
//try to get a new connection from this random node
Connection newConnection = randomNode.CreateConnection();
if(newConnection != null)
{
Interlocked.Increment(ref _connectionCount);
newConnection.OnConnectionChange +=
(c, ev) => { if(ev.Connected == false) Interlocked.Decrement(ref _connectionCount); };
connection = newConnection;
break;
}
}
}
return connection;
}
/// <summary>
/// Gets or creates connection to the cluster.
/// </summary>
/// <param name="partitionKey"> </param>
/// <returns> </returns>
/// <exception cref="CqlException">Can not connect to any node of the cluster! All connectivity to the cluster seems to be lost</exception>
public Connection GetOrCreateConnection(PartitionKey partitionKey)
{
//try based on partition first
if (partitionKey != null && partitionKey.IsSet)
{
var nodes = _nodes.GetResponsibleNodes(partitionKey).Where(n => n.IsUp).OrderBy(n => n.Load);
foreach (Node node in nodes)
{
Connection connection = node.GetOrCreateConnection(partitionKey);
if (connection != null)
return connection;
}
}
return _baseStrategy.GetOrCreateConnection(partitionKey);
}
public void CompositeTokenValue1()
{
/*
id1 | id2 | token_id1__id2
-----+-----+----------------------
10 | 20 | -9026262514124674157
*/
var key = new PartitionKey();
key.Set(new[] {CqlType.Int, CqlType.Text}, new object[] {10, "20"});
var calculatedToken = new MurmurToken();
calculatedToken.Parse(key.GetValue());
var tokenFromCassandra = new MurmurToken();
tokenFromCassandra.Parse("-9026262514124674157");
Assert.AreEqual(tokenFromCassandra, calculatedToken);
}
public void CompositeTokenValue2()
{
/*
id1 | id2 | token_id1__id2
-----+-----+----------------------
1 | 2 | 4093852821103061060
*/
var key = new PartitionKey();
key.Set(new[] { CqlType.Int, CqlType.Text }, new object[] { 1, "2" });
var calculatedToken = new MurmurToken();
calculatedToken.Parse(key.GetValue());
var tokenFromCassandra = new MurmurToken();
tokenFromCassandra.Parse("4093852821103061060");
Assert.AreEqual(tokenFromCassandra, calculatedToken);
}
public void SingleTokenValue()
{
/*
id1 | token_id1
-----+----------------------
1 | -4069959284402364209
*/
var key = new PartitionKey();
key.Set(CqlType.Int, 1);
var calculatedToken = new MurmurToken();
calculatedToken.Parse(key.GetValue());
var tokenFromCassandra = new MurmurToken();
tokenFromCassandra.Parse("-4069959284402364209");
Assert.AreEqual(tokenFromCassandra, calculatedToken);
}
public void CompositeTokenValueFromClass()
{
/*
id1 | id2 | token_id1__id2
-----+-----+----------------------
10 | 20 | -9026262514124674157
*/
var key = new PartitionKey();
key.Set(new CompositeKeyType { Id1 = 10, Id2 = "20", Val="test"});
var calculatedToken = new MurmurToken();
calculatedToken.Parse(key.GetValue());
var tokenFromCassandra = new MurmurToken();
tokenFromCassandra.Parse("-9026262514124674157");
Assert.AreEqual(tokenFromCassandra, calculatedToken);
}
/// <summary>
/// Gets or creates connection to the cluster.
/// </summary>
/// <param name="scope"> The scope. </param>
/// <param name="partitionKey"> The partition key. </param>
/// <returns> </returns>
/// <exception cref="CqlException">
/// Can not connect to any node of the cluster! All connectivity to the cluster seems to be
/// lost
/// </exception>
public Connection GetOrCreateConnection(ConnectionScope scope, PartitionKey partitionKey)
{
//provide connections on command level only
if(scope == ConnectionScope.Connection)
return null;
//try based on partition first
if(partitionKey != null && partitionKey.IsSet)
{
var nodes = _nodes.GetResponsibleNodes(partitionKey).Where(n => n.IsUp).OrderBy(n => n.Load);
foreach(Node node in nodes)
{
Connection connection = node.GetOrCreateConnection();
if(connection != null)
return connection;
}
}
return _baseStrategy.GetOrCreateConnection(scope, partitionKey);
}
public BigInteger? ComputeToken(PartitionKey partitionKey)
{
object[] keys = partitionKey.Keys;
if (1 == keys.Length)
{
ColumnType colType = keys[0].GetType().ToColumnType();
byte[] buffer = ValueSerialization.Serialize(colType, keys[0]);
return Hash(buffer, 0, buffer.Length);
}
var rawValues = new byte[keys.Length][];
for (int i = 0; i < keys.Length; i++)
{
ColumnType colType = keys[i].GetType().ToColumnType();
rawValues[i] = ValueSerialization.Serialize(colType, keys[i]);
}
int length = keys.Length * 3 + rawValues.Sum(val => val.Length);
using (var stream = new MemoryStream(length))
{
foreach (var rawValue in rawValues)
{
//write length of composite key part as short
var len = (short) rawValue.Length;
stream.WriteByte((byte) (len >> 8));
stream.WriteByte((byte) (len));
//write value
stream.Write(rawValue, 0, len);
//write terminator byte
stream.WriteByte(0);
}
byte[] buffer = stream.GetBuffer();
return Hash(buffer, 0, length);
}
}
internal static int ExecuteUpsertDocuments <T>(this Container container, Data <T> data, PartitionKey partitionKey, CancellationToken cancellationToken = default)
{
int affected = 0;
Data <T> records = new Data <T>(data.Items);
do
{
records.Items = data.Items.Skip(affected).ToList();
Task <StoredProcedureExecuteResponse <int> > task = container.Scripts.ExecuteStoredProcedureAsync <int>("upsertDocuments", partitionKey, new dynamic[] { records }, cancellationToken: cancellationToken);
task.Wait();
affected += task.Result.Resource;
} while (affected < data.Items.Count);
return(affected);
}
/// <summary>
/// Gets the underlying connection. Will reopen this CqlConnection, if the underlying connection has failed,
/// </summary>
/// <returns> An open connection </returns>
internal Connection GetConnection(PartitionKey partitionKey = default(PartitionKey))
{
if (_state == 0)
throw new CqlException("You must invoke Open or OpenAsync on a CqlConnection before other use.");
if (_state == 2)
throw new ObjectDisposedException("CqlConnection");
//reuse the connection if the connection strategy is exclusive
Connection connection = _cluster.Config.ConnectionStrategy == ConnectionStrategy.Exclusive
? _connection
: _cluster.GetOrCreateConnection(partitionKey);
if (connection == null)
throw new CqlException("Unable to obtain a Cql network connection.");
return connection;
}
public override TransactionalBatch CreateTransactionalBatch(PartitionKey partitionKey) => throw new NotImplementedException();
public override Task <ResponseMessage> ReplaceItemStreamAsync(Stream streamPayload, string id, PartitionKey partitionKey, ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default)
{
var returnValue = new ResponseMessage();
return(Task.FromResult(returnValue));
}
public bool TryExtract(IReadOnlyDictionary <string, string> headers, out PartitionKey?partitionKey)
{
partitionKey = new PartitionKey(scenarioContext.TestRunId.ToString());
return(true);
}
public DocumentClientReadWriteTestProvider()
{
_document = new HealthCheckDocument();
_partitionKey = new PartitionKey(_document.PartitionKey);
}
public async Task ContainerPartitionResourcePermissionTest(ConnectionMode connectionMode)
{
CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
{
ConnectionMode = connectionMode
};
CosmosClient cosmosClient = TestCommon.CreateCosmosClient(cosmosClientOptions);
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync("PermissionTest");
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await database.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await database.CreateContainerAsync(
id : containerId,
partitionKeyPath : "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
// Create items to read
ToDoActivity itemAccess = ToDoActivity.CreateRandomToDoActivity();
ToDoActivity itemNoAccess = ToDoActivity.CreateRandomToDoActivity();
await container.CreateItemAsync <ToDoActivity>(
itemAccess,
new PartitionKey(itemAccess.id));
await container.CreateItemAsync <ToDoActivity>(
itemNoAccess,
new PartitionKey(itemNoAccess.id));
//create permission
string permissionId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemAccess.id);
PermissionProperties permissionProperties = new PermissionProperties(
permissionId,
PermissionMode.Read,
container,
partitionKey);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: cosmosClientOptions, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(database.Id, containerId);
await tokenContainer.ReadItemAsync <ToDoActivity>(itemAccess.id, new PartitionKey(itemAccess.id));
try
{
await tokenContainer.ReadItemAsync <ToDoActivity>(itemNoAccess.id, new PartitionKey(itemNoAccess.id));
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions()
{
PartitionKey = new PartitionKey(itemAccess.id)
};
FeedIterator <ToDoActivity> feedIterator = tokenContainer.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> result = new List <ToDoActivity>();
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIterator.ReadNextAsync();
result.AddRange(toDoActivities);
}
Assert.AreEqual(1, result.Count);
// Test query with no service interop via gateway query plan to replicate x32 app
ContainerInternal containerCore = (ContainerInlineCore)tokenContainer;
MockCosmosQueryClient mock = new MockCosmosQueryClient(
clientContext: containerCore.ClientContext,
cosmosContainerCore: containerCore,
forceQueryPlanGatewayElseServiceInterop: true);
Container tokenGatewayQueryPlan = new ContainerInlineCore(
containerCore.ClientContext,
(DatabaseInternal)containerCore.Database,
containerCore.Id,
mock);
FeedIterator <ToDoActivity> feedIteratorGateway = tokenGatewayQueryPlan.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> resultGateway = new List <ToDoActivity>();
while (feedIteratorGateway.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIteratorGateway.ReadNextAsync();
resultGateway.AddRange(toDoActivities);
}
Assert.AreEqual(1, resultGateway.Count);
}
}
public UpsertItemStreamOperation(Stream streamPayload, TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(streamPayload, options, partitionKey)
{
}
public ReadItemOperation(string id, TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(options, partitionKey)
{
this.id = id;
}
public BigInteger? ComputeToken(PartitionKey partitionKey)
{
return null;
}
/// <summary>
/// Hash code
/// </summary>
public override int GetHashCode()
{
return(PartitionKey.GetHashCode() * RowKey.GetHashCode());
}
private async Task WriteBenchmarkCustomSync(DocumentClient writeClient, DocumentClient readClient)
{
Stopwatch stopwatch = new Stopwatch();
int i = 0;
int total = 100;
long lt = 0;
double ru = 0;
List <Result> result = new List <Result>();
string writeRegion = Helpers.ParseEndpoint(writeClient.WriteEndpoint);
string readRegion = Helpers.ParseEndpoint(readClient.ReadEndpoint);
string consistency = writeClient.ConsistencyLevel.ToString();
Console.WriteLine();
Console.WriteLine($"Test {total} writes in {writeRegion} with {consistency} consistency between all replicas except {readRegion} with Strong consistency. \r\nPress any key to continue\r\n...");
Console.ReadKey(true);
PartitionKey partitionKeyValue = new PartitionKey(PartitionKeyValue);
Console.WriteLine();
for (i = 0; i < total; i++)
{
SampleCustomer customer = customerGenerator.Generate();
stopwatch.Start();
//Write
ResourceResponse <Document> writeResponse = await writeClient.CreateDocumentAsync(containerUri, customer);
//Read
ResourceResponse <Document> readResponse = await readClient.ReadDocumentAsync(writeResponse.Resource.SelfLink,
new RequestOptions { PartitionKey = partitionKeyValue, SessionToken = writeResponse.SessionToken });
stopwatch.Stop();
lt = stopwatch.ElapsedMilliseconds;
ru = writeResponse.RequestCharge + readResponse.RequestCharge;
result.Add(new Result(lt, ru));
stopwatch.Reset();
Console.WriteLine($"Write/Read: Item {i} of {total}, Region: {writeRegion}, Latency: {lt} ms, Request Charge: {ru} RUs");
}
//Average at 99th Percentile
string _latency = Math.Round(result.OrderBy(o => o.Latency).Take(99).Average(o => o.Latency), 0).ToString();
string _ru = Math.Round(result.OrderBy(o => o.Latency).Take(99).Average(o => o.RU)).ToString();
results.Add(new ResultData
{
Test = $"Test with Custom Synchronization",
AvgLatency = _latency,
AvgRU = _ru
});
Console.WriteLine();
Console.WriteLine();
Console.WriteLine("Test Summary");
Console.WriteLine("-------------------------------------------------------------------------------------------------------------------------------------");
Console.WriteLine($"Test {total} writes in {writeRegion} with {consistency} consistency between all replicas except {readRegion} with Strong consistency");
Console.WriteLine();
Console.WriteLine($"Average Latency:\t{_latency} ms");
Console.WriteLine($"Average Request Units:\t{_ru} RUs");
Console.WriteLine();
Console.WriteLine("Press any key to continue...");
Console.ReadKey(true);
Console.WriteLine();
Console.WriteLine("All Tests Summary");
Console.WriteLine("-----------------------------------------------------------------------------------------------------");
foreach (ResultData r in results)
{
Console.WriteLine($"{r.Test}\tAvg Latency: {r.AvgLatency} ms\tAverage RU: {r.AvgRU}");
}
Console.WriteLine();
Console.WriteLine($"Test concluded. Press any key to continue\r\n...");
Console.ReadKey(true);
}
protected StreamUserOperation(Stream streamPayload, TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(options, partitionKey)
{
this.streamPayload = streamPayload;
}
internal static int ExecuteDeleteDocuments(this Container container, string query, PartitionKey partitionKey, CancellationToken cancellationToken = default)
{
int affected = 0;
ProcedureResponse response;
do
{
Task <StoredProcedureExecuteResponse <ProcedureResponse> > task = container.Scripts.ExecuteStoredProcedureAsync <ProcedureResponse>("deleteDocuments", partitionKey, new dynamic[] { query }, cancellationToken: cancellationToken);
task.Wait();
response = task.Result;
affected += response.Affected;
} while (response.Continuation);
return(affected);
}
public UpsertItemOperation(T item, TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(options, partitionKey)
{
this.item = item;
}
internal static void ExecuteExpireDocuments(this Container container, string query, int epoch, PartitionKey partitionKey, CancellationToken cancellationToken = default)
{
ProcedureResponse response;
do
{
Task <StoredProcedureExecuteResponse <ProcedureResponse> > task = container.Scripts.ExecuteStoredProcedureAsync <ProcedureResponse>("expireDocuments", partitionKey, new dynamic[] { query, epoch }, cancellationToken: cancellationToken);
task.Wait();
response = task.Result;
} while (response.Continuation);
}
public async Task ItemResourcePermissionTest()
{
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await this.cosmosDatabase.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerId, "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
string itemId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemId);
ItemResponse <dynamic> itemRespnose = await container.CreateItemAsync <dynamic>(new { id = itemId }, partitionKey);
Assert.AreEqual(HttpStatusCode.Created, itemRespnose.StatusCode);
//create permission
string permissionId = Guid.NewGuid().ToString();
PermissionProperties permissionProperties = new PermissionProperties(permissionId, PermissionMode.Read, container, partitionKey, itemId);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
//delete resource with PermissionMode.Read
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
try
{
ItemResponse <dynamic> response = await tokenCosmosClient
.GetDatabase(this.cosmosDatabase.Id)
.GetContainer(containerId)
.DeleteItemAsync <dynamic>(itemId, partitionKey);
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
}
//update permission to PermissionMode.All
permissionProperties = new PermissionProperties(permissionId, PermissionMode.All, container);
permissionResponse = await user.GetPermission(permissionId).ReplaceAsync(permissionProperties);
permission = permissionResponse.Resource;
//delete resource with PermissionMode.All
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
ItemResponse <dynamic> response = await tokenCosmosClient
.GetDatabase(this.cosmosDatabase.Id)
.GetContainer(containerId)
.DeleteItemAsync <dynamic>(itemId, partitionKey);
Assert.AreEqual(HttpStatusCode.NoContent, response.StatusCode);
}
}
// Add record to Country collection
private static async Task <bool> UploadCountryData_AddRecord(ILogger log, Country country, PartitionKey parkey, JToken token)
{
bool queryDone = false;
int tries = 0;
while (!queryDone)
{
try
{
log.LogInformation("---- " + country.Key + ": adding database record");
Document doc = await client.CreateDocumentAsync(CountryUrl, token, new RequestOptions()
{
PartitionKey = parkey
});
log.LogInformation("Country record " + country.Key + " created");
queryDone = true;
}
catch (DocumentClientException de)
{
var statusCode = (int)de.StatusCode;
if ((statusCode == 429 || statusCode == 503) && tries < 3)
{
log.LogInformation(">>>> Error 429/503(de): " + country.Key + " : RETRYING ADD AFTER SLEEP <<<<");
Thread.Sleep(de.RetryAfter);
}
else
{
log.LogError(de, "UploadCountryData(" + country.Key + ").db-add failed");
queryDone = true; // exit loop because of hard failure
return(false);
}
}
catch (System.AggregateException ae)
{
// See if a request rate too large occurred
if (ae.InnerException.GetType() == typeof(DocumentClientException))
{
var docExcep = ae.InnerException as DocumentClientException;
var statusCode = (int)docExcep.StatusCode;
if ((statusCode == 429 || statusCode == 503) && tries < 3)
{
log.LogInformation(">>>> Error 429/503(ae): " + country.Key + " : RETRYING ADD AFTER SLEEP <<<<");
Thread.Sleep(docExcep.RetryAfter);
}
else
{
log.LogError(ae, "UploadCountryData(" + country.Key + ").db-add db failed");
queryDone = true; // exit loop because of hard failure
return(false);
}
}
}
} // end while !queryDone
return(true);
}
public async Task ItemResourcePermissionTest()
{
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await this.cosmosDatabase.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerId, "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
string itemId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemId);
ItemResponse <dynamic> itemRespnose = await container.CreateItemAsync <dynamic>(new { id = itemId }, partitionKey);
Assert.AreEqual(HttpStatusCode.Created, itemRespnose.StatusCode);
//create permission
string permissionId = Guid.NewGuid().ToString();
PermissionProperties permissionProperties = new PermissionProperties(permissionId, PermissionMode.Read, container, partitionKey, itemId);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
//delete resource with PermissionMode.Read
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(this.cosmosDatabase.Id, containerId);
ItemResponse <dynamic> readPermissionItem = await tokenContainer.ReadItemAsync <dynamic>(itemId, partitionKey);
Assert.AreEqual(itemId, readPermissionItem.Resource.id.ToString());
try
{
ItemResponse <dynamic> response = await tokenContainer.DeleteItemAsync <dynamic>(
itemId,
partitionKey);
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
}
//update permission to PermissionMode.All
permissionProperties = new PermissionProperties(permissionId, PermissionMode.All, container);
permissionResponse = await user.GetPermission(permissionId).ReplaceAsync(permissionProperties);
permission = permissionResponse.Resource;
//delete resource with PermissionMode.All
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
using (FeedIterator <dynamic> feed = tokenCosmosClient
.GetDatabase(this.cosmosDatabase.Id)
.GetContainer(containerId)
.GetItemQueryIterator <dynamic>(new QueryDefinition("select * from t")))
{
while (feed.HasMoreResults)
{
FeedResponse <dynamic> response = await feed.ReadNextAsync();
Assert.IsNotNull(response);
}
}
}
}
public async Task UpsertAsync(TemporaryExposureKeyModel model)
{
_logger.LogInformation($"start {nameof(UpsertAsync)}");
var pk = new PartitionKey(model.PartitionKey);
await _db.TemporaryExposureKey.UpsertItemAsync(model, pk);
}
// If country record already exists in Country collection, remove it
private static async Task <bool> UploadCountryData_DeletePriorRecords(ILogger log, Country country, PartitionKey parkey)
{
// If country record already exists, remove it
log.LogInformation("---- " + country.Key + ": deleting prior Country DB records...");
var query = new SqlQuerySpec("SELECT * FROM Country c WHERE c.name = @name",
new SqlParameterCollection(new SqlParameter[] { new SqlParameter {
Name = "@name", Value = country.Name
} }));
var existingCountryRecords = client.CreateDocumentQuery <Microsoft.Azure.Documents.Document>(CountryUrl, query, new FeedOptions()
{
PartitionKey = parkey
});
bool queryDone = false;
int tries = 0;
if (existingCountryRecords != null)
{
int count = 0;
queryDone = false;
tries = 0;
while (!queryDone)
{
try
{
tries++;
RequestOptions options = new RequestOptions()
{
PartitionKey = parkey
};
if (existingCountryRecords != null)
{
foreach (Microsoft.Azure.Documents.Document c in existingCountryRecords)
{
await client.DeleteDocumentAsync(c.SelfLink, options);
count++;
}
if (count > 0)
{
log.LogInformation("---- " + country.Key + ": prior record deleted");
}
}
queryDone = true;
return(true);
}
catch (DocumentClientException de)
{
var statusCode = (int)de.StatusCode;
if ((statusCode == 429 || statusCode == 503) && tries < 3)
{
log.LogInformation(">>>> Error 429/503(de): " + country.Key + " : RETRYING DELETE AFTER SLEEP <<<<");
Thread.Sleep(de.RetryAfter);
}
else
{
log.LogError(de, "UploadCountryData(" + country.Key + ").db-delete failed");
queryDone = true; // exit loop because of hard failure
return(false);
}
}
catch (System.AggregateException ae)
{
// See if a request rate too large occurred
if (ae.InnerException.GetType() == typeof(DocumentClientException))
{
var docExcep = ae.InnerException as DocumentClientException;
var statusCode = (int)docExcep.StatusCode;
if ((statusCode == 429 || statusCode == 503) && tries < 3)
{
log.LogInformation(">>>> Error 429/503(ae): " + country.Key + " : RETRYING DELETE AFTER SLEEP <<<<");
Thread.Sleep(docExcep.RetryAfter);
}
else
{
log.LogError(ae, "UploadCountryData(" + country.Key + ").db-delete db failed");
queryDone = true; // exit loop because of hard failure
return(false);
}
}
}
} // end while !queryDone
} // end if records-to-delete
return(true);
}
public async Task TestQueryWithSpecialPartitionKeysAsync()
{
QueryWithSpecialPartitionKeysArgs[] queryWithSpecialPartitionKeyArgsList = new QueryWithSpecialPartitionKeysArgs[]
{
new QueryWithSpecialPartitionKeysArgs()
{
Name = "Guid",
Value = Guid.NewGuid(),
ValueToPartitionKey = val => val.ToString(),
},
//new QueryWithSpecialPartitionKeysArgs()
//{
// Name = "DateTime",
// Value = DateTime.Now,
// ValueToPartitionKey = val =>
// {
// string str = JsonConvert.SerializeObject(
// val,
// new JsonSerializerSettings()
// {
// Converters = new List<JsonConverter> { new IsoDateTimeConverter() }
// });
// return str.Substring(1, str.Length - 2);
// },
//},
new QueryWithSpecialPartitionKeysArgs()
{
Name = "Enum",
Value = HttpStatusCode.OK,
ValueToPartitionKey = val => (int)val,
},
new QueryWithSpecialPartitionKeysArgs()
{
Name = "CustomEnum",
Value = HttpStatusCode.OK,
ValueToPartitionKey = val => val.ToString(),
},
new QueryWithSpecialPartitionKeysArgs()
{
Name = "ResourceId",
Value = "testid",
ValueToPartitionKey = val => val,
},
new QueryWithSpecialPartitionKeysArgs()
{
Name = "CustomDateTime",
Value = new DateTime(2016, 11, 12),
ValueToPartitionKey = val => EpochDateTimeConverter.DateTimeToEpoch((DateTime)val),
},
};
foreach (QueryWithSpecialPartitionKeysArgs testArg in queryWithSpecialPartitionKeyArgsList)
{
// For this test we need to split direct and gateway runs into separate collections,
// since the query callback inserts some documents (thus has side effects).
await this.CreateIngestQueryDeleteAsync <QueryWithSpecialPartitionKeysArgs>(
ConnectionModes.Direct,
CollectionTypes.SinglePartition,
QueryTestsBase.NoDocuments,
ImplementationAsync,
testArg,
"/" + testArg.Name);
await this.CreateIngestQueryDeleteAsync <QueryWithSpecialPartitionKeysArgs>(
ConnectionModes.Direct,
CollectionTypes.MultiPartition,
QueryTestsBase.NoDocuments,
ImplementationAsync,
testArg,
"/" + testArg.Name);
await this.CreateIngestQueryDeleteAsync <QueryWithSpecialPartitionKeysArgs>(
ConnectionModes.Gateway,
CollectionTypes.SinglePartition,
QueryTestsBase.NoDocuments,
ImplementationAsync,
testArg,
"/" + testArg.Name);
await this.CreateIngestQueryDeleteAsync <QueryWithSpecialPartitionKeysArgs>(
ConnectionModes.Gateway,
CollectionTypes.MultiPartition,
QueryTestsBase.NoDocuments,
ImplementationAsync,
testArg,
"/" + testArg.Name);
}
async Task ImplementationAsync(
Container container,
IReadOnlyList <CosmosObject> documents,
QueryWithSpecialPartitionKeysArgs testArgs)
{
QueryWithSpecialPartitionKeysArgs args = testArgs;
SpecialPropertyDocument specialPropertyDocument = new SpecialPropertyDocument
{
Id = Guid.NewGuid().ToString()
};
specialPropertyDocument.GetType().GetProperty(args.Name).SetValue(specialPropertyDocument, args.Value);
object getPropertyValueFunction(SpecialPropertyDocument d) => d.GetType().GetProperty(args.Name).GetValue(d);
ItemResponse <SpecialPropertyDocument> response = await container.CreateItemAsync <SpecialPropertyDocument>(specialPropertyDocument);
dynamic returnedDoc = response.Resource;
Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc));
PartitionKey key = new PartitionKey(args.ValueToPartitionKey(args.Value));
response = await container.ReadItemAsync <SpecialPropertyDocument>(response.Resource.Id, new Cosmos.PartitionKey(key));
returnedDoc = response.Resource;
Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc));
returnedDoc = (await this.RunSinglePartitionQuery <SpecialPropertyDocument>(
container,
"SELECT * FROM t")).Single();
Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc));
string query;
switch (args.Name)
{
case "Guid":
query = $"SELECT * FROM T WHERE T.Guid = '{(Guid)args.Value}'";
break;
case "Enum":
query = $"SELECT * FROM T WHERE T.Enum = '{(HttpStatusCode)args.Value}'";
break;
case "DateTime":
query = $"SELECT * FROM T WHERE T.DateTime = '{(DateTime)args.Value}'";
break;
case "CustomEnum":
query = $"SELECT * FROM T WHERE T.CustomEnum = '{(HttpStatusCode)args.Value}'";
break;
case "ResourceId":
query = $"SELECT * FROM T WHERE T.ResourceId = '{(string)args.Value}'";
break;
case "CustomDateTime":
query = $"SELECT * FROM T WHERE T.CustomDateTime = '{(DateTime)args.Value}'";
break;
default:
query = null;
break;
}
returnedDoc = (await container.GetItemQueryIterator <SpecialPropertyDocument>(
query,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
PartitionKey = new Cosmos.PartitionKey(args.ValueToPartitionKey),
}).ReadNextAsync()).First();
Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc));
}
}
public async Task DeleteAsync(TemporaryExposureKeyModel model)
{
_logger.LogInformation($"start {nameof(DeleteAsync)}");
var pk = new PartitionKey(model.PartitionKey);
await _db.TemporaryExposureKey.DeleteItemAsync <TemporaryExposureKeyModel>(model.id, pk);
}
public bool TryExtract(IReadOnlyDictionary <string, string> headers, out PartitionKey?partitionKey)
{
partitionKey = new PartitionKey(testContext.TestRunId.ToString());
testContext.ExtractorWasCalled = true;
return(true);
}
public bool UseRowKey()
{
return(PartitionKey.Substring(2, 2) == "05" ||
PartitionKey.Substring(2, 2) == "06");
}
/// <summary>
/// Insert a document stream into the database.
/// </summary>
/// <param name="stream">Stream to insert</param>
/// <param name="partitionKey">PartitionKey to send</param>
/// <returns><see cref="CosmosResponse"/> that tracks success status along with various performance parameters</returns>
public Task <CosmosResponse> InsertDocument(Stream stream, PartitionKey partitionKey)
{
return(InsertDocumentStreamInternal(stream, partitionKey));
}
public override Task <ItemResponse <T> > ReadItemAsync <T>(string id, PartitionKey partitionKey, ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default)
{
var returnValue = new CosmosItemResponseMock <T>();
return(Task.FromResult((ItemResponse <T>)returnValue));
}
public override int GetHashCode() => PartitionKey.GetHashCode();
public Connection GetOrCreateConnection(ConnectionScope scope, PartitionKey partitionKey)
{
Connection connection = null;
//provide connections on connection level only
if(scope == ConnectionScope.Command)
return null;
if(scope == ConnectionScope.Infrastructure)
{
//connection exist, go and find random one
if(_connectionCount > 0)
{
int count = _nodes.Count;
int offset = _rndGen.Next(count);
for(int i = 0; i < count; i++)
{
connection = _nodes[(offset + i)%count].GetConnection();
if(connection != null)
return connection;
}
}
}
else
{
//try pick an unused connection
while(_connections.TryPop(out connection))
{
if(connection.IsAvailable)
{
connection.AllowCleanup = false;
return connection;
}
}
}
//check if we may create another connection if we didn't find a connection yet
if(_config.MaxConnections <= 0 || _connectionCount < _config.MaxConnections)
{
//all connections in use, or non available, go and create one at random node
int count = _nodes.Count;
int offset = _rndGen.Next(count);
for(int i = 0; i < count; i++)
{
connection = _nodes[(offset + i)%count].CreateConnection();
if(connection != null)
{
Interlocked.Increment(ref _connectionCount);
connection.OnConnectionChange += (src, ev) =>
{
if(!ev.Connected)
Interlocked.Decrement(ref _connectionCount);
};
//if infrastructure scope, push connection to list of available connections for other use
if(scope == ConnectionScope.Infrastructure)
_connections.Push(connection);
else
//disable cleanup of this connection while it is in reserved for exclusive use
connection.AllowCleanup = false;
return connection;
}
}
}
//yikes nothing available
return connection;
}
protected UserOperation(TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(partitionKey, null, null)
{
this.options = options;
}
/// <summary>
/// Gets a connection to a reference in the cluster
/// </summary>
/// <param name="partitionKey"> </param>
/// <returns> </returns>
public Connection GetOrCreateConnection(PartitionKey partitionKey)
{
return _connectionSelector.GetOrCreateConnection(partitionKey);
}
public ReplaceItemOperation(string id, T item, TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(options, partitionKey)
{
this.id = id;
this.item = item;
}
public IQuery<PropertyBag> Execute(string cql, object dataSource, PartitionKey partitionKey)
{
return _command.Execute<PropertyBag>(cql, dataSource, partitionKey);
}
public ReplaceItemStreamOperation(string id, Stream streamPayload, TransactionalBatchItemRequestOptions options, PartitionKey partitionKey) : base(streamPayload, options, partitionKey)
{
this.id = id;
}
public PatchItemOperation(string id, IReadOnlyList <PatchOperation> patchOperations, TransactionalBatchPatchItemRequestOptions options, PartitionKey partitionKey) : base(options, partitionKey)
{
this.id = id;
this.patchOperations = patchOperations;
}
/// <summary>
/// Requests a connection to be used for a command. Will reuse connection level connection if available
/// </summary>
/// <returns> An open connection </returns>
internal Connection GetConnection(PartitionKey partitionKey = default(PartitionKey))
{
if(State != ConnectionState.Open)
throw new CqlException("CqlConnection is not open for use");
//reuse the connection if any available on connection level
Connection connection = _connection ??
Cluster.ConnectionStrategy.GetOrCreateConnection(ConnectionScope.Command,
partitionKey);
if(connection == null)
throw new CqlException("Unable to obtain a Cql network connection.");
return connection;
}
