// </RunBulkImport>
/// <summary>
/// Get documents ordered by some doc property. This is done using OrderBy stored procedure.
/// </summary>
// <RunOrderBy>
private static async Task RunOrderBy(Container container)
{
// 1. Create or get the stored procedure.
string body = File.ReadAllText(@"js\OrderBy.js");
string scriptId = "OrderBy";
await TryDeleteStoredProcedure(container, scriptId);
Scripts cosmosScripts = container.Scripts;
StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, body));
// 2. Prepare to run stored procedure.
string orderByFieldName = "FamilyId";
string filterQuery = string.Format(CultureInfo.InvariantCulture, "SELECT r.FamilyId FROM root r WHERE r.{0} > 10", orderByFieldName);
// Note: in order to do a range query (> 10) on this field, the collection must have a range index set for this path (see ReadOrCreateCollection).
int?continuationToken = null;
int batchCount = 0;
do
{
// 3. Run the stored procedure.
StoredProcedureExecuteResponse <OrderByResult> response = await cosmosScripts.ExecuteStoredProcedureAsync <OrderByResult>(
scriptId,
new PartitionKey("Andersen"),
new dynamic[] { filterQuery, orderByFieldName, continuationToken });
// 4. Process stored procedure response.
continuationToken = response.Resource.Continuation;
Console.WriteLine($"Printing documents filtered/ordered by '{filterQuery}' and ordered by '{orderByFieldName}', batch #{batchCount++}, count #{response.Resource.Result.Length}");
} while (continuationToken != null);
// 5. To take care of big response, loop until Response.continuation token is null (see OrderBy.js for details).
}
public async Task TestReducePageSizeScenario()
{
int partitionKey = 0;
// Create some docs to make sure that one separate response is returned for 1st execute of query before retries.
// These are to make sure continuation token is passed along during retries.
string sprocId = "createTwoDocs";
string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument(
__.getSelfLink(),
{ id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 },
err => { if (err) throw err;}
);}";
Scripts scripts = this.Container.Scripts;
StoredProcedureResponse storedProcedureResponse =
await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody));
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
processedDocCount += docs.Count();
foreach (var doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartFromBeginning()
.WithInstanceName("random")
.WithMaxItems(6)
.WithLeaseContainer(this.LeaseContainer).Build();
// Generate the payload
await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 0, new PartitionKey(partitionKey));
// Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction.
var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000));
await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey));
await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 3, new PartitionKey(partitionKey));
await processor.StartAsync();
// Letting processor initialize and pickup changes
var isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator);
}
// </RunDemoAsync>
/// <summary>
/// Runs a simple script which just does a server side query
/// </summary>
// <RunSimpleScript>
private static async Task RunSimpleScript(Container container)
{
// 1. Create stored procedure for script.
string scriptFileName = @"js\SimpleScript.js";
string scriptId = Path.GetFileNameWithoutExtension(scriptFileName);
await TryDeleteStoredProcedure(container, scriptId);
Scripts cosmosScripts = container.Scripts;
StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, File.ReadAllText(scriptFileName)));
// 2. Create a document.
SampleDocument doc = new SampleDocument
{
Id = Guid.NewGuid().ToString(),
LastName = "Estel",
Headquarters = "Russia",
Locations = new Location[] { new Location {
Country = "Russia", City = "Novosibirsk"
} },
Income = 50000
};
ItemResponse <SampleDocument> created = await container.CreateItemAsync(doc, new PartitionKey(doc.LastName));
// 3. Run the script. Pass "Hello, " as parameter.
// The script will take the 1st document and echo: Hello, <document as json>.
StoredProcedureExecuteResponse <string> response = await container.Scripts.ExecuteStoredProcedureAsync <string, string>(new PartitionKey(doc.LastName), scriptId, "Hello");
Console.WriteLine("Result from script: {0}\r\n", response.Resource);
await container.DeleteItemAsync <SampleDocument>(doc.Id, new PartitionKey(doc.LastName));
}
public async Task ExecuteNonePkTest()
{
// Create a container in v2 without a partition key
string containerId = "SprocPkNone" + Guid.NewGuid().ToString();
ContainerInternal containerNonePk = await NonPartitionedContainerHelper.CreateNonPartitionedContainer(this.database, containerId);
Scripts scriptsNonePk = containerNonePk.Scripts;
string sprocId = Guid.NewGuid().ToString();
string sprocBody = @"function() {
var context = getContext();
var response = context.getResponse();
var collection = context.getCollection();
var collectionLink = collection.getSelfLink();
var filterQuery = 'SELECT * FROM c';
collection.queryDocuments(collectionLink, filterQuery, { },
function(err, documents) {
response.setBody(documents);
}
);
}";
StoredProcedureResponse storedProcedureResponse =
await scriptsNonePk.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody));
Assert.AreEqual(HttpStatusCode.Created, storedProcedureResponse.StatusCode);
StoredProcedureTests.ValidateStoredProcedureSettings(sprocId, sprocBody, storedProcedureResponse);
// Insert document and then query
string testPartitionId = Guid.NewGuid().ToString();
var payload = new { id = testPartitionId, user = testPartitionId };
ItemResponse <dynamic> createItemResponse = await containerNonePk.CreateItemAsync <dynamic>(payload);
Assert.AreEqual(HttpStatusCode.Created, createItemResponse.StatusCode);
StoredProcedureProperties storedProcedure = storedProcedureResponse;
StoredProcedureExecuteResponse <JArray> sprocResponse = await scriptsNonePk.ExecuteStoredProcedureAsync <JArray>(
sprocId,
Cosmos.PartitionKey.None,
parameters : null);
Assert.AreEqual(HttpStatusCode.OK, sprocResponse.StatusCode);
JArray jArray = sprocResponse;
Assert.AreEqual(1, jArray.Count);
Assert.AreEqual(testPartitionId, jArray[0]["id"]);
StoredProcedureResponse deleteResponse = await scriptsNonePk.DeleteStoredProcedureAsync(sprocId);
Assert.AreEqual(HttpStatusCode.NoContent, deleteResponse.StatusCode);
}
public async Task ScriptsStoredProcedureTest(bool directMode)
{
CosmosClient client = directMode ? DirectCosmosClient : GatewayCosmosClient;
Database database = client.GetDatabase(DatabaseId);
Container container = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/pk");
Scripts scripts = container.Scripts;
List <string> createdIds = new List <string>()
{
"BasicQuerySp1",
"BasicQuerySp2",
"BasicQuerySp3"
};
//Basic query
List <StoredProcedureProperties> queryResults = await this.ToListAsync(
scripts.GetStoredProcedureQueryStreamIterator,
scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>,
"select * from T where STARTSWITH(T.id, \"BasicQuerySp\")",
CosmosBasicQueryTests.RequestOptions);
if (queryResults.Count < 3)
{
foreach (string id in createdIds)
{
StoredProcedureProperties properties = await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties()
{
Id = id,
Body = "function() {var x = 10;}"
});
}
queryResults = await this.ToListAsync(
scripts.GetStoredProcedureQueryStreamIterator,
scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>,
"select * from T where STARTSWITH(T.id, \"BasicQuerySp\")",
CosmosBasicQueryTests.RequestOptions);
}
CollectionAssert.AreEquivalent(createdIds, queryResults.Select(x => x.Id).ToList());
//Read All
List <StoredProcedureProperties> results = await this.ToListAsync(
scripts.GetStoredProcedureQueryStreamIterator,
scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>,
null,
CosmosBasicQueryTests.RequestOptions);
CollectionAssert.IsSubsetOf(createdIds, results.Select(x => x.Id).ToList());
}
public async Task IteratorTest()
{
using (CosmosClient cosmosClient = TestCommon.CreateCosmosClient(new CosmosClientOptions()
{
Serializer = new FaultySerializer()
}))
{
// Should not use the custom serializer for these operations
Scripts scripts = cosmosClient.GetContainer(this.database.Id, this.container.Id).Scripts;
string sprocBody = "function() { { var x = 42; } }";
int numberOfSprocs = 3;
string[] sprocIds = new string[numberOfSprocs];
for (int i = 0; i < numberOfSprocs; i++)
{
string sprocId = Guid.NewGuid().ToString();
sprocIds[i] = sprocId;
StoredProcedureResponse storedProcedureResponse =
await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody));
Assert.AreEqual(HttpStatusCode.Created, storedProcedureResponse.StatusCode);
}
List <string> readSprocIds = new List <string>();
FeedIterator <StoredProcedureProperties> iter = scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>();
while (iter.HasMoreResults)
{
FeedResponse <StoredProcedureProperties> currentResultSet = await iter.ReadNextAsync();
{
foreach (StoredProcedureProperties storedProcedureSettingsEntry in currentResultSet)
{
readSprocIds.Add(storedProcedureSettingsEntry.Id);
}
}
}
CollectionAssert.AreEquivalent(sprocIds, readSprocIds);
}
}
public async Task CreateStoredProcedure(string spName)
{
var spBody = File.ReadAllText($"StoredProcedures/{spName}.js");
Scripts scripts = _container.Scripts;
StoredProcedureProperties storedProcedure = new StoredProcedureProperties
{
Id = spName,
Body = spBody
};
StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(storedProcedure);
var spResult = storedProcedureResponse.Resource;
string message = "Create StoredProcedure";
Printer.PrintLine(message: message);
WriteLine($"SP Created, Id: {spResult.Id}, \n\tSelfLink: {spResult.SelfLink}");
Printer.PrintLine(noOfTimes: (101 + message.Length));
}
// </RunSimpleScript>
/// <summary>
/// Import many documents using stored procedure.
/// </summary>
// <RunBulkImport>
private static async Task RunBulkImport(Container container)
{
string inputDirectory = @".\Data\";
string inputFileMask = "*.json";
int maxFiles = 2000;
int maxScriptSize = 50000;
// 1. Get the files.
string[] fileNames = Directory.GetFiles(inputDirectory, inputFileMask);
DirectoryInfo di = new DirectoryInfo(inputDirectory);
FileInfo[] fileInfos = di.GetFiles(inputFileMask);
// 2. Prepare for import.
int currentCount = 0;
int fileCount = maxFiles != 0 ? Math.Min(maxFiles, fileNames.Length) : fileNames.Length;
// 3. Create stored procedure for this script.
string scriptId = "BulkImport";
string body = File.ReadAllText(@".\JS\BulkImport.js");
await TryDeleteStoredProcedure(container, scriptId);
Scripts cosmosScripts = container.Scripts;
StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, body));
// 4. Create a batch of docs (MAX is limited by request size (2M) and to script for execution.
// We send batches of documents to create to script.
// Each batch size is determined by MaxScriptSize.
// MaxScriptSize should be so that:
// -- it fits into one request (MAX request size is 16Kb).
// -- it doesn't cause the script to time out.
// -- it is possible to experiment with MaxScriptSize to get best performance given number of throttles, etc.
while (currentCount < fileCount)
{
// 5. Create args for current batch.
// Note that we could send a string with serialized JSON and JSON.parse it on the script side,
// but that would cause script to run longer. Since script has timeout, unload the script as much
// as we can and do the parsing by client and framework. The script will get JavaScript objects.
string argsJson = CreateBulkInsertScriptArguments(fileNames, currentCount, fileCount, maxScriptSize);
dynamic[] args = new dynamic[] { JsonConvert.DeserializeObject <dynamic>(argsJson) };
// 6. execute the batch.
StoredProcedureExecuteResponse <int> scriptResult = await cosmosScripts.ExecuteStoredProcedureAsync <int>(
scriptId,
new PartitionKey("Andersen"),
args);
// 7. Prepare for next batch.
int currentlyInserted = scriptResult.Resource;
currentCount += currentlyInserted;
}
// 8. Validate
int numDocs = 0;
using (FeedIterator <dynamic> setIterator = container.GetItemQueryIterator <dynamic>())
{
while (setIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await setIterator.ReadNextAsync();
numDocs += response.Count();
}
}
Console.WriteLine("Found {0} documents in the collection. There were originally {1} files in the Data directory\r\n", numDocs, fileCount);
}