// </RunBulkImport> /// <summary> /// Get documents ordered by some doc property. This is done using OrderBy stored procedure. /// </summary> // <RunOrderBy> private static async Task RunOrderBy(Container container) { // 1. Create or get the stored procedure. string body = File.ReadAllText(@"js\OrderBy.js"); string scriptId = "OrderBy"; await TryDeleteStoredProcedure(container, scriptId); Scripts cosmosScripts = container.Scripts; StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, body)); // 2. Prepare to run stored procedure. string orderByFieldName = "FamilyId"; string filterQuery = string.Format(CultureInfo.InvariantCulture, "SELECT r.FamilyId FROM root r WHERE r.{0} > 10", orderByFieldName); // Note: in order to do a range query (> 10) on this field, the collection must have a range index set for this path (see ReadOrCreateCollection). int?continuationToken = null; int batchCount = 0; do { // 3. Run the stored procedure. StoredProcedureExecuteResponse <OrderByResult> response = await cosmosScripts.ExecuteStoredProcedureAsync <OrderByResult>( scriptId, new PartitionKey("Andersen"), new dynamic[] { filterQuery, orderByFieldName, continuationToken }); // 4. Process stored procedure response. continuationToken = response.Resource.Continuation; Console.WriteLine($"Printing documents filtered/ordered by '{filterQuery}' and ordered by '{orderByFieldName}', batch #{batchCount++}, count #{response.Resource.Result.Length}"); } while (continuationToken != null); // 5. To take care of big response, loop until Response.continuation token is null (see OrderBy.js for details). }
public async Task TestReducePageSizeScenario() { int partitionKey = 0; // Create some docs to make sure that one separate response is returned for 1st execute of query before retries. // These are to make sure continuation token is passed along during retries. string sprocId = "createTwoDocs"; string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument( __.getSelfLink(), { id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 }, err => { if (err) throw err;} );}"; Scripts scripts = this.Container.Scripts; StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody)); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { processedDocCount += docs.Count(); foreach (var doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartFromBeginning() .WithInstanceName("random") .WithMaxItems(6) .WithLeaseContainer(this.LeaseContainer).Build(); // Generate the payload await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 0, new PartitionKey(partitionKey)); // Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction. var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000)); await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey)); await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 3, new PartitionKey(partitionKey)); await processor.StartAsync(); // Letting processor initialize and pickup changes var isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator); }
// </RunDemoAsync> /// <summary> /// Runs a simple script which just does a server side query /// </summary> // <RunSimpleScript> private static async Task RunSimpleScript(Container container) { // 1. Create stored procedure for script. string scriptFileName = @"js\SimpleScript.js"; string scriptId = Path.GetFileNameWithoutExtension(scriptFileName); await TryDeleteStoredProcedure(container, scriptId); Scripts cosmosScripts = container.Scripts; StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, File.ReadAllText(scriptFileName))); // 2. Create a document. SampleDocument doc = new SampleDocument { Id = Guid.NewGuid().ToString(), LastName = "Estel", Headquarters = "Russia", Locations = new Location[] { new Location { Country = "Russia", City = "Novosibirsk" } }, Income = 50000 }; ItemResponse <SampleDocument> created = await container.CreateItemAsync(doc, new PartitionKey(doc.LastName)); // 3. Run the script. Pass "Hello, " as parameter. // The script will take the 1st document and echo: Hello, <document as json>. StoredProcedureExecuteResponse <string> response = await container.Scripts.ExecuteStoredProcedureAsync <string, string>(new PartitionKey(doc.LastName), scriptId, "Hello"); Console.WriteLine("Result from script: {0}\r\n", response.Resource); await container.DeleteItemAsync <SampleDocument>(doc.Id, new PartitionKey(doc.LastName)); }
public async Task ExecuteNonePkTest() { // Create a container in v2 without a partition key string containerId = "SprocPkNone" + Guid.NewGuid().ToString(); ContainerInternal containerNonePk = await NonPartitionedContainerHelper.CreateNonPartitionedContainer(this.database, containerId); Scripts scriptsNonePk = containerNonePk.Scripts; string sprocId = Guid.NewGuid().ToString(); string sprocBody = @"function() { var context = getContext(); var response = context.getResponse(); var collection = context.getCollection(); var collectionLink = collection.getSelfLink(); var filterQuery = 'SELECT * FROM c'; collection.queryDocuments(collectionLink, filterQuery, { }, function(err, documents) { response.setBody(documents); } ); }"; StoredProcedureResponse storedProcedureResponse = await scriptsNonePk.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody)); Assert.AreEqual(HttpStatusCode.Created, storedProcedureResponse.StatusCode); StoredProcedureTests.ValidateStoredProcedureSettings(sprocId, sprocBody, storedProcedureResponse); // Insert document and then query string testPartitionId = Guid.NewGuid().ToString(); var payload = new { id = testPartitionId, user = testPartitionId }; ItemResponse <dynamic> createItemResponse = await containerNonePk.CreateItemAsync <dynamic>(payload); Assert.AreEqual(HttpStatusCode.Created, createItemResponse.StatusCode); StoredProcedureProperties storedProcedure = storedProcedureResponse; StoredProcedureExecuteResponse <JArray> sprocResponse = await scriptsNonePk.ExecuteStoredProcedureAsync <JArray>( sprocId, Cosmos.PartitionKey.None, parameters : null); Assert.AreEqual(HttpStatusCode.OK, sprocResponse.StatusCode); JArray jArray = sprocResponse; Assert.AreEqual(1, jArray.Count); Assert.AreEqual(testPartitionId, jArray[0]["id"]); StoredProcedureResponse deleteResponse = await scriptsNonePk.DeleteStoredProcedureAsync(sprocId); Assert.AreEqual(HttpStatusCode.NoContent, deleteResponse.StatusCode); }
public async Task ScriptsStoredProcedureTest(bool directMode) { CosmosClient client = directMode ? DirectCosmosClient : GatewayCosmosClient; Database database = client.GetDatabase(DatabaseId); Container container = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/pk"); Scripts scripts = container.Scripts; List <string> createdIds = new List <string>() { "BasicQuerySp1", "BasicQuerySp2", "BasicQuerySp3" }; //Basic query List <StoredProcedureProperties> queryResults = await this.ToListAsync( scripts.GetStoredProcedureQueryStreamIterator, scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>, "select * from T where STARTSWITH(T.id, \"BasicQuerySp\")", CosmosBasicQueryTests.RequestOptions); if (queryResults.Count < 3) { foreach (string id in createdIds) { StoredProcedureProperties properties = await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties() { Id = id, Body = "function() {var x = 10;}" }); } queryResults = await this.ToListAsync( scripts.GetStoredProcedureQueryStreamIterator, scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>, "select * from T where STARTSWITH(T.id, \"BasicQuerySp\")", CosmosBasicQueryTests.RequestOptions); } CollectionAssert.AreEquivalent(createdIds, queryResults.Select(x => x.Id).ToList()); //Read All List <StoredProcedureProperties> results = await this.ToListAsync( scripts.GetStoredProcedureQueryStreamIterator, scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>, null, CosmosBasicQueryTests.RequestOptions); CollectionAssert.IsSubsetOf(createdIds, results.Select(x => x.Id).ToList()); }
public async Task IteratorTest() { using (CosmosClient cosmosClient = TestCommon.CreateCosmosClient(new CosmosClientOptions() { Serializer = new FaultySerializer() })) { // Should not use the custom serializer for these operations Scripts scripts = cosmosClient.GetContainer(this.database.Id, this.container.Id).Scripts; string sprocBody = "function() { { var x = 42; } }"; int numberOfSprocs = 3; string[] sprocIds = new string[numberOfSprocs]; for (int i = 0; i < numberOfSprocs; i++) { string sprocId = Guid.NewGuid().ToString(); sprocIds[i] = sprocId; StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody)); Assert.AreEqual(HttpStatusCode.Created, storedProcedureResponse.StatusCode); } List <string> readSprocIds = new List <string>(); FeedIterator <StoredProcedureProperties> iter = scripts.GetStoredProcedureQueryIterator <StoredProcedureProperties>(); while (iter.HasMoreResults) { FeedResponse <StoredProcedureProperties> currentResultSet = await iter.ReadNextAsync(); { foreach (StoredProcedureProperties storedProcedureSettingsEntry in currentResultSet) { readSprocIds.Add(storedProcedureSettingsEntry.Id); } } } CollectionAssert.AreEquivalent(sprocIds, readSprocIds); } }
public async Task CreateStoredProcedure(string spName) { var spBody = File.ReadAllText($"StoredProcedures/{spName}.js"); Scripts scripts = _container.Scripts; StoredProcedureProperties storedProcedure = new StoredProcedureProperties { Id = spName, Body = spBody }; StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(storedProcedure); var spResult = storedProcedureResponse.Resource; string message = "Create StoredProcedure"; Printer.PrintLine(message: message); WriteLine($"SP Created, Id: {spResult.Id}, \n\tSelfLink: {spResult.SelfLink}"); Printer.PrintLine(noOfTimes: (101 + message.Length)); }
// </RunSimpleScript> /// <summary> /// Import many documents using stored procedure. /// </summary> // <RunBulkImport> private static async Task RunBulkImport(Container container) { string inputDirectory = @".\Data\"; string inputFileMask = "*.json"; int maxFiles = 2000; int maxScriptSize = 50000; // 1. Get the files. string[] fileNames = Directory.GetFiles(inputDirectory, inputFileMask); DirectoryInfo di = new DirectoryInfo(inputDirectory); FileInfo[] fileInfos = di.GetFiles(inputFileMask); // 2. Prepare for import. int currentCount = 0; int fileCount = maxFiles != 0 ? Math.Min(maxFiles, fileNames.Length) : fileNames.Length; // 3. Create stored procedure for this script. string scriptId = "BulkImport"; string body = File.ReadAllText(@".\JS\BulkImport.js"); await TryDeleteStoredProcedure(container, scriptId); Scripts cosmosScripts = container.Scripts; StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, body)); // 4. Create a batch of docs (MAX is limited by request size (2M) and to script for execution. // We send batches of documents to create to script. // Each batch size is determined by MaxScriptSize. // MaxScriptSize should be so that: // -- it fits into one request (MAX request size is 16Kb). // -- it doesn't cause the script to time out. // -- it is possible to experiment with MaxScriptSize to get best performance given number of throttles, etc. while (currentCount < fileCount) { // 5. Create args for current batch. // Note that we could send a string with serialized JSON and JSON.parse it on the script side, // but that would cause script to run longer. Since script has timeout, unload the script as much // as we can and do the parsing by client and framework. The script will get JavaScript objects. string argsJson = CreateBulkInsertScriptArguments(fileNames, currentCount, fileCount, maxScriptSize); dynamic[] args = new dynamic[] { JsonConvert.DeserializeObject <dynamic>(argsJson) }; // 6. execute the batch. StoredProcedureExecuteResponse <int> scriptResult = await cosmosScripts.ExecuteStoredProcedureAsync <int>( scriptId, new PartitionKey("Andersen"), args); // 7. Prepare for next batch. int currentlyInserted = scriptResult.Resource; currentCount += currentlyInserted; } // 8. Validate int numDocs = 0; using (FeedIterator <dynamic> setIterator = container.GetItemQueryIterator <dynamic>()) { while (setIterator.HasMoreResults) { FeedResponse <dynamic> response = await setIterator.ReadNextAsync(); numDocs += response.Count(); } } Console.WriteLine("Found {0} documents in the collection. There were originally {1} files in the Data directory\r\n", numDocs, fileCount); }