コード例 #1
0
        public async Task TestSparkSessionJobWithPublicConstructor()
        {
            SparkSessionClient client = CreateClient();

            // Start the Spark session
            SparkSessionOptions   createParams     = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording);
            SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams);

            SparkSessionOperation anotherSessionOperation = InstrumentOperation(new SparkSessionOperation(int.Parse(sessionOperation.Id), client));
            SparkSession          sessionCreateResponse   = await anotherSessionOperation.WaitForCompletionAsync();

            // Verify the Spark session completes successfully
            Assert.True(LivyStates.Idle == sessionCreateResponse.State,
                        string.Format(
                            "Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
                            sessionCreateResponse.Id,
                            sessionCreateResponse.State,
                            sessionCreateResponse.Result,
                            string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>())
                            )
                        );

            // Execute Spark statement in the session
            var sparkStatementOptions = new SparkStatementOptions {
                Kind = SparkStatementLanguageType.Spark,
                Code = @"print(""Hello world"")"
            };
            SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions);

            SparkStatementOperation anotherStatementOperation = InstrumentOperation(new SparkStatementOperation(int.Parse(sessionOperation.Id), int.Parse(statementOperation.Id), client));
            SparkStatement          createStatementResponse   = await anotherStatementOperation.WaitForCompletionAsync();

            Assert.NotNull(createStatementResponse);

            // Verify the Spark statement completes successfully
            Assert.True(LivyStatementStates.Available == createStatementResponse.State,
                        string.Format(
                            "Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
                            createStatementResponse.Id,
                            createStatementResponse.State,
                            createStatementResponse.Output.ErrorValue)
                        );

            // Verify the output
            Dictionary <string, object> outputData = createStatementResponse.Output.Data as Dictionary <string, object>;

            Assert.AreEqual("Hello world", outputData["text/plain"] as string);

            // Get the list of Spark statements and check that the executed statement exists
            Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id);

            Assert.NotNull(listStatementResponse.Value);
            Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));

            // Get the list of Spark session and check that the created session exists
            List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client);

            Assert.NotNull(listSessionResponse);
            Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
        }
        public async Task TestSparkSessionJob()
        {
            SparkSessionClient client = CreateClient();

            // Start the Spark session
            SparkSessionOptions   createParams     = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording);
            SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams);

            SparkSession sessionCreateResponse = await sessionOperation.WaitForCompletionAsync();

            // Verify the Spark session completes successfully
            Assert.True("idle".Equals(sessionCreateResponse.State, StringComparison.OrdinalIgnoreCase) && sessionCreateResponse.Result == SparkSessionResultType.Succeeded,
                        string.Format(
                            "Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
                            sessionCreateResponse.Id,
                            sessionCreateResponse.State,
                            sessionCreateResponse.Result,
                            string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>())
                            )
                        );

            // Execute Spark statement in the session
            var sparkStatementOptions = new SparkStatementOptions {
                Kind = SparkStatementLanguageType.Spark,
                Code = @"print(""Hello world\n"")"
            };
            SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions);

            SparkStatement createStatementResponse = await statementOperation.WaitForCompletionAsync();

            Assert.NotNull(createStatementResponse);

            // Verify the Spark statement completes successfully
            Assert.True("ok".Equals(createStatementResponse.State, StringComparison.OrdinalIgnoreCase),
                        string.Format(
                            "Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
                            createStatementResponse.Id,
                            createStatementResponse.State,
                            createStatementResponse.Output.ErrorValue)
                        );

            // Verify the output
            Dictionary <string, string> outputData = JsonSerializer.Deserialize <Dictionary <string, string> >(createStatementResponse.Output.Data as string);

            Assert.Equals("Hello world", outputData["text/plain"]);

            // Get the list of Spark statements and check that the executed statement exists
            Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id);

            Assert.NotNull(listStatementResponse.Value);
            Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));

            // Get the list of Spark session and check that the created session exists
            List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client);

            Assert.NotNull(listSessionResponse);
            Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
        }
コード例 #3
0
        public async Task ExecuteSparkStatementSync()
        {
            #region Snippet:CreateSparkSessionClientAsync
            // Replace the strings below with the spark and endpoint information
            string sparkPoolName = "<my-spark-pool-name>";
            /*@@*/ sparkPoolName = TestEnvironment.SparkPoolName;

            string endpoint = "<my-endpoint-url>";
            /*@@*/ endpoint = TestEnvironment.EndpointUrl;

            SparkSessionClient client = new SparkSessionClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
            #endregion

            #region Snippet:CreateSparkSessionAsync
            SparkSessionOptions request = new SparkSessionOptions(name: $"session-{Guid.NewGuid()}")
            {
                DriverMemory   = "28g",
                DriverCores    = 4,
                ExecutorMemory = "28g",
                ExecutorCores  = 4,
                ExecutorCount  = 2
            };

            SparkSessionOperation createSessionOperation = await client.StartCreateSparkSessionAsync(request);

            SparkSession sessionCreated = await createSessionOperation.WaitForCompletionAsync();

            #endregion

            #region Snippet:GetSparkSessionAsync
            SparkSession session = await client.GetSparkSessionAsync(sessionCreated.Id);

            Debug.WriteLine($"Session is returned with name {session.Name} and state {session.State}");
            #endregion

            #region Snippet:CreateSparkStatementAsync
            SparkStatementOptions sparkStatementRequest = new SparkStatementOptions
            {
                Kind = SparkStatementLanguageType.Spark,
                Code = @"print(""Hello world\n"")"
            };

            SparkStatementOperation createStatementOperation = await client.StartCreateSparkStatementAsync(sessionCreated.Id, sparkStatementRequest);

            SparkStatement statementCreated = await createStatementOperation.WaitForCompletionAsync();

            #endregion

            #region Snippet:GetSparkStatementAsync
            SparkStatement statement = await client.GetSparkStatementAsync(sessionCreated.Id, statementCreated.Id);

            Debug.WriteLine($"Statement is returned with id {statement.Id} and state {statement.State}");
            #endregion

            #region Snippet:CancelSparkStatementAsync
            SparkStatementCancellationResult cancellationResult = client.CancelSparkStatement(sessionCreated.Id, statementCreated.Id);
            Debug.WriteLine($"Statement is cancelled with message {cancellationResult.Message}");
            #endregion

            #region Snippet:CancelSparkSessionAsync
            Response operation = client.CancelSparkSession(sessionCreated.Id);
            #endregion
        }