public async Task TestSparkSessionJobWithPublicConstructor() { SparkSessionClient client = CreateClient(); // Start the Spark session SparkSessionOptions createParams = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording); SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams); SparkSessionOperation anotherSessionOperation = InstrumentOperation(new SparkSessionOperation(int.Parse(sessionOperation.Id), client)); SparkSession sessionCreateResponse = await anotherSessionOperation.WaitForCompletionAsync(); // Verify the Spark session completes successfully Assert.True(LivyStates.Idle == sessionCreateResponse.State, string.Format( "Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}", sessionCreateResponse.Id, sessionCreateResponse.State, sessionCreateResponse.Result, string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>()) ) ); // Execute Spark statement in the session var sparkStatementOptions = new SparkStatementOptions { Kind = SparkStatementLanguageType.Spark, Code = @"print(""Hello world"")" }; SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions); SparkStatementOperation anotherStatementOperation = InstrumentOperation(new SparkStatementOperation(int.Parse(sessionOperation.Id), int.Parse(statementOperation.Id), client)); SparkStatement createStatementResponse = await anotherStatementOperation.WaitForCompletionAsync(); Assert.NotNull(createStatementResponse); // Verify the Spark statement completes successfully Assert.True(LivyStatementStates.Available == createStatementResponse.State, string.Format( "Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}", createStatementResponse.Id, createStatementResponse.State, createStatementResponse.Output.ErrorValue) ); // Verify the output Dictionary <string, object> outputData = createStatementResponse.Output.Data as Dictionary <string, object>; Assert.AreEqual("Hello world", outputData["text/plain"] as string); // Get the list of Spark statements and check that the executed statement exists Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id); Assert.NotNull(listStatementResponse.Value); Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id)); // Get the list of Spark session and check that the created session exists List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client); Assert.NotNull(listSessionResponse); Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id)); }
public async Task TestSparkSessionJob() { SparkSessionClient client = CreateClient(); // Start the Spark session SparkSessionOptions createParams = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording); SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams); SparkSession sessionCreateResponse = await sessionOperation.WaitForCompletionAsync(); // Verify the Spark session completes successfully Assert.True("idle".Equals(sessionCreateResponse.State, StringComparison.OrdinalIgnoreCase) && sessionCreateResponse.Result == SparkSessionResultType.Succeeded, string.Format( "Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}", sessionCreateResponse.Id, sessionCreateResponse.State, sessionCreateResponse.Result, string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>()) ) ); // Execute Spark statement in the session var sparkStatementOptions = new SparkStatementOptions { Kind = SparkStatementLanguageType.Spark, Code = @"print(""Hello world\n"")" }; SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions); SparkStatement createStatementResponse = await statementOperation.WaitForCompletionAsync(); Assert.NotNull(createStatementResponse); // Verify the Spark statement completes successfully Assert.True("ok".Equals(createStatementResponse.State, StringComparison.OrdinalIgnoreCase), string.Format( "Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}", createStatementResponse.Id, createStatementResponse.State, createStatementResponse.Output.ErrorValue) ); // Verify the output Dictionary <string, string> outputData = JsonSerializer.Deserialize <Dictionary <string, string> >(createStatementResponse.Output.Data as string); Assert.Equals("Hello world", outputData["text/plain"]); // Get the list of Spark statements and check that the executed statement exists Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id); Assert.NotNull(listStatementResponse.Value); Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id)); // Get the list of Spark session and check that the created session exists List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client); Assert.NotNull(listSessionResponse); Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id)); }