public async Task TestSparkSessionJob()
{
// Start the Spark session
SparkSessionOptions createParams = this.CreateSparkSessionRequestParameters();
SparkSession sessionCreateResponse = (await SparkSessionClient.CreateSparkSessionAsync(createParams)).Value;
// Poll the spark session initialization until it is successful.
SparkSession getSessionResponse = await this.PollSparkSessionAsync(sessionCreateResponse);
// Verify the Spark session completes successfully
Assert.True("idle".Equals(getSessionResponse.State, StringComparison.OrdinalIgnoreCase) && getSessionResponse.Result == SparkSessionResultType.Succeeded,
string.Format(
"Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
getSessionResponse.Id,
getSessionResponse.State,
getSessionResponse.Result,
string.Join(", ", getSessionResponse.Errors ?? new List <SparkServiceError>())
)
);
// Execute Spark statement in the session
SparkStatement createStatementResponse = (await SparkSessionClient.CreateSparkStatementAsync(
getSessionResponse.Id,
new SparkStatementOptions
{
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
})).Value;
SparkStatement getStatementResponse = await this.PollSparkSessionStatementAsync(
getSessionResponse.Id,
createStatementResponse);
Assert.NotNull(getSessionResponse);
// Verify the Spark statement completes successfully
Assert.True("ok".Equals(getStatementResponse.State, StringComparison.OrdinalIgnoreCase),
string.Format(
"Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
getStatementResponse.Id,
getStatementResponse.State,
getStatementResponse.Output.ErrorValue)
);
// Verify the output
Dictionary <string, string> outputData = JsonSerializer.Deserialize <Dictionary <string, string> >(getStatementResponse.Output.Data as string);
Assert.Equals("Hello world", outputData["text/plain"]);
// Get the list of Spark statements and check that the executed statement exists
Response <SparkStatementCollection> listStatementResponse = await SparkSessionClient.GetSparkStatementsAsync(sessionCreateResponse.Id);
Assert.NotNull(listStatementResponse.Value);
Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));
// Get the list of Spark session and check that the created session exists
List <SparkSession> listSessionResponse = await this.ListSparkSessionsAsync();
Assert.NotNull(listSessionResponse);
Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
}
