public async Task TestSparkBatchJob()
{
SparkBatchClient client = CreateClient();
// Submit the Spark job
SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment);
SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams);
SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync();
// Verify the Spark batch job completes successfully
Assert.True("success".Equals(jobCreateResponse.State, StringComparison.OrdinalIgnoreCase) && jobCreateResponse.Result == SparkBatchJobResultType.Succeeded,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
jobCreateResponse.Id,
jobCreateResponse.State,
jobCreateResponse.Result,
string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Get the list of Spark batch jobs and check that the submitted job exists
List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client);
Assert.NotNull(listJobResponse);
Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id));
}
public async Task TestSparkBatchJobCompletesWhenJobStarts()
{
SparkBatchClient client = CreateClient();
// Submit the Spark job
SparkBatchJobOptions createParams = SparkTestUtilities.CreateSparkJobRequestParameters(Recording, TestEnvironment);
SparkBatchOperation createOperation = await client.StartCreateSparkBatchJobAsync(createParams);
SparkBatchJob jobCreateResponse = await createOperation.WaitForCompletionAsync();
// Verify the Spark batch job submission starts successfully
Assert.True(LivyStates.Starting == jobCreateResponse.State || LivyStates.Running == jobCreateResponse.State || LivyStates.Success == jobCreateResponse.State,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Error (if any): {2}",
jobCreateResponse.Id,
jobCreateResponse.State,
string.Join(", ", jobCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Get the list of Spark batch jobs and check that the submitted job exists
List <SparkBatchJob> listJobResponse = await SparkTestUtilities.ListSparkBatchJobsAsync(client);
Assert.NotNull(listJobResponse);
Assert.IsTrue(listJobResponse.Any(job => job.Id == jobCreateResponse.Id));
}
public async Task TestSparkSessionJobWithPublicConstructor()
{
SparkSessionClient client = CreateClient();
// Start the Spark session
SparkSessionOptions createParams = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording);
SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams);
SparkSessionOperation anotherSessionOperation = InstrumentOperation(new SparkSessionOperation(int.Parse(sessionOperation.Id), client));
SparkSession sessionCreateResponse = await anotherSessionOperation.WaitForCompletionAsync();
// Verify the Spark session completes successfully
Assert.True(LivyStates.Idle == sessionCreateResponse.State,
string.Format(
"Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
sessionCreateResponse.Id,
sessionCreateResponse.State,
sessionCreateResponse.Result,
string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Execute Spark statement in the session
var sparkStatementOptions = new SparkStatementOptions {
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world"")"
};
SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions);
SparkStatementOperation anotherStatementOperation = InstrumentOperation(new SparkStatementOperation(int.Parse(sessionOperation.Id), int.Parse(statementOperation.Id), client));
SparkStatement createStatementResponse = await anotherStatementOperation.WaitForCompletionAsync();
Assert.NotNull(createStatementResponse);
// Verify the Spark statement completes successfully
Assert.True(LivyStatementStates.Available == createStatementResponse.State,
string.Format(
"Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
createStatementResponse.Id,
createStatementResponse.State,
createStatementResponse.Output.ErrorValue)
);
// Verify the output
Dictionary <string, object> outputData = createStatementResponse.Output.Data as Dictionary <string, object>;
Assert.AreEqual("Hello world", outputData["text/plain"] as string);
// Get the list of Spark statements and check that the executed statement exists
Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id);
Assert.NotNull(listStatementResponse.Value);
Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));
// Get the list of Spark session and check that the created session exists
List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client);
Assert.NotNull(listSessionResponse);
Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
}
public async Task TestSparkSessionJob()
{
SparkSessionClient client = CreateClient();
// Start the Spark session
SparkSessionOptions createParams = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording);
SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams);
SparkSession sessionCreateResponse = await sessionOperation.WaitForCompletionAsync();
// Verify the Spark session completes successfully
Assert.True("idle".Equals(sessionCreateResponse.State, StringComparison.OrdinalIgnoreCase) && sessionCreateResponse.Result == SparkSessionResultType.Succeeded,
string.Format(
"Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
sessionCreateResponse.Id,
sessionCreateResponse.State,
sessionCreateResponse.Result,
string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Execute Spark statement in the session
var sparkStatementOptions = new SparkStatementOptions {
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions);
SparkStatement createStatementResponse = await statementOperation.WaitForCompletionAsync();
Assert.NotNull(createStatementResponse);
// Verify the Spark statement completes successfully
Assert.True("ok".Equals(createStatementResponse.State, StringComparison.OrdinalIgnoreCase),
string.Format(
"Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
createStatementResponse.Id,
createStatementResponse.State,
createStatementResponse.Output.ErrorValue)
);
// Verify the output
Dictionary <string, string> outputData = JsonSerializer.Deserialize <Dictionary <string, string> >(createStatementResponse.Output.Data as string);
Assert.Equals("Hello world", outputData["text/plain"]);
// Get the list of Spark statements and check that the executed statement exists
Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id);
Assert.NotNull(listStatementResponse.Value);
Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));
// Get the list of Spark session and check that the created session exists
List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client);
Assert.NotNull(listSessionResponse);
Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
}