public void ExecuteSparkStatementSync()
{
// Environment variable with the Synapse workspace endpoint.
string endpoint = TestEnvironment.EndpointUrl;
// Environment variable with the Synapse Spark pool name.
string sparkPoolName = TestEnvironment.SparkPoolName;
#region Snippet:CreateSparkSessionClient
SparkSessionClient client = new SparkSessionClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:CreateSparkSession
SparkSessionOptions request = new SparkSessionOptions(name: $"session-{Guid.NewGuid()}")
{
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkSession sessionCreated = client.CreateSparkSession(request);
// Waiting session creation completion
sessionCreated = PollSparkSession(client, sessionCreated);
#endregion
#region Snippet:GetSparkSession
SparkSession session = client.GetSparkSession(sessionCreated.Id);
Debug.WriteLine($"Session is returned with name {session.Name} and state {session.State}");
#endregion
#region Snippet:CreateSparkStatement
SparkStatementOptions sparkStatementRequest = new SparkStatementOptions
{
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatement statementCreated = client.CreateSparkStatement(sessionCreated.Id, sparkStatementRequest);
// Wait operation completion
statementCreated = PollSparkStatement(client, sessionCreated.Id, statementCreated);
#endregion
#region Snippet:GetSparkStatement
SparkStatement statement = client.GetSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is returned with id {statement.Id} and state {statement.State}");
#endregion
#region Snippet:CancelSparkStatement
SparkStatementCancellationResult cancellationResult = client.CancelSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is cancelled with message {cancellationResult.Msg}");
#endregion
#region Snippet:CancelSparkSession
Response operation = client.CancelSparkSession(sessionCreated.Id);
#endregion
}
private static SparkStatement PollSparkStatement(
SparkSessionClient client,
int sessionId,
SparkStatement statement)
{
return(Poll(
statement,
s => null,
s => s.State,
s => client.GetSparkStatement(sessionId, s.Id),
ExecutingStates,
isFinalState: false));
}
public void ExecuteSparkStatementSync()
{
#region Snippet:CreateSparkSessionClient
// Replace the strings below with the spark and endpoint information
string sparkPoolName = "<my-spark-pool-name>";
/*@@*/ sparkPoolName = TestEnvironment.SparkPoolName;
string endpoint = "<my-endpoint-url>";
/*@@*/ endpoint = TestEnvironment.EndpointUrl;
SparkSessionClient client = new SparkSessionClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:CreateSparkSession
SparkSessionOptions request = new SparkSessionOptions(name: $"session-{Guid.NewGuid()}")
{
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkSessionOperation createSessionOperation = client.StartCreateSparkSession(request);
while (!createSessionOperation.HasCompleted)
{
System.Threading.Thread.Sleep(2000);
createSessionOperation.UpdateStatus();
}
SparkSession sessionCreated = createSessionOperation.Value;
#endregion
#region Snippet:GetSparkSession
SparkSession session = client.GetSparkSession(sessionCreated.Id);
Debug.WriteLine($"Session is returned with name {session.Name} and state {session.State}");
#endregion
#region Snippet:CreateSparkStatement
SparkStatementOptions sparkStatementRequest = new SparkStatementOptions
{
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatementOperation createStatementOperation = client.StartCreateSparkStatement(sessionCreated.Id, sparkStatementRequest);
while (!createStatementOperation.HasCompleted)
{
System.Threading.Thread.Sleep(2000);
createStatementOperation.UpdateStatus();
}
SparkStatement statementCreated = createStatementOperation.Value;
#endregion
#region Snippet:GetSparkStatement
SparkStatement statement = client.GetSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is returned with id {statement.Id} and state {statement.State}");
#endregion
#region Snippet:CancelSparkStatement
SparkStatementCancellationResult cancellationResult = client.CancelSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is cancelled with message {cancellationResult.Message}");
#endregion
#region Snippet:CancelSparkSession
Response operation = client.CancelSparkSession(sessionCreated.Id);
#endregion
}
public SparkStatement GetSparkSessionStatement(int sessionId, int statementId)
{
return(_sparkSessionClient.GetSparkStatement(sessionId, statementId));
}