public async Task TestSparkSessionJobWithPublicConstructor()
{
SparkSessionClient client = CreateClient();
// Start the Spark session
SparkSessionOptions createParams = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording);
SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams);
SparkSessionOperation anotherSessionOperation = InstrumentOperation(new SparkSessionOperation(int.Parse(sessionOperation.Id), client));
SparkSession sessionCreateResponse = await anotherSessionOperation.WaitForCompletionAsync();
// Verify the Spark session completes successfully
Assert.True(LivyStates.Idle == sessionCreateResponse.State,
string.Format(
"Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
sessionCreateResponse.Id,
sessionCreateResponse.State,
sessionCreateResponse.Result,
string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Execute Spark statement in the session
var sparkStatementOptions = new SparkStatementOptions {
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world"")"
};
SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions);
SparkStatementOperation anotherStatementOperation = InstrumentOperation(new SparkStatementOperation(int.Parse(sessionOperation.Id), int.Parse(statementOperation.Id), client));
SparkStatement createStatementResponse = await anotherStatementOperation.WaitForCompletionAsync();
Assert.NotNull(createStatementResponse);
// Verify the Spark statement completes successfully
Assert.True(LivyStatementStates.Available == createStatementResponse.State,
string.Format(
"Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
createStatementResponse.Id,
createStatementResponse.State,
createStatementResponse.Output.ErrorValue)
);
// Verify the output
Dictionary <string, object> outputData = createStatementResponse.Output.Data as Dictionary <string, object>;
Assert.AreEqual("Hello world", outputData["text/plain"] as string);
// Get the list of Spark statements and check that the executed statement exists
Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id);
Assert.NotNull(listStatementResponse.Value);
Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));
// Get the list of Spark session and check that the created session exists
List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client);
Assert.NotNull(listSessionResponse);
Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
}
public void ExecuteSparkStatementSync()
{
// Environment variable with the Synapse workspace endpoint.
string endpoint = TestEnvironment.EndpointUrl;
// Environment variable with the Synapse Spark pool name.
string sparkPoolName = TestEnvironment.SparkPoolName;
#region Snippet:CreateSparkSessionClient
SparkSessionClient client = new SparkSessionClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:CreateSparkSession
SparkSessionOptions request = new SparkSessionOptions(name: $"session-{Guid.NewGuid()}")
{
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkSession sessionCreated = client.CreateSparkSession(request);
// Waiting session creation completion
sessionCreated = PollSparkSession(client, sessionCreated);
#endregion
#region Snippet:GetSparkSession
SparkSession session = client.GetSparkSession(sessionCreated.Id);
Debug.WriteLine($"Session is returned with name {session.Name} and state {session.State}");
#endregion
#region Snippet:CreateSparkStatement
SparkStatementOptions sparkStatementRequest = new SparkStatementOptions
{
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatement statementCreated = client.CreateSparkStatement(sessionCreated.Id, sparkStatementRequest);
// Wait operation completion
statementCreated = PollSparkStatement(client, sessionCreated.Id, statementCreated);
#endregion
#region Snippet:GetSparkStatement
SparkStatement statement = client.GetSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is returned with id {statement.Id} and state {statement.State}");
#endregion
#region Snippet:CancelSparkStatement
SparkStatementCancellationResult cancellationResult = client.CancelSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is cancelled with message {cancellationResult.Msg}");
#endregion
#region Snippet:CancelSparkSession
Response operation = client.CancelSparkSession(sessionCreated.Id);
#endregion
}
public async Task TestSparkSessionJob()
{
SparkSessionClient client = CreateClient();
// Start the Spark session
SparkSessionOptions createParams = SparkTestUtilities.CreateSparkSessionRequestParameters(Recording);
SparkSessionOperation sessionOperation = await client.StartCreateSparkSessionAsync(createParams);
SparkSession sessionCreateResponse = await sessionOperation.WaitForCompletionAsync();
// Verify the Spark session completes successfully
Assert.True("idle".Equals(sessionCreateResponse.State, StringComparison.OrdinalIgnoreCase) && sessionCreateResponse.Result == SparkSessionResultType.Succeeded,
string.Format(
"Session: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
sessionCreateResponse.Id,
sessionCreateResponse.State,
sessionCreateResponse.Result,
string.Join(", ", sessionCreateResponse.Errors ?? new List <SparkServiceError>())
)
);
// Execute Spark statement in the session
var sparkStatementOptions = new SparkStatementOptions {
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatementOperation statementOperation = await client.StartCreateSparkStatementAsync(sessionCreateResponse.Id, sparkStatementOptions);
SparkStatement createStatementResponse = await statementOperation.WaitForCompletionAsync();
Assert.NotNull(createStatementResponse);
// Verify the Spark statement completes successfully
Assert.True("ok".Equals(createStatementResponse.State, StringComparison.OrdinalIgnoreCase),
string.Format(
"Spark statement: {0} did not return success. Current job state: {1}. Error (if any): {2}",
createStatementResponse.Id,
createStatementResponse.State,
createStatementResponse.Output.ErrorValue)
);
// Verify the output
Dictionary <string, string> outputData = JsonSerializer.Deserialize <Dictionary <string, string> >(createStatementResponse.Output.Data as string);
Assert.Equals("Hello world", outputData["text/plain"]);
// Get the list of Spark statements and check that the executed statement exists
Response <SparkStatementCollection> listStatementResponse = await client.GetSparkStatementsAsync(sessionCreateResponse.Id);
Assert.NotNull(listStatementResponse.Value);
Assert.IsTrue(listStatementResponse.Value.Statements.Any(stmt => stmt.Id == createStatementResponse.Id));
// Get the list of Spark session and check that the created session exists
List <SparkSession> listSessionResponse = await SparkTestUtilities.ListSparkSessionsAsync(client);
Assert.NotNull(listSessionResponse);
Assert.IsTrue(listSessionResponse.Any(session => session.Id == sessionCreateResponse.Id));
}
public SparkSession CreateSparkSession(SparkSessionOptions sparkSessionOptions, bool waitForCompletion)
{
var session = _sparkSessionClient.CreateSparkSession(sparkSessionOptions);
if (!waitForCompletion)
{
return(session);
}
return(PollSparkSession(session));
}
public override void ExecuteCmdlet()
{
if (this.IsParameterBound(c => c.SparkPoolObject))
{
var resourceIdentifier = new ResourceIdentifier(this.SparkPoolObject.Id);
this.WorkspaceName = resourceIdentifier.ParentResource;
this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1);
this.SparkPoolName = resourceIdentifier.ResourceName;
}
if (this.ReferenceFile != null)
{
for (int i = 0; i < this.ReferenceFile.Length; i++)
{
this.ReferenceFile[i] = Utils.NormalizeUrl(this.ReferenceFile[i]);
}
}
Utils.CategorizedFiles(this.ReferenceFile, out IList <string> jars, out IList <string> files);
var livyRequest = new SparkSessionOptions(this.Name)
{
ExecutorMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g",
ExecutorCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores,
DriverMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g",
DriverCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores,
ExecutorCount = this.ExecutorCount
};
jars?.ForEach(item => livyRequest.Jars.Add(item));
files?.ForEach(item => livyRequest.Files.Add(item));
this.Configuration?.ToDictionary()?.ForEach(item => livyRequest.Configuration.Add(item));
if (this.ShouldProcess(this.SparkPoolName, string.Format(Resources.StartingSynapseSparkSession, this.SparkPoolName, this.WorkspaceName)))
{
var sparkSession = SynapseAnalyticsClient.CreateSparkSession(livyRequest);
PSSynapseSparkSession psSparkSession = null;
if (this.IsParameterBound(c => c.Language))
{
this.Language = LanguageType.Parse(this.Language);
psSparkSession = new PSSynapseSparkSession(this.Language, sparkSession);
}
else
{
psSparkSession = new PSSynapseSparkSession(sparkSession);
}
WriteObject(psSparkSession);
}
}
public virtual async Task <Response <SparkSession> > CreateSparkSessionAsync(SparkSessionOptions sparkSessionOptions, bool?detailed = null, CancellationToken cancellationToken = default)
{
using var scope = _clientDiagnostics.CreateScope("SparkSessionClient.CreateSparkSession");
scope.Start();
try
{
return(await RestClient.CreateSparkSessionAsync(sparkSessionOptions, detailed, cancellationToken).ConfigureAwait(false));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public virtual Response <SparkSession> CreateSparkSession(SparkSessionOptions sparkSessionOptions, bool?detailed = null, CancellationToken cancellationToken = default)
{
using var scope = _clientDiagnostics.CreateScope("SparkSessionClient.CreateSparkSession");
scope.Start();
try
{
return(RestClient.CreateSparkSession(sparkSessionOptions, detailed, cancellationToken));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public void ExecuteSparkStatementSync()
{
#region Snippet:CreateSparkSessionClient
// Replace the strings below with the spark and endpoint information
string sparkPoolName = "<my-spark-pool-name>";
/*@@*/ sparkPoolName = TestEnvironment.SparkPoolName;
string endpoint = "<my-endpoint-url>";
/*@@*/ endpoint = TestEnvironment.EndpointUrl;
SparkSessionClient client = new SparkSessionClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:CreateSparkSession
SparkSessionOptions request = new SparkSessionOptions(name: $"session-{Guid.NewGuid()}")
{
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkSessionOperation createSessionOperation = client.StartCreateSparkSession(request);
while (!createSessionOperation.HasCompleted)
{
System.Threading.Thread.Sleep(2000);
createSessionOperation.UpdateStatus();
}
SparkSession sessionCreated = createSessionOperation.Value;
#endregion
#region Snippet:GetSparkSession
SparkSession session = client.GetSparkSession(sessionCreated.Id);
Debug.WriteLine($"Session is returned with name {session.Name} and state {session.State}");
#endregion
#region Snippet:CreateSparkStatement
SparkStatementOptions sparkStatementRequest = new SparkStatementOptions
{
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatementOperation createStatementOperation = client.StartCreateSparkStatement(sessionCreated.Id, sparkStatementRequest);
while (!createStatementOperation.HasCompleted)
{
System.Threading.Thread.Sleep(2000);
createStatementOperation.UpdateStatus();
}
SparkStatement statementCreated = createStatementOperation.Value;
#endregion
#region Snippet:GetSparkStatement
SparkStatement statement = client.GetSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is returned with id {statement.Id} and state {statement.State}");
#endregion
#region Snippet:CancelSparkStatement
SparkStatementCancellationResult cancellationResult = client.CancelSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is cancelled with message {cancellationResult.Message}");
#endregion
#region Snippet:CancelSparkSession
Response operation = client.CancelSparkSession(sessionCreated.Id);
#endregion
}
private async Task <SparkSessionOperation> StartCreateSparkSessionInternal(bool async, SparkSessionOptions sparkSessionOptions, bool?detailed = null, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(SparkSessionClient)}.{nameof(StartCreateSparkSession)}");
scope.Start();
try
{
Response <SparkSession> session;
if (async)
{
session = await RestClient.CreateSparkSessionAsync(sparkSessionOptions, detailed, cancellationToken).ConfigureAwait(false);
}
else
{
session = RestClient.CreateSparkSession(sparkSessionOptions, detailed, cancellationToken);
}
return(new SparkSessionOperation(this, _clientDiagnostics, session));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public virtual SparkSessionOperation StartCreateSparkSession(SparkSessionOptions sparkSessionOptions, bool?detailed = null, CancellationToken cancellationToken = default) => StartCreateSparkSessionInternal(false, sparkSessionOptions, detailed, cancellationToken).EnsureCompleted();
public virtual async Task <SparkSessionOperation> StartCreateSparkSessionAsync(SparkSessionOptions sparkSessionOptions, bool?detailed = null, CancellationToken cancellationToken = default) => await StartCreateSparkSessionInternal(true, sparkSessionOptions, detailed, cancellationToken).ConfigureAwait(false);
public async Task ExecuteSparkStatementSync()
{
#region Snippet:CreateSparkSessionClientAsync
#if SNIPPET
// Replace the strings below with the spark and endpoint information
string sparkPoolName = "<my-spark-pool-name>";
string endpoint = "<my-endpoint-url>";
#else
string sparkPoolName = TestEnvironment.SparkPoolName;
string endpoint = TestEnvironment.EndpointUrl;
#endif
SparkSessionClient client = new SparkSessionClient(new Uri(endpoint), sparkPoolName, new DefaultAzureCredential());
#endregion
#region Snippet:CreateSparkSessionAsync
SparkSessionOptions request = new SparkSessionOptions(name: $"session-{Guid.NewGuid()}")
{
DriverMemory = "28g",
DriverCores = 4,
ExecutorMemory = "28g",
ExecutorCores = 4,
ExecutorCount = 2
};
SparkSessionOperation createSessionOperation = await client.StartCreateSparkSessionAsync(request);
SparkSession sessionCreated = await createSessionOperation.WaitForCompletionAsync();
#endregion
#region Snippet:GetSparkSessionAsync
SparkSession session = await client.GetSparkSessionAsync(sessionCreated.Id);
Debug.WriteLine($"Session is returned with name {session.Name} and state {session.State}");
#endregion
#region Snippet:CreateSparkStatementAsync
SparkStatementOptions sparkStatementRequest = new SparkStatementOptions
{
Kind = SparkStatementLanguageType.Spark,
Code = @"print(""Hello world\n"")"
};
SparkStatementOperation createStatementOperation = await client.StartCreateSparkStatementAsync(sessionCreated.Id, sparkStatementRequest);
SparkStatement statementCreated = await createStatementOperation.WaitForCompletionAsync();
#endregion
#region Snippet:GetSparkStatementAsync
SparkStatement statement = await client.GetSparkStatementAsync(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is returned with id {statement.Id} and state {statement.State}");
#endregion
#region Snippet:CancelSparkStatementAsync
SparkStatementCancellationResult cancellationResult = client.CancelSparkStatement(sessionCreated.Id, statementCreated.Id);
Debug.WriteLine($"Statement is cancelled with message {cancellationResult.Message}");
#endregion
#region Snippet:CancelSparkSessionAsync
Response operation = client.CancelSparkSession(sessionCreated.Id);
#endregion
}
public SparkSession CreateSparkSession(SparkSessionOptions sparkSessionOptions)
{
var session = _sparkSessionClient.StartCreateSparkSession(sparkSessionOptions);
return(session.Poll().Value);
}
