protected override void ProcessRecord()
{
var colour = Console.ForegroundColor;
var executeResponse = _api.ExecuteAsync(new LivyBatchRequest(SparkJar)
{
Args = new string[] { Args },
ClassName = ClassName,
}).Result;
var state = executeResponse.State;
while (state == SparkJobState.Busy || state == SparkJobState.Starting)
{
var state2 = _api.GetBatchStateAsync(executeResponse.SessionId).Result;
WriteObject($"Processing record with state {state2.State}");
}
}
static async Task Main(string[] args)
{
Console.WriteLine("Getting build params ...");
IConfiguration config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true, true)
.Build();
Console.WriteLine("Starting EMR test ...");
Console.WriteLine("Getting batches ...");
var livy = new LivyRestApi(config["Host"]);
var batches = await livy.ListAsync();
Console.WriteLine("Finished getting batches. There are {0}...", batches.Total);
Console.WriteLine("Invoking async software ...");
var response = await livy.ExecuteAsync(new LivyBatchRequest(config["File"])
{
ClassName = config["Class"],
NumExecutors = 2,
DriverCores = 2,
ExecutorCores = 2
});
Console.WriteLine("Software async response is {0}", response.SessionId);
Console.WriteLine("Invoking sync software ...");
var responseSync = await livy.ExecuteWorkflowAsync(new LivyBatchRequest(config["File"])
{
ClassName = config["Class"],
NumExecutors = 2,
DriverCores = 2,
ExecutorCores = 2
}, TimeSpan.FromMinutes(1));
Console.WriteLine("Software async response is {0}", response.SessionId);
Console.WriteLine("Press ENTER to finish ...");
Console.ReadLine();
}
