public async Task <Guid> SubmitJobAsync(string jobName, string script, Dictionary <string, object> variables = null, bool showScriptOnOutput = false)
{
if (variables != null)
{
StringBuilder sb = new StringBuilder();
foreach (var kvp in variables)
{
sb.AppendLine($"DECLARE @{kvp.Key} = \"{kvp.Value}\";");
}
sb.Append(script);
script = sb.ToString();
}
if (showScriptOnOutput)
{
Console.WriteLine(script);
}
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var jobParameters = new JobInformation(jobName, JobType.USql, properties, priority: 1, degreeOfParallelism: 1, jobId: jobId);
await _dlaJobClient.Job.CreateAsync(_dlaAccountName, jobId, jobParameters);
return(jobId);
}
private static Guid SubmitJobByPath(
string scriptPath,
string fileName,
string dataSource,
string jobName,
int degreeOfParallelism = 250)
{
var script = File.ReadAllText(scriptPath);
script = script.Replace("[FILENAME]", fileName);
script = script.Replace("[DATASOURCE]", dataSource);
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var parameters = new JobInformation(
jobName,
JobType.USql,
properties,
priority: 1,
degreeOfParallelism: degreeOfParallelism,
jobId: jobId);
var jobInfo = _adlaJobClient.Job.Create("shield", jobId, parameters);
return(jobId);
}
// Submit a U-SQL job by providing a path to the script
public static string SubmitJobByPath(string scriptPath, string jobName)
{
var script = File.ReadAllText(scriptPath);
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var parameters = new JobInformation(jobName, JobType.USql, properties, priority: 1000, degreeOfParallelism: 1);
var jobInfo = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
return(jobId.ToString());
}
// Submit a U-SQL job by providing script contents.
// Returns the job ID
public static string SubmitJobByScript(string script, string jobName)
{
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var parameters = new JobInformation(jobName, JobType.USql, properties);
var jobInfo = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
return(jobId.ToString());
}
public Guid SubmitJobByPath(string scriptPath, string jobName)
{
var script = File.ReadAllText(scriptPath);
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var parameters = new JobInformation(jobName, JobType.USql, properties, priority: 1, degreeOfParallelism: 1, jobId: jobId);
var jobInfo = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
Console.WriteLine($"{jobInfo.StartTime}");
return(jobId);
}
public static Guid SubmitJobByPath(string scriptPath, string jobName)
{
System.IO.Stream stream = new System.IO.MemoryStream();
_adlsFileSystemClient.FileSystem.Open(_adlsAccountName, scriptPath).CopyToAsync(stream);
string script = StreamToString(stream);
stream.Dispose();
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var parameters = new JobInformation(jobName, JobType.USql, properties, priority: 1, degreeOfParallelism: 1, jobId: jobId);
var jobInfo = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
return(jobId);
}
public JobResult WaitForJob(Guid jobId)
{
var jobInfo = _adlaJobClient.Job.Get(_adlaAccountName, jobId);
while (jobInfo.State != JobState.Ended)
{
jobInfo = _adlaJobClient.Job.Get(_adlaAccountName, jobId);
USqlJobProperties p = (USqlJobProperties)jobInfo.Properties;
if (p != null)
{
Console.WriteLine($"{jobInfo.Name} - {p.TotalCompilationTime} - {p.TotalPauseTime} - {p.TotalQueuedTime} - {p.TotalRunningTime}");
}
Thread.Sleep(1000);
}
return(jobInfo.Result.Value);
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "Post", Route = "")] HttpRequestMessage req, TraceWriter log)
{
log.Info("Run USQL Job Azure Function Called.");
var requestBody = new RunUSQLJobBody();
try
{
requestBody = await req.Content.ReadAsAsync <RunUSQLJobBody>();
if (!requestBody.IsValid())
{
throw new ArgumentException("Supplied body contains parameter which is null or empty string.");
}
log.Info("usqlScript:" + requestBody.usql);
string adlaAccountName = ConfigurationManager.AppSettings.Get("adlaAccountName");
string tenantName = ConfigurationManager.AppSettings.Get("tenantName");
string appId = ConfigurationManager.AppSettings.Get("appId");
string appSecret = ConfigurationManager.AppSettings.Get("appSecret");
log.Info("Tenant: " + tenantName);
log.Info("AppId: " + appId);
//authenticate against adla account
var adlCreds = AuthenticateAzure(tenantName, appId, appSecret);
var adlaJobClient = new DataLakeAnalyticsJobManagementClient(adlCreds);
//create usql job frame
var jobId = Guid.NewGuid();
var adlaJobProperties = new USqlJobProperties(requestBody.usql);
var adlaJobParameters = new JobInformation("Auto Procedure Creator - " + requestBody.dataflowname, JobType.USql, adlaJobProperties, priority: 1, degreeOfParallelism: 1, jobId: jobId);
//create job
var jobCreator = adlaJobClient.Job.Create(adlaAccountName, jobId, adlaJobParameters);
return(req.CreateResponse(HttpStatusCode.OK, "ADLA job submitted. Id: " + jobId.ToString()));
}
catch (Exception ex)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, ex));
}
}
// private static DataLakeAnalyticsJobManagementClient _adlaJobClient;
static void Main(string[] args)
{
string secret_key = "oVOdbXKU28RE5oH2symmWXp3sYOoG1geV8x2q4nrgm0=";
string TENANT = "6c5ccde4-18b5-4936-ae88-995adc1f3c22";
string CLIENTID = "0649e374-afa1-44e6-a061-c44944a156d1";
string _adlaAccountName = "socialmediaadla";
Uri ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/");
Uri ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/");
var armCreds = GetCreds_SPI_SecretKey(TENANT, ARM_TOKEN_AUDIENCE, CLIENTID, secret_key);
var adlCreds = GetCreds_SPI_SecretKey(TENANT, ADL_TOKEN_AUDIENCE, CLIENTID, secret_key);
DataLakeAnalyticsJobManagementClient _adlaJobClient = new DataLakeAnalyticsJobManagementClient(armCreds);
var jobId = Guid.NewGuid();
string scriptPath = @"C:\GitRepo\bensonwang743\socialmedia\SocialMediaDataPipeline\ADLASample\usql\SampleUSQLScript.txt";
var script = File.ReadAllText(scriptPath);
var properties = new USqlJobProperties(script);
var parameters = new JobInformation("test", JobType.USql, properties);
var jobInfo = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
}
private static Guid CreateDataSource(
string dataSource,
string scriptPath = @"C:\Data Lake Local\CreateScript.txt",
int degreeOfParallelism = 250)
{
var script = File.ReadAllText(scriptPath);
script = script.Replace("[DATASOURCE]", dataSource);
var jobId = Guid.NewGuid();
var properties = new USqlJobProperties(script);
var parameters = new JobInformation(
$"Create {dataSource}",
JobType.USql,
properties,
priority: 1,
degreeOfParallelism: degreeOfParallelism,
jobId: jobId);
_adlaJobClient.Job.Create("shield", jobId, parameters);
return(jobId);
}
public override void ExecuteCmdlet()
{
// error handling for not passing or passing both script and script path
if ((string.IsNullOrEmpty(Script) && string.IsNullOrEmpty(ScriptPath)) ||
(!string.IsNullOrEmpty(Script) && !string.IsNullOrEmpty(ScriptPath)))
{
throw new CloudException(Resources.AmbiguousScriptParameter);
}
// get the script
if (string.IsNullOrEmpty(Script))
{
var powerShellDestinationPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath(ScriptPath);
if (!File.Exists(powerShellDestinationPath))
{
throw new CloudException(string.Format(Resources.ScriptFilePathDoesNotExist,
powerShellDestinationPath));
}
Script = File.ReadAllText(powerShellDestinationPath);
}
JobType jobType;
JobProperties properties;
if (USql)
{
jobType = JobType.USql;
var sqlIpProperties = new USqlJobProperties
{
Script = Script
};
if (!string.IsNullOrEmpty(CompileMode))
{
CompileMode toUse;
if (Enum.TryParse(CompileMode, out toUse))
{
sqlIpProperties.CompileMode = toUse;
}
}
if (!string.IsNullOrEmpty(Runtime))
{
sqlIpProperties.RuntimeVersion = Runtime;
}
properties = sqlIpProperties;
}
else if (Hive)
{
jobType = JobType.Hive;
properties = new HiveJobProperties
{
Script = Script
};
}
else
{
throw new CloudException(Resources.InvalidJobType);
}
var jobInfo = new JobInformation
{
JobId = Guid.NewGuid(),
Name = Name,
Properties = properties,
Type = jobType,
DegreeOfParallelism = DegreeOfParallelism,
Priority = Priority
};
WriteObject(CompileOnly
? DataLakeAnalyticsClient.BuildJob(Account, jobInfo)
: DataLakeAnalyticsClient.SubmitJob(Account, jobInfo));
}
public override void ExecuteCmdlet()
{
// error handling for not passing or passing both script and script path
if ((string.IsNullOrEmpty(Script) && string.IsNullOrEmpty(ScriptPath)) ||
(!string.IsNullOrEmpty(Script) && !string.IsNullOrEmpty(ScriptPath)))
{
throw new CloudException(Resources.AmbiguousScriptParameter);
}
// get the script
if (string.IsNullOrEmpty(Script))
{
var powerShellDestinationPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath(ScriptPath);
if (!File.Exists(powerShellDestinationPath))
{
throw new CloudException(string.Format(Resources.ScriptFilePathDoesNotExist,
powerShellDestinationPath));
}
Script = File.ReadAllText(powerShellDestinationPath);
}
JobType jobType;
JobProperties properties;
if (USql)
{
jobType = JobType.USql;
var sqlIpProperties = new USqlJobProperties
{
Script = Script
};
if (!string.IsNullOrEmpty(CompileMode))
{
CompileMode toUse;
if (Enum.TryParse(CompileMode, out toUse))
{
sqlIpProperties.CompileMode = toUse;
}
}
if (!string.IsNullOrEmpty(Runtime))
{
sqlIpProperties.RuntimeVersion = Runtime;
}
properties = sqlIpProperties;
}
else if (Hive)
{
jobType = JobType.Hive;
properties = new HiveJobProperties
{
Script = Script
};
}
else
{
throw new CloudException(Resources.InvalidJobType);
}
var jobInfo = new JobInformation
{
JobId = DataLakeAnalyticsClient.JobIdQueue.Count == 0 ? Guid.NewGuid() : DataLakeAnalyticsClient.JobIdQueue.Dequeue(),
Name = Name,
Properties = properties,
Type = jobType,
DegreeOfParallelism = DegreeOfParallelism,
Priority = Priority
};
WriteObject(CompileOnly
? DataLakeAnalyticsClient.BuildJob(Account, jobInfo)
: DataLakeAnalyticsClient.SubmitJob(Account, jobInfo));
}
public override void ExecuteCmdlet()
{
if (DegreeOfParallelism < 1)
{
WriteWarning(Resources.InvalidDegreeOfParallelism);
}
// error handling for not passing or passing both script and script path
if ((string.IsNullOrEmpty(Script) && string.IsNullOrEmpty(ScriptPath)) ||
(!string.IsNullOrEmpty(Script) && !string.IsNullOrEmpty(ScriptPath)))
{
throw new CloudException(Resources.AmbiguousScriptParameter);
}
// get the script
if (string.IsNullOrEmpty(Script))
{
var powerShellDestinationPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath(ScriptPath);
if (!File.Exists(powerShellDestinationPath))
{
throw new CloudException(string.Format(Resources.ScriptFilePathDoesNotExist,
powerShellDestinationPath));
}
Script = File.ReadAllText(powerShellDestinationPath);
}
JobType jobType;
JobProperties properties;
if (USql)
{
jobType = JobType.USql;
var sqlIpProperties = new USqlJobProperties
{
Script = Script
};
if (!string.IsNullOrEmpty(CompileMode))
{
CompileMode toUse;
if (Enum.TryParse(CompileMode, out toUse))
{
sqlIpProperties.CompileMode = toUse;
}
}
if (!string.IsNullOrEmpty(Runtime))
{
sqlIpProperties.RuntimeVersion = Runtime;
}
properties = sqlIpProperties;
}
else
{
throw new CloudException(Resources.InvalidJobType);
}
var jobInfo = new JobInformation
(
jobId: DataLakeAnalyticsClient.JobIdQueue.Count == 0 ? Guid.NewGuid() : DataLakeAnalyticsClient.JobIdQueue.Dequeue(),
name: Name,
properties: properties,
type: jobType,
degreeOfParallelism: DegreeOfParallelism,
priority: Priority
);
if (ParameterSetName.Equals(USqlJobParameterSetNameAndRecurrence) ||
ParameterSetName.Equals(USqlJobParameterSetNameAndPipeline) ||
ParameterSetName.Equals(USqlJobWithScriptPathAndRecurrence) ||
ParameterSetName.Equals(USqlJobWithScriptPathAndPipeline))
{
jobInfo.Related = new JobRelationshipProperties
{
RecurrenceId = RecurrenceId,
RecurrenceName = RecurrenceName
};
if (ParameterSetName.Equals(USqlJobParameterSetNameAndPipeline) ||
ParameterSetName.Equals(USqlJobWithScriptPathAndPipeline))
{
jobInfo.Related.PipelineId = PipelineId;
jobInfo.Related.PipelineName = PipelineName;
jobInfo.Related.PipelineUri = PipelineUri;
jobInfo.Related.RunId = RunId;
}
}
WriteObject(CompileOnly
? DataLakeAnalyticsClient.BuildJob(Account, jobInfo)
: DataLakeAnalyticsClient.SubmitJob(Account, jobInfo));
}