public void schedule_job_and_get_status(string jobId)
{
var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
var stats = jobClient.Job.GetStatisticsWithHttpMessagesAsync(TestsConfiguration.AccountAnalytics, Guid.Parse(jobId)).GetAwaiter().GetResult();
stats.Response.IsSuccessStatusCode.Should().BeTrue();
}
public static void SetupClients(ServiceClientCredentials tokenCreds, string subscriptionId)
{
_adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds);
_adlaClient.SubscriptionId = subscriptionId;
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
}
public void create_job_and_pass_parameters()
{
var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
var ji = new JobInformation("a job", JobType.USql, new USqlJobProperties(_script_with_parameters));
// TODO: how to pass parameters marked as external without injection into the script?
//var jobInfo = jobClient.Job.Create(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji);
}
public void create_job()
{
var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
var ji = new JobInformation("a job create_job", JobType.USql, new USqlJobProperties(_script));
var jobInfo = jobClient.Job.CreateWithHttpMessagesAsync(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji).GetAwaiter().GetResult();
jobInfo.Response.IsSuccessStatusCode.Should().BeTrue();
}
public void create_job_using_extension()
{
var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
var ji = new JobInformation("a job create_job_using_extension", JobType.USql, new USqlJobProperties(_script), Guid.Empty);
var jobInfo = jobClient.Job.Create(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji);
jobInfo.Should().NotBeNull();
}
public DataLakeAnalyticsJobManager(ServiceClientCredentials serviceClientCredentials, string subscriptionId, string dlaAccountName)
{
_serviceClientCredentials = serviceClientCredentials;
_subscriptionId = subscriptionId;
_dlaAccountName = dlaAccountName;
_dlaJobClient = new DataLakeAnalyticsJobManagementClient(serviceClientCredentials);
}
public DataLakeAnalyticsManagementHelper(TestBase testBase, MockContext context)
{
this.testBase = testBase;
resourceManagementClient = this.testBase.GetResourceManagementClient(context);
storageManagementClient = this.testBase.GetStorageManagementClient(context);
dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreAccountManagementClient(context);
dataLakeAnalyticsManagementClient = this.testBase.GetDataLakeAnalyticsAccountManagementClient(context);
dataLakeAnalyticsJobManagementClient = this.testBase.GetDataLakeAnalyticsJobManagementClient(context);
}
//Set up clients
private void setupClients(TokenCredentials tokenCreds, string subscriptionId)
{
//System.NET.Http w wersji 4.0 a nie 4.1
_adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds);
_adlsClient.SubscriptionId = subscriptionId;
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);
}
public DataLakeAnalyticsCatalogManagementHelper(TestBase testBase)
{
this.testBase = testBase;
resourceManagementClient = ClientManagementUtilities.GetResourceManagementClient(this.testBase);
dataLakeStoreManagementClient = ClientManagementUtilities.GetDataLakeStoreManagementClient(this.testBase);
dataLakeAnalyticsManagementClient =
ClientManagementUtilities.GetDataLakeAnalyticsManagementClient(this.testBase);
dataLakeAnalyticsJobManagementClient =
ClientManagementUtilities.GetDataLakeAnalyticsJobManagementClient(this.testBase);
}
public DataLakeAnalyticsCatalogManagementHelper(TestBase testBase)
{
this.testBase = testBase;
resourceManagementClient = ClientManagementUtilities.GetResourceManagementClient(this.testBase);
dataLakeStoreManagementClient = ClientManagementUtilities.GetDataLakeStoreManagementClient(this.testBase);
dataLakeAnalyticsManagementClient =
ClientManagementUtilities.GetDataLakeAnalyticsManagementClient(this.testBase);
dataLakeAnalyticsJobManagementClient =
ClientManagementUtilities.GetDataLakeAnalyticsJobManagementClient(this.testBase);
}
protected DataLakeAnalyticsTestsBase()
{
helper = new EnvironmentSetupHelper();
dataLakeAnalyticsManagementClient = GetDataLakeAnalyticsManagementClient();
dataLakeAnalyticsJobManagementClient = GetDataLakeAnalyticsJobManagementClient();
dataLakeAnalyticsCatalogManagementClient = GetDataLakeAnalyticsCatalogManagementClient();
resourceManagementClient = GetResourceManagementClient();
dataLakeStoreManagementClient = GetDataLakeStoreManagementClient();
this.resourceGroupName = TestUtilities.GenerateName("abarg1");
this.dataLakeAnalyticsAccountName = TestUtilities.GenerateName("testaba1");
this.dataLakeAccountName = TestUtilities.GenerateName("datalake01");
}
internal void RunJobToCompletion(DataLakeAnalyticsJobManagementClient jobClient, string resourceGroupName, string dataLakeAnalyticsAccountName, Guid jobIdToUse, string scriptToRun)
{
var createOrBuildParams = new JobInfoBuildOrCreateParameters
{
Job = new JobInformation
{
Name = TestUtilities.GenerateName("testjob1"),
JobId = jobIdToUse,
Type = JobType.USql,
DegreeOfParallelism = 2,
Properties = new USqlProperties
{
Type = JobType.USql,
Script = scriptToRun
}
}
};
var jobCreateResponse = jobClient.Jobs.Create(resourceGroupName,
dataLakeAnalyticsAccountName, createOrBuildParams);
Assert.NotNull(jobCreateResponse);
// Poll the job until it finishes
JobInfoGetResponse getJobResponse = jobClient.Jobs.Get(resourceGroupName,
dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
Assert.NotNull(getJobResponse);
int maxWaitInSeconds = 180; // 3 minutes should be long enough
int curWaitInSeconds = 0;
while (getJobResponse.Job.State != JobState.Ended && curWaitInSeconds < maxWaitInSeconds)
{
// wait 5 seconds before polling again
TestUtilities.Wait(5000);
curWaitInSeconds += 5;
getJobResponse = jobClient.Jobs.Get(resourceGroupName,
dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
Assert.NotNull(getJobResponse);
}
Assert.True(curWaitInSeconds <= maxWaitInSeconds);
// Verify the job completes successfully
Assert.True(
getJobResponse.Job.State == JobState.Ended && getJobResponse.Job.Result == JobResult.Succeeded,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
getJobResponse.Job.JobId, getJobResponse.Job.State, getJobResponse.Job.Result,
getJobResponse.Job.ErrorMessage));
}
public void create_job_and_wait()
{
var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
var ji = new JobInformation("a job create_job_and_wait", JobType.USql, new USqlJobProperties(_script), Guid.Empty);
var jobInfo = jobClient.Job.Create(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji);
bool done = false;
do
{
Thread.Sleep(TimeSpan.FromSeconds(3));
var jobWaitInfo = jobClient.Job.Get(TestsConfiguration.AccountAnalytics, jobInfo.JobId.Value);
done = jobWaitInfo.State == JobState.Ended;
Debug.WriteLine($"State: {jobWaitInfo.State?.ToString() ?? "null"}");
} while (!done);
}
public DataLakeAnalyticsClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_accountClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient <DataLakeAnalyticsAccountManagementClient>(context,
AzureEnvironment.Endpoint.ResourceManager);
_subscriptionId = context.Subscription.Id;
_jobClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient <DataLakeAnalyticsJobManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);
_catalogClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient <DataLakeAnalyticsCatalogManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);
}
public DataLakeAnalyticsClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_accountClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient<DataLakeAnalyticsAccountManagementClient>(context,
AzureEnvironment.Endpoint.ResourceManager);
_subscriptionId = context.Subscription.Id;
_jobClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient<DataLakeAnalyticsJobManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);
_catalogClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient<DataLakeAnalyticsCatalogManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);
}
internal Guid RunJobToCompletion(DataLakeAnalyticsJobManagementClient jobClient, string dataLakeAnalyticsAccountName, Guid jobIdToUse, string scriptToRun)
{
var createOrBuildParams = new JobInformation
{
Name = TestUtilities.GenerateName("testjob1"),
Type = JobType.USql,
DegreeOfParallelism = 2,
Properties = new USqlJobProperties
{
// Type = JobType.USql,
Script = scriptToRun
},
JobId = jobIdToUse
};
var jobCreateResponse = jobClient.Job.Create(dataLakeAnalyticsAccountName, jobIdToUse, createOrBuildParams);
Assert.NotNull(jobCreateResponse);
// Poll the job until it finishes
var getJobResponse = jobClient.Job.Get(dataLakeAnalyticsAccountName, jobCreateResponse.JobId.GetValueOrDefault());
Assert.NotNull(getJobResponse);
int maxWaitInSeconds = 180; // 3 minutes should be long enough
int curWaitInSeconds = 0;
while (getJobResponse.State != JobState.Ended && curWaitInSeconds < maxWaitInSeconds)
{
// wait 5 seconds before polling again
TestUtilities.Wait(5000);
curWaitInSeconds += 5;
getJobResponse = jobClient.Job.Get(dataLakeAnalyticsAccountName, jobCreateResponse.JobId.GetValueOrDefault());
Assert.NotNull(getJobResponse);
}
Assert.True(curWaitInSeconds <= maxWaitInSeconds);
// Verify the job completes successfully
Assert.True(
getJobResponse.State == JobState.Ended && getJobResponse.Result == JobResult.Succeeded,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
getJobResponse.JobId, getJobResponse.State, getJobResponse.Result,
getJobResponse.ErrorMessage != null && getJobResponse.ErrorMessage.Count > 0 ? getJobResponse.ErrorMessage[0].Details : "no error information returned"));
return(getJobResponse.JobId.GetValueOrDefault());
}
//Set up clients
public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
{
_adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds)
{
SubscriptionId = subscriptionId
};
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);
_adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(tokenCreds);
_adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds)
{
SubscriptionId = subscriptionId
};
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "Post", Route = "")] HttpRequestMessage req, TraceWriter log)
{
log.Info("Run USQL Job Azure Function Called.");
var requestBody = new RunUSQLJobBody();
try
{
requestBody = await req.Content.ReadAsAsync <RunUSQLJobBody>();
if (!requestBody.IsValid())
{
throw new ArgumentException("Supplied body contains parameter which is null or empty string.");
}
log.Info("usqlScript:" + requestBody.usql);
string adlaAccountName = ConfigurationManager.AppSettings.Get("adlaAccountName");
string tenantName = ConfigurationManager.AppSettings.Get("tenantName");
string appId = ConfigurationManager.AppSettings.Get("appId");
string appSecret = ConfigurationManager.AppSettings.Get("appSecret");
log.Info("Tenant: " + tenantName);
log.Info("AppId: " + appId);
//authenticate against adla account
var adlCreds = AuthenticateAzure(tenantName, appId, appSecret);
var adlaJobClient = new DataLakeAnalyticsJobManagementClient(adlCreds);
//create usql job frame
var jobId = Guid.NewGuid();
var adlaJobProperties = new USqlJobProperties(requestBody.usql);
var adlaJobParameters = new JobInformation("Auto Procedure Creator - " + requestBody.dataflowname, JobType.USql, adlaJobProperties, priority: 1, degreeOfParallelism: 1, jobId: jobId);
//create job
var jobCreator = adlaJobClient.Job.Create(adlaAccountName, jobId, adlaJobParameters);
return(req.CreateResponse(HttpStatusCode.OK, "ADLA job submitted. Id: " + jobId.ToString()));
}
catch (Exception ex)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, ex));
}
}
// private static DataLakeAnalyticsJobManagementClient _adlaJobClient;
static void Main(string[] args)
{
string secret_key = "oVOdbXKU28RE5oH2symmWXp3sYOoG1geV8x2q4nrgm0=";
string TENANT = "6c5ccde4-18b5-4936-ae88-995adc1f3c22";
string CLIENTID = "0649e374-afa1-44e6-a061-c44944a156d1";
string _adlaAccountName = "socialmediaadla";
Uri ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/");
Uri ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/");
var armCreds = GetCreds_SPI_SecretKey(TENANT, ARM_TOKEN_AUDIENCE, CLIENTID, secret_key);
var adlCreds = GetCreds_SPI_SecretKey(TENANT, ADL_TOKEN_AUDIENCE, CLIENTID, secret_key);
DataLakeAnalyticsJobManagementClient _adlaJobClient = new DataLakeAnalyticsJobManagementClient(armCreds);
var jobId = Guid.NewGuid();
string scriptPath = @"C:\GitRepo\bensonwang743\socialmedia\SocialMediaDataPipeline\ADLASample\usql\SampleUSQLScript.txt";
var script = File.ReadAllText(scriptPath);
var properties = new USqlJobProperties(script);
var parameters = new JobInformation("test", JobType.USql, properties);
var jobInfo = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
}
public DataLakeAnalyticsClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_accountClient = AzureSession.ClientFactory.CreateClient<DataLakeAnalyticsManagementClient>(context,
AzureEnvironment.Endpoint.ResourceManager);
_accountClient.UserAgentSuffix = " - PowerShell Client";
_subscriptionId = context.Subscription.Id;
var creds = AzureSession.AuthenticationFactory.GetSubscriptionCloudCredentials(context);
_jobClient = AzureSession.ClientFactory.CreateCustomClient<DataLakeAnalyticsJobManagementClient>(creds,
context.Environment.GetEndpoint(
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));
_catalogClient =
AzureSession.ClientFactory.CreateCustomClient<DataLakeAnalyticsCatalogManagementClient>(creds,
context.Environment.GetEndpoint(
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));
_jobClient.UserAgentSuffix = " - PowerShell Client";
_catalogClient.UserAgentSuffix = " - PowerShell Client";
}
protected AdlaTestsBase()
{
helper = new EnvironmentSetupHelper();
dataLakeAnalyticsManagementClient = GetDataLakeAnalyticsManagementClient();
dataLakeAnalyticsJobManagementClient = GetDataLakeAnalyticsJobManagementClient();
dataLakeAnalyticsCatalogManagementClient = GetDataLakeAnalyticsCatalogManagementClient();
resourceManagementClient = GetResourceManagementClient();
dataLakeStoreManagementClient = GetDataLakeStoreManagementClient();
storageManagementClient = GetStorageManagementClient();
this.resourceGroupName = TestUtilities.GenerateName("abarg1");
this.dataLakeAnalyticsAccountName = TestUtilities.GenerateName("testaba1");
this.dataLakeStoreAccountName = TestUtilities.GenerateName("datalake01");
this.secondDataLakeStoreAccountName = TestUtilities.GenerateName("datalake02");
this.azureBlobStoreName = TestUtilities.GenerateName("azureblob01");
this.dbName = TestUtilities.GenerateName("adladb01");
this.tableName = TestUtilities.GenerateName("adlatable01");
this.tvfName = TestUtilities.GenerateName("adlatvf01");
this.procName = TestUtilities.GenerateName("adlaproc01");
this.viewName = TestUtilities.GenerateName("adlaview01");
this.secretName = TestUtilities.GenerateName("adlasecret01");
this.secretPwd = TestUtilities.GenerateName("adlasecretpwd01");
this.credName = TestUtilities.GenerateName("adlacred01");
}
public DataLakeAnalyticsClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_accountClient = AzureSession.ClientFactory.CreateClient <DataLakeAnalyticsManagementClient>(context,
AzureEnvironment.Endpoint.ResourceManager);
_accountClient.UserAgentSuffix = " - PowerShell Client";
_subscriptionId = context.Subscription.Id;
var creds = AzureSession.AuthenticationFactory.GetSubscriptionCloudCredentials(context);
_jobClient = AzureSession.ClientFactory.CreateCustomClient <DataLakeAnalyticsJobManagementClient>(creds,
context.Environment.GetEndpoint(
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));
_catalogClient =
AzureSession.ClientFactory.CreateCustomClient <DataLakeAnalyticsCatalogManagementClient>(creds,
context.Environment.GetEndpoint(
AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));
_jobClient.UserAgentSuffix = " - PowerShell Client";
_catalogClient.UserAgentSuffix = " - PowerShell Client";
}
internal void RunJobToCompletion(DataLakeAnalyticsJobManagementClient jobClient, string resourceGroupName, string dataLakeAnalyticsAccountName, Guid jobIdToUse, string scriptToRun)
{
var createOrBuildParams = new JobInfoBuildOrCreateParameters
{
Job = new JobInformation
{
Name = TestUtilities.GenerateName("testjob1"),
JobId = jobIdToUse,
Type = JobType.USql,
DegreeOfParallelism = 2,
Properties = new USqlProperties
{
Type = JobType.USql,
Script = scriptToRun
}
}
};
var jobCreateResponse = jobClient.Jobs.Create(resourceGroupName,
dataLakeAnalyticsAccountName, createOrBuildParams);
Assert.NotNull(jobCreateResponse);
// Poll the job until it finishes
JobInfoGetResponse getJobResponse = jobClient.Jobs.Get(resourceGroupName,
dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
Assert.NotNull(getJobResponse);
int maxWaitInSeconds = 180; // 3 minutes should be long enough
int curWaitInSeconds = 0;
while (getJobResponse.Job.State != JobState.Ended && curWaitInSeconds < maxWaitInSeconds)
{
// wait 5 seconds before polling again
TestUtilities.Wait(5000);
curWaitInSeconds += 5;
getJobResponse = jobClient.Jobs.Get(resourceGroupName,
dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
Assert.NotNull(getJobResponse);
}
Assert.True(curWaitInSeconds <= maxWaitInSeconds);
// Verify the job completes successfully
Assert.True(
getJobResponse.Job.State == JobState.Ended && getJobResponse.Job.Result == JobResult.Succeeded,
string.Format(
"Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
getJobResponse.Job.JobId, getJobResponse.Job.State, getJobResponse.Job.Result,
getJobResponse.Job.ErrorMessage));
}
static void Main(string[] args)
{
string adlaAccountName = "<ADLA account name>";
string resourceGroupName = "<resource group name>";
string subscriptionId = "<subscription ID>";
string domain = "<AAD tenant ID / domain>";
var armTokenAudience = new Uri(@"https://management.core.windows.net/");
var adlTokenAudience = new Uri(@"https://datalake.azure.net/");
var aadTokenAudience = new Uri(@"https://graph.windows.net/");
// ----------------------------------------
// Perform authentication to get credentials
// ----------------------------------------
// INTERACTIVE WITH CACHE
var tokenCache = new TokenCache();
tokenCache.BeforeAccess = BeforeTokenCacheAccess;
tokenCache.AfterAccess = AfterTokenCacheAccess;
var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, tokenCache, PromptBehavior.Auto);
var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, tokenCache, PromptBehavior.Auto);
var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, tokenCache, PromptBehavior.Auto);
// INTERACTIVE WITHOUT CACHE
// var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, PromptBehavior.Auto);
// var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, PromptBehavior.Auto);
// var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, PromptBehavior.Auto);
// NON-INTERACTIVE WITH SECRET KEY
// string clientId = "<service principal / application client ID>";
// string secretKey = "<service principal / application secret key>";
// var armCreds = GetCredsServicePrincipalSecretKey(domain, armTokenAudience, clientId, secretKey);
// var adlCreds = GetCredsServicePrincipalSecretKey(domain, adlTokenAudience, clientId, secretKey);
// var aadCreds = GetCredsServicePrincipalSecretKey(domain, aadTokenAudience, clientId, secretKey);
// NON-INTERACTIVE WITH CERT
// string clientId = "<service principal / application client ID>";
// var certificate = new X509Certificate2(@"<path to (PFX) certificate file>", "<certificate password>");
// var armCreds = GetCredsServicePrincipalCertificate(domain, armTokenAudience, clientId, certificate);
// var adlCreds = GetCredsServicePrincipalCertificate(domain, adlTokenAudience, clientId, certificate);
// var aadCreds = GetCredsServicePrincipalCertificate(domain, aadTokenAudience, clientId, certificate);
// ----------------------------------------
// Create the REST clients using the credentials
// ----------------------------------------
var adlaAccountClient = new DataLakeAnalyticsAccountManagementClient(armCreds);
adlaAccountClient.SubscriptionId = subscriptionId;
var adlsAccountClient = new DataLakeStoreAccountManagementClient(armCreds);
adlsAccountClient.SubscriptionId = subscriptionId;
var adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(adlCreds);
var adlaJobClient = new DataLakeAnalyticsJobManagementClient(adlCreds);
var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(adlCreds);
var graphClient = new GraphRbacManagementClient(aadCreds);
graphClient.TenantID = domain;
// ----------------------------------------
// Perform operations with the REST clients
// ----------------------------------------
var account = adlaAccountClient.Account.Get(resourceGroupName, adlaAccountName);
Console.WriteLine($"My account's location is: {account.Location}!");
// string upn = "*****@*****.**";
// string displayName = graphClient.Users.Get(upn).DisplayName;
// Console.WriteLine($"The display name for {upn} is {displayName}!");
Console.ReadLine();
}
private static void Main(string[] args)
{
if (args == null || args.Length != 4)
{
throw new Exception("Invalid arguments.");
}
var dataSource = args[0];
var year = int.Parse(args[1]);
var month = int.Parse(args[2]);
var day = int.Parse(args[3]);
_adlsAccountName = "aegisdatalakestore";
// TODO: Replace this value with the name of your existing Data Lake Store account.
_resourceGroupName = "shieldanalytics";
// TODO: Replace this value with the name of the resource group containing your Data Lake Store account.
_location = "East US 2";
_subId = "a9038fb7-3b06-4cac-bbf5-755254b9960d";
const string localFolderPath = @"C:\Data Lake Local\";
// TODO: Make sure this exists and can be overwritten.
var localFilePath = localFolderPath + "file.txt"; // TODO: Make sure this exists and can be overwritten.
const string remoteFolderPath = "/data_lake_path/";
var remoteFilePath = remoteFolderPath + "file.txt";
// User login via interactive popup
// Use the client ID of an existing AAD "Native Client" application.
//SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
//const string domain = "ryanair.com";
//// Replace this string with the user's Azure Active Directory tenant ID or domain name, if needed.
//const string nativeClientAppClientId = "1950a258-227b-4e31-a9cf-717495945fc2";
//var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientAppClientId,
// new Uri("urn:ietf:wg:oauth:2.0:oob"));
//var creds = UserTokenProvider.LoginWithPromptAsync(domain, activeDirectoryClientSettings).Result;
// Service principal / application authentication with certificate
// Use the client ID and certificate of an existing AAD "Web App" application.
var fs = new FileStream(@"C:\Users\mooneyp\Downloads\cert.pfx", FileMode.Open);
var certBytes = new byte[fs.Length];
fs.Read(certBytes, 0, (int)fs.Length);
fs.Close();
var cert = new X509Certificate2(certBytes, "M3c54n1c4L\"1");
Console.WriteLine(cert.GetPublicKey());
Console.WriteLine(cert.GetPublicKeyString());
//SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
//var domain = "ryanair.com";
//var webApp_clientId = "e414f5b8-91af-410d-8740-330e077cbb5f";
//var clientCert = cert;
//var clientAssertionCertificate = new ClientAssertionCertificate(webApp_clientId, clientCert);
//var creds =
// ApplicationTokenProvider.LoginSilentWithCertificateAsync(domain, clientAssertionCertificate).Result;
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var domain = "ryanair.com";
var webApp_clientId = "1fd3b517-4200-4f5b-a7c6-1d73727c44d9";
var clientSecret = "/BtVXlmItAy3JXKCBx9nOLmiqNabcGcPuQ6ga4CxMa0=";
var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
_adlsClient = new DataLakeStoreAccountManagementClient(creds);
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(creds);
_adlsClient.SubscriptionId = _subId;
var jobId = CreateDataSource(dataSource);
Console.WriteLine("Creating {0}...", dataSource);
WaitForJob(jobId);
Console.WriteLine("{0} created.", dataSource);
Console.WriteLine("Reading files...");
var files = ListItems($"/events/v5/{year}/{month}/{day}");
if (!files.Any())
{
Console.WriteLine("Nothing to load.");
Console.ReadLine();
return;
}
var counter = 1;
foreach (var file in files)
{
Console.WriteLine($"Loading {dataSource} file #{counter}...");
jobId = SubmitJobByPath(
localFolderPath + "ExtractScript.txt",
file.PathSuffix,
dataSource,
$"Load {dataSource} file #{counter}");
WaitForJob(jobId);
Console.WriteLine($"{dataSource} file #{counter} loaded.");
counter++;
}
Console.WriteLine("All files loaded.");
Console.ReadLine();
}
static void Main(string[] args)
{
//_clientId = "a3bdd828-cb26-41e7-8437-e5d73ced7ab9";
//_clientSecret = "";
//_clientId = "263cd72a-ed4f-4deb-be71-24762bf7b6d2";
//_clientSecret = "";
//_clientId = "86797b72-1a07-4c52-89a6-2cb3320f5cd8";
//_clientSecret = "";
//_clientId = "94439138-c707-45e1-832e-0ddbd85ec671";
//_clientSecret = "";
//_adlaAccountName = "ideas-ppe-c14";
//_adlsAccountName = "ideas-ppe-c14";
//_adlaAccountName = "sandbox-c08";
//_adlsAccountName = "sandbox-c08";
// Authenticate
var authContext = new AuthenticationContext(string.Format("https://login.windows.net/{0}", _tenantId));
// swap these lines to switch between certificate and secret authentication
var clientCredentials = new ClientCredential(_clientId, _clientSecret);
//var clientCredentials = new ClientAssertionCertificate(_clientId, Utility.GetCertificateFromStore(_clientCertificateThumbprint, ""));
var authResult = authContext.AcquireTokenAsync("https://management.core.windows.net/", clientCredentials).GetAwaiter().GetResult();
if (authResult == null)
{
throw new InvalidOperationException("Failed to obtain the JWT token");
}
// Get access token based upon access request.
var tokenCred = new TokenCredentials(authResult.AccessToken);
var internalJobClient = new DataLakeInternalAnalyticsScopeJobManagementClient(tokenCred);
var publicJobClient = new DataLakeAnalyticsJobManagementClient(tokenCred);
var publicStoreClient = new DataLakeStoreFileSystemManagementClient(tokenCred);
string localInputFile = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "files\\origin.tsv");
string localClusterReferenceFile = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "files\\cluster-reference.txt");
publicStoreClient.FileSystem.UploadFile(_adlsAccountName, sourcePath: localInputFile, destinationPath: _ClusterInputFilePath, overwrite: true);
publicStoreClient.FileSystem.UploadFile(_adlsAccountName, sourcePath: localClusterReferenceFile, destinationPath: _ClusterReferenceFilePath, overwrite: true);
// Set External Parameters
Dictionary <string, string> extParameters = new Dictionary <string, string>();
extParameters.Add("UserName", string.Format("\"{0}\"", _userName));
extParameters.Add("Parameter1", "\"ParameterOne\"");
// Job Submission
string scriptFile = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "files\\origin-resources.script");
string scriptText = File.ReadAllText(scriptFile);
string dataRoot = string.Format("adl://{0}.{1}/", _adlaAccountName, _accountSuffix);
string scriptUpdatedFile = string.Empty;
ScopeCompiler.Parser.ParseInformation parserInfo = ScopeClient.Scope.ExtractJobResources(scriptFile, null, dataRoot, authResult.AccessToken, extParameters, " -on adl ", null, out scriptUpdatedFile);
scriptText = File.ReadAllText(scriptUpdatedFile);
// Local Compile
//ScopeClient.Scope.Compile(dataRoot: dataRoot, accessToken: authResult.AccessToken,
//scopeScriptFilePath: scriptUpdatedFile, parameters: extParameters, compilerOptions: null);
List <ScopeJobResource> localRes = parserInfo.GetResources(true).Select(f => new ScopeJobResource(Path.GetFileNameWithoutExtension(f), f)).ToList();
List <ScopeJobResource> clusterRes = parserInfo.GetResources(false).Select(f => new ScopeJobResource(Path.GetFileNameWithoutExtension(f), f)).ToList();
// Submit job
// Includes the recurrence name so that this job can be analyzed in Job Insights in the Azure Portal
Guid relationGuid = Guid.Parse(_tenantId);
var scopeProperties = new CreateScopeJobProperties(scriptText, resources: clusterRes);
var scopeParameters = new CreateScopeJobParameters(JobType.Scope, scopeProperties, _ScopeScriptName, priority: 900, related: new JobRelationshipProperties(relationGuid, recurrenceName: "Submit Scope Example Recurring Job"));
var jobId = Guid.NewGuid();
JobInformation jobInfo = internalJobClient.Extension.Create(_adlaAccountName, jobId, scopeParameters,
new DataLakeStoreAccountInformation(name: _adlaAccountName, suffix: _accountSuffix), localRes);
//JobInformation jobInfo = publicJobClient.Job.Create(_adlaAccountName, jobId, scopeParameters);
Console.WriteLine("{0} Submitted Job Id:{1}", DateTime.Now, jobInfo.JobId);
// Use the public job client to get status
if (publicJobClient.Job.Exists(_adlaAccountName, jobInfo.JobId.Value))
{
// Get Job Information
JobInformation job = publicJobClient.Job.Get(_adlaAccountName, jobInfo.JobId.Value);
// Wait for Job to Finish, with timeout
var timeout = DateTime.UtcNow.AddMinutes(20);
Console.WriteLine("{0} Waiting for job to finish", DateTime.Now);
while (job.Result == JobResult.None && DateTime.UtcNow < timeout)
{
job = publicJobClient.Job.Get(_adlaAccountName, jobInfo.JobId.Value);
Console.WriteLine("{0} - {1}", DateTime.Now, job.State);
System.Threading.Thread.Sleep(5 * 1000);
}
// Cancel if over X minutes
if (DateTime.UtcNow >= timeout)
{
publicJobClient.Job.Cancel(_adlaAccountName, jobInfo.JobId.Value);
}
Console.WriteLine("{0} Job Result: {1}", DateTime.Now, job.Result);
}
if (System.Diagnostics.Debugger.IsAttached)
{
Console.WriteLine("press enter key to exit");
Console.ReadLine();
}
}
