Пример #1
0
        public void schedule_job_and_get_status(string jobId)
        {
            var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
            var stats     = jobClient.Job.GetStatisticsWithHttpMessagesAsync(TestsConfiguration.AccountAnalytics, Guid.Parse(jobId)).GetAwaiter().GetResult();

            stats.Response.IsSuccessStatusCode.Should().BeTrue();
        }
Пример #2
0
 public static void SetupClients(ServiceClientCredentials tokenCreds, string subscriptionId)
 {
     _adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds);
     _adlaClient.SubscriptionId = subscriptionId;
     _adlaJobClient             = new DataLakeAnalyticsJobManagementClient(tokenCreds);
     _adlsFileSystemClient      = new DataLakeStoreFileSystemManagementClient(tokenCreds);
 }
Пример #3
0
 public void create_job_and_pass_parameters()
 {
     var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
     var ji        = new JobInformation("a job", JobType.USql, new USqlJobProperties(_script_with_parameters));
     // TODO: how to pass parameters marked as external without injection into the script?
     //var jobInfo = jobClient.Job.Create(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji);
 }
Пример #4
0
        public void create_job()
        {
            var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
            var ji        = new JobInformation("a job create_job", JobType.USql, new USqlJobProperties(_script));
            var jobInfo   = jobClient.Job.CreateWithHttpMessagesAsync(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji).GetAwaiter().GetResult();

            jobInfo.Response.IsSuccessStatusCode.Should().BeTrue();
        }
Пример #5
0
        public void create_job_using_extension()
        {
            var jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
            var ji        = new JobInformation("a job create_job_using_extension", JobType.USql, new USqlJobProperties(_script), Guid.Empty);
            var jobInfo   = jobClient.Job.Create(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji);

            jobInfo.Should().NotBeNull();
        }
Пример #6
0
        public DataLakeAnalyticsJobManager(ServiceClientCredentials serviceClientCredentials, string subscriptionId, string dlaAccountName)
        {
            _serviceClientCredentials = serviceClientCredentials;

            _subscriptionId = subscriptionId;
            _dlaAccountName = dlaAccountName;

            _dlaJobClient = new DataLakeAnalyticsJobManagementClient(serviceClientCredentials);
        }
 public DataLakeAnalyticsManagementHelper(TestBase testBase, MockContext context)
 {
     this.testBase                        = testBase;
     resourceManagementClient             = this.testBase.GetResourceManagementClient(context);
     storageManagementClient              = this.testBase.GetStorageManagementClient(context);
     dataLakeStoreManagementClient        = this.testBase.GetDataLakeStoreAccountManagementClient(context);
     dataLakeAnalyticsManagementClient    = this.testBase.GetDataLakeAnalyticsAccountManagementClient(context);
     dataLakeAnalyticsJobManagementClient = this.testBase.GetDataLakeAnalyticsJobManagementClient(context);
 }
Пример #8
0
        //Set up clients
        private void setupClients(TokenCredentials tokenCreds, string subscriptionId)
        {
            //System.NET.Http w wersji 4.0 a nie 4.1
            _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds);
            _adlsClient.SubscriptionId = subscriptionId;

            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
            _adlaJobClient        = new DataLakeAnalyticsJobManagementClient(tokenCreds);
        }
Пример #9
0
 public DataLakeAnalyticsCatalogManagementHelper(TestBase testBase)
 {
     this.testBase                     = testBase;
     resourceManagementClient          = ClientManagementUtilities.GetResourceManagementClient(this.testBase);
     dataLakeStoreManagementClient     = ClientManagementUtilities.GetDataLakeStoreManagementClient(this.testBase);
     dataLakeAnalyticsManagementClient =
         ClientManagementUtilities.GetDataLakeAnalyticsManagementClient(this.testBase);
     dataLakeAnalyticsJobManagementClient =
         ClientManagementUtilities.GetDataLakeAnalyticsJobManagementClient(this.testBase);
 }
 public DataLakeAnalyticsCatalogManagementHelper(TestBase testBase)
 {
     this.testBase = testBase;
     resourceManagementClient = ClientManagementUtilities.GetResourceManagementClient(this.testBase);
     dataLakeStoreManagementClient = ClientManagementUtilities.GetDataLakeStoreManagementClient(this.testBase);
     dataLakeAnalyticsManagementClient =
         ClientManagementUtilities.GetDataLakeAnalyticsManagementClient(this.testBase);
     dataLakeAnalyticsJobManagementClient =
         ClientManagementUtilities.GetDataLakeAnalyticsJobManagementClient(this.testBase);
 }
Пример #11
0
 protected DataLakeAnalyticsTestsBase()
 {
     helper = new EnvironmentSetupHelper();
     dataLakeAnalyticsManagementClient        = GetDataLakeAnalyticsManagementClient();
     dataLakeAnalyticsJobManagementClient     = GetDataLakeAnalyticsJobManagementClient();
     dataLakeAnalyticsCatalogManagementClient = GetDataLakeAnalyticsCatalogManagementClient();
     resourceManagementClient          = GetResourceManagementClient();
     dataLakeStoreManagementClient     = GetDataLakeStoreManagementClient();
     this.resourceGroupName            = TestUtilities.GenerateName("abarg1");
     this.dataLakeAnalyticsAccountName = TestUtilities.GenerateName("testaba1");
     this.dataLakeAccountName          = TestUtilities.GenerateName("datalake01");
 }
Пример #12
0
        internal void RunJobToCompletion(DataLakeAnalyticsJobManagementClient jobClient, string resourceGroupName, string dataLakeAnalyticsAccountName, Guid jobIdToUse, string scriptToRun)
        {
            var createOrBuildParams = new JobInfoBuildOrCreateParameters
            {
                Job = new JobInformation
                {
                    Name  = TestUtilities.GenerateName("testjob1"),
                    JobId = jobIdToUse,
                    Type  = JobType.USql,
                    DegreeOfParallelism = 2,
                    Properties          = new USqlProperties
                    {
                        Type   = JobType.USql,
                        Script = scriptToRun
                    }
                }
            };
            var jobCreateResponse = jobClient.Jobs.Create(resourceGroupName,
                                                          dataLakeAnalyticsAccountName, createOrBuildParams);

            Assert.NotNull(jobCreateResponse);

            // Poll the job until it finishes
            JobInfoGetResponse getJobResponse = jobClient.Jobs.Get(resourceGroupName,
                                                                   dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);

            Assert.NotNull(getJobResponse);

            int maxWaitInSeconds = 180; // 3 minutes should be long enough
            int curWaitInSeconds = 0;

            while (getJobResponse.Job.State != JobState.Ended && curWaitInSeconds < maxWaitInSeconds)
            {
                // wait 5 seconds before polling again
                TestUtilities.Wait(5000);
                curWaitInSeconds += 5;
                getJobResponse    = jobClient.Jobs.Get(resourceGroupName,
                                                       dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
                Assert.NotNull(getJobResponse);
            }

            Assert.True(curWaitInSeconds <= maxWaitInSeconds);

            // Verify the job completes successfully
            Assert.True(
                getJobResponse.Job.State == JobState.Ended && getJobResponse.Job.Result == JobResult.Succeeded,
                string.Format(
                    "Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
                    getJobResponse.Job.JobId, getJobResponse.Job.State, getJobResponse.Job.Result,
                    getJobResponse.Job.ErrorMessage));
        }
Пример #13
0
        public void create_job_and_wait()
        {
            var  jobClient = new DataLakeAnalyticsJobManagementClient(_serviceClientCredentials);
            var  ji        = new JobInformation("a job create_job_and_wait", JobType.USql, new USqlJobProperties(_script), Guid.Empty);
            var  jobInfo   = jobClient.Job.Create(TestsConfiguration.AccountAnalytics, Guid.NewGuid(), ji);
            bool done      = false;

            do
            {
                Thread.Sleep(TimeSpan.FromSeconds(3));
                var jobWaitInfo = jobClient.Job.Get(TestsConfiguration.AccountAnalytics, jobInfo.JobId.Value);

                done = jobWaitInfo.State == JobState.Ended;
                Debug.WriteLine($"State: {jobWaitInfo.State?.ToString() ?? "null"}");
            } while (!done);
        }
        public DataLakeAnalyticsClient(AzureContext context)
        {
            if (context == null)
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }

            _accountClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient <DataLakeAnalyticsAccountManagementClient>(context,
                                                                                                                     AzureEnvironment.Endpoint.ResourceManager);
            _subscriptionId = context.Subscription.Id;

            _jobClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient <DataLakeAnalyticsJobManagementClient>(context,
                                                                                                             AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);

            _catalogClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient <DataLakeAnalyticsCatalogManagementClient>(context,
                                                                                                                     AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);
        }
        public DataLakeAnalyticsClient(AzureContext context)
        {
            if (context == null)
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }
            
            _accountClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient<DataLakeAnalyticsAccountManagementClient>(context,
                AzureEnvironment.Endpoint.ResourceManager);
            _subscriptionId = context.Subscription.Id;

            _jobClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient<DataLakeAnalyticsJobManagementClient>(context,
                AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);

            _catalogClient = DataLakeAnalyticsCmdletBase.CreateAdlaClient<DataLakeAnalyticsCatalogManagementClient>(context,
                AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix, true);
        }
        internal Guid RunJobToCompletion(DataLakeAnalyticsJobManagementClient jobClient, string dataLakeAnalyticsAccountName, Guid jobIdToUse, string scriptToRun)
        {
            var createOrBuildParams = new JobInformation
            {
                Name = TestUtilities.GenerateName("testjob1"),
                Type = JobType.USql,
                DegreeOfParallelism = 2,
                Properties          = new USqlJobProperties
                {
                    // Type = JobType.USql,
                    Script = scriptToRun
                },
                JobId = jobIdToUse
            };
            var jobCreateResponse = jobClient.Job.Create(dataLakeAnalyticsAccountName, jobIdToUse, createOrBuildParams);

            Assert.NotNull(jobCreateResponse);

            // Poll the job until it finishes
            var getJobResponse = jobClient.Job.Get(dataLakeAnalyticsAccountName, jobCreateResponse.JobId.GetValueOrDefault());

            Assert.NotNull(getJobResponse);

            int maxWaitInSeconds = 180; // 3 minutes should be long enough
            int curWaitInSeconds = 0;

            while (getJobResponse.State != JobState.Ended && curWaitInSeconds < maxWaitInSeconds)
            {
                // wait 5 seconds before polling again
                TestUtilities.Wait(5000);
                curWaitInSeconds += 5;
                getJobResponse    = jobClient.Job.Get(dataLakeAnalyticsAccountName, jobCreateResponse.JobId.GetValueOrDefault());
                Assert.NotNull(getJobResponse);
            }

            Assert.True(curWaitInSeconds <= maxWaitInSeconds);

            // Verify the job completes successfully
            Assert.True(
                getJobResponse.State == JobState.Ended && getJobResponse.Result == JobResult.Succeeded,
                string.Format(
                    "Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
                    getJobResponse.JobId, getJobResponse.State, getJobResponse.Result,
                    getJobResponse.ErrorMessage != null && getJobResponse.ErrorMessage.Count > 0 ? getJobResponse.ErrorMessage[0].Details : "no error information returned"));
            return(getJobResponse.JobId.GetValueOrDefault());
        }
Пример #17
0
        //Set up clients
        public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
        {
            _adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds)
            {
                SubscriptionId = subscriptionId
            };

            _adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);

            _adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(tokenCreds);

            _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds)
            {
                SubscriptionId = subscriptionId
            };

            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
        }
Пример #18
0
        public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "Post", Route = "")] HttpRequestMessage req, TraceWriter log)
        {
            log.Info("Run USQL Job Azure Function Called.");

            var requestBody = new RunUSQLJobBody();

            try
            {
                requestBody = await req.Content.ReadAsAsync <RunUSQLJobBody>();

                if (!requestBody.IsValid())
                {
                    throw new ArgumentException("Supplied body contains parameter which is null or empty string.");
                }

                log.Info("usqlScript:" + requestBody.usql);

                string adlaAccountName = ConfigurationManager.AppSettings.Get("adlaAccountName");
                string tenantName      = ConfigurationManager.AppSettings.Get("tenantName");
                string appId           = ConfigurationManager.AppSettings.Get("appId");
                string appSecret       = ConfigurationManager.AppSettings.Get("appSecret");

                log.Info("Tenant: " + tenantName);
                log.Info("AppId: " + appId);

                //authenticate against adla account
                var adlCreds      = AuthenticateAzure(tenantName, appId, appSecret);
                var adlaJobClient = new DataLakeAnalyticsJobManagementClient(adlCreds);

                //create usql job frame
                var jobId             = Guid.NewGuid();
                var adlaJobProperties = new USqlJobProperties(requestBody.usql);
                var adlaJobParameters = new JobInformation("Auto Procedure Creator - " + requestBody.dataflowname, JobType.USql, adlaJobProperties, priority: 1, degreeOfParallelism: 1, jobId: jobId);

                //create job
                var jobCreator = adlaJobClient.Job.Create(adlaAccountName, jobId, adlaJobParameters);

                return(req.CreateResponse(HttpStatusCode.OK, "ADLA job submitted. Id: " + jobId.ToString()));
            }
            catch (Exception ex)
            {
                return(req.CreateResponse(HttpStatusCode.BadRequest, ex));
            }
        }
Пример #19
0
        // private static DataLakeAnalyticsJobManagementClient _adlaJobClient;
        static void Main(string[] args)
        {
            string secret_key         = "oVOdbXKU28RE5oH2symmWXp3sYOoG1geV8x2q4nrgm0=";
            string TENANT             = "6c5ccde4-18b5-4936-ae88-995adc1f3c22";
            string CLIENTID           = "0649e374-afa1-44e6-a061-c44944a156d1";
            string _adlaAccountName   = "socialmediaadla";
            Uri    ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/");
            Uri    ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/");

            var armCreds = GetCreds_SPI_SecretKey(TENANT, ARM_TOKEN_AUDIENCE, CLIENTID, secret_key);
            var adlCreds = GetCreds_SPI_SecretKey(TENANT, ADL_TOKEN_AUDIENCE, CLIENTID, secret_key);
            DataLakeAnalyticsJobManagementClient _adlaJobClient = new DataLakeAnalyticsJobManagementClient(armCreds);
            var    jobId      = Guid.NewGuid();
            string scriptPath = @"C:\GitRepo\bensonwang743\socialmedia\SocialMediaDataPipeline\ADLASample\usql\SampleUSQLScript.txt";
            var    script     = File.ReadAllText(scriptPath);
            var    properties = new USqlJobProperties(script);
            var    parameters = new JobInformation("test", JobType.USql, properties);
            var    jobInfo    = _adlaJobClient.Job.Create(_adlaAccountName, jobId, parameters);
        }
        public DataLakeAnalyticsClient(AzureContext context)
        {
            if (context == null)
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }

            _accountClient = AzureSession.ClientFactory.CreateClient<DataLakeAnalyticsManagementClient>(context,
                AzureEnvironment.Endpoint.ResourceManager);
            _accountClient.UserAgentSuffix = " - PowerShell Client";
            _subscriptionId = context.Subscription.Id;
            var creds = AzureSession.AuthenticationFactory.GetSubscriptionCloudCredentials(context);
            _jobClient = AzureSession.ClientFactory.CreateCustomClient<DataLakeAnalyticsJobManagementClient>(creds,
                context.Environment.GetEndpoint(
                    AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));
            _catalogClient =
                AzureSession.ClientFactory.CreateCustomClient<DataLakeAnalyticsCatalogManagementClient>(creds,
                    context.Environment.GetEndpoint(
                        AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));

            _jobClient.UserAgentSuffix = " - PowerShell Client";
            _catalogClient.UserAgentSuffix = " - PowerShell Client";
        }
Пример #21
0
 protected AdlaTestsBase()
 {
     helper = new EnvironmentSetupHelper();
     dataLakeAnalyticsManagementClient        = GetDataLakeAnalyticsManagementClient();
     dataLakeAnalyticsJobManagementClient     = GetDataLakeAnalyticsJobManagementClient();
     dataLakeAnalyticsCatalogManagementClient = GetDataLakeAnalyticsCatalogManagementClient();
     resourceManagementClient            = GetResourceManagementClient();
     dataLakeStoreManagementClient       = GetDataLakeStoreManagementClient();
     storageManagementClient             = GetStorageManagementClient();
     this.resourceGroupName              = TestUtilities.GenerateName("abarg1");
     this.dataLakeAnalyticsAccountName   = TestUtilities.GenerateName("testaba1");
     this.dataLakeStoreAccountName       = TestUtilities.GenerateName("datalake01");
     this.secondDataLakeStoreAccountName = TestUtilities.GenerateName("datalake02");
     this.azureBlobStoreName             = TestUtilities.GenerateName("azureblob01");
     this.dbName     = TestUtilities.GenerateName("adladb01");
     this.tableName  = TestUtilities.GenerateName("adlatable01");
     this.tvfName    = TestUtilities.GenerateName("adlatvf01");
     this.procName   = TestUtilities.GenerateName("adlaproc01");
     this.viewName   = TestUtilities.GenerateName("adlaview01");
     this.secretName = TestUtilities.GenerateName("adlasecret01");
     this.secretPwd  = TestUtilities.GenerateName("adlasecretpwd01");
     this.credName   = TestUtilities.GenerateName("adlacred01");
 }
        public DataLakeAnalyticsClient(AzureContext context)
        {
            if (context == null)
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }

            _accountClient = AzureSession.ClientFactory.CreateClient <DataLakeAnalyticsManagementClient>(context,
                                                                                                         AzureEnvironment.Endpoint.ResourceManager);
            _accountClient.UserAgentSuffix = " - PowerShell Client";
            _subscriptionId = context.Subscription.Id;
            var creds = AzureSession.AuthenticationFactory.GetSubscriptionCloudCredentials(context);

            _jobClient = AzureSession.ClientFactory.CreateCustomClient <DataLakeAnalyticsJobManagementClient>(creds,
                                                                                                              context.Environment.GetEndpoint(
                                                                                                                  AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));
            _catalogClient =
                AzureSession.ClientFactory.CreateCustomClient <DataLakeAnalyticsCatalogManagementClient>(creds,
                                                                                                         context.Environment.GetEndpoint(
                                                                                                             AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix));

            _jobClient.UserAgentSuffix     = " - PowerShell Client";
            _catalogClient.UserAgentSuffix = " - PowerShell Client";
        }
        internal void RunJobToCompletion(DataLakeAnalyticsJobManagementClient jobClient, string resourceGroupName, string dataLakeAnalyticsAccountName, Guid jobIdToUse, string scriptToRun)
        {
            var createOrBuildParams = new JobInfoBuildOrCreateParameters
            {
                Job = new JobInformation
                {
                    Name = TestUtilities.GenerateName("testjob1"),
                    JobId = jobIdToUse,
                    Type = JobType.USql,
                    DegreeOfParallelism = 2,
                    Properties = new USqlProperties
                    {
                        Type = JobType.USql,
                        Script = scriptToRun
                    }
                }
            };
            var jobCreateResponse = jobClient.Jobs.Create(resourceGroupName,
                dataLakeAnalyticsAccountName, createOrBuildParams);

            Assert.NotNull(jobCreateResponse);

            // Poll the job until it finishes
            JobInfoGetResponse getJobResponse = jobClient.Jobs.Get(resourceGroupName,
                dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
            Assert.NotNull(getJobResponse);

            int maxWaitInSeconds = 180; // 3 minutes should be long enough
            int curWaitInSeconds = 0;
            while (getJobResponse.Job.State != JobState.Ended && curWaitInSeconds < maxWaitInSeconds)
            {
                // wait 5 seconds before polling again
                TestUtilities.Wait(5000);
                curWaitInSeconds += 5;
                getJobResponse = jobClient.Jobs.Get(resourceGroupName,
                    dataLakeAnalyticsAccountName, jobCreateResponse.Job.JobId);
                Assert.NotNull(getJobResponse);
            }

            Assert.True(curWaitInSeconds <= maxWaitInSeconds);

            // Verify the job completes successfully
            Assert.True(
                getJobResponse.Job.State == JobState.Ended && getJobResponse.Job.Result == JobResult.Succeeded,
                string.Format(
                    "Job: {0} did not return success. Current job state: {1}. Actual result: {2}. Error (if any): {3}",
                    getJobResponse.Job.JobId, getJobResponse.Job.State, getJobResponse.Job.Result,
                    getJobResponse.Job.ErrorMessage));
        }
Пример #24
0
        static void Main(string[] args)
        {
            string adlaAccountName   = "<ADLA account name>";
            string resourceGroupName = "<resource group name>";
            string subscriptionId    = "<subscription ID>";

            string domain           = "<AAD tenant ID / domain>";
            var    armTokenAudience = new Uri(@"https://management.core.windows.net/");
            var    adlTokenAudience = new Uri(@"https://datalake.azure.net/");
            var    aadTokenAudience = new Uri(@"https://graph.windows.net/");

            // ----------------------------------------
            // Perform authentication to get credentials
            // ----------------------------------------

            // INTERACTIVE WITH CACHE
            var tokenCache = new TokenCache();

            tokenCache.BeforeAccess = BeforeTokenCacheAccess;
            tokenCache.AfterAccess  = AfterTokenCacheAccess;
            var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, tokenCache, PromptBehavior.Auto);
            var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, tokenCache, PromptBehavior.Auto);
            var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, tokenCache, PromptBehavior.Auto);

            // INTERACTIVE WITHOUT CACHE
            // var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, PromptBehavior.Auto);
            // var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, PromptBehavior.Auto);
            // var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, PromptBehavior.Auto);

            // NON-INTERACTIVE WITH SECRET KEY
            // string clientId = "<service principal / application client ID>";
            // string secretKey = "<service principal / application secret key>";
            // var armCreds = GetCredsServicePrincipalSecretKey(domain, armTokenAudience, clientId, secretKey);
            // var adlCreds = GetCredsServicePrincipalSecretKey(domain, adlTokenAudience, clientId, secretKey);
            // var aadCreds = GetCredsServicePrincipalSecretKey(domain, aadTokenAudience, clientId, secretKey);

            // NON-INTERACTIVE WITH CERT
            // string clientId = "<service principal / application client ID>";
            // var certificate = new X509Certificate2(@"<path to (PFX) certificate file>", "<certificate password>");
            // var armCreds = GetCredsServicePrincipalCertificate(domain, armTokenAudience, clientId, certificate);
            // var adlCreds = GetCredsServicePrincipalCertificate(domain, adlTokenAudience, clientId, certificate);
            // var aadCreds = GetCredsServicePrincipalCertificate(domain, aadTokenAudience, clientId, certificate);

            // ----------------------------------------
            // Create the REST clients using the credentials
            // ----------------------------------------

            var adlaAccountClient = new DataLakeAnalyticsAccountManagementClient(armCreds);

            adlaAccountClient.SubscriptionId = subscriptionId;

            var adlsAccountClient = new DataLakeStoreAccountManagementClient(armCreds);

            adlsAccountClient.SubscriptionId = subscriptionId;

            var adlaCatalogClient    = new DataLakeAnalyticsCatalogManagementClient(adlCreds);
            var adlaJobClient        = new DataLakeAnalyticsJobManagementClient(adlCreds);
            var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(adlCreds);

            var graphClient = new GraphRbacManagementClient(aadCreds);

            graphClient.TenantID = domain;

            // ----------------------------------------
            // Perform operations with the REST clients
            // ----------------------------------------

            var account = adlaAccountClient.Account.Get(resourceGroupName, adlaAccountName);

            Console.WriteLine($"My account's location is: {account.Location}!");

            // string upn = "*****@*****.**";
            // string displayName = graphClient.Users.Get(upn).DisplayName;
            // Console.WriteLine($"The display name for {upn} is {displayName}!");

            Console.ReadLine();
        }
Пример #25
0
        private static void Main(string[] args)
        {
            if (args == null || args.Length != 4)
            {
                throw new Exception("Invalid arguments.");
            }

            var dataSource = args[0];
            var year       = int.Parse(args[1]);
            var month      = int.Parse(args[2]);
            var day        = int.Parse(args[3]);

            _adlsAccountName = "aegisdatalakestore";
            // TODO: Replace this value with the name of your existing Data Lake Store account.
            _resourceGroupName = "shieldanalytics";
            // TODO: Replace this value with the name of the resource group containing your Data Lake Store account.
            _location = "East US 2";
            _subId    = "a9038fb7-3b06-4cac-bbf5-755254b9960d";

            const string localFolderPath = @"C:\Data Lake Local\";
            // TODO: Make sure this exists and can be overwritten.
            var          localFilePath    = localFolderPath + "file.txt"; // TODO: Make sure this exists and can be overwritten.
            const string remoteFolderPath = "/data_lake_path/";
            var          remoteFilePath   = remoteFolderPath + "file.txt";

            // User login via interactive popup
            // Use the client ID of an existing AAD "Native Client" application.
            //SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            //const string domain = "ryanair.com";
            //// Replace this string with the user's Azure Active Directory tenant ID or domain name, if needed.
            //const string nativeClientAppClientId = "1950a258-227b-4e31-a9cf-717495945fc2";
            //var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientAppClientId,
            //    new Uri("urn:ietf:wg:oauth:2.0:oob"));
            //var creds = UserTokenProvider.LoginWithPromptAsync(domain, activeDirectoryClientSettings).Result;

            // Service principal / application authentication with certificate
            // Use the client ID and certificate of an existing AAD "Web App" application.

            var fs        = new FileStream(@"C:\Users\mooneyp\Downloads\cert.pfx", FileMode.Open);
            var certBytes = new byte[fs.Length];

            fs.Read(certBytes, 0, (int)fs.Length);
            fs.Close();
            var cert = new X509Certificate2(certBytes, "M3c54n1c4L\"1");

            Console.WriteLine(cert.GetPublicKey());
            Console.WriteLine(cert.GetPublicKeyString());

            //SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            //var domain = "ryanair.com";
            //var webApp_clientId = "e414f5b8-91af-410d-8740-330e077cbb5f";
            //var clientCert = cert;
            //var clientAssertionCertificate = new ClientAssertionCertificate(webApp_clientId, clientCert);
            //var creds =
            //    ApplicationTokenProvider.LoginSilentWithCertificateAsync(domain, clientAssertionCertificate).Result;

            SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            var domain           = "ryanair.com";
            var webApp_clientId  = "1fd3b517-4200-4f5b-a7c6-1d73727c44d9";
            var clientSecret     = "/BtVXlmItAy3JXKCBx9nOLmiqNabcGcPuQ6ga4CxMa0=";
            var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
            var creds            = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;

            _adlsClient           = new DataLakeStoreAccountManagementClient(creds);
            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
            _adlaJobClient        = new DataLakeAnalyticsJobManagementClient(creds);

            _adlsClient.SubscriptionId = _subId;

            var jobId = CreateDataSource(dataSource);

            Console.WriteLine("Creating {0}...", dataSource);
            WaitForJob(jobId);
            Console.WriteLine("{0} created.", dataSource);

            Console.WriteLine("Reading files...");
            var files = ListItems($"/events/v5/{year}/{month}/{day}");

            if (!files.Any())
            {
                Console.WriteLine("Nothing to load.");
                Console.ReadLine();
                return;
            }

            var counter = 1;

            foreach (var file in files)
            {
                Console.WriteLine($"Loading {dataSource} file #{counter}...");

                jobId = SubmitJobByPath(
                    localFolderPath + "ExtractScript.txt",
                    file.PathSuffix,
                    dataSource,
                    $"Load {dataSource} file #{counter}");

                WaitForJob(jobId);
                Console.WriteLine($"{dataSource} file #{counter} loaded.");
                counter++;
            }

            Console.WriteLine("All files loaded.");
            Console.ReadLine();
        }
Пример #26
0
        static void Main(string[] args)
        {
            //_clientId = "a3bdd828-cb26-41e7-8437-e5d73ced7ab9";
            //_clientSecret = "";

            //_clientId = "263cd72a-ed4f-4deb-be71-24762bf7b6d2";
            //_clientSecret = "";

            //_clientId = "86797b72-1a07-4c52-89a6-2cb3320f5cd8";
            //_clientSecret = "";

            //_clientId = "94439138-c707-45e1-832e-0ddbd85ec671";
            //_clientSecret = "";

            //_adlaAccountName = "ideas-ppe-c14";
            //_adlsAccountName = "ideas-ppe-c14";

            //_adlaAccountName = "sandbox-c08";
            //_adlsAccountName = "sandbox-c08";


            // Authenticate
            var authContext = new AuthenticationContext(string.Format("https://login.windows.net/{0}", _tenantId));

            // swap these lines to switch between certificate and secret authentication
            var clientCredentials = new ClientCredential(_clientId, _clientSecret);
            //var clientCredentials = new ClientAssertionCertificate(_clientId, Utility.GetCertificateFromStore(_clientCertificateThumbprint, ""));

            var authResult = authContext.AcquireTokenAsync("https://management.core.windows.net/", clientCredentials).GetAwaiter().GetResult();

            if (authResult == null)
            {
                throw new InvalidOperationException("Failed to obtain the JWT token");
            }

            // Get access token based upon access request.
            var tokenCred = new TokenCredentials(authResult.AccessToken);

            var internalJobClient = new DataLakeInternalAnalyticsScopeJobManagementClient(tokenCred);
            var publicJobClient   = new DataLakeAnalyticsJobManagementClient(tokenCred);
            var publicStoreClient = new DataLakeStoreFileSystemManagementClient(tokenCred);

            string localInputFile            = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "files\\origin.tsv");
            string localClusterReferenceFile = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "files\\cluster-reference.txt");

            publicStoreClient.FileSystem.UploadFile(_adlsAccountName, sourcePath: localInputFile, destinationPath: _ClusterInputFilePath, overwrite: true);
            publicStoreClient.FileSystem.UploadFile(_adlsAccountName, sourcePath: localClusterReferenceFile, destinationPath: _ClusterReferenceFilePath, overwrite: true);

            // Set External Parameters
            Dictionary <string, string> extParameters = new Dictionary <string, string>();

            extParameters.Add("UserName", string.Format("\"{0}\"", _userName));
            extParameters.Add("Parameter1", "\"ParameterOne\"");

            // Job Submission
            string scriptFile        = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "files\\origin-resources.script");
            string scriptText        = File.ReadAllText(scriptFile);
            string dataRoot          = string.Format("adl://{0}.{1}/", _adlaAccountName, _accountSuffix);
            string scriptUpdatedFile = string.Empty;

            ScopeCompiler.Parser.ParseInformation parserInfo = ScopeClient.Scope.ExtractJobResources(scriptFile, null, dataRoot, authResult.AccessToken, extParameters, " -on adl ", null, out scriptUpdatedFile);
            scriptText = File.ReadAllText(scriptUpdatedFile);

            // Local Compile
            //ScopeClient.Scope.Compile(dataRoot: dataRoot, accessToken: authResult.AccessToken,
            //scopeScriptFilePath: scriptUpdatedFile, parameters: extParameters, compilerOptions: null);

            List <ScopeJobResource> localRes   = parserInfo.GetResources(true).Select(f => new ScopeJobResource(Path.GetFileNameWithoutExtension(f), f)).ToList();
            List <ScopeJobResource> clusterRes = parserInfo.GetResources(false).Select(f => new ScopeJobResource(Path.GetFileNameWithoutExtension(f), f)).ToList();

            // Submit job
            // Includes the recurrence name so that this job can be analyzed in Job Insights in the Azure Portal
            Guid relationGuid    = Guid.Parse(_tenantId);
            var  scopeProperties = new CreateScopeJobProperties(scriptText, resources: clusterRes);
            var  scopeParameters = new CreateScopeJobParameters(JobType.Scope, scopeProperties, _ScopeScriptName, priority: 900, related: new JobRelationshipProperties(relationGuid, recurrenceName: "Submit Scope Example Recurring Job"));

            var            jobId   = Guid.NewGuid();
            JobInformation jobInfo = internalJobClient.Extension.Create(_adlaAccountName, jobId, scopeParameters,
                                                                        new DataLakeStoreAccountInformation(name: _adlaAccountName, suffix: _accountSuffix), localRes);

            //JobInformation jobInfo = publicJobClient.Job.Create(_adlaAccountName, jobId, scopeParameters);

            Console.WriteLine("{0} Submitted Job Id:{1}", DateTime.Now, jobInfo.JobId);

            // Use the public job client to get status
            if (publicJobClient.Job.Exists(_adlaAccountName, jobInfo.JobId.Value))
            {
                // Get Job Information
                JobInformation job = publicJobClient.Job.Get(_adlaAccountName, jobInfo.JobId.Value);

                // Wait for Job to Finish, with timeout
                var timeout = DateTime.UtcNow.AddMinutes(20);
                Console.WriteLine("{0} Waiting for job to finish", DateTime.Now);

                while (job.Result == JobResult.None && DateTime.UtcNow < timeout)
                {
                    job = publicJobClient.Job.Get(_adlaAccountName, jobInfo.JobId.Value);
                    Console.WriteLine("{0} - {1}", DateTime.Now, job.State);
                    System.Threading.Thread.Sleep(5 * 1000);
                }

                // Cancel if over X minutes
                if (DateTime.UtcNow >= timeout)
                {
                    publicJobClient.Job.Cancel(_adlaAccountName, jobInfo.JobId.Value);
                }

                Console.WriteLine("{0} Job Result: {1}", DateTime.Now, job.Result);
            }

            if (System.Diagnostics.Debugger.IsAttached)
            {
                Console.WriteLine("press enter key to exit");
                Console.ReadLine();
            }
        }