Beispiel #1
0
        private void ConnectMenuItem_Click(object sender, RoutedEventArgs e)
        {
            try
            {
                // User login via interactive popup
                // Authenticate using an an Azure AD domain and client ID that is available by default for all Azure subscriptions
                SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
                tenant = this.txtTenantID.Text;
                var nativeClientApp_clientId      = "1950a258-227b-4e31-a9cf-717495945fc2";
                var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientApp_clientId, new Uri("urn:ietf:wg:oauth:2.0:oob"));
                var creds = UserTokenProvider.LoginWithPromptAsync(tenant, activeDirectoryClientSettings).Result;
                this.uploadMenuItem.IsEnabled      = true;
                this.downloadMenuItem.IsEnabled    = true;
                this.listFilesMenuItem.IsEnabled   = true;
                this.getFileInfoMenuItem.IsEnabled = true;

                // Create client objects and set the subscription ID
                adlsClient           = new DataLakeStoreAccountManagementClient(creds);
                adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
            }
            catch (Exception ex)
            {
                MessageBox.Show($"Exception {ex.Message}", "Connect Failure", MessageBoxButton.OK, MessageBoxImage.Error);
            }
        }
Beispiel #2
0
        //Set up clients
        public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
        {
            _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds);
            _adlsClient.SubscriptionId = subscriptionId;

            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
        }
Beispiel #3
0
    static void Main(string[] args)
    {
        _adlsAccountName   = "test333";
        _resourceGroupName = "ivanrg";
        _location          = "East US 2";
        _subId             = "xxxx";

        string TENANT   = "xxxx";
        string CLIENTID = "xxxx";

        System.Uri ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/");
        System.Uri ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/");
        string     secret_key         = "xxxx";
        var        armCreds           = GetCreds_SPI_SecretKey(TENANT, ARM_TOKEN_AUDIENCE, CLIENTID, secret_key);

        // Create client objects and set the subscription ID
        _adlsClient = new DataLakeStoreAccountManagementClient(armCreds)
        {
            SubscriptionId = _subId
        };

        // Create Data Lake Storage Gen1 account
        var adlsParameters = new DataLakeStoreAccount(location: _location);

        _adlsClient.Account.Create(_resourceGroupName, _adlsAccountName, adlsParameters);

        Console.WriteLine("--completed--");
        Console.ReadLine();
    }
 public DataLakeStoreAndFileSystemManagementHelper(TestBase testBase, MockContext context)
 {
     this.testBase                 = testBase;
     resourceManagementClient      = this.testBase.GetResourceManagementClient(context);
     dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreAccountManagementClient(context);
     dataLakeStoreFileSystemClient = this.testBase.GetDataLakeStoreFileSystemManagementClient(context);
 }
Beispiel #5
0
        private void InitializeADLS()
        {
            this.adlsAccountName = ConfigurationManager.AppSettings["adlsAccountName"];
            string subId            = ConfigurationManager.AppSettings["subId"];
            string remoteFolderPath = "/Hackfest/";

            remoteFileName = "test" + partitionID + ".txt";
            remoteFilePath = remoteFolderPath + remoteFileName;

            SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            var domain           = ConfigurationManager.AppSettings["domain"];
            var webApp_clientId  = ConfigurationManager.AppSettings["webApp_clientId"];
            var clientSecret     = ConfigurationManager.AppSettings["clientSecret"];
            var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
            var creds            = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;

            adlsClient                = new DataLakeStoreAccountManagementClient(creds);
            adlsFileSystemClient      = new DataLakeStoreFileSystemManagementClient(creds);
            adlsClient.SubscriptionId = subId;

            //create directory if not already exist
            if (!adlsFileSystemClient.FileSystem.PathExists(adlsAccountName, remoteFolderPath))
            {
                adlsFileSystemClient.FileSystem.Mkdirs(adlsAccountName, remoteFolderPath);
            }
            //create/overwrite the file
            string header = "";
            var    stream = new MemoryStream(Encoding.UTF8.GetBytes(header));

            adlsFileSystemClient.FileSystem.Create(adlsAccountName, remoteFilePath, stream, true);
        }
 public DataLakeAnalyticsManagementHelper(TestBase testBase, MockContext context)
 {
     this.testBase                        = testBase;
     resourceManagementClient             = this.testBase.GetResourceManagementClient(context);
     storageManagementClient              = this.testBase.GetStorageManagementClient(context);
     dataLakeStoreManagementClient        = this.testBase.GetDataLakeStoreAccountManagementClient(context);
     dataLakeAnalyticsManagementClient    = this.testBase.GetDataLakeAnalyticsAccountManagementClient(context);
     dataLakeAnalyticsJobManagementClient = this.testBase.GetDataLakeAnalyticsJobManagementClient(context);
 }
Beispiel #7
0
        //Set up clients
        private void setupClients(TokenCredentials tokenCreds, string subscriptionId)
        {
            //System.NET.Http w wersji 4.0 a nie 4.1
            _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds);
            _adlsClient.SubscriptionId = subscriptionId;

            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
            _adlaJobClient        = new DataLakeAnalyticsJobManagementClient(tokenCreds);
        }
        public DataLakeStoreClient(AzureContext context)
        {
            if (context == null)
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }

            _subscriptionId = context.Subscription.Id;
            _client = DataLakeStoreCmdletBase.CreateAdlsClient<DataLakeStoreAccountManagementClient>(context,
                AzureEnvironment.Endpoint.ResourceManager);
        }
Beispiel #9
0
        static void Main(string[] args)
        {
            adlsAccountName = "shellpocadlsgen1";
            string[] ad = new string[]
            {
                "[dbo].[item_dim].csv",
                "[dbo].[location_dim].csv",
                "[dbo].[manufacturer_dim].csv",
                "[dbo].[Retail_Sales_Transaction_Data].csv",
                "[dbo].[retailer_dim].csv",
                "[dbo].[store_dim].csv"
            };
            string[] src = new string[]
            {
                "item.csv",
                "location.csv",
                "manufacturer.csv",
                "trans.csv",
                "retailer.csv",
                "store.csv"
            };
            subscriptionId = "45110a54-85eb-46b4-8f41-c5d80533039c";
            SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            var domain           = "happiestminds.onmicrosoft.com";
            var clientId         = "98e780fd-9707-4690-a51e-7a05698f3f1e";
            var clientSecret     = "k0kmlvC7soT3cuwnxYC/10G4lOFQuHt0wt3CIFDPrbM=";
            var clientCredential = new ClientCredential(clientId, clientSecret);
            var creds            = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;

            adlsClient                = new DataLakeStoreAccountManagementClient(creds);
            adlsFileSystemClient      = new DataLakeStoreFileSystemManagementClient(creds);
            adlsClient.SubscriptionId = subscriptionId;
            //string sourceFolderPath = @"D:\ADLS\";
            //string sourceFolderPath = @"\\192.168.56.1\adls\";
            string sourceFolderPath = @"\ADLS\";
            string y = DateTime.Now.ToString("yyyy");
            string m = DateTime.Now.ToString("MMMM");
            string d = DateTime.Now.ToString("dd");

            System.Console.WriteLine(y);
            System.Console.WriteLine(m);
            System.Console.WriteLine(d);
            //string dataLakeStoreFolderPath = "/Development/"+y+"/"+m+"/"+d+"/";
            string dataLakeStoreFolderPath = "/Development/" + y + "/" + m + "/10/";

            for (int i = 0; i < 6; i++)
            {
                string sourceFilePath        = Path.Combine(sourceFolderPath, src[i]);
                string dataLakeStoreFilePath = Path.Combine(dataLakeStoreFolderPath, ad[i]);
                //adlsFileSystemClient.FileSystem.UploadFile(adlsAccountName, sourceFilePath, dataLakeStoreFilePath, 1, false, true);
                adlsFileSystemClient.FileSystem.DownloadFile(adlsAccountName, dataLakeStoreFilePath, sourceFilePath, 1, false, true);
            }
            Console.WriteLine("6. Finished!");
        }
        public DataLakeStoreClient(IAzureContext context)
        {
            if (context == null)
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }

            _client = DataLakeStoreCmdletBase.CreateAdlsClient <DataLakeStoreAccountManagementClient>(context,
                                                                                                      AzureEnvironment.Endpoint.ResourceManager);
            _subscriptionId = context.Subscription.GetId();
        }
Beispiel #11
0
        public AdlsHelper(IActivityLogger logger, AdlsInfo adlsInfo)
        {
            _logger   = logger;
            _adlsName = adlsInfo.AdlsName;
            var creds = GetAccountCredentials(adlsInfo);

            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);

            var adlsAccountManagementClient = new DataLakeStoreAccountManagementClient(creds)
            {
                SubscriptionId = adlsInfo.AzureSubscriptionId
            };

            adlsAccountManagementClient.Account.List();
        }
Beispiel #12
0
        //Set up clients
        public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
        {
            _adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds)
            {
                SubscriptionId = subscriptionId
            };

            _adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);

            _adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(tokenCreds);

            _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds)
            {
                SubscriptionId = subscriptionId
            };

            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
        }
Beispiel #13
0
        private static void Main(string[] args)
        {
            _adlsAccountName   = "dipanbappnexus"; // TODO: Replace this value with the name of your existing Data Lake Store account.
            _resourceGroupName = "dipanbappnexus"; // TODO: Replace this value with the name of the resource group containing your Data Lake Store account.
            _location          = "East US 2";
            _subId             = "66935119xxxxxxxxxxxx2-10d7e8e377cf";

            string localFolderPath  = @"C:\WORKAREA\Enlist\GitHub\CodeMonkey\Practise\CreateADLApplication\CreateADLApplication\Data\"; // TODO: Make sure this exists and can be overwritten.
            string localFilePath    = localFolderPath + "file.txt";                                                                     // TODO: Make sure this exists and can be overwritten.
            string remoteFolderPath = "/data_lake_path/";
            string remoteFilePath   = remoteFolderPath + "file.txt";

            try
            {
                // Service principal / appplication authentication with client secret / key
                // Use the client ID of an existing AAD "Web App" application.
                SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
                var domain           = "72f988bf-86f1-xxxxxxxxxx-2d7cd011db47";      //This is the tenant ID
                var webApp_clientId  = "76c6a67b-xxxxxxxxx-08b81c66c654";            //This is the Application ID
                var clientSecret     = "oXW9O+R6Kxxxxxxxxxxxxxx8dC8yhWCg1mYafdGfE="; //This is the secret key
                var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
                var creds            = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;

                // Create client objects and set the subscription ID
                _adlsClient                = new DataLakeStoreAccountManagementClient(creds);
                _adlsFileSystemClient      = new DataLakeStoreFileSystemManagementClient(creds);
                _adlsClient.SubscriptionId = _subId;

                var accounts = ListAdlStoreAccounts();
                foreach (var account in accounts)
                {
                    Console.WriteLine(account.Name.ToString());
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }



            Console.ReadLine();
        }
        public static void Run([TimerTrigger("*/5 * * * * *")] TimerInfo myTimer, [EventHub("eventhub", Connection = "eventhubconnection")] out string eventHubMessage, TraceWriter log)
        {
            log.Info($"C# Timer trigger function executed at: {DateTime.Now}");

            // login to azure ad
            // Service principal / appplication authentication with client secret / key
            // Use the client ID of an existing AAD "Web App" application.
            SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());

            var domain           = "";
            var webApp_clientId  = "";
            var clientSecret     = "";
            var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
            var credsTask        = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential);

            Task.WaitAll(credsTask);
            var creds = credsTask.Result;

            // Create client objects and set the subscription ID
            _adlsClient = new DataLakeStoreAccountManagementClient(creds)
            {
                SubscriptionId = _subId
            };
            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);

            var directory = "testDir";

            CreateDirectory(directory);
            var randomGuid = Guid.NewGuid();
            var filePath   = directory + "\\" + randomGuid + ".txt";

            CreateFile(filePath);

            // notify eventhub queue about the new file;
            var message = new
            {
                Timestamp = DateTime.UtcNow,
                Filename  = filePath
            };

            eventHubMessage = JsonConvert.SerializeObject(message, Formatting.Indented);
        }
Beispiel #15
0
        static async Task MainAsync(string[] args)
        {
            try
            {
                _adlsAccountName   = "uneidel"; // TODO: Replace this value with the name of your existing Data Lake Store account.
                _resourceGroupName = "Sample";  // TODO: Replace this value with the name of the resource group containing your Data Lake Store account.
                _location          = "North Europe";
                _subId             = "69edde7d-b2fe-4eae-911d-8a9a491bee69";

                string localFolderPath  = @"C:\local_path\";                         // TODO: Make sure this exists and can be overwritten.
                string localFilePath    = Path.Combine(localFolderPath, "file.txt"); // TODO: Make sure this exists and can be overwritten.
                string remoteFolderPath = "/";
                string remoteFilePath   = Path.Combine(remoteFolderPath, "ratings.dat");

                // Service principal / appplication authentication with client secret / key
                // Use the client ID of an existing AAD "Web App" application.
                SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());

                var domain           = "microsoft.onmicrosoft.com";
                var webApp_clientId  = "cbf9fd14-9fcb-4316-8362-1b8a987d34cb";
                var clientSecret     = "NsHKalzZYrOou0916K1971WY5DPWGiLTl5tElXLUBlQ=";
                var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
                var creds            = await ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential);

                // Create client objects and set the subscription ID
                _adlsClient = new DataLakeStoreAccountManagementClient(creds)
                {
                    SubscriptionId = _subId
                };
                _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
                //var accs = await ListAdlStoreAccounts();
                DownloadFile(remoteFilePath, "c:\\temp\\foo1.txt");
            }
            catch (Exception ex)
            {
                var foo = ex;
            }
        }
Beispiel #16
0
        private static void Main(string[] args)
        {
            if (args == null || args.Length != 4)
            {
                throw new Exception("Invalid arguments.");
            }

            var dataSource = args[0];
            var year       = int.Parse(args[1]);
            var month      = int.Parse(args[2]);
            var day        = int.Parse(args[3]);

            _adlsAccountName = "aegisdatalakestore";
            // TODO: Replace this value with the name of your existing Data Lake Store account.
            _resourceGroupName = "shieldanalytics";
            // TODO: Replace this value with the name of the resource group containing your Data Lake Store account.
            _location = "East US 2";
            _subId    = "a9038fb7-3b06-4cac-bbf5-755254b9960d";

            const string localFolderPath = @"C:\Data Lake Local\";
            // TODO: Make sure this exists and can be overwritten.
            var          localFilePath    = localFolderPath + "file.txt"; // TODO: Make sure this exists and can be overwritten.
            const string remoteFolderPath = "/data_lake_path/";
            var          remoteFilePath   = remoteFolderPath + "file.txt";

            // User login via interactive popup
            // Use the client ID of an existing AAD "Native Client" application.
            //SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            //const string domain = "ryanair.com";
            //// Replace this string with the user's Azure Active Directory tenant ID or domain name, if needed.
            //const string nativeClientAppClientId = "1950a258-227b-4e31-a9cf-717495945fc2";
            //var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientAppClientId,
            //    new Uri("urn:ietf:wg:oauth:2.0:oob"));
            //var creds = UserTokenProvider.LoginWithPromptAsync(domain, activeDirectoryClientSettings).Result;

            // Service principal / application authentication with certificate
            // Use the client ID and certificate of an existing AAD "Web App" application.

            var fs        = new FileStream(@"C:\Users\mooneyp\Downloads\cert.pfx", FileMode.Open);
            var certBytes = new byte[fs.Length];

            fs.Read(certBytes, 0, (int)fs.Length);
            fs.Close();
            var cert = new X509Certificate2(certBytes, "M3c54n1c4L\"1");

            Console.WriteLine(cert.GetPublicKey());
            Console.WriteLine(cert.GetPublicKeyString());

            //SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            //var domain = "ryanair.com";
            //var webApp_clientId = "e414f5b8-91af-410d-8740-330e077cbb5f";
            //var clientCert = cert;
            //var clientAssertionCertificate = new ClientAssertionCertificate(webApp_clientId, clientCert);
            //var creds =
            //    ApplicationTokenProvider.LoginSilentWithCertificateAsync(domain, clientAssertionCertificate).Result;

            SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
            var domain           = "ryanair.com";
            var webApp_clientId  = "1fd3b517-4200-4f5b-a7c6-1d73727c44d9";
            var clientSecret     = "/BtVXlmItAy3JXKCBx9nOLmiqNabcGcPuQ6ga4CxMa0=";
            var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
            var creds            = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;

            _adlsClient           = new DataLakeStoreAccountManagementClient(creds);
            _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
            _adlaJobClient        = new DataLakeAnalyticsJobManagementClient(creds);

            _adlsClient.SubscriptionId = _subId;

            var jobId = CreateDataSource(dataSource);

            Console.WriteLine("Creating {0}...", dataSource);
            WaitForJob(jobId);
            Console.WriteLine("{0} created.", dataSource);

            Console.WriteLine("Reading files...");
            var files = ListItems($"/events/v5/{year}/{month}/{day}");

            if (!files.Any())
            {
                Console.WriteLine("Nothing to load.");
                Console.ReadLine();
                return;
            }

            var counter = 1;

            foreach (var file in files)
            {
                Console.WriteLine($"Loading {dataSource} file #{counter}...");

                jobId = SubmitJobByPath(
                    localFolderPath + "ExtractScript.txt",
                    file.PathSuffix,
                    dataSource,
                    $"Load {dataSource} file #{counter}");

                WaitForJob(jobId);
                Console.WriteLine($"{dataSource} file #{counter} loaded.");
                counter++;
            }

            Console.WriteLine("All files loaded.");
            Console.ReadLine();
        }
Beispiel #17
0
        static void Main(string[] args)
        {
            string adlaAccountName   = "<ADLA account name>";
            string resourceGroupName = "<resource group name>";
            string subscriptionId    = "<subscription ID>";

            string domain           = "<AAD tenant ID / domain>";
            var    armTokenAudience = new Uri(@"https://management.core.windows.net/");
            var    adlTokenAudience = new Uri(@"https://datalake.azure.net/");
            var    aadTokenAudience = new Uri(@"https://graph.windows.net/");

            // ----------------------------------------
            // Perform authentication to get credentials
            // ----------------------------------------

            // INTERACTIVE WITH CACHE
            var tokenCache = new TokenCache();

            tokenCache.BeforeAccess = BeforeTokenCacheAccess;
            tokenCache.AfterAccess  = AfterTokenCacheAccess;
            var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, tokenCache, PromptBehavior.Auto);
            var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, tokenCache, PromptBehavior.Auto);
            var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, tokenCache, PromptBehavior.Auto);

            // INTERACTIVE WITHOUT CACHE
            // var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, PromptBehavior.Auto);
            // var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, PromptBehavior.Auto);
            // var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, PromptBehavior.Auto);

            // NON-INTERACTIVE WITH SECRET KEY
            // string clientId = "<service principal / application client ID>";
            // string secretKey = "<service principal / application secret key>";
            // var armCreds = GetCredsServicePrincipalSecretKey(domain, armTokenAudience, clientId, secretKey);
            // var adlCreds = GetCredsServicePrincipalSecretKey(domain, adlTokenAudience, clientId, secretKey);
            // var aadCreds = GetCredsServicePrincipalSecretKey(domain, aadTokenAudience, clientId, secretKey);

            // NON-INTERACTIVE WITH CERT
            // string clientId = "<service principal / application client ID>";
            // var certificate = new X509Certificate2(@"<path to (PFX) certificate file>", "<certificate password>");
            // var armCreds = GetCredsServicePrincipalCertificate(domain, armTokenAudience, clientId, certificate);
            // var adlCreds = GetCredsServicePrincipalCertificate(domain, adlTokenAudience, clientId, certificate);
            // var aadCreds = GetCredsServicePrincipalCertificate(domain, aadTokenAudience, clientId, certificate);

            // ----------------------------------------
            // Create the REST clients using the credentials
            // ----------------------------------------

            var adlaAccountClient = new DataLakeAnalyticsAccountManagementClient(armCreds);

            adlaAccountClient.SubscriptionId = subscriptionId;

            var adlsAccountClient = new DataLakeStoreAccountManagementClient(armCreds);

            adlsAccountClient.SubscriptionId = subscriptionId;

            var adlaCatalogClient    = new DataLakeAnalyticsCatalogManagementClient(adlCreds);
            var adlaJobClient        = new DataLakeAnalyticsJobManagementClient(adlCreds);
            var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(adlCreds);

            var graphClient = new GraphRbacManagementClient(aadCreds);

            graphClient.TenantID = domain;

            // ----------------------------------------
            // Perform operations with the REST clients
            // ----------------------------------------

            var account = adlaAccountClient.Account.Get(resourceGroupName, adlaAccountName);

            Console.WriteLine($"My account's location is: {account.Location}!");

            // string upn = "*****@*****.**";
            // string displayName = graphClient.Users.Get(upn).DisplayName;
            // Console.WriteLine($"The display name for {upn} is {displayName}!");

            Console.ReadLine();
        }