private void ConnectMenuItem_Click(object sender, RoutedEventArgs e) { try { // User login via interactive popup // Authenticate using an an Azure AD domain and client ID that is available by default for all Azure subscriptions SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); tenant = this.txtTenantID.Text; var nativeClientApp_clientId = "1950a258-227b-4e31-a9cf-717495945fc2"; var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientApp_clientId, new Uri("urn:ietf:wg:oauth:2.0:oob")); var creds = UserTokenProvider.LoginWithPromptAsync(tenant, activeDirectoryClientSettings).Result; this.uploadMenuItem.IsEnabled = true; this.downloadMenuItem.IsEnabled = true; this.listFilesMenuItem.IsEnabled = true; this.getFileInfoMenuItem.IsEnabled = true; // Create client objects and set the subscription ID adlsClient = new DataLakeStoreAccountManagementClient(creds); adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); } catch (Exception ex) { MessageBox.Show($"Exception {ex.Message}", "Connect Failure", MessageBoxButton.OK, MessageBoxImage.Error); } }
//Set up clients public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId) { _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds); _adlsClient.SubscriptionId = subscriptionId; _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds); }
static void Main(string[] args) { _adlsAccountName = "test333"; _resourceGroupName = "ivanrg"; _location = "East US 2"; _subId = "xxxx"; string TENANT = "xxxx"; string CLIENTID = "xxxx"; System.Uri ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/"); System.Uri ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/"); string secret_key = "xxxx"; var armCreds = GetCreds_SPI_SecretKey(TENANT, ARM_TOKEN_AUDIENCE, CLIENTID, secret_key); // Create client objects and set the subscription ID _adlsClient = new DataLakeStoreAccountManagementClient(armCreds) { SubscriptionId = _subId }; // Create Data Lake Storage Gen1 account var adlsParameters = new DataLakeStoreAccount(location: _location); _adlsClient.Account.Create(_resourceGroupName, _adlsAccountName, adlsParameters); Console.WriteLine("--completed--"); Console.ReadLine(); }
public DataLakeStoreAndFileSystemManagementHelper(TestBase testBase, MockContext context) { this.testBase = testBase; resourceManagementClient = this.testBase.GetResourceManagementClient(context); dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreAccountManagementClient(context); dataLakeStoreFileSystemClient = this.testBase.GetDataLakeStoreFileSystemManagementClient(context); }
private void InitializeADLS() { this.adlsAccountName = ConfigurationManager.AppSettings["adlsAccountName"]; string subId = ConfigurationManager.AppSettings["subId"]; string remoteFolderPath = "/Hackfest/"; remoteFileName = "test" + partitionID + ".txt"; remoteFilePath = remoteFolderPath + remoteFileName; SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var domain = ConfigurationManager.AppSettings["domain"]; var webApp_clientId = ConfigurationManager.AppSettings["webApp_clientId"]; var clientSecret = ConfigurationManager.AppSettings["clientSecret"]; var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result; adlsClient = new DataLakeStoreAccountManagementClient(creds); adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); adlsClient.SubscriptionId = subId; //create directory if not already exist if (!adlsFileSystemClient.FileSystem.PathExists(adlsAccountName, remoteFolderPath)) { adlsFileSystemClient.FileSystem.Mkdirs(adlsAccountName, remoteFolderPath); } //create/overwrite the file string header = ""; var stream = new MemoryStream(Encoding.UTF8.GetBytes(header)); adlsFileSystemClient.FileSystem.Create(adlsAccountName, remoteFilePath, stream, true); }
public DataLakeAnalyticsManagementHelper(TestBase testBase, MockContext context) { this.testBase = testBase; resourceManagementClient = this.testBase.GetResourceManagementClient(context); storageManagementClient = this.testBase.GetStorageManagementClient(context); dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreAccountManagementClient(context); dataLakeAnalyticsManagementClient = this.testBase.GetDataLakeAnalyticsAccountManagementClient(context); dataLakeAnalyticsJobManagementClient = this.testBase.GetDataLakeAnalyticsJobManagementClient(context); }
//Set up clients private void setupClients(TokenCredentials tokenCreds, string subscriptionId) { //System.NET.Http w wersji 4.0 a nie 4.1 _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds); _adlsClient.SubscriptionId = subscriptionId; _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds); _adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds); }
public DataLakeStoreClient(AzureContext context) { if (context == null) { throw new ApplicationException(Resources.InvalidDefaultSubscription); } _subscriptionId = context.Subscription.Id; _client = DataLakeStoreCmdletBase.CreateAdlsClient<DataLakeStoreAccountManagementClient>(context, AzureEnvironment.Endpoint.ResourceManager); }
static void Main(string[] args) { adlsAccountName = "shellpocadlsgen1"; string[] ad = new string[] { "[dbo].[item_dim].csv", "[dbo].[location_dim].csv", "[dbo].[manufacturer_dim].csv", "[dbo].[Retail_Sales_Transaction_Data].csv", "[dbo].[retailer_dim].csv", "[dbo].[store_dim].csv" }; string[] src = new string[] { "item.csv", "location.csv", "manufacturer.csv", "trans.csv", "retailer.csv", "store.csv" }; subscriptionId = "45110a54-85eb-46b4-8f41-c5d80533039c"; SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var domain = "happiestminds.onmicrosoft.com"; var clientId = "98e780fd-9707-4690-a51e-7a05698f3f1e"; var clientSecret = "k0kmlvC7soT3cuwnxYC/10G4lOFQuHt0wt3CIFDPrbM="; var clientCredential = new ClientCredential(clientId, clientSecret); var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result; adlsClient = new DataLakeStoreAccountManagementClient(creds); adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); adlsClient.SubscriptionId = subscriptionId; //string sourceFolderPath = @"D:\ADLS\"; //string sourceFolderPath = @"\\192.168.56.1\adls\"; string sourceFolderPath = @"\ADLS\"; string y = DateTime.Now.ToString("yyyy"); string m = DateTime.Now.ToString("MMMM"); string d = DateTime.Now.ToString("dd"); System.Console.WriteLine(y); System.Console.WriteLine(m); System.Console.WriteLine(d); //string dataLakeStoreFolderPath = "/Development/"+y+"/"+m+"/"+d+"/"; string dataLakeStoreFolderPath = "/Development/" + y + "/" + m + "/10/"; for (int i = 0; i < 6; i++) { string sourceFilePath = Path.Combine(sourceFolderPath, src[i]); string dataLakeStoreFilePath = Path.Combine(dataLakeStoreFolderPath, ad[i]); //adlsFileSystemClient.FileSystem.UploadFile(adlsAccountName, sourceFilePath, dataLakeStoreFilePath, 1, false, true); adlsFileSystemClient.FileSystem.DownloadFile(adlsAccountName, dataLakeStoreFilePath, sourceFilePath, 1, false, true); } Console.WriteLine("6. Finished!"); }
public DataLakeStoreClient(IAzureContext context) { if (context == null) { throw new ApplicationException(Resources.InvalidDefaultSubscription); } _client = DataLakeStoreCmdletBase.CreateAdlsClient <DataLakeStoreAccountManagementClient>(context, AzureEnvironment.Endpoint.ResourceManager); _subscriptionId = context.Subscription.GetId(); }
public AdlsHelper(IActivityLogger logger, AdlsInfo adlsInfo) { _logger = logger; _adlsName = adlsInfo.AdlsName; var creds = GetAccountCredentials(adlsInfo); _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); var adlsAccountManagementClient = new DataLakeStoreAccountManagementClient(creds) { SubscriptionId = adlsInfo.AzureSubscriptionId }; adlsAccountManagementClient.Account.List(); }
//Set up clients public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId) { _adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds) { SubscriptionId = subscriptionId }; _adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds); _adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(tokenCreds); _adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds) { SubscriptionId = subscriptionId }; _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds); }
private static void Main(string[] args) { _adlsAccountName = "dipanbappnexus"; // TODO: Replace this value with the name of your existing Data Lake Store account. _resourceGroupName = "dipanbappnexus"; // TODO: Replace this value with the name of the resource group containing your Data Lake Store account. _location = "East US 2"; _subId = "66935119xxxxxxxxxxxx2-10d7e8e377cf"; string localFolderPath = @"C:\WORKAREA\Enlist\GitHub\CodeMonkey\Practise\CreateADLApplication\CreateADLApplication\Data\"; // TODO: Make sure this exists and can be overwritten. string localFilePath = localFolderPath + "file.txt"; // TODO: Make sure this exists and can be overwritten. string remoteFolderPath = "/data_lake_path/"; string remoteFilePath = remoteFolderPath + "file.txt"; try { // Service principal / appplication authentication with client secret / key // Use the client ID of an existing AAD "Web App" application. SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var domain = "72f988bf-86f1-xxxxxxxxxx-2d7cd011db47"; //This is the tenant ID var webApp_clientId = "76c6a67b-xxxxxxxxx-08b81c66c654"; //This is the Application ID var clientSecret = "oXW9O+R6Kxxxxxxxxxxxxxx8dC8yhWCg1mYafdGfE="; //This is the secret key var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result; // Create client objects and set the subscription ID _adlsClient = new DataLakeStoreAccountManagementClient(creds); _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); _adlsClient.SubscriptionId = _subId; var accounts = ListAdlStoreAccounts(); foreach (var account in accounts) { Console.WriteLine(account.Name.ToString()); } } catch (Exception ex) { Console.WriteLine(ex.Message); } Console.ReadLine(); }
public static void Run([TimerTrigger("*/5 * * * * *")] TimerInfo myTimer, [EventHub("eventhub", Connection = "eventhubconnection")] out string eventHubMessage, TraceWriter log) { log.Info($"C# Timer trigger function executed at: {DateTime.Now}"); // login to azure ad // Service principal / appplication authentication with client secret / key // Use the client ID of an existing AAD "Web App" application. SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var domain = ""; var webApp_clientId = ""; var clientSecret = ""; var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var credsTask = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential); Task.WaitAll(credsTask); var creds = credsTask.Result; // Create client objects and set the subscription ID _adlsClient = new DataLakeStoreAccountManagementClient(creds) { SubscriptionId = _subId }; _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); var directory = "testDir"; CreateDirectory(directory); var randomGuid = Guid.NewGuid(); var filePath = directory + "\\" + randomGuid + ".txt"; CreateFile(filePath); // notify eventhub queue about the new file; var message = new { Timestamp = DateTime.UtcNow, Filename = filePath }; eventHubMessage = JsonConvert.SerializeObject(message, Formatting.Indented); }
static async Task MainAsync(string[] args) { try { _adlsAccountName = "uneidel"; // TODO: Replace this value with the name of your existing Data Lake Store account. _resourceGroupName = "Sample"; // TODO: Replace this value with the name of the resource group containing your Data Lake Store account. _location = "North Europe"; _subId = "69edde7d-b2fe-4eae-911d-8a9a491bee69"; string localFolderPath = @"C:\local_path\"; // TODO: Make sure this exists and can be overwritten. string localFilePath = Path.Combine(localFolderPath, "file.txt"); // TODO: Make sure this exists and can be overwritten. string remoteFolderPath = "/"; string remoteFilePath = Path.Combine(remoteFolderPath, "ratings.dat"); // Service principal / appplication authentication with client secret / key // Use the client ID of an existing AAD "Web App" application. SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var domain = "microsoft.onmicrosoft.com"; var webApp_clientId = "cbf9fd14-9fcb-4316-8362-1b8a987d34cb"; var clientSecret = "NsHKalzZYrOou0916K1971WY5DPWGiLTl5tElXLUBlQ="; var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var creds = await ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential); // Create client objects and set the subscription ID _adlsClient = new DataLakeStoreAccountManagementClient(creds) { SubscriptionId = _subId }; _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); //var accs = await ListAdlStoreAccounts(); DownloadFile(remoteFilePath, "c:\\temp\\foo1.txt"); } catch (Exception ex) { var foo = ex; } }
private static void Main(string[] args) { if (args == null || args.Length != 4) { throw new Exception("Invalid arguments."); } var dataSource = args[0]; var year = int.Parse(args[1]); var month = int.Parse(args[2]); var day = int.Parse(args[3]); _adlsAccountName = "aegisdatalakestore"; // TODO: Replace this value with the name of your existing Data Lake Store account. _resourceGroupName = "shieldanalytics"; // TODO: Replace this value with the name of the resource group containing your Data Lake Store account. _location = "East US 2"; _subId = "a9038fb7-3b06-4cac-bbf5-755254b9960d"; const string localFolderPath = @"C:\Data Lake Local\"; // TODO: Make sure this exists and can be overwritten. var localFilePath = localFolderPath + "file.txt"; // TODO: Make sure this exists and can be overwritten. const string remoteFolderPath = "/data_lake_path/"; var remoteFilePath = remoteFolderPath + "file.txt"; // User login via interactive popup // Use the client ID of an existing AAD "Native Client" application. //SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); //const string domain = "ryanair.com"; //// Replace this string with the user's Azure Active Directory tenant ID or domain name, if needed. //const string nativeClientAppClientId = "1950a258-227b-4e31-a9cf-717495945fc2"; //var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientAppClientId, // new Uri("urn:ietf:wg:oauth:2.0:oob")); //var creds = UserTokenProvider.LoginWithPromptAsync(domain, activeDirectoryClientSettings).Result; // Service principal / application authentication with certificate // Use the client ID and certificate of an existing AAD "Web App" application. var fs = new FileStream(@"C:\Users\mooneyp\Downloads\cert.pfx", FileMode.Open); var certBytes = new byte[fs.Length]; fs.Read(certBytes, 0, (int)fs.Length); fs.Close(); var cert = new X509Certificate2(certBytes, "M3c54n1c4L\"1"); Console.WriteLine(cert.GetPublicKey()); Console.WriteLine(cert.GetPublicKeyString()); //SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); //var domain = "ryanair.com"; //var webApp_clientId = "e414f5b8-91af-410d-8740-330e077cbb5f"; //var clientCert = cert; //var clientAssertionCertificate = new ClientAssertionCertificate(webApp_clientId, clientCert); //var creds = // ApplicationTokenProvider.LoginSilentWithCertificateAsync(domain, clientAssertionCertificate).Result; SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var domain = "ryanair.com"; var webApp_clientId = "1fd3b517-4200-4f5b-a7c6-1d73727c44d9"; var clientSecret = "/BtVXlmItAy3JXKCBx9nOLmiqNabcGcPuQ6ga4CxMa0="; var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result; _adlsClient = new DataLakeStoreAccountManagementClient(creds); _adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); _adlaJobClient = new DataLakeAnalyticsJobManagementClient(creds); _adlsClient.SubscriptionId = _subId; var jobId = CreateDataSource(dataSource); Console.WriteLine("Creating {0}...", dataSource); WaitForJob(jobId); Console.WriteLine("{0} created.", dataSource); Console.WriteLine("Reading files..."); var files = ListItems($"/events/v5/{year}/{month}/{day}"); if (!files.Any()) { Console.WriteLine("Nothing to load."); Console.ReadLine(); return; } var counter = 1; foreach (var file in files) { Console.WriteLine($"Loading {dataSource} file #{counter}..."); jobId = SubmitJobByPath( localFolderPath + "ExtractScript.txt", file.PathSuffix, dataSource, $"Load {dataSource} file #{counter}"); WaitForJob(jobId); Console.WriteLine($"{dataSource} file #{counter} loaded."); counter++; } Console.WriteLine("All files loaded."); Console.ReadLine(); }
static void Main(string[] args) { string adlaAccountName = "<ADLA account name>"; string resourceGroupName = "<resource group name>"; string subscriptionId = "<subscription ID>"; string domain = "<AAD tenant ID / domain>"; var armTokenAudience = new Uri(@"https://management.core.windows.net/"); var adlTokenAudience = new Uri(@"https://datalake.azure.net/"); var aadTokenAudience = new Uri(@"https://graph.windows.net/"); // ---------------------------------------- // Perform authentication to get credentials // ---------------------------------------- // INTERACTIVE WITH CACHE var tokenCache = new TokenCache(); tokenCache.BeforeAccess = BeforeTokenCacheAccess; tokenCache.AfterAccess = AfterTokenCacheAccess; var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, tokenCache, PromptBehavior.Auto); var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, tokenCache, PromptBehavior.Auto); var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, tokenCache, PromptBehavior.Auto); // INTERACTIVE WITHOUT CACHE // var armCreds = GetCredsInteractivePopup(domain, armTokenAudience, PromptBehavior.Auto); // var adlCreds = GetCredsInteractivePopup(domain, adlTokenAudience, PromptBehavior.Auto); // var aadCreds = GetCredsInteractivePopup(domain, aadTokenAudience, PromptBehavior.Auto); // NON-INTERACTIVE WITH SECRET KEY // string clientId = "<service principal / application client ID>"; // string secretKey = "<service principal / application secret key>"; // var armCreds = GetCredsServicePrincipalSecretKey(domain, armTokenAudience, clientId, secretKey); // var adlCreds = GetCredsServicePrincipalSecretKey(domain, adlTokenAudience, clientId, secretKey); // var aadCreds = GetCredsServicePrincipalSecretKey(domain, aadTokenAudience, clientId, secretKey); // NON-INTERACTIVE WITH CERT // string clientId = "<service principal / application client ID>"; // var certificate = new X509Certificate2(@"<path to (PFX) certificate file>", "<certificate password>"); // var armCreds = GetCredsServicePrincipalCertificate(domain, armTokenAudience, clientId, certificate); // var adlCreds = GetCredsServicePrincipalCertificate(domain, adlTokenAudience, clientId, certificate); // var aadCreds = GetCredsServicePrincipalCertificate(domain, aadTokenAudience, clientId, certificate); // ---------------------------------------- // Create the REST clients using the credentials // ---------------------------------------- var adlaAccountClient = new DataLakeAnalyticsAccountManagementClient(armCreds); adlaAccountClient.SubscriptionId = subscriptionId; var adlsAccountClient = new DataLakeStoreAccountManagementClient(armCreds); adlsAccountClient.SubscriptionId = subscriptionId; var adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(adlCreds); var adlaJobClient = new DataLakeAnalyticsJobManagementClient(adlCreds); var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(adlCreds); var graphClient = new GraphRbacManagementClient(aadCreds); graphClient.TenantID = domain; // ---------------------------------------- // Perform operations with the REST clients // ---------------------------------------- var account = adlaAccountClient.Account.Get(resourceGroupName, adlaAccountName); Console.WriteLine($"My account's location is: {account.Location}!"); // string upn = "*****@*****.**"; // string displayName = graphClient.Users.Get(upn).DisplayName; // Console.WriteLine($"The display name for {upn} is {displayName}!"); Console.ReadLine(); }