private void InitializeADLS()
{
this.adlsAccountName = ConfigurationManager.AppSettings["adlsAccountName"];
string subId = ConfigurationManager.AppSettings["subId"];
string remoteFolderPath = "/Hackfest/";
remoteFileName = "test" + partitionID + ".txt";
remoteFilePath = remoteFolderPath + remoteFileName;
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var domain = ConfigurationManager.AppSettings["domain"];
var webApp_clientId = ConfigurationManager.AppSettings["webApp_clientId"];
var clientSecret = ConfigurationManager.AppSettings["clientSecret"];
var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
adlsClient = new DataLakeStoreAccountManagementClient(creds);
adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
adlsClient.SubscriptionId = subId;
//create directory if not already exist
if (!adlsFileSystemClient.FileSystem.PathExists(adlsAccountName, remoteFolderPath))
{
adlsFileSystemClient.FileSystem.Mkdirs(adlsAccountName, remoteFolderPath);
}
//create/overwrite the file
string header = "";
var stream = new MemoryStream(Encoding.UTF8.GetBytes(header));
adlsFileSystemClient.FileSystem.Create(adlsAccountName, remoteFilePath, stream, true);
}
private void ConnectMenuItem_Click(object sender, RoutedEventArgs e)
{
try
{
// User login via interactive popup
// Authenticate using an an Azure AD domain and client ID that is available by default for all Azure subscriptions
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
tenant = this.txtTenantID.Text;
var nativeClientApp_clientId = "1950a258-227b-4e31-a9cf-717495945fc2";
var activeDirectoryClientSettings = ActiveDirectoryClientSettings.UsePromptOnly(nativeClientApp_clientId, new Uri("urn:ietf:wg:oauth:2.0:oob"));
var creds = UserTokenProvider.LoginWithPromptAsync(tenant, activeDirectoryClientSettings).Result;
this.uploadMenuItem.IsEnabled = true;
this.downloadMenuItem.IsEnabled = true;
this.listFilesMenuItem.IsEnabled = true;
this.getFileInfoMenuItem.IsEnabled = true;
// Create client objects and set the subscription ID
adlsClient = new DataLakeStoreAccountManagementClient(creds);
adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
}
catch (Exception ex)
{
MessageBox.Show($"Exception {ex.Message}", "Connect Failure", MessageBoxButton.OK, MessageBoxImage.Error);
}
}
public DataLakeStoreFileSystemManagementHelper(TestBase testBase)
{
this.testBase = testBase;
resourceManagementClient = this.testBase.GetResourceManagementClient();
dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreManagementClient();
dataLakeStoreFileSystemClient = this.testBase.GetDataLakeStoreFileSystemManagementClient();
}
public DataLakeStoreAndFileSystemManagementHelper(TestBase testBase, MockContext context)
{
this.testBase = testBase;
resourceManagementClient = this.testBase.GetResourceManagementClient(context);
dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreAccountManagementClient(context);
dataLakeStoreFileSystemClient = this.testBase.GetDataLakeStoreFileSystemManagementClient(context);
}
//Set up clients
public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
{
_adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds);
_adlsClient.SubscriptionId = subscriptionId;
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
}
private static void Main(string[] args)
{
// TODO: 1. Provide the name of your existing Data Lake Store account.
_adlsAccountName = "hello";
// TODO: 1. Replace with the subscription ID of the Azure Subscription containing your Data Lake Store account.
_subId = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc";
string fileName = "MSFT-history.csv";
string localFolderPath = AppDomain.CurrentDomain.BaseDirectory;
string localFilePath = localFolderPath + fileName;
string remoteFolderPath = "/stock_prices/";
string remoteFilePath = remoteFolderPath + fileName;
var creds = InteractiveLogin();
// TODO: 3. Create file system client object
_adlsFileSystemClient = //...
UploadFile(localFilePath, remoteFilePath);
Console.WriteLine("File uploaded");
var files = ListItems(remoteFolderPath);
Console.WriteLine("Folder contents:");
foreach (var file in files)
{
Console.WriteLine(file.PathSuffix);
}
DownloadFile(remoteFilePath, localFilePath);
Console.WriteLine("File downloaded");
}
public static void SetupClients(ServiceClientCredentials tokenCreds, string subscriptionId)
{
_adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds);
_adlaClient.SubscriptionId = subscriptionId;
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
}
//Set up clients
public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
{
_dataLakeStoreFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
var storageCreds = new StorageCredentials(wasbAccount, wasbKey);
_cloudBlobClient = new CloudBlobClient(new Uri(string.Format("https://{0}.blob.core.windows.net/", wasbAccount)), storageCreds);
}
public static void Run([ServiceBusTrigger("events", "data-lake", AccessRights.Listen, Connection = "ServiceBusConnectionString")] BrokeredMessage message, TraceWriter log)
{
var messageId = message.MessageId;
var messageBody = new StreamReader(message.GetBody <Stream>(), Encoding.UTF8);
try
{
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var clientCredential = new ClientCredential(servicePrincipalId, servicePrincipalKey);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
if (message.Properties["merge"].ToString() == "false")
{
adlsFileSystemClient.FileSystem.ConcurrentAppend(adlAccountName, updateFilePath, messageBody.BaseStream, appendMode: AppendModeType.Autocreate);
log.Info($"The message has been sent to the data lake messageId: {messageId}", "DATA_LAKE_UPDATE");
}
else if (message.Properties["merge"].ToString() == "true")
{
adlsFileSystemClient.FileSystem.ConcurrentAppend(adlAccountName, mergeFilePath, messageBody.BaseStream, appendMode: AppendModeType.Autocreate);
log.Info($"The message has been sent to the data lake messageId: {messageId}", "DATA_LAKE_(UN)MERGE");
}
}
catch (Exception e)
{
log.Error("There was an error processing the message: ", e, "DATA_LAKE_ERROR");
}
}
public DataLakeStoreFileSystemManagementHelper(TestBase testBase)
{
this.testBase = testBase;
resourceManagementClient = this.testBase.GetResourceManagementClient();
dataLakeStoreManagementClient = this.testBase.GetDataLakeStoreManagementClient();
dataLakeStoreFileSystemClient = this.testBase.GetDataLakeStoreFileSystemManagementClient();
}
public async Task <IEnumerable <BlobMeta> > GetMetaAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(ids);
DataLakeStoreFileSystemManagementClient client = await GetFsClient();
return(await Task.WhenAll(ids.Select(id => GetMetaAsync(id, client))));
}
public async Task DeleteAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(ids);
DataLakeStoreFileSystemManagementClient client = await GetFsClient();
await Task.WhenAll(ids.Select(id => client.FileSystem.DeleteAsync(_accountName, id)));
}
private async Task <BlobMeta> GetMetaAsync(string id, DataLakeStoreFileSystemManagementClient client)
{
FileStatusResult fsr = await client.FileSystem.GetFileStatusAsync(_accountName, id);
var meta = new BlobMeta(fsr.FileStatus.Length.Value, null);
return(meta);
}
protected DataLakeStoreTestsBase()
{
helper = new EnvironmentSetupHelper();
dataLakeStoreManagementClient = GetDataLakeStoreManagementClient();
dataLakeStoreFileSystemManagementClient = GetDataLakeStoreFileSystemManagementClient();
resourceManagementClient = GetResourceManagementClient();
this.resourceGroupName = TestUtilities.GenerateName("datalakerg1");
this.dataLakeAccountName = TestUtilities.GenerateName("testdatalake1");
}
private async Task CheckAuthentication()
{
if (!IsAuthenticated)
{
_serviceClientCredentials = await ApplicationTokenProvider.LoginSilentAsync(_domain, _clientCredential);
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(_serviceClientCredentials);
}
}
public static bool DownloadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false)
{
var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, isDownload: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive);
var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient);
var uploader = new DataLakeStoreUploader(parameters, frontend);
uploader.Execute();
return(true);
}
//Set up clients
private void setupClients(TokenCredentials tokenCreds, string subscriptionId)
{
//System.NET.Http w wersji 4.0 a nie 4.1
_adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds);
_adlsClient.SubscriptionId = subscriptionId;
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);
}
static void Main(string[] args)
{
adlsAccountName = "shellpocadlsgen1";
string[] ad = new string[]
{
"[dbo].[item_dim].csv",
"[dbo].[location_dim].csv",
"[dbo].[manufacturer_dim].csv",
"[dbo].[Retail_Sales_Transaction_Data].csv",
"[dbo].[retailer_dim].csv",
"[dbo].[store_dim].csv"
};
string[] src = new string[]
{
"item.csv",
"location.csv",
"manufacturer.csv",
"trans.csv",
"retailer.csv",
"store.csv"
};
subscriptionId = "45110a54-85eb-46b4-8f41-c5d80533039c";
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var domain = "happiestminds.onmicrosoft.com";
var clientId = "98e780fd-9707-4690-a51e-7a05698f3f1e";
var clientSecret = "k0kmlvC7soT3cuwnxYC/10G4lOFQuHt0wt3CIFDPrbM=";
var clientCredential = new ClientCredential(clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
adlsClient = new DataLakeStoreAccountManagementClient(creds);
adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
adlsClient.SubscriptionId = subscriptionId;
//string sourceFolderPath = @"D:\ADLS\";
//string sourceFolderPath = @"\\192.168.56.1\adls\";
string sourceFolderPath = @"\ADLS\";
string y = DateTime.Now.ToString("yyyy");
string m = DateTime.Now.ToString("MMMM");
string d = DateTime.Now.ToString("dd");
System.Console.WriteLine(y);
System.Console.WriteLine(m);
System.Console.WriteLine(d);
//string dataLakeStoreFolderPath = "/Development/"+y+"/"+m+"/"+d+"/";
string dataLakeStoreFolderPath = "/Development/" + y + "/" + m + "/10/";
for (int i = 0; i < 6; i++)
{
string sourceFilePath = Path.Combine(sourceFolderPath, src[i]);
string dataLakeStoreFilePath = Path.Combine(dataLakeStoreFolderPath, ad[i]);
//adlsFileSystemClient.FileSystem.UploadFile(adlsAccountName, sourceFilePath, dataLakeStoreFilePath, 1, false, true);
adlsFileSystemClient.FileSystem.DownloadFile(adlsAccountName, dataLakeStoreFilePath, sourceFilePath, 1, false, true);
}
Console.WriteLine("6. Finished!");
}
public DataLakeStoreOutputManager(IOutputFormatter serializer, IFilePathAlgorithm filePathAlgorithm, ServiceClientCredentials serviceClientCredentials, string subscriptionId, string adlsAccountName)
{
_serializer = serializer;
_filePathAlgorithm = filePathAlgorithm;
_serviceClientCredentials = serviceClientCredentials;
_subscriptionId = subscriptionId;
_adlsAccountName = adlsAccountName;
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(_serviceClientCredentials);
}
public DataLakeStoreOutputManager(IMessageFormatter serializer, IFolderStructure folderStructure, ServiceClientCredentials serviceClientCredentials, string subscriptionId, string dlsAccountName)
{
_serializer = serializer;
_folderStructure = folderStructure;
_serviceClientCredentials = serviceClientCredentials;
_subscriptionId = subscriptionId;
_dlsAccountName = dlsAccountName;
_dlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(serviceClientCredentials);
}
public DataLakeStoreFileSystemClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_client = DataLakeStoreCmdletBase.CreateAdlsClient<DataLakeStoreFileSystemManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix, true);
uniqueActivityIdGenerator = new Random();
}
public DataLakeStoreFileSystemClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
var creds = AzureSession.AuthenticationFactory.GetSubscriptionCloudCredentials(context);
_client = AzureSession.ClientFactory.CreateCustomClient<DataLakeStoreFileSystemManagementClient>(creds,
context.Environment.GetEndpoint(AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix));
_client.UserAgentSuffix = " - PowerShell Client";
uniqueActivityIdGenerator = new Random();
}
private async Task <DataLakeStoreFileSystemManagementClient> GetFsClient()
{
if (_fsClient != null)
{
return(_fsClient);
}
ServiceClientCredentials creds = await GetCreds();
_fsClient = new DataLakeStoreFileSystemManagementClient(creds);
return(_fsClient);
}
public async Task <IEnumerable <BlobId> > ListAsync(ListOptions options, CancellationToken cancellationToken)
{
if (options == null)
{
options = new ListOptions();
}
DataLakeStoreFileSystemManagementClient client = await GetFsClient();
var browser = new DirectoryBrowser(client, _accountName);
return(await browser.Browse(options, cancellationToken));
}
public DataLakeStoreFileSystemClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
var creds = AzureSession.AuthenticationFactory.GetSubscriptionCloudCredentials(context);
_client = AzureSession.ClientFactory.CreateCustomClient <DataLakeStoreFileSystemManagementClient>(creds,
context.Environment.GetEndpoint(AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix));
_client.UserAgentSuffix = " - PowerShell Client";
uniqueActivityIdGenerator = new Random();
}
static void Main(string[] args)
{
try
{
// Parse the connection string and return a reference to the storage account.
// Retrieve the storage account from the connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
// Retrieve a reference to the table.
CloudTable table = tableClient.GetTableReference("datalogger");
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var credTask = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential);
string sourceDirectory = @"F:\OutputJson\";
DirectoryInfo dir = new DirectoryInfo(sourceDirectory);
int iteration = 0;
while (dir.EnumerateFiles().Skip(iteration * 1000).Take(1000).Count() > 0)
{
try
{
Console.WriteLine(string.Format("Processing Iteration: {0}" + Environment.NewLine, iteration));
List <DriveFile> files = new List <DriveFile>();
using (fileSystemClient = new DataLakeStoreFileSystemManagementClient(credTask.Result))
{
foreach (var file in dir.EnumerateFiles().Skip(iteration * 1000).Take(1000))
{
DriveFile driveFile = new DriveFile {
FilePath = sourceDirectory + Path.GetFileName(file.ToString()), FileType = FileType.Signal
};
files.Add(driveFile);
}
Task.WaitAll(StoreFiles(files, table).ToArray());
}
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Error on Batch Processing: Batch: {0} Error :{1}", iteration, ex.Message));
}
iteration++;
}
}
catch (Exception ex)
{
Console.WriteLine("Error at " + ex.InnerException);
}
}
public AdlsHelper(IActivityLogger logger, AdlsInfo adlsInfo)
{
_logger = logger;
_adlsName = adlsInfo.AdlsName;
var creds = GetAccountCredentials(adlsInfo);
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
var adlsAccountManagementClient = new DataLakeStoreAccountManagementClient(creds)
{
SubscriptionId = adlsInfo.AzureSubscriptionId
};
adlsAccountManagementClient.Account.List();
}
public async Task <Stream> OpenReadAsync(string id, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
DataLakeStoreFileSystemManagementClient client = await GetFsClient();
try
{
return(await client.FileSystem.OpenAsync(_accountName, id));
}
catch (CloudException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
throw new StorageException(ErrorCode.NotFound, ex);
}
}
private async Task <BlobMeta> GetMetaAsync(string id, DataLakeStoreFileSystemManagementClient client)
{
FileStatusResult fsr;
try
{
fsr = await client.FileSystem.GetFileStatusAsync(_accountName, id);
}
catch (AdlsErrorException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
var meta = new BlobMeta(fsr.FileStatus.Length.Value, null);
return(meta);
}
public DataLakeStoreFileSystemClient(AzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_client = DataLakeStoreCmdletBase.CreateAdlsClient<DataLakeStoreFileSystemManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix, true);
uniqueActivityIdGenerator = new Random();
// We need to override the default .NET value for max connections to a host to our number of threads, if necessary.
// Otherwise we won't achieve the parallelism we want.
// This is also required before the first call on the data lake store client.
ServicePointManager.DefaultConnectionLimit = Math.Max(MaxConnectionLimit,
ServicePointManager.DefaultConnectionLimit);
}
//Set up clients
public static void SetupClients(TokenCredentials tokenCreds, string subscriptionId)
{
_adlaClient = new DataLakeAnalyticsAccountManagementClient(tokenCreds)
{
SubscriptionId = subscriptionId
};
_adlaJobClient = new DataLakeAnalyticsJobManagementClient(tokenCreds);
_adlaCatalogClient = new DataLakeAnalyticsCatalogManagementClient(tokenCreds);
_adlsClient = new DataLakeStoreAccountManagementClient(tokenCreds)
{
SubscriptionId = subscriptionId
};
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(tokenCreds);
}
public DataLakeStoreFileSystemClient(IAzureContext context)
{
if (context == null)
{
throw new ApplicationException(Resources.InvalidDefaultSubscription);
}
_client = DataLakeStoreCmdletBase.CreateAdlsClient <DataLakeStoreFileSystemManagementClient>(context,
AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix, true);
uniqueActivityIdGenerator = new Random();
// We need to override the default .NET value for max connections to a host to our number of threads, if necessary.
// Otherwise we won't achieve the parallelism we want.
// This is also required before the first call on the data lake store client.
ServicePointManager.DefaultConnectionLimit = Math.Max(MaxConnectionLimit,
ServicePointManager.DefaultConnectionLimit);
}
private static string adlsAccountFQDN = "andrewnames.azuredatalakestore.net"; // full account FQDN, not just the account name like example.azure.datalakestore.net
public static void Main(string[] args)
{
// Obtain AAD token
var creds = new ClientCredential(applicationId, clientSecret);
var clientCreds = ApplicationTokenProvider.LoginSilentAsync(tenantId, creds).GetAwaiter().GetResult();
// Create ADLS client object
try
{
string fileName = "/Test/Training.zip";
var filePathToWrite = Path.Combine(Directory.GetCurrentDirectory(), "Training.zip");
var extracPathToWrite = Path.Combine(Directory.GetCurrentDirectory());
var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(clientCreds);
var srcPath = fileName;
var destPath = filePathToWrite;
using (var stream = adlsFileSystemClient.FileSystem.OpenAsync("andrewnames", srcPath))
using (var fileStream = new FileStream(destPath, FileMode.Create))
{
var streamTask = stream.Result.CopyToAsync(fileStream);
while (!streamTask.IsCompleted)
{
Console.Clear();
Console.WriteLine("************training content downloading in process. It will take about two minutes..please wait..." + DateTime.Now.ToString("h:mm:ss tt"));
Thread.Sleep(1000);
}
ZipFile.ExtractToDirectory(destPath, extracPathToWrite);
}
}
catch (AdlsException e)
{
PrintAdlsException(e);
}
Console.WriteLine("Done. Press ENTER to continue ...");
Console.ReadLine();
}
