private static void Step2_CreateModelJson()
{
var json = ModelJsonBuilder.Run(
new List <Type>
{
typeof(Product),
typeof(Order),
typeof(OrderRow),
},
new List <Tuple <Type, string, Type, string> >
{
new Tuple <Type, string, Type, string>(typeof(Order), nameof(Order.OrderID), typeof(OrderRow), nameof(OrderRow.OrderID)),
new Tuple <Type, string, Type, string>(typeof(Product), nameof(Product.ProductID), typeof(OrderRow), nameof(OrderRow.ProductID))
},
_csvFileList);
string fileName = "model.json";
if (_configuration.GetValue <bool>("DumpToLocalFile"))
{
File.WriteAllText(fileName, json);
}
if (_configuration.GetValue <bool>("UploadToADLSGen2"))
{
byte[] buffer = Encoding.UTF8.GetBytes(json);
var adlsFileClient = _ADLSDirectoryClient.GetFileClient("model.json");
Console.WriteLine($"Uploading {adlsFileClient.Uri.ToString()}");
adlsFileClient.Create();
adlsFileClient.Append(new MemoryStream(buffer), 0);
adlsFileClient.Flush(buffer.Length);
}
}
// </Snippet_DownloadFromDirectory>
#endregion
#region Download a binary file from a directory
// ---------------------------------------------------------
// Download file from directory (binary)
//----------------------------------------------------------
// <Snippet_DownloadBinaryFromDirectory>
public async Task DownloadFile(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient =
directoryClient.GetFileClient("my-image.png");
Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync();
BinaryReader reader = new BinaryReader(downloadResponse.Value.Content);
FileStream fileStream =
File.OpenWrite("C:\\Users\\contoso\\my-image-downloaded.png");
int bufferSize = 4096;
byte[] buffer = new byte[bufferSize];
int count;
while ((count = reader.Read(buffer, 0, buffer.Length)) != 0)
{
fileStream.Write(buffer, 0, count);
}
await fileStream.FlushAsync();
fileStream.Close();
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = "SevenZip/{account}/{container}/{directory}/{filename}")] HttpRequest req,
string account, string container, string directory, string filename, ILogger log)
{
//Retrieve File from storage
var lakeClient = GetDataLakeServiceClient(HttpUtility.UrlDecode(account));
var fileSystemClient = lakeClient.GetFileSystemClient(HttpUtility.UrlDecode(container));
DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient(HttpUtility.UrlDecode(directory));
var DownloadFile = directoryClient.GetFileClient(HttpUtility.UrlDecode(filename));
var ReadStream = await DownloadFile.OpenReadAsync();
var response = req.HttpContext.Response;
response.StatusCode = 200;
response.ContentType = "application/json-data-stream";
using (var archive = SevenZipArchive.Open(ReadStream, null))
{
foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
{
foreach (IArchiveEntry e in archive.Entries)
{
e.WriteTo(response.Body);
}
}
}
return(new EmptyResult());
}
public async Task CreateFileClientAsync_Directory()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Create a DataLake Filesystem
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
await filesystem.CreateAsync();
//Create a DataLake Directory
DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory"));
await directory.CreateAsync();
// Create a DataLake File using a DataLake Directory
DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file"));
await file.CreateAsync();
// Verify we created one file
AsyncPageable <PathItem> response = filesystem.GetPathsAsync();
IList <PathItem> paths = await response.ToListAsync();
Assert.AreEqual(1, paths.Count);
// Cleanup
await filesystem.DeleteAsync();
}
#pragma warning disable CA1806 // Do not ignore method results
public override void Run(CancellationToken cancellationToken)
{
var serviceClient = new DataLakeServiceClient(s_testEnvironment.DataLakeServiceUri);
new DataLakeServiceClient(s_testEnvironment.DataLakeServiceUri, s_tokenCredential);
new DataLakeServiceClient(s_testEnvironment.DataLakeServiceUri, s_testEnvironment.DataLakeCredential);
var fileSystemClient = new DataLakeFileSystemClient(s_fileSystemUri);
new DataLakeFileSystemClient(s_fileSystemUri, s_tokenCredential);
new DataLakeFileSystemClient(s_fileSystemUri, s_testEnvironment.DataLakeCredential);
var directoryClient = new DataLakeDirectoryClient(s_directoryUri);
new DataLakeDirectoryClient(s_directoryUri, s_tokenCredential);
new DataLakeDirectoryClient(s_directoryUri, s_testEnvironment.DataLakeCredential);
new DataLakeFileClient(s_fileUri);
new DataLakeFileClient(s_fileUri, s_tokenCredential);
new DataLakeFileClient(s_fileUri, s_testEnvironment.DataLakeCredential);
new DataLakePathClient(s_fileUri);
new DataLakePathClient(s_fileUri, s_tokenCredential);
new DataLakePathClient(s_fileUri, s_testEnvironment.DataLakeCredential);
serviceClient.GetFileSystemClient("foo");
fileSystemClient.GetDirectoryClient("foo");
fileSystemClient.GetFileClient("foo");
directoryClient.GetFileClient("foo");
}
public void CreateFileClient_Directory()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
#region Snippet:SampleSnippetDataLakeFileSystemClient_Create
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Create a DataLake Filesystem
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
filesystem.Create();
#endregion Snippet:SampleSnippetDataLakeFileSystemClient_Create
#region Snippet:SampleSnippetDataLakeFileClient_Create_Directory
// Create a DataLake Directory
DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory"));
directory.Create();
// Create a DataLake File using a DataLake Directory
DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file"));
file.Create();
#endregion Snippet:SampleSnippetDataLakeFileClient_Create_Directory
// Verify we created one file
Assert.AreEqual(1, filesystem.ListPaths().Count());
// Cleanup
filesystem.Delete();
}
public IDisposable GetNewFile(out DataLakeFileClient file, DataLakeServiceClient service = default, string fileSystemName = default, string directoryName = default, string fileName = default)
{
IDisposable disposingFileSystem = GetNewFileSystem(out DataLakeFileSystemClient fileSystem, service, fileSystemName);
DataLakeDirectoryClient directory = InstrumentClient(fileSystem.GetDirectoryClient(directoryName ?? GetNewDirectoryName()));
_ = directory.CreateAsync().Result;
file = InstrumentClient(directory.GetFileClient(fileName ?? GetNewFileName()));
_ = file.CreateAsync().Result;
return(disposingFileSystem);
}
// </Snippet_UploadFile>
#endregion
#region Upload files to a directory in bulk
// ---------------------------------------------------------
// Upload files to the directory - bulk uploads
//----------------------------------------------------------
// <Snippet_UploadFileBulk>
public async Task UploadFileBulk(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient = directoryClient.GetFileClient("uploaded-file.txt");
FileStream fileStream =
File.OpenRead("C:\\Users\\contoso\\file-to-upload.txt");
await fileClient.UploadAsync(fileStream);
}
static async Task ApplyACLsToSampleData(DataLakeDirectoryClient rootDirectoryClient, AppSettings settings)
{
Console.WriteLine("Applying Execute and Read ACLs to root directory...");
await ApplyACLsForDirectory(rootDirectoryClient, RolePermissions.Execute | RolePermissions.Read, settings);
Console.WriteLine(@"Applying Execute and Read ACLs to root ""Files For Organization""...");
var filesForOrganizationClient = rootDirectoryClient.GetFileClient("Files for Organization.txt");
await ApplyACLsForFile(filesForOrganizationClient, RolePermissions.Execute | RolePermissions.Read, settings);
Console.WriteLine("Applying Execute And Read ACLs to Shared Documents directory recursively...");
var sharedDocumentsDirectoryClient = rootDirectoryClient.GetSubDirectoryClient("Shared Documents");
await ApplyACLsForDirectory(sharedDocumentsDirectoryClient, RolePermissions.Execute | RolePermissions.Read, settings, recursive : true);
Console.WriteLine("Applying Execute and Read ACLs to User Documents directory...");
var userDocumentsDirectoryClient = rootDirectoryClient.GetSubDirectoryClient("User Documents");
await ApplyACLsForDirectory(userDocumentsDirectoryClient, RolePermissions.Execute | RolePermissions.Read, settings);
Console.WriteLine("Applying Execute and Read ACLs to Alice's document directory...");
var aliceDirectoryClient = userDocumentsDirectoryClient.GetSubDirectoryClient("Alice");
await ApplyACLsForDirectory(aliceDirectoryClient, RolePermissions.Execute | RolePermissions.Read, settings);
Console.WriteLine(@"Applying Execute and Read ACLs to ""Alice.txt""...");
var aliceTxtFile = aliceDirectoryClient.GetFileClient("alice.txt");
await ApplyACLsForFile(aliceTxtFile, RolePermissions.Execute | RolePermissions.Read, settings);
Console.WriteLine("Applying Execute and Read ACLs to John's document directory recursively...");
var johnDirectoryClient = userDocumentsDirectoryClient.GetSubDirectoryClient("John");
await ApplyACLsForDirectory(johnDirectoryClient, RolePermissions.Execute | RolePermissions.Read, settings, recursive : true);
Console.WriteLine("Applying Execute and Read ACLs to Bob's document directory recursively...");
var bobDirectoryClient = userDocumentsDirectoryClient.GetSubDirectoryClient("Bob");
await ApplyACLsForDirectory(bobDirectoryClient, RolePermissions.Execute | RolePermissions.Read, settings, recursive : true);
Console.WriteLine(@"Removing Execute and Read ACLs from ""c.txt""");
var cClient = bobDirectoryClient.GetSubDirectoryClient("Reports").GetFileClient("c.txt");
await RemoveACLsForFile(cClient, settings);
Console.WriteLine(@"Removing Execute and Read ACLs from Bob's Sales directory recursively...");
var salesClient = bobDirectoryClient.GetSubDirectoryClient("Sales");
await RemoveACLsForDirectory(salesClient, settings, recursive : true);
}
public async Task <dynamic> DownloadJsonData(DataLakeFileSystemClient fileSystemClient, string directory, string filePath)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient(directory);
DataLakeFileClient fileClient =
directoryClient.GetFileClient(filePath);
Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync();
var streamContent = new StreamContent(downloadResponse.Value.Content);
var stringContent = await streamContent.ReadAsStringAsync();
return(JsonConvert.DeserializeObject <dynamic>(stringContent));
}
// </Snippet_ACLDirectory>
#endregion
#region Get and set file ACLs
// ---------------------------------------------------------
// Get and set ACLs on files
//----------------------------------------------------------
// <Snippet_FileACL>
public async Task ManageFileACLs(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient =
directoryClient.GetFileClient("hello.txt");
PathAccessControl FileAccessControl =
await fileClient.GetAccessControlAsync();
foreach (var item in FileAccessControl.AccessControlList)
{
Console.WriteLine(item.ToString());
}
IList <PathAccessControlItem> accessControlList
= PathAccessControlExtensions.ParseAccessControlList
("user::rwx,group::r-x,other::rw-");
fileClient.SetAccessControlList(accessControlList);
}
static async Task UploadSampleDataIfNotExistsAsync(string localDirectory, DataLakeDirectoryClient directoryClient)
{
// Upload all sample data files in this directory
foreach (string filePath in Directory.GetFiles(localDirectory))
{
string fileName = Path.GetFileName(filePath);
DataLakeFileClient fileClient = directoryClient.GetFileClient(fileName);
if (!await fileClient.ExistsAsync())
{
await fileClient.UploadAsync(filePath);
}
}
// Recursively create subdirectories, and upload all sample data files in those subdirectories
foreach (string directory in Directory.GetDirectories(localDirectory))
{
string directoryName = Path.GetFileNameWithoutExtension(directory);
DataLakeDirectoryClient subDirectoryClient = directoryClient.GetSubDirectoryClient(directoryName);
await subDirectoryClient.CreateIfNotExistsAsync();
await UploadSampleDataIfNotExistsAsync(directory, subDirectoryClient);
}
}
public async Task DownloadFile(DataLakeFileSystemClient fileSystemClient, string keywordFilename, string outputFilePath)
{
DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient(stAccountDirectory);
DataLakeFileClient fileClient = directoryClient.GetFileClient(keywordFilename);
Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync();
BinaryReader reader = new BinaryReader(downloadResponse.Value.Content);
FileStream fileStream = File.OpenWrite(outputFilePath);
int bufferSize = 4096;
byte[] buffer = new byte[bufferSize];
int count;
while ((count = reader.Read(buffer, 0, buffer.Length)) != 0)
{
fileStream.Write(buffer, 0, count);
}
await fileStream.FlushAsync();
fileStream.Close();
}
// </Snippet_UploadFileBulk>
#endregion
#region Download a file from a directory
// ---------------------------------------------------------
// Download file from directory
//----------------------------------------------------------
// <Snippet_DownloadFromDirectory>
public async Task DownloadFile2(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient = directoryClient.GetFileClient("my-image.png");
Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync();
StreamReader reader = new StreamReader(downloadResponse.Value.Content);
FileStream fileStream = File.OpenWrite("C:\\Users\\contoso\\my-image-downloaded.png");
string output = await reader.ReadToEndAsync();
byte[] data = System.Text.Encoding.UTF8.GetBytes(output);
fileStream.Write(data, 0, data.Length);
await fileStream.FlushAsync();
fileStream.Close();
}
private static async Task DownloadAsync(string fileSystem, string directory, string fileName)
{
DataLakeFileSystemClient fileSystemClient = _dataLakeClient.GetFileSystemClient(fileSystem);
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient(directory);
DataLakeFileClient fileClient =
directoryClient.GetFileClient(fileName);
Console.WriteLine("\nStarting download...");
Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync();
var reader = new BinaryReader(downloadResponse.Value.Content);
FileStream fileStream =
File.OpenWrite($"{fileName}");
const int bufferSize = 4096;
var buffer = new byte[bufferSize];
int count;
while ((count = reader.Read(buffer, 0, buffer.Length)) != 0)
{
fileStream.Write(buffer, 0, count);
}
await fileStream.FlushAsync();
fileStream.Close();
Console.WriteLine($"File downloaded to {fileName}\n");
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = "SevenZipV2/{account}/{container}/{directory}/{filename}")] HttpRequest req,
string account, string container, string directory, string filename, ILogger log)
{
//Retrieve File from storage
var lakeClient = SevenZip.GetDataLakeServiceClient(HttpUtility.UrlDecode(account));
var fileSystemClient = lakeClient.GetFileSystemClient(HttpUtility.UrlDecode(container));
DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient(HttpUtility.UrlDecode(directory));
var DownloadFile = directoryClient.GetFileClient(HttpUtility.UrlDecode(filename));
var ReadStream = await DownloadFile.OpenReadAsync();
//Begin to send first http response
var response = req.HttpContext.Response;
response.StatusCode = 200;
response.ContentType = "application/json-data-stream";
await response.Body.WriteAsync(Encoding.UTF8.GetBytes("["));
using (var archive = SevenZipArchive.Open(ReadStream, null))
{
//Retrieve uncompressed Stream
IArchiveEntry e = archive.Entries.FirstOrDefault();
var un7ZipStream = e.OpenEntryStream();
log.LogInformation($"Stream Opened {filename}");
XmlReaderSettings settings = new XmlReaderSettings();
settings.Async = true;
settings.IgnoreWhitespace = true;
using (XmlReader reader = XmlReader.Create(un7ZipStream, settings))
{
bool keepReading = reader.Read();
bool firstrow = true;
while (keepReading)
{
if ((reader.NodeType == XmlNodeType.Element) && (reader.Name == "row") && reader.HasAttributes)
{
dynamic exo = new ExpandoObject();
for (int attInd = 0; attInd < reader.AttributeCount; attInd++)
{
reader.MoveToAttribute(attInd);
((IDictionary <String, Object>)exo).Add(reader.Name, reader.Value);
/*if(reader.Name == "Id"&&int.Parse(reader.Value)%1000==0)
* {
* log.LogInformation(reader.Value);
* } */
}
await response.Body.WriteAsync(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(exo)));
}
if (reader.Read())
{
if ((reader.NodeType == XmlNodeType.Element) && (reader.Name == "row") && reader.HasAttributes)
{
if (!firstrow)
{
await response.Body.WriteAsync(Encoding.UTF8.GetBytes(","));
}
else
{
firstrow = false;
}
}
}
else
{
keepReading = false;
}
}
}
}
await response.Body.WriteAsync(Encoding.UTF8.GetBytes("]"));
return(new EmptyResult());
}
public void Traverse()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-traverse" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-traverse"));
filesystem.Create();
try
{
// Create a bunch of directories and files within the directories
DataLakeDirectoryClient first = filesystem.CreateDirectory("first");
first.CreateSubDirectory("a");
first.CreateSubDirectory("b");
DataLakeDirectoryClient second = filesystem.CreateDirectory("second");
second.CreateSubDirectory("c");
second.CreateSubDirectory("d");
filesystem.CreateDirectory("third");
DataLakeDirectoryClient fourth = filesystem.CreateDirectory("fourth");
DataLakeDirectoryClient deepest = fourth.CreateSubDirectory("e");
// Upload a DataLake file named "file"
DataLakeFileClient file = deepest.GetFileClient("file");
file.Create();
using (FileStream stream = File.OpenRead(originalPath))
{
file.Append(stream, 0);
}
// Keep track of all the names we encounter
List <string> names = new List <string>();
foreach (PathItem pathItem in filesystem.ListPaths(recursive: true))
{
names.Add(pathItem.Name);
}
// Verify we've seen everything
Assert.AreEqual(10, names.Count);
Assert.Contains("first", names);
Assert.Contains("second", names);
Assert.Contains("third", names);
Assert.Contains("fourth", names);
Assert.Contains("first/a", names);
Assert.Contains("first/b", names);
Assert.Contains("second/c", names);
Assert.Contains("second/d", names);
Assert.Contains("fourth/e", names);
Assert.Contains("fourth/e/file", names);
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
