private static string[] ExtractFileDetail(DriveFile driveFile)
{
FileInfo fileInfo = new FileInfo(driveFile.FilePath);
if (driveFile.FileType == FileType.Signal || driveFile.FileType == FileType.Event)
{
string[] fileNameTokens = fileInfo.Name.Split('.')[0].Split('_');
if (fileNameTokens.Length == 3)
{
return(fileNameTokens);
}
}
return(null);
}
static void Main(string[] args)
{
try
{
// Parse the connection string and return a reference to the storage account.
// Retrieve the storage account from the connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
// Retrieve a reference to the table.
CloudTable table = tableClient.GetTableReference("datalogger");
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var credTask = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential);
string sourceDirectory = @"F:\OutputJson\";
DirectoryInfo dir = new DirectoryInfo(sourceDirectory);
int iteration = 0;
while (dir.EnumerateFiles().Skip(iteration * 1000).Take(1000).Count() > 0)
{
try
{
Console.WriteLine(string.Format("Processing Iteration: {0}" + Environment.NewLine, iteration));
List <DriveFile> files = new List <DriveFile>();
using (fileSystemClient = new DataLakeStoreFileSystemManagementClient(credTask.Result))
{
foreach (var file in dir.EnumerateFiles().Skip(iteration * 1000).Take(1000))
{
DriveFile driveFile = new DriveFile {
FilePath = sourceDirectory + Path.GetFileName(file.ToString()), FileType = FileType.Signal
};
files.Add(driveFile);
}
Task.WaitAll(StoreFiles(files, table).ToArray());
}
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Error on Batch Processing: Batch: {0} Error :{1}", iteration, ex.Message));
}
iteration++;
}
}
catch (Exception ex)
{
Console.WriteLine("Error at " + ex.InnerException);
}
}