private static string[] ExtractFileDetail(DriveFile driveFile) { FileInfo fileInfo = new FileInfo(driveFile.FilePath); if (driveFile.FileType == FileType.Signal || driveFile.FileType == FileType.Event) { string[] fileNameTokens = fileInfo.Name.Split('.')[0].Split('_'); if (fileNameTokens.Length == 3) { return(fileNameTokens); } } return(null); }
static void Main(string[] args) { try { // Parse the connection string and return a reference to the storage account. // Retrieve the storage account from the connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); // Retrieve a reference to the table. CloudTable table = tableClient.GetTableReference("datalogger"); SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); var credTask = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential); string sourceDirectory = @"F:\OutputJson\"; DirectoryInfo dir = new DirectoryInfo(sourceDirectory); int iteration = 0; while (dir.EnumerateFiles().Skip(iteration * 1000).Take(1000).Count() > 0) { try { Console.WriteLine(string.Format("Processing Iteration: {0}" + Environment.NewLine, iteration)); List <DriveFile> files = new List <DriveFile>(); using (fileSystemClient = new DataLakeStoreFileSystemManagementClient(credTask.Result)) { foreach (var file in dir.EnumerateFiles().Skip(iteration * 1000).Take(1000)) { DriveFile driveFile = new DriveFile { FilePath = sourceDirectory + Path.GetFileName(file.ToString()), FileType = FileType.Signal }; files.Add(driveFile); } Task.WaitAll(StoreFiles(files, table).ToArray()); } } catch (Exception ex) { Console.WriteLine(string.Format("Error on Batch Processing: Batch: {0} Error :{1}", iteration, ex.Message)); } iteration++; } } catch (Exception ex) { Console.WriteLine("Error at " + ex.InnerException); } }