public void GetProjects_AzureDataLake() { string configPath = "Assets/CliConfigurationSecrets/AzureDataLakeProjectTest.json"; if (!File.Exists(configPath)) { throw new FileNotFoundException(configPath); } // Gets config file, fails if not exist ConfigurationService configService = new ConfigurationService(); CliConfiguration config = configService.GetConfiguration(configPath); DataStore dataStore = config.DataStores[0]; AzureDataLakeCrawler sut = new AzureDataLakeCrawler( dataStore.AccountName, dataStore.TenantId, dataStore.ClientId, dataStore.ClientSecret, dataStore.AzureFileSystemName); List <Core.Models.v0_2.Project> actual = sut.GetProjects(new ProjectReader( new ProjectParser())); Assert.NotNull(actual); Assert.Single(actual); Assert.Equal("TestProject", actual[0].Name); }
public void HandleCollate( List <string> datastores, bool silent, string?outdir, IConsole console) { // Abort if not configuration found if (this.config == null) { Console.WriteLine("Unable to read 'configuration.json' file. To create a configuration file, use the 'setup' action"); return; } //bool silentMode = silent ??= false; bool silentMode = silent; // Set output directory if none specified outdir ??= Path.Combine( Directory.GetCurrentDirectory(), "catalog.json"); Console.WriteLine($"Will write output to {outdir}"); // If no store specified then crawl them all if (datastores.Count == 0) { datastores = config.DataStores.Select(ds => ds.Name).ToList(); } Console.Write($"Plannig to crawl: "); foreach (var datastore in datastores) { Console.Write($"{datastore} "); } // Confirm crawl should continue char shouldCont = 'Y'; if (!silentMode) { Console.WriteLine(); Console.WriteLine("Continue? (Y|N)"); shouldCont = Convert.ToChar(Console.Read()); } if (shouldCont != 'Y') { Console.WriteLine("Aborting..."); return; } // Start crawling all specified data stores List <Metadata> middenMetadatas = new List <Metadata>(); List <Project> mippenProjects = new List <Project>(); foreach (string store in datastores) { var currStore = config .DataStores .FirstOrDefault(s => s.Name == store); if (currStore == null) { Console.WriteLine($"No data store with name {store} in config file"); continue; } Console.WriteLine($"Crawling Data Store: {currStore.Name}"); ICrawl?crawler = null; switch (currStore.Type) { case DataStoreTypes.LocalFileSystem: if (currStore.Path is not null) { crawler = new LocalFileSystemCrawler( currStore.Path); } else { Console.WriteLine( $"Not enough information provided to crawl {currStore.Name}"); } break; case DataStoreTypes.AzureDataLakeGen2: if ( currStore.AccountName is not null && currStore.TenantId is not null && currStore.ClientId is not null && currStore.ClientSecret is not null && currStore.AzureFileSystemName is not null) { crawler = new AzureDataLakeCrawler( currStore.AccountName, currStore.TenantId, currStore.ClientId, currStore.ClientSecret, currStore.AzureFileSystemName); } else { Console.WriteLine( $"Not enough information provided to crawl {currStore.Name}"); } break;