public void GetProjects_AzureDataLake()
{
string configPath = "Assets/CliConfigurationSecrets/AzureDataLakeProjectTest.json";
if (!File.Exists(configPath))
{
throw new FileNotFoundException(configPath);
}
// Gets config file, fails if not exist
ConfigurationService configService = new ConfigurationService();
CliConfiguration config = configService.GetConfiguration(configPath);
DataStore dataStore = config.DataStores[0];
AzureDataLakeCrawler sut = new AzureDataLakeCrawler(
dataStore.AccountName,
dataStore.TenantId,
dataStore.ClientId,
dataStore.ClientSecret,
dataStore.AzureFileSystemName);
List <Core.Models.v0_2.Project> actual = sut.GetProjects(new ProjectReader(
new ProjectParser()));
Assert.NotNull(actual);
Assert.Single(actual);
Assert.Equal("TestProject", actual[0].Name);
}
public void HandleCollate(
List <string> datastores,
bool silent,
string?outdir,
IConsole console)
{
// Abort if not configuration found
if (this.config == null)
{
Console.WriteLine("Unable to read 'configuration.json' file. To create a configuration file, use the 'setup' action");
return;
}
//bool silentMode = silent ??= false;
bool silentMode = silent;
// Set output directory if none specified
outdir ??= Path.Combine(
Directory.GetCurrentDirectory(), "catalog.json");
Console.WriteLine($"Will write output to {outdir}");
// If no store specified then crawl them all
if (datastores.Count == 0)
{
datastores = config.DataStores.Select(ds => ds.Name).ToList();
}
Console.Write($"Plannig to crawl: ");
foreach (var datastore in datastores)
{
Console.Write($"{datastore} ");
}
// Confirm crawl should continue
char shouldCont = 'Y';
if (!silentMode)
{
Console.WriteLine();
Console.WriteLine("Continue? (Y|N)");
shouldCont = Convert.ToChar(Console.Read());
}
if (shouldCont != 'Y')
{
Console.WriteLine("Aborting...");
return;
}
// Start crawling all specified data stores
List <Metadata> middenMetadatas = new List <Metadata>();
List <Project> mippenProjects = new List <Project>();
foreach (string store in datastores)
{
var currStore = config
.DataStores
.FirstOrDefault(s => s.Name == store);
if (currStore == null)
{
Console.WriteLine($"No data store with name {store} in config file");
continue;
}
Console.WriteLine($"Crawling Data Store: {currStore.Name}");
ICrawl?crawler = null;
switch (currStore.Type)
{
case DataStoreTypes.LocalFileSystem:
if (currStore.Path is not null)
{
crawler = new LocalFileSystemCrawler(
currStore.Path);
}
else
{
Console.WriteLine(
$"Not enough information provided to crawl {currStore.Name}");
}
break;
case DataStoreTypes.AzureDataLakeGen2:
if (
currStore.AccountName is not null &&
currStore.TenantId is not null &&
currStore.ClientId is not null &&
currStore.ClientSecret is not null &&
currStore.AzureFileSystemName is not null)
{
crawler = new AzureDataLakeCrawler(
currStore.AccountName,
currStore.TenantId,
currStore.ClientId,
currStore.ClientSecret,
currStore.AzureFileSystemName);
}
else
{
Console.WriteLine(
$"Not enough information provided to crawl {currStore.Name}");
}
break;
