private async Task RunPartition(int partition, CancellationToken token)
{
DateTime timestamp = DateTime.Now;
IDatalakeStore dataLakeStore = new DatalakeStore(_testOption, _loggerFactory.CreateLogger <DatalakeStore>());;
int count = 0;
while (!token.IsCancellationRequested)
{
var data = new
{
Partition = partition,
Timestamp = DateTime.Now,
Type = "trace",
RecordId = count,
Data = $"This is {count} record id"
};
count++;
if (count % 100 == 0)
{
timestamp = timestamp + TimeSpan.FromDays(1);
}
string json = Json.Default.Serialize(data) + Environment.NewLine;
string path = $"telemetry/data/{timestamp.Year:D4}/{timestamp.Month:D2}/{timestamp.Day:D2}/{partition:D6}/trace.json";
await dataLakeStore.Append(path, Encoding.UTF8.GetBytes(json), token);
Interlocked.Increment(ref _totalCount);
}
}
private async Task InitializeFileSystem()
{
IDatalakeFileSystem management = new DatalakeFileSystem(_testOption, _loggerFactory.CreateLogger <DatalakeFileSystem>());
await management.CreateIfNotExist(_testOption.ContainerName, CancellationToken.None);
IDatalakeStore dataLakeStore = new DatalakeStore(_testOption, _loggerFactory.CreateLogger <DatalakeStore>());
await ClearContainer(dataLakeStore);
}
public static IServiceCollection ConfigureDirectoryService(this IServiceCollection service)
{
service.VerifyNotNull(nameof(service));
service.AddSingleton <IMemoryCache, MemoryCache>();
service.AddSingleton <DirectoryService>(service =>
{
ApplicationOption option = service.GetRequiredService <ApplicationOption>();
ILoggerFactory loggerFactory = service.GetRequiredService <ILoggerFactory>();
IMemoryCache memoryCache = service.GetRequiredService <IMemoryCache>();
var datalakeOption = new DatalakeStoreOption
{
AccountName = option.Storage.AccountName,
ContainerName = option.Storage.ContainerName,
AccountKey = option.Storage.AccountKey,
BasePath = option.Storage.BasePath
};
var store = new DatalakeStore(datalakeOption, loggerFactory.CreateLogger <DatalakeStore>());
var document = new DocumentStorage(store, memoryCache);
return(new DirectoryService(document));
});
service.AddSingleton <IdentityService>(service =>
{
ApplicationOption option = service.GetRequiredService <ApplicationOption>();
ILoggerFactory loggerFactory = service.GetRequiredService <ILoggerFactory>();
IMemoryCache memoryCache = service.GetRequiredService <IMemoryCache>();
var datalakeOption = new DatalakeStoreOption
{
AccountName = option.IdentityStorage.AccountName,
ContainerName = option.IdentityStorage.ContainerName,
AccountKey = option.IdentityStorage.AccountKey,
BasePath = option.IdentityStorage.BasePath
};
var store = new DatalakeStore(datalakeOption, loggerFactory.CreateLogger <DatalakeStore>());
var document = new DocumentStorage(store, memoryCache);
return(new IdentityService(document, loggerFactory.CreateLogger <IdentityService>()));
});
service.AddSingleton <SigningService>(service =>
{
DirectoryService directoryService = service.GetRequiredService <DirectoryService>();
IdentityService identityService = service.GetRequiredService <IdentityService>();
ILoggerFactory loggerFactory = service.GetRequiredService <ILoggerFactory>();
return(new SigningService(directoryService, identityService, loggerFactory.CreateLogger <SigningService>()));
});
return(service);
}
