private async Task RunPartition(int partition, CancellationToken token)
{
DateTime timestamp = DateTime.Now;
IDataLakeStore dataLakeStore = new DataLakeStore(_testOption, _loggerFactory.CreateLogger <DataLakeStore>());;
int count = 0;
while (!token.IsCancellationRequested)
{
var data = new
{
Partition = partition,
Timestamp = DateTime.Now,
Type = "trace",
RecordId = count,
Data = $"This is {count} record id"
};
count++;
if (count % 100 == 0)
{
timestamp = timestamp + TimeSpan.FromDays(1);
}
string json = Json.Default.Serialize(data) + Environment.NewLine;
string path = $"telemetry/data/{timestamp.Year:D4}/{timestamp.Month:D2}/{timestamp.Day:D2}/{partition:D6}/trace.json";
await dataLakeStore.Append(path, Encoding.UTF8.GetBytes(json), token);
Interlocked.Increment(ref _totalCount);
}
}
private async Task InitializeFileSystem()
{
IDataLakeManagement management = new DataLakeManagement(_testOption, _loggerFactory.CreateLogger <DataLakeManagement>());
await management.CreateIfNotExist(_testOption.ContainerName, CancellationToken.None);
IDataLakeStore dataLakeStore = new DataLakeStore(_testOption, _loggerFactory.CreateLogger <DataLakeStore>());
await ClearContainer(dataLakeStore);
}
public new IArtifactStorage?Create(string nameSpace)
{
nameSpace.VerifyNotEmpty(nameof(nameSpace));
if (!base.TryGetValue(nameSpace, out DataLakeNamespace? subject))
{
return(null);
}
var storage = new DataLakeStore(subject.Store, _loggerFactory.CreateLogger <DataLakeStore>());
return(new ArtifactStorage(storage, subject.PathRoot, _loggerFactory.CreateLogger <ArtifactStorage>()));
}