private void WriteData(List <Datum> data, FileLocalDataStore localDataStore, Action <FileLocalDataStore> postWriteAction = null)
{
for (int i = 0; i < data.Count; ++i)
{
localDataStore.WriteDatumAsync(data[i], CancellationToken.None).Wait();
postWriteAction?.Invoke(localDataStore);
}
}
private void WriteData(List <Datum> data, FileLocalDataStore localDataStore, Action <FileLocalDataStore> postWriteAction = null)
{
foreach (Datum datum in data)
{
localDataStore.WriteDatum(datum, CancellationToken.None);
postWriteAction?.Invoke(localDataStore);
}
}
private string WriteLocalDataStore(List <Datum> data, CompressionLevel compressionLevel, Action <FileLocalDataStore> postWriteAction = null)
{
Protocol protocol = CreateProtocol(compressionLevel);
FileLocalDataStore localDataStore = protocol.LocalDataStore as FileLocalDataStore;
protocol.LocalDataStore.Start();
WriteData(data, localDataStore, postWriteAction);
string path = localDataStore.Path;
localDataStore.Stop();
return(path);
}
private async Task TarTestAsync(int numFiles)
{
InitServiceHelper();
List <Datum> data = GenerateData();
// write the data store multiple times
FileLocalDataStore localDataStore = null;
string[] paths = await WriteLocalDataStoreAsync(data, CompressionLevel.Optimal, (obj) =>
{
localDataStore = obj;
}, numFiles);
Assert.Equal(numFiles, paths.Length);
// write the tar file
string tarPath = Path.Combine(localDataStore.Protocol.StorageDirectory, Guid.NewGuid().ToString());
localDataStore.CreateTarFromLocalData(tarPath);
// untar
FileStream tarFile = new FileStream(tarPath, FileMode.Open, FileAccess.Read);
TarArchive tarArchive = TarArchive.CreateInputTarArchive(tarFile);
string untarDirectory = Path.Combine(localDataStore.Protocol.StorageDirectory, Guid.NewGuid().ToString());
tarArchive.ExtractContents(untarDirectory);
tarArchive.Close();
tarFile.Close();
// check that the same number of files were created
Assert.Equal(numFiles, Directory.GetFiles(untarDirectory, "*.json.gz", SearchOption.AllDirectories).Length);
// check that the files' contents are byte-equal
foreach (string path in paths)
{
byte[] originalBytes = File.ReadAllBytes(path);
string untarredPath = Directory.GetFiles(untarDirectory, Path.GetFileName(path), SearchOption.AllDirectories).Single();
byte[] untarredBytes = File.ReadAllBytes(untarredPath);
Assert.True(originalBytes.SequenceEqual(untarredBytes));
}
Directory.Delete(untarDirectory, true);
File.Delete(tarPath);
}
public void OptimalCompressionRemoteWriteClearsFilesTest()
{
InitServiceHelper();
List <Datum> data = GenerateData();
FileLocalDataStore localDataStore = null;
WriteLocalDataStore(data, CompressionLevel.Optimal, (obj) =>
{
localDataStore = obj;
});
Assert.AreEqual(Directory.GetFiles(localDataStore.StorageDirectory).Length, 1);
localDataStore.WriteToRemoteAsync(CancellationToken.None).Wait();
// there should still be 1 file (the new open file)
Assert.AreEqual(Directory.GetFiles(localDataStore.StorageDirectory).Length, 1);
}
public async Task FastestCompressionRemoteWriteClearsFilesTest()
{
InitServiceHelper();
List <Datum> data = GenerateData();
FileLocalDataStore localDataStore = null;
await WriteLocalDataStoreAsync(data, CompressionLevel.Fastest, (obj) =>
{
localDataStore = obj;
});
Assert.Equal(Directory.GetFiles(localDataStore.StorageDirectory).Length, 1);
await localDataStore.WriteToRemoteAsync(CancellationToken.None);
// there should still be 1 file (the new open file)
Assert.Equal(Directory.GetFiles(localDataStore.StorageDirectory).Length, 1);
}
private async Task <Protocol> CreateProtocolAsync(CompressionLevel compressionLevel)
{
FileLocalDataStore localDataStore = new FileLocalDataStore
{
CompressionLevel = compressionLevel
};
ConsoleRemoteDataStore remoteDataStore = new ConsoleRemoteDataStore
{
WriteDelayMS = 1000000
};
Protocol protocol = await Protocol.CreateAsync("test");
protocol.Id = Guid.NewGuid().ToString();
protocol.LocalDataStore = localDataStore;
protocol.RemoteDataStore = remoteDataStore;
return(protocol);
}
private async Task <string[]> WriteLocalDataStoreAsync(List <Datum> data, CompressionLevel compressionLevel, Action <FileLocalDataStore> postWriteAction = null, int numTimes = 1)
{
Protocol protocol = await CreateProtocolAsync(compressionLevel);
FileLocalDataStore localDataStore = protocol.LocalDataStore as FileLocalDataStore;
for (int i = 0; i < numTimes; ++i)
{
await localDataStore.StartAsync();
WriteData(data, localDataStore, postWriteAction);
await localDataStore.StopAsync();
Assert.Equal(localDataStore.TotalDataWritten, localDataStore.TotalDataBuffered);
}
Assert.Equal(numTimes, localDataStore.PathsPreparedForRemote.Count);
return(localDataStore.PathsPreparedForRemote.ToArray());
}
private Protocol CreateProtocol(CompressionLevel compressionLevel)
{
FileLocalDataStore localDataStore = new FileLocalDataStore()
{
CompressionLevel = compressionLevel
};
ConsoleRemoteDataStore remoteDataStore = new ConsoleRemoteDataStore()
{
WriteDelayMS = 1000000
};
Protocol protocol = new Protocol("test")
{
Id = Guid.NewGuid().ToString(),
LocalDataStore = localDataStore,
RemoteDataStore = remoteDataStore
};
return(protocol);
}
