/// <summary>
/// Crop a store, generating a new store.
/// </summary>
/// <param name="store">Store name.</param>
/// <param name="path">Store path.</param>
/// <param name="output">Output store name.</param>
/// <param name="outputPath">Output store path.</param>
/// <param name="start">Start time relative to beginning.</param>
/// <param name="length">Length relative to start.</param>
/// <returns>Success flag.</returns>
internal static int CropStore(string store, string path, string output, string outputPath, string start, string length)
{
Console.WriteLine($"Cropping store (store={store}, path={path}, output={output}, outputpath={outputPath}, start={start}, length={length})");
var startTime = TimeSpan.Zero; // start at beginning if unspecified
if (!string.IsNullOrWhiteSpace(start) && !TimeSpan.TryParse(start, CultureInfo.InvariantCulture, out startTime))
{
throw new Exception($"Could not parse start time '{start}' (see https://docs.microsoft.com/en-us/dotnet/api/system.timespan.parse for valid format).");
}
var lengthRelativeInterval = RelativeTimeInterval.LeftBounded(TimeSpan.Zero); // continue to end if unspecified
if (!string.IsNullOrWhiteSpace(length))
{
if (TimeSpan.TryParse(length, CultureInfo.InvariantCulture, out var lengthTimeSpan))
{
lengthRelativeInterval = RelativeTimeInterval.Future(lengthTimeSpan);
}
else
{
throw new Exception($"Could not parse length '{length}' (see https://docs.microsoft.com/en-us/dotnet/api/system.timespan.parse for valid format).");
}
}
if (PsiStore.Exists(output, outputPath) && !ConfirmStoreOverwrite(output))
{
return(-1);
}
PsiStore.Crop((store, path), (output, outputPath), startTime, lengthRelativeInterval, false, new Progress <double>(p => Console.WriteLine($"Progress: {p * 100.0:F2}%")), Console.WriteLine);
return(0);
}
public async Task SessionCreateDerivedPartitionCancellation()
{
var dataset = new Dataset();
var session = dataset.CreateSession();
// generate a test store
var originalStoreName = "OriginalStore";
GenerateTestStore(originalStoreName, StorePath);
// add a partition
var partition0 = session.AddPsiStorePartition(originalStoreName, StorePath, "Partition_0");
Assert.AreEqual(1, session.Partitions.Count);
Assert.AreEqual("Partition_0", session.Partitions[0].Name);
int multiplier = 7;
try
{
// create a cancellation token source that will automatically request cancellation after 500 ms
var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
// create a derived partition which contains the values from the original stream multiplied by a multiplier
await session.CreateDerivedPsiPartitionAsync(
(pipeline, importer, exporter, parameter) =>
{
var inputStream = importer.OpenStream <int>("Root");
var derivedStream = inputStream.Sample(TimeSpan.FromMinutes(1), RelativeTimeInterval.Infinite).Select(x => x *parameter).Write("DerivedStream", exporter);
// add a dummy source and propose a long time interval so that the operation will block (and eventually be canceled)
var generator = Generators.Repeat(pipeline, 0, int.MaxValue, TimeSpan.FromMilliseconds(1000));
var replayTimeInterval = TimeInterval.LeftBounded(importer.MessageOriginatingTimeInterval.Left);
pipeline.ProposeReplayTime(replayTimeInterval);
},
multiplier,
"Partition_1",
false,
"DerivedStore",
StorePath,
null,
null,
cts.Token);
}
catch (OperationCanceledException)
{
// should NOT have created a new partition (but original partition should be intact)
Assert.AreEqual(1, session.Partitions.Count);
Assert.IsTrue(PsiStore.Exists("OriginalStore", StorePath));
Assert.IsFalse(PsiStore.Exists("DerivedStore", StorePath));
// verify original partition metadata
var originalPartition = session.Partitions[0] as Partition <PsiStoreStreamReader>;
Assert.AreEqual("Partition_0", originalPartition.Name);
Assert.AreEqual("OriginalStore", originalPartition.StoreName);
Assert.AreEqual($"{StorePath}{Path.DirectorySeparatorChar}OriginalStore.0000", originalPartition.StorePath);
throw;
}
}
/// <summary>
/// Concatenate a set of stores, generating a new store.
/// </summary>
/// <param name="stores">Store names (semicolon separated).</param>
/// <param name="path">Store path.</param>
/// <param name="output">Output store name.</param>
/// <param name="outputPath">Output store path.</param>
/// <returns>Success flag.</returns>
internal static int ConcatenateStores(string stores, string path, string output, string outputPath)
{
Console.WriteLine($"Concatenating stores (stores={stores}, path={path}, output={output}, outputpath={outputPath})");
if (PsiStore.Exists(output, outputPath) && !ConfirmStoreOverwrite(output))
{
return(-1);
}
PsiStore.Concatenate(stores.Split(';').Select(s => (s, path)), (output, outputPath), false, new Progress <double>(p => Console.WriteLine($"Progress: {p * 100.0:F2}%")), Console.WriteLine);
return(0);
}
public async Task SessionCreateDerivedPartition()
{
var dataset = new Dataset();
var session = dataset.CreateSession();
// generate a test store
var originalStoreName = "OriginalStore";
GenerateTestStore(originalStoreName, StorePath);
// add a partition
var partition0 = session.AddPsiStorePartition(originalStoreName, StorePath, "Partition_0");
Assert.AreEqual(1, session.Partitions.Count);
Assert.AreEqual("Partition_0", session.Partitions[0].Name);
int multiplier = 7;
// create a derived partition which contains the values from the original stream multiplied by a multiplier
await session.CreateDerivedPsiPartitionAsync(
(pipeline, importer, exporter, parameter) =>
{
var inputStream = importer.OpenStream <int>("Root");
var derivedStream = inputStream.Select(x => x *parameter).Write("DerivedStream", exporter);
},
multiplier,
"Partition_1",
false,
"DerivedStore",
StorePath);
// should have created a new store partition
Assert.AreEqual(2, session.Partitions.Count);
Assert.IsTrue(PsiStore.Exists("OriginalStore", StorePath));
Assert.IsTrue(PsiStore.Exists("DerivedStore", StorePath));
// verify partition metadata
var originalPartition = session.Partitions[0] as Partition <PsiStoreStreamReader>;
Assert.AreEqual("Partition_0", originalPartition.Name);
Assert.AreEqual("OriginalStore", originalPartition.StoreName);
Assert.AreEqual($"{StorePath}{Path.DirectorySeparatorChar}OriginalStore.0000", originalPartition.StorePath);
var derivedPartition = session.Partitions[1] as Partition <PsiStoreStreamReader>;
Assert.AreEqual("Partition_1", derivedPartition.Name);
Assert.AreEqual("DerivedStore", derivedPartition.StoreName);
Assert.AreEqual(StorePath, derivedPartition.StorePath);
// collections to capture stream values
var originalValues = new List <int>();
var originalTimes = new List <DateTime>();
var derivedValues = new List <int>();
var derivedTimes = new List <DateTime>();
// read stream values from the partitions
using (var pipeline = Pipeline.Create())
{
var originalPartitionImporter = PsiStore.Open(pipeline, originalPartition.StoreName, originalPartition.StorePath);
originalPartitionImporter.OpenStream <int>("Root").Do(
(i, e) =>
{
originalValues.Add(i);
originalTimes.Add(e.OriginatingTime);
});
var derivedPartitionImporter = PsiStore.Open(pipeline, derivedPartition.StoreName, derivedPartition.StorePath);
derivedPartitionImporter.OpenStream <int>("DerivedStream").Do(
(i, e) =>
{
derivedValues.Add(i);
derivedTimes.Add(e.OriginatingTime);
});
pipeline.Run();
}
// verify that we read the data
Assert.AreEqual(10, originalValues.Count);
Assert.AreEqual(10, originalTimes.Count);
Assert.AreEqual(10, derivedValues.Count);
Assert.AreEqual(10, derivedTimes.Count);
// verify values from both streams are what we expect
CollectionAssert.AreEqual(originalValues.Select(x => x * multiplier).ToList(), derivedValues);
CollectionAssert.AreEqual(originalTimes, derivedTimes);
}
