public void MemoryWarehouseShouldStoreAndReturnAndAppendAndReturnPallet()
{
var warehouse = new Warehouse();
var payload = new List <string>()
{
"Test Test test"
};
var scope = new ApplicationScope("TestApp");
var key = new WarehouseKey($"key", scope);
var receipt = warehouse.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
var returnedValue = warehouse.Retrieve <string>(key).ToList();
returnedValue.Should().Contain(payload);
var nextText = " 123456789";
payload.Add(nextText);
warehouse.Append(key, new[] { nextText }, receipt.Policies);
var newReturnedValue = warehouse.Retrieve <string>(key);
newReturnedValue.Should().Contain(payload);
}
public ShelfManifest GetManifest(WarehouseKey key)
{
// we can't support returning this data for real yet. It'd be good to pull this data from the file system
var manifest = new ShelfManifest(new[] { LoadingDockPolicy.Persistent }, -1);
// TODO: pull some metadata from File system
return(manifest);
}
public Receipt Store <T>(WarehouseKey key, IEnumerable <T> data, IEnumerable <LoadingDockPolicy> loadingDockPolicies)
{
// when items are stored, store them in the local warehouse. Policy syncing will happen somewhere else
var receipt = warehouse.Store(key, data, loadingDockPolicies);
RaiseStatusUpdated($"Data stored: {key}. Data Checksum: {receipt.SHA256Checksum}");
return(receipt);
}
public WarehouseTestEnvironment Store(string key, string payload, LoadingDockPolicy[] loadingDockPolicies, int warehouseIndex = 0)
{
var warehouseKey = new WarehouseKey(key, scope: StorageScope);
Log($"[Action] Storing. Key: {key}. Payload: {payload.Substring(0, LesserOfTwo(payload.Length,50))}");
var storageReceipt = WarehouseServers[warehouseIndex].Store(warehouseKey, new[] { payload }, loadingDockPolicies);
StorageReceipts.Add(key, storageReceipt);
storageReceipt.WasSuccessful.Should().BeTrue();
return(this);
}
public void PayloadSigningShouldRoundtripQuickly()
{
var warehouse = new Warehouse();
var payload = new[] { "Test Test test" };
var scope = new ApplicationScope("TestApp");
var key = new WarehouseKey($"key", scope);
var receipt = warehouse.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
var returnedValue = warehouse.Retrieve <string>(key).ToList();
Warehouse.VerifyChecksum(returnedValue, receipt.SHA256Checksum).Should().BeTrue();
}
public void MemoryWarehouseShouldStoreAndReturnPallet()
{
var warehouse = new Warehouse();
var payload = new[] { "Test Test test" };
var scope = new ApplicationScope("TestApp");
var key = new WarehouseKey($"key", scope);
var receipt = warehouse.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
var returnedValue = warehouse.Retrieve <string>(key).ToList();
returnedValue.Should().Contain(payload);
}
public void Store(WarehouseKey key, IEnumerable <string> payload, IProducerConsumerCollection <LoadingDockPolicy> enforcedPolicies)
{
var convertedPayload = new List <byte>();
foreach (var rec in payload)
{
convertedPayload.AddRange(Encoding.UTF8.GetBytes(rec));
}
SupportedPolicies.ForEach((ldp) => enforcedPolicies.TryAdd(ldp));
FileSystemProvider.WriteToFileSystem($"\\{key.Scope.Identifier}\\{key.Id}", convertedPayload.ToArray());
}
public IEnumerable <string> Retrieve(WarehouseKey key)
{
// TODO: don't do this asynchronously YET. I'm delaying converting everything to async, but not yet.
var rawData = FileSystemProvider.ReadFromFileSystem(GetFilePath(key)).Result;
if (rawData == null || rawData.Length == 0)
{
// no data, just return an empty file
// TODO: should this be an exception? if there isn't any data. Or is it possible an empty file was created
yield return("");
}
else
{
yield return(Encoding.UTF8.GetString(rawData));
}
}
public void MultiThreadedWriteReadPerformanceTest()
{
var warehouse = new Warehouse();
var appScope = new ApplicationScope("Test");
Parallel.ForEach(Enumerable.Range(1, 100),
new ParallelOptions {
MaxDegreeOfParallelism = 10
},
(index) => {
var payload = new[] { index.ToString() };
var key = new WarehouseKey($"key_{index}", appScope);
warehouse.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
output.WriteLine($"Index stored: {index}");
warehouse.Retrieve <string>(key).Should().Contain(payload);
});
}
public WarehouseTestEnvironment AssertStored(string testPayloadKey, string testPayload, int warehouseIndex = 0)
{
var storeKey = new WarehouseKey(testPayloadKey, StorageScope);
var warehouse = WarehouseServers[warehouseIndex];
var retrievedData = warehouse.Retrieve <string>(storeKey);
retrievedData.Should().NotBeEmpty();
// retrieve the receipt created when stored, and check the hashes (this is an overlap of other tests, but a bit of a sanity check)
var receipt = StorageReceipts[testPayloadKey];
Log($"[ASSERTION] Stored. Found: {retrievedData.First()}. Expected: {testPayload}");
Warehouse.VerifyChecksum(new[] { testPayload }, receipt.SHA256Checksum);
return(this);
}
public WarehouseTestEnvironment AssertStoragePolicy(string testPayloadKey, int warehouseIndex = 0)
{
var storeKey = new WarehouseKey(testPayloadKey, StorageScope);
var receipt = StorageReceipts[testPayloadKey];
if (receipt == null)
{
throw new ApplicationException("No receipt was found for test payloadf");
}
var manifest = WarehouseServers[warehouseIndex].GetManifest(storeKey);
Log($"[ASSERTION] Policies. Found: {string.Join(',',manifest.StoragePolicies)}. Expected: {string.Join(',', receipt.Policies)}");
// all of the policies being enforced should be in the manifest
manifest.StoragePolicies.Should().Contain(receipt.Policies);
return(this);
}
public void StoreAndRetrieve_ReturnsDataCorrectly()
{
var lighthouseServer = new LighthouseServer(Output.WriteLine);
lighthouseServer.Start();
var warehouseServer = new WarehouseServer();
// bind the warehouserServer to the lighthouseServer
lighthouseServer.Launch(warehouseServer);
var key = new WarehouseKey("testData", StorageScope.Global);
// get a shelf that can hold data for the duration of the session
var payload = new[] { "data" };
warehouseServer.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
var retrievedValues = warehouseServer.Retrieve <string>(key);
payload.Should().BeEquivalentTo(payload);
}
public void MultiThreadedAppendReadToOneKeyPerformanceTest()
{
var warehouse = new Warehouse();
var appScope = new ApplicationScope("Test");
var key = new WarehouseKey("key", appScope);
var payload = new[] { "initial" };
warehouse.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
Parallel.ForEach(Enumerable.Range(1, 100),
new ParallelOptions {
MaxDegreeOfParallelism = 10
},
(index) => {
warehouse.Append(key, new[] { (index + 100).ToString() }, new[] { LoadingDockPolicy.Ephemeral });
output.WriteLine($"Index stored: {index}");
});
warehouse.Retrieve <string>(key).Count().Should().Be(101);
}
public void PayloadSigningShouldRoundtrip()
{
var warehouse = new Warehouse();
var payload = Enumerable.Range(1, 500).Select(i => $"record{i}-{Guid.NewGuid()}").ToArray();
var scope = new ApplicationScope("TestApp");
var key = new WarehouseKey($"key", scope);
Stopwatch timer = new Stopwatch();
timer.Start();
var receipt = warehouse.Store(key, payload, new[] { LoadingDockPolicy.Ephemeral });
var returnedValue = warehouse.Retrieve <string>(key).ToList();
Warehouse.VerifyChecksum(returnedValue, receipt.SHA256Checksum).Should().BeTrue();
timer.Stop();
// the amount of time to store, and retrieve a few kilobytes
output.WriteLine($"Runtime was {Encoding.UTF8.GetByteCount(payload.SelectMany(st => st).ToArray())} bytes in {timer.ElapsedMilliseconds}.");
timer.ElapsedMilliseconds.Should().BeLessThan(100);
}
public IEnumerable <T> Retrieve <T>(WarehouseKey key)
{
return(warehouse.Retrieve <T>(key));
}
public bool CanRetrieve(WarehouseKey key)
{
var file = GetFilePath(key);
return(FileSystemProvider.FileExists(file));
}
public void Append(WarehouseKey key, IEnumerable <string> additionalPayload)
{
throw new NotImplementedException();
}
string GetFilePath(WarehouseKey key)
{
return($"\\{key.Scope.Identifier}\\{key.Id}");
}
public WarehouseKeyManifest GetManifest(WarehouseKey key)
{
return(warehouse.GetManifest(key));
}
public void Append <T>(WarehouseKey key, IEnumerable <T> data, IEnumerable <LoadingDockPolicy> loadingDockPolicies)
{
// when items are stored, store them in the local warehouse. Policy syncing will happen somewhere else
warehouse.Append(key, data, loadingDockPolicies);
}
