public void TestFlushThrowsOnCacheStoreException()
{
var serverCache = Ignition.GetIgnite().CreateCache <int, int>(new CacheConfiguration
{
Name = TestUtils.TestName,
CacheStoreFactory = new BlockingCacheStore(),
WriteThrough = true
});
var options = new DataStreamerClientOptions
{
AllowOverwrite = true // Required for cache store to be invoked.
};
var streamer = Client.GetDataStreamer <int, int>(serverCache.Name, options);
streamer.Remove(1);
var ex = Assert.Throws <AggregateException>(() => streamer.Flush());
StringAssert.Contains("Failed to finish operation (too many remaps)", ex.GetBaseException().Message);
// Streamer is closed because of the flush failure.
Assert.IsTrue(streamer.IsClosed);
}
/** <inheritDoc /> */
public IDataStreamerClient <TK, TV> GetDataStreamer <TK, TV>(string cacheName,
DataStreamerClientOptions <TK, TV> options)
{
IgniteArgumentCheck.NotNullOrEmpty(cacheName, "cacheName");
return(new DataStreamerClient <TK, TV>(_socket, cacheName, options));
}
public void TestInvalidOptionValuesCauseArgumentException()
{
var opts = new DataStreamerClientOptions();
Assert.Throws <ArgumentException>(() => opts.PerNodeBufferSize = -1);
Assert.Throws <ArgumentException>(() => opts.PerNodeParallelOperations = -1);
}
public void TestAutoFlushOnFullBuffer()
{
var cache = GetClientCache <string>();
var keys = TestUtils.GetPrimaryKeys(GetIgnite(), cache.Name).Take(10).ToArray();
// Set server buffers to 1 so that server always flushes the data.
var options = new DataStreamerClientOptions <int, int>
{
PerNodeBufferSize = 3
};
using (var streamer = Client.GetDataStreamer(
cache.Name,
options))
{
streamer.Add(keys[1], 1);
Assert.AreEqual(0, cache.GetSize());
streamer.Add(keys[2], 2);
Assert.AreEqual(0, cache.GetSize());
streamer.Add(keys[3], 3);
TestUtils.WaitForTrueCondition(() => cache.GetSize() == 3);
}
}
public void TestSkipStoreDoesNotInvokeCacheStore([Values(true, false)] bool allowOverwrite)
{
var serverCache = Ignition.GetIgnite().CreateCache <int, int>(new CacheConfiguration
{
Name = TestUtils.TestName,
CacheStoreFactory = new BlockingCacheStore(),
WriteThrough = true
});
var options = new DataStreamerClientOptions
{
SkipStore = true,
AllowOverwrite = allowOverwrite
};
BlockingCacheStore.Block();
using (var streamer = Client.GetDataStreamer <int, int>(serverCache.Name, options))
{
foreach (var x in Enumerable.Range(1, 300))
{
streamer.Add(x, -x);
}
}
Assert.AreEqual(300, serverCache.GetSize());
Assert.AreEqual(-100, serverCache[100]);
}
public void TestDataStreamerNoPermissionThrowsSecurityViolationClientException([Values(true, false)] bool add)
{
using (var client = StartClient(AllowAllLogin))
{
client.GetOrCreateCache <int, int>(ForbiddenCache);
}
using (var client = StartClient())
{
var options = new DataStreamerClientOptions {
AllowOverwrite = true
};
var streamer = client.GetDataStreamer <int, int>(ForbiddenCache, options);
if (add)
{
streamer.Add(1, 1);
}
else
{
streamer.Remove(1);
}
var ex = Assert.Throws <AggregateException>(() => streamer.Flush());
var clientEx = (IgniteClientException)ex.GetBaseException();
Assert.AreEqual(ClientStatusCode.SecurityViolation, clientEx.StatusCode);
Assert.AreEqual("Client is not authorized to perform this operation", clientEx.Message);
}
}
public void TestAddRemoveOverwrite()
{
var cache = GetClientCache <int>();
cache.PutAll(Enumerable.Range(1, 10).ToDictionary(x => x, x => x + 1));
var options = new DataStreamerClientOptions {
AllowOverwrite = true
};
using (var streamer = Client.GetDataStreamer <int, object>(cache.Name, options))
{
streamer.Add(1, 11);
streamer.Add(20, 20);
foreach (var key in new[] { 2, 4, 6, 7, 8, 9 })
{
streamer.Remove(key);
}
// Remove with null
streamer.Add(10, null);
}
var resKeys = cache.GetAll(Enumerable.Range(1, 30))
.Select(x => x.Key)
.OrderBy(x => x)
.ToArray();
Assert.AreEqual(11, cache.Get(1));
Assert.AreEqual(20, cache.Get(20));
Assert.AreEqual(4, cache.GetSize());
Assert.AreEqual(new[] { 1, 3, 5, 20 }, resKeys);
}
public void TestFlushThrowsOnExceptionInStreamReceiverReadBinary()
{
var options = new DataStreamerClientOptions <int, int>
{
Receiver = new StreamReceiverReadBinaryThrowException()
};
var streamer = Client.GetDataStreamer(CacheName, options);
streamer.Add(1, 1);
var ex = Assert.Throws <AggregateException>(() => streamer.Flush());
var clientEx = (IgniteClientException)ex.GetBaseException();
StringAssert.Contains("Failed to finish operation (too many remaps)", clientEx.Message);
}
public void TestStreamReceiver()
{
var cache = GetClientCache <int>();
var options = new DataStreamerClientOptions <int, int>
{
Receiver = new StreamReceiverAddOne()
};
using (var streamer = Client.GetDataStreamer(cache.Name, options))
{
streamer.Add(1, 1);
}
Assert.AreEqual(2, cache[1]);
}
public void TestAllOperationsThrowWhenStreamerIsClosed()
{
var options = new DataStreamerClientOptions
{
AllowOverwrite = true
};
var streamer = Client.GetDataStreamer <int, int>(CacheName, options);
streamer.Close(true);
Assert.Throws <ObjectDisposedException>(() => streamer.Add(1, 1));
Assert.Throws <ObjectDisposedException>(() => streamer.Remove(1));
Assert.Throws <ObjectDisposedException>(() => streamer.Flush());
Assert.Throws <ObjectDisposedException>(() => streamer.FlushAsync());
}
public void TestAutoFlushInterval()
{
var cache = GetClientCache <int>();
var options = new DataStreamerClientOptions
{
AutoFlushInterval = TimeSpan.FromSeconds(0.1)
};
using (var streamer = Client.GetDataStreamer <int, int>(cache.Name, options))
{
streamer.Add(1, 1);
TestUtils.WaitForTrueCondition(() => cache.ContainsKey(1));
streamer.Add(2, 2);
TestUtils.WaitForTrueCondition(() => cache.ContainsKey(2));
}
}
public void TestStreamReceiverKeepBinary()
{
var cache = GetClientCache <Test>().WithKeepBinary <int, IBinaryObject>();
var options = new DataStreamerClientOptions <int, IBinaryObject>
{
Receiver = new StreamReceiverAddTwoKeepBinary(),
ReceiverKeepBinary = true
};
using (var streamer = Client.GetDataStreamer(cache.Name, options))
{
streamer.Add(1, Client.GetBinary().ToBinary <IBinaryObject>(new Test {
Val = 3
}));
}
Assert.AreEqual(5, cache[1].Deserialize <Test>().Val);
}
public void TestDisconnectedBuffersGetFlushedOnExplicitFlush()
{
var server = StartServer();
var client = StartClient();
var cache = CreateCache(client);
var options = new DataStreamerClientOptions
{
PerNodeBufferSize = 3
};
using (var streamer = client.GetDataStreamer <int, int>(cache.Name, options))
{
// Fill the buffer for the initial server node.
streamer.Add(-1, -1);
streamer.Add(-2, -2);
StartServer();
server.Dispose();
// Perform cache operation to detect connection failure.
Assert.Catch(() => cache.Put(1, 3));
// Fill the buffer to force flush.
streamer.Add(1, 1);
streamer.Add(2, 2);
streamer.Add(3, 3);
// Automatic flush does not involve old buffer.
TestUtils.WaitForTrueCondition(() => cache.ContainsKey(1));
Assert.AreEqual(3, cache.GetSize());
Assert.IsTrue(cache.ContainsKeys(new[] { 1, 2, 3 }));
// Explicit flush includes old buffer.
streamer.Add(4, 4);
streamer.Flush();
Assert.AreEqual(6, cache.GetSize());
Assert.IsTrue(cache.ContainsKeys(new[] { -1, -2, 4 }));
}
}
/// <summary>
/// Gets the flags.
/// </summary>
private static Flags GetFlags(DataStreamerClientOptions options)
{
var flags = Flags.Flush | Flags.Close;
if (options.AllowOverwrite)
{
flags |= Flags.AllowOverwrite;
}
if (options.SkipStore)
{
flags |= Flags.SkipStore;
}
if (options.ReceiverKeepBinary)
{
flags |= Flags.KeepBinary;
}
return(flags);
}
/// <summary>
/// Initializes a new instance of <see cref="DataStreamerClient{TK,TV}"/>.
/// </summary>
/// <param name="socket">Socket.</param>
/// <param name="cacheName">Cache name.</param>
/// <param name="options">Options.</param>
public DataStreamerClient(
ClientFailoverSocket socket,
string cacheName,
DataStreamerClientOptions <TK, TV> options)
{
Debug.Assert(socket != null);
Debug.Assert(!string.IsNullOrEmpty(cacheName));
// Copy to prevent modification.
_options = new DataStreamerClientOptions <TK, TV>(options);
_socket = socket;
_cacheName = cacheName;
_cacheId = BinaryUtils.GetCacheId(cacheName);
_flags = GetFlags(_options);
var interval = _options.AutoFlushInterval;
if (interval != TimeSpan.Zero)
{
_autoFlushTimer = new Timer(_ => AutoFlush(), null, interval, interval);
}
}
public void TestAutoFlushClosesStreamerWhenCacheDoesNotExist()
{
var options = new DataStreamerClientOptions
{
AutoFlushInterval = TimeSpan.FromSeconds(0.2)
};
var streamer = Client.GetDataStreamer <int, int>("bad-cache-name", options);
streamer.Add(1, 1);
Assert.IsFalse(streamer.IsClosed);
TestUtils.WaitForTrueCondition(() => streamer.IsClosed);
var ex = Assert.Throws <IgniteClientException>(() => streamer.Flush());
Assert.AreEqual("Streamer is closed with error, check inner exception for details.", ex.Message);
Assert.IsNotNull(ex.InnerException);
var inner = ((AggregateException)ex.InnerException).GetBaseException();
StringAssert.StartsWith("Cache does not exist", inner.Message);
}
public void TestExceedingPerNodeParallelOperationsBlocksAddMethod()
{
var serverCache = Ignition.GetIgnite().CreateCache <int, int>(new CacheConfiguration
{
Name = TestUtils.TestName,
CacheStoreFactory = new BlockingCacheStore(),
WriteThrough = true
});
var options = new DataStreamerClientOptions
{
PerNodeParallelOperations = 2,
PerNodeBufferSize = 1,
AllowOverwrite = true // Required for cache store to be invoked.
};
// Get primary keys for one of the nodes.
var keys = TestUtils.GetPrimaryKeys(Ignition.GetIgnite(), serverCache.Name).Take(5).ToArray();
using (var streamer = Client.GetDataStreamer <int, int>(serverCache.Name, options))
{
// Block writes and add data.
BlockingCacheStore.Block();
streamer.Add(keys[1], 1);
streamer.Add(keys[2], 2);
// ReSharper disable once AccessToDisposedClosure
var task = Task.Factory.StartNew(() => streamer.Add(keys[3], 3));
// Task is blocked because two streamer operations are already in progress.
Assert.IsFalse(TestUtils.WaitForCondition(() => task.IsCompleted, 500));
BlockingCacheStore.Unblock();
TestUtils.WaitForTrueCondition(() => task.IsCompleted, 500);
}
Assert.AreEqual(3, serverCache.GetSize());
}
/** <inheritDoc /> */
public IDataStreamerClient <TK, TV> GetDataStreamer <TK, TV>(string cacheName, DataStreamerClientOptions options)
{
return(GetDataStreamer(cacheName, new DataStreamerClientOptions <TK, TV>(options)));
}
public void TestStreamerDoesNotLoseDataOnRandomTopologyChanges()
{
const int maxNodes = 4;
const int topologyChanges = 16;
var nodes = new Queue <IIgnite>();
nodes.Enqueue(StartServer());
var client = StartClient(maxPort: 10809);
var cache = CreateCache(client);
var options = new DataStreamerClientOptions {
AllowOverwrite = true
};
var streamer = client.GetDataStreamer <int, int>(cache.Name, options);
var id = 0;
var cancel = false;
var adderTask = Task.Factory.StartNew(() =>
{
// ReSharper disable once AccessToModifiedClosure
while (!cancel)
{
id++;
streamer.Add(id, id);
if (id % 500 == 0)
{
// Sleep once in a while to reduce streamed data size.
Thread.Sleep(100);
}
}
});
for (int i = 0; i < topologyChanges; i++)
{
Thread.Sleep(100);
if (nodes.Count <= 2 || (nodes.Count < maxNodes && TestUtils.Random.Next(2) == 0))
{
nodes.Enqueue(StartServer());
}
else
{
nodes.Dequeue().Dispose();
}
}
cancel = true;
adderTask.Wait(TimeSpan.FromSeconds(15));
streamer.Close(cancel: false);
var streamerImpl = (DataStreamerClient <int, int>)streamer;
TestUtils.WaitForTrueCondition(
() => id == cache.GetSize(),
() => string.Format("Expected: {0}, actual: {1}, sent: {2}, alloc: {3}, pool: {4}", id,
cache.GetSize(), streamerImpl.EntriesSent, streamerImpl.ArraysAllocated,
streamerImpl.ArraysPooled),
timeout: 3000);
DataStreamerClientTest.CheckArrayPoolLeak(streamer);
Assert.Greater(id, 10000);
Assert.AreEqual(1, cache[1]);
Assert.AreEqual(id, cache[id]);
}
