///
/// Storing in riak
/// Each new metric
/// 1. Pull riak for quantum ("cpu_usage.16903330000.5s")
/// 2. Update quantum with average / min / max etc
/// 3. Save
/// 4. Pull 5s quantums older than 1 hour
/// 5. Merge into 5 minute quantums
/// 6. Delete old 5s quantums, save new 5 minute quantums
/// 7. Pull 5m quantums older than 1 day
/// 8. Merge into 1 hour quantums
/// 9. Delete 5m quantums, save new 1 hour quantums
/// 10. Same for week, month, and finally year
///
///
///
private async Task <RiakObject> CreateNewDb(string streamId)
{
var id = new RiakObjectId("default", "Timeseries", streamId);
var obj = new RiakObject(id, new Quantum[] { });
var options = new RiakPutOptions {
IfNoneMatch = true, IfNotModified = false, ReturnHead = true, ReturnBody = true
};
options.SetW(Quorum.WellKnown.Quorum);
_logger.Debug("Saving new timeseries db with id {0}", id.Key);
var result = await _client.Async.Put(obj, options).ConfigureAwait(false);
if (!result.IsSuccess && result.ErrorMessage != "modified")
{
_logger.Warn("Failed to create new ts db for id {0}. Error: {1}", id.Key, result.ErrorMessage);
return(null);
}
var goptions = new RiakGetOptions {
BasicQuorum = true, NotFoundOk = false
};
goptions.SetRw(Quorum.WellKnown.Quorum);
_logger.Debug("Getting timeseries db of id {0}", id.Key);
result = await _client.Async.Get(id, goptions).ConfigureAwait(false);
return(result.Value);
}
public async Task Save(String endpoint, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", endpoint);
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = await _riak.Async.Get(id, options);
if (exists.IsSuccess)
{
var checkpoint = exists.Value.GetObject<Checkpoint>();
checkpoint.Position = position;
exists.Value.SetObject(checkpoint);
var putOpt = new RiakPutOptions { IfNotModified = true, IfNoneMatch = false };
putOpt.SetW(Quorum.WellKnown.All);
await _riak.Async.Put(exists.Value, putOpt);
}
else
{
var checkpoint = new Checkpoint
{
Id = endpoint,
Position = position,
};
var putOpt = new RiakPutOptions { IfNotModified = false, IfNoneMatch = true };
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, checkpoint);
await _riak.Async.Put(o, putOpt);
}
}
public void Heartbeat(string endpoint, Int32 bucket, DateTime Timestamp, long?position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
var competer = exists.Value.GetObject <Competer>();
if (competer.Discriminator != Discriminator)
{
throw new DiscriminatorException();
}
competer.Heartbeat = Timestamp;
competer.Position = position ?? competer.Position;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions {
IfNotModified = true, IfNoneMatch = false
};
putOpt.SetW(Quorum.WellKnown.All);
_riak.Put(exists.Value, putOpt);
}
}
/// <summary>
/// Persist a <see cref="CorrugatedIron.Models.RiakObject"/> to Riak using the specific <see cref="CorrugatedIron.Models.RiakPutOptions" />.
/// </summary>
/// <param name='value'>
/// The <see cref="CorrugatedIron.Models.RiakObject"/> to save.
/// </param>
/// <param name='options'>
/// Put options
/// </param>
public RiakResult <RiakObject> Put(RiakObject value, RiakPutOptions options = null)
{
options = options ?? new RiakPutOptions();
var request = value.ToMessage();
options.Populate(request);
var result = UseConnection(conn => conn.PbcWriteRead <RpbPutReq, RpbPutResp>(request));
if (!result.IsSuccess)
{
return(RiakResult <RiakObject> .Error(result.ResultCode, result.ErrorMessage));
}
var finalResult = options.ReturnBody
? new RiakObject(value.Bucket, value.Key, result.Value.Content.First(), result.Value.VectorClock)
: value;
if (options.ReturnBody && result.Value.Content.Count > 1)
{
finalResult.Siblings = result.Value.Content.Select(c =>
new RiakObject(value.Bucket, value.Key, c, result.Value.VectorClock)).ToList();
}
value.MarkClean();
return(RiakResult <RiakObject> .Success(finalResult));
}
/// <summary>
/// Persist an <see href="System.Collections.Generic.IEnumerable{T}"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to Riak.
/// </summary>
/// <param name='values'>
/// The <see href="System.Collections.Generic.IEnumerable{T}"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to save.
/// </param>
/// <param name='options'>
/// Put options.
/// </param>
/// <returns>An <see cref="System.Collections.Generic.IEnumerable{T}"/> of <see cref="RiakResult{T}"/>
/// is returned. You should verify the success or failure of each result separately.</returns>
/// <remarks>Riak does not support multi put behavior. CorrugatedIron's multi put functionality wraps multiple
/// put requests and returns results as an IEnumerable{RiakResult{RiakObject}}. Callers should be aware that
/// this may result in partial success - all results should be evaluated individually in the calling application.
/// In addition, applications should plan for multiple failures or multiple cases of siblings being present.</remarks>
public IEnumerable <RiakObject> Put(IEnumerable <RiakObject> values, RiakPutOptions options = null)
{
return(Async.Put(values, options)
.ToEnumerable()
.Where(x => !x.IsLeft)
.Select(x => x.Right)
.ToList());
}
public Boolean Adopt(String endpoint, Int32 bucket, DateTime Timestamp)
{
Logger.Info("Endpoint {0} attempting to adopt bucket {1}", endpoint, bucket);
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (!exists.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, exists.ErrorMessage);
return(false);
}
var competer = exists.Value.GetObject <Competer>();
competer.Heartbeat = Timestamp;
competer.Discriminator = Discriminator;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions {
IfNotModified = true, IfNoneMatch = false
};
putOpt.SetW(Quorum.WellKnown.All);
var putResult = _riak.Put(exists.Value, putOpt);
if (!putResult.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, putResult.ErrorMessage);
return(false);
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject <Competer>();
if (competer.Discriminator == Discriminator)
{
Logger.Info("Endpoint {0} successfully adopted bucket {1}", endpoint, bucket);
}
else
{
Logger.Info("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
}
return(competer.Discriminator == Discriminator);
}
public bool CheckOrSave(string endpoint, Int32 bucket, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
return(false);
}
var competer = new Competer
{
Id = $"{endpoint}:{bucket}",
Discriminator = Discriminator,
Endpoint = endpoint,
Bucket = bucket,
Position = position,
Heartbeat = DateTime.UtcNow
};
var putOpt = new RiakPutOptions {
IfNotModified = false, IfNoneMatch = true
};
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, competer);
var result = _riak.Put(o, putOpt);
if (!result.IsSuccess)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, result.ErrorMessage);
return(false);
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject <Competer>();
if (competer.Discriminator != Discriminator)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
return(false);
}
Logger.Info("Endpoint {0} successfully claimed bucket {1}", endpoint, bucket);
return(true);
}
public bool CheckOrSave(string endpoint, Int32 bucket, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess) return false;
var competer = new Competer
{
Id = $"{endpoint}:{bucket}",
Discriminator = Discriminator,
Endpoint = endpoint,
Bucket = bucket,
Position = position,
Heartbeat = DateTime.UtcNow
};
var putOpt = new RiakPutOptions { IfNotModified = false, IfNoneMatch = true };
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, competer);
var result = _riak.Put(o, putOpt);
if (!result.IsSuccess)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, result.ErrorMessage);
return false;
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject<Competer>();
if (competer.Discriminator != Discriminator)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
return false;
}
Logger.Info("Endpoint {0} successfully claimed bucket {1}", endpoint, bucket);
return true;
}
public void TestKVOperations()
{
string key = Guid.NewGuid().ToString();
var id = new RiakObjectId(TestBucketType, TestBucket, key);
var obj = new RiakObject(id, Value);
// put
var putOptions = new RiakPutOptions();
putOptions.SetReturnBody(true);
putOptions.SetDw(3);
putOptions.SetTimeout(new Timeout(TimeSpan.FromSeconds(60)));
var putResult = Client.Put(obj, putOptions);
Assert.True(putResult.IsSuccess, putResult.ErrorMessage);
Assert.AreEqual(TestBucketType, putResult.Value.BucketType);
// get
var getResult = Client.Get(id);
Assert.True(getResult.IsSuccess);
Assert.AreEqual(TestBucketType, getResult.Value.BucketType);
// delete
var deleteOptions = new RiakDeleteOptions();
deleteOptions.Vclock = getResult.Value.VectorClock;
deleteOptions.SetDw(3);
var deleteResult = Client.Delete(id, new RiakDeleteOptions().SetDw(3));
Assert.True(deleteResult.IsSuccess);
// multiget
var ids = new List <RiakObjectId>();
for (int i = 0; i < 3; i++)
{
obj = new RiakObject(new RiakObjectId(TestBucketType, TestBucket, key + i), Value);
Client.Put(obj, new RiakPutOptions().SetReturnBody(false).SetDw(3));
ids.Add(obj.ToRiakObjectId());
}
var multiGetResult = Client.Get(ids).ToList();
Assert.True(multiGetResult.All(r => r.IsSuccess));
Assert.True(multiGetResult.All(r => r.Value.BucketType == TestBucketType));
}
private async Task <bool> UpdateTimeDb(string streamId, Func <IEnumerable <Quantum>, IEnumerable <Quantum> > updateAction)
{
var retries = 0;
while (retries < 5)
{
var options = new RiakGetOptions {
BasicQuorum = true, NotFoundOk = false
};
options.SetRw(Quorum.WellKnown.Quorum);
var id = new RiakObjectId("default", "Timeseries", streamId);
_logger.Debug("Getting timeseries db of id {0}", id.Key);
var result = await _client.Async.Get(id, options).ConfigureAwait(false);
RiakObject timedb = result.Value;
if (!result.IsSuccess)
{
timedb = await CreateNewDb(streamId).ConfigureAwait(false);
}
if (timedb == null)
{
_logger.Error("Could not find or create ts db for metric {0}", streamId);
continue;
}
var quantums = timedb.GetObject <IEnumerable <Quantum> >();
quantums = updateAction(quantums);
timedb.SetObject(quantums);
var uptOptions = new RiakPutOptions {
IfNoneMatch = false, IfNotModified = true, ReturnHead = true, ReturnBody = false
};
uptOptions.SetW(Quorum.WellKnown.Quorum);
var updated = await _client.Async.Put(timedb, uptOptions).ConfigureAwait(false);
if (updated.IsSuccess)
{
return(true);
}
retries++;
}
return(false);
}
public async Task End(Exception ex = null)
{
if (ex != null)
{
return;
}
// New and update options commented out for now.. causing too many events to be dropped because riak doing weird things
var options = new RiakPutOptions { /*IfNoneMatch = true, IfNotModified = false, */
ReturnHead = true, ReturnBody = false
};
options.SetW(Quorum.WellKnown.Quorum);
var saved = _client.Async.Put(_saves.Values, options);
var uptOptions = new RiakPutOptions {/* IfNoneMatch = false, IfNotModified = true,*/
ReturnHead = true, ReturnBody = false
};
uptOptions.SetW(Quorum.WellKnown.Quorum);
var updated = _client.Async.Put(_updates.Values, uptOptions);
var deleteOpt = new RiakDeleteOptions();
deleteOpt.SetR(Quorum.WellKnown.Quorum);
var deleted = _client.Async.Delete(_deletions, deleteOpt);
await Task.WhenAll(saved, updated, deleted);
var exceptions = saved.Result.Where(x => !x.IsSuccess).Select(x => x.Exception)
.Concat(updated.Result.Where(x => !x.IsSuccess).Select(x => x.Exception))
.Concat(deleted.Result.Where(x => !x.IsSuccess).Select(x => x.Exception));
if (exceptions.Any())
{
// Backout saves because if the event created new objects running the event again will cause more errors due to match_found
await BackOutSaves(_saves.Keys);
throw new StorageException("Failed to commit", exceptions);
}
}
public void TestKVOperations()
{
string key = Guid.NewGuid().ToString();
var id = new RiakObjectId(TestBucketType, TestBucket, key);
var obj = new RiakObject(id, Value);
// put
var putOptions = new RiakPutOptions();
putOptions.SetReturnBody(true);
putOptions.SetDw(3);
putOptions.SetTimeout(new Timeout(TimeSpan.FromSeconds(60)));
var putResult = Client.Put(obj, putOptions);
Assert.True(putResult.IsSuccess, putResult.ErrorMessage);
Assert.AreEqual(TestBucketType, putResult.Value.BucketType);
// get
var getResult = Client.Get(id);
Assert.True(getResult.IsSuccess);
Assert.AreEqual(TestBucketType, getResult.Value.BucketType);
// delete
var deleteOptions = new RiakDeleteOptions();
deleteOptions.Vclock = getResult.Value.VectorClock;
deleteOptions.SetDw(3);
var deleteResult = Client.Delete(id, new RiakDeleteOptions().SetDw(3));
Assert.True(deleteResult.IsSuccess);
// multiget
var ids = new List<RiakObjectId>();
for (int i = 0; i < 3; i++)
{
obj = new RiakObject(new RiakObjectId(TestBucketType, TestBucket, key + i), Value);
Client.Put(obj, new RiakPutOptions().SetReturnBody(false).SetDw(3));
ids.Add(obj.ToRiakObjectId());
}
var multiGetResult = Client.Get(ids).ToList();
Assert.True(multiGetResult.All(r => r.IsSuccess));
Assert.True(multiGetResult.All(r => r.Value.BucketType == TestBucketType));
}
public Task <RiakResult <RiakObject> > Put(RiakObject value, RiakPutOptions options = null)
{
if (!IsValidBucketOrKey(value.Bucket))
{
return(RiakResult <RiakObject> .ErrorTask(ResultCode.InvalidRequest, InvalidBucketErrorMessage, false));
}
if (!IsValidBucketOrKey(value.Key))
{
return(RiakResult <RiakObject> .ErrorTask(ResultCode.InvalidRequest, InvalidKeyErrorMessage, false));
}
var request = value.ToMessage();
options = options ?? new RiakPutOptions();
options.Populate(request);
return(UseConnection(conn => conn.PbcWriteRead <RpbPutReq, RpbPutResp>(request))
.ContinueWith((Task <RiakResult <RpbPutResp> > finishedTask) => {
var result = finishedTask.Result;
if (!result.IsSuccess)
{
return RiakResult <RiakObject> .Error(result.ResultCode, result.ErrorMessage, result.NodeOffline);
}
var finalResult = options.ReturnBody
? new RiakObject(value.Bucket, value.Key, result.Value.content.First(), result.Value.vclock)
: value;
if (options.ReturnBody && result.Value.content.Count > 1)
{
finalResult.Siblings = result.Value.content.Select(
c => new RiakObject(value.Bucket, value.Key, c, result.Value.vclock)).ToList();
}
return RiakResult <RiakObject> .Success(finalResult);
}));
}
public async Task Save(String endpoint, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", endpoint);
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = await _riak.Async.Get(id, options);
if (exists.IsSuccess)
{
var checkpoint = exists.Value.GetObject <Checkpoint>();
checkpoint.Position = position;
exists.Value.SetObject(checkpoint);
var putOpt = new RiakPutOptions {
IfNotModified = true, IfNoneMatch = false
};
putOpt.SetW(Quorum.WellKnown.All);
await _riak.Async.Put(exists.Value, putOpt);
}
else
{
var checkpoint = new Checkpoint
{
Id = endpoint,
Position = position,
};
var putOpt = new RiakPutOptions {
IfNotModified = false, IfNoneMatch = true
};
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, checkpoint);
await _riak.Async.Put(o, putOpt);
}
}
/// <summary>
/// Persist a <see cref="CorrugatedIron.Models.RiakObject"/> to Riak using the specific <see cref="CorrugatedIron.Models.RiakPutOptions" />.
/// </summary>
/// <param name='value'>
/// The <see cref="CorrugatedIron.Models.RiakObject"/> to save.
/// </param>
/// <param name='options'>
/// Put options
/// </param>
public RiakObject Put(RiakObject value, RiakPutOptions options = null)
{
var result = Async.Put(value, options).ConfigureAwait(false).GetAwaiter().GetResult();
return(result.IsLeft ? null : result.Right);
}
public void Put(RiakObject value, Action <RiakResult <RiakObject> > callback, RiakPutOptions options)
{
ExecAsync(() => callback(_client.Put(value, options)));
}
public void Put(IEnumerable <RiakObject> values, Action <IEnumerable <RiakResult <RiakObject> > > callback, RiakPutOptions options)
{
ExecAsync(() => callback(_client.Put(values, options)));
}
public Task <RiakResult <RiakObject> > Put(RiakObject value, RiakPutOptions options)
{
return(Task.Factory.StartNew(() => _client.Put(value, options)));
}
public Task <IEnumerable <RiakResult <RiakObject> > > Put(IEnumerable <RiakObject> values, RiakPutOptions options)
{
return(Task.Factory.StartNew(() => _client.Put(values, options)));
}
public void Heartbeat(string endpoint, Int32 bucket, DateTime Timestamp, long? position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
var competer = exists.Value.GetObject<Competer>();
if (competer.Discriminator != Discriminator)
throw new DiscriminatorException();
competer.Heartbeat = Timestamp;
competer.Position = position ?? competer.Position;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions { IfNotModified = true, IfNoneMatch = false };
putOpt.SetW(Quorum.WellKnown.All);
_riak.Put(exists.Value, putOpt);
}
}
public Boolean Adopt(String endpoint, Int32 bucket, DateTime Timestamp)
{
Logger.Info("Endpoint {0} attempting to adopt bucket {1}", endpoint, bucket);
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (!exists.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, exists.ErrorMessage);
return false;
}
var competer = exists.Value.GetObject<Competer>();
competer.Heartbeat = Timestamp;
competer.Discriminator = Discriminator;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions { IfNotModified = true, IfNoneMatch = false };
putOpt.SetW(Quorum.WellKnown.All);
var putResult = _riak.Put(exists.Value, putOpt);
if (!putResult.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, putResult.ErrorMessage);
return false;
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject<Competer>();
if (competer.Discriminator == Discriminator)
Logger.Info("Endpoint {0} successfully adopted bucket {1}", endpoint, bucket);
else
Logger.Info("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
return competer.Discriminator == Discriminator;
}
public void Put(IEnumerable <RiakObject> values, Action <IEnumerable <RiakResult <RiakObject> > > callback, RiakPutOptions options)
{
ExecAsync(Put(values, options), callback);
}
public Task <IEnumerable <RiakResult <RiakObject> > > Put(IEnumerable <RiakObject> values, RiakPutOptions options = null)
{
options = options ?? new RiakPutOptions();
return(UseConnection(conn => {
return AfterAll(values.Select(v => {
if (!IsValidBucketOrKey(v.Bucket))
{
return RiakResult <RpbPutResp> .ErrorTask(ResultCode.InvalidRequest, InvalidBucketErrorMessage, false);
}
if (!IsValidBucketOrKey(v.Key))
{
return RiakResult <RpbPutResp> .ErrorTask(ResultCode.InvalidRequest, InvalidKeyErrorMessage, false);
}
var msg = v.ToMessage();
options.Populate(msg);
return conn.PbcWriteRead <RpbPutReq, RpbPutResp>(msg);
})).ContinueWith((Task <IEnumerable <RiakResult <RpbPutResp> > > finishedTask) => {
return RiakResult <IEnumerable <RiakResult <RpbPutResp> > > .Success(finishedTask.Result);
});
}).ContinueWith((Task <RiakResult <IEnumerable <RiakResult <RpbPutResp> > > > finishedTask) => {
var results = finishedTask.Result;
return results.Value.Zip(values, Tuple.Create).Select(t => {
if (t.Item1.IsSuccess)
{
var finalResult = options.ReturnBody
? new RiakObject(t.Item2.Bucket, t.Item2.Key, t.Item1.Value.content.First(), t.Item1.Value.vclock)
: t.Item2;
if (options.ReturnBody && t.Item1.Value.content.Count > 1)
{
finalResult.Siblings = t.Item1.Value.content.Select(
c => new RiakObject(t.Item2.Bucket, t.Item2.Key, c, t.Item1.Value.vclock)).ToList();
}
return RiakResult <RiakObject> .Success(finalResult);
}
return RiakResult <RiakObject> .Error(t.Item1.ResultCode, t.Item1.ErrorMessage, t.Item1.NodeOffline);
});
}));
}
public IObservable <Either <RiakException, RiakObject> > Put(IEnumerable <RiakObject> values, RiakPutOptions options = null)
{
var observables = Observable.Create <Either <RiakException, RiakObject> >(async obs =>
{
try
{
options = options ?? new RiakPutOptions();
foreach (var v in values)
{
if (!IsValidBucketOrKey(v.Bucket))
{
obs.OnNext(new Either <RiakException, RiakObject>(new RiakException((uint)ResultCode.InvalidRequest, InvalidBucketErrorMessage, false)));
continue;
}
if (!IsValidBucketOrKey(v.Key))
{
obs.OnNext(new Either <RiakException, RiakObject>(new RiakException((uint)ResultCode.InvalidRequest, InvalidKeyErrorMessage, false)));
continue;
}
var msg = v.ToMessage();
options.Populate(msg);
try
{
var result = await
_connection.PbcWriteRead <RpbPutReq, RpbPutResp>(_endPoint, msg)
.ConfigureAwait(false);
var finalResult = options.ReturnBody
? new RiakObject(v.Bucket, v.Key, result.content.First(), result.vclock)
: v;
if (options.ReturnBody && result.content.Count > 1)
{
finalResult.Siblings = result.content
.Select(c => new RiakObject(v.Bucket, v.Key, c, result.vclock))
.ToList();
}
obs.OnNext(new Either <RiakException, RiakObject>(finalResult));
}
catch (RiakException riakException)
{
obs.OnNext(new Either <RiakException, RiakObject>(riakException));
continue;
}
}
obs.OnCompleted();
}
catch (Exception exception)
{
obs.OnError(exception);
}
return(Disposable.Empty);
});
return(observables);
}
public void Put(RiakObject value, Action <RiakResult <RiakObject> > callback, RiakPutOptions options)
{
ExecAsync(Put(value, options), callback);
}
/// <summary>
/// Persist an <see href="System.Collections.Generic.IEnumerable{T}"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to Riak.
/// </summary>
/// <param name='values'>
/// The <see href="System.Collections.Generic.IEnumerable{T}"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to save.
/// </param>
/// <param name='options'>
/// Put options.
/// </param>
/// <returns>An <see cref="System.Collections.Generic.IEnumerable{T}"/> of <see cref="RiakResult{T}"/>
/// is returned. You should verify the success or failure of each result separately.</returns>
/// <remarks>Riak does not support multi put behavior. CorrugatedIron's multi put functionality wraps multiple
/// put requests and returns results as an IEnumerable{RiakResult{RiakObject}}. Callers should be aware that
/// this may result in partial success - all results should be evaluated individually in the calling application.
/// In addition, applications should plan for multiple failures or multiple cases of siblings being present.</remarks>
public IEnumerable <RiakResult <RiakObject> > Put(IEnumerable <RiakObject> values, RiakPutOptions options = null)
{
return(WaitFor(_client.Put(values, options)));
}
public async Task End(Exception ex = null)
{
if (ex != null)
{
return;
}
Logger.Debug("UOW End - {0} saves, {1} updates, {2} deletions", _saves.Count, _updates.Count, _deletions.Count);
// New and update options commented out for now.. causing too many events to be dropped because riak doing weird things
// - Dont comment them out, figure out riak weirdness!
IEnumerable <RiakResult> exceptions;
using (CommitTimer.NewContext())
{
var options = new RiakPutOptions
{
IfNoneMatch = true,
IfNotModified = false,
ReturnHead = true,
ReturnBody = false
};
options.SetW(Quorum.WellKnown.Quorum);
var saved = _client.Async.Put(_saves.Values, options);
var uptOptions = new RiakPutOptions
{
IfNoneMatch = false,
IfNotModified = true,
ReturnHead = true,
ReturnBody = false
};
uptOptions.SetW(Quorum.WellKnown.Quorum);
var updated = _client.Async.Put(_updates.Values, uptOptions);
var deleteOpt = new RiakDeleteOptions();
deleteOpt.SetR(Quorum.WellKnown.Quorum);
var deleted = _client.Async.Delete(_deletions, deleteOpt);
await Task.WhenAll(saved, updated, deleted).ConfigureAwait(false);
exceptions = saved.Result.Where(x => !x.IsSuccess)
.Concat(updated.Result.Where(x => !x.IsSuccess))
.Concat(deleted.Result.Where(x => !x.IsSuccess));
}
// Remove updated and deleted from cache (regardless of success)
foreach (var update in _updates)
{
MemCache.Evict(update.Key.ToString());
}
foreach (var delete in _deletions)
{
MemCache.Evict(delete.Key.ToString());
}
NLog.MappedDiagnosticsLogicalContext.Clear();
if (exceptions.Any())
{
ExceptionsMeter.Mark();
var grouped = exceptions.GroupBy(x => x.ErrorMessage).Select(x => x.First());
Logger.Warn("Exceptions when saving. Details: {0}", grouped.Select(x => $"{x.ResultCode} {x.ErrorMessage}").Aggregate((cur, next) => $"{cur}, {next}"));
foreach (var tracked in _tracked)
{
MemCache.Evict(tracked.Key.ToString());
}
// Backout saves because if the event created new objects running the event again will cause more errors due to match_found
await BackOutSaves(_saves.Keys).ConfigureAwait(false);
throw new StorageException("Failed to commit", grouped.Select(x => x.Exception));
}
}
/// <summary>
/// Persist an <see href="System.Collections.Generic.IEnumerable<T>"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to Riak.
/// </summary>
/// <param name='values'>
/// The <see href="System.Collections.Generic.IEnumerable<T>"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to save.
/// </param>
/// <param name='options'>
/// Put options.
/// </param>
public IEnumerable <RiakResult <RiakObject> > Put(IEnumerable <RiakObject> values, RiakPutOptions options = null)
{
options = options ?? new RiakPutOptions();
values = values.ToList();
var messages = values.Select(v =>
{
var m = v.ToMessage();
options.Populate(m);
return(m);
}).ToList();
var results = UseConnection(conn =>
{
var responses = messages.Select(conn.PbcWriteRead <RpbPutReq, RpbPutResp>).ToList();
return(RiakResult <IEnumerable <RiakResult <RpbPutResp> > > .Success(responses));
});
var resultsArray = results.Value.ToArray();
for (var i = 0; i < resultsArray.Length; i++)
{
if (resultsArray[i].IsSuccess)
{
values.ElementAt(i).MarkClean();
}
}
return(results.Value.Zip(values, Tuple.Create).Select(t =>
{
if (t.Item1.IsSuccess)
{
var finalResult = options.ReturnBody
? new RiakObject(t.Item2.Bucket, t.Item2.Key, t.Item1.Value.Content.First(), t.Item1.Value.VectorClock)
: t.Item2;
if (options.ReturnBody && t.Item1.Value.Content.Count > 1)
{
finalResult.Siblings = t.Item1.Value.Content.Select(c =>
new RiakObject(t.Item2.Bucket, t.Item2.Key, c, t.Item1.Value.VectorClock)).ToList();
}
return RiakResult <RiakObject> .Success(finalResult);
}
return RiakResult <RiakObject> .Error(t.Item1.ResultCode, t.Item1.ErrorMessage);
}));
}
public async Task <Either <RiakException, RiakObject> > Put(RiakObject value, RiakPutOptions options = null)
{
if (!IsValidBucketOrKey(value.Bucket))
{
return(new Either <RiakException, RiakObject>(new RiakException((uint)ResultCode.InvalidRequest, InvalidBucketErrorMessage, false)));
}
if (!IsValidBucketOrKey(value.Key))
{
return(new Either <RiakException, RiakObject>(new RiakException((uint)ResultCode.InvalidRequest, InvalidKeyErrorMessage, false)));
}
options = options ?? new RiakPutOptions();
var request = value.ToMessage();
options.Populate(request);
try
{
var result =
await _connection.PbcWriteRead <RpbPutReq, RpbPutResp>(_endPoint, request).ConfigureAwait(false);
var finalResult = options.ReturnBody
? new RiakObject(value.Bucket, value.Key, result.content.First(), result.vclock)
: value;
if (options.ReturnBody && result.content.Count > 1)
{
finalResult.Siblings = result.content.Select(c =>
new RiakObject(value.Bucket, value.Key, c, result.vclock)).ToList();
}
return(new Either <RiakException, RiakObject>(finalResult));
}
catch (RiakException riakException)
{
return(new Either <RiakException, RiakObject>(riakException));
}
}
/// <summary>
/// Persist a <see cref="CorrugatedIron.Models.RiakObject"/> to Riak using the specific <see cref="CorrugatedIron.Models.RiakPutOptions" />.
/// </summary>
/// <param name='value'>
/// The <see cref="CorrugatedIron.Models.RiakObject"/> to save.
/// </param>
/// <param name='options'>
/// Put options
/// </param>
public RiakResult <RiakObject> Put(RiakObject value, RiakPutOptions options = null)
{
return(WaitFor(_client.Put(value, options)));
}
