public async Task Save(String endpoint, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", endpoint);
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = await _riak.Async.Get(id, options);
if (exists.IsSuccess)
{
var checkpoint = exists.Value.GetObject<Checkpoint>();
checkpoint.Position = position;
exists.Value.SetObject(checkpoint);
var putOpt = new RiakPutOptions { IfNotModified = true, IfNoneMatch = false };
putOpt.SetW(Quorum.WellKnown.All);
await _riak.Async.Put(exists.Value, putOpt);
}
else
{
var checkpoint = new Checkpoint
{
Id = endpoint,
Position = position,
};
var putOpt = new RiakPutOptions { IfNotModified = false, IfNoneMatch = true };
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, checkpoint);
await _riak.Async.Put(o, putOpt);
}
}
public void Heartbeat(string endpoint, Int32 bucket, DateTime Timestamp, long?position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
var competer = exists.Value.GetObject <Competer>();
if (competer.Discriminator != Discriminator)
{
throw new DiscriminatorException();
}
competer.Heartbeat = Timestamp;
competer.Position = position ?? competer.Position;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions {
IfNotModified = true, IfNoneMatch = false
};
putOpt.SetW(Quorum.WellKnown.All);
_riak.Put(exists.Value, putOpt);
}
}
///
/// Storing in riak
/// Each new metric
/// 1. Pull riak for quantum ("cpu_usage.16903330000.5s")
/// 2. Update quantum with average / min / max etc
/// 3. Save
/// 4. Pull 5s quantums older than 1 hour
/// 5. Merge into 5 minute quantums
/// 6. Delete old 5s quantums, save new 5 minute quantums
/// 7. Pull 5m quantums older than 1 day
/// 8. Merge into 1 hour quantums
/// 9. Delete 5m quantums, save new 1 hour quantums
/// 10. Same for week, month, and finally year
///
///
///
private async Task <RiakObject> CreateNewDb(string streamId)
{
var id = new RiakObjectId("default", "Timeseries", streamId);
var obj = new RiakObject(id, new Quantum[] { });
var options = new RiakPutOptions {
IfNoneMatch = true, IfNotModified = false, ReturnHead = true, ReturnBody = true
};
options.SetW(Quorum.WellKnown.Quorum);
_logger.Debug("Saving new timeseries db with id {0}", id.Key);
var result = await _client.Async.Put(obj, options).ConfigureAwait(false);
if (!result.IsSuccess && result.ErrorMessage != "modified")
{
_logger.Warn("Failed to create new ts db for id {0}. Error: {1}", id.Key, result.ErrorMessage);
return(null);
}
var goptions = new RiakGetOptions {
BasicQuorum = true, NotFoundOk = false
};
goptions.SetRw(Quorum.WellKnown.Quorum);
_logger.Debug("Getting timeseries db of id {0}", id.Key);
result = await _client.Async.Get(id, goptions).ConfigureAwait(false);
return(result.Value);
}
/// <summary>
/// Get the specified <paramref name="key"/> from the <paramref name="bucket"/>.
/// Optionally can be read from rVal instances. By default, the server's
/// r-value will be used, but can be overridden by rVal.
/// </summary>
/// <param name='bucket'>
/// The name of the bucket containing the <paramref name="key"/>
/// </param>
/// <param name='key'>
/// The key.
/// </param>
/// <param name='options'>The <see cref="CorrugatedIron.Models.RiakGetOptions" /> responsible for
/// configuring the semantics of this single get request. These options will override any previously
/// defined bucket configuration properties.</param>
/// <remarks>If a node does not respond, that does not necessarily mean that the
/// <paramref name="bucket"/>/<paramref name="key"/> combination is not available. It simply means
/// that fewer than R/PR nodes responded to the read request. See <see cref="CorrugatedIron.Models.RiakGetOptions" />
/// for information on how different options change Riak's default behavior.
/// </remarks>
public RiakResult <RiakObject> Get(string bucket, string key, RiakGetOptions options = null)
{
if (!IsValidBucketOrKey(bucket))
{
return(RiakResult <RiakObject> .Error(ResultCode.InvalidRequest, InvalidBucketErrorMessage, false));
}
if (!IsValidBucketOrKey(key))
{
return(RiakResult <RiakObject> .Error(ResultCode.InvalidRequest, InvalidKeyErrorMessage, false));
}
var request = new RpbGetReq {
bucket = bucket.ToRiakString(), key = key.ToRiakString()
};
options = options ?? new RiakGetOptions();
options.Populate(request);
var result = UseConnection(conn => conn.PbcWriteRead <RpbGetReq, RpbGetResp>(request));
if (!result.IsSuccess)
{
return(RiakResult <RiakObject> .Error(result.ResultCode, result.ErrorMessage, result.NodeOffline));
}
if (result.Value.vclock == null)
{
return(RiakResult <RiakObject> .Error(ResultCode.NotFound, "Unable to find value in Riak", false));
}
var o = new RiakObject(bucket, key, result.Value.content, result.Value.vclock);
return(RiakResult <RiakObject> .Success(o));
}
/// <summary>
/// Retrieve multiple objects from Riak.
/// </summary>
/// <param name='bucketKeyPairs'>
/// An <see href="System.Collections.Generic.IEnumerable<T>"/> of <see cref="CorrugatedIron.Models.RiakObjectId"/> to be retrieved
/// </param>
/// <param name='options'>The <see cref="CorrugatedIron.Models.RiakGetOptions" /> responsible for
/// configuring the semantics of this single get request. These options will override any previously
/// defined bucket configuration properties.</param>
/// <returns>An <see cref="System.Collections.Generic.IEnumerable{T}"/> of <see cref="RiakResult{T}"/>
/// is returned. You should verify the success or failure of each result separately.</returns>
/// <remarks>Riak does not support multi get behavior. CorrugatedIron's multi get functionality wraps multiple
/// get requests and returns results as an IEnumerable{RiakResult{RiakObject}}. Callers should be aware that
/// this may result in partial success - all results should be evaluated individually in the calling application.
/// In addition, applications should plan for multiple failures or multiple cases of siblings being present.</remarks>
public IEnumerable <RiakResult <RiakObject> > Get(IEnumerable <RiakObjectId> bucketKeyPairs,
RiakGetOptions options = null)
{
bucketKeyPairs = bucketKeyPairs.ToList();
options = options ?? new RiakGetOptions();
var results = UseConnection(conn =>
{
var responses = bucketKeyPairs.Select(bkp =>
{
// modified closure FTW
var bk = bkp;
if (!IsValidBucketOrKey(bk.Bucket))
{
return(RiakResult <RpbGetResp> .Error(ResultCode.InvalidRequest, InvalidBucketErrorMessage, false));
}
if (!IsValidBucketOrKey(bk.Key))
{
return(RiakResult <RpbGetResp> .Error(ResultCode.InvalidRequest, InvalidKeyErrorMessage, false));
}
var req = new RpbGetReq {
bucket = bk.Bucket.ToRiakString(), key = bk.Key.ToRiakString()
};
options.Populate(req);
return(conn.PbcWriteRead <RpbGetReq, RpbGetResp>(req));
}).ToList();
return(RiakResult <IEnumerable <RiakResult <RpbGetResp> > > .Success(responses));
});
return(results.Value.Zip(bucketKeyPairs, Tuple.Create).Select(result =>
{
if (!result.Item1.IsSuccess)
{
return RiakResult <RiakObject> .Error(result.Item1.ResultCode, result.Item1.ErrorMessage, result.Item1.NodeOffline);
}
if (result.Item1.Value.vclock == null)
{
return RiakResult <RiakObject> .Error(ResultCode.NotFound, "Unable to find value in Riak", false);
}
var o = new RiakObject(result.Item2.Bucket, result.Item2.Key, result.Item1.Value.content.First(), result.Item1.Value.vclock);
if (result.Item1.Value.content.Count > 1)
{
o.Siblings = result.Item1.Value.content.Select(c =>
new RiakObject(result.Item2.Bucket, result.Item2.Key, c, result.Item1.Value.vclock)).ToList();
}
return RiakResult <RiakObject> .Success(o);
}));
}
public Boolean Adopt(String endpoint, Int32 bucket, DateTime Timestamp)
{
Logger.Info("Endpoint {0} attempting to adopt bucket {1}", endpoint, bucket);
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (!exists.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, exists.ErrorMessage);
return(false);
}
var competer = exists.Value.GetObject <Competer>();
competer.Heartbeat = Timestamp;
competer.Discriminator = Discriminator;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions {
IfNotModified = true, IfNoneMatch = false
};
putOpt.SetW(Quorum.WellKnown.All);
var putResult = _riak.Put(exists.Value, putOpt);
if (!putResult.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, putResult.ErrorMessage);
return(false);
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject <Competer>();
if (competer.Discriminator == Discriminator)
{
Logger.Info("Endpoint {0} successfully adopted bucket {1}", endpoint, bucket);
}
else
{
Logger.Info("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
}
return(competer.Discriminator == Discriminator);
}
public void ReadParameters()
{
RiakObjectId id = PutRufus();
var opts = new RiakGetOptions();
opts.SetR(3);
var rslt = client.Get(id, opts);
CheckResult(rslt);
Console.WriteLine(Encoding.UTF8.GetString(rslt.Value.Value));
}
public bool CheckOrSave(string endpoint, Int32 bucket, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
return(false);
}
var competer = new Competer
{
Id = $"{endpoint}:{bucket}",
Discriminator = Discriminator,
Endpoint = endpoint,
Bucket = bucket,
Position = position,
Heartbeat = DateTime.UtcNow
};
var putOpt = new RiakPutOptions {
IfNotModified = false, IfNoneMatch = true
};
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, competer);
var result = _riak.Put(o, putOpt);
if (!result.IsSuccess)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, result.ErrorMessage);
return(false);
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject <Competer>();
if (competer.Discriminator != Discriminator)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
return(false);
}
Logger.Info("Endpoint {0} successfully claimed bucket {1}", endpoint, bucket);
return(true);
}
public async Task <IEnumerable <Datapoint> > Retrieve(string streamId, long? @from = null, long?to = null)
{
if ([email protected])
{
@from = long.MinValue;
}
if (!to.HasValue)
{
to = long.MaxValue;
}
var options = new RiakGetOptions {
BasicQuorum = true, NotFoundOk = false
};
options.SetRw(Quorum.WellKnown.Quorum);
var id = new RiakObjectId("default", "Timeseries", streamId);
_logger.Debug("Getting timeseries db of id {0}", id.Key);
var result = await _client.Async.Get(id, options).ConfigureAwait(false);
if (!result.IsSuccess)
{
return new Datapoint[] { }
}
;
var timeDb = result.Value.GetObject <IEnumerable <Quantum> >();
var ret = new List <Datapoint>();
foreach (var timeKey in timeDb.Where(x => x.Timestamp >= @from && x.Timestamp < to))
{
var dpId = new RiakObjectId("default", "Timeseries", timeKey.Id);
_logger.Debug("Getting datapoint of id {0}", dpId.Key);
var datapoint = await _client.Async.Get(id, options).ConfigureAwait(false);
if (!datapoint.IsSuccess)
{
_logger.Warn($"Failed to find datapoint of id {dpId.Key}. Error: {datapoint.ErrorMessage}");
continue;
}
ret.Add(datapoint.Value.GetObject <Datapoint>());
}
return(ret);
}
}
public DateTime? LastHeartbeat(string endpoint, Int32 bucket)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
var competer = exists.Value.GetObject<Competer>();
return competer.Heartbeat;
}
return null;
}
public async Task<EventStore.ClientAPI.Position> Load(String endpoint)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", endpoint);
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = await _riak.Async.Get(id, options);
if (exists.IsSuccess)
{
var position = exists.Value.GetObject<Checkpoint>();
return new EventStore.ClientAPI.Position(position.Position, position.Position);
}
return EventStore.ClientAPI.Position.Start;
}
public bool CheckOrSave(string endpoint, Int32 bucket, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess) return false;
var competer = new Competer
{
Id = $"{endpoint}:{bucket}",
Discriminator = Discriminator,
Endpoint = endpoint,
Bucket = bucket,
Position = position,
Heartbeat = DateTime.UtcNow
};
var putOpt = new RiakPutOptions { IfNotModified = false, IfNoneMatch = true };
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, competer);
var result = _riak.Put(o, putOpt);
if (!result.IsSuccess)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, result.ErrorMessage);
return false;
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject<Competer>();
if (competer.Discriminator != Discriminator)
{
Logger.Info("Endpoint {0} failed to claim bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
return false;
}
Logger.Info("Endpoint {0} successfully claimed bucket {1}", endpoint, bucket);
return true;
}
private async Task <bool> UpdateTimeDb(string streamId, Func <IEnumerable <Quantum>, IEnumerable <Quantum> > updateAction)
{
var retries = 0;
while (retries < 5)
{
var options = new RiakGetOptions {
BasicQuorum = true, NotFoundOk = false
};
options.SetRw(Quorum.WellKnown.Quorum);
var id = new RiakObjectId("default", "Timeseries", streamId);
_logger.Debug("Getting timeseries db of id {0}", id.Key);
var result = await _client.Async.Get(id, options).ConfigureAwait(false);
RiakObject timedb = result.Value;
if (!result.IsSuccess)
{
timedb = await CreateNewDb(streamId).ConfigureAwait(false);
}
if (timedb == null)
{
_logger.Error("Could not find or create ts db for metric {0}", streamId);
continue;
}
var quantums = timedb.GetObject <IEnumerable <Quantum> >();
quantums = updateAction(quantums);
timedb.SetObject(quantums);
var uptOptions = new RiakPutOptions {
IfNoneMatch = false, IfNotModified = true, ReturnHead = true, ReturnBody = false
};
uptOptions.SetW(Quorum.WellKnown.Quorum);
var updated = await _client.Async.Put(timedb, uptOptions).ConfigureAwait(false);
if (updated.IsSuccess)
{
return(true);
}
retries++;
}
return(false);
}
public async Task <EventStore.ClientAPI.Position> Load(String endpoint)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", endpoint);
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = await _riak.Async.Get(id, options);
if (exists.IsSuccess)
{
var position = exists.Value.GetObject <Checkpoint>();
return(new EventStore.ClientAPI.Position(position.Position, position.Position));
}
return(EventStore.ClientAPI.Position.Start);
}
public async Task Save(String endpoint, long position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", endpoint);
var options = new RiakGetOptions {
};
options.SetRw(Quorum.WellKnown.All);
var exists = await _riak.Async.Get(id, options);
if (exists.IsSuccess)
{
var checkpoint = exists.Value.GetObject <Checkpoint>();
checkpoint.Position = position;
exists.Value.SetObject(checkpoint);
var putOpt = new RiakPutOptions {
IfNotModified = true, IfNoneMatch = false
};
putOpt.SetW(Quorum.WellKnown.All);
await _riak.Async.Put(exists.Value, putOpt);
}
else
{
var checkpoint = new Checkpoint
{
Id = endpoint,
Position = position,
};
var putOpt = new RiakPutOptions {
IfNotModified = false, IfNoneMatch = true
};
putOpt.SetW(Quorum.WellKnown.All);
var o = new RiakObject(id, checkpoint);
await _riak.Async.Put(o, putOpt);
}
}
public Boolean Adopt(String endpoint, Int32 bucket, DateTime Timestamp)
{
Logger.Info("Endpoint {0} attempting to adopt bucket {1}", endpoint, bucket);
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (!exists.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, exists.ErrorMessage);
return false;
}
var competer = exists.Value.GetObject<Competer>();
competer.Heartbeat = Timestamp;
competer.Discriminator = Discriminator;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions { IfNotModified = true, IfNoneMatch = false };
putOpt.SetW(Quorum.WellKnown.All);
var putResult = _riak.Put(exists.Value, putOpt);
if (!putResult.IsSuccess)
{
Logger.Error("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, putResult.ErrorMessage);
return false;
}
Thread.Sleep(new Random().Next(100, 2000));
do
{
exists = _riak.Get(id, options);
} while (!exists.IsSuccess);
competer = exists.Value.GetObject<Competer>();
if (competer.Discriminator == Discriminator)
Logger.Info("Endpoint {0} successfully adopted bucket {1}", endpoint, bucket);
else
Logger.Info("Endpoint {0} failed to adopt bucket {1}. Error: {2}", endpoint, bucket, "Discriminator mismatch");
return competer.Discriminator == Discriminator;
}
public void Heartbeat(string endpoint, Int32 bucket, DateTime Timestamp, long? position)
{
var id = new RiakObjectId($"{Settings.Bucket}.system", $"{endpoint}:{bucket}");
var options = new RiakGetOptions { };
options.SetRw(Quorum.WellKnown.All);
var exists = _riak.Get(id, options);
if (exists.IsSuccess)
{
var competer = exists.Value.GetObject<Competer>();
if (competer.Discriminator != Discriminator)
throw new DiscriminatorException();
competer.Heartbeat = Timestamp;
competer.Position = position ?? competer.Position;
exists.Value.SetObject(competer);
var putOpt = new RiakPutOptions { IfNotModified = true, IfNoneMatch = false };
putOpt.SetW(Quorum.WellKnown.All);
_riak.Put(exists.Value, putOpt);
}
}
