public async Task KeepFeedingAFollowerAndNeverDreamsOfPower()
{
var t = new CancellationTokenSource();
var leaderId = Guid.NewGuid();
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
// to set the term to 1
await _server.AppendEntriesAsync(new AppendEntriesRequest()
{
CurrentTerm = 1,
Entries = new byte[0][],
LeaderCommitIndex = 20,
LeaderId = leaderId,
PreviousLogIndex = -1,
PreviousLogTerm = 0
});
_server.LastHeartBeat = new AlwaysRecentTimestamp();
_server.LastHeartBeatSent = new AlwaysRecentTimestamp();
TheTrace.TraceInformation("OK, now this is before wait...");
Thread.Sleep(1000);
TheTrace.TraceInformation("Wait finished.");
Assert.Equal(1, _server.State.CurrentTerm);
TheTrace.TraceInformation("Checked Term.");
Assert.Equal(Role.Follower, _server.Role);
TheTrace.TraceInformation("Checked Role.");
t.Cancel();
}
private void SetupPeerAppendLogJobs(IEnumerable <Peer> peers)
{
foreach (var w in _workers.GetWorkers(Queues.PeerAppendLog))
{
w.Start();
}
_workers.GetWorkers(Queues.ProcessCommandQueue).Single().Start();
foreach (var p in peers)
{
var localP = p;
var q = Queues.PeerAppendLog + localP.Address;
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] setting up peer append log for queue {q}");
var todo = PeerAppendLog(localP);
_workers.Enqueue(q,
new Job(todo,
TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(3, (i) => TimeSpan.FromMilliseconds(i * i * 50)),
TimeSpan.FromMilliseconds(30)));
}
// Applying commands received from the clients
Func <CancellationToken, Task> pcq = ProcessCommandsQueue;
_workers.Enqueue(Queues.ProcessCommandQueue,
new Job(pcq,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardRangeArrived>();
TheTrace.TraceInformation("Got {0}->{1} from {2}", shardKeyArrived.InclusiveStartKey,
shardKeyArrived.InclusiveEndKey, shardKeyArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString());
var entities = table.ExecuteQuery(new TableQuery().Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", "ge", shardKeyArrived.InclusiveStartKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", "le", shardKeyArrived.InclusiveEndKey))));
bool hasAnything = false;
foreach (var entity in entities)
{
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
private static Func <Task> SendCommand(int run)
{
return(async() =>
{
var r = new Random();
while (_run == run)
{
try
{
for (int i = 0; i < r.Next(1, 10); i++)
{
var payload = BitConverter.GetBytes(r.Next(0, 100000)).Concat(BitConverter.GetBytes(r.Next(0, 100000))).ToArray();
await _cluster.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = payload
});
}
TheTrace.TraceInformation("Sent commands");
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
}
await Task.Delay(r.Next(0, 50));
}
});
}
private void LoadState(Guid?seedId = null)
{
using (var tx = _env.BeginReadOnlyTransaction())
{
LoadLastTermAndIndex(tx);
// snapshot
Snapshot ss;
if (TryGetLastSnapshot(out ss))
{
LogOffset = ss.LastIncludedIndex + 1;
}
// state
_state = new PersistentState(seedId);
Bufferable val;
if (tx.TryGet(_stateDb, StateDbKeys.Id, out val))
{
_state.Id = val;
if (tx.TryGet(_stateDb, StateDbKeys.CurrentTerm, out val)) // this should be always TRUE
{
_state.CurrentTerm = val;
}
if (tx.TryGet(_stateDb, StateDbKeys.LastVotedFor, out val))
{
Guid g = val;
_state.LastVotedForId = g == Guid.Empty ? (Guid?)null : g;
}
}
}
TheTrace.TraceInformation($"Seed Id was {seedId} and now {_state.Id}");
}
private IEnumerable <DynamicTableEntity> PreprocessEntities(IEnumerable <DynamicTableEntity> entities, ShardKeyArrived shardKeyArrived, string shardKeyTime)
{
var minDateTime = DateTimeOffset.MaxValue;
var n = 0;
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning(
"SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
yield return(entity);
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (n > 0)
{
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
}
public async Task <bool> TryLockAsync(
LockToken token,
int tries = 16, // this is NOT retry - it is try
int retryTimeoutMilliseconds = 15000,
int timeoutMilliseconds = 15000)
{
if (tries < 1)
{
tries = 1;
}
var blob = await GetBlobAsync(token.ResourceId);
for (int i = 0; i < tries; i++)
{
try
{
await blob.AcquireLeaseAsync(
TimeSpan.FromMilliseconds(timeoutMilliseconds),
token.TokenId.ToString("N"));
return(true);
}
catch (Exception e)
{
TheTrace.TraceInformation("Lock attempt - already locked: {0}", e);
// ignore
}
await Task.Delay(TimeSpan.FromMilliseconds(retryTimeoutMilliseconds / tries));
}
return(false);
}
public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
var key = source.ToTypeKey();
if (source.IsActive.GetValueOrDefault(true))
{
var consume = EventHubConsumer.Consumers.AddOrUpdate(key,
new Lazy <EventHubConsumer>(() =>
{
TheTrace.TraceInformation("Just added this eventHub consumer {0}", key);
return(new EventHubConsumer(_pusher, source.ToSummary()));
}),
(kk, vv) => vv);
// to make sure it gets accessed and created if new otherwise system is 'lazy'
TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName);
}
else
{
Lazy <EventHubConsumer> consumer = null;
if (EventHubConsumer.Consumers.TryRemove(key, out consumer))
{
consumer.Value.Dispose();
TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key);
}
}
return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false)));
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
await _pusher.PushAll(PreprocessEntities(entities, shardKeyArrived, shardKeyTime), shardKeyArrived.Source).ConfigureAwait(false);
}).ConfigureAwait(false);
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
bool hasAnything = false;
foreach (var entity in entities)
{
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public async Task <bool> UpdateMappingAsync(string baseUrl, string indexName, string typeName, string mapping)
{
TheTrace.TraceInformation("Adding {0} type to {1} index.", typeName, indexName);
baseUrl = baseUrl.TrimEnd('/');
var url = string.Format(MappingFormat, baseUrl, indexName, typeName);
var response = await _httpClient.PutAsync(url,
new StringContent(mapping, Encoding.UTF8, "application/json"));
var text = await response.Content.ReadAsStringAsync();
var mappings = _existingIndices[indexName];
if (response.IsSuccessStatusCode)
{
mappings.TryAdd(typeName, null);
return(true);
}
else
{
throw new ApplicationException(string.Format("Error {0}: {1}",
response.StatusCode,
text));
}
}
private async Task PushbatchAsync()
{
if (_stringBuilder.Length == 0)
{
return;
}
try
{
var responseMessage = await _httpClient.PostAsync(_esUrl + "_bulk",
new StringContent(_stringBuilder.ToString(),
Encoding.UTF8, "application/json"));
_stringBuilder.Clear();
_numberOfRecords = 0;
responseMessage.EnsureSuccessStatusCode();
TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushing to {0}", _esUrl);
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
throw;
}
}
public async Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (!string.IsNullOrEmpty(source.StopOffsetPoint) && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
{
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
var lockToken = new LockToken(source.ToTypeKey());
int seconds =
Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds));
if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000)))
{
TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey());
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
try
{
var events = await DoSchedule(source);
return(new Tuple <IEnumerable <Event>, bool>(events, true));
}
finally
{
Task.Run(() => _lockStore.ReleaseLockAsync(lockToken)).Wait();
}
}
public async Task PushAsync(DynamicTableEntity entity, DiagnosticsSourceSummary source)
{
if (source.Filter == null)
{
source.Filter = string.Empty;
}
if (!_filters.ContainsKey(source.Filter))
{
_filters.Add(source.Filter, new SimpleFilter(source.Filter));
}
if (!_filters[source.Filter].Satisfies(entity))
{
return;
}
var op = new
{
index = new
{
_index = source.IndexName ?? _indexNamer.BuildName(entity.Timestamp, source.DynamicProperties["MappingName"].ToString().ToLowerInvariant()),
_type = source.DynamicProperties["MappingName"].ToString(),
_id = entity.PartitionKey + entity.RowKey
}
};
var doc = new JObject();
doc.Add("@timestamp", entity.Timestamp);
doc.Add("PartitionKey", entity.PartitionKey);
doc.Add("RowKey", entity.RowKey);
doc.Add("cb_type", source.TypeName);
foreach (var property in entity.Properties)
{
if (property.Key != DiagnosticsSource.CustomAttributesFieldName)
{
doc[property.Key] = JToken.FromObject(property.Value.PropertyAsObject);
}
}
if (entity.Properties.ContainsKey(DiagnosticsSource.CustomAttributesFieldName))
{
foreach (var keyValue in GetNameValues(entity.Properties[DiagnosticsSource.CustomAttributesFieldName].StringValue))
{
doc[keyValue.Key] = keyValue.Value;
}
}
_batch.AddDoc(JsonConvert.SerializeObject(op).Replace("\r\n", " "), doc.ToString().Replace("\r\n", " "));
if (_batch.Count >= _batchSize)
{
await PushbatchAsync();
TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushed records to ElasticSearch for {0}-{1}",
source.PartitionKey,
source.RowKey);
}
}
private async Task KeepExtendingLeaseAsync(Func <Task> extendLeaseAsync, TimeSpan howLong, CancellationToken cancellationToken, string resource)
{
await EnsureExists();
var thisLong = new TimeSpan(2 * howLong.Ticks / 3); // RATM: how long? This long, what you reap is what you sew!
await Task.Delay(thisLong, cancellationToken);
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
await extendLeaseAsync();
TheTrace.TraceInformation("Extended the lifetime of the lease for {0}...", resource);
await Task.Delay(thisLong, cancellationToken);
}
catch (Exception exception)
{
if (!cancellationToken.IsCancellationRequested) // it is OK if cancellation requested, it would have been cancellation
{
TheTrace.TraceError(exception.ToString());
}
break;
}
}
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
if (source.LastOffsetPoint == null)
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O");
}
var lastOffset = DateTimeOffset.Parse(source.LastOffsetPoint);
var events = new List <Event>();
var graceMinutes = source.GracePeriodMinutes ?? 3;
var now = DateTimeOffset.UtcNow.DropSecondAndMilliseconds();
var newLastOffset = lastOffset;
int n = 1; // start from a minute after
while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n)))
{
newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n))
.DropSecondAndMilliseconds(); // just to be sure
var shardKeys = GetShardKeys(newLastOffset);
events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived {
Source = source.ToSummary(), ShardKey = shardKey
})));
if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun)
{
break;
}
n++;
TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First());
}
source.LastOffsetPoint = newLastOffset.ToString("O");
return(Task.FromResult((IEnumerable <Event>)events));
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
_telemetryProvider.WriteTelemetry(
"BlobFileActor receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
//var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(blobFileArrived.Source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(blobFileArrived.Source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, blobFileArrived.Source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
var hasAnything = false;
var minDateTime = DateTimeOffset.UtcNow;
var records = parser.Parse(() => stream, blob.Uri, blobFileArrived.Source, new ParseCursor(blobFileArrived.Position ?? 0)
{
EndPosition = blobFileArrived.EndPosition ?? 0
});
var seenPages = await _pusher.PushAll(records, blobFileArrived.Source).ConfigureAwait(false);
hasAnything = seenPages > 0;
if (hasAnything)
{
_telemetryProvider.WriteTelemetry(
"BlobFileActor message processing duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
blobFileArrived.Source.TypeName);
}
}, blobFileArrived.Source.TypeName);
return(Enumerable.Empty <Event>());
}
private async Task Candidacy(CancellationToken c)
{
while (_role == Role.Candidate)
{
var forMe = 1; // vote for yourself
var againstMe = 0;
var peers = _peerManager.GetPeers().ToArray();
var concensus = (peers.Length / 2) + 1;
var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(3, (i) => TimeSpan.FromMilliseconds(i * i * 30));
var policy = Policy.TimeoutAsync(_settings.CandidacyTimeout).WrapAsync(retry);
var request = new RequestVoteRequest()
{
CandidateId = State.Id,
CurrentTerm = State.CurrentTerm,
LastLogIndex = _logPersister.LastIndex,
LastLogTerm = _logPersister.LastEntryTerm
};
var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.RequestVoteAsync(request))));
var maxTerm = 0L;
foreach (var r in all)
{
if (r.Outcome == OutcomeType.Successful)
{
if (r.Result.CurrentTerm > maxTerm)
{
maxTerm = r.Result.CurrentTerm;
}
if (r.Result != null && r.Result.VoteGranted)
{
forMe++;
}
else
{
againstMe++;
}
}
}
if (againstMe >= concensus)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. I got rejected with {againstMe} votes :/");
BecomeFollower(maxTerm);
}
else if (forMe >= concensus)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. I got elected with {forMe} votes! :)");
BecomeLeader();
}
else
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. Non-conclusive with {forMe} for me and {againstMe} against me.");
}
}
}
public void Dispose()
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Disposing server.");
_workers.Stop();
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Disposing server. Workers stopped.");
_logPersister.Dispose();
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Disposing server. Log Persister stopped.");
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var blobPath = source.GetProperty <string>("BlobPath");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath);
blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with /
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
DateTimeOffset maxOffset = offset.TimeOffset;
FileOffset newOffset = null;
var events = new List <Event>();
foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob)
.Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
{
if (blob.Properties.LastModified > offset.TimeOffset)
{
var filename = blob.Uri.ToString();
newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0);
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(),
BlobId = filename,
Position = 0,
EndPosition = blob.Properties.Length
}));
TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename);
}
}
source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
private void BecomeFollower(long term)
{
State.LastVotedForId = null;
DestroyPeerAppendLogJobs();
_lastHeartbeat.Set(); // important not to become candidate again at least for another timeout
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] About to set term from {State.CurrentTerm} to {term}");
State.CurrentTerm = term;
OnRoleChanged(_role = Role.Follower);
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var minDateTime = DateTimeOffset.MaxValue;
var hasAnything = false;
int n = 0;
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning("SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (hasAnything)
{
await _pusher.FlushAsync();
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
});
return(Enumerable.Empty <Event>());
}
private void BecomeCandidate()
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] BecomeCandidate start");
State.IncrementTerm();
State.LastVotedForId = State.Id;
_leaderAddress = null;
DestroyPeerAppendLogJobs();
OnRoleChanged(_role = Role.Candidate);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] BecomeCandidate end");
}
public async Task <bool> CreateIndexIfNotExistsAsync(string baseUrl, string indexName, string jsonCommand = "")
{
if (_existingIndices.ContainsKey(indexName))
{
return(false);
}
baseUrl = baseUrl.TrimEnd('/');
string searchUrl = string.Format(IndexSearchFormat, baseUrl, indexName);
TheTrace.TraceInformation("Just wanna check if this index exists: {0}. URL: {1}", indexName, searchUrl);
var getResponse = await _httpClient.GetAsync(searchUrl);
var getText = await getResponse.Content.ReadAsStringAsync();
if (getResponse.IsSuccessStatusCode)
{
_existingIndices.TryAdd(indexName, new ConcurrentDictionary <string, string>());
return(false);
}
if (getResponse.StatusCode == HttpStatusCode.NotFound)
{
TheTrace.TraceInformation("It sent Back this {0} and text => {1}", (int)getResponse.StatusCode, getText);
var url = string.Format(IndexFormat, baseUrl, indexName);
var putResponse = await _httpClient.PutAsync(url, new StringContent(jsonCommand, Encoding.UTF8, "application/json"));
var putText = "[NO CONTENT]";
if (putResponse.Content != null)
{
putText = await putResponse.Content.ReadAsStringAsync();
}
if (putResponse.IsSuccessStatusCode || (putResponse.StatusCode == HttpStatusCode.BadRequest && putText.Contains("already exists")))
{
_existingIndices.TryAdd(indexName, new ConcurrentDictionary <string, string>());
return(true);
}
else
{
throw new ApplicationException(string.Format("Error in PUT {0}: {1}",
putResponse.StatusCode,
putText));
}
}
else
{
TheTrace.TraceInformation("It sent Back this {0}", (int)getResponse.StatusCode);
throw new ApplicationException(string.Format("Error in GET: {0}: {1}",
getResponse.StatusCode,
getText));
}
}
public async Task <bool> TryLockAsync(
LockToken token,
int tries = 16, // this is NOT retry - it is try
int retryTimeoutMilliseconds = 15000,
int timeoutMilliseconds = 15000,
int aquireTimeoutMilliseconds = 15000)
{
await EnsureExists();
if (tries < 1)
{
tries = 1;
}
var blob = await GetBlobAsync(token.ResourceId);
for (var i = 0; i < tries; i++)
{
try
{
using (var cancellationTokenSource = new CancellationTokenSource(aquireTimeoutMilliseconds))
{
await blob.AcquireLeaseAsync(
TimeSpan.FromMilliseconds(Math.Min(MaxLockPossibleMilliseconds, timeoutMilliseconds)),
token.TokenId.ToString("N"), null, null, null, cancellationTokenSource.Token);
}
if (timeoutMilliseconds > MaxLockPossibleMilliseconds)
{
KeepExtendingLeaseAsync(async() =>
{
await blob.AcquireLeaseAsync(
TimeSpan.FromMilliseconds(MaxLockPossibleMilliseconds),
token.TokenId.ToString("N"));
}, TimeSpan.FromMilliseconds(MaxLockPossibleMilliseconds), token.RenewCancellation.Token, token.ResourceId); // DO NOT WAIT THIS!!!
}
return(true);
}
catch (Exception e)
{
TheTrace.TraceInformation("Lock attempt - already locked: {0}", e);
// ignore
}
if (i < tries - 1)
{
await Task.Delay(TimeSpan.FromMilliseconds(retryTimeoutMilliseconds / tries));
}
}
return(false);
}
private async Task <bool> Process(CancellationToken cancellationToken)
{
var result = await _queueOperator.NextAsync(
new QueueName(_actorDescriptor.SourceQueueName));
var cancellationTokenSource = new CancellationTokenSource();
if (result.IsSuccessful)
{
TheTrace.TraceInformation("Receieved a message. Id: {0} Queue: {1} ", result.PollingResult.Id, _actorDescriptor.SourceQueueName);
var actor = (IProcessorActor)_serviceLocator.GetService(_actorDescriptor.ActorType);
try
{
// this is NOT supposed to be awaited upon!!
_queueOperator.KeepExtendingLeaseAsync(result.PollingResult, TimeSpan.FromSeconds(30),
cancellationTokenSource.Token).SafeObserve();
// Would have been great to make this fixed memory foot-print with real iterable vs. list
var events = (await actor.ProcessAsync(result.PollingResult)).ToArray(); // the enumerable has to be turned into a list anyway. it does further on
var groups = events.GroupBy(x => x.QueueName);
foreach (var gr in groups)
{
await _queueOperator.PushBatchAsync(gr);
TryDisposeMessages(gr);
}
cancellationTokenSource.Cancel();
await _queueOperator.CommitAsync(result.PollingResult);
TheTrace.TraceInformation("Processing succeeded. Id: {0} Queue: {1} ", result.PollingResult.Id, _actorDescriptor.SourceQueueName);
}
catch (Exception exception)
{
TheTrace.TraceInformation("Processing failed. Id: {0} Queue: {1} ", result.PollingResult.Id, _actorDescriptor.SourceQueueName);
TheTrace.TraceError(exception.ToString());
cancellationTokenSource.Cancel();
_queueOperator.AbandonAsync(result.PollingResult).SafeObserve().Wait();
}
finally
{
if (result.IsSuccessful)
{
TryDisposeMessage(result.PollingResult);
}
_serviceLocator.ReleaseService(actor);
}
}
return(result.IsSuccessful);
}
private void SetupPool()
{
var names = new List <string>();
foreach (var p in _peerManager.GetPeers())
{
names.Add(Queues.PeerAppendLog + p.Address);
}
names.Add(Queues.LogCommit);
names.Add(Queues.HeartBeatReceiveAndCandidacy);
names.Add(Queues.HeartBeatSend);
names.Add(Queues.ProcessCommandQueue);
names.Add(Queues.CreateSnapshot);
_workers = new WorkerPool(names.ToArray());
_workers.Start();
// LogCommit
Func <CancellationToken, Task> logCommit = LogCommit;
_workers.Enqueue(Queues.LogCommit,
new Job(logCommit,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
// receiving heartbeat
Func <CancellationToken, Task> hbr = HeartBeatReceive;
_workers.Enqueue(Queues.HeartBeatReceiveAndCandidacy,
new Job(hbr,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
// sending heartbeat
Func <CancellationToken, Task> hbs = HeartBeatSend;
_workers.Enqueue(Queues.HeartBeatSend,
new Job(hbs,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
// Applying commands received from the clients
Func <CancellationToken, Task> cs = CreateSnapshot;
_workers.Enqueue(Queues.CreateSnapshot,
new Job(cs,
TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(2, (i) => TimeSpan.FromMilliseconds(i * i * 50)),
_settings.ElectionTimeoutMin.Multiply(0.2)));
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Setup finished.");
}
/// <inheritdoc />
public Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
var peers = _peerManager.GetPeers();
var peer = peers.Where(x => x.Id == request.CandidateId).FirstOrDefault();
var peerName = peer?.ShortName ?? request.CandidateId.ToString();
lock (State)
{
if (request.CurrentTerm > State.CurrentTerm)
{
BecomeFollower(request.CurrentTerm);
}
// Reply false if term < currentTerm
if (State.CurrentTerm > request.CurrentTerm)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Rejecting vote of {peerName} due to backward term");
return(Task.FromResult(new RequestVoteResponse()
{
CurrentTerm = State.CurrentTerm,
VoteGranted = false
}));
}
// If votedFor is null or candidateId, and candidate’s log is at least as up-to-date as receiver’s log, grant vote(§5.2, §5.4)
if (!State.LastVotedForId.HasValue && _logPersister.LastIndex <= request.LastLogIndex)
{
State.LastVotedForId = request.CandidateId;
// If election timeout elapses without receiving AppendEntries RPC from current leader OR GRANTING VOTE TO CANDIDATE: convert to candidate
_lastHeartbeat.Set();
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Voting for {peerName} for term {request.CurrentTerm}");
return(Task.FromResult(new RequestVoteResponse()
{
CurrentTerm = State.CurrentTerm,
VoteGranted = true
}));
}
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Rejecting vote of {peerName} for term {request.CurrentTerm} as it did not fulfil");
// assume the rest we send back no
return(Task.FromResult(new RequestVoteResponse()
{
CurrentTerm = State.CurrentTerm,
VoteGranted = false
}));
}
}
private async Task <bool> Process(CancellationToken cancellationToken)
{
var result = await _queueOperator.NextAsync(
new QueueName(_actorDescriptor.SourceQueueName));
var cancellationTokenSource = new CancellationTokenSource();
if (result.IsSuccessful)
{
TheTrace.TraceInformation("Receieved a message. Id: {0} Queue: {1} ", result.PollingResult.Id, _actorDescriptor.SourceQueueName);
var actor = (IProcessorActor)_serviceLocator.GetService(_actorDescriptor.ActorType);
try
{
_queueOperator.KeepExtendingLeaseAsync(result.PollingResult, TimeSpan.FromSeconds(30),
cancellationTokenSource.Token).SafeObserve();
var events = (await actor.ProcessAsync(result.PollingResult)).ToArray();
cancellationTokenSource.Cancel();
await _queueOperator.CommitAsync(result.PollingResult);
var groups = events.GroupBy(x => x.QueueName);
foreach (var gr in groups)
{
await _queueOperator.PushBatchAsync(gr);
TryDisposeMessages(gr);
}
TheTrace.TraceInformation("Processing succeeded. Id: {0} Queue: {1} ", result.PollingResult.Id, _actorDescriptor.SourceQueueName);
}
catch (Exception exception)
{
TheTrace.TraceInformation("Processing failed. Id: {0} Queue: {1} ", result.PollingResult.Id, _actorDescriptor.SourceQueueName);
TheTrace.TraceError(exception.ToString());
cancellationTokenSource.Cancel();
_queueOperator.AbandonAsync(result.PollingResult).SafeObserve().Wait();
}
finally
{
if (result.IsSuccessful)
{
TryDisposeMessage(result.PollingResult);
}
_serviceLocator.ReleaseService(actor);
}
}
return(result.IsSuccessful);
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parserTypeName = blobFileArrived.Source.DynamicProperties["Parser"].ToString();
if (parserTypeName == null)
{
throw new NullReferenceException("parserTypeName");
}
var parserType = Assembly.GetExecutingAssembly().GetType(parserTypeName) ?? Type.GetType(parserTypeName);
if (parserType == null)
{
throw new InvalidOperationException("Parser type was null: " + parserTypeName);
}
var parser = (IParser)Activator.CreateInstance(parserType);
bool hasAnything = false;
foreach (var entity in parser.Parse(stream, blob.Uri))
{
await _pusher.PushAsync(entity, blobFileArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}