public async Task <bool> TryLockAsync(
LockToken token,
int tries = 16, // this is NOT retry - it is try
int retryTimeoutMilliseconds = 15000,
int timeoutMilliseconds = 15000)
{
if (tries < 1)
{
tries = 1;
}
var blob = await GetBlobAsync(token.ResourceId);
for (int i = 0; i < tries; i++)
{
try
{
await blob.AcquireLeaseAsync(
TimeSpan.FromMilliseconds(timeoutMilliseconds),
token.TokenId.ToString("N"));
return(true);
}
catch (Exception e)
{
TheTrace.TraceInformation("Lock attempt - already locked: {0}", e);
// ignore
}
await Task.Delay(TimeSpan.FromMilliseconds(retryTimeoutMilliseconds / tries));
}
return(false);
}
private static Func <Task> SendCommand(int run)
{
return(async() =>
{
var r = new Random();
while (_run == run)
{
try
{
for (int i = 0; i < r.Next(1, 10); i++)
{
var payload = BitConverter.GetBytes(r.Next(0, 100000)).Concat(BitConverter.GetBytes(r.Next(0, 100000))).ToArray();
await _cluster.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = payload
});
}
TheTrace.TraceInformation("Sent commands");
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
}
await Task.Delay(r.Next(0, 50));
}
});
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
await _pusher.PushAll(PreprocessEntities(entities, shardKeyArrived, shardKeyTime), shardKeyArrived.Source).ConfigureAwait(false);
}).ConfigureAwait(false);
return(Enumerable.Empty <Event>());
}
private IEnumerable <DynamicTableEntity> PreprocessEntities(IEnumerable <DynamicTableEntity> entities, ShardKeyArrived shardKeyArrived, string shardKeyTime)
{
var minDateTime = DateTimeOffset.MaxValue;
var n = 0;
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning(
"SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
yield return(entity);
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (n > 0)
{
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
}
public async Task <bool> UpdateMappingAsync(string baseUrl, string indexName, string typeName, string mapping)
{
TheTrace.TraceInformation("Adding {0} type to {1} index.", typeName, indexName);
baseUrl = baseUrl.TrimEnd('/');
var url = string.Format(MappingFormat, baseUrl, indexName, typeName);
var response = await _httpClient.PutAsync(url,
new StringContent(mapping, Encoding.UTF8, "application/json"));
var text = await response.Content.ReadAsStringAsync();
var mappings = _existingIndices[indexName];
if (response.IsSuccessStatusCode)
{
mappings.TryAdd(typeName, null);
return(true);
}
else
{
throw new ApplicationException(string.Format("Error {0}: {1}",
response.StatusCode,
text));
}
}
private async Task DoScheduleAsync(PeckSource source)
{
var oldTimestamp = source.Timestamp;
try
{
await _eventQueueOperator.PushAsync(new Event(new PeckSourceScheduled()
{
Source = source
})
{
QueueName = QueueName.FromTopicName("PeckSourceScheduled").ToString()
});
source.LastOffset = DateTimeOffset.UtcNow;
await _table.ExecuteAsync(TableOperation.InsertOrMerge(source));
}
catch (Exception e)
{
source.LastOffset = oldTimestamp;
TheTrace.TraceError("Error scheduling source {0} : {1}", source.Name, e);
try
{
_table.Execute(TableOperation.InsertOrMerge(source));
}
catch (Exception ex)
{
TheTrace.TraceError("Error saving error (!) in table. Source {0} : {1}", source.Name, ex);
}
}
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
bool hasAnything = false;
foreach (var entity in entities)
{
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public void CanAdd2And2EvenIfItFails()
{
var maths = new Worker("maths");
var ran = false;
var timesForExceptions = 4;
var job = new Job <int>(c =>
{
if (timesForExceptions-- > 0)
{
throw new ApplicationException();
}
ran = true;
return(Task.FromResult(2 + 2));
},
TheTrace.LogPolicy("this").RetryForeverAsync()
);
maths.Start();
maths.Enqueue(job);
while (!job.IsFinished)
{
Thread.Sleep(100);
}
Assert.True(ran);
}
private async Task PushbatchAsync()
{
if (_stringBuilder.Length == 0)
{
return;
}
try
{
var responseMessage = await _httpClient.PostAsync(_esUrl + "_bulk",
new StringContent(_stringBuilder.ToString(),
Encoding.UTF8, "application/json"));
_stringBuilder.Clear();
_numberOfRecords = 0;
responseMessage.EnsureSuccessStatusCode();
TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushing to {0}", _esUrl);
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
throw;
}
}
public async Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (!string.IsNullOrEmpty(source.StopOffsetPoint) && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
{
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
var lockToken = new LockToken(source.ToTypeKey());
int seconds =
Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds));
if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000)))
{
TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey());
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
try
{
var events = await DoSchedule(source);
return(new Tuple <IEnumerable <Event>, bool>(events, true));
}
finally
{
Task.Run(() => _lockStore.ReleaseLockAsync(lockToken)).Wait();
}
}
public async Task PushAsync(DynamicTableEntity entity, DiagnosticsSourceSummary source)
{
if (source.Filter == null)
{
source.Filter = string.Empty;
}
if (!_filters.ContainsKey(source.Filter))
{
_filters.Add(source.Filter, new SimpleFilter(source.Filter));
}
if (!_filters[source.Filter].Satisfies(entity))
{
return;
}
var op = new
{
index = new
{
_index = source.IndexName ?? _indexNamer.BuildName(entity.Timestamp, source.DynamicProperties["MappingName"].ToString().ToLowerInvariant()),
_type = source.DynamicProperties["MappingName"].ToString(),
_id = entity.PartitionKey + entity.RowKey
}
};
var doc = new JObject();
doc.Add("@timestamp", entity.Timestamp);
doc.Add("PartitionKey", entity.PartitionKey);
doc.Add("RowKey", entity.RowKey);
doc.Add("cb_type", source.TypeName);
foreach (var property in entity.Properties)
{
if (property.Key != DiagnosticsSource.CustomAttributesFieldName)
{
doc[property.Key] = JToken.FromObject(property.Value.PropertyAsObject);
}
}
if (entity.Properties.ContainsKey(DiagnosticsSource.CustomAttributesFieldName))
{
foreach (var keyValue in GetNameValues(entity.Properties[DiagnosticsSource.CustomAttributesFieldName].StringValue))
{
doc[keyValue.Key] = keyValue.Value;
}
}
_batch.AddDoc(JsonConvert.SerializeObject(op).Replace("\r\n", " "), doc.ToString().Replace("\r\n", " "));
if (_batch.Count >= _batchSize)
{
await PushbatchAsync();
TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushed records to ElasticSearch for {0}-{1}",
source.PartitionKey,
source.RowKey);
}
}
private void LoadState(Guid?seedId = null)
{
using (var tx = _env.BeginReadOnlyTransaction())
{
LoadLastTermAndIndex(tx);
// snapshot
Snapshot ss;
if (TryGetLastSnapshot(out ss))
{
LogOffset = ss.LastIncludedIndex + 1;
}
// state
_state = new PersistentState(seedId);
Bufferable val;
if (tx.TryGet(_stateDb, StateDbKeys.Id, out val))
{
_state.Id = val;
if (tx.TryGet(_stateDb, StateDbKeys.CurrentTerm, out val)) // this should be always TRUE
{
_state.CurrentTerm = val;
}
if (tx.TryGet(_stateDb, StateDbKeys.LastVotedFor, out val))
{
Guid g = val;
_state.LastVotedForId = g == Guid.Empty ? (Guid?)null : g;
}
}
}
TheTrace.TraceInformation($"Seed Id was {seedId} and now {_state.Id}");
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardRangeArrived>();
TheTrace.TraceInformation("Got {0}->{1} from {2}", shardKeyArrived.InclusiveStartKey,
shardKeyArrived.InclusiveEndKey, shardKeyArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString());
var entities = table.ExecuteQuery(new TableQuery().Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", "ge", shardKeyArrived.InclusiveStartKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", "le", shardKeyArrived.InclusiveEndKey))));
bool hasAnything = false;
foreach (var entity in entities)
{
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public async Task KeepFeedingAFollowerAndNeverDreamsOfPower()
{
var t = new CancellationTokenSource();
var leaderId = Guid.NewGuid();
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
// to set the term to 1
await _server.AppendEntriesAsync(new AppendEntriesRequest()
{
CurrentTerm = 1,
Entries = new byte[0][],
LeaderCommitIndex = 20,
LeaderId = leaderId,
PreviousLogIndex = -1,
PreviousLogTerm = 0
});
_server.LastHeartBeat = new AlwaysRecentTimestamp();
_server.LastHeartBeatSent = new AlwaysRecentTimestamp();
TheTrace.TraceInformation("OK, now this is before wait...");
Thread.Sleep(1000);
TheTrace.TraceInformation("Wait finished.");
Assert.Equal(1, _server.State.CurrentTerm);
TheTrace.TraceInformation("Checked Term.");
Assert.Equal(Role.Follower, _server.Role);
TheTrace.TraceInformation("Checked Role.");
t.Cancel();
}
public async Task KeepExtendingLeaseAsync(Event message, TimeSpan howLong, CancellationToken cancellationToken)
{
await Task.Delay(new TimeSpan(2 * howLong.Ticks / 3), cancellationToken);
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var underlyingMessage = (BrokeredMessage)message.UnderlyingMessage;
await underlyingMessage.RenewLockAsync();
await Task.Delay(new TimeSpan(2 * howLong.Ticks / 3), cancellationToken);
}
catch (Exception exception)
{
TheTrace.TraceError(exception.ToString());
break;
}
}
}
public async Task <PollerResult <Event> > NextAsync(QueueName name)
{
try
{
BrokeredMessage message = null;
if (name.IsSimpleQueue)
{
var client = _clientProvider.GetQueueClient(name);
message = await client.ReceiveAsync(_longPollingTimeout);
}
else
{
var client = _clientProvider.GetSubscriptionClient(name);
message = await client.ReceiveAsync(_longPollingTimeout);
}
return(new PollerResult <Event>(message != null,
message == null
? null
: message.ToEvent(name)
));
}
catch (Exception e)
{
TheTrace.TraceWarning(e.ToString());
return(new PollerResult <Event>(false, null));
}
}
/// <summary>
/// Note: RunAsync and Start ping-pong between each other.
/// </summary>
/// <returns></returns>
private async Task RunAsync()
{
bool result = false;
try
{
_cancellationTokenSource = new CancellationTokenSource();
result = await _work(_cancellationTokenSource.Token);
}
catch (Exception exception)
{
TheTrace.TraceWarning(exception.ToString());
}
if (result)
{
_interval.Reset();
}
else
{
await Task.Delay(_interval.Next());
}
if (_isWorking)
{
Start();
}
}
private async Task KeepExtendingLeaseAsync(Func <Task> extendLeaseAsync, TimeSpan howLong, CancellationToken cancellationToken, string resource)
{
await EnsureExists();
var thisLong = new TimeSpan(2 * howLong.Ticks / 3); // RATM: how long? This long, what you reap is what you sew!
await Task.Delay(thisLong, cancellationToken);
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
await extendLeaseAsync();
TheTrace.TraceInformation("Extended the lifetime of the lease for {0}...", resource);
await Task.Delay(thisLong, cancellationToken);
}
catch (Exception exception)
{
if (!cancellationToken.IsCancellationRequested) // it is OK if cancellation requested, it would have been cancellation
{
TheTrace.TraceError(exception.ToString());
}
break;
}
}
}
public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
var key = source.ToTypeKey();
if (source.IsActive.GetValueOrDefault(true))
{
var consume = EventHubConsumer.Consumers.AddOrUpdate(key,
new Lazy <EventHubConsumer>(() =>
{
TheTrace.TraceInformation("Just added this eventHub consumer {0}", key);
return(new EventHubConsumer(_pusher, source.ToSummary()));
}),
(kk, vv) => vv);
// to make sure it gets accessed and created if new otherwise system is 'lazy'
TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName);
}
else
{
Lazy <EventHubConsumer> consumer = null;
if (EventHubConsumer.Consumers.TryRemove(key, out consumer))
{
consumer.Value.Dispose();
TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key);
}
}
return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false)));
}
private void SetupPeerAppendLogJobs(IEnumerable <Peer> peers)
{
foreach (var w in _workers.GetWorkers(Queues.PeerAppendLog))
{
w.Start();
}
_workers.GetWorkers(Queues.ProcessCommandQueue).Single().Start();
foreach (var p in peers)
{
var localP = p;
var q = Queues.PeerAppendLog + localP.Address;
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] setting up peer append log for queue {q}");
var todo = PeerAppendLog(localP);
_workers.Enqueue(q,
new Job(todo,
TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(3, (i) => TimeSpan.FromMilliseconds(i * i * 50)),
TimeSpan.FromMilliseconds(30)));
}
// Applying commands received from the clients
Func <CancellationToken, Task> pcq = ProcessCommandsQueue;
_workers.Enqueue(Queues.ProcessCommandQueue,
new Job(pcq,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
if (source.LastOffsetPoint == null)
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O");
}
var lastOffset = DateTimeOffset.Parse(source.LastOffsetPoint);
var events = new List <Event>();
var graceMinutes = source.GracePeriodMinutes ?? 3;
var now = DateTimeOffset.UtcNow.DropSecondAndMilliseconds();
var newLastOffset = lastOffset;
int n = 1; // start from a minute after
while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n)))
{
newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n))
.DropSecondAndMilliseconds(); // just to be sure
var shardKeys = GetShardKeys(newLastOffset);
events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived {
Source = source.ToSummary(), ShardKey = shardKey
})));
if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun)
{
break;
}
n++;
TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First());
}
source.LastOffsetPoint = newLastOffset.ToString("O");
return(Task.FromResult((IEnumerable <Event>)events));
}
public async Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived)
{
//var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
CloudStorageAccount account = null;
if (!String.IsNullOrWhiteSpace(shardKeyArrived.Source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(shardKeyArrived.Source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
try
{
account = new CloudStorageAccount(accountSas, shardKeyArrived.Source.AccountName, endpointSuffix: "", useHttps: true);
}
catch (Exception ex)
{
TheTrace.TraceError(ex.ToString());
}
}
else
{
account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
}
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString());
return(await table.ExecuteQueryAsync(new TableQuery().Where(TableQuery.GenerateFilterCondition("PartitionKey", "eq", shardKeyArrived.ShardKey))).ConfigureAwait(false));
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
_telemetryProvider.WriteTelemetry(
"BlobFileActor receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
//var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(blobFileArrived.Source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(blobFileArrived.Source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, blobFileArrived.Source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
var hasAnything = false;
var minDateTime = DateTimeOffset.UtcNow;
var records = parser.Parse(() => stream, blob.Uri, blobFileArrived.Source, new ParseCursor(blobFileArrived.Position ?? 0)
{
EndPosition = blobFileArrived.EndPosition ?? 0
});
var seenPages = await _pusher.PushAll(records, blobFileArrived.Source).ConfigureAwait(false);
hasAnything = seenPages > 0;
if (hasAnything)
{
_telemetryProvider.WriteTelemetry(
"BlobFileActor message processing duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
blobFileArrived.Source.TypeName);
}
}, blobFileArrived.Source.TypeName);
return(Enumerable.Empty <Event>());
}
private async Task Candidacy(CancellationToken c)
{
while (_role == Role.Candidate)
{
var forMe = 1; // vote for yourself
var againstMe = 0;
var peers = _peerManager.GetPeers().ToArray();
var concensus = (peers.Length / 2) + 1;
var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(3, (i) => TimeSpan.FromMilliseconds(i * i * 30));
var policy = Policy.TimeoutAsync(_settings.CandidacyTimeout).WrapAsync(retry);
var request = new RequestVoteRequest()
{
CandidateId = State.Id,
CurrentTerm = State.CurrentTerm,
LastLogIndex = _logPersister.LastIndex,
LastLogTerm = _logPersister.LastEntryTerm
};
var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.RequestVoteAsync(request))));
var maxTerm = 0L;
foreach (var r in all)
{
if (r.Outcome == OutcomeType.Successful)
{
if (r.Result.CurrentTerm > maxTerm)
{
maxTerm = r.Result.CurrentTerm;
}
if (r.Result != null && r.Result.VoteGranted)
{
forMe++;
}
else
{
againstMe++;
}
}
}
if (againstMe >= concensus)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. I got rejected with {againstMe} votes :/");
BecomeFollower(maxTerm);
}
else if (forMe >= concensus)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. I got elected with {forMe} votes! :)");
BecomeLeader();
}
else
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. Non-conclusive with {forMe} for me and {againstMe} against me.");
}
}
}
public void Dispose()
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Disposing server.");
_workers.Stop();
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Disposing server. Workers stopped.");
_logPersister.Dispose();
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Disposing server. Log Persister stopped.");
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var blobPath = source.GetProperty <string>("BlobPath");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath);
blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with /
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
DateTimeOffset maxOffset = offset.TimeOffset;
FileOffset newOffset = null;
var events = new List <Event>();
foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob)
.Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
{
if (blob.Properties.LastModified > offset.TimeOffset)
{
var filename = blob.Uri.ToString();
newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0);
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(),
BlobId = filename,
Position = 0,
EndPosition = blob.Properties.Length
}));
TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename);
}
}
source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
private void BecomeFollower(long term)
{
State.LastVotedForId = null;
DestroyPeerAppendLogJobs();
_lastHeartbeat.Set(); // important not to become candidate again at least for another timeout
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] About to set term from {State.CurrentTerm} to {term}");
State.CurrentTerm = term;
OnRoleChanged(_role = Role.Follower);
}
private async Task HeartBeatSend(CancellationToken c)
{
if (_role != Role.Leader)
{
return;
}
if (_lastHeartbeatSent.Since() < _settings.ElectionTimeoutMin.Multiply(0.2))
{
return;
}
var currentTerm = State.CurrentTerm; // create a var. Could change during the method leading to confusing logs.
var req = new AppendEntriesRequest()
{
CurrentTerm = currentTerm,
Entries = new byte[0][],
LeaderCommitIndex = _volatileState.CommitIndex,
LeaderId = State.Id,
PreviousLogIndex = long.MaxValue,
PreviousLogTerm = long.MaxValue
};
var peers = _peerManager.GetPeers().ToArray();
var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync();
var policy = Policy.TimeoutAsync(_settings.ElectionTimeoutMin.Multiply(0.2)).WrapAsync(retry);
var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.AppendEntriesAsync(req))));
var maxTerm = currentTerm;
foreach (var r in all)
{
if (r.Outcome == OutcomeType.Successful)
{
if (!r.Result.IsSuccess)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Got this reason for unsuccessful AppendEntriesAsync from a peer: {r.Result.Reason}");
}
// NOTE: We do NOT change leadership if they send higher term, since they could be candidates whom will not become leaders
// we actually do not need to do anything with the result other than logging it
if (r.Result.CurrentTerm > maxTerm)
{
maxTerm = r.Result.CurrentTerm;
}
}
}
if (maxTerm > State.CurrentTerm)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Revolution brewing. Terms as high as {maxTerm} (vs my {currentTerm}) were seen.");
}
_lastHeartbeatSent.Set();
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var minDateTime = DateTimeOffset.MaxValue;
var hasAnything = false;
int n = 0;
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning("SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (hasAnything)
{
await _pusher.FlushAsync();
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
});
return(Enumerable.Empty <Event>());
}
private void BecomeCandidate()
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] BecomeCandidate start");
State.IncrementTerm();
State.LastVotedForId = State.Id;
_leaderAddress = null;
DestroyPeerAppendLogJobs();
OnRoleChanged(_role = Role.Candidate);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] BecomeCandidate end");
}
