private void ReceiveGetKeyIds()
{
var keys = _dataEntries.Where(kvp => !(kvp.Value.Item1.Data is DeletedData))
.Select(x => x.Key)
.ToImmutableHashSet();
Sender.Tell(new GetKeysIdsResult(keys));
}
public async Task SyncGroupAsync(CancellationToken cancellationToken)
{
if (_disposeCount > 0)
{
throw new ObjectDisposedException($"Consumer {{GroupId:{GroupId},MemberId:{MemberId}}} is no longer valid");
}
var groupAssignments = await _joinSemaphore.LockAsync(
async() => {
if (IsLeader)
{
var encoder = Encoders[ProtocolType];
var assigner = encoder.GetAssignor(_groupProtocol);
var assignments = await assigner.AssignMembersAsync(Router, GroupId, GenerationId, _memberMetadata, cancellationToken).ConfigureAwait(false);
return(assignments.Select(pair => new SyncGroupRequest.GroupAssignment(pair.Key, pair.Value)));
}
return(null);
}, _disposeToken.Token).ConfigureAwait(false);
await Task.WhenAll(_batches.Values.Select(b => b.CommitMarkedIgnoringDisposedAsync(cancellationToken))).ConfigureAwait(false);
SyncGroupResponse response;
try {
var request = new SyncGroupRequest(GroupId, GenerationId, MemberId, groupAssignments);
response = await Router.SyncGroupAsync(request, new RequestContext(protocolType : ProtocolType), Configuration.GroupCoordinationRetry, cancellationToken).ConfigureAwait(false);
} catch (RequestException ex) {
switch (ex.ErrorCode)
{
case ErrorCode.REBALANCE_IN_PROGRESS:
Router.Log.Info(() => LogEvent.Create(ex.Message));
TriggerRejoin();
return;
case ErrorCode.GROUP_AUTHORIZATION_FAILED:
case ErrorCode.UNKNOWN_MEMBER_ID:
Router.Log.Warn(() => LogEvent.Create(ex));
_leaveOnDispose = false; // no point in attempting to leave the group since it will fail
_disposeToken.Cancel();
break;
}
throw;
}
_syncSemaphore.Lock(() => {
_assignment = response.member_assignment;
var validPartitions = response.member_assignment.PartitionAssignments.ToImmutableHashSet();
var invalidPartitions = _batches.Where(pair => !validPartitions.Contains(pair.Key)).ToList();
foreach (var invalidPartition in invalidPartitions)
{
invalidPartition.Value.Dispose();
}
_batches = _batches.RemoveRange(invalidPartitions.Select(pair => pair.Key));
}, _disposeToken.Token);
if (Interlocked.Increment(ref _syncCount) == 1)
{
_fetchSemaphore.Release();
}
}
private void OnOutputGroupChanged(IImmutableDictionary <string, IOutputGroup> e)
{
foreach (var outputGroupPair in e.Where(pair => KnownOutputGroups.Contains(pair.Key)))
{
_outputGroups[outputGroupPair.Key] = new HashSet <string>(outputGroupPair.Value.Outputs.Select(output => output.Key));
}
}
/// <summary>
/// Writes a <see cref="Packet"/> object to the currrent stream.
/// </summary>
/// <param name="packet">The <see cref="Packet"/> object.</param>
public void Write(Packet packet)
{
var range = new Range <long>();
range.Minimum = BaseStream.Position;
packet.Header.CopyTo(BaseStream);
packet.Body.CopyTo(BaseStream);
range.Maximum = BaseStream.Position - 1;
var overwrittenPackets = packets
.Where(x => range.IsInsideRange(new Range <long>
{
Minimum = x.Key,
Maximum = x.Key + x.Value - 1
}))
.Select(x => x.Key);
foreach (var key in overwrittenPackets)
{
packets = packets.Remove(key);
}
packets = packets.Add(range.Minimum, range.Maximum - range.Minimum + 1);
}
private IImmutableDictionary <string, ITypeSpecBuilder> IntrospectPlaceholders(IImmutableDictionary <string, ITypeSpecBuilder> metamodel, Func <IIntrospector> getIntrospector)
{
var ph = metamodel.Where(i => string.IsNullOrEmpty(i.Value.FullName)).Select(i => i.Value.Type);
var mm = ph.AsParallel().SelectMany(type => IntrospectSpecification(type, metamodel, getIntrospector).metamodel).Distinct(new TypeSpecKeyComparer()).ToDictionary(kvp => kvp.Key, kvp => kvp.Value).ToImmutableDictionary();
return(mm.Any(i => string.IsNullOrEmpty(i.Value.FullName))
? IntrospectPlaceholders(mm, getIntrospector)
: mm);
}
private State(
IImmutableDictionary <char, IImmutableSet <char> > waitingForByState)
{
WaitingForByState = waitingForByState;
NextAvailable = WaitingForByState
.Where(kv => kv.Value.IsEmpty())
.OrderBy(kv => kv.Key)
.Select(kv => kv.Key)
.ToArray();
}
private IImmutableDictionary <string, ITypeSpecBuilder> IntrospectSystemTypes(Type[] systemTypes, IImmutableDictionary <string, ITypeSpecBuilder> specDictionary)
{
var placeholders = GetPlaceholders(systemTypes, ClassStrategy);
var pending = specDictionary.Where(i => i.Value.IsPendingIntrospection).Select(i => i.Value.Type);
var toIntrospect = placeholders.Select(kvp => kvp.Value.Type).Union(pending).ToArray();
return(placeholders.Any()
? IntrospectTypes(toIntrospect, specDictionary.AddRange(placeholders))
: specDictionary);
}
private void ReceiveStatus(IImmutableDictionary <string, ByteString> otherDigests, int chunk, int totChunks)
{
if (_log.IsDebugEnabled)
{
_log.Debug("Received gossip status from [{0}], chunk {1}/{2} containing [{3}]", Sender.Path.Address, chunk + 1, totChunks, string.Join(", ", otherDigests.Keys));
}
// if no data was send we do nothing
if (otherDigests.Count == 0)
{
return;
}
var otherDifferentKeys = otherDigests
.Where(x => IsOtherDifferent(x.Key, x.Value))
.Select(x => x.Key)
.ToImmutableHashSet();
var otherKeys = otherDigests.Keys.ToImmutableHashSet();
var myKeys = (totChunks == 1
? _dataEntries.Keys
: _dataEntries.Keys.Where(x => Math.Abs(x.GetHashCode()) % totChunks == chunk))
.ToImmutableHashSet();
var otherMissingKeys = myKeys.Except(otherKeys);
var keys = otherDifferentKeys
.Union(otherMissingKeys)
.Take(_settings.MaxDeltaElements)
.ToArray();
if (keys.Length != 0)
{
if (_log.IsDebugEnabled)
{
_log.Debug("Sending gossip to [{0}]: {1}", Sender.Path.Address, string.Join(", ", keys));
}
var g = new Gossip(keys.Select(k => new KeyValuePair <string, DataEnvelope>(k, GetData(k))).ToImmutableDictionary(), !otherDifferentKeys.IsEmpty);
Sender.Tell(g);
}
var myMissingKeys = otherKeys.Except(myKeys);
if (!myMissingKeys.IsEmpty)
{
if (Context.System.Log.IsDebugEnabled)
{
Context.System.Log.Debug("Sending gossip status to {0}, requesting missing {1}", Sender.Path.Address, string.Join(", ", myMissingKeys));
}
var status = new Internal.Status(myMissingKeys.Select(x => new KeyValuePair <string, ByteString>(x, NotFoundDigest)).ToImmutableDictionary(), chunk, totChunks);
Sender.Tell(status);
}
}
/// <summary>
/// Adds an element ID to this set, which enables JavaScript to change the classes on the element.
/// </summary>
internal void AddElementId(string id)
{
foreach (var keyValuePair in ConditionsByClassName.Where(i => i.Value != null))
{
keyValuePair.Value.AddJsModificationStatement(
expression => "if( {0} ) {1} else {2}".FormatWith(
expression,
"{ " + "$( '#{0}' ).addClass( '{1}' );".FormatWith(id, keyValuePair.Key) + " }",
"{ " + "$( '#{0}' ).removeClass( '{1}' );".FormatWith(id, keyValuePair.Key) + " }"));
}
}
/// <summary>
/// Adds an element ID to this set, which enables JavaScript to change the classes on the element.
/// </summary>
internal void AddElementId(string id)
{
foreach (var keyValuePair in JsModificationStatementAdderAndInclusionPredicatePairsByClassName.Where(i => i.Value != null))
{
keyValuePair.Value.Item1(
classIncludedExpression =>
"if( {0} ) {1} else {2}".FormatWith(
classIncludedExpression,
"{ " + "$( '#{0}' ).addClass( '{1}' );".FormatWith(id, keyValuePair.Key) + " }",
"{ " + "$( '#{0}' ).removeClass( '{1}' );".FormatWith(id, keyValuePair.Key) + " }"));
}
}
public ConstantTypeOptimisingVelocityExpressionFactory(IBinderFactory binderFactory, IImmutableDictionary <string, object> globals)
: base(binderFactory)
{
var nullGlobals = globals?.Where(x => x.Value == null);
if (nullGlobals?.Any() ?? false)
{
throw new ArgumentOutOfRangeException(nameof(globals), $"The following global variables must not be null: {String.Join(", ", nullGlobals.Select(x => x.Key))}");
}
_globals = globals;
}
private void OnOutputGroupChanged(IImmutableDictionary <string, IOutputGroup> e)
{
foreach (var outputGroupPair in e.Where(pair => KnownOutputGroups.Contains(pair.Key)))
{
var outputs = outputGroupPair.Value
.Outputs
.Where(output => !output.Value.ContainsKey(CopyToOutputDirectory) ||
!string.Equals(output.Value[CopyToOutputDirectory], Never, StringComparison.InvariantCultureIgnoreCase))
.Select(output => output.Key);
_outputGroups[outputGroupPair.Key] = new HashSet <string>(outputs);
}
}
private IImmutableDictionary <string, ITypeSpecBuilder> IntrospectPlaceholders(IImmutableDictionary <string, ITypeSpecBuilder> metamodel)
{
var ph = metamodel.Where(i => string.IsNullOrEmpty(i.Value.FullName)).Select(i => i.Value.Type);
var mm = ph.AsParallel().SelectMany(type => IntrospectSpecification(type, metamodel).Item2).Distinct(new DE()).ToDictionary(kvp => kvp.Key, kvp => kvp.Value).ToImmutableDictionary();
if (mm.Any(i => string.IsNullOrEmpty(i.Value.FullName)))
{
return(IntrospectPlaceholders(mm));
}
return(mm);
}
async Task <IImmutableDictionary <string, IModuleIdentity> > GetModuleIdentitiesAsync(Diff diff)
{
IList <string> addedOrUpdatedModuleNames = diff.AddedOrUpdated.Select(m => ModuleIdentityHelper.GetModuleIdentityName(m.Name)).ToList();
IEnumerable <string> removedModuleNames = diff.Removed.Select(ModuleIdentityHelper.GetModuleIdentityName);
IImmutableDictionary <string, Identity> identities = (await this.identityManager.GetIdentities()).ToImmutableDictionary(i => i.ModuleId);
// Create identities for all modules that are in the deployment but aren't in iotedged.
IEnumerable <string> createIdentities = addedOrUpdatedModuleNames.Where(m => !identities.ContainsKey(m));
// Update identities for all modules that are in the deployment and are in iotedged (except for Edge Agent which gets special
// treatment in iotedged).
//
// NOTE: This update can potentiatlly be made more efficient by checking that an update is actually needed, i.e. if auth type
// is not SAS and/or if the credentials are not what iotedged expects it to be.
IEnumerable <Identity> updateIdentities = addedOrUpdatedModuleNames
.Where(m => identities.ContainsKey(m) && m != Constants.EdgeAgentModuleIdentityName)
.Select(m => identities[m]);
// Remove identities which exist in iotedged but don't exist in the deployment anymore. We exclude however, identities that
// aren't managed by Edge since these have been created by some out-of-band process and Edge doesn't "own" the identity.
IEnumerable <string> removeIdentities = removedModuleNames.Where(
m => identities.ContainsKey(m) &&
Constants.ModuleIdentityEdgeManagedByValue.Equals(identities[m].ManagedBy, StringComparison.OrdinalIgnoreCase));
// First remove identities (so that we don't go over the IoTHub limit).
await Task.WhenAll(removeIdentities.Select(i => this.identityManager.DeleteIdentityAsync(i)));
// Create/update identities.
IEnumerable <Task <Identity> > createTasks = createIdentities.Select(i => this.identityManager.CreateIdentityAsync(i, Constants.ModuleIdentityEdgeManagedByValue));
IEnumerable <Task <Identity> > updateTasks = updateIdentities.Select(i => this.identityManager.UpdateIdentityAsync(i.ModuleId, i.GenerationId, i.ManagedBy));
Identity[] upsertedIdentities = await Task.WhenAll(createTasks.Concat(updateTasks));
List <IModuleIdentity> moduleIdentities;
moduleIdentities = upsertedIdentities.Select(m => this.GetModuleIdentity(m)).ToList();
// Add back the unchanged identities (including Edge Agent).
var upsertedIdentityList = moduleIdentities.Select(i => i.ModuleId).ToList();
var unchangedIdentities = identities.Where(i => !removedModuleNames.Contains(i.Key) && !upsertedIdentityList.Contains(i.Key));
moduleIdentities.AddRange(unchangedIdentities.Select(i => this.GetModuleIdentity(i.Value)));
return(moduleIdentities.ToImmutableDictionary(m => ModuleIdentityHelper.GetModuleName(m.ModuleId)));
}
private static bool ExcludePrivateFieldsBasedOnRewrittenNodes(
IImmutableDictionary <ISymbol, PrivateField> privateFields,
IImmutableDictionary <SyntaxNode, ISymbol> references,
ISet <SyntaxNode> rewrittenNodes)
{
var partiallyRewrittenSymbols = references
.Where(r => !rewrittenNodes.Contains(r.Key))
.Select(r => r.Value)
.ToImmutableHashSet();
foreach (var partiallyRewrittenSymbol in partiallyRewrittenSymbols)
{
privateFields[partiallyRewrittenSymbol].Excluded = true;
}
var allSymbolsToRewrite = references.Values.ToImmutableHashSet();
return(partiallyRewrittenSymbols.Count == allSymbolsToRewrite.Count);
}
private void InitRemovedNodePruning()
{
var removedSet = _removedNodes.Where(x => (_allReachableClockTime - x.Value) > _maxPruningDisseminationNanos)
.Select(x => x.Key)
.ToImmutableHashSet();
if (removedSet.Any())
{
foreach (var entry in _dataEntries)
{
var key = entry.Key;
var envelope = entry.Value.Item1;
foreach (var removed in removedSet)
{
Action init = () =>
{
var newEnvelope = envelope.InitRemovedNodePruning(removed, _selfUniqueAddress);
Context.System.Log.Debug("Initiating pruning of {0} with data {1}", removed, key);
SetData(key, newEnvelope);
};
if (envelope.NeedPruningFrom(removed))
{
if (envelope.Data is IRemovedNodePruning)
{
var res = envelope.Pruning[removed];
if (res == null)
{
init();
}
else if (res.Phase is PruningInitialized && res.Owner == _selfUniqueAddress)
{
init();
}
}
}
}
}
}
}
public override int GetHashCode()
{
var hash = 17;
hash = hash * 23 + TagName.GetHashCode();
foreach (
var attribute in _attributes.Where(attribute => !string.Equals(attribute.Key, "style") && !string.Equals(attribute.Key, "class"))
.OrderBy(attribute => attribute.Key))
{
hash = hash * 23 + attribute.Key.GetHashCode();
hash = hash * 23 + attribute.Value.GetHashCode();
}
foreach (var style in Styles.OrderBy(style => style.Key))
{
hash = hash * 23 + style.Key.GetHashCode();
hash = hash * 23 + style.Value.GetHashCode();
}
hash = Classes.OrderBy(c => c).Aggregate(hash, (current, @class) => current * 23 + @class.GetHashCode());
return(Contents.Aggregate(hash, (current, content) => current * 23 + content.GetHashCode()));
}
private static bool ExcludePrivateFieldsBasedOnRewrittenNodes(
IImmutableDictionary<ISymbol, PrivateField> privateFields,
IImmutableDictionary<SyntaxNode, ISymbol> references,
ISet<SyntaxNode> rewrittenNodes)
{
var partiallyRewrittenSymbols = references
.Where(r => !rewrittenNodes.Contains(r.Key))
.Select(r => r.Value)
.ToImmutableHashSet();
foreach (var partiallyRewrittenSymbol in partiallyRewrittenSymbols)
{
privateFields[partiallyRewrittenSymbol].Excluded = true;
}
var allSymbolsToRewrite = references.Values.ToImmutableHashSet();
return partiallyRewrittenSymbols.Count == allSymbolsToRewrite.Count;
}
/// <summary>
/// Reads the next PAR2 packet of the given packet type.
/// </summary>
/// <param name="type">The packet type.</param>
/// <returns>A <see cref="Packet"/> object of the given type, read from the stream;
/// <see cref="null"/> null if there are no more packets of the given type.</returns>
public Packet ReadPacket(PacketType type)
{
return(packets.Where(x => BaseStream.Position <= x.Key && x.Value == type).Select(x => ReadPacket(x.Key)).FirstOrDefault());
}
public static IImmutableDictionary <K, T> filter <K, T>(IImmutableDictionary <K, T> map, Func <T, bool> predicate) => map.Where(kv => predicate(kv.Value)).ToImmutableDictionary();
