public StateSyncItem(Keccak hash, NodeDataType nodeType, int level, uint rightness) { Hash = hash; NodeDataType = nodeType; Level = (byte)level; Rightness = rightness; }
private static StateSyncItem PushItem(StateSyncFeed.IPendingSyncItems items, NodeDataType nodeDataType, int level, uint rightness, int progress = 0) { StateSyncItem stateSyncItem1 = new StateSyncItem(Keccak.Zero, nodeDataType, level, rightness); items.PushToSelectedStream(stateSyncItem1, progress); return(stateSyncItem1); }
public StateSyncItem(Keccak hash, NodeDataType nodeType, int level, float priority) { Hash = hash; NodeDataType = nodeType; Level = level; Priority = priority; }
public StateSyncItem(Keccak hash, byte[]?accountPathNibbles, byte[]?pathNibbles, NodeDataType nodeType, int level = 0, uint rightness = 0) { Hash = hash; AccountPathNibbles = accountPathNibbles ?? Array.Empty <byte>(); PathNibbles = pathNibbles ?? Array.Empty <byte>(); NodeDataType = nodeType; Level = (byte)level; Rightness = rightness; }
public NodeChange(Guid guid, object value, NodeDataType type) { if (guid == null) { throw new ArgumentNullException(nameof(guid)); } if (value == null) { throw new ArgumentNullException(nameof(value)); } }
/// <summary> /// Get the type that this enum value represents /// </summary> /// <param name="nDT"></param> /// <returns></returns> public static Type RepresentedType(this NodeDataType nDT) { if (nDT == NodeDataType.NodeSupport) { return(typeof(NodeSupport)); } else { return(null); } }
/// <summary> /// Get the next NodeDataType value after the one specified /// </summary> /// <param name="type"></param> /// <returns></returns> public static NodeDataType Next(this NodeDataType type) { NodeDataType[] values = (NodeDataType[])Enum.GetValues(typeof(NodeDataType)); int i = Array.IndexOf(values, type) + 1; if (i == values.Length) { i = 0; } return(values[i]); }
/// <summary> /// Initializes a new instance of <see cref="SourceFieldNode"/> class. /// </summary> /// <param name="dataType">The type of the constant defined by the current node.</param> /// <param name="fieldName">Name of the field.</param> /// <param name="value">The constant value.</param> /// <param name="setName">The name of the set of data to which this node belongs</param> /// <param name="propertyPath">The property path.</param> /// <param name="systemName">Name of the system.</param> /// <param name="objectName">Name of the object.</param> /// <param name="parentNode">The parent node.</param> public SourceFieldNode(NodeDataType dataType, string fieldName, object value, string setName, string propertyPath, string systemName = null, string objectName = null, SourceFieldNode parentNode = null, bool isGroupingField = false) : this(dataType) { InitialFieldName = fieldName; FieldName = DefineFieldName(fieldName, systemName); Value = value; PropertyPath = propertyPath; SetName = setName; ParentNode = parentNode; ObjectName = objectName; IsGroupingField = isGroupingField; }
/// <summary> /// Get an instance of the given configuration from config service. /// </summary> /// <typeparam name="TConfig">Type of configuration.</typeparam> /// <param name="configKey">Config file key.</param> /// <param name="nodeDataType">Node data type.</param> /// <returns>The requested configuration.</returns> public TConfig GetConfigFromServiceByKey <TConfig>(string configKey, NodeDataType nodeDataType = NodeDataType.Json) where TConfig : class, new() { IConfigDefinition configFile = this._configRepository.GetConfigFileByKey(configKey); if (configFile != null) { configFile.IsFromService = true; return(this._configAccessor.GetConfigValue <TConfig>(configFile, nodeDataType)); } return(null); }
//! //! Maybe in the future turn this into a gradient //! (when you implement cross-link interpretation) //! public Color GetCorrectColor(NodeDataType portReference) { /* * Flux, * Int, * Float, * String, * Bool, * Vector2, * Vector3, * Vector4, * Quaternion */ switch (portReference) { case NodeDataType.Flux: return(dataTypeColors[1]); case NodeDataType.Int: return(dataTypeColors[2]); case NodeDataType.Float: return(dataTypeColors[3]); case NodeDataType.String: return(dataTypeColors[4]); case NodeDataType.Bool: return(dataTypeColors[5]); case NodeDataType.Vector2: return(dataTypeColors[6]); case NodeDataType.Vector3: return(dataTypeColors[7]); case NodeDataType.Vector4: return(dataTypeColors[8]); case NodeDataType.Quaternion: return(dataTypeColors[9]); } return(dataTypeColors[0]); }
/// <summary> /// Deserialize config data. /// </summary> /// <typeparam name="TConfigType">Config type.</typeparam> /// <param name="data">Config value data.</param> /// <param name="dataType">Node data type.</param> /// <returns>Config type instance.</returns> private TConfigType DeserializeData <TConfigType>(string data, NodeDataType dataType) { if (string.IsNullOrEmpty(data)) { return(default(TConfigType)); } if (dataType == NodeDataType.Json) { return(this._serializer.DeserializeObject <TConfigType>(data)); } else if (dataType == NodeDataType.Xml) { return(this._serializer.DeserializeXml <TConfigType>(data)); } else { return(this._serializer.DeserializeObject <TConfigType>(this._serializer.SerializeObject(data))); } }
public byte[]?[] GetNodeData(IList <Keccak> keys, NodeDataType includedTypes = NodeDataType.State | NodeDataType.Code) { byte[]?[] values = new byte[keys.Count][]; for (int i = 0; i < keys.Count; i++) { IDb stateDb = _stateDb.Innermost; IDb codeDb = _codeDb.Innermost; values[i] = null; if ((includedTypes & NodeDataType.State) == NodeDataType.State) { values[i] = stateDb.Get(keys[i]); } if (values[i] == null && (includedTypes & NodeDataType.Code) == NodeDataType.Code) { values[i] = codeDb.Get(keys[i]); } } return(values); }
private float CalculatePriority(NodeDataType nodeDataType, byte level, uint rightness) { if (nodeDataType == NodeDataType.Code) { return(0f); } switch (nodeDataType) { case NodeDataType.Storage: MaxStorageLevel = Math.Max(MaxStorageLevel, level); _maxStorageRightness = Math.Max(_maxStorageRightness, rightness); break; case NodeDataType.State: MaxStateLevel = Math.Max(MaxStateLevel, level); _maxRightness = Math.Max(_maxRightness, rightness); break; } float priority = CalculatePriority(level, rightness); return(priority); }
/// <summary> /// Initializes a new instance of the <see cref="FindFirstFunctionNode"/> class. /// </summary> /// <param name="parameter1">Input List value.</param> /// <param name="parameter2">Input Display Field Value.</param> /// <param name="parameter3">Input Filter Value.</param> /// <param name="dataType"></param> /// /// public FindFirstFunctionNode(ExpressionNode parameter1, ExpressionNode parameter2, ExpressionNode parameter3, NodeDataType dataType) : base(ExpressionNodeType.FindFirst, parameter1, parameter2, parameter3) { this.DataType = dataType; }
/// <summary> /// Get config value. /// </summary> /// <typeparam name="TConfigType">Config type.</typeparam> /// <param name="configDefinition">Config definition.</param> /// <param name="nodeDataType">Node data type.</param> /// <returns>Config instance.</returns> public TConfigType GetConfigValue <TConfigType>(IConfigDefinition configDefinition, NodeDataType nodeDataType = NodeDataType.Json) where TConfigType : class, new() { if (configDefinition == null) { return(null); } var configCollectionKey = $"{configDefinition.SystemName}_{configDefinition.ConfigName}".ToLower(); if (ConfigCollection.ContainsKey(configCollectionKey)) { return(ConfigCollection[configCollectionKey] as TConfigType); } TConfigType configValue; if (configDefinition.IsFromService) { configValue = this._configServiceProvider.GetConfig <TConfigType>(configDefinition.SystemName, configDefinition.ConfigName, nodeDataType); this._configServiceProvider.WatchDataChange <TConfigType>(configDefinition.SystemName, configDefinition.ConfigName, value => this.AddOrUpdateConfigCollection(configDefinition, value), nodeDataType); } else { configValue = this._configFileProvider.GetConfig <TConfigType>(configDefinition.ConfigName); } this.AddOrUpdateConfigCollection(configDefinition, configValue); return(configValue); }
/// <summary> /// Is this a displacement result data type? /// </summary> /// <param name="type"></param> /// <returns></returns> public static bool IsDisplacementData(this NodeDataType type) { return(type >= NodeDataType.Displacement_Ux && type <= NodeDataType.Displacmenet_Rzz); }
public (NodeDataHandlerResult Result, int NodesConsumed) HandleResponse(StateSyncBatch batch) { int requestLength = batch.RequestedNodes?.Length ?? 0; int responseLength = batch.Responses?.Length ?? 0; void AddAgainAllItems() { for (int i = 0; i < requestLength; i++) { AddNode(batch.RequestedNodes[i], null, "missing", true); } } try { lock (_handleWatch) { _handleWatch.Restart(); bool requestWasMade = batch.AssignedPeer?.Current != null; NodeDataHandlerResult result; if (!requestWasMade) { AddAgainAllItems(); if (_logger.IsTrace) { _logger.Trace($"Batch was not assigned to any peer."); } Interlocked.Increment(ref _notAssignedCount); result = NodeDataHandlerResult.NotAssigned; return(result, 0); } bool isMissingRequestData = batch.RequestedNodes == null; bool isMissingResponseData = batch.Responses == null; bool hasValidFormat = !isMissingRequestData && !isMissingResponseData; if (!hasValidFormat) { AddAgainAllItems(); if (_logger.IsDebug) { _logger.Debug($"Batch response had invalid format"); } Interlocked.Increment(ref _invalidFormatCount); result = NodeDataHandlerResult.InvalidFormat; return(result, 0); } if (_logger.IsTrace) { _logger.Trace($"Received node data - {responseLength} items in response to {requestLength}"); } int nonEmptyResponses = 0; int invalidNodes = 0; for (int i = 0; i < batch.RequestedNodes.Length; i++) { StateSyncItem currentStateSyncItem = batch.RequestedNodes[i]; /* if the peer has limit on number of requests in a batch then the response will possibly be * shorter than the request */ if (batch.Responses.Length < i + 1) { AddNode(currentStateSyncItem, null, "missing", true); continue; } /* if the peer does not have details of this particular node */ byte[] currentResponseItem = batch.Responses[i]; if (currentResponseItem == null) { AddNode(batch.RequestedNodes[i], null, "missing", true); continue; } /* node sent data that is not consistent with its hash - it happens surprisingly often */ if (Keccak.Compute(currentResponseItem) != currentStateSyncItem.Hash) { if (_logger.IsDebug) { _logger.Debug($"Peer sent invalid data (batch {requestLength}->{responseLength}) of length {batch.Responses[i]?.Length} of type {batch.RequestedNodes[i].NodeDataType} at level {batch.RequestedNodes[i].Level} of type {batch.RequestedNodes[i].NodeDataType} Keccak({batch.Responses[i].ToHexString()}) != {batch.RequestedNodes[i].Hash}"); } invalidNodes++; continue; } nonEmptyResponses++; NodeDataType nodeDataType = currentStateSyncItem.NodeDataType; if (nodeDataType == NodeDataType.Code) { SaveNode(currentStateSyncItem, currentResponseItem); continue; } TrieNode trieNode = new TrieNode(NodeType.Unknown, new Rlp(currentResponseItem)); trieNode.ResolveNode(null); switch (trieNode.NodeType) { case NodeType.Unknown: invalidNodes++; if (_logger.IsError) { _logger.Error($"Node {currentStateSyncItem.Hash} resolved to {nameof(NodeType.Unknown)}"); } break; case NodeType.Branch: trieNode.BuildLookupTable(); DependentItem dependentBranch = new DependentItem(currentStateSyncItem, currentResponseItem, 0); HashSet <Keccak> alreadyProcessedChildHashes = new HashSet <Keccak>(); for (int childIndex = 0; childIndex < 16; childIndex++) { Keccak child = trieNode.GetChildHash(childIndex); if (alreadyProcessedChildHashes.Contains(child)) { continue; } alreadyProcessedChildHashes.Add(child); if (child != null) { AddNodeResult addChildResult = AddNode(new StateSyncItem(child, nodeDataType, currentStateSyncItem.Level + 1, CalculatePriority(currentStateSyncItem)), dependentBranch, "branch child"); if (addChildResult != AddNodeResult.AlreadySaved) { dependentBranch.Counter++; } } } if (dependentBranch.Counter == 0) { SaveNode(currentStateSyncItem, currentResponseItem); } break; case NodeType.Extension: Keccak next = trieNode[0].Keccak; if (next != null) { DependentItem dependentItem = new DependentItem(currentStateSyncItem, currentResponseItem, 1); AddNodeResult addResult = AddNode(new StateSyncItem(next, nodeDataType, currentStateSyncItem.Level + 1, currentStateSyncItem.Priority), dependentItem, "extension child"); if (addResult == AddNodeResult.AlreadySaved) { SaveNode(currentStateSyncItem, currentResponseItem); } } else { /* this happens when we have a short RLP format of the node * that would not be stored as Keccak but full RLP*/ SaveNode(currentStateSyncItem, currentResponseItem); } break; case NodeType.Leaf: if (nodeDataType == NodeDataType.State) { DependentItem dependentItem = new DependentItem(currentStateSyncItem, currentResponseItem, 0, true); Account account = _accountDecoder.Decode(new Rlp.DecoderContext(trieNode.Value)); if (account.CodeHash != Keccak.OfAnEmptyString) { // prepare a branch without the code DB // this only protects against being same as storage root? if (account.CodeHash == account.StorageRoot) { lock (_codesSameAsNodes) { _codesSameAsNodes.Add(account.CodeHash); } } else { AddNodeResult addCodeResult = AddNode(new StateSyncItem(account.CodeHash, NodeDataType.Code, 0, 0), dependentItem, "code"); if (addCodeResult != AddNodeResult.AlreadySaved) { dependentItem.Counter++; } } } if (account.StorageRoot != Keccak.EmptyTreeHash) { AddNodeResult addStorageNodeResult = AddNode(new StateSyncItem(account.StorageRoot, NodeDataType.Storage, 0, 0), dependentItem, "storage"); if (addStorageNodeResult != AddNodeResult.AlreadySaved) { dependentItem.Counter++; } } if (dependentItem.Counter == 0) { Interlocked.Increment(ref _savedAccounts); SaveNode(currentStateSyncItem, currentResponseItem); } } else { SaveNode(currentStateSyncItem, currentResponseItem); } break; default: if (_logger.IsError) { _logger.Error($"Unknown value {currentStateSyncItem.NodeDataType} of {nameof(NodeDataType)} at {currentStateSyncItem.Hash}"); } invalidNodes++; continue; } } lock (_stateDbLock) { Rlp rlp = Rlp.Encode( Rlp.Encode(_consumedNodesCount), Rlp.Encode(_savedStorageCount), Rlp.Encode(_savedStateCount), Rlp.Encode(_savedNodesCount), Rlp.Encode(_savedAccounts), Rlp.Encode(_savedCode), Rlp.Encode(_requestedNodesCount), Rlp.Encode(_dbChecks), Rlp.Encode(_stateWasThere), Rlp.Encode(_stateWasNotThere), Rlp.Encode(_dataSize)); lock (_codeDbLock) { _codeDb[_fastSyncProgressKey.Bytes] = rlp.Bytes; _codeDb.Commit(); _stateDb.Commit(); } } Interlocked.Add(ref _consumedNodesCount, nonEmptyResponses); if (_logger.IsTrace) { _logger.Trace($"After handling response (non-empty responses {nonEmptyResponses}) of {batch.RequestedNodes.Length} from ({Stream0.Count}|{Stream1.Count}|{Stream2.Count}) nodes"); } /* magic formula is ratio of our desired batch size - 1024 to Geth max batch size 384 times some missing nodes ratio */ bool isEmptish = (decimal)nonEmptyResponses / requestLength < 384m / 1024m * 0.75m; if (isEmptish) { Interlocked.Increment(ref _emptishCount); } /* here we are very forgiving for Geth nodes that send bad data fast */ bool isBadQuality = nonEmptyResponses > 64 && (decimal)invalidNodes / requestLength > 0.50m; if (isBadQuality) { Interlocked.Increment(ref _badQualityCount); } bool isEmpty = nonEmptyResponses == 0 && !isBadQuality; if (isEmpty) { if (_logger.IsWarn) { _logger.Warn($"Peer sent no data in response to a request of length {batch.RequestedNodes.Length}"); } result = NodeDataHandlerResult.NoData; return(result, 0); } if (!isEmptish && !isBadQuality) { Interlocked.Increment(ref _okCount); } result = isEmptish ? NodeDataHandlerResult.Emptish : isBadQuality ? NodeDataHandlerResult.BadQuality : NodeDataHandlerResult.OK; if (DateTime.UtcNow - _lastReportTime > TimeSpan.FromSeconds(1)) { decimal requestedNodesPerSecond = 1000m * (_requestedNodesCount - _lastRequestedNodesCount) / (decimal)(DateTime.UtcNow - _lastReportTime).TotalMilliseconds; decimal savedNodesPerSecond = 1000m * (_savedNodesCount - _lastSavedNodesCount) / (decimal)(DateTime.UtcNow - _lastReportTime).TotalMilliseconds; _lastSavedNodesCount = _savedNodesCount; _lastRequestedNodesCount = _requestedNodesCount; _lastReportTime = DateTime.UtcNow; if (_logger.IsInfo) { _logger.Info($"SNPS: {savedNodesPerSecond,6:F0} | RNPS: {requestedNodesPerSecond,6:F0} | Saved nodes {_savedNodesCount} / requested {_requestedNodesCount} ({(decimal) _savedNodesCount / _requestedNodesCount:P2}), saved accounts {_savedAccounts}, size: {(decimal)_dataSize / 1000 / 1000:F2}MB, enqueued nodes {Stream0.Count:D5}|{Stream1.Count:D5}|{Stream2.Count:D5}"); } if (_logger.IsInfo) { _logger.Info($"AVTIH: {_averageTimeInHandler:F2}ms | P: {_pendingRequests.Count} | Request results - OK: {(decimal) _okCount / TotalRequestsCount:p2}, Emptish: {(decimal) _emptishCount / TotalRequestsCount:p2}, BadQuality: {(decimal) _badQualityCount / TotalRequestsCount:p2}, InvalidFormat: {(decimal) _invalidFormatCount / TotalRequestsCount:p2}, NotAssigned {(decimal) _notAssignedCount / TotalRequestsCount:p2}"); } if (_logger.IsTrace) { _logger.Trace($"Requested {_requestedNodesCount}, consumed {_consumedNodesCount}, missed {_requestedNodesCount - _consumedNodesCount}, {_savedCode} contracts, {_savedStateCount - _savedAccounts} states, {_savedStorageCount} storage, DB checks {_stateWasThere}/{_stateWasNotThere + _stateWasThere} cached({_checkWasCached}+{_checkWasInDependencies})"); } if (_logger.IsTrace) { _logger.Trace($"Consume : {(decimal) _consumedNodesCount / _requestedNodesCount:p2}, Save : {(decimal) _savedNodesCount / _requestedNodesCount:p2}, DB Reads : {(decimal) _dbChecks / _requestedNodesCount:p2}"); } } long total = _handleWatch.ElapsedMilliseconds + _networkWatch.ElapsedMilliseconds; if (total != 0) { // calculate averages if (_logger.IsTrace) { _logger.Trace($"Prepare batch {_networkWatch.ElapsedMilliseconds}ms ({(decimal) _networkWatch.ElapsedMilliseconds / total:P0}) - Handle {_handleWatch.ElapsedMilliseconds}ms ({(decimal) _handleWatch.ElapsedMilliseconds / total:P0})"); } } _averageTimeInHandler = (_averageTimeInHandler * (ProcessedRequestsCount - 1) + _handleWatch.ElapsedMilliseconds) / ProcessedRequestsCount; return(result, nonEmptyResponses); } } finally { _handleWatch.Stop(); if (!_pendingRequests.TryRemove(batch, out _)) { _logger.Error("Cannot remove pending request"); } } }
public NodeInputAttribute(NodeDataType type) { InputType = type; }
private DestinationField CreateField(string name, string systemName, NodeDataType type, ExpressionObjectBase owner) { var field = new DestinationField(owner) { DataType = type, Name = name, IsKey = false, IsKeyVisible = true, IsKeyEnabled = true, ConnectorIn = { DataType = type, Name = name }, SystemName = systemName, SetName = SourceFieldSetNames.Item }; field.ConnectorIn.Validator = _connectionValidatorFactory(field); return field; }
/// <summary> /// Initializes a new instance of the <see cref="GetArrayItemFunctionNode"/> class. /// </summary> /// <param name="parameter1"> /// The parameter 1. /// </param> /// <param name="parameter2"> /// The parameter 2. /// </param> /// <param name="elementType"> /// The element type. /// </param> public GetArrayItemFunctionNode(ExpressionNode parameter1, ExpressionNode parameter2, NodeDataType elementType) : base(ExpressionNodeType.GetArrayItem, parameter1, parameter2) { _elementType = elementType; }
/// <summary> /// Get config from config service. /// </summary> /// <typeparam name="TConfigType">Config Type.</typeparam> /// <param name="systemName">System name.</param> /// <param name="configName">Config name.</param> /// <param name="nodeDataType">Node date type.</param> /// <returns>Config instance.</returns> public TConfigType GetConfig <TConfigType>(string systemName, string configName, NodeDataType nodeDataType = NodeDataType.Json) { var client = GetZookeeperClient(); if (client != null) { var path = $"/{systemName}/{configName}"; if (client.Exists(path)) { var data = client.GetData(path); return(DeserializeData <TConfigType>(data, nodeDataType)); } } return(default(TConfigType)); }
///// <summary> ///// Initializes a new instance of the <see cref="ConverterNode"/> class. ///// </summary> ///// <param name="parameter">The first parameter.</param> //public ConverterNode(ExpressionNode parameter) // : base(ExpressionNodeType.Converter, parameter) //{ //} /// <summary> /// Initializes a new instance of <see cref="ConstantNode"/> class. /// </summary> /// <param name="parameter">The type of the constant defined by the current node.</param> /// <param name="dataTypeIn">The type of the constant defined by the current node.</param> /// <param name="dataTypeOut">The type of the constant defined by the current node.</param> public ConverterNode(ExpressionNode parameter, NodeDataType dataTypeIn, NodeDataType dataTypeOut) : base(ExpressionNodeType.Converter, parameter) { DataTypeIn = dataTypeIn; DataTypeOut = dataTypeOut; }
private void HandleTrieNode(StateSyncItem currentStateSyncItem, byte[] currentResponseItem, ref int invalidNodes) { NodeDataType nodeDataType = currentStateSyncItem.NodeDataType; TrieNode trieNode = new TrieNode(NodeType.Unknown, currentResponseItem); trieNode.ResolveNode(null); switch (trieNode.NodeType) { case NodeType.Unknown: invalidNodes++; if (_logger.IsError) { _logger.Error($"Node {currentStateSyncItem.Hash} resolved to {nameof(NodeType.Unknown)}"); } break; case NodeType.Branch: DependentItem dependentBranch = new DependentItem(currentStateSyncItem, currentResponseItem, 0); // children may have the same hashes (e.g. a set of accounts with the same code at different addresses) HashSet <Keccak> alreadyProcessedChildHashes = new HashSet <Keccak>(); for (int childIndex = 15; childIndex >= 0; childIndex--) { Keccak childHash = trieNode.GetChildHash(childIndex); if (childHash != null && alreadyProcessedChildHashes.Contains(childHash)) { continue; } alreadyProcessedChildHashes.Add(childHash); if (childHash != null) { AddNodeResult addChildResult = AddNodeToPending(new StateSyncItem(childHash, nodeDataType, currentStateSyncItem.Level + 1, CalculateRightness(trieNode.NodeType, currentStateSyncItem, childIndex)) { BranchChildIndex = (short)childIndex, ParentBranchChildIndex = currentStateSyncItem.BranchChildIndex }, dependentBranch, "branch child"); if (addChildResult != AddNodeResult.AlreadySaved) { dependentBranch.Counter++; } else { _syncProgress.ReportSynced(currentStateSyncItem.Level + 1, currentStateSyncItem.BranchChildIndex, childIndex, currentStateSyncItem.NodeDataType, NodeProgressState.AlreadySaved); } } else { _syncProgress.ReportSynced(currentStateSyncItem.Level + 1, currentStateSyncItem.BranchChildIndex, childIndex, currentStateSyncItem.NodeDataType, NodeProgressState.Empty); } } if (dependentBranch.Counter == 0) { SaveNode(currentStateSyncItem, currentResponseItem); } break; case NodeType.Extension: Keccak next = trieNode[0].Keccak; if (next != null) { DependentItem dependentItem = new DependentItem(currentStateSyncItem, currentResponseItem, 1); AddNodeResult addResult = AddNodeToPending(new StateSyncItem(next, nodeDataType, currentStateSyncItem.Level + trieNode.Path.Length, CalculateRightness(trieNode.NodeType, currentStateSyncItem, 0)) { ParentBranchChildIndex = currentStateSyncItem.BranchChildIndex }, dependentItem, "extension child"); if (addResult == AddNodeResult.AlreadySaved) { SaveNode(currentStateSyncItem, currentResponseItem); } } else { /* this happens when we have a short RLP format of the node * that would not be stored as Keccak but full RLP*/ SaveNode(currentStateSyncItem, currentResponseItem); } break; case NodeType.Leaf: if (nodeDataType == NodeDataType.State) { _pendingItems.MaxStateLevel = 64; DependentItem dependentItem = new DependentItem(currentStateSyncItem, currentResponseItem, 0, true); (Keccak codeHash, Keccak storageRoot) = AccountDecoder.DecodeHashesOnly(new RlpStream(trieNode.Value)); if (codeHash != Keccak.OfAnEmptyString) { // prepare a branch without the code DB // this only protects against being same as storage root? if (codeHash == storageRoot) { lock (_codesSameAsNodes) { _codesSameAsNodes.Add(codeHash); } } else { AddNodeResult addCodeResult = AddNodeToPending(new StateSyncItem(codeHash, NodeDataType.Code, 0, currentStateSyncItem.Rightness), dependentItem, "code"); if (addCodeResult != AddNodeResult.AlreadySaved) { dependentItem.Counter++; } } } if (storageRoot != Keccak.EmptyTreeHash) { AddNodeResult addStorageNodeResult = AddNodeToPending(new StateSyncItem(storageRoot, NodeDataType.Storage, 0, currentStateSyncItem.Rightness), dependentItem, "storage"); if (addStorageNodeResult != AddNodeResult.AlreadySaved) { dependentItem.Counter++; } } if (dependentItem.Counter == 0) { Interlocked.Increment(ref _data.SavedAccounts); SaveNode(currentStateSyncItem, currentResponseItem); } } else { _pendingItems.MaxStorageLevel = 64; SaveNode(currentStateSyncItem, currentResponseItem); } break; default: if (_logger.IsError) { _logger.Error($"Unknown value {currentStateSyncItem.NodeDataType} of {nameof(NodeDataType)} at {currentStateSyncItem.Hash}"); } invalidNodes++; break; } }
public override SyncResponseHandlingResult HandleResponse(StateSyncBatch batch) { if (batch == EmptyBatch) { _logger.Error("Received empty batch as a response"); } if (!_pendingRequests.TryRemove(batch, out _)) { if (_logger.IsDebug) { _logger.Debug($"Cannot remove pending request {batch}"); } return(SyncResponseHandlingResult.OK); } else { if (_logger.IsTrace) { _logger.Trace($"Removing pending request {batch}"); } } int requestLength = batch.RequestedNodes?.Length ?? 0; int responseLength = batch.Responses?.Length ?? 0; void AddAgainAllItems() { for (int i = 0; i < requestLength; i++) { AddNodeToPending(batch.RequestedNodes[i], null, "missing", true); } } try { lock (_handleWatch) { if (DateTime.UtcNow - _lastReview > TimeSpan.FromSeconds(60)) { _lastReview = DateTime.UtcNow; string reviewMessage = _pendingItems.RecalculatePriorities(); if (_logger.IsInfo) { _logger.Info(reviewMessage); } } _handleWatch.Restart(); bool requestWasMade = batch.Responses != null; if (!requestWasMade) { AddAgainAllItems(); if (_logger.IsTrace) { _logger.Trace($"Batch was not assigned to any peer."); } Interlocked.Increment(ref _data.NotAssignedCount); return(SyncResponseHandlingResult.NotAssigned); } bool isMissingRequestData = batch.RequestedNodes == null; bool isMissingResponseData = batch.Responses == null; bool hasValidFormat = !isMissingRequestData && !isMissingResponseData; if (!hasValidFormat) { _hintsToResetRoot++; AddAgainAllItems(); if (_logger.IsWarn) { _logger.Warn($"Batch response had invalid format"); } Interlocked.Increment(ref _data.InvalidFormatCount); return(isMissingRequestData ? SyncResponseHandlingResult.InternalError : SyncResponseHandlingResult.NotAssigned); } if (_logger.IsTrace) { _logger.Trace($"Received node data - {responseLength} items in response to {requestLength}"); } int nonEmptyResponses = 0; int invalidNodes = 0; for (int i = 0; i < batch.RequestedNodes.Length; i++) { StateSyncItem currentStateSyncItem = batch.RequestedNodes[i]; /* if the peer has limit on number of requests in a batch then the response will possibly be * shorter than the request */ if (batch.Responses.Length < i + 1) { AddNodeToPending(currentStateSyncItem, null, "missing", true); continue; } /* if the peer does not have details of this particular node */ byte[] currentResponseItem = batch.Responses[i]; if (currentResponseItem == null) { AddNodeToPending(batch.RequestedNodes[i], null, "missing", true); continue; } /* node sent data that is not consistent with its hash - it happens surprisingly often */ if (!ValueKeccak.Compute(currentResponseItem).BytesAsSpan.SequenceEqual(currentStateSyncItem.Hash.Bytes)) { AddNodeToPending(currentStateSyncItem, null, "missing", true); if (_logger.IsTrace) { _logger.Trace($"Peer sent invalid data (batch {requestLength}->{responseLength}) of length {batch.Responses[i]?.Length} of type {batch.RequestedNodes[i].NodeDataType} at level {batch.RequestedNodes[i].Level} of type {batch.RequestedNodes[i].NodeDataType} Keccak({batch.Responses[i].ToHexString()}) != {batch.RequestedNodes[i].Hash}"); } invalidNodes++; continue; } nonEmptyResponses++; NodeDataType nodeDataType = currentStateSyncItem.NodeDataType; if (nodeDataType == NodeDataType.Code) { SaveNode(currentStateSyncItem, currentResponseItem); continue; } HandleTrieNode(currentStateSyncItem, currentResponseItem, ref invalidNodes); } Interlocked.Add(ref _data.ConsumedNodesCount, nonEmptyResponses); StoreProgressInDb(); if (_logger.IsTrace) { _logger.Trace($"After handling response (non-empty responses {nonEmptyResponses}) of {batch.RequestedNodes.Length} from ({_pendingItems.Description}) nodes"); } /* magic formula is ratio of our desired batch size - 1024 to Geth max batch size 384 times some missing nodes ratio */ bool isEmptish = (decimal)nonEmptyResponses / Math.Max(requestLength, 1) < 384m / 1024m * 0.75m; if (isEmptish) { Interlocked.Increment(ref _hintsToResetRoot); Interlocked.Increment(ref _data.EmptishCount); } else { Interlocked.Exchange(ref _hintsToResetRoot, 0); } /* here we are very forgiving for Geth nodes that send bad data fast */ bool isBadQuality = nonEmptyResponses > 64 && (decimal)invalidNodes / Math.Max(requestLength, 1) > 0.50m; if (isBadQuality) { Interlocked.Increment(ref _data.BadQualityCount); } bool isEmpty = nonEmptyResponses == 0 && !isBadQuality; if (isEmpty) { if (_logger.IsDebug) { _logger.Debug($"Peer sent no data in response to a request of length {batch.RequestedNodes.Length}"); } return(SyncResponseHandlingResult.NoProgress); } if (!isEmptish && !isBadQuality) { Interlocked.Increment(ref _data.OkCount); } SyncResponseHandlingResult result = isEmptish ? SyncResponseHandlingResult.Emptish : isBadQuality ? SyncResponseHandlingResult.LesserQuality : SyncResponseHandlingResult.OK; _data.DisplayProgressReport(_pendingRequests.Count, _logger); long total = _handleWatch.ElapsedMilliseconds + _networkWatch.ElapsedMilliseconds; if (total != 0) { // calculate averages if (_logger.IsTrace) { _logger.Trace($"Prepare batch {_networkWatch.ElapsedMilliseconds}ms ({(decimal) _networkWatch.ElapsedMilliseconds / total:P0}) - Handle {_handleWatch.ElapsedMilliseconds}ms ({(decimal) _handleWatch.ElapsedMilliseconds / total:P0})"); } } if (_handleWatch.ElapsedMilliseconds > 250) { if (_logger.IsDebug) { _logger.Debug($"Handle watch {_handleWatch.ElapsedMilliseconds}, DB reads {_data.DbChecks - _data.LastDbReads}, ratio {(decimal) _handleWatch.ElapsedMilliseconds / Math.Max(1, _data.DbChecks - _data.LastDbReads)}"); } } _data.LastDbReads = _data.DbChecks; _data.AverageTimeInHandler = (_data.AverageTimeInHandler * (_data.ProcessedRequestsCount - 1) + _handleWatch.ElapsedMilliseconds) / _data.ProcessedRequestsCount; Interlocked.Add(ref _data.HandledNodesCount, nonEmptyResponses); return(result); } } catch (Exception e) { _logger.Error("Error when handling state sync response", e); return(SyncResponseHandlingResult.InternalError); } finally { _handleWatch.Stop(); } }
/// <summary> /// Initializes a new instance of the <see cref="GetContextValueFunctionNode"/> class. /// </summary> /// <param name="id"> /// The node id. /// </param> /// <param name="valueType"> /// The value type. /// </param> public GetContextValueFunctionNode(Guid id, NodeDataType valueType) : base(ExpressionNodeType.GetContextValue) { Id = id; ValueType = valueType; }
/// <summary> /// Gets a value indicating if this instance has a parent field of a particular type /// </summary> /// <param name="type">The field data type</param> /// <value><c>true</c> if this instance has direct or indirect parent of a particular type; otherwise, <c>false</c>.</value> public bool HasParentFieldOfType(NodeDataType type) { if (ParentField != null) { if (ParentField.DataType == type) return true; return ParentField.HasParentFieldOfType(type); } return false; }
/// <summary> /// Ares the types compatible. /// </summary> /// <param name="sourceType">Type of the source.</param> /// <param name="destType">Type of the dest.</param> /// <returns><c>true</c> if XXXX, <c>false</c> otherwise.</returns> private static bool AreTypesCompatible(NodeDataType sourceType, NodeDataType destType) { if (sourceType == NodeDataType.List && destType == NodeDataType.String) { return false; } if (destType == NodeDataType.String) { switch (sourceType) { case NodeDataType.Array: return false; case NodeDataType.ByteArray: return false; case NodeDataType.FixedArray: return false; } } if (sourceType == destType || destType == NodeDataType.String || destType == NodeDataType.Field || destType == NodeDataType.Object) { return true; } if (destType == NodeDataType.Formattable) { return sourceType == NodeDataType.Byte || sourceType == NodeDataType.DateTime || sourceType == NodeDataType.Decimal || sourceType == NodeDataType.Double || sourceType == NodeDataType.Int || sourceType == NodeDataType.Guid; } switch (sourceType) { case NodeDataType.Field: return false; case NodeDataType.String: return false; case NodeDataType.Double: return destType == NodeDataType.Decimal || destType == NodeDataType.Int || destType == NodeDataType.Result; case NodeDataType.Decimal: return destType == NodeDataType.Double || destType == NodeDataType.Result; case NodeDataType.DateTime: return false; case NodeDataType.Int: return destType == NodeDataType.Double || destType == NodeDataType.Decimal || destType == NodeDataType.Result || destType == NodeDataType.CrossReference || destType == NodeDataType.ReverseCrossReference; case NodeDataType.Boolean: return false; case NodeDataType.CrossReference: case NodeDataType.ReverseCrossReference: return destType == NodeDataType.Int; case NodeDataType.SPCChart: return false; case NodeDataType.GageRR: return false; case NodeDataType.Array: return destType == NodeDataType.List || destType == NodeDataType.Checklist || destType == NodeDataType.Sample; case NodeDataType.Entity: return destType == NodeDataType.List; case NodeDataType.List: return destType == NodeDataType.Array; case NodeDataType.Null: return destType == NodeDataType.Boolean || destType == NodeDataType.CrossReference || destType == NodeDataType.DateTime || destType == NodeDataType.Decimal || destType == NodeDataType.Double || destType == NodeDataType.Int || destType == NodeDataType.String || destType == NodeDataType.Result; case NodeDataType.Result: return destType == NodeDataType.Double || destType == NodeDataType.Decimal || destType == NodeDataType.Int || destType == NodeDataType.Result; } return false; }
/// <summary> /// Does this data type constitute a property of the node itself? /// </summary> /// <param name="type"></param> /// <returns></returns> public static bool IsNodeData(this NodeDataType type) { return(type == NodeDataType.X || type == NodeDataType.Y || type == NodeDataType.Z); }
/// <summary> /// Initializes a new instance of <see cref="ConstantNode"/> class. /// </summary> /// <param name="dataType">The type of the constant defined by the current node.</param> /// <param name="value">The constant value.</param> public ConstantNode(NodeDataType dataType, object value) : this(dataType) { Value = value; }
/// <summary> /// Initializes a new instance of the <see cref="ExpressionConnector"/> class. /// </summary> /// <param name="connectorType">Type of the connector.</param> /// <param name="connector">The connector.</param> /// <exception cref="System.ArgumentNullException">connector</exception> public ExpressionConnector(ConnectorType connectorType, IConnector connector) { if (connector == null) throw new ArgumentNullException("connector"); Id = connector.Id; _connectorType = connectorType; _dataType = connector.DataType; _name = connector.Name; _connector = connector; _connector.PropertyChanged += OnConnectorPropertyChanged; }
/// <summary> /// Initializes a new instance of <see cref="SourceFieldNode"/> class. /// </summary> /// <param name="dataType">The type of the constant defined by the current node.</param> public SourceFieldNode(NodeDataType dataType) : base(ExpressionNodeType.SourceField) { DataType = dataType; }
/// <summary> /// Watch data change, exec callback method. /// </summary> /// <typeparam name="TConfigType">Config type.</typeparam> /// <param name="systemName">System name.</param> /// <param name="configName">Config name.</param> /// <param name="callback">Callback action.</param> /// <param name="dataType">Node data type, Default is json.</param> public void WatchDataChange <TConfigType>(string systemName, string configName, Action <TConfigType> callback, NodeDataType dataType = NodeDataType.Json) { var client = GetZookeeperClient(); if (client != null) { client.Watch($"/{systemName}/{configName}", context => { var data = context.GetData(); callback(DeserializeData <TConfigType>(data, dataType)); }); } }
/// <summary> /// Get an instance of the given configuration from config service. /// </summary> /// <typeparam name="TConfig">Type of configuration.</typeparam> /// <param name="configName">Config name.</param> /// <param name="nodeDataType">Node data type.</param> /// <returns>The requested configuration.</returns> public TConfig GetConfigFromService <TConfig>(string configName, NodeDataType nodeDataType = NodeDataType.Json) where TConfig : class, new() { return(GetConfigFromService <TConfig>(configName, string.Empty, nodeDataType)); }
/// <summary> /// Initializes a new instance of <see cref="ConstantNode"/> class. /// </summary> /// <param name="dataType">The type of the constant defined by the current node.</param> public ConstantNode(NodeDataType dataType) :base(ExpressionNodeType.Constant) { DataType = dataType; }
/// <summary> /// Get an instance of the given configuration from config service. /// </summary> /// <typeparam name="TConfig">Type of configuration.</typeparam> /// <param name="configName">Config name.</param> /// <param name="systemName">System name.</param> /// <param name="nodeDataType">Node data type.</param> /// <returns>The requested configuration.</returns> public TConfig GetConfigFromService <TConfig>(string configName, string systemName, NodeDataType nodeDataType = NodeDataType.Json) where TConfig : class, new() { if (string.IsNullOrWhiteSpace(configName)) { throw new ArgumentNullException("configName"); } if (string.IsNullOrWhiteSpace(systemName)) { systemName = this._configRepository.DefaultSystemName; } var configFile = new ConfigDefinition { SystemName = systemName, ConfigName = configName, IsFromService = true }; return(this._configAccessor.GetConfigValue <TConfig>(configFile, nodeDataType)); }
//public ExpressionConnector(ConnectorType connectorType, NodeDataType nodeDataType, string name) //{ // _connectorType = connectorType; // _dataType = nodeDataType; // _name = name; //} /// <summary> /// Initializes a new instance of the <see cref="ExpressionConnector"/> class. /// </summary> /// <param name="connectorType">Type of the connector.</param> /// <param name="expressionField">The expression field.</param> /// <exception cref="System.ArgumentNullException">expressionField</exception> public ExpressionConnector(ConnectorType connectorType, IExpressionField expressionField) { if (expressionField == null) throw new ArgumentNullException("expressionField"); Id = expressionField.Id; _connectorType = connectorType; _dataType = expressionField.DataType; _name = expressionField.Name; _field = expressionField; _connector = expressionField.Connector; if (_connector != null) { _connector.PropertyChanged += OnConnectorPropertyChanged; } }
/// <summary> /// Creates the field. /// </summary> /// <param name="name">The field name.</param> /// <param name="type">The field type.</param> /// <param name="owner">The owner.</param> /// <returns>The <see cref="DestinationField"/>.</returns> private DestinationField CreateField(string name, NodeDataType type, ExpressionObjectBase owner) { var field = new DestinationField(owner) { DataType = type, Name = name, IsKeyVisible = false, IsKeyEnabled = false, ConnectorIn = { DataType = type, Name = name }, SystemName = name, SetName = SourceFieldSetNames.Item }; return field; }
/// <summary> /// Initializes a new instance of the <see cref="FormatValueFunctionNode"/> class. /// </summary> /// <param name="parameter1"> /// The parameter 1. /// </param> /// <param name="parameter2"> /// The parameter 2. /// </param> /// <param name="dataType"> /// The data type. /// </param> public FormatValueFunctionNode(ExpressionNode parameter1, ExpressionNode parameter2, NodeDataType dataType) : base(ExpressionNodeType.FormatValue, parameter1, parameter2) { DataType = dataType; }
/// <summary> /// Gets the constant value. /// </summary> /// <param name="value">The value.</param> /// <param name="type">The type.</param> /// <returns>System.String.</returns> private string GetConstantValue(string value, NodeDataType type) { var result = string.Empty; switch (type) { case NodeDataType.Boolean: { bool boolValue; if (!bool.TryParse(value, out boolValue)) boolValue = false; result = boolValue ? "true" : "false"; } break; case NodeDataType.Double: { double doubleValue; if (!double.TryParse(value, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out doubleValue)) doubleValue = 0.0; if (double.IsNaN(doubleValue)) result = "double.NaN"; else if (double.IsNegativeInfinity(doubleValue)) result = "double.NegativeInfinity"; else if (double.IsPositiveInfinity(doubleValue)) result = "double.PositiveInfinity"; else result = doubleValue.ToString("R", CultureInfo.InvariantCulture); } break; case NodeDataType.Decimal: { decimal decimalValue; if (!decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out decimalValue)) decimalValue = 0m; result = decimalValue.ToString(CultureInfo.InvariantCulture) + "m"; } break; case NodeDataType.Int: { int intValue; if (!int.TryParse(value, NumberStyles.Integer | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out intValue)) intValue = 0; result = intValue.ToString(CultureInfo.InvariantCulture); } break; case NodeDataType.DateTime: result = string.Format("SafeTypeConverter.Convert<DateTime>(\"{0}\")", value); break; case NodeDataType.ChartType: case NodeDataType.String: result = string.Format("\"{0}\"", Escape(value)); break; case NodeDataType.Null: result = "null"; break; case NodeDataType.ProcessState: { Guid stateGuid; if (Guid.TryParse(value, out stateGuid)) { result = string.Format( CultureInfo.InvariantCulture, "Cebos.Veyron.SharedTypes.Extensions.DynamicExpressionHelper.GetStateName(new System.Guid(\"{0}\"))", stateGuid); } else { result = "null"; } } break; } return result; }
public void ReportSynced(int level, int parentIndex, int childIndex, NodeDataType nodeDataType, NodeProgressState nodeProgressState) { if (level > 2 || nodeDataType != NodeDataType.State) { return; } switch (level) { case 0: for (int i = 0; i < 256; i++) { UpdateState(i, nodeProgressState); } break; case 1: ReportSyncedLevel1(childIndex, nodeProgressState); break; case 2: ReportSyncedLevel2(parentIndex, childIndex, nodeProgressState); break; } int savedBranches = 0; for (int i = 0; i < _syncProgress.Length; i++) { if (_syncProgress[i] != NodeProgressState.Unknown && _syncProgress[i] != NodeProgressState.Requested) { savedBranches += 1; } } decimal currentProgress = (decimal)savedBranches / _syncProgress.Length; if (currentProgress == LastProgress) { return; } Metrics.StateBranchProgress = (int)(currentProgress * 100); LastProgress = currentProgress; if (nodeProgressState == NodeProgressState.Empty) { return; } string detailsString = string.Empty; if (_logger.IsInfo) { StringBuilder builder = new StringBuilder(); for (int i = 0; i < _syncProgress.Length; i++) { if (i % 64 == 0) { builder.AppendLine(); } switch (_syncProgress[i]) { case NodeProgressState.Unknown: builder.Append('?'); break; case NodeProgressState.Empty: builder.Append('0'); break; case NodeProgressState.AlreadySaved: builder.Append('1'); break; case NodeProgressState.Saved: builder.Append('+'); break; case NodeProgressState.Requested: builder.Append('*'); break; default: throw new ArgumentOutOfRangeException(); } } detailsString = builder.ToString(); } if (_logger.IsInfo) { _logger.Info($"Branch sync progress (do not extrapolate): {(decimal) savedBranches / _syncProgress.Length:p2} of block {CurrentSyncBlock}{detailsString}"); } }
/// <summary> /// Initializes a new instance of the <see cref="MaxFunctionNode"/> class. /// </summary> /// <param name="parameter1"> /// The first parameter. /// </param> /// <param name="parameter2"> /// The second parameter. /// </param> /// <param name="dataType"> /// The data type. /// </param> public MaxFunctionNode(ExpressionNode parameter1, ExpressionNode parameter2, NodeDataType dataType) : base(ExpressionNodeType.Max, parameter1, parameter2) { DataType = dataType; }
private string GetConstantValue(string value, NodeDataType type) { string result = String.Empty; switch (type) { case NodeDataType.Boolean: case NodeDataType.Double: case NodeDataType.Decimal: case NodeDataType.Int: result = value; break; case NodeDataType.DateTime: result = String.Format("functions.ParseDate(\"{0}\")", value); break; case NodeDataType.String: result = String.Format("\"{0}\"", value); break; } return result; }
public NodeOutputAttribute(NodeDataType type) { OutputType = type; }
public static string GetCSharpType(NodeDataType dataType) { switch (dataType) { case NodeDataType.String: return "string"; case NodeDataType.Double: case NodeDataType.Result: return "double?"; case NodeDataType.Int: return "int?"; case NodeDataType.Boolean: return "bool?"; case NodeDataType.Decimal: return "decimal?"; case NodeDataType.DateTime: return "DateTime?"; case NodeDataType.CrossReference: return "int?"; case NodeDataType.ReverseCrossReference: return "int?"; case NodeDataType.List: return "IEnumerable"; case NodeDataType.Byte: return "byte"; case NodeDataType.ByteArray: return "byte[]"; case NodeDataType.Char: return "char"; default: return "object"; } }
/// <summary> /// Creates the Result node /// </summary> /// <param name="connectorIn">The input connector</param> /// <param name="dataType">The data type</param> /// <param name="fieldName">The field name</param> /// <returns>ResultNode</returns> private ResultNode CreateResultNode(IConnectorIn connectorIn, NodeDataType dataType, string fieldName) { return new ResultNode { FieldName = fieldName, DataType = dataType, Expression = ToExpressionNode( connectorIn.Connection.Source.Connection.Source.Owner, connectorIn.Connection.Source.Connection) }; }