public Builder(PredicateSyntaxStrategy <T> owner, object key, StateTableStore table, bool trackIncrementalSteps, string?name, IEqualityComparer <T> comparer) { _owner = owner; _name = name; _comparer = comparer; _key = key; _filterTable = table.GetStateTableOrEmpty <SyntaxNode>(_owner._filterKey).ToBuilder(stepName: null, trackIncrementalSteps); _transformTable = table.GetStateTableOrEmpty <T>(_key).ToBuilder(_name, trackIncrementalSteps); }
public NodeStateTable <T> GetLatestStateTableForNode <T>(IIncrementalGeneratorNode <T> source) { // if we've already evaluated a node during this build, we can just return the existing result if (_stateTableBuilder.TryGetTable(source, out var table)) { return((NodeStateTable <T>)table); } // get the previous table, if there was one for this node NodeStateTable <T> previousTable = _previousTable._tables.GetStateTableOrEmpty <T>(source); // request the node update its state based on the current driver table and store the new result var newTable = source.UpdateStateTable(this, previousTable, _cancellationToken); _stateTableBuilder.SetTable(source, newTable); return(newTable); }
public Builder(SyntaxReceiverInputNode owner, DriverStateTable driverStateTable, bool trackIncrementalSteps) { _owner = owner; _nodeStateTable = driverStateTable.GetStateTableOrEmpty <ISyntaxContextReceiver?>(_owner).ToBuilder(stepName: null, trackIncrementalSteps); try { _receiver = owner._receiverCreator(); } catch (Exception e) { throw new UserFunctionException(e); } if (_receiver is object) { _walker = new GeneratorSyntaxWalker(_receiver); } }
public NodeStateTable <T> UpdateStateTable(DriverStateTable.Builder graphState, NodeStateTable <T> previousTable, CancellationToken cancellationToken) { var stopwatch = SharedStopwatch.StartNew(); var inputItems = _getInput(graphState); TimeSpan elapsedTime = stopwatch.Elapsed; // create a mutable hashset of the new items we can check against HashSet <T> itemsSet = new HashSet <T>(_inputComparer); foreach (var item in inputItems) { var added = itemsSet.Add(item); Debug.Assert(added); } var builder = graphState.CreateTableBuilder(previousTable, _name, _comparer); // We always have no inputs steps into an InputNode, but we track the difference between "no inputs" (empty collection) and "no step information" (default value) var noInputStepsStepInfo = builder.TrackIncrementalSteps ? ImmutableArray <(IncrementalGeneratorRunStep, int)> .Empty : default;
private (ImmutableArray <TInput>, ImmutableArray <(IncrementalGeneratorRunStep InputStep, int OutputIndex)>) GetValuesAndInputs( NodeStateTable <TInput> sourceTable, NodeStateTable <ImmutableArray <TInput> > previousTable, NodeStateTable <ImmutableArray <TInput> > .Builder newTable) { // Do an initial pass to both get the steps, and determine how many entries we'll have. var sourceInputsBuilder = newTable.TrackIncrementalSteps ? ArrayBuilder <(IncrementalGeneratorRunStep InputStep, int OutputIndex)> .GetInstance() : null; var entryCount = 0; foreach (var entry in sourceTable) { // Always keep track of its step information, regardless of if the entry was removed or not, so we // can accurately report how long it took and what actually happened (for testing validation). sourceInputsBuilder?.Add((entry.Step !, entry.OutputIndex)); if (entry.State != EntryState.Removed) { entryCount++; } } var sourceInputs = sourceInputsBuilder != null?sourceInputsBuilder.ToImmutableAndFree() : default; // First, see if we can reuse the entries from previousTable. // If not, produce the actual values we need from sourceTable. var result = tryReusePreviousTableValues(entryCount) ?? computeCurrentTableValues(entryCount); return(result, sourceInputs); ImmutableArray <TInput>?tryReusePreviousTableValues(int entryCount) { if (previousTable.Count != 1) { return(null); } var previousItems = previousTable.Single().item; // If they don't have the same length, we clearly can't reuse them. if (previousItems.Length != entryCount) { return(null); } var indexInPrevious = 0; foreach (var entry in sourceTable) { if (entry.State == EntryState.Removed) { continue; } // If the entries aren't the same, we can't reuse. if (!EqualityComparer <TInput> .Default.Equals(entry.Item, previousItems[indexInPrevious])) { return(null); } indexInPrevious++; } // We better have the exact same count as previousItems as we checked that above. Debug.Assert(indexInPrevious == previousItems.Length); // Looks good, we can reuse this. return(previousItems); } ImmutableArray <TInput> computeCurrentTableValues(int entryCount) { // Important: we initialize with the exact capacity we need here so that we don't make a pointless // scratch array that may be very large and may cause GC churn when it cannot be returned to the pool. var builder = ArrayBuilder <TInput> .GetInstance(entryCount); foreach (var entry in sourceTable) { if (entry.State == EntryState.Removed) { continue; } builder.Add(entry.Item); } Debug.Assert(builder.Count == entryCount); return(builder.ToImmutableAndFree()); } }
public NodeStateTable <ImmutableArray <TInput> > UpdateStateTable(DriverStateTable.Builder builder, NodeStateTable <ImmutableArray <TInput> > previousTable, CancellationToken cancellationToken) { // grab the source inputs var sourceTable = builder.GetLatestStateTableForNode(_sourceNode); // Semantics of a batch transform: // Batches will always exist (a batch of the empty table is still []) // There is only ever one input, the batch of the upstream table // - Output is cached when upstream is all cached // - Added when the previous table was empty // - Modified otherwise // update the table var newTable = builder.CreateTableBuilder(previousTable, _name, _comparer); // If this execution is tracking steps, then the source table should have also tracked steps or be the empty table. Debug.Assert(!newTable.TrackIncrementalSteps || (sourceTable.HasTrackedSteps || sourceTable.IsEmpty)); var stopwatch = SharedStopwatch.StartNew(); var(sourceValues, sourceInputs) = GetValuesAndInputs(sourceTable, previousTable, newTable); if (previousTable.IsEmpty) { newTable.AddEntry(sourceValues, EntryState.Added, stopwatch.Elapsed, sourceInputs, EntryState.Added); } else if (!sourceTable.IsCached || !newTable.TryUseCachedEntries(stopwatch.Elapsed, sourceInputs)) { if (!newTable.TryModifyEntry(sourceValues, _comparer, stopwatch.Elapsed, sourceInputs, EntryState.Modified)) { newTable.AddEntry(sourceValues, EntryState.Added, stopwatch.Elapsed, sourceInputs, EntryState.Added); } } return(newTable.ToImmutableAndFree()); }
public NodeStateTable <TOutput> UpdateStateTable(DriverStateTable.Builder builder, NodeStateTable <TOutput> previousTable, CancellationToken cancellationToken) { // grab the source inputs var sourceTable = builder.GetLatestStateTableForNode(_sourceNode); if (sourceTable.IsCached) { if (builder.DriverState.TrackIncrementalSteps) { return(previousTable.CreateCachedTableWithUpdatedSteps(sourceTable, _name)); } return(previousTable); } // Semantics of a transform: // Element-wise comparison of upstream table // - Cached or Removed: no transform, just use previous values // - Added: perform transform and add // - Modified: perform transform and do element wise comparison with previous results var newTable = builder.CreateTableBuilder(previousTable, _name); foreach (var entry in sourceTable) { var inputs = newTable.TrackIncrementalSteps ? ImmutableArray.Create((entry.Step !, entry.OutputIndex)) : default;
public NodeStateTable <TOutput> UpdateStateTable(DriverStateTable.Builder graphState, NodeStateTable <TOutput> previousTable, CancellationToken cancellationToken) { string stepName = Kind == IncrementalGeneratorOutputKind.Source ? WellKnownGeneratorOutputs.SourceOutput : WellKnownGeneratorOutputs.ImplementationSourceOutput; var sourceTable = graphState.GetLatestStateTableForNode(_source); if (sourceTable.IsCached) { if (graphState.DriverState.TrackIncrementalSteps) { return(previousTable.CreateCachedTableWithUpdatedSteps(sourceTable, stepName)); } return(previousTable); } var nodeTable = graphState.CreateTableBuilder(previousTable, stepName); foreach (var entry in sourceTable) { var inputs = nodeTable.TrackIncrementalSteps ? ImmutableArray.Create((entry.Step !, entry.OutputIndex)) : default;
public IStateTable GetSyntaxInputTable(SyntaxInputNode syntaxInputNode, NodeStateTable <SyntaxTree> syntaxTreeTable) { Debug.Assert(_syntaxInputNodes.Contains(syntaxInputNode)); // when we don't have a value for this node, we update all the syntax inputs at once if (!_tableBuilder.Contains(syntaxInputNode)) { // CONSIDER: when the compilation is the same as previous, the syntax trees must also be the same. // if we have a previous state table for a node, we can just short circuit knowing that it is up to date // This step isn't part of the tree, so we can skip recording. var compilationIsCached = _compilation == _previous._compilation; // get a builder for each input node var syntaxInputBuilders = ArrayBuilder <(SyntaxInputNode node, ISyntaxInputBuilder builder)> .GetInstance(_syntaxInputNodes.Length); foreach (var node in _syntaxInputNodes) { // We don't cache the tracked incremental steps in a manner that we can easily rehydrate between runs, // so we disable the cached compilation perf optimization when incremental step tracking is enabled. if (compilationIsCached && !_enableTracking && _previous._tables.TryGetValue(node, out var previousStateTable)) { _tableBuilder.SetTable(node, previousStateTable); } else { syntaxInputBuilders.Add((node, node.GetBuilder(_previous._tables, _enableTracking))); _syntaxTimes[node] = TimeSpan.Zero; } } if (syntaxInputBuilders.Count > 0) { // at this point we need to grab the syntax trees from the new compilation, and optionally diff them against the old ones NodeStateTable <SyntaxTree> syntaxTreeState = syntaxTreeTable; // update each tree for the builders, sharing the semantic model foreach (var(tree, state, syntaxTreeIndex, stepInfo) in syntaxTreeState) { var root = new Lazy <SyntaxNode>(() => tree.GetRoot(_cancellationToken)); var model = state != EntryState.Removed ? new Lazy <SemanticModel>(() => _compilation.GetSemanticModel(tree)) : null; for (int i = 0; i < syntaxInputBuilders.Count; i++) { var currentNode = syntaxInputBuilders[i].node; try { Stopwatch sw = Stopwatch.StartNew(); try { _cancellationToken.ThrowIfCancellationRequested(); syntaxInputBuilders[i].builder.VisitTree(root, state, model, _cancellationToken); } finally { var elapsed = sw.Elapsed; // if this node isn't the one that caused the update, ensure we remember it and remove the time it took from the requester if (currentNode != syntaxInputNode) { _syntaxTimes[syntaxInputNode] = _syntaxTimes[syntaxInputNode].Subtract(elapsed); _syntaxTimes[currentNode] = _syntaxTimes[currentNode].Add(elapsed); } } } catch (UserFunctionException ufe) { // we're evaluating this node ahead of time, so we can't just throw the exception // instead we'll hold onto it, and throw the exception when a downstream node actually // attempts to read the value _syntaxExceptions[currentNode] = ufe; syntaxInputBuilders.RemoveAt(i); i--; } } } // save the updated inputs foreach ((var node, ISyntaxInputBuilder builder) in syntaxInputBuilders) { builder.SaveStateAndFree(_tableBuilder); Debug.Assert(_tableBuilder.Contains(node)); } } syntaxInputBuilders.Free(); } // if we don't have an entry for this node, it must have thrown an exception if (!_tableBuilder.TryGetTable(syntaxInputNode, out var result)) { throw _syntaxExceptions[syntaxInputNode]; } return(result); }
public NodeStateTable <TOutput> UpdateStateTable(DriverStateTable.Builder builder, NodeStateTable <TOutput> previousTable, CancellationToken cancellationToken) { // grab the source inputs var sourceTable = builder.GetLatestStateTableForNode(_sourceNode); if (sourceTable.IsCached) { return(previousTable); } // Semantics of a transform: // Element-wise comparison of upstream table // - Cached or Removed: no transform, just use previous values // - Added: perform transform and add // - Modified: perform transform and do element wise comparison with previous results var newTable = previousTable.ToBuilder(); foreach (var entry in sourceTable) { if (entry.state == EntryState.Removed) { newTable.RemoveEntries(); } else if (entry.state != EntryState.Cached || !newTable.TryUseCachedEntries()) { // generate the new entries var newOutputs = _func(entry.item, cancellationToken); if (entry.state != EntryState.Modified || !newTable.TryModifyEntries(newOutputs, _comparer)) { newTable.AddEntries(newOutputs, EntryState.Added); } } } return(newTable.ToImmutableAndFree()); }
public NodeStateTable <T> UpdateStateTable(DriverStateTable.Builder graphState, NodeStateTable <T> previousTable, CancellationToken cancellationToken) { var inputItems = _getInput(graphState); // create a mutable hashset of the new items we can check against HashSet <T> itemsSet = new HashSet <T>(_comparer); foreach (var item in inputItems) { var added = itemsSet.Add(item); Debug.Assert(added); } var builder = previousTable.ToBuilder(); // for each item in the previous table, check if its still in the new items foreach ((var oldItem, _) in previousTable) { if (itemsSet.Remove(oldItem)) { // we're iterating the table, so know that it has entries var usedCache = builder.TryUseCachedEntries(); Debug.Assert(usedCache); } else { builder.RemoveEntries(); } } // any remaining new items are added foreach (var newItem in itemsSet) { builder.AddEntry(newItem, EntryState.Added); } return(builder.ToImmutableAndFree()); }
public NodeStateTable <T> UpdateStateTable(DriverStateTable.Builder graphState, NodeStateTable <T> previousTable, CancellationToken cancellationToken) { var inputItems = _getInput(graphState); // create a mutable hashset of the new items we can check against HashSet <T> itemsSet = new HashSet <T>(_comparer); foreach (var item in inputItems) { var added = itemsSet.Add(item); Debug.Assert(added); } var builder = previousTable.ToBuilder(); // for each item in the previous table, check if its still in the new items int itemIndex = 0; foreach ((var oldItem, _) in previousTable) { if (itemsSet.Remove(oldItem)) { // we're iterating the table, so know that it has entries var usedCache = builder.TryUseCachedEntries(); Debug.Assert(usedCache); } else if (inputItems.Length == previousTable.Count) { // When the number of items matches the previous iteration, we use a heuristic to mark the input as modified // This allows us to correctly 'replace' items even when they aren't actually the same. In the case that the // item really isn't modified, but a new item, we still function correctly as we mostly treat them the same, // but will perform an extra comparison that is omitted in the pure 'added' case. var modified = builder.TryModifyEntry(inputItems[itemIndex], _comparer); Debug.Assert(modified); itemsSet.Remove(inputItems[itemIndex]); } else { builder.RemoveEntries(); } itemIndex++; } // any remaining new items are added foreach (var newItem in itemsSet) { builder.AddEntry(newItem, EntryState.Added); } return(builder.ToImmutableAndFree()); }
public Builder(SyntaxInputNode <T> owner, DriverStateTable table, bool trackIncrementalSteps) { _owner = owner; _filterTable = table.GetStateTableOrEmpty <SyntaxNode>(_owner._filterKey).ToBuilder(stepName: null, trackIncrementalSteps); _transformTable = table.GetStateTableOrEmpty <T>(_owner).ToBuilder(_owner.Name, trackIncrementalSteps); }
public NodeStateTable <T> .Builder CreateTableBuilder <T>( NodeStateTable <T> previousTable, string?stepName, IEqualityComparer <T>?equalityComparer, int?tableCapacity = null) { return(previousTable.ToBuilder(stepName, DriverState.TrackIncrementalSteps, equalityComparer, tableCapacity)); }
public NodeStateTable <T> UpdateStateTable(DriverStateTable.Builder graphState, NodeStateTable <T> previousTable, CancellationToken cancellationToken) { return((NodeStateTable <T>)graphState.GetSyntaxInputTable(this)); }
public NodeStateTable <ImmutableArray <TInput> > UpdateStateTable(DriverStateTable.Builder builder, NodeStateTable <ImmutableArray <TInput> > previousTable, CancellationToken cancellationToken) { // grab the source inputs var sourceTable = builder.GetLatestStateTableForNode(_sourceNode); // Semantics of a batch transform: // Batches will always exist (a batch of the empty table is still []) // There is only ever one input, the batch of the upstream table // - Output is cached when upstream is all cached // - Added when the previous table was empty // - Modified otherwise var source = sourceTable.Batch(); // update the table var newTable = previousTable.ToBuilder(); if (!sourceTable.IsCached || !newTable.TryUseCachedEntries()) { if (!newTable.TryModifyEntry(source, _comparer)) { newTable.AddEntry(source, EntryState.Added); } } return(newTable.ToImmutableAndFree()); }
public Builder(SyntaxInputNode <T> owner, DriverStateTable table) { _owner = owner; _filterTable = table.GetStateTableOrEmpty <SyntaxNode>(_owner._filterKey).ToBuilder(); _transformTable = table.GetStateTableOrEmpty <T>(_owner).ToBuilder(); }
public NodeStateTable <TOutput> UpdateStateTable(DriverStateTable.Builder graphState, NodeStateTable <TOutput> previousTable, CancellationToken cancellationToken) { var sourceTable = graphState.GetLatestStateTableForNode(_source); if (sourceTable.IsCached) { return(previousTable); } var nodeTable = previousTable.ToBuilder(); foreach (var entry in sourceTable) { if (entry.state == EntryState.Removed) { nodeTable.RemoveEntries(); } else if (entry.state != EntryState.Cached || !nodeTable.TryUseCachedEntries()) { // we don't currently handle modified any differently than added at the output // we just run the action and mark the new source as added. In theory we could compare // the diagnostics and sources produced and compare them, to see if they are any different // than before. var sourcesBuilder = new AdditionalSourcesCollection(_sourceExtension); var diagnostics = DiagnosticBag.GetInstance(); SourceProductionContext context = new SourceProductionContext(sourcesBuilder, diagnostics, cancellationToken); try { _action(context, entry.item); nodeTable.AddEntry((sourcesBuilder.ToImmutable(), diagnostics.ToReadOnly()), EntryState.Added); } finally { sourcesBuilder.Free(); diagnostics.Free(); } } } return(nodeTable.ToImmutableAndFree()); }
public NodeStateTable <T> .Builder CreateTableBuilder <T>(NodeStateTable <T> previousTable, string?stepName) { return(previousTable.ToBuilder(stepName, DriverState.TrackIncrementalSteps)); }
public NodeStateTable <ImmutableArray <TInput> > UpdateStateTable(DriverStateTable.Builder builder, NodeStateTable <ImmutableArray <TInput> > previousTable, CancellationToken cancellationToken) { // grab the source inputs var sourceTable = builder.GetLatestStateTableForNode(_sourceNode); // Semantics of a batch transform: // Batches will always exist (a batch of the empty table is still []) // There is only ever one input, the batch of the upstream table // - Output is cached when upstream is all cached // - Added when the previous table was empty // - Modified otherwise // update the table var newTable = builder.CreateTableBuilder(previousTable, _name); // If this execution is tracking steps, then the source table should have also tracked steps or be the empty table. Debug.Assert(!newTable.TrackIncrementalSteps || (sourceTable.HasTrackedSteps || sourceTable.IsEmpty)); var stopwatch = SharedStopwatch.StartNew(); var batchedSourceEntries = sourceTable.Batch(); var sourceValues = batchedSourceEntries.SelectAsArray(sourceEntry => sourceEntry.State != EntryState.Removed, sourceEntry => sourceEntry.Item); var sourceInputs = newTable.TrackIncrementalSteps ? batchedSourceEntries.SelectAsArray(sourceEntry => (sourceEntry.Step !, sourceEntry.OutputIndex)) : default;
public IStateTable GetSyntaxInputTable(ISyntaxInputNode syntaxInputNode) { Debug.Assert(_syntaxInputNodes.Contains(syntaxInputNode)); // when we don't have a value for this node, we update all the syntax inputs at once if (!_tableBuilder.ContainsKey(syntaxInputNode)) { // CONSIDER: when the compilation is the same as previous, the syntax trees must also be the same. // if we have a previous state table for a node, we can just short circuit knowing that it is up to date // This step isn't part of the tree, so we can skip recording. var compilationIsCached = GetLatestStateTableForNode(SharedInputNodes.Compilation).IsCached; // get a builder for each input node var syntaxInputBuilders = ArrayBuilder <ISyntaxInputBuilder> .GetInstance(_syntaxInputNodes.Length); foreach (var node in _syntaxInputNodes) { // TODO: We don't cache the tracked incremental steps in a manner that we can easily rehydrate between runs, // so we'll disable the cached compilation perf optimization when incremental step tracking is enabled. if (compilationIsCached && !DriverState.TrackIncrementalSteps && _previousTable._tables.TryGetValue(node, out var previousStateTable)) { _tableBuilder.Add(node, previousStateTable); } else { syntaxInputBuilders.Add(node.GetBuilder(_previousTable, DriverState.TrackIncrementalSteps)); } } if (syntaxInputBuilders.Count == 0) { // bring over the previously cached syntax tree inputs _tableBuilder[SharedInputNodes.SyntaxTrees] = _previousTable._tables[SharedInputNodes.SyntaxTrees]; } else { GeneratorRunStateTable.Builder temporaryRunStateBuilder = new GeneratorRunStateTable.Builder(DriverState.TrackIncrementalSteps); NodeStateTable <SyntaxTree> syntaxTreeState = GetLatestStateTableForNode(SharedInputNodes.SyntaxTrees); // update each tree for the builders, sharing the semantic model foreach ((var tree, var state, var syntaxTreeIndex, var stepInfo) in syntaxTreeState) { var root = new Lazy <SyntaxNode>(() => tree.GetRoot(_cancellationToken)); var model = state != EntryState.Removed ? Compilation.GetSemanticModel(tree) : null; for (int i = 0; i < syntaxInputBuilders.Count; i++) { try { _cancellationToken.ThrowIfCancellationRequested(); syntaxInputBuilders[i].VisitTree(root, state, model, _cancellationToken); } catch (UserFunctionException ufe) { // we're evaluating this node ahead of time, so we can't just throw the exception // instead we'll hold onto it, and throw the exception when a downstream node actually // attempts to read the value _syntaxExceptions[syntaxInputBuilders[i].SyntaxInputNode] = ufe; syntaxInputBuilders.RemoveAt(i); i--; } } } // save the updated inputs foreach (ISyntaxInputBuilder builder in syntaxInputBuilders) { builder.SaveStateAndFree(_tableBuilder); Debug.Assert(_tableBuilder.ContainsKey(builder.SyntaxInputNode)); } } syntaxInputBuilders.Free(); } // if we don't have an entry for this node, it must have thrown an exception if (!_tableBuilder.ContainsKey(syntaxInputNode)) { throw _syntaxExceptions[syntaxInputNode]; } return(_tableBuilder[syntaxInputNode]); }