public Guid TriggerAndOverride(string queueName, FunctionSnapshot function, IDictionary<string, string> arguments, Guid? parentId, ExecutionReason reason) { if (function == null) { throw new ArgumentNullException("function"); } Guid id = Guid.NewGuid(); CallAndOverrideMessage message = new CallAndOverrideMessage { Id = id, FunctionId = function.HostFunctionId, Arguments = arguments, ParentId = parentId, Reason = reason }; string functionId = new FunctionIdentifier(function.QueueName, function.HostFunctionId).ToString(); FunctionInstanceSnapshot snapshot = CreateSnapshot(id, arguments, parentId, DateTimeOffset.UtcNow, functionId, function.FullName, function.ShortName); _logger.LogFunctionQueued(snapshot); _sender.Enqueue(queueName, message); return id; }
// This method runs concurrently with other index processing. // Ensure all logic here is idempotent. public void ProcessFunctionStarted(FunctionStartedMessage message) { if (message == null) { throw new ArgumentNullException("message"); } FunctionInstanceSnapshot snapshot = CreateSnapshot(message); _functionInstanceLogger.LogFunctionStarted(snapshot); string functionId = new FunctionIdentifier(message.SharedQueueName, message.Function.Id).ToString(); Guid functionInstanceId = message.FunctionInstanceId; DateTimeOffset startTime = message.StartTime; WebJobRunIdentifier webJobRunId = message.WebJobRunIdentifier; Guid? parentId = message.ParentId; // Race to write index entries for function started. if (!HasLoggedFunctionCompleted(functionInstanceId)) { CreateOrUpdateIndexEntries(snapshot, startTime, webJobRunId); } // If the function has since completed, we lost the race. // Delete any function started index entries. // Note that this code does not depend on whether or not the index entries were previously written by this // method, as this processing could have been aborted and resumed at another point in time. In that case, // we still own cleaning up any dangling function started index entries. DateTimeOffset? functionCompletedTime = GetFunctionCompletedTime(functionInstanceId); bool hasLoggedFunctionCompleted = functionCompletedTime.HasValue; if (hasLoggedFunctionCompleted) { DeleteFunctionStartedIndexEntriesIfNeeded(functionInstanceId, message.StartTime, functionCompletedTime.Value, functionId, parentId, webJobRunId); } }
// This method runs concurrently with other index processing. // Ensure all logic here is idempotent. public void ProcessFunctionCompleted(FunctionCompletedMessage message) { if (message == null) { throw new ArgumentNullException("message"); } FunctionInstanceSnapshot snapshot = CreateSnapshot(message); // The completed message includes the full parameter logs; delete the extra blob used for running status // updates. DeleteParameterLogBlob(snapshot.ParameterLogBlob); snapshot.ParameterLogBlob = null; _functionInstanceLogger.LogFunctionCompleted(snapshot); Guid functionInstanceId = message.FunctionInstanceId; DateTimeOffset endTime = message.EndTime; string functionId = new FunctionIdentifier(message.SharedQueueName, message.Function.Id).ToString(); Guid? parentId = message.ParentId; WebJobRunIdentifier webJobRunId = message.WebJobRunIdentifier; DeleteFunctionStartedIndexEntriesIfNeeded(functionInstanceId, message.StartTime, endTime, functionId, parentId, webJobRunId); CreateOrUpdateIndexEntries(snapshot, endTime, webJobRunId); // Increment is non-idempotent. If the process dies before deleting the message that triggered it, it can // occur multiple times. // If we wanted to make this operation idempotent, one option would be to store the list of function // instance IDs that succeeded & failed, rather than just the counters, so duplicate operations could be // detected. // For now, we just do a non-idempotent increment last, which makes it very unlikely that the queue message // would not subsequently get deleted. if (message.Succeeded) { _statisticsWriter.IncrementSuccess(functionId); } else { _statisticsWriter.IncrementFailure(functionId); } }
public void ProcessHostStarted(HostStartedMessage message) { if (message == null) { throw new ArgumentNullException("message"); } string hostId = message.SharedQueueName; DateTimeOffset hostVersion = message.EnqueuedOn; DateTime hostVersionUtc = hostVersion.UtcDateTime; IEnumerable<FunctionDescriptor> messageFunctions = message.Functions ?? Enumerable.Empty<FunctionDescriptor>(); HostSnapshot newSnapshot = new HostSnapshot { HostVersion = hostVersion, FunctionIds = CreateFunctionIds(messageFunctions) }; if (_hostIndexManager.UpdateOrCreateIfLatest(hostId, newSnapshot)) { IEnumerable<VersionedMetadata> existingFunctions = _functionIndexManager.List(hostId); IEnumerable<VersionedMetadata> removedFunctions = existingFunctions .Where((f) => !newSnapshot.FunctionIds.Any(i => f.Id == i)); foreach (VersionedMetadata removedFunction in removedFunctions) { // Remove all functions no longer in our list (unless they exist with a later host version than // ours). string fullId = new FunctionIdentifier(hostId, removedFunction.Id).ToString(); _functionIndexManager.DeleteIfLatest(fullId, hostVersion, removedFunction.ETag, removedFunction.Version); } HeartbeatDescriptor heartbeat = message.Heartbeat; IEnumerable<FunctionDescriptor> addedFunctions = messageFunctions .Where((d) => !existingFunctions.Any(f => d.Id == f.Id)); foreach (FunctionDescriptor addedFunction in addedFunctions) { // Create any functions just appearing in our list (or update existing ones if they're earlier than // ours). FunctionSnapshot snapshot = CreateFunctionSnapshot(hostId, heartbeat, addedFunction, hostVersion); _functionIndexManager.CreateOrUpdateIfLatest(snapshot); } // Update any functions appearing in both lists provided they're still earlier than ours (or create them // if they've since been deleted). var possiblyUpdatedFunctions = existingFunctions .Join(messageFunctions, (f) => f.Id, (d) => d.Id, (f, d) => new { Descriptor = d, HostVersion = f.Version, ETag = f.ETag }); foreach (var possiblyUpdatedFunction in possiblyUpdatedFunctions) { FunctionSnapshot snapshot = CreateFunctionSnapshot(hostId, heartbeat, possiblyUpdatedFunction.Descriptor, hostVersion); _functionIndexManager.UpdateOrCreateIfLatest(snapshot, possiblyUpdatedFunction.ETag, possiblyUpdatedFunction.HostVersion); } } // Delete any functions we may have added or updated that are no longer in the index. // The create and update calls above may have occured after another instance started processing a later // version of this host. If that instance had already read the existing function list before we added the // function, it would think the function had already been deleted. In the end, we can't leave a function // around unless it's still in the host index after we've added or updated it. HostSnapshot finalSnapshot = _hostIndexManager.Read(hostId); IEnumerable<FunctionDescriptor> functionsRemovedAfterThisHostVersion; if (finalSnapshot == null) { functionsRemovedAfterThisHostVersion = messageFunctions; } else if (finalSnapshot.HostVersion.UtcDateTime > hostVersionUtc) { // Note that we base the list of functions to delete on what's in the HostStartedMessage, not what's in // the addedFunctions and possibleUpdatedFunctions variables, as this instance could have been aborted // and resumed and could lose state like those local variables. functionsRemovedAfterThisHostVersion = messageFunctions.Where( f => !finalSnapshot.FunctionIds.Any((i) => f.Id == i)); } else { functionsRemovedAfterThisHostVersion = Enumerable.Empty<FunctionDescriptor>(); } foreach (FunctionDescriptor functionNoLongerInSnapshot in functionsRemovedAfterThisHostVersion) { string fullId = new FunctionIdentifier(hostId, functionNoLongerInSnapshot.Id).ToString(); _functionIndexManager.DeleteIfLatest(fullId, hostVersionUtc); } _functionIndexVersionManager.UpdateOrCreateIfLatest(newSnapshot.HostVersion); }