public Guid TriggerAndOverride(string queueName, FunctionSnapshot function,
IDictionary<string, string> arguments, Guid? parentId, ExecutionReason reason)
{
if (function == null)
{
throw new ArgumentNullException("function");
}
Guid id = Guid.NewGuid();
CallAndOverrideMessage message = new CallAndOverrideMessage
{
Id = id,
FunctionId = function.HostFunctionId,
Arguments = arguments,
ParentId = parentId,
Reason = reason
};
string functionId = new FunctionIdentifier(function.QueueName, function.HostFunctionId).ToString();
FunctionInstanceSnapshot snapshot = CreateSnapshot(id, arguments, parentId, DateTimeOffset.UtcNow,
functionId, function.FullName, function.ShortName);
_logger.LogFunctionQueued(snapshot);
_sender.Enqueue(queueName, message);
return id;
}
// This method runs concurrently with other index processing.
// Ensure all logic here is idempotent.
public void ProcessFunctionStarted(FunctionStartedMessage message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
FunctionInstanceSnapshot snapshot = CreateSnapshot(message);
_functionInstanceLogger.LogFunctionStarted(snapshot);
string functionId = new FunctionIdentifier(message.SharedQueueName, message.Function.Id).ToString();
Guid functionInstanceId = message.FunctionInstanceId;
DateTimeOffset startTime = message.StartTime;
WebJobRunIdentifier webJobRunId = message.WebJobRunIdentifier;
Guid? parentId = message.ParentId;
// Race to write index entries for function started.
if (!HasLoggedFunctionCompleted(functionInstanceId))
{
CreateOrUpdateIndexEntries(snapshot, startTime, webJobRunId);
}
// If the function has since completed, we lost the race.
// Delete any function started index entries.
// Note that this code does not depend on whether or not the index entries were previously written by this
// method, as this processing could have been aborted and resumed at another point in time. In that case,
// we still own cleaning up any dangling function started index entries.
DateTimeOffset? functionCompletedTime = GetFunctionCompletedTime(functionInstanceId);
bool hasLoggedFunctionCompleted = functionCompletedTime.HasValue;
if (hasLoggedFunctionCompleted)
{
DeleteFunctionStartedIndexEntriesIfNeeded(functionInstanceId, message.StartTime,
functionCompletedTime.Value, functionId, parentId, webJobRunId);
}
}
// This method runs concurrently with other index processing.
// Ensure all logic here is idempotent.
public void ProcessFunctionCompleted(FunctionCompletedMessage message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
FunctionInstanceSnapshot snapshot = CreateSnapshot(message);
// The completed message includes the full parameter logs; delete the extra blob used for running status
// updates.
DeleteParameterLogBlob(snapshot.ParameterLogBlob);
snapshot.ParameterLogBlob = null;
_functionInstanceLogger.LogFunctionCompleted(snapshot);
Guid functionInstanceId = message.FunctionInstanceId;
DateTimeOffset endTime = message.EndTime;
string functionId = new FunctionIdentifier(message.SharedQueueName, message.Function.Id).ToString();
Guid? parentId = message.ParentId;
WebJobRunIdentifier webJobRunId = message.WebJobRunIdentifier;
DeleteFunctionStartedIndexEntriesIfNeeded(functionInstanceId, message.StartTime, endTime, functionId,
parentId, webJobRunId);
CreateOrUpdateIndexEntries(snapshot, endTime, webJobRunId);
// Increment is non-idempotent. If the process dies before deleting the message that triggered it, it can
// occur multiple times.
// If we wanted to make this operation idempotent, one option would be to store the list of function
// instance IDs that succeeded & failed, rather than just the counters, so duplicate operations could be
// detected.
// For now, we just do a non-idempotent increment last, which makes it very unlikely that the queue message
// would not subsequently get deleted.
if (message.Succeeded)
{
_statisticsWriter.IncrementSuccess(functionId);
}
else
{
_statisticsWriter.IncrementFailure(functionId);
}
}
public void ProcessHostStarted(HostStartedMessage message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
string hostId = message.SharedQueueName;
DateTimeOffset hostVersion = message.EnqueuedOn;
DateTime hostVersionUtc = hostVersion.UtcDateTime;
IEnumerable<FunctionDescriptor> messageFunctions = message.Functions ?? Enumerable.Empty<FunctionDescriptor>();
HostSnapshot newSnapshot = new HostSnapshot
{
HostVersion = hostVersion,
FunctionIds = CreateFunctionIds(messageFunctions)
};
if (_hostIndexManager.UpdateOrCreateIfLatest(hostId, newSnapshot))
{
IEnumerable<VersionedMetadata> existingFunctions = _functionIndexManager.List(hostId);
IEnumerable<VersionedMetadata> removedFunctions = existingFunctions
.Where((f) => !newSnapshot.FunctionIds.Any(i => f.Id == i));
foreach (VersionedMetadata removedFunction in removedFunctions)
{
// Remove all functions no longer in our list (unless they exist with a later host version than
// ours).
string fullId = new FunctionIdentifier(hostId, removedFunction.Id).ToString();
_functionIndexManager.DeleteIfLatest(fullId, hostVersion, removedFunction.ETag,
removedFunction.Version);
}
HeartbeatDescriptor heartbeat = message.Heartbeat;
IEnumerable<FunctionDescriptor> addedFunctions = messageFunctions
.Where((d) => !existingFunctions.Any(f => d.Id == f.Id));
foreach (FunctionDescriptor addedFunction in addedFunctions)
{
// Create any functions just appearing in our list (or update existing ones if they're earlier than
// ours).
FunctionSnapshot snapshot = CreateFunctionSnapshot(hostId, heartbeat, addedFunction, hostVersion);
_functionIndexManager.CreateOrUpdateIfLatest(snapshot);
}
// Update any functions appearing in both lists provided they're still earlier than ours (or create them
// if they've since been deleted).
var possiblyUpdatedFunctions = existingFunctions
.Join(messageFunctions, (f) => f.Id, (d) => d.Id, (f, d) => new
{
Descriptor = d,
HostVersion = f.Version,
ETag = f.ETag
});
foreach (var possiblyUpdatedFunction in possiblyUpdatedFunctions)
{
FunctionSnapshot snapshot = CreateFunctionSnapshot(hostId, heartbeat,
possiblyUpdatedFunction.Descriptor, hostVersion);
_functionIndexManager.UpdateOrCreateIfLatest(snapshot, possiblyUpdatedFunction.ETag,
possiblyUpdatedFunction.HostVersion);
}
}
// Delete any functions we may have added or updated that are no longer in the index.
// The create and update calls above may have occured after another instance started processing a later
// version of this host. If that instance had already read the existing function list before we added the
// function, it would think the function had already been deleted. In the end, we can't leave a function
// around unless it's still in the host index after we've added or updated it.
HostSnapshot finalSnapshot = _hostIndexManager.Read(hostId);
IEnumerable<FunctionDescriptor> functionsRemovedAfterThisHostVersion;
if (finalSnapshot == null)
{
functionsRemovedAfterThisHostVersion = messageFunctions;
}
else if (finalSnapshot.HostVersion.UtcDateTime > hostVersionUtc)
{
// Note that we base the list of functions to delete on what's in the HostStartedMessage, not what's in
// the addedFunctions and possibleUpdatedFunctions variables, as this instance could have been aborted
// and resumed and could lose state like those local variables.
functionsRemovedAfterThisHostVersion = messageFunctions.Where(
f => !finalSnapshot.FunctionIds.Any((i) => f.Id == i));
}
else
{
functionsRemovedAfterThisHostVersion = Enumerable.Empty<FunctionDescriptor>();
}
foreach (FunctionDescriptor functionNoLongerInSnapshot in functionsRemovedAfterThisHostVersion)
{
string fullId = new FunctionIdentifier(hostId, functionNoLongerInSnapshot.Id).ToString();
_functionIndexManager.DeleteIfLatest(fullId, hostVersionUtc);
}
_functionIndexVersionManager.UpdateOrCreateIfLatest(newSnapshot.HostVersion);
}
