public Guid TriggerAndOverride(string queueName, FunctionSnapshot function,
IDictionary <string, string> arguments, Guid?parentId, ExecutionReason reason)
{
if (function == null)
{
throw new ArgumentNullException("function");
}
Guid id = Guid.NewGuid();
CallAndOverrideMessage message = new CallAndOverrideMessage
{
Id = id,
FunctionId = function.HostFunctionId,
Arguments = arguments,
ParentId = parentId,
Reason = reason
};
string functionId = new FunctionIdentifier(function.QueueName, function.HostFunctionId).ToString();
FunctionInstanceSnapshot snapshot = CreateSnapshot(id, arguments, parentId, DateTimeOffset.UtcNow,
functionId, function.FullName, function.ShortName);
_logger.LogFunctionQueued(snapshot);
_sender.Enqueue(queueName, message);
return(id);
}
private static bool TryResolveParameters(FunctionSnapshot function, FunctionInstanceSnapshot snapshot, out IEnumerable <FunctionParameterViewModel> resolvedParameters)
{
List <FunctionParameterViewModel> parameters = new List <FunctionParameterViewModel>();
foreach (KeyValuePair <string, ParameterSnapshot> parameter in function.Parameters)
{
if (!snapshot.Arguments.ContainsKey(parameter.Key))
{
resolvedParameters = null;
return(false);
}
FunctionParameterViewModel parameterModel = new FunctionParameterViewModel
{
Name = parameter.Key,
Description = parameter.Value.Prompt,
Value = snapshot.Arguments[parameter.Key].Value
};
parameters.Add(parameterModel);
}
resolvedParameters = parameters;
return(true);
}
// This method runs concurrently with other index processing.
// Ensure all logic here is idempotent.
public void ProcessFunctionStarted(FunctionStartedMessage message)
{
FunctionInstanceSnapshot snapshot = CreateSnapshot(message);
_functionInstanceLogger.LogFunctionStarted(snapshot);
string functionId = new FunctionIdentifier(message.SharedQueueName, message.Function.Id).ToString();
Guid functionInstanceId = message.FunctionInstanceId;
DateTimeOffset startTime = message.StartTime;
WebJobRunIdentifier webJobRunId = message.WebJobRunIdentifier;
Guid? parentId = message.ParentId;
// Race to write index entries for function started.
if (!HasLoggedFunctionCompleted(functionInstanceId))
{
CreateOrUpdateIndexEntries(snapshot, startTime, webJobRunId);
}
// If the function has since completed, we lost the race.
// Delete any function started index entries.
// Note that this code does not depend on whether or not the index entries were previously written by this
// method, as this processing could have been aborted and resumed at another point in time. In that case,
// we still own cleaning up any dangling function started index entries.
DateTimeOffset?functionCompletedTime = GetFunctionCompletedTime(functionInstanceId);
bool hasLoggedFunctionCompleted = functionCompletedTime.HasValue;
if (hasLoggedFunctionCompleted)
{
DeleteFunctionStartedIndexEntriesIfNeeded(functionInstanceId, message.StartTime,
functionCompletedTime.Value, functionId, parentId, webJobRunId);
}
}
public IHttpActionResult GetFunctionInvocation(Guid functionInvocationId)
{
var func = _functionInstanceLookup.Lookup(functionInvocationId);
if (func == null)
{
return(NotFound());
}
var model = new FunctionInstanceDetailsViewModel();
model.Invocation = new InvocationLogViewModel(func, HostInstanceHasHeartbeat(func));
model.TriggerReason = new TriggerReasonViewModel(func);
model.Trigger = model.TriggerReason.ToString();
model.IsAborted = model.Invocation.Status == ViewModels.FunctionInstanceStatus.Running &&
_aborter.HasRequestedHostInstanceAbort(func.InstanceQueueName);
model.Parameters = CreateParameterModels(_account, func);
// fetch ancestor
var parentGuid = func.ParentId;
if (parentGuid.HasValue)
{
FunctionInstanceSnapshot ancestor = _functionInstanceLookup.Lookup(parentGuid.Value);
if (ancestor != null)
{
bool?hasValidHeartbeat = HostInstanceHasHeartbeat(ancestor);
model.Ancestor = new InvocationLogViewModel(ancestor, hasValidHeartbeat);
}
}
return(Ok(model));
}
private static FunctionInstanceSnapshot CreateSnapshot(FunctionCompletedMessage message)
{
FunctionInstanceSnapshot entity = CreateSnapshot((FunctionStartedMessage)message);
entity.ParameterLogs = message.ParameterLogs;
entity.EndTime = message.EndTime;
entity.Succeeded = message.Succeeded;
entity.ExceptionType = message.Failure != null ? message.Failure.ExceptionType : null;
entity.ExceptionMessage = message.Failure != null ? message.Failure.ExceptionDetails : null;
return(entity);
}
private bool?HostInstanceHasHeartbeat(FunctionInstanceSnapshot snapshot)
{
HeartbeatDescriptor heartbeat = snapshot.Heartbeat;
if (heartbeat == null)
{
return(null);
}
return(HostInstanceHasHeartbeat(heartbeat.SharedContainerName, heartbeat.SharedDirectoryName,
heartbeat.InstanceBlobName, heartbeat.ExpirationInSeconds));
}
internal static BlobBoundParameterModel CreateExtendedBlobModel(FunctionInstanceSnapshot snapshot, FunctionInstanceArgument argument)
{
Debug.Assert(argument != null);
if (argument.Value == null)
{
return(null);
}
string[] components = argument.Value.Split(new char[] { '/' });
if (components.Length != 2)
{
return(null);
}
var blobParam = new BlobBoundParameterModel();
blobParam.IsOutput = argument.IsBlobOutput;
blobParam.ConnectionStringKey = ConnectionStringProvider.GetPrefixedConnectionStringName(argument.AccountName);
CloudStorageAccount account = argument.GetStorageAccount();
if (account == null)
{
blobParam.IsConnectionStringMissing = true;
return(blobParam);
}
CloudBlockBlob blob = account
.CreateCloudBlobClient()
.GetContainerReference(components[0])
.GetBlockBlobReference(components[1]);
Guid?blobWriter = GetBlobWriter(blob);
if (!blobWriter.HasValue)
{
blobParam.IsBlobMissing = true;
}
else
{
blobParam.OwnerId = blobWriter.Value;
if (blobWriter.Value == snapshot.Id)
{
blobParam.IsBlobOwnedByCurrentFunctionInstance = true;
}
}
return(blobParam);
}
private InvocationLogViewModel CreateInvocationEntry(RecentInvocationEntry entry)
{
Debug.Assert(entry != null);
var metadataSnapshot = new FunctionInstanceSnapshot();
metadataSnapshot.Id = entry.Id;
metadataSnapshot.DisplayTitle = entry.DisplayTitle;
metadataSnapshot.StartTime = entry.StartTime;
metadataSnapshot.EndTime = entry.EndTime;
metadataSnapshot.Succeeded = entry.Succeeded;
metadataSnapshot.Heartbeat = entry.Heartbeat;
return(new InvocationLogViewModel(metadataSnapshot, HostInstanceHasHeartbeat(metadataSnapshot)));
}
public static void AddParameterModels(string parameterName, FunctionInstanceArgument argument, ParameterLog log,
FunctionInstanceSnapshot snapshot, ICollection <ParamModel> parameterModels)
{
ParamModel model = new ParamModel
{
Name = parameterName,
ArgInvokeString = argument.Value
};
if (log != null)
{
model.Status = Format(log);
}
if (argument.IsBlob)
{
model.ExtendedBlobModel = LogAnalysis.CreateExtendedBlobModel(snapshot, argument);
}
parameterModels.Add(model);
// Special-case IBinder, which adds sub-parameters.
BinderParameterLog binderLog = log as BinderParameterLog;
if (binderLog != null)
{
IEnumerable <BinderParameterLogItem> items = binderLog.Items;
if (items != null)
{
int count = items.Count();
model.Status = String.Format(CultureInfo.CurrentCulture, "Bound {0} object{1}.", count, count != 1 ? "s" : String.Empty);
foreach (BinderParameterLogItem item in items)
{
if (item == null)
{
continue;
}
ParameterSnapshot itemSnapshot = HostIndexer.CreateParameterSnapshot(item.Descriptor);
string itemName = parameterName + ": " + itemSnapshot.AttributeText;
FunctionInstanceArgument itemArgument =
FunctionIndexer.CreateFunctionInstanceArgument(item.Value, item.Descriptor);
AddParameterModels(itemName, itemArgument, item.Log, snapshot, parameterModels);
}
}
}
}
private void CreateOrUpdateIndexEntries(FunctionInstanceSnapshot snapshot, DateTimeOffset timestamp, WebJobRunIdentifier webJobRunId)
{
_recentInvocationsWriter.CreateOrUpdate(snapshot, timestamp);
_recentInvocationsByFunctionWriter.CreateOrUpdate(snapshot, timestamp);
if (webJobRunId != null)
{
_recentInvocationsByJobRunWriter.CreateOrUpdate(snapshot, webJobRunId, timestamp);
}
if (snapshot.ParentId.HasValue)
{
_recentInvocationsByParentWriter.CreateOrUpdate(snapshot, timestamp);
}
}
internal InvocationLogViewModel(FunctionInstanceSnapshot snapshot, bool?heartbeatIsValid)
{
Id = snapshot.Id;
FunctionName = snapshot.FunctionShortName;
FunctionId = snapshot.FunctionId;
FunctionFullName = snapshot.FunctionFullName;
FunctionDisplayTitle = snapshot.DisplayTitle;
HostInstanceId = snapshot.HostInstanceId;
InstanceQueueName = snapshot.InstanceQueueName;
if (snapshot.WebSiteName != null &&
snapshot.WebSiteName == Environment.GetEnvironmentVariable(WebSitesKnownKeyNames.WebSiteNameKey))
{
ExecutingJobRunId = new WebJobRunIdentifierViewModel((WebJobTypes)Enum.Parse(typeof(WebJobTypes),
snapshot.WebJobType), snapshot.WebJobName, snapshot.WebJobRunId);
}
if (heartbeatIsValid.HasValue)
{
Status = snapshot.GetStatusWithHeartbeat(heartbeatIsValid.Value);
}
else
{
Status = snapshot.GetStatusWithoutHeartbeat();
}
switch (Status)
{
case FunctionInstanceStatus.Running:
WhenUtc = snapshot.StartTime.Value.UtcDateTime;
Duration = DateTimeOffset.UtcNow - snapshot.StartTime;
break;
case FunctionInstanceStatus.CompletedSuccess:
WhenUtc = snapshot.EndTime.Value.UtcDateTime;
Duration = snapshot.GetFinalDuration();
break;
case FunctionInstanceStatus.CompletedFailed:
WhenUtc = snapshot.EndTime.Value.UtcDateTime;
Duration = snapshot.GetFinalDuration();
ExceptionType = snapshot.ExceptionType;
ExceptionMessage = snapshot.ExceptionMessage;
break;
case FunctionInstanceStatus.NeverFinished:
WhenUtc = snapshot.StartTime.Value.UtcDateTime;
Duration = null;
break;
}
}
private FunctionSnapshot GetFunctionFromInstance(string id, out Guid parsed, out FunctionInstanceSnapshot snapshot)
{
if (!Guid.TryParse(id, out parsed))
{
snapshot = null;
return(null);
}
snapshot = _functionInstanceLookup.Lookup(parsed);
if (snapshot == null)
{
return(null);
}
return(GetFunction(snapshot.FunctionId));
}
private bool?HostInstanceHasHeartbeat(FunctionInstanceSnapshot snapshot)
{
if (snapshot.FunctionInstanceHeartbeatExpiry.HasValue)
{
var now = DateTime.UtcNow;
return(snapshot.FunctionInstanceHeartbeatExpiry.Value > now);
}
HeartbeatDescriptor heartbeat = snapshot.Heartbeat;
if (heartbeat == null)
{
return(null);
}
return(HostInstanceHasHeartbeat(heartbeat.SharedContainerName, heartbeat.SharedDirectoryName,
heartbeat.InstanceBlobName, heartbeat.ExpirationInSeconds));
}
// This method runs concurrently with other index processing.
// Ensure all logic here is idempotent.
public void ProcessFunctionCompleted(FunctionCompletedMessage message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
FunctionInstanceSnapshot snapshot = CreateSnapshot(message);
// The completed message includes the full parameter logs; delete the extra blob used for running status
// updates.
DeleteParameterLogBlob(snapshot.ParameterLogBlob);
snapshot.ParameterLogBlob = null;
_functionInstanceLogger.LogFunctionCompleted(snapshot);
Guid functionInstanceId = message.FunctionInstanceId;
DateTimeOffset endTime = message.EndTime;
string functionId = new FunctionIdentifier(message.SharedQueueName, message.Function.Id).ToString();
Guid? parentId = message.ParentId;
WebJobRunIdentifier webJobRunId = message.WebJobRunIdentifier;
DeleteFunctionStartedIndexEntriesIfNeeded(functionInstanceId, message.StartTime, endTime, functionId,
parentId, webJobRunId);
CreateOrUpdateIndexEntries(snapshot, endTime, webJobRunId);
// Increment is non-idempotent. If the process dies before deleting the message that triggered it, it can
// occur multiple times.
// If we wanted to make this operation idempotent, one option would be to store the list of function
// instance IDs that succeeded & failed, rather than just the counters, so duplicate operations could be
// detected.
// For now, we just do a non-idempotent increment last, which makes it very unlikely that the queue message
// would not subsequently get deleted.
if (message.Succeeded)
{
_statisticsWriter.IncrementSuccess(functionId);
}
else
{
_statisticsWriter.IncrementFailure(functionId);
}
}
public void FunctionInstanceSnapshot_BuildDisplayTitle_CompletelyRemovesLineBreaks(string argumentValue, string expectedDisplayTitle)
{
FunctionInstanceSnapshot message = new FunctionInstanceSnapshot
{
Arguments = new Dictionary <string, FunctionInstanceArgument>
{
{ "message", new FunctionInstanceArgument()
{
Value = argumentValue
} }
},
};
string displayTitle = message.DisplayTitle;
Assert.Equal(expectedDisplayTitle, displayTitle);
Assert.DoesNotContain("\r", displayTitle);
Assert.DoesNotContain("\n", displayTitle);
}
public static FunctionInstanceStatus GetStatusWithHeartbeat(this FunctionInstanceSnapshot snapshot, bool?heartbeatIsValid)
{
if (snapshot == null)
{
throw new ArgumentNullException("snapshot");
}
if (snapshot.EndTime.HasValue)
{
if (snapshot.Succeeded.Value)
{
return(FunctionInstanceStatus.CompletedSuccess);
}
else
{
return(FunctionInstanceStatus.CompletedFailed);
}
}
else if (heartbeatIsValid.HasValue)
{
if (heartbeatIsValid.Value)
{
return(FunctionInstanceStatus.Running);
}
else
{
return(FunctionInstanceStatus.NeverFinished);
}
}
else if (snapshot.StartTime.HasValue)
{
return(FunctionInstanceStatus.Running);
}
else
{
return(FunctionInstanceStatus.Queued);
}
}
// null if job hasn't finished yet.
public static TimeSpan?GetFinalDuration(this FunctionInstanceSnapshot snapshot)
{
if (snapshot == null)
{
throw new ArgumentNullException("snapshot");
}
DateTimeOffset?endTime = snapshot.EndTime;
if (!endTime.HasValue)
{
return(null);
}
DateTimeOffset?startTime = snapshot.StartTime;
if (!startTime.HasValue)
{
return(null);
}
return(endTime.Value - startTime.Value);
}
public static FunctionInstanceStatus GetStatusWithoutHeartbeat(this FunctionInstanceSnapshot snapshot)
{
return(GetStatusWithHeartbeat(snapshot, null));
}
private DateTimeOffset?GetFunctionCompletedTime(Guid functionInstanceId)
{
FunctionInstanceSnapshot primaryLog = _functionInstanceLookup.Lookup(functionInstanceId);
return(primaryLog != null ? primaryLog.EndTime : null);
}
// Get Live information from current watcher values.
internal static IDictionary <string, ParameterLog> GetParameterLogs(CloudStorageAccount account, FunctionInstanceSnapshot snapshot)
{
if (snapshot.ParameterLogs != null)
{
return(snapshot.ParameterLogs);
}
if (snapshot.ParameterLogBlob == null)
{
return(null);
}
CloudBlockBlob blob = snapshot.ParameterLogBlob.GetBlockBlob(account);
string contents;
try
{
contents = blob.DownloadText();
}
catch (StorageException exception)
{
// common case, no status information written.
if (exception.IsNotFound())
{
return(null);
}
else
{
throw;
}
}
try
{
return(JsonConvert.DeserializeObject <IDictionary <string, ParameterLog> >(contents,
JsonSerialization.Settings));
}
catch
{
// Not fatal.
// This could happen if the app wrote a corrupted log.
return(null);
}
}
private static ParameterModel[] CreateParameterModels(CloudStorageAccount account, FunctionInstanceSnapshot snapshot)
{
List <ParameterModel> models = new List <ParameterModel>();
IDictionary <string, FunctionInstanceArgument> parameters = snapshot.Arguments;
IDictionary <string, ParameterLog> parameterLogs = LogAnalysis.GetParameterLogs(account, snapshot);
foreach (KeyValuePair <string, FunctionInstanceArgument> parameter in parameters)
{
string name = parameter.Key;
FunctionInstanceArgument argument = parameter.Value;
ParameterLog log;
if (parameterLogs != null && parameterLogs.ContainsKey(name))
{
log = parameterLogs[name];
}
else
{
log = null;
}
LogAnalysis.AddParameterModels(name, argument, log, snapshot, models);
}
return(models.ToArray());
}
private static ParameterModel[] CreateParameterModels(CloudStorageAccount account, FunctionInstanceSnapshot snapshot)
{
List <ParameterModel> models = new List <ParameterModel>();
// It's important that we order any virtual parameters (e.g. Singleton parameters) to the end
// so they don't interfere with actual function parameter ordering
var parameters = snapshot.Arguments.OrderBy(p => p.Value.IsVirtualParameter);
IDictionary <string, ParameterLog> parameterLogs = LogAnalysis.GetParameterLogs(account, snapshot);
foreach (KeyValuePair <string, FunctionInstanceArgument> parameter in parameters)
{
string name = parameter.Key;
FunctionInstanceArgument argument = parameter.Value;
ParameterLog log;
if (parameterLogs != null && parameterLogs.ContainsKey(name))
{
log = parameterLogs[name];
}
else
{
log = null;
}
LogAnalysis.AddParameterModels(name, argument, log, snapshot, models);
}
return(models.ToArray());
}
internal TriggerReasonViewModel(FunctionInstanceSnapshot underlyingObject)
{
UnderlyingObject = underlyingObject;
}
public void CreateSnapshot_CreatesExpectedSnapshot()
{
FunctionDescriptor function = new FunctionDescriptor
{
Id = "FunctionId",
FullName = "FullName",
ShortName = "ShortName",
Parameters = new ParameterDescriptor[]
{
new ParameterDescriptor {
Name = "param1"
},
new ParameterDescriptor {
Name = "param2"
}
}
};
FunctionStartedMessage message = new FunctionStartedMessage
{
FunctionInstanceId = Guid.NewGuid(),
HostInstanceId = Guid.NewGuid(),
InstanceQueueName = "InstanceQueueName",
Reason = ExecutionReason.AutomaticTrigger,
ReasonDetails = "A trigger fired!",
Heartbeat = new HeartbeatDescriptor
{
InstanceBlobName = "InstanceBlobName",
SharedContainerName = "SharedContainerName",
SharedDirectoryName = "SharedDirectoryName",
ExpirationInSeconds = 5
},
SharedQueueName = "SharedQueueName",
Function = function,
Arguments = new Dictionary <string, string>
{
{ "param1", "foo" },
{ "param2", "bar" }
},
ParentId = Guid.NewGuid(),
StartTime = DateTime.Now,
OutputBlob = new LocalBlobDescriptor {
BlobName = "OutputBlobName", ContainerName = "OutputBlobContainerName"
},
ParameterLogBlob = new LocalBlobDescriptor {
BlobName = "ParameterLogBlobName", ContainerName = "ParameterLogBlobContainerName"
},
WebJobRunIdentifier = new WebJobRunIdentifier {
JobName = "JobName", JobType = WebJobTypes.Triggered, RunId = "RunId", WebSiteName = "WebSiteName"
}
};
FunctionInstanceSnapshot snapshot = FunctionIndexer.CreateSnapshot(message);
Assert.Equal(message.FunctionInstanceId, snapshot.Id);
Assert.Equal(message.HostInstanceId, snapshot.HostInstanceId);
Assert.Equal(message.InstanceQueueName, snapshot.InstanceQueueName);
Assert.Same(message.Heartbeat, snapshot.Heartbeat);
Assert.Equal("SharedQueueName_FunctionId", snapshot.FunctionId);
Assert.Equal(message.Function.FullName, snapshot.FunctionFullName);
Assert.Equal(message.Function.ShortName, snapshot.FunctionShortName);
Assert.Equal(2, snapshot.Arguments.Count);
Assert.Equal("foo", snapshot.Arguments["param1"].Value);
Assert.Equal("bar", snapshot.Arguments["param2"].Value);
Assert.Equal(message.ParentId, snapshot.ParentId);
Assert.Equal(message.ReasonDetails, snapshot.Reason);
Assert.Equal(message.StartTime, snapshot.QueueTime);
Assert.Equal(message.StartTime, snapshot.StartTime);
Assert.Equal(message.OutputBlob, snapshot.OutputBlob);
Assert.Same(message.ParameterLogBlob, snapshot.ParameterLogBlob);
Assert.Equal(message.WebJobRunIdentifier.WebSiteName, snapshot.WebSiteName);
Assert.Equal(message.WebJobRunIdentifier.JobType.ToString(), snapshot.WebJobType);
Assert.Equal(message.WebJobRunIdentifier.JobName, snapshot.WebJobName);
Assert.Equal(message.WebJobRunIdentifier.RunId, snapshot.WebJobRunId);
}
