public void VsTelemetrySession_ToVsTelemetryEvent()
{
// Arrange
var actionTelemetryData = new VSActionsTelemetryEvent(
projectIds: new[] { Guid.NewGuid().ToString() },
operationType: NuGetOperationType.Install,
source: OperationSource.UI,
startTime: DateTimeOffset.Now.AddSeconds(-2),
status: NuGetOperationStatus.Failed,
packageCount: 1,
endTime: DateTimeOffset.Now,
duration: 1.30);
var operationId = Guid.NewGuid().ToString();
actionTelemetryData["OperationId"] = operationId;
// Act
var vsTelemetryEvent = VSTelemetrySession.ToVsTelemetryEvent(actionTelemetryData);
// Assert
Assert.True(vsTelemetryEvent.Name.StartsWith(VSTelemetrySession.VSEventNamePrefix, ignoreCase: true, culture: CultureInfo.InvariantCulture));
Assert.True(vsTelemetryEvent.Properties.Keys.All(
p => p.StartsWith(VSTelemetrySession.VSPropertyNamePrefix, ignoreCase: true, culture: CultureInfo.InvariantCulture)));
}
public void ActionsTelemetryService_EmitActionEvent_OperationNoOp(NuGetOperationType operationType)
{
// Arrange
var telemetrySession = new Mock <ITelemetrySession>();
TelemetryEvent lastTelemetryEvent = null;
telemetrySession
.Setup(x => x.PostEvent(It.IsAny <TelemetryEvent>()))
.Callback <TelemetryEvent>(x => lastTelemetryEvent = x);
var operationId = Guid.NewGuid().ToString();
var actionTelemetryData = new VSActionsTelemetryEvent(
operationId,
projectIds: new[] { Guid.NewGuid().ToString() },
operationType: operationType,
source: OperationSource.PMC,
startTime: DateTimeOffset.Now.AddSeconds(-1),
status: NuGetOperationStatus.NoOp,
packageCount: 1,
endTime: DateTimeOffset.Now,
duration: .40);
var service = new NuGetVSTelemetryService(telemetrySession.Object);
// Act
service.EmitTelemetryEvent(actionTelemetryData);
// Assert
VerifyTelemetryEventData(operationId, actionTelemetryData, lastTelemetryEvent);
}
internal static void AddUiActionEngineTelemetryProperties(
VSActionsTelemetryEvent actionTelemetryEvent,
bool continueAfterPreview,
bool acceptedLicense,
UserAction userAction,
int?selectedIndex,
int?recommendedCount,
bool?recommendPackages,
(string modelVersion, string vsixVersion)?recommenderVersion,
private void VerifyTelemetryEventData(string operationId, VSActionsTelemetryEvent expected, TelemetryEvent actual)
{
Assert.NotNull(actual);
Assert.Equal(ActionsTelemetryEvent.NugetActionEventName, actual.Name);
Assert.Equal(10, actual.Count);
Assert.Equal(expected.OperationType.ToString(), actual["OperationType"].ToString());
Assert.Equal(expected.Source.ToString(), actual["Source"].ToString());
TestTelemetryUtility.VerifyTelemetryEventData(operationId, expected, actual);
}
private async Task PerformActionImplAsync(
IServiceBroker serviceBroker,
INuGetProjectManagerService projectManagerService,
INuGetUI uiService,
ResolveActionsAsync resolveActionsAsync,
NuGetOperationType operationType,
UserAction userAction,
CancellationToken cancellationToken)
{
var status = NuGetOperationStatus.Succeeded;
var startTime = DateTimeOffset.Now;
var packageCount = 0;
var continueAfterPreview = true;
var acceptedLicense = true;
List <string> removedPackages = null;
var existingPackages = new HashSet <Tuple <string, string> >();
List <Tuple <string, string> > addedPackages = null;
List <Tuple <string, string> > updatedPackagesOld = null;
List <Tuple <string, string> > updatedPackagesNew = null;
// Enable granular level telemetry events for nuget ui operation
uiService.ProjectContext.OperationId = Guid.NewGuid();
Stopwatch packageEnumerationTime = new Stopwatch();
packageEnumerationTime.Start();
try
{
// collect the install state of the existing packages
foreach (IProjectContextInfo project in uiService.Projects)
{
IEnumerable <IPackageReferenceContextInfo> installedPackages = await project.GetInstalledPackagesAsync(
uiService.UIContext.ServiceBroker,
cancellationToken);
foreach (IPackageReferenceContextInfo package in installedPackages)
{
Tuple <string, string> packageInfo = new Tuple <string, string>(
package.Identity.Id,
(package.Identity.Version == null ? "" : package.Identity.Version.ToNormalizedString()));
if (!existingPackages.Contains(packageInfo))
{
existingPackages.Add(packageInfo);
}
}
}
}
catch (Exception)
{
// don't teardown the process if we have a telemetry failure
}
packageEnumerationTime.Stop();
await _lockService.ExecuteNuGetOperationAsync(async() =>
{
try
{
uiService.BeginOperation();
using (INuGetProjectUpgraderService projectUpgrader = await serviceBroker.GetProxyAsync <INuGetProjectUpgraderService>(
NuGetServices.ProjectUpgraderService,
cancellationToken))
{
bool isAcceptedFormat = await CheckPackageManagementFormatAsync(projectUpgrader, uiService, cancellationToken);
if (!isAcceptedFormat)
{
return;
}
}
TelemetryServiceUtility.StartOrResumeTimer();
IReadOnlyList <ProjectAction> actions = await resolveActionsAsync(projectManagerService);
IReadOnlyList <PreviewResult> results = await GetPreviewResultsAsync(projectManagerService, actions, cancellationToken);
if (operationType == NuGetOperationType.Uninstall)
{
// removed packages don't have version info
removedPackages = results.SelectMany(result => result.Deleted)
.Select(package => package.Id)
.Distinct()
.ToList();
packageCount = removedPackages.Count;
}
else
{
// log rich info about added packages
addedPackages = results.SelectMany(result => result.Added)
.Select(package => new Tuple <string, string>(package.Id, (package.Version == null ? "" : package.Version.ToNormalizedString())))
.Distinct()
.ToList();
var addCount = addedPackages.Count;
//updated packages can have an old and a new id.
updatedPackagesOld = results.SelectMany(result => result.Updated)
.Select(package => new Tuple <string, string>(package.Old.Id, (package.Old.Version == null ? "" : package.Old.Version.ToNormalizedString())))
.Distinct()
.ToList();
updatedPackagesNew = results.SelectMany(result => result.Updated)
.Select(package => new Tuple <string, string>(package.New.Id, (package.New.Version == null ? "" : package.New.Version.ToNormalizedString())))
.Distinct()
.ToList();
var updateCount = updatedPackagesNew.Count;
// update packages count
packageCount = addCount + updateCount;
if (updateCount > 0)
{
// set operation type to update when there are packages being updated
operationType = NuGetOperationType.Update;
}
}
TelemetryServiceUtility.StopTimer();
// Show the preview window.
if (uiService.DisplayPreviewWindow)
{
bool shouldContinue = uiService.PromptForPreviewAcceptance(results);
if (!shouldContinue)
{
continueAfterPreview = false;
return;
}
}
TelemetryServiceUtility.StartOrResumeTimer();
// Show the license acceptance window.
bool accepted = await CheckLicenseAcceptanceAsync(uiService, results, cancellationToken);
TelemetryServiceUtility.StartOrResumeTimer();
if (!accepted)
{
acceptedLicense = false;
return;
}
// Warn about the fact that the "dotnet" TFM is deprecated.
if (uiService.DisplayDeprecatedFrameworkWindow)
{
bool shouldContinue = await ShouldContinueDueToDotnetDeprecationAsync(projectManagerService, uiService, cancellationToken);
TelemetryServiceUtility.StartOrResumeTimer();
if (!shouldContinue)
{
return;
}
}
if (!cancellationToken.IsCancellationRequested)
{
await projectManagerService.ExecuteActionsAsync(
actions,
cancellationToken);
string[] projectIds = actions
.Select(action => action.ProjectId)
.Distinct()
.ToArray();
uiService.UIContext.RaiseProjectActionsExecuted(projectIds);
}
else
{
status = NuGetOperationStatus.Cancelled;
}
}
catch (System.Net.Http.HttpRequestException ex)
{
status = NuGetOperationStatus.Failed;
if (ex.InnerException != null)
{
uiService.ShowError(ex.InnerException);
}
else
{
uiService.ShowError(ex);
}
}
catch (Exception ex)
{
status = NuGetOperationStatus.Failed;
uiService.ShowError(ex);
}
finally
{
TelemetryServiceUtility.StopTimer();
var duration = TelemetryServiceUtility.GetTimerElapsedTime();
uiService.ProjectContext.Log(MessageLevel.Info,
string.Format(CultureInfo.CurrentCulture, Resources.Operation_TotalTime, duration));
uiService.EndOperation();
// don't show "Succeeded" if we actually cancelled...
if ((!continueAfterPreview) || (!acceptedLicense))
{
if (status == NuGetOperationStatus.Succeeded)
{
status = NuGetOperationStatus.Cancelled;
}
}
var plc = new PackageLoadContext(isSolution: false, uiService.UIContext);
IReadOnlyCollection <string> frameworks = await plc.GetSupportedFrameworksAsync();
string[] projectIds = (await ProjectUtility.GetSortedProjectIdsAsync(
uiService.UIContext.ServiceBroker,
uiService.Projects,
cancellationToken)).ToArray();
var packageSourceMapping = PackageSourceMapping.GetPackageSourceMapping(uiService.Settings);
bool isPackageSourceMappingEnabled = packageSourceMapping?.IsEnabled ?? false;
var actionTelemetryEvent = new VSActionsTelemetryEvent(
uiService.ProjectContext.OperationId.ToString(),
projectIds,
operationType,
OperationSource.UI,
startTime,
status,
packageCount,
DateTimeOffset.Now,
duration.TotalSeconds,
isPackageSourceMappingEnabled: isPackageSourceMappingEnabled);
var nuGetUI = uiService as NuGetUI;
AddUiActionEngineTelemetryProperties(
actionTelemetryEvent,
continueAfterPreview,
acceptedLicense,
userAction,
nuGetUI?.SelectedIndex,
nuGetUI?.RecommendedCount,
nuGetUI?.RecommendPackages,
nuGetUI?.RecommenderVersion,
nuGetUI?.TopLevelVulnerablePackagesCount ?? 0,
nuGetUI?.TopLevelVulnerablePackagesMaxSeverities?.ToList() ?? new List <int>(),
existingPackages,
addedPackages,
removedPackages,
updatedPackagesOld,
updatedPackagesNew,
frameworks);
actionTelemetryEvent["InstalledPackageEnumerationTimeInMilliseconds"] = packageEnumerationTime.ElapsedMilliseconds;
TelemetryActivity.EmitTelemetryEvent(actionTelemetryEvent);
}
}, cancellationToken);
}
private static void AddUiActionEngineTelemetryProperties(
VSActionsTelemetryEvent actionTelemetryEvent,
bool continueAfterPreview,
bool acceptedLicense,
UserAction userAction,
HashSet <Tuple <string, string> > existingPackages,
List <Tuple <string, string> > addedPackages,
List <string> removedPackages,
List <Tuple <string, string> > updatedPackagesOld,
List <Tuple <string, string> > updatedPackagesNew)
{
TelemetryEvent ToTelemetryPackage(Tuple <string, string> package)
{
var subEvent = new TelemetryEvent(eventName: null);
subEvent.AddPiiData("id", package.Item1?.ToLowerInvariant() ?? "(empty package id)");
subEvent["version"] = package.Item2;
return(subEvent);
}
List <TelemetryEvent> ToTelemetryPackageList(List <Tuple <string, string> > packages)
{
var list = new List <TelemetryEvent>(packages.Count);
list.AddRange(packages.Select(ToTelemetryPackage));
return(list);
}
// log possible cancel reasons
if (!continueAfterPreview)
{
actionTelemetryEvent["CancelAfterPreview"] = "True";
}
if (!acceptedLicense)
{
actionTelemetryEvent["AcceptedLicense"] = "False";
}
// log the single top level package the user is installing or removing
if (userAction != null)
{
// userAction.Version can be null for deleted packages.
actionTelemetryEvent.ComplexData["SelectedPackage"] = ToTelemetryPackage(new Tuple <string, string>(userAction.PackageId, userAction.Version?.ToNormalizedString() ?? string.Empty));
}
// log the installed package state
if (existingPackages != null && existingPackages.Count > 0)
{
var packages = new List <TelemetryEvent>();
foreach (var package in existingPackages)
{
packages.Add(ToTelemetryPackage(package));
}
actionTelemetryEvent.ComplexData["ExistingPackages"] = packages;
}
// other packages can be added, removed, or upgraded as part of bulk upgrade or as part of satisfying package dependencies, so log that also
if (addedPackages != null && addedPackages.Count > 0)
{
var packages = new List <TelemetryEvent>();
foreach (var package in addedPackages)
{
packages.Add(ToTelemetryPackage(package));
}
actionTelemetryEvent.ComplexData["AddedPackages"] = packages;
}
if (removedPackages != null && removedPackages.Count > 0)
{
var packages = new List <TelemetryPiiProperty>();
foreach (var package in removedPackages)
{
packages.Add(new TelemetryPiiProperty(package?.ToLowerInvariant() ?? "(empty package id)"));
}
actionTelemetryEvent.ComplexData["RemovedPackages"] = packages;
}
// two collections for updated packages: pre and post upgrade
if (updatedPackagesNew != null && updatedPackagesNew.Count > 0)
{
actionTelemetryEvent.ComplexData["UpdatedPackagesNew"] = ToTelemetryPackageList(updatedPackagesNew);
}
if (updatedPackagesOld != null && updatedPackagesOld.Count > 0)
{
actionTelemetryEvent.ComplexData["UpdatedPackagesOld"] = ToTelemetryPackageList(updatedPackagesOld);
}
}
public void AddUiActionEngineTelemetryProperties_AddsVulnerabilityInfo_Succeeds()
{
// Arrange
var telemetrySession = new Mock <ITelemetrySession>();
TelemetryEvent lastTelemetryEvent = null;
telemetrySession
.Setup(x => x.PostEvent(It.IsAny <TelemetryEvent>()))
.Callback <TelemetryEvent>(x => lastTelemetryEvent = x);
var operationId = Guid.NewGuid().ToString();
var actionTelemetryData = new VSActionsTelemetryEvent(
operationId,
projectIds: new[] { Guid.NewGuid().ToString() },
operationType: NuGetOperationType.Install,
source: OperationSource.PMC,
startTime: DateTimeOffset.Now.AddSeconds(-1),
status: NuGetOperationStatus.NoOp,
packageCount: 1,
endTime: DateTimeOffset.Now,
duration: .40,
isPackageSourceMappingEnabled: false);
UIActionEngine.AddUiActionEngineTelemetryProperties(
actionTelemetryEvent: actionTelemetryData,
continueAfterPreview: true,
acceptedLicense: true,
userAction: UserAction.CreateInstallAction("mypackageId", new NuGetVersion(1, 0, 0)),
selectedIndex: 0,
recommendedCount: 0,
recommendPackages: false,
recommenderVersion: null,
topLevelVulnerablePackagesCount: 3,
topLevelVulnerablePackagesMaxSeverities: new List <int> {
1, 1, 3
}, // each package has its own max severity
existingPackages: null,
addedPackages: null,
removedPackages: null,
updatedPackagesOld: null,
updatedPackagesNew: null,
targetFrameworks: null);
// Act
var service = new NuGetVSTelemetryService(telemetrySession.Object);
service.EmitTelemetryEvent(actionTelemetryData);
// Assert
Assert.NotNull(lastTelemetryEvent);
Assert.NotNull(lastTelemetryEvent.ComplexData["TopLevelVulnerablePackagesMaxSeverities"] as List <int>);
var pkgSeverities = lastTelemetryEvent.ComplexData["TopLevelVulnerablePackagesMaxSeverities"] as List <int>;
Assert.Equal(lastTelemetryEvent["TopLevelVulnerablePackagesCount"], pkgSeverities.Count());
Assert.Collection(pkgSeverities,
item => Assert.Equal(1, item),
item => Assert.Equal(1, item),
item => Assert.Equal(3, item));
Assert.Equal(3, pkgSeverities.Count());
}
public void HyperlinkClicked_CorrelatesSearchSelectionAndAction_Succeeds()
{
// Arrange
var telemetrySession = new Mock <ITelemetrySession>();
TelemetryEvent lastTelemetryEvent = null;
_ = telemetrySession
.Setup(x => x.PostEvent(It.IsAny <TelemetryEvent>()))
.Callback <TelemetryEvent>(x => lastTelemetryEvent = x);
var service = new NuGetVSTelemetryService(telemetrySession.Object);
var testPackageId = "testPackage.id";
var testPackageVersion = new NuGetVersion(1, 0, 0);
var evtHyperlink = new HyperlinkClickedTelemetryEvent(
HyperlinkType.DeprecationAlternativeDetails,
ContractsItemFilter.All,
isSolutionView: false,
testPackageId);
var evtSearch = new SearchSelectionTelemetryEvent(
parentId: It.IsAny <Guid>(),
recommendedCount: It.IsAny <int>(),
itemIndex: It.IsAny <int>(),
packageId: testPackageId,
packageVersion: testPackageVersion,
isPackageVulnerable: It.IsAny <bool>(),
isPackageDeprecated: true,
hasDeprecationAlternativePackage: true);
var evtActions = new VSActionsTelemetryEvent(
operationId: It.IsAny <string>(),
projectIds: new[] { Guid.NewGuid().ToString() },
operationType: NuGetOperationType.Install,
source: OperationSource.PMC,
startTime: DateTimeOffset.Now.AddSeconds(-1),
status: NuGetOperationStatus.NoOp,
packageCount: 1,
endTime: DateTimeOffset.Now,
duration: .40,
isPackageSourceMappingEnabled: false);
// Simulate UIActionEngine.AddUiActionEngineTelemetryProperties()
var pkgAdded = new TelemetryEvent(eventName: null);
pkgAdded.AddPiiData("id", VSTelemetryServiceUtility.NormalizePackageId(testPackageId));
pkgAdded.AddPiiData("version", testPackageVersion.ToNormalizedString());
var packages = new List <TelemetryEvent>
{
pkgAdded
};
evtActions.ComplexData["AddedPackages"] = packages;
// Act
service.EmitTelemetryEvent(evtHyperlink);
var hyperlinkEmitted = lastTelemetryEvent;
service.EmitTelemetryEvent(evtSearch);
var searchEmitted = lastTelemetryEvent;
service.EmitTelemetryEvent(evtActions);
var actionEmitted = lastTelemetryEvent;
// Assert
var packageIdHyperlink = hyperlinkEmitted.GetPiiData().First(x => x.Key == HyperlinkClickedTelemetryEvent.AlternativePackageIdPropertyName).Value;
var packageIdSearch = searchEmitted.GetPiiData().First(x => x.Key == "PackageId").Value;
var packageIdsAction = (IEnumerable <TelemetryEvent>)actionEmitted.ComplexData["AddedPackages"];
var packageIds = packageIdsAction.Select(x => x.GetPiiData().First(x => x.Key == "id").Value);
Assert.Equal(packageIdHyperlink, packageIdSearch);
Assert.Contains(packageIdHyperlink, packageIds);
}
