public void VsTelemetrySession_ToVsTelemetryEvent() { // Arrange var actionTelemetryData = new VSActionsTelemetryEvent( projectIds: new[] { Guid.NewGuid().ToString() }, operationType: NuGetOperationType.Install, source: OperationSource.UI, startTime: DateTimeOffset.Now.AddSeconds(-2), status: NuGetOperationStatus.Failed, packageCount: 1, endTime: DateTimeOffset.Now, duration: 1.30); var operationId = Guid.NewGuid().ToString(); actionTelemetryData["OperationId"] = operationId; // Act var vsTelemetryEvent = VSTelemetrySession.ToVsTelemetryEvent(actionTelemetryData); // Assert Assert.True(vsTelemetryEvent.Name.StartsWith(VSTelemetrySession.VSEventNamePrefix, ignoreCase: true, culture: CultureInfo.InvariantCulture)); Assert.True(vsTelemetryEvent.Properties.Keys.All( p => p.StartsWith(VSTelemetrySession.VSPropertyNamePrefix, ignoreCase: true, culture: CultureInfo.InvariantCulture))); }
public void ActionsTelemetryService_EmitActionEvent_OperationNoOp(NuGetOperationType operationType) { // Arrange var telemetrySession = new Mock <ITelemetrySession>(); TelemetryEvent lastTelemetryEvent = null; telemetrySession .Setup(x => x.PostEvent(It.IsAny <TelemetryEvent>())) .Callback <TelemetryEvent>(x => lastTelemetryEvent = x); var operationId = Guid.NewGuid().ToString(); var actionTelemetryData = new VSActionsTelemetryEvent( operationId, projectIds: new[] { Guid.NewGuid().ToString() }, operationType: operationType, source: OperationSource.PMC, startTime: DateTimeOffset.Now.AddSeconds(-1), status: NuGetOperationStatus.NoOp, packageCount: 1, endTime: DateTimeOffset.Now, duration: .40); var service = new NuGetVSTelemetryService(telemetrySession.Object); // Act service.EmitTelemetryEvent(actionTelemetryData); // Assert VerifyTelemetryEventData(operationId, actionTelemetryData, lastTelemetryEvent); }
internal static void AddUiActionEngineTelemetryProperties( VSActionsTelemetryEvent actionTelemetryEvent, bool continueAfterPreview, bool acceptedLicense, UserAction userAction, int?selectedIndex, int?recommendedCount, bool?recommendPackages, (string modelVersion, string vsixVersion)?recommenderVersion,
private void VerifyTelemetryEventData(string operationId, VSActionsTelemetryEvent expected, TelemetryEvent actual) { Assert.NotNull(actual); Assert.Equal(ActionsTelemetryEvent.NugetActionEventName, actual.Name); Assert.Equal(10, actual.Count); Assert.Equal(expected.OperationType.ToString(), actual["OperationType"].ToString()); Assert.Equal(expected.Source.ToString(), actual["Source"].ToString()); TestTelemetryUtility.VerifyTelemetryEventData(operationId, expected, actual); }
private async Task PerformActionImplAsync( IServiceBroker serviceBroker, INuGetProjectManagerService projectManagerService, INuGetUI uiService, ResolveActionsAsync resolveActionsAsync, NuGetOperationType operationType, UserAction userAction, CancellationToken cancellationToken) { var status = NuGetOperationStatus.Succeeded; var startTime = DateTimeOffset.Now; var packageCount = 0; var continueAfterPreview = true; var acceptedLicense = true; List <string> removedPackages = null; var existingPackages = new HashSet <Tuple <string, string> >(); List <Tuple <string, string> > addedPackages = null; List <Tuple <string, string> > updatedPackagesOld = null; List <Tuple <string, string> > updatedPackagesNew = null; // Enable granular level telemetry events for nuget ui operation uiService.ProjectContext.OperationId = Guid.NewGuid(); Stopwatch packageEnumerationTime = new Stopwatch(); packageEnumerationTime.Start(); try { // collect the install state of the existing packages foreach (IProjectContextInfo project in uiService.Projects) { IEnumerable <IPackageReferenceContextInfo> installedPackages = await project.GetInstalledPackagesAsync( uiService.UIContext.ServiceBroker, cancellationToken); foreach (IPackageReferenceContextInfo package in installedPackages) { Tuple <string, string> packageInfo = new Tuple <string, string>( package.Identity.Id, (package.Identity.Version == null ? "" : package.Identity.Version.ToNormalizedString())); if (!existingPackages.Contains(packageInfo)) { existingPackages.Add(packageInfo); } } } } catch (Exception) { // don't teardown the process if we have a telemetry failure } packageEnumerationTime.Stop(); await _lockService.ExecuteNuGetOperationAsync(async() => { try { uiService.BeginOperation(); using (INuGetProjectUpgraderService projectUpgrader = await serviceBroker.GetProxyAsync <INuGetProjectUpgraderService>( NuGetServices.ProjectUpgraderService, cancellationToken)) { bool isAcceptedFormat = await CheckPackageManagementFormatAsync(projectUpgrader, uiService, cancellationToken); if (!isAcceptedFormat) { return; } } TelemetryServiceUtility.StartOrResumeTimer(); IReadOnlyList <ProjectAction> actions = await resolveActionsAsync(projectManagerService); IReadOnlyList <PreviewResult> results = await GetPreviewResultsAsync(projectManagerService, actions, cancellationToken); if (operationType == NuGetOperationType.Uninstall) { // removed packages don't have version info removedPackages = results.SelectMany(result => result.Deleted) .Select(package => package.Id) .Distinct() .ToList(); packageCount = removedPackages.Count; } else { // log rich info about added packages addedPackages = results.SelectMany(result => result.Added) .Select(package => new Tuple <string, string>(package.Id, (package.Version == null ? "" : package.Version.ToNormalizedString()))) .Distinct() .ToList(); var addCount = addedPackages.Count; //updated packages can have an old and a new id. updatedPackagesOld = results.SelectMany(result => result.Updated) .Select(package => new Tuple <string, string>(package.Old.Id, (package.Old.Version == null ? "" : package.Old.Version.ToNormalizedString()))) .Distinct() .ToList(); updatedPackagesNew = results.SelectMany(result => result.Updated) .Select(package => new Tuple <string, string>(package.New.Id, (package.New.Version == null ? "" : package.New.Version.ToNormalizedString()))) .Distinct() .ToList(); var updateCount = updatedPackagesNew.Count; // update packages count packageCount = addCount + updateCount; if (updateCount > 0) { // set operation type to update when there are packages being updated operationType = NuGetOperationType.Update; } } TelemetryServiceUtility.StopTimer(); // Show the preview window. if (uiService.DisplayPreviewWindow) { bool shouldContinue = uiService.PromptForPreviewAcceptance(results); if (!shouldContinue) { continueAfterPreview = false; return; } } TelemetryServiceUtility.StartOrResumeTimer(); // Show the license acceptance window. bool accepted = await CheckLicenseAcceptanceAsync(uiService, results, cancellationToken); TelemetryServiceUtility.StartOrResumeTimer(); if (!accepted) { acceptedLicense = false; return; } // Warn about the fact that the "dotnet" TFM is deprecated. if (uiService.DisplayDeprecatedFrameworkWindow) { bool shouldContinue = await ShouldContinueDueToDotnetDeprecationAsync(projectManagerService, uiService, cancellationToken); TelemetryServiceUtility.StartOrResumeTimer(); if (!shouldContinue) { return; } } if (!cancellationToken.IsCancellationRequested) { await projectManagerService.ExecuteActionsAsync( actions, cancellationToken); string[] projectIds = actions .Select(action => action.ProjectId) .Distinct() .ToArray(); uiService.UIContext.RaiseProjectActionsExecuted(projectIds); } else { status = NuGetOperationStatus.Cancelled; } } catch (System.Net.Http.HttpRequestException ex) { status = NuGetOperationStatus.Failed; if (ex.InnerException != null) { uiService.ShowError(ex.InnerException); } else { uiService.ShowError(ex); } } catch (Exception ex) { status = NuGetOperationStatus.Failed; uiService.ShowError(ex); } finally { TelemetryServiceUtility.StopTimer(); var duration = TelemetryServiceUtility.GetTimerElapsedTime(); uiService.ProjectContext.Log(MessageLevel.Info, string.Format(CultureInfo.CurrentCulture, Resources.Operation_TotalTime, duration)); uiService.EndOperation(); // don't show "Succeeded" if we actually cancelled... if ((!continueAfterPreview) || (!acceptedLicense)) { if (status == NuGetOperationStatus.Succeeded) { status = NuGetOperationStatus.Cancelled; } } var plc = new PackageLoadContext(isSolution: false, uiService.UIContext); IReadOnlyCollection <string> frameworks = await plc.GetSupportedFrameworksAsync(); string[] projectIds = (await ProjectUtility.GetSortedProjectIdsAsync( uiService.UIContext.ServiceBroker, uiService.Projects, cancellationToken)).ToArray(); var packageSourceMapping = PackageSourceMapping.GetPackageSourceMapping(uiService.Settings); bool isPackageSourceMappingEnabled = packageSourceMapping?.IsEnabled ?? false; var actionTelemetryEvent = new VSActionsTelemetryEvent( uiService.ProjectContext.OperationId.ToString(), projectIds, operationType, OperationSource.UI, startTime, status, packageCount, DateTimeOffset.Now, duration.TotalSeconds, isPackageSourceMappingEnabled: isPackageSourceMappingEnabled); var nuGetUI = uiService as NuGetUI; AddUiActionEngineTelemetryProperties( actionTelemetryEvent, continueAfterPreview, acceptedLicense, userAction, nuGetUI?.SelectedIndex, nuGetUI?.RecommendedCount, nuGetUI?.RecommendPackages, nuGetUI?.RecommenderVersion, nuGetUI?.TopLevelVulnerablePackagesCount ?? 0, nuGetUI?.TopLevelVulnerablePackagesMaxSeverities?.ToList() ?? new List <int>(), existingPackages, addedPackages, removedPackages, updatedPackagesOld, updatedPackagesNew, frameworks); actionTelemetryEvent["InstalledPackageEnumerationTimeInMilliseconds"] = packageEnumerationTime.ElapsedMilliseconds; TelemetryActivity.EmitTelemetryEvent(actionTelemetryEvent); } }, cancellationToken); }
private static void AddUiActionEngineTelemetryProperties( VSActionsTelemetryEvent actionTelemetryEvent, bool continueAfterPreview, bool acceptedLicense, UserAction userAction, HashSet <Tuple <string, string> > existingPackages, List <Tuple <string, string> > addedPackages, List <string> removedPackages, List <Tuple <string, string> > updatedPackagesOld, List <Tuple <string, string> > updatedPackagesNew) { TelemetryEvent ToTelemetryPackage(Tuple <string, string> package) { var subEvent = new TelemetryEvent(eventName: null); subEvent.AddPiiData("id", package.Item1?.ToLowerInvariant() ?? "(empty package id)"); subEvent["version"] = package.Item2; return(subEvent); } List <TelemetryEvent> ToTelemetryPackageList(List <Tuple <string, string> > packages) { var list = new List <TelemetryEvent>(packages.Count); list.AddRange(packages.Select(ToTelemetryPackage)); return(list); } // log possible cancel reasons if (!continueAfterPreview) { actionTelemetryEvent["CancelAfterPreview"] = "True"; } if (!acceptedLicense) { actionTelemetryEvent["AcceptedLicense"] = "False"; } // log the single top level package the user is installing or removing if (userAction != null) { // userAction.Version can be null for deleted packages. actionTelemetryEvent.ComplexData["SelectedPackage"] = ToTelemetryPackage(new Tuple <string, string>(userAction.PackageId, userAction.Version?.ToNormalizedString() ?? string.Empty)); } // log the installed package state if (existingPackages != null && existingPackages.Count > 0) { var packages = new List <TelemetryEvent>(); foreach (var package in existingPackages) { packages.Add(ToTelemetryPackage(package)); } actionTelemetryEvent.ComplexData["ExistingPackages"] = packages; } // other packages can be added, removed, or upgraded as part of bulk upgrade or as part of satisfying package dependencies, so log that also if (addedPackages != null && addedPackages.Count > 0) { var packages = new List <TelemetryEvent>(); foreach (var package in addedPackages) { packages.Add(ToTelemetryPackage(package)); } actionTelemetryEvent.ComplexData["AddedPackages"] = packages; } if (removedPackages != null && removedPackages.Count > 0) { var packages = new List <TelemetryPiiProperty>(); foreach (var package in removedPackages) { packages.Add(new TelemetryPiiProperty(package?.ToLowerInvariant() ?? "(empty package id)")); } actionTelemetryEvent.ComplexData["RemovedPackages"] = packages; } // two collections for updated packages: pre and post upgrade if (updatedPackagesNew != null && updatedPackagesNew.Count > 0) { actionTelemetryEvent.ComplexData["UpdatedPackagesNew"] = ToTelemetryPackageList(updatedPackagesNew); } if (updatedPackagesOld != null && updatedPackagesOld.Count > 0) { actionTelemetryEvent.ComplexData["UpdatedPackagesOld"] = ToTelemetryPackageList(updatedPackagesOld); } }
public void AddUiActionEngineTelemetryProperties_AddsVulnerabilityInfo_Succeeds() { // Arrange var telemetrySession = new Mock <ITelemetrySession>(); TelemetryEvent lastTelemetryEvent = null; telemetrySession .Setup(x => x.PostEvent(It.IsAny <TelemetryEvent>())) .Callback <TelemetryEvent>(x => lastTelemetryEvent = x); var operationId = Guid.NewGuid().ToString(); var actionTelemetryData = new VSActionsTelemetryEvent( operationId, projectIds: new[] { Guid.NewGuid().ToString() }, operationType: NuGetOperationType.Install, source: OperationSource.PMC, startTime: DateTimeOffset.Now.AddSeconds(-1), status: NuGetOperationStatus.NoOp, packageCount: 1, endTime: DateTimeOffset.Now, duration: .40, isPackageSourceMappingEnabled: false); UIActionEngine.AddUiActionEngineTelemetryProperties( actionTelemetryEvent: actionTelemetryData, continueAfterPreview: true, acceptedLicense: true, userAction: UserAction.CreateInstallAction("mypackageId", new NuGetVersion(1, 0, 0)), selectedIndex: 0, recommendedCount: 0, recommendPackages: false, recommenderVersion: null, topLevelVulnerablePackagesCount: 3, topLevelVulnerablePackagesMaxSeverities: new List <int> { 1, 1, 3 }, // each package has its own max severity existingPackages: null, addedPackages: null, removedPackages: null, updatedPackagesOld: null, updatedPackagesNew: null, targetFrameworks: null); // Act var service = new NuGetVSTelemetryService(telemetrySession.Object); service.EmitTelemetryEvent(actionTelemetryData); // Assert Assert.NotNull(lastTelemetryEvent); Assert.NotNull(lastTelemetryEvent.ComplexData["TopLevelVulnerablePackagesMaxSeverities"] as List <int>); var pkgSeverities = lastTelemetryEvent.ComplexData["TopLevelVulnerablePackagesMaxSeverities"] as List <int>; Assert.Equal(lastTelemetryEvent["TopLevelVulnerablePackagesCount"], pkgSeverities.Count()); Assert.Collection(pkgSeverities, item => Assert.Equal(1, item), item => Assert.Equal(1, item), item => Assert.Equal(3, item)); Assert.Equal(3, pkgSeverities.Count()); }
public void HyperlinkClicked_CorrelatesSearchSelectionAndAction_Succeeds() { // Arrange var telemetrySession = new Mock <ITelemetrySession>(); TelemetryEvent lastTelemetryEvent = null; _ = telemetrySession .Setup(x => x.PostEvent(It.IsAny <TelemetryEvent>())) .Callback <TelemetryEvent>(x => lastTelemetryEvent = x); var service = new NuGetVSTelemetryService(telemetrySession.Object); var testPackageId = "testPackage.id"; var testPackageVersion = new NuGetVersion(1, 0, 0); var evtHyperlink = new HyperlinkClickedTelemetryEvent( HyperlinkType.DeprecationAlternativeDetails, ContractsItemFilter.All, isSolutionView: false, testPackageId); var evtSearch = new SearchSelectionTelemetryEvent( parentId: It.IsAny <Guid>(), recommendedCount: It.IsAny <int>(), itemIndex: It.IsAny <int>(), packageId: testPackageId, packageVersion: testPackageVersion, isPackageVulnerable: It.IsAny <bool>(), isPackageDeprecated: true, hasDeprecationAlternativePackage: true); var evtActions = new VSActionsTelemetryEvent( operationId: It.IsAny <string>(), projectIds: new[] { Guid.NewGuid().ToString() }, operationType: NuGetOperationType.Install, source: OperationSource.PMC, startTime: DateTimeOffset.Now.AddSeconds(-1), status: NuGetOperationStatus.NoOp, packageCount: 1, endTime: DateTimeOffset.Now, duration: .40, isPackageSourceMappingEnabled: false); // Simulate UIActionEngine.AddUiActionEngineTelemetryProperties() var pkgAdded = new TelemetryEvent(eventName: null); pkgAdded.AddPiiData("id", VSTelemetryServiceUtility.NormalizePackageId(testPackageId)); pkgAdded.AddPiiData("version", testPackageVersion.ToNormalizedString()); var packages = new List <TelemetryEvent> { pkgAdded }; evtActions.ComplexData["AddedPackages"] = packages; // Act service.EmitTelemetryEvent(evtHyperlink); var hyperlinkEmitted = lastTelemetryEvent; service.EmitTelemetryEvent(evtSearch); var searchEmitted = lastTelemetryEvent; service.EmitTelemetryEvent(evtActions); var actionEmitted = lastTelemetryEvent; // Assert var packageIdHyperlink = hyperlinkEmitted.GetPiiData().First(x => x.Key == HyperlinkClickedTelemetryEvent.AlternativePackageIdPropertyName).Value; var packageIdSearch = searchEmitted.GetPiiData().First(x => x.Key == "PackageId").Value; var packageIdsAction = (IEnumerable <TelemetryEvent>)actionEmitted.ComplexData["AddedPackages"]; var packageIds = packageIdsAction.Select(x => x.GetPiiData().First(x => x.Key == "id").Value); Assert.Equal(packageIdHyperlink, packageIdSearch); Assert.Contains(packageIdHyperlink, packageIds); }