private void DoIntegrityCheckBetweenCatalogAndLucene() { DateTime startTime = Convert.ToDateTime(File.ReadAllText(CursorFileFullPath)); DateTime endTime = GetLastCommitTimeStampForLucene(); HashSet <PackageEntry> catalogPackages = V3Utility.GetCatalogPackages(CatalogRootUrl, CatalogStorageAccount, startTime, endTime); List <PackageEntry> missingPackagesFromLucene = GetMissingPackagesFromLuceneAndRegistrationBlob(catalogPackages); if (missingPackagesFromLucene != null & missingPackagesFromLucene.Count > 0) { string missingPackagesString = string.Join(",", missingPackagesFromLucene.Select(e => e.ToString()).ToArray()); Console.WriteLine("Missing Packages : {0}", missingPackagesString); new SendAlertMailTask { AlertSubject = string.Format("Packages missing in V3 Lucene Index/RegistrationBlob"), Details = string.Format("List of packages that are found in catalog and missing either in Lucene Index/Registration Blob: {0}", missingPackagesString), AlertName = "Packages missing in V3 Lucene Index", Component = "V3 SearchService/RegistrationBlob", Level = "Error" }.ExecuteCommand(); } else { //Update cursor only if validation succeeds. File.WriteAllText(CursorFileFullPath, endTime.ToString()); } }
private void CheckLagBetweenCatalogAndLucene() { DateTime lastCommitToIndex = GetLastCommitTimeStampForLucene(); DateTime lastCommitToCatalog = V3Utility.GetValueFromCatalogIndex(CatalogRootUrl, "commitTimeStamp"); double actualLag = lastCommitToCatalog.Subtract(lastCommitToIndex).TotalMinutes; //Allowed lag should be taken from config and it should be in seconds. But right now, the lag noticed is around 10 minutes. So setting it to 20 //Until we fix the issue https://github.com/NuGet/NuGetGallery/issues/2479 //double allowedLag = thresholdValues.V3SearchIndexCommitTimeStampLagInMinutes; double allowedLag = 20; Console.WriteLine("Lag between catalog and Lucene : {0}", actualLag); if (actualLag > allowedLag) { new SendAlertMailTask { AlertSubject = string.Format("V3 search lagging behind catalog by {0} minutes", actualLag), Details = string.Format("The commit timestamp for V3 Search Lucene Index is {0} and the commit timestamp for Catalog is {1}", lastCommitToIndex.ToString(), lastCommitToCatalog.ToString()), AlertName = "V3 Search Luence Index Lagging behind V2 DB.", Component = "V3 SearchService", Level = "Error" }.ExecuteCommand(); } }
private HashSet <PackageEntry> GetCatalogPackages() { return(V3Utility.GetCatalogPackages(CatalogRootUrl, CatalogStorageAccount)); }
private DateTime GetLastCreatedCursorFromCatalog() { return(V3Utility.GetValueFromCatalogIndex(CatalogRootUrl, "nuget:lastCreated")); }
private DateTime GetCommitTimeStampFromCatalog() { return(V3Utility.GetValueFromCatalogIndex(CatalogRootUrl, "commitTimeStamp")); }