public static JToken QuerySearch(NuGetSearcherManager searcherManager, string q, bool countOnly, string projectType, string supportedFramework, bool includePrerelease, string sortBy, int skip, int take, bool ignoreFilter)
        {
            var searcher = searcherManager.Get();
            try
            {
                Filter filter = ignoreFilter ? null : searcherManager.GetFilter(searcher, includePrerelease, null);

                Query query = LuceneQueryCreator.Parse(q, false);

                if (countOnly)
                {
                    return DocumentCountImpl(searcher, query, filter);
                }
                else
                {
                    IDictionary<string, int> rankings = searcherManager.GetRankings(projectType);

                    return ListDocumentsImpl(searcher, query, rankings, filter, sortBy, skip, take, searcherManager);
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
Example #2
0
        private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient)
        {
            var searcher = searcherManager.Get();

            try
            {
                // Track number of documents in index
                searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs());

                // Track time between Lucene commit and reopen
                string temp;
                if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp))
                {
                    var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal);

                    searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag,
                                                      (searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds,
                                                      new Dictionary <string, string>()
                    {
                        { SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") },
                        { SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") }
                    });
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
 public static async Task RankingsAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
 {
     await responseWriter.WriteResponseAsync(
         context,
         HttpStatusCode.OK,
         jsonWriter => ServiceInfoImpl.Rankings(jsonWriter, searcherManager));
 }
        public static JToken QuerySearch(NuGetSearcherManager searcherManager, string scheme, string q, bool countOnly, string projectType, string supportedFramework, bool includePrerelease, int skip, int take, bool includeExplanation, bool applyFilter)
        {
            var searcher = searcherManager.Get();
            try
            {
                Query query = MakeQuery(q, searcherManager);
                TopDocs topDocs;

                if (applyFilter)
                {
                    Filter filter = searcherManager.GetFilter(searcher, includePrerelease, supportedFramework);

                    //TODO: uncomment these lines when we have an index that contains the appropriate @type field in every document
                    //Filter typeFilter = new CachingWrapperFilter(new TypeFilter("http://schema.nuget.org/schema#NuGetClassicPackage"));
                    //filter = new ChainedFilter(new Filter[] { filter, typeFilter }, ChainedFilter.Logic.AND);

                    topDocs = searcher.Search(query, filter, skip + take);
                }
                else
                {
                    topDocs = searcher.Search(query, skip + take);
                }

                return MakeResult(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query);
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
 public static async Task Stats(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
 {
     await responseWriter.WriteResponseAsync(
         context,
         HttpStatusCode.OK,
         jsonWriter => ServiceInfoImpl.Stats(jsonWriter, searcherManager));
 }
        public static void Search(JsonWriter jsonWriter,
            NuGetSearcherManager searcherManager,
            string scheme,
            string q,
            bool includePrerelease,
            int skip,
            int take,
            string feed,
            bool includeExplanation)
        {
            var searcher = searcherManager.Get();

            try
            {
                Query query = MakeSearchQuery(q, searcher);

                Filter filter = null;

                if (searcher.TryGetFilter(false, includePrerelease, feed, out filter))
                {
                    // Filter before running the query (make the search set smaller)
                    query = new FilteredQuery(query, filter);
                }

                TopDocs topDocs = searcher.Search(query, skip + take);

                ResponseFormatter.WriteSearchResult(jsonWriter, searcher, scheme, topDocs, skip, take, includePrerelease, includeExplanation, query);
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
 public static async Task RankingsAsync(IOwinContext context, NuGetSearcherManager searcherManager)
 {
     await ResponseHelpers.WriteResponseAsync(
         context,
         HttpStatusCode.OK,
         jsonWriter => ServiceInfoImpl.Rankings(jsonWriter, searcherManager));
 }
        public NuGetSearcherManager CreateSearcherManager()
        {
            NuGetSearcherManager searcherManager;

            string luceneDirectory = System.Configuration.ConfigurationManager.AppSettings.Get("Local.Lucene.Directory");

            if (!string.IsNullOrEmpty(luceneDirectory))
            {
                string dataDirectory = System.Configuration.ConfigurationManager.AppSettings.Get("Local.Data.Directory");
                searcherManager = NuGetSearcherManager.CreateLocal(luceneDirectory, dataDirectory);
            }
            else
            {
                string storagePrimary       = System.Configuration.ConfigurationManager.AppSettings.Get("Storage.Primary");
                string searchIndexContainer = System.Configuration.ConfigurationManager.AppSettings.Get("Search.IndexContainer");
                string searchDataContainer  = System.Configuration.ConfigurationManager.AppSettings.Get("Search.DataContainer");

                searcherManager = NuGetSearcherManager.CreateAzure(storagePrimary, searchIndexContainer, searchDataContainer);
            }

            string registrationBaseAddress = System.Configuration.ConfigurationManager.AppSettings.Get("Search.RegistrationBaseAddress");

            searcherManager.RegistrationBaseAddress["http"]  = MakeRegistrationBaseAddress("http", registrationBaseAddress);
            searcherManager.RegistrationBaseAddress["https"] = MakeRegistrationBaseAddress("https", registrationBaseAddress);
            return(searcherManager);
        }
        private static NuGetSearcherManager InitNuGetSearcherManager(string indexName)
        {
            var searcherManager = new NuGetSearcherManager(indexName, new Mock<ILogger>().Object, directory: null, loader: null, azureDirectorySynchronizer: null);

            searcherManager.RegistrationBaseAddress[Constants.SchemeName] = new Uri(Constants.BaseUri);

            return searcherManager;
        }
        public static JToken Query(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            Trace.TraceInformation("Search: {0}", context.Request.QueryString);

            string q = context.Request.Query["q"] ?? string.Empty;

            string projectType = context.Request.Query["projectType"] ?? string.Empty;

            string sortBy = context.Request.Query["sortBy"] ?? string.Empty;

            bool includePrerelease;
            if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
            {
                includePrerelease = false;
            }

            bool countOnly;
            if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
            {
                countOnly = false;
            }

            string feed = context.Request.Query["feed"] ?? "none";

            int skip;
            if (!int.TryParse(context.Request.Query["skip"], out skip))
            {
                skip = 0;
            }

            int take;
            if (!int.TryParse(context.Request.Query["take"], out take))
            {
                take = 20;
            }

            bool ignoreFilter;
            if (!bool.TryParse(context.Request.Query["ignoreFilter"], out ignoreFilter))
            {
                ignoreFilter = false;
            }

            //string fxName = context.Request.Query["supportedFramework"];
            //FrameworkName supportedFramework = null;
            //if (!String.IsNullOrEmpty(fxName))
            //{
            //    supportedFramework = VersionUtility.ParseFrameworkName(fxName);
            //}
            //if (supportedFramework == null || !SearcherManager.GetFrameworks().Contains(supportedFramework))
            //{
            //    supportedFramework = FrameworksList.AnyFramework;
            //}

            string supportedFramework = string.Empty;

            return QuerySearch(searcherManager, q, countOnly, projectType, supportedFramework, includePrerelease, sortBy, skip, take, ignoreFilter);
        }
        public static async Task FindAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
        {
            var id = context.Request.Query["id"] ?? string.Empty;
            var scheme = context.Request.Uri.Scheme;

            await responseWriter.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => ServiceImpl.Find(jsonWriter, searcherManager, id, scheme));
        }
        public static async Task FindAsync(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            var id     = context.Request.Query["id"] ?? string.Empty;
            var scheme = context.Request.Uri.Scheme;

            await ResponseHelpers.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => ServiceImpl.Find(jsonWriter, searcherManager, id, scheme));
        }
 public static void Stats(JsonWriter jsonWriter, NuGetSearcherManager searcherManager)
 {
     var searcher = searcherManager.Get();
     try
     {
         ResponseFormatter.WriteStatsResult(jsonWriter, searcher);
     }
     finally
     {
         searcherManager.Release(searcher);
     }
 }
        public static void Search(JsonWriter jsonWriter, NuGetSearcherManager searcherManager, string q, bool countOnly, bool includePrerelease, string sortBy, int skip, int take, string feed, bool ignoreFilter, bool luceneQuery)
        {
            if (jsonWriter == null)
            {
                throw new ArgumentNullException(nameof(jsonWriter));
            }

            if (searcherManager == null)
            {
                throw new ArgumentNullException(nameof(searcherManager));
            }

            var searcher = searcherManager.Get();
            try
            {
                // The old V2 search service would treat "id:" queries (~match) in the same way as it did "packageid:" (==match).
                // If "id:" is in the query, replace it.
                if (luceneQuery && !string.IsNullOrEmpty(q) && q.StartsWith("id:", StringComparison.OrdinalIgnoreCase))
                {
                    q = "packageid:" + q.Substring(3);
                }

                // Build the query
                Query query = NuGetQuery.MakeQuery(q, searcher.Owners);

                // Build filter
                bool includeUnlisted = ignoreFilter;
                includePrerelease = ignoreFilter || includePrerelease;
                feed = ignoreFilter ? null : feed;

                Filter filter = null;
                if (!ignoreFilter && searcher.TryGetFilter(includeUnlisted, includePrerelease, feed, out filter))
                {
                    // Filter before running the query (make the search set smaller)
                    query = new FilteredQuery(query, filter);
                }

                if (countOnly)
                {
                    DocumentCountImpl(jsonWriter, searcher, query);
                }
                else
                {
                    ListDocumentsImpl(jsonWriter, searcher, query, sortBy, skip, take);
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
        public static JToken Query(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            int skip;
            if (!int.TryParse(context.Request.Query["skip"], out skip))
            {
                skip = 0;
            }

            int take;
            if (!int.TryParse(context.Request.Query["take"], out take))
            {
                take = 20;
            }

            bool countOnly;
            if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
            {
                countOnly = false;
            }

            bool includePrerelease;
            if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
            {
                includePrerelease = false;
            }

            bool includeExplanation;
            if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
            {
                includeExplanation = false;
            }

            bool applyFilter;
            if (!bool.TryParse(context.Request.Query["filter"], out applyFilter))
            {
                applyFilter = true;
            }

            string projectType = context.Request.Query["projectType"] ?? string.Empty;

            string supportedFramework = context.Request.Query["supportedFramework"];

            string q = context.Request.Query["q"] ?? string.Empty;

            string scheme = context.Request.Uri.Scheme;

            return QuerySearch(searcherManager, scheme, q, countOnly, projectType, supportedFramework, includePrerelease, skip, take, includeExplanation, applyFilter);
        }
        public static async Task V3SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            var skip = GetSkip(context);
            var take = GetTake(context);
            var includePrerelease  = GetIncludePrerelease(context);
            var includeExplanation = GetIncludeExplanation(context);

            var q      = context.Request.Query["q"] ?? string.Empty;
            var feed   = context.Request.Query["feed"];
            var scheme = context.Request.Uri.Scheme;

            await ResponseHelpers.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => ServiceImpl.Search(jsonWriter, searcherManager, scheme, q, includePrerelease, skip, take, feed, includeExplanation));
        }
        public static async Task V3SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
        {
            var skip = GetSkip(context);
            var take = GetTake(context);
            var includePrerelease = GetIncludePrerelease(context);
            bool includeExplanation = GetBool(context, nameof(includeExplanation));

            var q = context.Request.Query["q"] ?? string.Empty;
            var feed = context.Request.Query["feed"];
            var scheme = context.Request.Uri.Scheme;

            await responseWriter.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => ServiceImpl.Search(jsonWriter, searcherManager, scheme, q, includePrerelease, skip, take, feed, includeExplanation));
        }
        public static async Task V2SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            var skip              = GetSkip(context);
            var take              = GetTake(context);
            var ignoreFilter      = GetIgnoreFilter(context);
            var countOnly         = GetCountOnly(context);
            var includePrerelease = GetIncludePrerelease(context);

            var q      = context.Request.Query["q"] ?? string.Empty;
            var sortBy = context.Request.Query["sortBy"] ?? string.Empty;
            var feed   = context.Request.Query["feed"];

            await ResponseHelpers.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => GalleryServiceImpl.Search(jsonWriter, searcherManager, q, countOnly, includePrerelease, sortBy, skip, take, feed, ignoreFilter));
        }
Example #19
0
        public static async Task V2SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            var skip = GetSkip(context);
            var take = GetTake(context);
            var ignoreFilter = GetIgnoreFilter(context);
            var countOnly = GetCountOnly(context);
            var includePrerelease = GetIncludePrerelease(context);

            var q = context.Request.Query["q"] ?? string.Empty;
            var sortBy = context.Request.Query["sortBy"] ?? string.Empty;
            var feed = context.Request.Query["feed"];

            await ResponseHelpers.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => GalleryServiceImpl.Search(jsonWriter, searcherManager, q, countOnly, includePrerelease, sortBy, skip, take, feed, ignoreFilter));
        }
        public void Configuration(IAppBuilder app)
        {
            //app.UseErrorPage();

            //  search test console

            app.Use(async (context, next) =>
            {
                if (String.Equals(context.Request.Path.Value, "/console", StringComparison.OrdinalIgnoreCase))
                {
                    // Redirect to trailing slash to maintain relative links
                    context.Response.Redirect(context.Request.PathBase + context.Request.Path + "/");
                    context.Response.StatusCode = 301;
                    return;
                }
                else if (String.Equals(context.Request.Path.Value, "/console/", StringComparison.OrdinalIgnoreCase))
                {
                    context.Request.Path = new PathString("/console/Index.html");
                }
                await next();
            });

            app.UseStaticFiles(new StaticFileOptions(new SharedOptions
            {
                RequestPath = new PathString("/console"),
                FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console")
            }));

            //  start the service running - the Lucene index needs to be reopened regularly on a background thread
            string searchIndexRefresh = System.Configuration.ConfigurationManager.AppSettings.Get("Search.IndexRefresh") ?? "15";
            int seconds;
            if (!int.TryParse(searchIndexRefresh, out seconds))
            {
                seconds = 15;
            }

            _searcherManager = CreateSearcherManager();

            _searcherManager.Open();

            _gate = 0;
            _timer = new Timer(new TimerCallback(ReopenCallback), 0, 0, seconds * 1000);

            app.Run(Invoke);
        }
        public static async Task AutoCompleteAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
        {
            var skip = GetSkip(context);
            var take = GetTake(context);
            var includePrerelease = GetIncludePrerelease(context);
            bool explanation = GetBool(context, nameof(explanation));

            var q = context.Request.Query["q"];
            var id = context.Request.Query["id"];
            if (q == null && id == null)
            {
                q = string.Empty;
            }

            await responseWriter.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => ServiceImpl.AutoComplete(jsonWriter, searcherManager, q, id, includePrerelease, skip, take, explanation));
        }
Example #22
0
        private bool InitializeSearcherManager(IndexingConfiguration config, Directory directory, ILoader loader, ILoggerFactory loggerFactory)
        {
            const int maxRetries = 10;

            try
            {
                Retry.Incremental(
                    () =>
                {
                    var stopwatch = Stopwatch.StartNew();

                    _searcherManager = NuGetSearcherManager.Create(config, loggerFactory, directory, loader);
                    _searcherManager.Open();

                    stopwatch.Stop();

                    _searchTelemetryClient.TrackMetric(
                        SearchTelemetryClient.MetricName.SearchIndexReopenDuration, stopwatch.Elapsed.TotalSeconds);

                    TrackIndexMetrics(_searcherManager, _searchTelemetryClient);
                },
                    shouldRetry: e =>
                {
                    // Retry on any exception (but log it)
                    _logger.LogError("Startup: An error occurred initializing searcher manager. Going to retry... Exception: {Exception}",
                                     e);
                    _searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, 1);

                    return(true);
                },
                    maxRetries: maxRetries,
                    waitIncrement: TimeSpan.FromSeconds(1));

                return(true);
            }
            catch (Exception e)
            {
                _logger.LogCritical("Startup: A critical error occurred initializing searcher manager. Number of retries exhausted. Exception: {Exception}", e);
                _searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, maxRetries);

                return(false);
            }
        }
        public static void AutoComplete(JsonWriter jsonWriter, NuGetSearcherManager searcherManager, string q, string id, bool includePrerelease, int skip, int take, bool includeExplanation)
        {
            var searcher = searcherManager.Get();
            try
            {
                Filter filter = null;

                if (q != null)
                {
                    Query query = MakeAutoCompleteQuery(q,
                        searcher.DocIdMapping,
                        searcher.Downloads,
                        searcher.Rankings,
                        searcher.QueryBoostingContext);

                    if (searcher.TryGetFilter(false, includePrerelease, null, out filter))
                    {
                        // Filter before running the query (make the search set smaller)
                        query = new FilteredQuery(query, filter);
                    }

                    TopDocs topDocs = searcher.Search(query, skip + take);
                    ResponseFormatter.WriteAutoCompleteResult(jsonWriter, searcher, topDocs, skip, take, includeExplanation, query);
                }
                else
                {
                    Query query = MakeAutoCompleteVersionQuery(id);

                    if (searcher.TryGetFilter(false, includePrerelease, null, out filter))
                    {
                        // Filter before running the query (make the search set smaller)
                        query = new FilteredQuery(query, filter);
                    }

                    TopDocs topDocs = searcher.Search(query, 1);
                    ResponseFormatter.WriteAutoCompleteVersionResult(jsonWriter, searcher, includePrerelease, topDocs);
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
        public static async Task AutoCompleteAsync(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            var skip = GetSkip(context);
            var take = GetTake(context);
            var includePrerelease  = GetIncludePrerelease(context);
            var includeExplanation = GetIncludeExplanation(context);

            var q  = context.Request.Query["q"];
            var id = context.Request.Query["id"];

            if (q == null && id == null)
            {
                q = string.Empty;
            }

            await ResponseHelpers.WriteResponseAsync(
                context,
                HttpStatusCode.OK,
                jsonWriter => ServiceImpl.AutoComplete(jsonWriter, searcherManager, q, id, includePrerelease, skip, take, includeExplanation));
        }
        public void Configuration(IAppBuilder app)
        {
            app.UseErrorPage();

            //  search test console

            app.Use(async(context, next) =>
            {
                if (String.Equals(context.Request.Path.Value, "/console", StringComparison.OrdinalIgnoreCase))
                {
                    // Redirect to trailing slash to maintain relative links
                    context.Response.Redirect(context.Request.PathBase + context.Request.Path + "/");
                    context.Response.StatusCode = 301;
                    return;
                }
                else if (String.Equals(context.Request.Path.Value, "/console/", StringComparison.OrdinalIgnoreCase))
                {
                    context.Request.Path = new PathString("/console/Index.html");
                }
                await next();
            });

            app.UseStaticFiles(new StaticFileOptions(new SharedOptions
            {
                RequestPath = new PathString("/console"),
                FileSystem  = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console")
            }));

            //  start the service running - the Lucene index needs to be reopened regularly on a background thread

            _searcherManager = CreateSearcherManager();

            _searcherManager.Open();

            _gate  = 0;
            _timer = new Timer(new TimerCallback(ReopenCallback), 0, 0, 180 * 1000);

            app.Run(Invoke);
        }
Example #26
0
        async Task CreateLuceneIndex(Uri catalogIndex, Func <StorageHttpMessageHandler> handlerFunc, CancellationToken cancellationToken)
        {
            Lucene.Net.Store.Directory luceneDirectory = new RAMDirectory();

            var collector = new SearchIndexFromCatalogCollector(
                catalogIndex,
                luceneDirectory,
                null,
                handlerFunc);

            await collector.Run(
                new MemoryCursor(DateTime.MinValue.ToUniversalTime()),
                new MemoryCursor(DateTime.MaxValue.ToUniversalTime()),
                cancellationToken);

            ILogger logger = new DebugLogger("Lucene");
            ILoader loader = new AuxillaryIndexLoader();

            SearcherManager = new NuGetSearcherManager("memory", logger, luceneDirectory, loader);
            SearcherManager.RegistrationBaseAddress["http"]  = new Uri(_baseAddress, Registration);
            SearcherManager.RegistrationBaseAddress["https"] = new Uri(_baseAddress, Registration);
            SearcherManager.Open();
        }
        public NuGetIndexSearcher(
            NuGetSearcherManager manager,
            IndexReader reader,
            IDictionary<string, string> commitUserData,
            IDictionary<string, Filter> curatedFeeds,
            Filter[][] latest,
            IReadOnlyDictionary<string, int[]> docIdMapping,
            Downloads downloads,
            VersionResult[] versions,
            RankingResult rankings,
            QueryBoostingContext context,
            OpenBitSet latestBitSet,
            OpenBitSet latestStableBitSet,
            OwnersResult owners)
            : base(reader)
        {
            Manager = manager;
            CommitUserData = commitUserData;

            _curatedFeeds = new Dictionary<string, Filter>(curatedFeeds.Count);
            foreach (var curatedFeedsFilter in curatedFeeds)
            {
                _curatedFeeds.Add(curatedFeedsFilter.Key, new CachingWrapperFilter(curatedFeedsFilter.Value));
            }

            _latest = latest;
            DocIdMapping = docIdMapping;
            Downloads = downloads;
            Versions = versions;
            Rankings = rankings;
            LatestBitSet = latestBitSet;
            LatestStableBitSet = latestStableBitSet;
            Owners = owners;
            QueryBoostingContext = context;
            LastReopen = DateTime.UtcNow;
        }
        public static JToken AutoCompleteMakeResult(IndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
        {
            JArray array = new JArray();

            for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
            {
                ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
                Document document = searcher.Doc(scoreDoc.Doc);
                string id = document.Get("Id");

                array.Add(id);
            }

            JObject result = new JObject();

            result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
            result.Add("totalHits", topDocs.TotalHits);
            result.Add("indexName", searcherManager.IndexName);
            result.Add("data", array);

            if (includeExplanation)
            {
                JArray explanations = new JArray();
                for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
                {
                    ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
                    Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
                    explanations.Add(explanation.ToString());
                }
                result.Add("explanations", explanations);
            }

            return result;
        }
 public static void Find(JsonWriter jsonWriter, NuGetSearcherManager searcherManager, string id, string scheme)
 {
     var searcher = searcherManager.Get();
     try
     {
         Query query = MakeFindQuery(id);
         TopDocs topDocs = searcher.Search(query, 1);
         ResponseFormatter.WriteFindResult(jsonWriter, searcher, scheme, topDocs);
     }
     finally
     {
         searcherManager.Release(searcher);
     }
 }
        public static JToken MakeResultData(NuGetIndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
        {
            Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];

            JArray array = new JArray();

            for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
            {
                ScoreDoc scoreDoc = topDocs.ScoreDocs[i];

                Document document = searcher.Doc(scoreDoc.Doc);

                string url = document.Get("Url");
                string id = document.Get("Id");
                string version = document.Get("Version");

                JObject obj = new JObject();
                obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri;
                obj["@type"] = "Package";
                obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
                obj["id"] = id;

                AddField(obj, document, "domain", "Domain");
                AddField(obj, document, "description", "Description");
                AddField(obj, document, "summary", "Summary");
                AddField(obj, document, "title", "Title");
                AddField(obj, document, "iconUrl", "IconUrl");
                AddField(obj, document, "licenseUrl", "LicenseUrl");
                AddField(obj, document, "projectUrl", "ProjectUrl");
                AddFieldAsArray(obj, document, "tags", "Tags");
                AddFieldAsArray(obj, document, "authors", "Authors");

                obj["version"] = version;
                obj["versions"] = searcherManager.GetVersions(searcher, scheme, scoreDoc.Doc);
                
                if (includeExplanation)
                {
                    Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
                    obj["explanation"] = explanation.ToString();
                }

                array.Add(obj);
            }

            return array;
        }
        static JToken MakeResult(NuGetIndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
        {
            JToken data = MakeResultData(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query);

            JObject result = new JObject();

            result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
            result.Add("totalHits", topDocs.TotalHits);
            result.Add("lastReopen", searcherManager.LastReopen.ToString("o"));
            result.Add("index", searcherManager.IndexName);
            result.Add("data", data);

            return result;
        }
Example #32
0
        private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient)
        {
            var searcher = searcherManager.Get();
            try
            {
                // Track number of documents in index
                searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs());

                // Track time between Lucene commit and reopen
                string temp;
                if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp))
                {
                    var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal);

                    searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag,
                        (searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds,
                        new Dictionary<string, string>()
                        {
                            { SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") },
                            { SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") }
                        });
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
Example #33
0
        private bool InitializeSearcherManager(IConfiguration configuration, Directory directory, ILoader loader, ILoggerFactory loggerFactory)
        {
            const int maxRetries = 10;

            try
            {
                Retry.Incremental(
                    () =>
                    {
                        var stopwatch = Stopwatch.StartNew();

                        _searcherManager = NuGetSearcherManager.Create(configuration, loggerFactory, directory, loader);
                        _searcherManager.Open();

                        stopwatch.Stop();

                        _searchTelemetryClient.TrackMetric(
                            SearchTelemetryClient.MetricName.SearchIndexReopenDuration, stopwatch.Elapsed.TotalSeconds);

                        TrackIndexMetrics(_searcherManager, _searchTelemetryClient);
                    },
                    shouldRetry: e =>
                    {
                        // Retry on any exception (but log it)
                        _logger.LogError("Startup: An error occurred initializing searcher manager. Going to retry...", e);
                        _searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, 1);

                        return true;
                    },
                    maxRetries: maxRetries,
                    waitIncrement: TimeSpan.FromSeconds(1));

                return true;
            }
            catch (Exception e)
            {
                _logger.LogCritical("Startup: A critical error occurred initializing searcher manager. Number of retries exhausted.", e);
                _searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, maxRetries);

                return false;
            }
        }
 public static Query MakeQuery(string q, NuGetSearcherManager searcherManager)
 {
     Query query = LuceneQueryCreator.Parse(q, false);
     Query boostedQuery = new RankingScoreQuery(query, searcherManager.GetRankings());
     return boostedQuery;
 }
        static JToken FindSearch(NuGetSearcherManager searcherManager, string id, string scheme)
        {
            if (id == null)
            {
                return null;
            }

            var searcher = searcherManager.Get();
            try
            {
                string analyzedId = id.ToLowerInvariant();
                Query query = new TermQuery(new Term("Id", analyzedId));
                TopDocs topDocs = searcher.Search(query, 1);

                if (topDocs.TotalHits > 0)
                {
                    Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
                    JObject obj = new JObject();
                    obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
                    return obj;
                }
                else
                {
                    return null;
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
        public static JToken AutoComplete(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            var searcher = searcherManager.Get();
            try
            {
                int skip;
                if (!int.TryParse(context.Request.Query["skip"], out skip))
                {
                    skip = 0;
                }

                int take;
                if (!int.TryParse(context.Request.Query["take"], out take))
                {
                    take = 20;
                }

                bool includePrerelease;
                if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
                {
                    includePrerelease = false;
                }

                bool includeExplanation;
                if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
                {
                    includeExplanation = false;
                }

                string supportedFramework = context.Request.Query["supportedFramework"];

                string q = context.Request.Query["q"]; 
                string id = context.Request.Query["id"];

                if (q == null && id == null)
                {
                    q = string.Empty;
                }

                return AutoCompleteSearch(searcherManager, q, id, supportedFramework, includePrerelease, skip, take, includeExplanation);
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
        public static JToken Find(IOwinContext context, NuGetSearcherManager searcherManager)
        {
            string id = context.Request.Query["id"];

            string scheme = context.Request.Uri.Scheme;

            return FindSearch(searcherManager, id, scheme);
        }
Example #38
0
 public static async Task Stats(IOwinContext context, NuGetSearcherManager searcherManager)
 {
     await ResponseHelpers.WriteResponseAsync(
         context,
         HttpStatusCode.OK,
         jsonWriter => ServiceInfoImpl.Stats(jsonWriter, searcherManager));
 }
        static Query AutoCompleteMakeQuery(string q, NuGetSearcherManager searcherManager)
        {
            if (string.IsNullOrEmpty(q))
            {
                return new MatchAllDocsQuery();
            }

            QueryParser queryParser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "IdAutocomplete", new PackageAnalyzer());

            //TODO: we should be doing phrase queries to get the ordering right
            //const int MAX_NGRAM_LENGTH = 8;
            //q = (q.Length < MAX_NGRAM_LENGTH) ? q : q.Substring(0, MAX_NGRAM_LENGTH);
            //string phraseQuery = string.Format("IdAutocompletePhrase:\"/ {0}\"~20", q);
            //Query query = queryParser.Parse(phraseQuery);

            Query query = queryParser.Parse(q);

            Query boostedQuery = new RankingScoreQuery(query, searcherManager.GetRankings());
            return boostedQuery;
        }
        static JObject AutoCompleteMakeVersionResult(NuGetSearcherManager searcherManager, bool includePrerelease, TopDocs topDocs)
        {
            var searcher = searcherManager.Get();
            try
            {
                JObject result = new JObject();

                result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
                result.Add("indexName", searcherManager.IndexName);

                if (topDocs.TotalHits > 0)
                {
                    ScoreDoc scoreDoc = topDocs.ScoreDocs[0];
                    JArray versions = searcherManager.GetVersionLists(searcher, scoreDoc.Doc);
                    result.Add("totalHits", versions.Count());
                    result["data"] = versions;
                }
                else
                {
                    result.Add("totalHits", 0);
                    result["data"] = new JArray();
                }
                return result;
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }
        static JToken AutoCompleteSearch(NuGetSearcherManager searcherManager, string q, string id, string supportedFramework, bool includePrerelease, int skip, int take, bool includeExplanation)
        {
            var searcher = searcherManager.Get();
            try
            {
                Filter filter = searcherManager.GetFilter(searcher, includePrerelease, supportedFramework);

                if (q != null)
                {
                    Query query = AutoCompleteMakeQuery(q, searcherManager);
                    TopDocs topDocs = searcher.Search(query, filter, skip + take);
                    return AutoCompleteMakeResult(searcher, topDocs, skip, take, searcherManager, includeExplanation, query);
                }
                else
                {
                    Query query = new TermQuery(new Term("Id", id.ToLowerInvariant()));
                    TopDocs topDocs = searcher.Search(query, filter, 1);
                    return AutoCompleteMakeVersionResult(searcherManager, includePrerelease, topDocs);
                }
            }
            finally
            {
                searcherManager.Release(searcher);
            }
        }