public async Task <IEnumerable <JobExecution> > GetAndMarkPending(int max, DateTime until, CancellationToken ctx) { using var _ = QueryMetrics.TimeQuery(nameof(JobExecution).ToLowerInvariant(), "mark_and_get_pending"); using var transaction = await _dbContext.Database.BeginTransactionAsync(ctx); try { var jobs = await _dbContext.Set <JobExecution>() .Where(s => s.ScheduledAt <= until && s.Status == ExecutionStatus.Scheduled && s.Job.Enabled) .OrderBy(s => s.ScheduledAt) .Take(max) .Include(s => s.Job) .ToListAsync(ctx); jobs.ForEach(j => j.InFlight()); await _dbContext.SaveChangesAsync(ctx); return(jobs); } catch (Exception ex) { await transaction.RollbackAsync(CancellationToken.None); _logger.LogError(ex, "Failed to get pending job executions"); return(Enumerable.Empty <JobExecution>()); } finally { await transaction.CommitAsync(CancellationToken.None); } }
public void QueryMetrics_DiceCoefficientTest() { Assert.IsTrue(QueryMetrics.DiceCoefficient("a b", "b c") - 0.50 < 0.001); Assert.IsTrue(QueryMetrics.DiceCoefficient("a b c d e f g h", "a b c d e f g") - 0.933 < 0.001); Assert.IsTrue(QueryMetrics.DiceCoefficient("\"a\" BBB_b", "BB BBB BBB_b") - 0.40 < 0.001); Assert.IsTrue(QueryMetrics.DiceCoefficient(String.Empty, String.Empty) == 0.0); }
private static void ConsoleShowQueryMetrics(string paritionId, QueryMetrics qm) { ConsoleNewLine(); Console.WriteLine($"============ PARITION ID: {paritionId} ============"); ConsoleNewLine(); Console.WriteLine(qm); }
public Task UpdateAsync(T entity, CancellationToken ctx) { using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "update"); _dbContext.Entry(entity).CurrentValues.SetValues(entity); return(_dbContext.SaveChangesAsync(ctx)); }
private void OpenFileWithSelectedResult(object sender) { try { var result = sender as ListBoxItem; if (result != null) { string[] searchKeys = GetKeys(this.searchKey); var searchResult = result.Content as CodeSearchResult; FileOpener.OpenItem(searchResult); Task.Factory.StartNew(() => { Thread.Sleep(500); this.Dispatcher.BeginInvoke(HighlightStuffInvoker, searchResult.ProgramElement.FullFilePath, searchResult. ProgramElement.DefinitionLineNumber, searchResult.ProgramElement.RawSource, searchKeys); }); var matchDescription = QueryMetrics.DescribeQueryProgramElementMatch(searchResult.ProgramElement, searchBox.Text); LogEvents.OpeningCodeSearchResult(searchResult, SearchResults.IndexOf(searchResult) + 1, matchDescription); } } catch (ArgumentException aex) { LogEvents.UIGenericError(this, aex); MessageBox.Show(FileNotFoundPopupMessage, FileNotFoundPopupTitle, MessageBoxButton.OK); } catch (Exception ee) { LogEvents.UIGenericError(this, ee); MessageBox.Show(FileNotFoundPopupMessage, FileNotFoundPopupTitle, MessageBoxButton.OK); } }
public async Task AddAsync(T entity, CancellationToken ctx) { using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "insert"); await _dbContext.Set <T>().AddAsync(entity, ctx); await _dbContext.SaveChangesAsync(ctx); }
public async Task <T?> GetByIdAsync(Guid id, CancellationToken ctx) { using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "get_by_id"); var keyValues = new object[] { id }; return(await _dbContext.Set <T>().FindAsync(keyValues, ctx)); }
public void TestAddition() { QueryMetrics doubleQueryMetrics = MockQueryMetrics + MockQueryMetrics; // Spot check Assert.AreEqual(2 * BackendMetricsTests.MockBackendMetrics.IndexLookupTime, doubleQueryMetrics.BackendMetrics.IndexLookupTime); Assert.AreEqual(2 * IndexUtilizationInfoTests.MockIndexUtilizationInfo.PotentialSingleIndexes.Count, doubleQueryMetrics.IndexUtilizationInfo.PotentialSingleIndexes.Count); Assert.AreEqual(2 * ClientSideMetricsTests.MockClientSideMetrics.RequestCharge, doubleQueryMetrics.ClientSideMetrics.RequestCharge); }
public static void DefaultProduceAsyncCompleteDelegate( int numberOfDocuments, double requestCharge, QueryMetrics queryMetrics, long responseLengthInBytes, CancellationToken token) { return; }
public void QueryMetrics_QueryTypeTest() { Assert.AreEqual(QueryMetrics.ExamineQuery("a b").ToString(), "Plain,Plain"); Assert.AreEqual(QueryMetrics.ExamineQuery("\"a b c\"").ToString(), "Quoted,Quoted,Quoted"); Assert.AreEqual(QueryMetrics.ExamineQuery("a_aAa b").ToString(), "CamelcaseUnderscore,Plain"); Assert.AreEqual(QueryMetrics.ExamineQuery("\" a\" bBb").ToString(), "Quoted,Camelcase"); Assert.AreEqual(QueryMetrics.ExamineQuery("a\" \"b").ToString(), "Plain,Plain"); Assert.AreEqual(QueryMetrics.ExamineQuery("a\" B_b_b_B \"c").ToString(), "Plain,QuotedUnderscore,Plain"); Assert.AreEqual(QueryMetrics.ExamineQuery("\"a\" BBB_b").ToString(), "Quoted,AcronymUnderscore"); Assert.AreEqual(QueryMetrics.ExamineQuery("-Abb aAAAc").ToString(), "MinusCamelcase,CamelcaseAcronym"); }
public void TestCreateFromEnumerable() { QueryMetrics tripleQueryMetrics = QueryMetrics.CreateFromIEnumerable(new List <QueryMetrics>() { MockQueryMetrics, MockQueryMetrics, MockQueryMetrics }); // Spot check Assert.AreEqual(3 * BackendMetricsTests.MockBackendMetrics.IndexLookupTime, tripleQueryMetrics.BackendMetrics.IndexLookupTime); Assert.AreEqual(3 * IndexUtilizationInfoTests.MockIndexUtilizationInfo.PotentialSingleIndexes.Count, tripleQueryMetrics.IndexUtilizationInfo.PotentialSingleIndexes.Count); Assert.AreEqual(3 * ClientSideMetricsTests.MockClientSideMetrics.RequestCharge, tripleQueryMetrics.ClientSideMetrics.RequestCharge); }
public void TestAccumulator() { QueryMetrics.Accumulator accumulator = new QueryMetrics.Accumulator(); accumulator = accumulator.Accumulate(MockQueryMetrics); accumulator = accumulator.Accumulate(MockQueryMetrics); QueryMetrics doubleQueryMetrics = QueryMetrics.Accumulator.ToQueryMetrics(accumulator); // Spot check Assert.AreEqual(2 * BackendMetricsTests.MockBackendMetrics.IndexLookupTime, doubleQueryMetrics.BackendMetrics.IndexLookupTime); Assert.AreEqual(2 * IndexUtilizationInfoTests.MockIndexUtilizationInfo.PotentialSingleIndexes.Count, doubleQueryMetrics.IndexUtilizationInfo.PotentialSingleIndexes.Count); Assert.AreEqual(2 * ClientSideMetricsTests.MockClientSideMetrics.RequestCharge, doubleQueryMetrics.ClientSideMetrics.RequestCharge); }
public async Task UpdateManyAsync(IEnumerable <T> entities, CancellationToken ctx) { using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "batch_update"); using var transaction = await _dbContext.Database.BeginTransactionAsync(ctx); try { entities.ToList().ForEach(e => _dbContext.Entry(e).CurrentValues.SetValues(entities)); } catch (Exception ex) { await transaction.RollbackAsync(CancellationToken.None); _logger.LogError(ex, "Failed batch update transaction"); } finally { await transaction.CommitAsync(CancellationToken.None); } }
public async Task AddManyAsync(IEnumerable <T> entities, CancellationToken ctx) { using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "insert_many"); using var transaction = await _dbContext.Database.BeginTransactionAsync(ctx); try { await _dbContext.Set <T>().AddRangeAsync(entities, ctx); } catch (Exception ex) { await transaction.RollbackAsync(CancellationToken.None); _logger.LogError(ex, "Failed batch insert transaction"); } finally { await transaction.CommitAsync(CancellationToken.None); } }
public void LogCosmosQueryMetricsObject(ILogger logger, Guid correlationId, string message, QueryMetrics cosmosQueryMetrics) { logger.LogInformation("CorrelationId: {0} Message: {1} Object: {2}", correlationId, message, JsonConvert.SerializeObject(cosmosQueryMetrics)); }
public void LogCosmosQueryMetrics(ILogger logger, Guid correlationId, string message, QueryMetrics cosmosQueryMetrics) { logger.LogInformation("CorrelationId: {0}\n" + "Message: {1}\n" + "IndexHitRatio: {2}\n" + "OutputDocumentCount: {3}\n" + "Retries: {4}\n" + "RetrievedDocumentCount: {5}\n" + "TotalTime: {6}", correlationId, message, cosmosQueryMetrics.IndexHitRatio, cosmosQueryMetrics.OutputDocumentCount, cosmosQueryMetrics.Retries, cosmosQueryMetrics.RetrievedDocumentCount, cosmosQueryMetrics.TotalTime); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req, ILogger log, ExecutionContext context) { try { EnsureClient(context); string productCode = req.Query["productCode"]; SqlQuerySpec querySpec = null; if (!String.IsNullOrWhiteSpace(productCode)) { querySpec = new SqlQuerySpec( String.Concat( "SELECT VALUE COUNT(1) FROM Items i WHERE i.productCode IN ('", productCode, "')")); } else { querySpec = new SqlQuerySpec("SELECT VALUE COUNT(1)FROM Items i"); } IDocumentQuery <dynamic> query = cosmosDbClient.CreateDocumentQuery( collectionUri, querySpec, new FeedOptions() { EnableCrossPartitionQuery = true, PartitionKey = null, PopulateQueryMetrics = true, MaxItemCount = 50, MaxDegreeOfParallelism = 0, MaxBufferedItemCount = 100 }).AsDocumentQuery(); double totalRUs = 0; long count = 0; while (query.HasMoreResults) { FeedResponse <dynamic> feedResponse = await query.ExecuteNextAsync(); Console.WriteLine(feedResponse.RequestCharge); totalRUs += feedResponse.RequestCharge; IReadOnlyDictionary <string, QueryMetrics> partitionIdToQueryMetrics = feedResponse.QueryMetrics; foreach (KeyValuePair <string, QueryMetrics> kvp in partitionIdToQueryMetrics) { string partitionId = kvp.Key; QueryMetrics queryMetrics = kvp.Value; Console.WriteLine("{0}: {1}", partitionId, queryMetrics); } IEnumerator <dynamic> docEnumerator = feedResponse.GetEnumerator(); while (docEnumerator.MoveNext()) { count += (long)docEnumerator.Current; } } var responsePayload = new ResponseContract { Count = count, TotalRUs = totalRUs, }; log.LogInformation("Count: {0}, Total RUs: {1}", count, totalRUs); return(new OkObjectResult(JsonConvert.SerializeObject(responsePayload))); } catch (Exception error) { return(new ObjectResult(error.ToString()) { StatusCode = 500 }); } }
private async Task <DocumentProducer <T> > FetchAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { // TODO: This workflow could be simplified. FetchResult exceptionFetchResult = null; try { this.fetchSchedulingMetrics.Start(); this.fetchExecutionRangeAccumulator.BeginFetchRange(); FeedResponse <T> feedResponse = null; double requestCharge = 0; long responseLengthBytes = 0; QueryMetrics queryMetrics = QueryMetrics.Zero; do { int pageSize = (int)Math.Min(this.PageSize, (long)int.MaxValue); Debug.Assert(pageSize >= 0, string.Format("pageSize was negative ... this.PageSize: {0}", this.PageSize)); using (DocumentServiceRequest request = this.createRequestFunc(this.CurrentBackendContinuationToken, pageSize)) { retryPolicyInstance = retryPolicyInstance ?? this.createRetryPolicyFunc(); retryPolicyInstance.OnBeforeSendRequest(request); // Custom backoff and retry ExceptionDispatchInfo exception = null; try { cancellationToken.ThrowIfCancellationRequested(); feedResponse = await this.executeRequestFunc(request, cancellationToken); this.fetchExecutionRangeAccumulator.EndFetchRange(feedResponse.Count, Interlocked.Read(ref this.retries)); this.ActivityId = Guid.Parse(feedResponse.ActivityId); } catch (Exception ex) { exception = ExceptionDispatchInfo.Capture(ex); } if (exception != null) { cancellationToken.ThrowIfCancellationRequested(); ShouldRetryResult shouldRetryResult = await retryPolicyInstance.ShouldRetryAsync(exception.SourceException, cancellationToken); shouldRetryResult.ThrowIfDoneTrying(exception); this.ScheduleFetch(retryPolicyInstance, shouldRetryResult.BackoffTime); Interlocked.Increment(ref this.retries); return(this); } requestCharge += feedResponse.RequestCharge; responseLengthBytes += feedResponse.ResponseLengthBytes; if (feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics] != null) { queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics], new ClientSideMetrics(this.retries, requestCharge, this.fetchExecutionRangeAccumulator.GetExecutionRanges(), new List <Tuple <string, SchedulingTimeSpan> >()), this.activityId); // Reset the counters. Interlocked.Exchange(ref this.retries, 0); } this.UpdateRequestContinuationToken(feedResponse.ResponseContinuation); retryPolicyInstance = null; this.numDocumentsFetched += feedResponse.Count; } }while (!this.FetchedAll && feedResponse.Count <= 0); await this.CompleteFetchAsync(feedResponse, cancellationToken); this.produceAsyncCompleteCallback(this, feedResponse.Count, requestCharge, queryMetrics, responseLengthBytes, cancellationToken); } catch (Exception ex) { DefaultTrace.TraceWarning(string.Format( CultureInfo.InvariantCulture, "{0}, CorrelatedActivityId: {1}, ActivityId {2} | DocumentProducer Id: {3}, Exception in FetchAsync: {4}", DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture), this.correlatedActivityId, this.ActivityId, this.targetRange.Id, ex.Message)); exceptionFetchResult = new FetchResult(ExceptionDispatchInfo.Capture(ex)); } finally { this.fetchSchedulingMetrics.Stop(); if (this.FetchedAll) { // One more callback to send the scheduling metrics this.produceAsyncCompleteCallback( producer: this, size: 0, resourceUnitUsage: 0, queryMetrics: QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( QueryMetrics.Zero.ToDelimitedString(), new ClientSideMetrics( retries: 0, requestCharge: 0, fetchExecutionRanges: new List <FetchExecutionRange>(), partitionSchedulingTimeSpans: new List <Tuple <string, SchedulingTimeSpan> > { new Tuple <string, SchedulingTimeSpan>(this.targetRange.Id, this.fetchSchedulingMetrics.Elapsed) }), Guid.Empty), responseLengthBytes: 0, token: cancellationToken); } } if (exceptionFetchResult != null) { this.UpdateRequestContinuationToken(this.CurrentBackendContinuationToken); await this.itemBuffer.AddAsync(exceptionFetchResult, cancellationToken); } return(this); }
public void Search(String searchString, SimpleSearchCriteria searchCriteria = null, bool interactive = true) { if (!EnsureSolutionOpen()) { return; } try { var codeSearcher = new CodeSearcher(new IndexerSearcher()); if (String.IsNullOrEmpty(searchString)) { return; } var solutionKey = ServiceLocator.ResolveOptional <SolutionKey>(); //no opened solution if (solutionKey == null) { _searchResultListener.UpdateMessage("Sando searches only the currently open Solution. Please open a Solution and try again."); return; } searchString = ExtensionPointsRepository.Instance.GetQueryRewriterImplementation().RewriteQuery(searchString); PreRetrievalMetrics preMetrics = new PreRetrievalMetrics(ServiceLocator.Resolve <DocumentIndexer>().Reader, ServiceLocator.Resolve <Analyzer>()); LogEvents.PreSearch(this, preMetrics.MaxIdf(searchString), preMetrics.AvgIdf(searchString), preMetrics.AvgSqc(searchString), preMetrics.AvgVar(searchString)); LogEvents.PreSearchQueryAnalysis(this, QueryMetrics.ExamineQuery(searchString).ToString(), QueryMetrics.DiceCoefficient(QueryMetrics.SavedQuery, searchString)); QueryMetrics.SavedQuery = searchString; var criteria = GetCriteria(searchString, searchCriteria); var results = codeSearcher.Search(criteria, true).AsQueryable(); var resultsReorderer = ExtensionPointsRepository.Instance.GetResultsReordererImplementation(); results = resultsReorderer.ReorderSearchResults(results); var returnString = new StringBuilder(); if (criteria.IsQueryReformed()) { returnString.Append(criteria.GetQueryReformExplanation()); } if (!results.Any()) { returnString.Append("No results found. "); } else { returnString.Append(results.Count() + " results returned. "); } if (ServiceLocator.Resolve <InitialIndexingWatcher>().IsInitialIndexingInProgress()) { returnString.Append("Sando is still performing its initial index of this project, results may be incomplete."); } _searchResultListener.Update(searchString, results); _searchResultListener.UpdateMessage(returnString.ToString()); _searchResultListener.UpdateRecommendedQueries(criteria.GetRecommendedQueries()); LogEvents.PostSearch(this, results.Count(), criteria.NumberOfSearchResultsReturned, PostRetrievalMetrics.AvgScore(results.ToList()), PostRetrievalMetrics.StdDevScore(results.ToList())); } catch (Exception e) { _searchResultListener.UpdateMessage("Sando is experiencing difficulties. See log file for details."); LogEvents.UISandoSearchingError(this, e); } }
public QueryMetricsTraceDatum(QueryMetrics queryMetrics) { this.QueryMetrics = queryMetrics ?? throw new ArgumentNullException(nameof(queryMetrics)); }