public async Task <IEnumerable <JobExecution> > GetAndMarkPending(int max, DateTime until, CancellationToken ctx)
{
using var _ = QueryMetrics.TimeQuery(nameof(JobExecution).ToLowerInvariant(), "mark_and_get_pending");
using var transaction = await _dbContext.Database.BeginTransactionAsync(ctx);
try
{
var jobs = await _dbContext.Set <JobExecution>()
.Where(s => s.ScheduledAt <= until && s.Status == ExecutionStatus.Scheduled && s.Job.Enabled)
.OrderBy(s => s.ScheduledAt)
.Take(max)
.Include(s => s.Job)
.ToListAsync(ctx);
jobs.ForEach(j => j.InFlight());
await _dbContext.SaveChangesAsync(ctx);
return(jobs);
}
catch (Exception ex)
{
await transaction.RollbackAsync(CancellationToken.None);
_logger.LogError(ex, "Failed to get pending job executions");
return(Enumerable.Empty <JobExecution>());
}
finally
{
await transaction.CommitAsync(CancellationToken.None);
}
}
public void QueryMetrics_DiceCoefficientTest()
{
Assert.IsTrue(QueryMetrics.DiceCoefficient("a b", "b c") - 0.50 < 0.001);
Assert.IsTrue(QueryMetrics.DiceCoefficient("a b c d e f g h", "a b c d e f g") - 0.933 < 0.001);
Assert.IsTrue(QueryMetrics.DiceCoefficient("\"a\" BBB_b", "BB BBB BBB_b") - 0.40 < 0.001);
Assert.IsTrue(QueryMetrics.DiceCoefficient(String.Empty, String.Empty) == 0.0);
}
private static void ConsoleShowQueryMetrics(string paritionId, QueryMetrics qm)
{
ConsoleNewLine();
Console.WriteLine($"============ PARITION ID: {paritionId} ============");
ConsoleNewLine();
Console.WriteLine(qm);
}
public Task UpdateAsync(T entity, CancellationToken ctx)
{
using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "update");
_dbContext.Entry(entity).CurrentValues.SetValues(entity);
return(_dbContext.SaveChangesAsync(ctx));
}
private void OpenFileWithSelectedResult(object sender)
{
try
{
var result = sender as ListBoxItem;
if (result != null)
{
string[] searchKeys = GetKeys(this.searchKey);
var searchResult = result.Content as CodeSearchResult;
FileOpener.OpenItem(searchResult);
Task.Factory.StartNew(() =>
{
Thread.Sleep(500);
this.Dispatcher.BeginInvoke(HighlightStuffInvoker, searchResult.ProgramElement.FullFilePath, searchResult.
ProgramElement.DefinitionLineNumber, searchResult.ProgramElement.RawSource, searchKeys);
});
var matchDescription = QueryMetrics.DescribeQueryProgramElementMatch(searchResult.ProgramElement, searchBox.Text);
LogEvents.OpeningCodeSearchResult(searchResult, SearchResults.IndexOf(searchResult) + 1, matchDescription);
}
}
catch (ArgumentException aex)
{
LogEvents.UIGenericError(this, aex);
MessageBox.Show(FileNotFoundPopupMessage, FileNotFoundPopupTitle, MessageBoxButton.OK);
}
catch (Exception ee)
{
LogEvents.UIGenericError(this, ee);
MessageBox.Show(FileNotFoundPopupMessage, FileNotFoundPopupTitle, MessageBoxButton.OK);
}
}
public async Task AddAsync(T entity, CancellationToken ctx)
{
using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "insert");
await _dbContext.Set <T>().AddAsync(entity, ctx);
await _dbContext.SaveChangesAsync(ctx);
}
public async Task <T?> GetByIdAsync(Guid id, CancellationToken ctx)
{
using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "get_by_id");
var keyValues = new object[] { id };
return(await _dbContext.Set <T>().FindAsync(keyValues, ctx));
}
public void TestAddition()
{
QueryMetrics doubleQueryMetrics = MockQueryMetrics + MockQueryMetrics;
// Spot check
Assert.AreEqual(2 * BackendMetricsTests.MockBackendMetrics.IndexLookupTime, doubleQueryMetrics.BackendMetrics.IndexLookupTime);
Assert.AreEqual(2 * IndexUtilizationInfoTests.MockIndexUtilizationInfo.PotentialSingleIndexes.Count, doubleQueryMetrics.IndexUtilizationInfo.PotentialSingleIndexes.Count);
Assert.AreEqual(2 * ClientSideMetricsTests.MockClientSideMetrics.RequestCharge, doubleQueryMetrics.ClientSideMetrics.RequestCharge);
}
public static void DefaultProduceAsyncCompleteDelegate(
int numberOfDocuments,
double requestCharge,
QueryMetrics queryMetrics,
long responseLengthInBytes,
CancellationToken token)
{
return;
}
public void QueryMetrics_QueryTypeTest()
{
Assert.AreEqual(QueryMetrics.ExamineQuery("a b").ToString(), "Plain,Plain");
Assert.AreEqual(QueryMetrics.ExamineQuery("\"a b c\"").ToString(), "Quoted,Quoted,Quoted");
Assert.AreEqual(QueryMetrics.ExamineQuery("a_aAa b").ToString(), "CamelcaseUnderscore,Plain");
Assert.AreEqual(QueryMetrics.ExamineQuery("\" a\" bBb").ToString(), "Quoted,Camelcase");
Assert.AreEqual(QueryMetrics.ExamineQuery("a\" \"b").ToString(), "Plain,Plain");
Assert.AreEqual(QueryMetrics.ExamineQuery("a\" B_b_b_B \"c").ToString(), "Plain,QuotedUnderscore,Plain");
Assert.AreEqual(QueryMetrics.ExamineQuery("\"a\" BBB_b").ToString(), "Quoted,AcronymUnderscore");
Assert.AreEqual(QueryMetrics.ExamineQuery("-Abb aAAAc").ToString(), "MinusCamelcase,CamelcaseAcronym");
}
public void TestCreateFromEnumerable()
{
QueryMetrics tripleQueryMetrics = QueryMetrics.CreateFromIEnumerable(new List <QueryMetrics>()
{
MockQueryMetrics, MockQueryMetrics, MockQueryMetrics
});
// Spot check
Assert.AreEqual(3 * BackendMetricsTests.MockBackendMetrics.IndexLookupTime, tripleQueryMetrics.BackendMetrics.IndexLookupTime);
Assert.AreEqual(3 * IndexUtilizationInfoTests.MockIndexUtilizationInfo.PotentialSingleIndexes.Count, tripleQueryMetrics.IndexUtilizationInfo.PotentialSingleIndexes.Count);
Assert.AreEqual(3 * ClientSideMetricsTests.MockClientSideMetrics.RequestCharge, tripleQueryMetrics.ClientSideMetrics.RequestCharge);
}
public void TestAccumulator()
{
QueryMetrics.Accumulator accumulator = new QueryMetrics.Accumulator();
accumulator = accumulator.Accumulate(MockQueryMetrics);
accumulator = accumulator.Accumulate(MockQueryMetrics);
QueryMetrics doubleQueryMetrics = QueryMetrics.Accumulator.ToQueryMetrics(accumulator);
// Spot check
Assert.AreEqual(2 * BackendMetricsTests.MockBackendMetrics.IndexLookupTime, doubleQueryMetrics.BackendMetrics.IndexLookupTime);
Assert.AreEqual(2 * IndexUtilizationInfoTests.MockIndexUtilizationInfo.PotentialSingleIndexes.Count, doubleQueryMetrics.IndexUtilizationInfo.PotentialSingleIndexes.Count);
Assert.AreEqual(2 * ClientSideMetricsTests.MockClientSideMetrics.RequestCharge, doubleQueryMetrics.ClientSideMetrics.RequestCharge);
}
public async Task UpdateManyAsync(IEnumerable <T> entities, CancellationToken ctx)
{
using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "batch_update");
using var transaction = await _dbContext.Database.BeginTransactionAsync(ctx);
try
{
entities.ToList().ForEach(e => _dbContext.Entry(e).CurrentValues.SetValues(entities));
}
catch (Exception ex)
{
await transaction.RollbackAsync(CancellationToken.None);
_logger.LogError(ex, "Failed batch update transaction");
}
finally
{
await transaction.CommitAsync(CancellationToken.None);
}
}
public async Task AddManyAsync(IEnumerable <T> entities, CancellationToken ctx)
{
using var _ = QueryMetrics.TimeQuery(typeof(T).Name.ToLowerInvariant(), "insert_many");
using var transaction = await _dbContext.Database.BeginTransactionAsync(ctx);
try
{
await _dbContext.Set <T>().AddRangeAsync(entities, ctx);
}
catch (Exception ex)
{
await transaction.RollbackAsync(CancellationToken.None);
_logger.LogError(ex, "Failed batch insert transaction");
}
finally
{
await transaction.CommitAsync(CancellationToken.None);
}
}
public void LogCosmosQueryMetricsObject(ILogger logger, Guid correlationId, string message, QueryMetrics cosmosQueryMetrics)
{
logger.LogInformation("CorrelationId: {0} Message: {1} Object: {2}", correlationId, message, JsonConvert.SerializeObject(cosmosQueryMetrics));
}
public void LogCosmosQueryMetrics(ILogger logger, Guid correlationId, string message, QueryMetrics cosmosQueryMetrics)
{
logger.LogInformation("CorrelationId: {0}\n" +
"Message: {1}\n" +
"IndexHitRatio: {2}\n" +
"OutputDocumentCount: {3}\n" +
"Retries: {4}\n" +
"RetrievedDocumentCount: {5}\n" +
"TotalTime: {6}",
correlationId,
message,
cosmosQueryMetrics.IndexHitRatio,
cosmosQueryMetrics.OutputDocumentCount,
cosmosQueryMetrics.Retries,
cosmosQueryMetrics.RetrievedDocumentCount,
cosmosQueryMetrics.TotalTime);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req,
ILogger log,
ExecutionContext context)
{
try
{
EnsureClient(context);
string productCode = req.Query["productCode"];
SqlQuerySpec querySpec = null;
if (!String.IsNullOrWhiteSpace(productCode))
{
querySpec = new SqlQuerySpec(
String.Concat(
"SELECT VALUE COUNT(1) FROM Items i WHERE i.productCode IN ('",
productCode,
"')"));
}
else
{
querySpec = new SqlQuerySpec("SELECT VALUE COUNT(1)FROM Items i");
}
IDocumentQuery <dynamic> query = cosmosDbClient.CreateDocumentQuery(
collectionUri,
querySpec,
new FeedOptions()
{
EnableCrossPartitionQuery = true,
PartitionKey = null,
PopulateQueryMetrics = true,
MaxItemCount = 50,
MaxDegreeOfParallelism = 0,
MaxBufferedItemCount = 100
}).AsDocumentQuery();
double totalRUs = 0;
long count = 0;
while (query.HasMoreResults)
{
FeedResponse <dynamic> feedResponse = await query.ExecuteNextAsync();
Console.WriteLine(feedResponse.RequestCharge);
totalRUs += feedResponse.RequestCharge;
IReadOnlyDictionary <string, QueryMetrics> partitionIdToQueryMetrics = feedResponse.QueryMetrics;
foreach (KeyValuePair <string, QueryMetrics> kvp in partitionIdToQueryMetrics)
{
string partitionId = kvp.Key;
QueryMetrics queryMetrics = kvp.Value;
Console.WriteLine("{0}: {1}", partitionId, queryMetrics);
}
IEnumerator <dynamic> docEnumerator = feedResponse.GetEnumerator();
while (docEnumerator.MoveNext())
{
count += (long)docEnumerator.Current;
}
}
var responsePayload = new ResponseContract
{
Count = count,
TotalRUs = totalRUs,
};
log.LogInformation("Count: {0}, Total RUs: {1}", count, totalRUs);
return(new OkObjectResult(JsonConvert.SerializeObject(responsePayload)));
}
catch (Exception error)
{
return(new ObjectResult(error.ToString())
{
StatusCode = 500
});
}
}
private async Task <DocumentProducer <T> > FetchAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
// TODO: This workflow could be simplified.
FetchResult exceptionFetchResult = null;
try
{
this.fetchSchedulingMetrics.Start();
this.fetchExecutionRangeAccumulator.BeginFetchRange();
FeedResponse <T> feedResponse = null;
double requestCharge = 0;
long responseLengthBytes = 0;
QueryMetrics queryMetrics = QueryMetrics.Zero;
do
{
int pageSize = (int)Math.Min(this.PageSize, (long)int.MaxValue);
Debug.Assert(pageSize >= 0, string.Format("pageSize was negative ... this.PageSize: {0}", this.PageSize));
using (DocumentServiceRequest request = this.createRequestFunc(this.CurrentBackendContinuationToken, pageSize))
{
retryPolicyInstance = retryPolicyInstance ?? this.createRetryPolicyFunc();
retryPolicyInstance.OnBeforeSendRequest(request);
// Custom backoff and retry
ExceptionDispatchInfo exception = null;
try
{
cancellationToken.ThrowIfCancellationRequested();
feedResponse = await this.executeRequestFunc(request, cancellationToken);
this.fetchExecutionRangeAccumulator.EndFetchRange(feedResponse.Count, Interlocked.Read(ref this.retries));
this.ActivityId = Guid.Parse(feedResponse.ActivityId);
}
catch (Exception ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
}
if (exception != null)
{
cancellationToken.ThrowIfCancellationRequested();
ShouldRetryResult shouldRetryResult = await retryPolicyInstance.ShouldRetryAsync(exception.SourceException, cancellationToken);
shouldRetryResult.ThrowIfDoneTrying(exception);
this.ScheduleFetch(retryPolicyInstance, shouldRetryResult.BackoffTime);
Interlocked.Increment(ref this.retries);
return(this);
}
requestCharge += feedResponse.RequestCharge;
responseLengthBytes += feedResponse.ResponseLengthBytes;
if (feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics] != null)
{
queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(this.retries, requestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
new List <Tuple <string, SchedulingTimeSpan> >()),
this.activityId);
// Reset the counters.
Interlocked.Exchange(ref this.retries, 0);
}
this.UpdateRequestContinuationToken(feedResponse.ResponseContinuation);
retryPolicyInstance = null;
this.numDocumentsFetched += feedResponse.Count;
}
}while (!this.FetchedAll && feedResponse.Count <= 0);
await this.CompleteFetchAsync(feedResponse, cancellationToken);
this.produceAsyncCompleteCallback(this, feedResponse.Count, requestCharge, queryMetrics, responseLengthBytes, cancellationToken);
}
catch (Exception ex)
{
DefaultTrace.TraceWarning(string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {1}, ActivityId {2} | DocumentProducer Id: {3}, Exception in FetchAsync: {4}",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
this.correlatedActivityId,
this.ActivityId,
this.targetRange.Id,
ex.Message));
exceptionFetchResult = new FetchResult(ExceptionDispatchInfo.Capture(ex));
}
finally
{
this.fetchSchedulingMetrics.Stop();
if (this.FetchedAll)
{
// One more callback to send the scheduling metrics
this.produceAsyncCompleteCallback(
producer: this,
size: 0,
resourceUnitUsage: 0,
queryMetrics: QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
QueryMetrics.Zero.ToDelimitedString(),
new ClientSideMetrics(
retries: 0,
requestCharge: 0,
fetchExecutionRanges: new List <FetchExecutionRange>(),
partitionSchedulingTimeSpans: new List <Tuple <string, SchedulingTimeSpan> > {
new Tuple <string, SchedulingTimeSpan>(this.targetRange.Id, this.fetchSchedulingMetrics.Elapsed)
}),
Guid.Empty),
responseLengthBytes: 0,
token: cancellationToken);
}
}
if (exceptionFetchResult != null)
{
this.UpdateRequestContinuationToken(this.CurrentBackendContinuationToken);
await this.itemBuffer.AddAsync(exceptionFetchResult, cancellationToken);
}
return(this);
}
public void Search(String searchString, SimpleSearchCriteria searchCriteria = null, bool interactive = true)
{
if (!EnsureSolutionOpen())
{
return;
}
try
{
var codeSearcher = new CodeSearcher(new IndexerSearcher());
if (String.IsNullOrEmpty(searchString))
{
return;
}
var solutionKey = ServiceLocator.ResolveOptional <SolutionKey>(); //no opened solution
if (solutionKey == null)
{
_searchResultListener.UpdateMessage("Sando searches only the currently open Solution. Please open a Solution and try again.");
return;
}
searchString = ExtensionPointsRepository.Instance.GetQueryRewriterImplementation().RewriteQuery(searchString);
PreRetrievalMetrics preMetrics = new PreRetrievalMetrics(ServiceLocator.Resolve <DocumentIndexer>().Reader, ServiceLocator.Resolve <Analyzer>());
LogEvents.PreSearch(this, preMetrics.MaxIdf(searchString), preMetrics.AvgIdf(searchString), preMetrics.AvgSqc(searchString), preMetrics.AvgVar(searchString));
LogEvents.PreSearchQueryAnalysis(this, QueryMetrics.ExamineQuery(searchString).ToString(), QueryMetrics.DiceCoefficient(QueryMetrics.SavedQuery, searchString));
QueryMetrics.SavedQuery = searchString;
var criteria = GetCriteria(searchString, searchCriteria);
var results = codeSearcher.Search(criteria, true).AsQueryable();
var resultsReorderer = ExtensionPointsRepository.Instance.GetResultsReordererImplementation();
results = resultsReorderer.ReorderSearchResults(results);
var returnString = new StringBuilder();
if (criteria.IsQueryReformed())
{
returnString.Append(criteria.GetQueryReformExplanation());
}
if (!results.Any())
{
returnString.Append("No results found. ");
}
else
{
returnString.Append(results.Count() + " results returned. ");
}
if (ServiceLocator.Resolve <InitialIndexingWatcher>().IsInitialIndexingInProgress())
{
returnString.Append("Sando is still performing its initial index of this project, results may be incomplete.");
}
_searchResultListener.Update(searchString, results);
_searchResultListener.UpdateMessage(returnString.ToString());
_searchResultListener.UpdateRecommendedQueries(criteria.GetRecommendedQueries());
LogEvents.PostSearch(this, results.Count(), criteria.NumberOfSearchResultsReturned, PostRetrievalMetrics.AvgScore(results.ToList()), PostRetrievalMetrics.StdDevScore(results.ToList()));
}
catch (Exception e)
{
_searchResultListener.UpdateMessage("Sando is experiencing difficulties. See log file for details.");
LogEvents.UISandoSearchingError(this, e);
}
}
public QueryMetricsTraceDatum(QueryMetrics queryMetrics)
{
this.QueryMetrics = queryMetrics ?? throw new ArgumentNullException(nameof(queryMetrics));
}
