public async Task <ThreadApiResponse> CommentsForThread(SocialGistPostId post)
{
// Telemetry
telemetryClient.TrackEvent(TelemetryNames.SocialGist_Thread,
new Dictionary <string, string> {
{ "url", post.Url }
}
);
// Do the real work
var parameters = GetBaseParameters();
parameters.Add("url", post.Url);
parameters.Add("use_compression", true);
var baseUrl = "http://redditcomments.socialgist.com/";
var paramMessage = string.Join(",", parameters.Select(kvp => kvp.Key + ": " + kvp.Value.ToString()));
log.Info($"Request to: {baseUrl} with initial parameters: {paramMessage}.");
var uriBuilder = new UriBuilder(baseUrl);
uriBuilder.QueryParamsFromDictionary(parameters);
var result =
await httpClient.GetJsonAsync <ThreadApiResponse>(uriBuilder.Uri,
configuration.SocialGistApiRequestTimeout);
await CheckError(result);
return(result.Object);
}
public async Task <ThreadApiResponse> CommentsForThread(SocialGistPostId post)
{
// Telemetry
telemetryClient.TrackEvent(TelemetryNames.SocialGist_Thread,
new Dictionary <string, string> {
{ "url", post.Url }
}
);
// Do the real work
var parameters = GetBaseParameters();
parameters.Add("url", post.Url);
parameters.Add("use_compression", true);
var uriBuilder = new UriBuilder("http://redditcomments.socialgist.com/");
uriBuilder.QueryParamsFromDictionary(parameters);
var result =
await httpClient.GetJsonAsync <ThreadApiResponse>(uriBuilder.Uri,
configuration.SocialGistApiRequestTimeout);
await CheckError(result);
return(result.Object);
}
private void ApiHttpRequestAndLog(IHostingEnvironment env, ITelemetryClient telemetryClient)
{
try
{
telemetryClient.TrackEvent($"Start Up Api HttpRequest: {nameof(Environment.MachineName)} = {Environment.MachineName}");
telemetryClient.TrackEvent($"Start Up Api HttpRequest: {nameof(env.EnvironmentName)} = {env.EnvironmentName}");
telemetryClient.TrackEvent($"Start Up Api HttpRequest: {nameof(env.ApplicationName)} = {env.ApplicationName}");
var sw = new Stopwatch();
var uri = new Uri(Configuration["Tribal:APIAddress"]);
telemetryClient.TrackEvent($"Start Up Api HttpRequest: {nameof(uri.Host)} = {uri.Host}");
telemetryClient.TrackEvent($"Start Up Api HttpRequest: Address = {uri.OriginalString}");
using (var httpClient = new HttpClient())
{
telemetryClient.TrackEvent($"Start Up Api HttpRequest: Started at = {DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffffff")}");
sw.Start();
var statusCode = httpClient.GetAsync(uri).Result.StatusCode;
sw.Stop();
telemetryClient.TrackEvent($"Start Up Api HttpRequest: Ended at = {DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffffff")}");
telemetryClient.TrackEvent($"Start Up Api HttpRequest: Http Status Code = {statusCode}");
telemetryClient.TrackEvent($"Start Up Api HttpRequest: Time taken = {sw.ElapsedMilliseconds} ms");
}
}
catch (Exception e)
{
telemetryClient.TrackEvent($"Start Up Api HttpRequest: exception");
telemetryClient.TrackException(e);
}
}
public static void EnableEngine()
{
Logger.LogInfo("Enabling Engine...");
TelemetryClient.TrackEvent("EnableEngine", new Dictionary <string, string>(), new Dictionary <string, double>());
_engine.EnableEngine().RunSynchronously();
_efsWatcher.Enable();
}
public async Task <DeviceMetadata> GetMetadataAsync(string id)
{
if (FailedToDispatch)
{
return(null); // drop new messages if there is a dispatch problem
}
int current = 0;
while (true)
{
try
{
var request = new DeviceInfo()
{
Id = id
};
var response = await this.Client.GetMetadataAsync(request);
if (FailedToDispatch)
{
logger.LogDebug("gRPC client connected successfully.");
telemetryClient.TrackEvent($"gRPC client connected successfully from {System.Environment.MachineName}.");
FailedToDispatch = false;
}
return(response);
}
catch (RpcException ex)
{
if (ex.StatusCode == StatusCode.Unavailable)
{
logger.LogWarning("gRPC client failed to connect...");
telemetryClient.TrackEvent($"gRPC client failed to connect from {System.Environment.MachineName}.");
FailedToDispatch = true;
}
else
{
telemetryClient.TrackException(ex);
throw ex;
}
}
current += 1000;
if (current > CacheGrpcTimeout)
{
logger.LogError("gRPC client timed out.");
telemetryClient.TrackEvent($"gRPC client timed out from {System.Environment.MachineName}.");
throw new Exception("gRPC client timed out.");
}
await Task.Delay(1000);
}
}
/// <summary>
/// Generic method to download from blob and copy to app settings temp folder
/// </summary>
/// <param name="blobSynname"></param>
/// <param name="tempFilename"></param>
/// <param name="synonymFileName"></param>
/// <returns></returns>
public async Task <bool> DownloadFile(string blobFileName, string tempfolder)
{
//Set up the client
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
_storagename, //"ncsfindacourse", //"dfcdevfacostr",
_storagekey), true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
MemoryStream ms = new MemoryStream();
try
{
CloudBlobContainer container = blobClient.GetContainerReference(_containername);
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobFileName);
var existsAsync = blockBlob.ExistsAsync();
existsAsync.Wait();
if (existsAsync.Result)
{
using (var fileStream = System.IO.File.OpenWrite(System.IO.Path.Combine(tempfolder, blobFileName)))
{
await blockBlob.DownloadToStreamAsync(fileStream);
}
}
else
{
_telemetry.TrackEvent("DownloadFile", new Dictionary <string, string>()
{
{ "BlobFileName", blobFileName },
{ "Tempfolder", tempfolder }
});
return(false);
}
}
catch (StorageException stex)
{
_telemetry.TrackException(stex);
return(false);
}
catch (Exception ex)
{
_telemetry.TrackException(ex);
return(false);
}
return(true);
}
int IVsSolutionEvents.OnAfterOpenSolution(object pUnkReserved, int fNewSolution)
{
var port = Network.freeTcpPort();
Logger.LogInfo("Package: Selected {0} as the port for Engine server.", port);
TddStud10Host = new TddStud10HostProxy(port, FSharpOption <bool> .None);
TddStud10Host.Start();
var cfg = EngineConfigLoader.load(new EngineConfig(), FilePath.NewFilePath(GetSolutionPath()));
EngineLoader.Load(
this,
new EngineParams(
HostVersion,
cfg,
FilePath.NewFilePath(GetSolutionPath()),
DateTime.UtcNow
));
if (!cfg.IsDisabled)
{
EngineLoader.EnableEngine();
}
else
{
TelemetryClient.TrackEvent("EngineDisabledOnLoad", new Dictionary <string, string>(), new Dictionary <string, double>());
}
Logger.LogInfo("Triggering SnapshotGC on solution load.");
SnapshotGC.SweepAsync(FilePath.NewFilePath(Environment.ExpandEnvironmentVariables(cfg.SnapShotRoot)));
return(VSConstants.S_OK);
}
public async Task Process(SocialGistPostId post)
{
this.objectLogger.Log(post, "ThreadProcessor");
telemetryClient.TrackEvent(TelemetryNames.ThreadProcessor_ProcessThread);
var result = await socialgist.CommentsForThread(post);
var filteredPosts = new List <ResponsePost>()
{
result.Post
}.Where(x =>
!string.IsNullOrEmpty(x.Id) && !string.IsNullOrEmpty(x.Author) && x.CreatedUtc != 0);
var filteredComments = result.Comments.Where(x =>
!string.IsNullOrEmpty(x.Id) && !string.IsNullOrEmpty(x.Author) && x.CreatedUtc != 0);
if (filteredComments.Any() || filteredPosts.Any())
{
IEnumerable <IDocument> allDocuments = FilterAndWriteDocuments(filteredPosts.ToList().AsReadOnly(), filteredComments.ToList().AsReadOnly()).ToList();
// Perform all actions that need to be run over every post and comment
foreach (var processor in this.documentProcessors)
{
processor.ProcessDocuments(allDocuments);
}
}
else
{
log.Info($"No thread found with url {post.Url}");
}
}
private static RunStepResult DiscoverSequencePoints(IRunExecutorHost host, RunStartParams rsp, RunStepInfo rsi)
{
var sequencePoint = Instrumentation.GenerateSequencePointInfo(host, rsp);
if (sequencePoint != null)
{
sequencePoint.Serialize(rsp.DataFiles.SequencePointStore);
}
var totalSP = sequencePoint.Values.Aggregate(0, (acc, e) => acc + e.Count);
TelemetryClient.TrackEvent(rsi.name.Item, new Dictionary <string, string>(), new Dictionary <string, double> {
{ "SequencePointCount", totalSP }
});
return(RunStepStatus.Succeeded.ToRSR(RunData.NewSequencePoints(sequencePoint), "Binaries Instrumented - which ones - TBD"));
}
protected virtual void TrackEvent(string eventName, Action <Dictionary <string, string> > addProperties)
{
var telemetryProperties = new Dictionary <string, string>();
addProperties(telemetryProperties);
_telemetryClient.TrackEvent(eventName, telemetryProperties, metrics: null);
}
public bool IsSuccessfulResult <T>(IResult <T> result, ITelemetryClient telemetryClient,
string methodName, string value, DateTime dtStart)
{
if (result.HasValue && result.IsSuccess && !result.IsFailure)
{
//ASB TODO This is never used? I am commenting it out.
//var regionsOnly = result.Value.Items.Where(x => x.Opportunity.HasRegion);
telemetryClient.TrackEvent(
$"{methodName} for: {value} took: {(DateTime.Now - dtStart).TotalMilliseconds.ToString()} ms.");
}
else
{
telemetryClient.TrackEvent($"{methodName}: Invalid.");
return(false);
}
return(true);
}
private async Task UpdateLastAcceptedCertificatesRead(string plant, string code)
{
var result = await _mediator.Send(new UpdateDateTimeSettingCommand(code, TimeService.UtcNow));
if (result.ResultType != ServiceResult.ResultType.Ok)
{
_logger.LogWarning(
$"Autotransfer tags functions failed. Could not update {code}. ResultType {result.ResultType}");
_telemetryClient.TrackEvent("Synchronization Status",
new Dictionary <string, string>
{
{ "Status", "Failed" },
{ "Plant", plant },
{ "Type", "Autotransfer tags" },
{ "ResultType", result.ResultType.ToString() }
});
}
}
public static void TrackCollectorCacheMiss(this ITelemetryClient telemetryClient, Repository repository, string recordType, string recordValue)
{
Dictionary <string, string> properties = new Dictionary <string, string>()
{
{ "RecordType", recordType },
{ "RecordValue", recordValue },
{ "RepositoryName", repository.RepositoryName },
{ "RepositoryId", repository.RepositoryId.ToString() },
{ "OrganizationLogin", repository.OrganizationLogin },
{ "OrganizationId", repository.OrganizationId.ToString() },
};
telemetryClient.TrackEvent("CollectorCacheMiss", properties);
}
private static void SendSessionEndEvent(ITelemetryClient telemetryClient, DateTime functionStartDate, string outputPaths, Dictionary <string, string> sessionStartProperties, bool success)
{
DateTime endTime = DateTime.UtcNow;
TimeSpan sessionDuration = endTime - functionStartDate;
Dictionary <string, string> sessionEndProperties = new Dictionary <string, string>(sessionStartProperties)
{
{ "FunctionEndDate", $"{endTime:O}" },
{ "FunctionDuration", sessionDuration.ToString() },
{ "OutputPaths", outputPaths },
{ "Success", success.ToString() },
};
telemetryClient.TrackEvent("SessionEnd", sessionEndProperties);
}
private void ExecuteViewTddStud10Logs(object sender, EventArgs e)
{
var pkgPath = Assembly.getAssemblyLocation(System.Reflection.Assembly.GetExecutingAssembly());
var loggerPath = Path.Combine(pkgPath, string.Format(@"rtlogs\TddStud10.RealTimeEtwListener{0}.exe", DFizer.isDF() ? ".DF" : ""));
try
{
ProcessStartInfo info = new ProcessStartInfo(loggerPath);
info.UseShellExecute = true;
info.Verb = "runas";
Process.Start(info);
}
catch
{
Services
.GetService <SVsUIShell, IVsUIShell>()
.DisplayMessageBox(
Properties.Resources.ProductTitle,
string.Format(Properties.Resources.UnableToStartLogger, loggerPath));
}
TelemetryClient.TrackEvent("ViewTddStud10Logs", new Dictionary <string, string>(), new Dictionary <string, double>());
}
public void CompleteListLeaguesTimer(int?recordCount = null)
{
var elapsed = this.GetElapsedMilliseconds(EventNames.LeaguesListTiming.ToString());
var properties = BuildProperties();
var metrics = BuildMetrics(elapsed);
if (recordCount.HasValue)
{
metrics.Add("Records Found", recordCount.Value);
}
_telemetryClient.TrackEvent(EventNames.LeaguesListTiming.ToString(), properties, metrics);
}
private void ProcessMcPkgEvent(string messageJson)
{
var mcPkgEvent = JsonSerializer.Deserialize <McPkgTopic>(messageJson);
if (string.IsNullOrWhiteSpace(mcPkgEvent.Plant) ||
string.IsNullOrWhiteSpace(mcPkgEvent.CommPkgNo) ||
string.IsNullOrWhiteSpace(mcPkgEvent.McPkgNo) ||
(string.IsNullOrWhiteSpace(mcPkgEvent.McPkgNoOld) != (string.IsNullOrWhiteSpace(mcPkgEvent.CommPkgNoOld))))
{
throw new Exception($"Unable to deserialize JSON to McPkgEvent {messageJson}");
}
_telemetryClient.TrackEvent(IpoBusReceiverTelemetryEvent,
new Dictionary <string, string>
{
{ PcsServiceBusTelemetryConstants.Event, McPkgTopic.TopicName },
public static void TrackCollectorCacheHit(this ITelemetryClient telemetryClient, Repository repository, string recordType, string recordValue, string currentCollectorIdentifier, string cachingCollectorIdentifier, string decision)
{
Dictionary <string, string> properties = new Dictionary <string, string>()
{
{ "Decision", decision },
{ "CurrentCollectorIdentifier", currentCollectorIdentifier },
{ "CachingCollectorIdentifier", cachingCollectorIdentifier },
{ "RecordType", recordType },
{ "RecordValue", recordValue },
{ "RepositoryName", repository.RepositoryName },
{ "RepositoryId", repository.RepositoryId.ToString() },
{ "OrganizationLogin", repository.OrganizationLogin },
{ "OrganizationId", repository.OrganizationId.ToString() },
};
telemetryClient.TrackEvent("CollectorCacheHit", properties);
}
/// <summary>
/// Message receive handler, this method will increment a counter that keeps track of the
/// amount of messages that have been received
/// </summary>
public void HandleEvent(TelemetryEventType telemetryEventType, Guid?messageId, IMessageDiagnostics message)
{
var elapsedSpan = new TimeSpan(DateTime.UtcNow.Ticks - _createdAt.Ticks);
if (telemetryEventType == TelemetryEventType.PostConsume)
{
Interlocked.Increment(ref _postConsumeMessagesCount);
}
_telemetryClient.TrackEvent(
telemetryEventType,
messageId ?? Guid.Empty,
new
{
CreatedAt = elapsedSpan.Ticks,
Message = new { message.Id }
}
);
}
private void TrackEventCore(string eventName,
Dictionary <string, string> data,
Dictionary <string, double> metrics)
{
_client.TrackEvent(eventName, data, metrics);
}
public async Task <IHttpJsonResponseMessage <T> > GetJsonAsync <T>(
Uri requestUri,
TimeSpan?requestTimeout = null,
AuthenticationHeaderValue authenticationHeaderValue = null)
{
// SocialGist only accepts TLS 1.1 or higher. AzureFunctions (on Azure) default to SSL3 or TLS
// AzureFunctions runing locally use defaults that work with SocialGist but remote Azure Functions do not.
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;
log.Verbose($"Retrieving URL using protocol: {ServicePointManager.SecurityProtocol}");
telemetryClient.TrackEvent(TelemetryNames.HTTP_Get, new Dictionary <string, string> {
{ "URL", requestUri.ToString() }
});
var handler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
using (var client = new System.Net.Http.HttpClient(handler))
{
if (requestTimeout.HasValue)
{
client.Timeout = requestTimeout.Value;
}
if (authenticationHeaderValue != null)
{
client.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
}
string data = "";
HttpResponseMessage response = null;
int retryCount = 0;
while (retryCount < RETRY_COUNT)
{
try
{
telemetryClient.TrackEvent(TelemetryNames.HTTP_Try, new Dictionary <string, string> {
{ "URL", requestUri?.ToString() }, { "try number", retryCount.ToString() }
});
log.Verbose($"Retrieving URL {requestUri} on attempt {retryCount}");
// Track telemetry for each host query
using (var tracker = telemetryClient.StartTrackDependency(requestUri.Host, requestUri.AbsolutePath, "HTTP"))
{
tracker.Properties.Add("Request URL", requestUri?.ToString());
tracker.Properties.Add("Try attempt", retryCount.ToString());
try
{
response = await client.GetAsync(requestUri);
}
catch (TaskCanceledException)
{
tracker.IsSuccess = false;
tracker.ResultCode = "Task Canceled (HTTP Timeout)";
throw;
}
catch (HttpRequestException e)
{
tracker.IsSuccess = false;
tracker.ResultCode = $"HTTPException: {e.Message}";
throw;
}
tracker.IsSuccess = response.IsSuccessStatusCode;
tracker.ResultCode = response.StatusCode.ToString();
// Get data after setting telemetry status codes incase this errors out.
data = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
// Do not store this data unless the request failed. This data can get very large very fast.
tracker.Properties.Add("Server Response", data);
}
}
if (response.IsSuccessStatusCode)
{
this.objectLogger.Log(data, "HttpClient", requestUri.ToString());
// Successfully completed request
break;
}
else
{
telemetryClient.TrackEvent(TelemetryNames.HTTP_Error, new Dictionary <string, string> {
{ "URL", requestUri?.ToString() },
{ "StatusCode", response?.StatusCode.ToString() }
});
retryCount = HandleFailure(retryCount, requestUri, response);
}
}
catch (Exception e)
{
retryCount++;
telemetryClient.TrackEvent(TelemetryNames.HTTP_Error, new Dictionary <string, string> {
{ "URL", requestUri?.ToString() },
{ "StatusCode", response?.StatusCode.ToString() },
{ "Exception", e.ToString() }
});
if (retryCount < RETRY_COUNT)
{
log.Error($"Error executing web request for URL {requestUri} on attempt {retryCount}. Retrying.", e);
}
else
{
log.Error($"Error executing web request for URI {requestUri} on attempt {retryCount}. NOT retrying", e);
// Rethrow the exception to fail the current job
throw;
}
}
// Do not wait after the final try.
if (retryCount < RETRY_COUNT)
{
// Request failed. Wait a random time and try again. Random() uses a time-dependent seed.
int wait = new Random().Next((int)minWaitBeforeRetry.TotalMilliseconds, (int)maxWaitBeforeRetry.TotalMilliseconds);
log.Verbose($"Waiting {wait} milliseconds before retrying");
Thread.Sleep(wait);
}
}
log.Verbose($"URL {requestUri} retrieved.");
var metric = new MetricTelemetry();
metric.Name = TelemetryNames.HTTP_RetryCount;
metric.Sum = retryCount;
metric.Timestamp = DateTime.Now;
metric.Properties.Add("Domain", requestUri.Host);
telemetryClient.TrackMetric(metric);
try
{
var obj = JsonConvert.DeserializeObject <T>(data);
log.Verbose($"Result deserialized from URL {requestUri}");
return(new HttpJsonResponseMessage <T>
{
Object = obj,
ResponseMessage = response
});
}
catch (Exception e)
{
telemetryClient.TrackEvent(TelemetryNames.HTTP_JSON_Error, new Dictionary <string, string>
{
{ "URL", requestUri?.ToString() },
{ "StatusCode", response?.StatusCode.ToString() },
{ "Exception", e.ToString() },
{ "Response", data }
});
// Log and rethrow the exception
log.Error($"Error deserializing result from URL {requestUri}", e);
// Rethrow the exception to fail the current job
throw;
}
}
}
private void ApiPingAndLog(IHostingEnvironment env, ITelemetryClient telemetryClient)
{
Ping pinger = null;
try
{
telemetryClient.TrackEvent($"Start Up Api HttpRequest: {nameof(Environment.MachineName)} = {Environment.MachineName}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(env.EnvironmentName)} = {env.EnvironmentName}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(env.ApplicationName)} = {env.ApplicationName}");
var uri = new Uri(Configuration["Tribal:APIAddress"]);
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(uri.Host)} = {uri.Host}");
pinger = new Ping();
var reply = pinger.Send(uri.Host);
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(reply.Address)} = {reply?.Address}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(reply.Options.DontFragment)} = {reply?.Options?.DontFragment}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(reply.Options.Ttl)} = {reply?.Options?.Ttl}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(reply.RoundtripTime)} = {reply?.RoundtripTime}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(reply.Buffer.Length)} = {reply?.Buffer?.Length}");
telemetryClient.TrackEvent($"Start Up Api Ping: {nameof(reply.Status)} = {reply?.Status}");
}
catch (Exception e)
{
telemetryClient.TrackEvent($"Start Up Api Ping: exception");
telemetryClient.TrackException(e);
}
finally
{
if (pinger != null)
{
pinger.Dispose();
}
}
}
private void TrackResponsibleEvent(ResponsibleTopic responsibleEvent) => _telemetryClient.TrackEvent(PreservationBusReceiverTelemetryEvent,