public void Upgrade(EventUpgraderContext ctx) { if (ctx.Version > new Version(1, 0, 0, 850)) { return; } foreach (var doc in ctx.Documents.OfType <JObject>()) { var current = doc; while (current != null) { if (doc["ExtendedData"] is JObject extendedData) { if (extendedData["ExtraExceptionProperties"] != null) { extendedData.Rename("ExtraExceptionProperties", "__ExceptionInfo"); } if (extendedData["ExceptionInfo"] != null) { extendedData.Rename("ExceptionInfo", "__ExceptionInfo"); } if (extendedData["TraceInfo"] != null) { extendedData.Rename("TraceInfo", "TraceLog"); } } current = current["Inner"] as JObject; } } }
public void Upgrade(EventUpgraderContext ctx) { if (ctx.Version > new Version(1, 0, 0, 500)) { return; } foreach (var doc in ctx.Documents) { if (!(doc["ExceptionlessClientInfo"] is JObject clientInfo) || !clientInfo.HasValues || clientInfo["InstallDate"] == null) { return; } // This shouldn't hurt using DateTimeOffset to try and parse a date. It insures you won't lose any info. if (DateTimeOffset.TryParse(clientInfo["InstallDate"].ToString(), out var date)) { clientInfo.Remove("InstallDate"); clientInfo.Add("InstallDate", new JValue(date)); } else { clientInfo.Remove("InstallDate"); } } }
public void ParseErrors(string errorFilePath) { var json = File.ReadAllText(errorFilePath); var ctx = new EventUpgraderContext(json); _upgrader.Upgrade(ctx); ApprovalsUtility.VerifyFile(Path.ChangeExtension(errorFilePath, ".expected.json"), ctx.Documents.First.ToString()); var events = _parser.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0"); Assert.Equal(1, events.Count); }
public void ParseEvents(string errorFilePath) { var json = File.ReadAllText(errorFilePath); var ctx = new EventUpgraderContext(json); // TODO: Figure out what is wrong with 800000002e519522d83837a1 _eventUpgraderPluginManager.Upgrade(ctx); ApprovalsUtility.VerifyFile(Path.ChangeExtension(errorFilePath, ".expected.json"), ctx.Documents.First.ToString()); var events = _eventParserPluginManager.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0"); Assert.Equal(1, events.Count); }
public void ParseErrors(string errorFilePath) { string json = File.ReadAllText(errorFilePath); var ctx = new EventUpgraderContext(json); _upgrader.Upgrade(ctx); string expectedContent = File.ReadAllText(Path.ChangeExtension(errorFilePath, ".expected.json")); Assert.Equal(expectedContent, ctx.Documents.First.ToString()); var events = _parser.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0"); Assert.Single(events); }
public List <PersistentEvent> ParseEvents(string input, int apiVersion, string userAgent) { if (apiVersion != 1) { return(null); } try { var ctx = new EventUpgraderContext(input); _manager.Upgrade(ctx); return(ctx.Documents.FromJson <PersistentEvent>(_settings)); } catch (Exception ex) { Logger.Error().Message("Error parsing event: {0}", ex.Message).Exception(ex).Write(); return(null); } }
public async Task <List <PersistentEvent> > ParseEventsAsync(string input, int apiVersion, string userAgent) { if (apiVersion != 1) { return(null); } try { var ctx = new EventUpgraderContext(input); await _manager.UpgradeAsync(ctx).AnyContext(); return(ctx.Documents.FromJson <PersistentEvent>(_settings)); } catch (Exception ex) { _logger.Error(ex, "Error parsing event: {0}", ex.Message); return(null); } }
/// <summary> /// Runs all of the event upgrade plugins upgrade method. /// </summary> public void Upgrade(EventUpgraderContext context) { string metricPrefix = String.Concat(_metricPrefix, nameof(Upgrade).ToLower(), "."); foreach (var plugin in Plugins.Values.ToList()) { string metricName = String.Concat(metricPrefix, plugin.Name.ToLower()); try { _metricsClient.Time(() => plugin.Upgrade(context), metricName); } catch (Exception ex) { using (_logger.BeginScope(new Dictionary <string, object> { { "Context", context } })) _logger.LogError(ex, "Error calling upgrade in plugin {PluginName}: {Message}", plugin.Name, ex.Message); throw; } } }
public List <PersistentEvent> ParseEvents(string input, int apiVersion, string userAgent) { if (apiVersion != 1) { return(null); } try { var ctx = new EventUpgraderContext(input); _manager.Upgrade(ctx); var settings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore, ContractResolver = new ExtensionContractResolver() }; return(ctx.Documents.FromJson <PersistentEvent>(settings)); } catch (Exception ex) { Log.Error().Message("Error parsing event: {0}", ex.Message).Exception(ex).Write(); return(null); } }
public List <PersistentEvent> ParseEvents(string input, int apiVersion, string userAgent) { if (apiVersion != 1) { return(null); } try { var ctx = new EventUpgraderContext(input); _manager.Upgrade(ctx); var settings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore, ContractResolver = new ExtensionContractResolver() }; return(ctx.Documents.FromJson <PersistentEvent>(settings)); } catch (Exception ex) { ex.ToExceptionless().AddObject(input, "Error").AddObject(apiVersion, "Api Version").Submit(); return(null); } }
public void Upgrade(EventUpgraderContext ctx) { if (ctx.Version > new Version(1, 0, 0, 844)) { return; } foreach (var doc in ctx.Documents) { if (!(doc["RequestInfo"] is JObject requestInfo) || !requestInfo.HasValues) { return; } if (requestInfo["Cookies"] != null && requestInfo["Cookies"].HasValues) { if (requestInfo["Cookies"] is JObject cookies) { cookies.Remove(""); } } if (requestInfo["Form"] != null && requestInfo["Form"].HasValues) { if (requestInfo["Form"] is JObject form) { form.Remove(""); } } if (requestInfo["QueryString"] != null && requestInfo["QueryString"].HasValues) { if (requestInfo["QueryString"] is JObject queryString) { queryString.Remove(""); } } } }
public void Upgrade(EventUpgraderContext ctx) { if (ctx.Version != null) { return; } if (ctx.Documents.Count == 0 || !ctx.Documents.First().HasValues) { ctx.Version = new Version(); return; } var doc = ctx.Documents.First(); if (!(doc["ExceptionlessClientInfo"] is JObject clientInfo) || !clientInfo.HasValues || clientInfo["Version"] == null) { ctx.Version = new Version(); return; } if (clientInfo["Version"].ToString().Contains(" ")) { string version = clientInfo["Version"].ToString().Split(' ').First(); ctx.Version = new Version(version); return; } if (clientInfo["Version"].ToString().Contains("-")) { string version = clientInfo["Version"].ToString().Split('-').First(); ctx.Version = new Version(version); return; } // old version format ctx.Version = new Version(clientInfo["Version"].ToString()); }
protected override async Task <JobResult> RunInternalAsync(CancellationToken token) { OutputPublicIp(); QueueEntry <EventMigrationBatch> queueEntry = null; try { queueEntry = _queue.Dequeue(); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write(); int total = 0; var stopwatch = new Stopwatch(); stopwatch.Start(); var errorCollection = GetErrorCollection(); var knownStackIds = new List <string>(); var serializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore }; serializerSettings.AddModelConverters(); var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks)); var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList(); while (errors.Count > 0) { Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write(); var upgradedErrors = JArray.FromObject(errors); var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true); _eventUpgraderPluginManager.Upgrade(ctx); var upgradedEvents = upgradedErrors.FromJson <PersistentEvent>(serializerSettings); var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray(); if (stackIdsToCheck.Length > 0) { knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck)); } upgradedEvents.ForEach(e => { if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1)) { e.Date = DateTimeOffset.Now; } e.CreatedUtc = e.Date.ToUniversalTime().DateTime; if (!knownStackIds.Contains(e.StackId)) { // We haven't processed this stack id yet in this run. Check to see if this stack has already been imported.. e.IsFirstOccurrence = true; knownStackIds.Add(e.StackId); } var request = e.GetRequestInfo(); if (request != null) { e.AddRequestInfo(request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH)); } foreach (var ip in GetIpAddresses(e, request)) { var location = _geoIpResolver.ResolveIp(ip); if (location == null || !location.IsValid()) { continue; } e.Geo = location.ToString(); break; } if (e.Type == Event.KnownTypes.NotFound && request != null) { if (String.IsNullOrWhiteSpace(e.Source)) { e.Message = null; e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false); } return; } var error = e.GetError(); if (error == null) { Debugger.Break(); Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write(); return; } var stackingTarget = error.GetStackingTarget(); if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName())) { e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000); } var signature = new ErrorSignature(error); if (signature.SignatureInfo.Count <= 0) { return; } var targetInfo = new SettingsDictionary(signature.SignatureInfo); if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message")) { targetInfo["Message"] = error.GetStackingTarget().Error.Message; } error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo; }); try { _eventRepository.Add(upgradedEvents, sendNotification: false); } catch (Exception) { foreach (var persistentEvent in upgradedEvents) { try { _eventRepository.Add(persistentEvent, sendNotification: false); } catch (Exception ex) { //Debugger.Break(); Log.Error().Exception(ex).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write(); } } } total += upgradedEvents.Count; var lastId = upgradedEvents.Last().Id; _cache.Set("migration-errorid", lastId); errors = errorCollection.Find(Query.And(Query.GT(ErrorFieldNames.Id, ObjectId.Parse(lastId)), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks))) .SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)) .SetLimit(_batchSize).ToList(); } _cache.Set("migration-completedday", queueEntry.Value.EndTicks); queueEntry.Complete(); return(JobResult.Success); }
public void Upgrade(EventUpgraderContext ctx) { if (ctx.Version > new Version(2, 0)) { return; } foreach (var doc in ctx.Documents.OfType <JObject>()) { bool isNotFound = doc.GetPropertyStringValue("Code") == "404"; if (ctx.IsMigration) { doc.Rename("ErrorStackId", "StackId"); } else { doc.RenameOrRemoveIfNullOrEmpty("Id", "ReferenceId"); doc.Remove("OrganizationId"); doc.Remove("ProjectId"); doc.Remove("ErrorStackId"); } doc.RenameOrRemoveIfNullOrEmpty("OccurrenceDate", "Date"); doc.Remove("ExceptionlessClientInfo"); if (!doc.RemoveIfNullOrEmpty("Tags")) { var tags = doc.GetValue("Tags"); if (tags.Type == JTokenType.Array) { foreach (var tag in tags.ToList()) { string t = tag.ToString(); if (String.IsNullOrEmpty(t) || t.Length > 255) { tag.Remove(); } } } } doc.RenameOrRemoveIfNullOrEmpty("RequestInfo", "@request"); bool hasRequestInfo = doc["@request"] != null; if (!isNotFound) { doc.RenameOrRemoveIfNullOrEmpty("EnvironmentInfo", "@environment"); } else { doc.Remove("EnvironmentInfo"); } doc.RenameAll("ExtendedData", "Data"); var extendedData = doc.Property("Data") != null?doc.Property("Data").Value as JObject : null; if (extendedData != null) { if (!isNotFound) { extendedData.RenameOrRemoveIfNullOrEmpty("TraceLog", "@trace"); } else { extendedData.Remove("TraceLog"); } } if (isNotFound && hasRequestInfo) { doc.RemoveAll("Code", "Type", "Message", "Inner", "StackTrace", "TargetMethod", "Modules"); if (extendedData?["__ExceptionInfo"] != null) { extendedData.Remove("__ExceptionInfo"); } doc.Add("Type", new JValue("404")); } else { var error = new JObject(); if (!doc.RemoveIfNullOrEmpty("Message")) { error.Add("Message", doc["Message"].Value <string>()); } error.MoveOrRemoveIfNullOrEmpty(doc, "Code", "Type", "Inner", "StackTrace", "TargetMethod", "Modules"); // Copy the exception info from root extended data to the current errors extended data. if (extendedData?["__ExceptionInfo"] != null) { error.Add("Data", new JObject()); ((JObject)error["Data"]).MoveOrRemoveIfNullOrEmpty(extendedData, "__ExceptionInfo"); } string id = doc["Id"]?.Value <string>(); string projectId = doc["ProjectId"]?.Value <string>(); RenameAndValidateExtraExceptionProperties(id, error); var inner = error["Inner"] as JObject; while (inner != null) { RenameAndValidateExtraExceptionProperties(id, inner); inner = inner["Inner"] as JObject; } doc.Add("Type", new JValue(isNotFound ? "404" : "error")); doc.Add("@error", error); } string emailAddress = doc.GetPropertyStringValueAndRemove("UserEmail"); string userDescription = doc.GetPropertyStringValueAndRemove("UserDescription"); if (!String.IsNullOrWhiteSpace(emailAddress) && !String.IsNullOrWhiteSpace(userDescription)) { doc.Add("@user_description", JObject.FromObject(new UserDescription(emailAddress, userDescription))); } string identity = doc.GetPropertyStringValueAndRemove("UserName"); if (!String.IsNullOrWhiteSpace(identity)) { doc.Add("@user", JObject.FromObject(new UserInfo(identity))); } doc.RemoveAllIfNullOrEmpty("Data", "GenericArguments", "Parameters"); } }
private static int Main(string[] args) { OutputHeader(); // TODO: Hook up nlog to write to the console. Trace.Listeners.Add(new TextWriterTraceListener(Console.Out)); try { var ca = new ConsoleArguments(); if (Parser.ParseHelp(args)) { OutputUsageHelp(); PauseIfDebug(); return(0); } if (!Parser.ParseArguments(args, ca, Console.Error.WriteLine)) { OutputUsageHelp(); PauseIfDebug(); return(1); } Console.Clear(); OutputHeader(); const int BatchSize = 25; var container = CreateContainer(); var searchclient = container.GetInstance <IElasticClient>(); if (ca.DeleteExistingIndexes) { searchclient.DeleteIndex(i => i.AllIndices()); } var serializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore }; serializerSettings.AddModelConverters(); ISearchResponse <Stack> mostRecentStack = null; if (ca.Resume) { mostRecentStack = searchclient.Search <Stack>(d => d.AllIndices().Type(typeof(Stack)).SortDescending("_uid").Source(s => s.Include(e => e.Id)).Size(1)); } ISearchResponse <PersistentEvent> mostRecentEvent = null; if (ca.Resume) { mostRecentEvent = searchclient.Search <PersistentEvent>(d => d.AllIndices().Type(typeof(PersistentEvent)).SortDescending("_uid").Source(s => s.Include(e => e.Id)).Size(1)); } int total = 0; var stopwatch = new Stopwatch(); if (!ca.SkipStacks) { stopwatch.Start(); var errorStackCollection = GetErrorStackCollection(container); var query = mostRecentStack != null && mostRecentStack.Total > 0 ? Query.GT(ErrorStackFieldNames.Id, ObjectId.Parse(mostRecentStack.Hits.First().Id)) : Query.Null; var stacks = errorStackCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorStackFieldNames.Id)).SetLimit(BatchSize).ToList(); while (stacks.Count > 0) { stacks.ForEach(s => { s.Type = s.SignatureInfo != null && s.SignatureInfo.ContainsKey("HttpMethod") && s.SignatureInfo.ContainsKey("Path") ? "404" : "error"; if (s.Tags != null) { s.Tags.RemoveWhere(t => String.IsNullOrEmpty(t) || t.Length > 255); } if (s.Title != null && s.Title.Length > 1000) { s.Title = s.Title.Truncate(1000); } }); Console.SetCursorPosition(0, 4); Console.WriteLine("Migrating stacks {0:N0} total {1:N0}/s...", total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0); var response = searchclient.IndexMany(stacks, type: "stacks", index: ElasticSearchRepository <Stack> .StacksIndexName); if (!response.IsValid) { Debugger.Break(); } var lastId = stacks.Last().Id; stacks = errorStackCollection.Find(Query.GT(ErrorStackFieldNames.Id, ObjectId.Parse(lastId))).SetSortOrder(SortBy.Ascending(ErrorStackFieldNames.Id)).SetLimit(BatchSize).ToList(); total += stacks.Count; } } total = 0; stopwatch.Reset(); if (!ca.SkipErrors) { stopwatch.Start(); var eventUpgraderPluginManager = container.GetInstance <EventUpgraderPluginManager>(); var eventRepository = container.GetInstance <IEventRepository>(); var errorCollection = GetErrorCollection(container); //var json1 = JsonExtensions.ToJson(errorCollection.FindOneById(ObjectId.Parse("80000000e2cc694bd029a952")), Formatting.Indented); //var json2 = JsonExtensions.ToJson(errorCollection.FindOneById(ObjectId.Parse("80000000e2cc694bd029a953")), Formatting.Indented); //var ctx2 = new EventUpgraderContext("[" + json1 + "," + json2 + "]", new Version(1, 5), true); //eventUpgraderPluginManager.Upgrade(ctx2); //var ev2 = ctx2.Documents.ToObject<List<PersistentEvent>>(); //eventRepository.Add(ev2.First()); //eventRepository.Add(ev2.Last()); var query = mostRecentEvent != null && mostRecentEvent.Total > 0 ? Query.GT(ErrorFieldNames.Id, ObjectId.Parse(mostRecentEvent.Hits.First().Id)) : Query.Null; var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.Id)).SetLimit(BatchSize).ToList(); // TODO: When resuming, we need to get a list of existing stack ids from the events. var knownStackIds = new List <string>(); while (errors.Count > 0) { Console.SetCursorPosition(0, 5); Console.WriteLine("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0); var events = JArray.FromObject(errors); var ctx = new EventUpgraderContext(events, new Version(1, 5), true); eventUpgraderPluginManager.Upgrade(ctx); var ev = events.FromJson <PersistentEvent>(serializerSettings); ev.ForEach(e => { if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1)) { e.Date = DateTimeOffset.Now; } if (e.Type != Event.KnownTypes.Error) { return; } if (!knownStackIds.Contains(e.StackId)) { e.IsFirstOccurrence = true; knownStackIds.Add(e.StackId); } var request = e.GetRequestInfo(); if (request != null) { e.AddRequestInfo(request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH)); } var stacking = e.GetStackingTarget(); if (stacking != null && stacking.Method != null && !String.IsNullOrEmpty(stacking.Method.DeclaringTypeFullName)) { e.Source = stacking.Method.DeclaringTypeFullName.Truncate(2000); } }); try { eventRepository.Add(ev); } catch (Exception ex) { Debugger.Break(); } var lastId = ev.Last().Id; errors = errorCollection.Find(Query.GT(ErrorFieldNames.Id, ObjectId.Parse(lastId))).SetSortOrder(SortBy.Ascending(ErrorFieldNames.Id)).SetLimit(BatchSize).ToList(); total += events.Count; } } PauseIfDebug(); } catch (FileNotFoundException e) { Console.Error.WriteLine("{0} ({1})", e.Message, e.FileName); PauseIfDebug(); return(1); } catch (Exception e) { Console.Error.WriteLine(e.ToString()); PauseIfDebug(); return(1); } return(0); }
protected override async Task <JobResult> RunInternalAsync(CancellationToken token) { OutputPublicIp(); QueueEntry <EventMigrationBatch> queueEntry = null; try { queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1)); } catch (Exception ex) { if (!(ex is TimeoutException)) { Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write(); return(JobResult.FromException(ex)); } } if (queueEntry == null) { return(JobResult.Success); } Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write(); int total = 0; var stopwatch = new Stopwatch(); stopwatch.Start(); var errorCollection = GetErrorCollection(); var knownStackIds = new List <string>(); var userAgentParser = Parser.GetDefault(); var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks)); var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList(); int batch = 0; while (errors.Count > 0) { Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write(); var upgradedErrors = JArray.FromObject(errors); var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true); _eventUpgraderPluginManager.Upgrade(ctx); var upgradedEvents = upgradedErrors.FromJson <PersistentEvent>(_settings); var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray(); if (stackIdsToCheck.Length > 0) { knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck)); } upgradedEvents.ForEach(async e => { if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1)) { e.Date = DateTimeOffset.Now; } e.CreatedUtc = e.Date.ToUniversalTime().DateTime; // Truncate really large fields if (e.Message != null && e.Message.Length > 2000) { Log.Error().Project(e.ProjectId).Message("Event: {0} Message is Too Big: {1}", e.Id, e.Message.Length).Write(); e.Message = e.Message.Truncate(2000); } if (e.Source != null && e.Source.Length > 2000) { Log.Error().Project(e.ProjectId).Message("Event: {0} Source is Too Big: {1}", e.Id, e.Source.Length).Write(); e.Source = e.Source.Truncate(2000); } if (!knownStackIds.Contains(e.StackId)) { // We haven't processed this stack id yet in this run. Check to see if this stack has already been imported.. e.IsFirstOccurrence = true; knownStackIds.Add(e.StackId); } var request = e.GetRequestInfo(); if (request != null) { request = request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH); if (!String.IsNullOrEmpty(request.UserAgent)) { try { var info = userAgentParser.Parse(request.UserAgent); if (!String.Equals(info.UserAgent.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; if (!String.IsNullOrEmpty(info.UserAgent.Major)) { request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; } } if (!String.Equals(info.Device.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; } if (!String.Equals(info.OS.Family, "Other")) { request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; if (!String.IsNullOrEmpty(info.OS.Major)) { request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; } } request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider; } catch (Exception ex) { Log.Warn().Project(e.ProjectId).Message("Unable to parse user agent {0}. Exception: {1}", request.UserAgent, ex.Message).Write(); } } e.AddRequestInfo(request); } foreach (var ip in GetIpAddresses(e, request)) { var location = await _geoIpResolver.ResolveIpAsync(ip, token); if (location == null || !location.IsValid()) { continue; } e.Geo = location.ToString(); break; } if (e.Type == Event.KnownTypes.NotFound && request != null) { if (String.IsNullOrWhiteSpace(e.Source)) { e.Message = null; e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false); } return; } var error = e.GetError(); if (error == null) { Debugger.Break(); Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write(); return; } var stackingTarget = error.GetStackingTarget(); if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName())) { e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000); } var signature = new ErrorSignature(error); if (signature.SignatureInfo.Count <= 0) { return; } var targetInfo = new SettingsDictionary(signature.SignatureInfo); if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message")) { targetInfo["Message"] = stackingTarget.Error.Message; } error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo; }); Log.Info().Message("Saving events {0}-{1} {2:N0} total", errors.First().Id, errors.Last().Id, upgradedEvents.Count).Write(); try { _eventRepository.Add(upgradedEvents, sendNotification: false); } catch (Exception) { foreach (var persistentEvent in upgradedEvents) { try { _eventRepository.Add(persistentEvent, sendNotification: false); } catch (Exception ex) { //Debugger.Break(); Log.Error().Exception(ex).Project(persistentEvent.ProjectId).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write(); } } } batch++; total += upgradedEvents.Count; Log.Info().Message("Getting next batch of events").Write(); var sw = new Stopwatch(); sw.Start(); errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).SetSkip(_batchSize * batch).ToList(); sw.Stop(); Log.Info().Message("Finished getting next batch of events in {0}ms", sw.ElapsedMilliseconds).Write(); } Log.Info().Message("Finished processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write(); _cache.Set("migration-completedperiod", queueEntry.Value.EndTicks); queueEntry.Complete(); return(JobResult.Success); }