public void Upgrade(EventUpgraderContext ctx)
{
if (ctx.Version > new Version(1, 0, 0, 850))
{
return;
}
foreach (var doc in ctx.Documents.OfType <JObject>())
{
var current = doc;
while (current != null)
{
if (doc["ExtendedData"] is JObject extendedData)
{
if (extendedData["ExtraExceptionProperties"] != null)
{
extendedData.Rename("ExtraExceptionProperties", "__ExceptionInfo");
}
if (extendedData["ExceptionInfo"] != null)
{
extendedData.Rename("ExceptionInfo", "__ExceptionInfo");
}
if (extendedData["TraceInfo"] != null)
{
extendedData.Rename("TraceInfo", "TraceLog");
}
}
current = current["Inner"] as JObject;
}
}
}
public void Upgrade(EventUpgraderContext ctx)
{
if (ctx.Version > new Version(1, 0, 0, 500))
{
return;
}
foreach (var doc in ctx.Documents)
{
if (!(doc["ExceptionlessClientInfo"] is JObject clientInfo) || !clientInfo.HasValues || clientInfo["InstallDate"] == null)
{
return;
}
// This shouldn't hurt using DateTimeOffset to try and parse a date. It insures you won't lose any info.
if (DateTimeOffset.TryParse(clientInfo["InstallDate"].ToString(), out var date))
{
clientInfo.Remove("InstallDate");
clientInfo.Add("InstallDate", new JValue(date));
}
else
{
clientInfo.Remove("InstallDate");
}
}
}
public void ParseErrors(string errorFilePath)
{
var json = File.ReadAllText(errorFilePath);
var ctx = new EventUpgraderContext(json);
_upgrader.Upgrade(ctx);
ApprovalsUtility.VerifyFile(Path.ChangeExtension(errorFilePath, ".expected.json"), ctx.Documents.First.ToString());
var events = _parser.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0");
Assert.Equal(1, events.Count);
}
public void ParseEvents(string errorFilePath)
{
var json = File.ReadAllText(errorFilePath);
var ctx = new EventUpgraderContext(json);
// TODO: Figure out what is wrong with 800000002e519522d83837a1
_eventUpgraderPluginManager.Upgrade(ctx);
ApprovalsUtility.VerifyFile(Path.ChangeExtension(errorFilePath, ".expected.json"), ctx.Documents.First.ToString());
var events = _eventParserPluginManager.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0");
Assert.Equal(1, events.Count);
}
public void ParseErrors(string errorFilePath)
{
string json = File.ReadAllText(errorFilePath);
var ctx = new EventUpgraderContext(json);
_upgrader.Upgrade(ctx);
string expectedContent = File.ReadAllText(Path.ChangeExtension(errorFilePath, ".expected.json"));
Assert.Equal(expectedContent, ctx.Documents.First.ToString());
var events = _parser.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0");
Assert.Single(events);
}
public List <PersistentEvent> ParseEvents(string input, int apiVersion, string userAgent)
{
if (apiVersion != 1)
{
return(null);
}
try {
var ctx = new EventUpgraderContext(input);
_manager.Upgrade(ctx);
return(ctx.Documents.FromJson <PersistentEvent>(_settings));
} catch (Exception ex) {
Logger.Error().Message("Error parsing event: {0}", ex.Message).Exception(ex).Write();
return(null);
}
}
public async Task <List <PersistentEvent> > ParseEventsAsync(string input, int apiVersion, string userAgent)
{
if (apiVersion != 1)
{
return(null);
}
try {
var ctx = new EventUpgraderContext(input);
await _manager.UpgradeAsync(ctx).AnyContext();
return(ctx.Documents.FromJson <PersistentEvent>(_settings));
} catch (Exception ex) {
_logger.Error(ex, "Error parsing event: {0}", ex.Message);
return(null);
}
}
/// <summary>
/// Runs all of the event upgrade plugins upgrade method.
/// </summary>
public void Upgrade(EventUpgraderContext context)
{
string metricPrefix = String.Concat(_metricPrefix, nameof(Upgrade).ToLower(), ".");
foreach (var plugin in Plugins.Values.ToList())
{
string metricName = String.Concat(metricPrefix, plugin.Name.ToLower());
try {
_metricsClient.Time(() => plugin.Upgrade(context), metricName);
}
catch (Exception ex) {
using (_logger.BeginScope(new Dictionary <string, object> {
{ "Context", context }
}))
_logger.LogError(ex, "Error calling upgrade in plugin {PluginName}: {Message}", plugin.Name, ex.Message);
throw;
}
}
}
public List <PersistentEvent> ParseEvents(string input, int apiVersion, string userAgent)
{
if (apiVersion != 1)
{
return(null);
}
try {
var ctx = new EventUpgraderContext(input);
_manager.Upgrade(ctx);
var settings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore,
ContractResolver = new ExtensionContractResolver()
};
return(ctx.Documents.FromJson <PersistentEvent>(settings));
} catch (Exception ex) {
Log.Error().Message("Error parsing event: {0}", ex.Message).Exception(ex).Write();
return(null);
}
}
public List <PersistentEvent> ParseEvents(string input, int apiVersion, string userAgent)
{
if (apiVersion != 1)
{
return(null);
}
try {
var ctx = new EventUpgraderContext(input);
_manager.Upgrade(ctx);
var settings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore,
ContractResolver = new ExtensionContractResolver()
};
return(ctx.Documents.FromJson <PersistentEvent>(settings));
} catch (Exception ex) {
ex.ToExceptionless().AddObject(input, "Error").AddObject(apiVersion, "Api Version").Submit();
return(null);
}
}
public void Upgrade(EventUpgraderContext ctx)
{
if (ctx.Version > new Version(1, 0, 0, 844))
{
return;
}
foreach (var doc in ctx.Documents)
{
if (!(doc["RequestInfo"] is JObject requestInfo) || !requestInfo.HasValues)
{
return;
}
if (requestInfo["Cookies"] != null && requestInfo["Cookies"].HasValues)
{
if (requestInfo["Cookies"] is JObject cookies)
{
cookies.Remove("");
}
}
if (requestInfo["Form"] != null && requestInfo["Form"].HasValues)
{
if (requestInfo["Form"] is JObject form)
{
form.Remove("");
}
}
if (requestInfo["QueryString"] != null && requestInfo["QueryString"].HasValues)
{
if (requestInfo["QueryString"] is JObject queryString)
{
queryString.Remove("");
}
}
}
}
public void Upgrade(EventUpgraderContext ctx)
{
if (ctx.Version != null)
{
return;
}
if (ctx.Documents.Count == 0 || !ctx.Documents.First().HasValues)
{
ctx.Version = new Version();
return;
}
var doc = ctx.Documents.First();
if (!(doc["ExceptionlessClientInfo"] is JObject clientInfo) || !clientInfo.HasValues || clientInfo["Version"] == null)
{
ctx.Version = new Version();
return;
}
if (clientInfo["Version"].ToString().Contains(" "))
{
string version = clientInfo["Version"].ToString().Split(' ').First();
ctx.Version = new Version(version);
return;
}
if (clientInfo["Version"].ToString().Contains("-"))
{
string version = clientInfo["Version"].ToString().Split('-').First();
ctx.Version = new Version(version);
return;
}
// old version format
ctx.Version = new Version(clientInfo["Version"].ToString());
}
protected override async Task <JobResult> RunInternalAsync(CancellationToken token)
{
OutputPublicIp();
QueueEntry <EventMigrationBatch> queueEntry = null;
try {
queueEntry = _queue.Dequeue();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
int total = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
var errorCollection = GetErrorCollection();
var knownStackIds = new List <string>();
var serializerSettings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore
};
serializerSettings.AddModelConverters();
var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks));
var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList();
while (errors.Count > 0)
{
Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write();
var upgradedErrors = JArray.FromObject(errors);
var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true);
_eventUpgraderPluginManager.Upgrade(ctx);
var upgradedEvents = upgradedErrors.FromJson <PersistentEvent>(serializerSettings);
var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray();
if (stackIdsToCheck.Length > 0)
{
knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck));
}
upgradedEvents.ForEach(e => {
if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1))
{
e.Date = DateTimeOffset.Now;
}
e.CreatedUtc = e.Date.ToUniversalTime().DateTime;
if (!knownStackIds.Contains(e.StackId))
{
// We haven't processed this stack id yet in this run. Check to see if this stack has already been imported..
e.IsFirstOccurrence = true;
knownStackIds.Add(e.StackId);
}
var request = e.GetRequestInfo();
if (request != null)
{
e.AddRequestInfo(request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH));
}
foreach (var ip in GetIpAddresses(e, request))
{
var location = _geoIpResolver.ResolveIp(ip);
if (location == null || !location.IsValid())
{
continue;
}
e.Geo = location.ToString();
break;
}
if (e.Type == Event.KnownTypes.NotFound && request != null)
{
if (String.IsNullOrWhiteSpace(e.Source))
{
e.Message = null;
e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false);
}
return;
}
var error = e.GetError();
if (error == null)
{
Debugger.Break();
Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write();
return;
}
var stackingTarget = error.GetStackingTarget();
if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName()))
{
e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000);
}
var signature = new ErrorSignature(error);
if (signature.SignatureInfo.Count <= 0)
{
return;
}
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message"))
{
targetInfo["Message"] = error.GetStackingTarget().Error.Message;
}
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
});
try {
_eventRepository.Add(upgradedEvents, sendNotification: false);
} catch (Exception) {
foreach (var persistentEvent in upgradedEvents)
{
try {
_eventRepository.Add(persistentEvent, sendNotification: false);
} catch (Exception ex) {
//Debugger.Break();
Log.Error().Exception(ex).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write();
}
}
}
total += upgradedEvents.Count;
var lastId = upgradedEvents.Last().Id;
_cache.Set("migration-errorid", lastId);
errors = errorCollection.Find(Query.And(Query.GT(ErrorFieldNames.Id, ObjectId.Parse(lastId)), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks)))
.SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC))
.SetLimit(_batchSize).ToList();
}
_cache.Set("migration-completedday", queueEntry.Value.EndTicks);
queueEntry.Complete();
return(JobResult.Success);
}
public void Upgrade(EventUpgraderContext ctx)
{
if (ctx.Version > new Version(2, 0))
{
return;
}
foreach (var doc in ctx.Documents.OfType <JObject>())
{
bool isNotFound = doc.GetPropertyStringValue("Code") == "404";
if (ctx.IsMigration)
{
doc.Rename("ErrorStackId", "StackId");
}
else
{
doc.RenameOrRemoveIfNullOrEmpty("Id", "ReferenceId");
doc.Remove("OrganizationId");
doc.Remove("ProjectId");
doc.Remove("ErrorStackId");
}
doc.RenameOrRemoveIfNullOrEmpty("OccurrenceDate", "Date");
doc.Remove("ExceptionlessClientInfo");
if (!doc.RemoveIfNullOrEmpty("Tags"))
{
var tags = doc.GetValue("Tags");
if (tags.Type == JTokenType.Array)
{
foreach (var tag in tags.ToList())
{
string t = tag.ToString();
if (String.IsNullOrEmpty(t) || t.Length > 255)
{
tag.Remove();
}
}
}
}
doc.RenameOrRemoveIfNullOrEmpty("RequestInfo", "@request");
bool hasRequestInfo = doc["@request"] != null;
if (!isNotFound)
{
doc.RenameOrRemoveIfNullOrEmpty("EnvironmentInfo", "@environment");
}
else
{
doc.Remove("EnvironmentInfo");
}
doc.RenameAll("ExtendedData", "Data");
var extendedData = doc.Property("Data") != null?doc.Property("Data").Value as JObject : null;
if (extendedData != null)
{
if (!isNotFound)
{
extendedData.RenameOrRemoveIfNullOrEmpty("TraceLog", "@trace");
}
else
{
extendedData.Remove("TraceLog");
}
}
if (isNotFound && hasRequestInfo)
{
doc.RemoveAll("Code", "Type", "Message", "Inner", "StackTrace", "TargetMethod", "Modules");
if (extendedData?["__ExceptionInfo"] != null)
{
extendedData.Remove("__ExceptionInfo");
}
doc.Add("Type", new JValue("404"));
}
else
{
var error = new JObject();
if (!doc.RemoveIfNullOrEmpty("Message"))
{
error.Add("Message", doc["Message"].Value <string>());
}
error.MoveOrRemoveIfNullOrEmpty(doc, "Code", "Type", "Inner", "StackTrace", "TargetMethod", "Modules");
// Copy the exception info from root extended data to the current errors extended data.
if (extendedData?["__ExceptionInfo"] != null)
{
error.Add("Data", new JObject());
((JObject)error["Data"]).MoveOrRemoveIfNullOrEmpty(extendedData, "__ExceptionInfo");
}
string id = doc["Id"]?.Value <string>();
string projectId = doc["ProjectId"]?.Value <string>();
RenameAndValidateExtraExceptionProperties(id, error);
var inner = error["Inner"] as JObject;
while (inner != null)
{
RenameAndValidateExtraExceptionProperties(id, inner);
inner = inner["Inner"] as JObject;
}
doc.Add("Type", new JValue(isNotFound ? "404" : "error"));
doc.Add("@error", error);
}
string emailAddress = doc.GetPropertyStringValueAndRemove("UserEmail");
string userDescription = doc.GetPropertyStringValueAndRemove("UserDescription");
if (!String.IsNullOrWhiteSpace(emailAddress) && !String.IsNullOrWhiteSpace(userDescription))
{
doc.Add("@user_description", JObject.FromObject(new UserDescription(emailAddress, userDescription)));
}
string identity = doc.GetPropertyStringValueAndRemove("UserName");
if (!String.IsNullOrWhiteSpace(identity))
{
doc.Add("@user", JObject.FromObject(new UserInfo(identity)));
}
doc.RemoveAllIfNullOrEmpty("Data", "GenericArguments", "Parameters");
}
}
private static int Main(string[] args)
{
OutputHeader();
// TODO: Hook up nlog to write to the console.
Trace.Listeners.Add(new TextWriterTraceListener(Console.Out));
try {
var ca = new ConsoleArguments();
if (Parser.ParseHelp(args))
{
OutputUsageHelp();
PauseIfDebug();
return(0);
}
if (!Parser.ParseArguments(args, ca, Console.Error.WriteLine))
{
OutputUsageHelp();
PauseIfDebug();
return(1);
}
Console.Clear();
OutputHeader();
const int BatchSize = 25;
var container = CreateContainer();
var searchclient = container.GetInstance <IElasticClient>();
if (ca.DeleteExistingIndexes)
{
searchclient.DeleteIndex(i => i.AllIndices());
}
var serializerSettings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore
};
serializerSettings.AddModelConverters();
ISearchResponse <Stack> mostRecentStack = null;
if (ca.Resume)
{
mostRecentStack = searchclient.Search <Stack>(d => d.AllIndices().Type(typeof(Stack)).SortDescending("_uid").Source(s => s.Include(e => e.Id)).Size(1));
}
ISearchResponse <PersistentEvent> mostRecentEvent = null;
if (ca.Resume)
{
mostRecentEvent = searchclient.Search <PersistentEvent>(d => d.AllIndices().Type(typeof(PersistentEvent)).SortDescending("_uid").Source(s => s.Include(e => e.Id)).Size(1));
}
int total = 0;
var stopwatch = new Stopwatch();
if (!ca.SkipStacks)
{
stopwatch.Start();
var errorStackCollection = GetErrorStackCollection(container);
var query = mostRecentStack != null && mostRecentStack.Total > 0 ? Query.GT(ErrorStackFieldNames.Id, ObjectId.Parse(mostRecentStack.Hits.First().Id)) : Query.Null;
var stacks = errorStackCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorStackFieldNames.Id)).SetLimit(BatchSize).ToList();
while (stacks.Count > 0)
{
stacks.ForEach(s => {
s.Type = s.SignatureInfo != null && s.SignatureInfo.ContainsKey("HttpMethod") && s.SignatureInfo.ContainsKey("Path") ? "404" : "error";
if (s.Tags != null)
{
s.Tags.RemoveWhere(t => String.IsNullOrEmpty(t) || t.Length > 255);
}
if (s.Title != null && s.Title.Length > 1000)
{
s.Title = s.Title.Truncate(1000);
}
});
Console.SetCursorPosition(0, 4);
Console.WriteLine("Migrating stacks {0:N0} total {1:N0}/s...", total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0);
var response = searchclient.IndexMany(stacks, type: "stacks", index: ElasticSearchRepository <Stack> .StacksIndexName);
if (!response.IsValid)
{
Debugger.Break();
}
var lastId = stacks.Last().Id;
stacks = errorStackCollection.Find(Query.GT(ErrorStackFieldNames.Id, ObjectId.Parse(lastId))).SetSortOrder(SortBy.Ascending(ErrorStackFieldNames.Id)).SetLimit(BatchSize).ToList();
total += stacks.Count;
}
}
total = 0;
stopwatch.Reset();
if (!ca.SkipErrors)
{
stopwatch.Start();
var eventUpgraderPluginManager = container.GetInstance <EventUpgraderPluginManager>();
var eventRepository = container.GetInstance <IEventRepository>();
var errorCollection = GetErrorCollection(container);
//var json1 = JsonExtensions.ToJson(errorCollection.FindOneById(ObjectId.Parse("80000000e2cc694bd029a952")), Formatting.Indented);
//var json2 = JsonExtensions.ToJson(errorCollection.FindOneById(ObjectId.Parse("80000000e2cc694bd029a953")), Formatting.Indented);
//var ctx2 = new EventUpgraderContext("[" + json1 + "," + json2 + "]", new Version(1, 5), true);
//eventUpgraderPluginManager.Upgrade(ctx2);
//var ev2 = ctx2.Documents.ToObject<List<PersistentEvent>>();
//eventRepository.Add(ev2.First());
//eventRepository.Add(ev2.Last());
var query = mostRecentEvent != null && mostRecentEvent.Total > 0 ? Query.GT(ErrorFieldNames.Id, ObjectId.Parse(mostRecentEvent.Hits.First().Id)) : Query.Null;
var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.Id)).SetLimit(BatchSize).ToList();
// TODO: When resuming, we need to get a list of existing stack ids from the events.
var knownStackIds = new List <string>();
while (errors.Count > 0)
{
Console.SetCursorPosition(0, 5);
Console.WriteLine("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0);
var events = JArray.FromObject(errors);
var ctx = new EventUpgraderContext(events, new Version(1, 5), true);
eventUpgraderPluginManager.Upgrade(ctx);
var ev = events.FromJson <PersistentEvent>(serializerSettings);
ev.ForEach(e => {
if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1))
{
e.Date = DateTimeOffset.Now;
}
if (e.Type != Event.KnownTypes.Error)
{
return;
}
if (!knownStackIds.Contains(e.StackId))
{
e.IsFirstOccurrence = true;
knownStackIds.Add(e.StackId);
}
var request = e.GetRequestInfo();
if (request != null)
{
e.AddRequestInfo(request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH));
}
var stacking = e.GetStackingTarget();
if (stacking != null && stacking.Method != null && !String.IsNullOrEmpty(stacking.Method.DeclaringTypeFullName))
{
e.Source = stacking.Method.DeclaringTypeFullName.Truncate(2000);
}
});
try {
eventRepository.Add(ev);
} catch (Exception ex) {
Debugger.Break();
}
var lastId = ev.Last().Id;
errors = errorCollection.Find(Query.GT(ErrorFieldNames.Id, ObjectId.Parse(lastId))).SetSortOrder(SortBy.Ascending(ErrorFieldNames.Id)).SetLimit(BatchSize).ToList();
total += events.Count;
}
}
PauseIfDebug();
} catch (FileNotFoundException e) {
Console.Error.WriteLine("{0} ({1})", e.Message, e.FileName);
PauseIfDebug();
return(1);
} catch (Exception e) {
Console.Error.WriteLine(e.ToString());
PauseIfDebug();
return(1);
}
return(0);
}
protected override async Task <JobResult> RunInternalAsync(CancellationToken token)
{
OutputPublicIp();
QueueEntry <EventMigrationBatch> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
int total = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
var errorCollection = GetErrorCollection();
var knownStackIds = new List <string>();
var userAgentParser = Parser.GetDefault();
var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks));
var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList();
int batch = 0;
while (errors.Count > 0)
{
Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write();
var upgradedErrors = JArray.FromObject(errors);
var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true);
_eventUpgraderPluginManager.Upgrade(ctx);
var upgradedEvents = upgradedErrors.FromJson <PersistentEvent>(_settings);
var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray();
if (stackIdsToCheck.Length > 0)
{
knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck));
}
upgradedEvents.ForEach(async e => {
if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1))
{
e.Date = DateTimeOffset.Now;
}
e.CreatedUtc = e.Date.ToUniversalTime().DateTime;
// Truncate really large fields
if (e.Message != null && e.Message.Length > 2000)
{
Log.Error().Project(e.ProjectId).Message("Event: {0} Message is Too Big: {1}", e.Id, e.Message.Length).Write();
e.Message = e.Message.Truncate(2000);
}
if (e.Source != null && e.Source.Length > 2000)
{
Log.Error().Project(e.ProjectId).Message("Event: {0} Source is Too Big: {1}", e.Id, e.Source.Length).Write();
e.Source = e.Source.Truncate(2000);
}
if (!knownStackIds.Contains(e.StackId))
{
// We haven't processed this stack id yet in this run. Check to see if this stack has already been imported..
e.IsFirstOccurrence = true;
knownStackIds.Add(e.StackId);
}
var request = e.GetRequestInfo();
if (request != null)
{
request = request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH);
if (!String.IsNullOrEmpty(request.UserAgent))
{
try {
var info = userAgentParser.Parse(request.UserAgent);
if (!String.Equals(info.UserAgent.Family, "Other"))
{
request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family;
if (!String.IsNullOrEmpty(info.UserAgent.Major))
{
request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major;
}
}
if (!String.Equals(info.Device.Family, "Other"))
{
request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family;
}
if (!String.Equals(info.OS.Family, "Other"))
{
request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family;
if (!String.IsNullOrEmpty(info.OS.Major))
{
request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major;
}
}
request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider;
} catch (Exception ex) {
Log.Warn().Project(e.ProjectId).Message("Unable to parse user agent {0}. Exception: {1}", request.UserAgent, ex.Message).Write();
}
}
e.AddRequestInfo(request);
}
foreach (var ip in GetIpAddresses(e, request))
{
var location = await _geoIpResolver.ResolveIpAsync(ip, token);
if (location == null || !location.IsValid())
{
continue;
}
e.Geo = location.ToString();
break;
}
if (e.Type == Event.KnownTypes.NotFound && request != null)
{
if (String.IsNullOrWhiteSpace(e.Source))
{
e.Message = null;
e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false);
}
return;
}
var error = e.GetError();
if (error == null)
{
Debugger.Break();
Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write();
return;
}
var stackingTarget = error.GetStackingTarget();
if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName()))
{
e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000);
}
var signature = new ErrorSignature(error);
if (signature.SignatureInfo.Count <= 0)
{
return;
}
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message"))
{
targetInfo["Message"] = stackingTarget.Error.Message;
}
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
});
Log.Info().Message("Saving events {0}-{1} {2:N0} total", errors.First().Id, errors.Last().Id, upgradedEvents.Count).Write();
try {
_eventRepository.Add(upgradedEvents, sendNotification: false);
} catch (Exception) {
foreach (var persistentEvent in upgradedEvents)
{
try {
_eventRepository.Add(persistentEvent, sendNotification: false);
} catch (Exception ex) {
//Debugger.Break();
Log.Error().Exception(ex).Project(persistentEvent.ProjectId).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write();
}
}
}
batch++;
total += upgradedEvents.Count;
Log.Info().Message("Getting next batch of events").Write();
var sw = new Stopwatch();
sw.Start();
errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).SetSkip(_batchSize * batch).ToList();
sw.Stop();
Log.Info().Message("Finished getting next batch of events in {0}ms", sw.ElapsedMilliseconds).Write();
}
Log.Info().Message("Finished processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
_cache.Set("migration-completedperiod", queueEntry.Value.EndTicks);
queueEntry.Complete();
return(JobResult.Success);
}
