public override Task EventProcessingAsync(EventContext context)
{
if (!context.Event.IsError())
{
return(Task.CompletedTask);
}
var error = context.Event.GetError();
if (error == null)
{
return(Task.CompletedTask);
}
if (String.IsNullOrWhiteSpace(context.Event.Message))
{
context.Event.Message = error.Message;
}
if (context.StackSignatureData.Count > 0)
{
return(Task.CompletedTask);
}
string[] commonUserMethods = { "DataContext.SubmitChanges", "Entities.SaveChanges" };
if (context.HasProperty("CommonMethods"))
{
commonUserMethods = context.GetProperty <string>("CommonMethods").SplitAndTrim(new [] { ',' });
}
string[] userNamespaces = null;
if (context.HasProperty("UserNamespaces"))
{
userNamespaces = context.GetProperty <string>("UserNamespaces").SplitAndTrim(new [] { ',' });
}
var signature = new ErrorSignature(error, userCommonMethods: commonUserMethods, userNamespaces: userNamespaces);
if (signature.SignatureInfo.Count <= 0)
{
return(Task.CompletedTask);
}
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
var stackingTarget = error.GetStackingTarget();
if (stackingTarget?.Error?.StackTrace?.Count > 0 && !targetInfo.ContainsKey("Message"))
{
targetInfo["Message"] = stackingTarget.Error.Message;
}
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
foreach (string key in signature.SignatureInfo.Keys)
{
context.StackSignatureData.Add(key, signature.SignatureInfo[key]);
}
return(Task.CompletedTask);
}
/// <summary>
/// set a setting
/// </summary>
public void SetSetting(string name, object value, string groupId = null)
{
string key = name;
if (!string.IsNullOrEmpty(groupId))
{
key = $"{groupId}.{name}";
}
string strValue = value?.ToString();
if (SettingsDictionary.ContainsKey(key))
{
SettingsDictionary[key] = strValue;
}
else
{
SettingsDictionary.Add(key, strValue);
}
SetGroupSetting(groupId, name, value);
if (AutoSave)
{
SaveSettings();
}
}
/// <summary>
/// Must override this, this is the bit that matches up the designer properties to the dictionary values
/// </summary>
/// <param name="context"></param>
/// <param name="collection"></param>
/// <returns></returns>
public override SettingsPropertyValueCollection GetPropertyValues(SettingsContext context, SettingsPropertyCollection collection)
{
//load the file
if (!_loaded)
{
_loaded = true;
LoadValuesFromFile();
}
//collection that will be returned.
SettingsPropertyValueCollection values = new SettingsPropertyValueCollection();
//itterate thought the properties we get from the designer, checking to see if the setting is in the dictionary
foreach (SettingsProperty setting in collection)
{
SettingsPropertyValue value = new SettingsPropertyValue(setting);
value.IsDirty = false;
if (SettingsDictionary.ContainsKey(setting.Name))
{
value.SerializedValue = SettingsDictionary[setting.Name].value;
}
else //use defaults in the case where there are no settings yet
{
value.SerializedValue = setting.DefaultValue == null ? string.Empty : setting.DefaultValue.ToString();
}
values.Add(value);
}
return(values);
}
/// <summary>
/// Must override this, this is the bit that matches up the designer properties to the dictionary values
/// </summary>
/// <param name="context"></param>
/// <param name="collection"></param>
/// <returns></returns>
public override SettingsPropertyValueCollection GetPropertyValues(SettingsContext context, SettingsPropertyCollection collection)
{
//load the file
if (!_loaded)
{
_loaded = true;
LoadValuesFromFile();
}
//collection that will be returned.
SettingsPropertyValueCollection values = new SettingsPropertyValueCollection();
//itterate thought the properties we get from the designer, checking to see if the setting is in the dictionary
foreach (SettingsProperty setting in collection)
{
SettingsPropertyValue value = new SettingsPropertyValue(setting);
value.IsDirty = false;
//need the type of the value for the strong typing
var t = Type.GetType(setting.PropertyType.FullName);
if (SettingsDictionary.ContainsKey(setting.Name))
{
value.SerializedValue = SettingsDictionary[setting.Name].value;
value.PropertyValue = Convert.ChangeType(SettingsDictionary[setting.Name].value, t);
}
else //use defaults in the case where there are no settings yet
{
value.SerializedValue = setting.DefaultValue;
value.PropertyValue = Convert.ChangeType(setting.DefaultValue, t);
}
values.Add(value);
}
return(values);
}
private void LoadSettings()
{
FileNameField = (SettingsDictionary.ContainsKey("FileNameField") ? SettingsDictionary["FileNameField"] : "");
FolderNameField = (SettingsDictionary.ContainsKey("FolderNameField") ? SettingsDictionary["FolderNameField"] : "");
DefaultFolder = (SettingsDictionary.ContainsKey("DefaultFolder") ? SettingsDictionary["DefaultFolder"] : "");
ActionName = (SettingsDictionary.ContainsKey("ActionName") ? SettingsDictionary["ActionName"] : "");
}
private static void BuildDiff(DiffDictionary <string, string> dictionary, SettingsDictionary settingsA, SettingsDictionary settingsB)
{
foreach (var settingA in settingsA)
{
string oldValue, newValue;
if (settingsB.ContainsKey(settingA.Key))
{
oldValue = settingA.Value;
newValue = settingsB[settingA.Key];
}
else
{
oldValue = settingA.Value;
newValue = settingsB[settingA.Key] = default(string);
}
if (oldValue != newValue)
{
dictionary[settingA.Key] = new Diff <string> {
NewValue = newValue,
OldValue = oldValue
};
}
}
}
public override Task EventProcessingAsync(EventContext context) {
if (!context.Event.IsError())
return Task.CompletedTask;
Error error = context.Event.GetError();
if (error == null)
return Task.CompletedTask;
if (String.IsNullOrWhiteSpace(context.Event.Message))
context.Event.Message = error.Message;
string[] commonUserMethods = { "DataContext.SubmitChanges", "Entities.SaveChanges" };
if (context.HasProperty("CommonMethods"))
commonUserMethods = context.GetProperty<string>("CommonMethods").SplitAndTrim(',');
string[] userNamespaces = null;
if (context.HasProperty("UserNamespaces"))
userNamespaces = context.GetProperty<string>("UserNamespaces").SplitAndTrim(',');
var signature = new ErrorSignature(error, userCommonMethods: commonUserMethods, userNamespaces: userNamespaces);
if (signature.SignatureInfo.Count <= 0)
return Task.CompletedTask;
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
var stackingTarget = error.GetStackingTarget();
if (stackingTarget?.Error?.StackTrace != null && stackingTarget.Error.StackTrace.Count > 0 && !targetInfo.ContainsKey("Message"))
targetInfo["Message"] = stackingTarget.Error.Message;
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
foreach (var key in signature.SignatureInfo.Keys)
context.StackSignatureData.Add(key, signature.SignatureInfo[key]);
return Task.CompletedTask;
}
/// <summary>
/// Must override this, this is the bit that does the saving to file. Called when Settings.Save() is called
/// </summary>
/// <param name="context"></param>
/// <param name="collection"></param>
public override void SetPropertyValues(SettingsContext context, SettingsPropertyValueCollection collection)
{
//grab the values from the collection parameter and update the values in our dictionary.
foreach (SettingsPropertyValue value in collection)
{
var setting = new SettingStruct()
{
value = (value.PropertyValue == null ? String.Empty : value.SerializedValue.ToString()),
name = value.Name,
serializeAs = value.Property.SerializeAs.ToString()
};
if (!SettingsDictionary.ContainsKey(value.Name))
{
SettingsDictionary.Add(value.Name, setting);
}
else
{
SettingsDictionary[value.Name] = setting;
}
}
//now that our local dictionary is up-to-date, save it to disk.
SaveValuesToFile();
}
public override void LoadSettings()
{
base.LoadSettings();
SenderAccount = (SettingsDictionary.ContainsKey("SenderAccount") ? SettingsDictionary["SenderAccount"]:"");
SenderAccountPassword = (SettingsDictionary.ContainsKey("SenderAccountPassword") ? SettingsDictionary["SenderAccountPassword"] : "");
SmtpServerName = (SettingsDictionary.ContainsKey("SmtpServer") ? SettingsDictionary["SmtpServer"] : "");
SmtpPort = (SettingsDictionary.ContainsKey("SmtpPort") ? Convert.ToInt16(SettingsDictionary["SmtpPort"]) : 587);
UseSSL = (SettingsDictionary.ContainsKey("UseSSL") ? Convert.ToBoolean(SettingsDictionary["UseSSL"]) : true);
UseDefaultCredentials = (SettingsDictionary.ContainsKey("UseDefaultCredentials") ? Convert.ToBoolean(SettingsDictionary["UseDefaultCredentials"]) : true);
}
public override void Execute(XmlNodeList dataNodes)
{
if (SettingsDictionary.ContainsKey("ConfirmationRequired") && (Convert.ToBoolean(SettingsDictionary["ConfirmationRequired"])))
{
if (MessageBox.Show("Should I proceed with Deleting " + dataNodes.Count + " records ?", "Delete", MessageBoxButtons.YesNo) == DialogResult.No)
{
return;
}
}
base.Execute(dataNodes);
}
public override void Execute(params EasyDynamicObject[] dataObjects)
{
if (SettingsDictionary.ContainsKey("ConfirmationRequired") && (Convert.ToBoolean(SettingsDictionary["ConfirmationRequired"])))
{
if (MessageBox.Show("Should I proceed with Deleting " + dataObjects.Length + " records ?", "Delete", MessageBoxButtons.YesNo) == DialogResult.No)
{
return;
}
}
base.Execute(dataObjects);
}
public virtual void LoadSettings()
{
Recipient = (SettingsDictionary.ContainsKey("Recipient") ? SettingsDictionary["Recipient"] : "");
CC = (SettingsDictionary.ContainsKey("CC") ? SettingsDictionary["CC"] : "");
BCC = (SettingsDictionary.ContainsKey("BCC") ? SettingsDictionary["BCC"] : "");
Subject = (SettingsDictionary.ContainsKey("Subject") ? SettingsDictionary["Subject"] : "");
Body = (SettingsDictionary.ContainsKey("Body") ? SettingsDictionary["Body"] : "");
if (Body.StartsWith("@"))
{
if (File.Exists(Body.Substring(1)))
{
Body = File.ReadAllText(Body.Substring(1));
}
}
IndividualEmails = (SettingsDictionary.ContainsKey("IndividualEmails") ? Convert.ToBoolean(SettingsDictionary["IndividualEmails"]) : false);
}
private bool CanExecute(string fileName, string folderName)
{
if (String.IsNullOrWhiteSpace(fileName))
{
return(false);
}
if ((!fileName.Contains(Path.DirectorySeparatorChar)) && (!String.IsNullOrWhiteSpace(folderName)))
{
fileName = Path.Combine(folderName, fileName);
}
if ((!fileName.Contains(Path.DirectorySeparatorChar)) && (SettingsDictionary.ContainsKey("DefaultFolder")))
{
fileName = Path.Combine(SettingsDictionary["DefaultFolder"], fileName);
}
WorkingFileName = fileName;
return(File.Exists(fileName));
}
public void set(string key, string value)
{
SystemSetting setting;
if (SettingsDictionary.ContainsKey(key))
{
setting = SettingsDictionary[key];
}
else
{
setting = new SystemSetting();
setting.Name = key;
SettingsDictionary[key] = setting;
}
setting.Value = value;
setting.Commit();
}
public string get(string key)
{
string result;
if (SettingsDictionary == null)
{
globalInstance.LoadData();
}
if (SettingsDictionary.ContainsKey(key))
{
result = SettingsDictionary[key].Value;
}
else
{
result = null;
}
return(result);
}
/// <summary>
/// Must override this, this is the bit that matches up the designer properties to the dictionary values
/// </summary>
/// <param name="context"></param>
/// <param name="collection"></param>
/// <returns></returns>
public override SettingsPropertyValueCollection GetPropertyValues(SettingsContext context, SettingsPropertyCollection collection)
{
//load the file
if (!_loaded)
{
_loaded = true;
LoadValuesFromFile();
}
//collection that will be returned.
SettingsPropertyValueCollection values = new SettingsPropertyValueCollection();
//itterate thought the properties we get from the designer, checking to see if the setting is in the dictionary
foreach (SettingsProperty setting in collection)
{
SettingsPropertyValue value = new SettingsPropertyValue(setting);
value.IsDirty = false;
//need the type of the value for the strong typing
var t = GetType(setting.PropertyType.FullName);
if (SettingsDictionary.ContainsKey(setting.Name))
{
value.SerializedValue = SettingsDictionary[setting.Name].value;
// http://stackoverflow.com/questions/393731/generic-conversion-function-doesnt-seem-to-work-with-guids
value.PropertyValue = TypeDescriptor.GetConverter(t).ConvertFromInvariantString(SettingsDictionary[setting.Name].value);
//value.PropertyValue = Convert.ChangeType(SettingsDictionary[setting.Name].value, t);
}
else //use defaults in the case where there are no settings yet
{
value.SerializedValue = setting.DefaultValue;
// http://stackoverflow.com/questions/393731/generic-conversion-function-doesnt-seem-to-work-with-guids
value.PropertyValue = TypeDescriptor.GetConverter(t).ConvertFromInvariantString((string)setting.DefaultValue);
// value.PropertyValue = Convert.ChangeType(setting.DefaultValue, t);
}
values.Add(value);
}
return(values);
}
internal static void ProcessServerConfigResponse(IConfigurationAndLogAccessor accessors, SettingsDictionary serverConfig, string storeId)
{
if (serverConfig == null)
{
return;
}
try {
// only allow one save at a time
using (new SingleGlobalInstance(String.Concat(storeId, CachedServerConfigFile).GetHashCode().ToString(), 500)) {
// retry loop
for (int retry = 0; retry < 2; retry++)
{
using (var dir = new IsolatedStorageDirectory(storeId)) {
try {
dir.WriteFile(CachedServerConfigFile, serverConfig);
break;
} catch (Exception ex) {
// File is being used by another process or thread or the file does not exist.
accessors.Log.FormattedError(ex, "Unable to save server config to local storage: {0}", ex.Message);
Thread.Sleep(50);
}
}
}
}
} catch (Exception ex) {
accessors.Log.Error(ex, "An error occurred while saving client configuration");
}
// apply the config values from the server to the current client configuration
foreach (string k in serverConfig.Keys)
{
accessors.Configuration[k] = serverConfig[k];
}
// if a set of data exclusions are not sent down, then it means that there aren't any exclusions.
if (!serverConfig.ContainsKey("@@DataExclusions"))
{
accessors.Configuration["@@DataExclusions"] = String.Empty;
}
}
private static void BuildDiff(DiffDictionary<string, string> dictionary, SettingsDictionary settingsA, SettingsDictionary settingsB) {
foreach (var settingA in settingsA) {
string oldValue, newValue;
if (settingsB.ContainsKey(settingA.Key)) {
oldValue = settingA.Value;
newValue = settingsB[settingA.Key];
}
else {
oldValue = settingA.Value;
newValue = settingsB[settingA.Key] = default(string);
}
if (oldValue != newValue) {
dictionary[settingA.Key] = new Diff<string> {
NewValue = newValue,
OldValue = oldValue
};
}
}
}
public void LoadDefaultValues()
{
List <Composites.Settings> settingsList = GetDefaultSettings(Professionbuddy.Instance.PbBehavior);
foreach (Composites.Settings setting in settingsList)
{
if (!SettingsDictionary.ContainsKey(setting.SettingName))
{
SettingsDictionary[setting.SettingName] = new PbProfileSettingEntry
{
Value = GetValue(setting.Type, setting.DefaultValue)
}
}
;
SettingsDictionary[setting.SettingName].Summary = setting.Summary;
SettingsDictionary[setting.SettingName].Category = setting.Category;
SettingsDictionary[setting.SettingName].Global = setting.Global;
SettingsDictionary[setting.SettingName].Hidden = setting.Hidden;
}
// remove unused settings..
SettingsDictionary = SettingsDictionary.Where(kv => settingsList.Any(s => s.SettingName == kv.Key)).ToDictionary(kv => kv.Key,
kv => kv.Value);
}
public void CanGetSettingsMultithreaded()
{
var settings = new SettingsDictionary();
var result = Parallel.For(0, 20, index => {
for (int i = 0; i < 10; i++)
{
string key = $"setting-{i}";
if (!settings.ContainsKey(key))
{
settings.Add(key, (index * i).ToString());
}
else
{
settings[key] = (index * i).ToString();
}
}
});
while (!result.IsCompleted)
{
Thread.Yield();
}
}
/// <summary>
/// Must override this, this is the bit that does the saving to file. Called when Settings.Save() is called
/// </summary>
/// <param name="context"></param>
/// <param name="collection"></param>
public override void SetPropertyValues(SettingsContext context, SettingsPropertyValueCollection collection)
{
//grab the values from the collection parameter and update the values in our dictionary.
foreach (SettingsPropertyValue value in collection)
{
try
{
Type t = GetType(value.Property.PropertyType.FullName);
var setting = new SettingStruct()
{
value = (value.PropertyValue == null ? String.Empty
//: value.PropertyValue.ToString()),
: (string)(TypeDescriptor.GetConverter(t).ConvertToInvariantString(value.PropertyValue))),
name = value.Name,
serializeAs = value.Property.SerializeAs.ToString()
};
if (!SettingsDictionary.ContainsKey(value.Name))
{
SettingsDictionary.Add(value.Name, setting);
}
else
{
SettingsDictionary[value.Name] = setting;
}
}
catch (Exception ee)
{
Console.WriteLine(ee.Message);
}
}
//now that our local dictionary is up-to-date, save it to disk.
SaveValuesToFile();
}
protected override async Task<JobResult> RunInternalAsync(CancellationToken token) {
OutputPublicIp();
QueueEntry<EventMigrationBatch> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException)) {
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return JobResult.FromException(ex);
}
}
if (queueEntry == null)
return JobResult.Success;
Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
int total = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
var errorCollection = GetErrorCollection();
var knownStackIds = new List<string>();
var userAgentParser = Parser.GetDefault();
var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks));
var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList();
int batch = 0;
while (errors.Count > 0) {
Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write();
var upgradedErrors = JArray.FromObject(errors);
var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true);
_eventUpgraderPluginManager.Upgrade(ctx);
var upgradedEvents = upgradedErrors.FromJson<PersistentEvent>(_settings);
var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray();
if (stackIdsToCheck.Length > 0)
knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck));
upgradedEvents.ForEach(async e => {
if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1))
e.Date = DateTimeOffset.Now;
e.CreatedUtc = e.Date.ToUniversalTime().DateTime;
// Truncate really large fields
if (e.Message != null && e.Message.Length > 2000) {
Log.Error().Project(e.ProjectId).Message("Event: {0} Message is Too Big: {1}", e.Id, e.Message.Length).Write();
e.Message = e.Message.Truncate(2000);
}
if (e.Source != null && e.Source.Length > 2000) {
Log.Error().Project(e.ProjectId).Message("Event: {0} Source is Too Big: {1}", e.Id, e.Source.Length).Write();
e.Source = e.Source.Truncate(2000);
}
if (!knownStackIds.Contains(e.StackId)) {
// We haven't processed this stack id yet in this run. Check to see if this stack has already been imported..
e.IsFirstOccurrence = true;
knownStackIds.Add(e.StackId);
}
var request = e.GetRequestInfo();
if (request != null) {
request = request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH);
if (!String.IsNullOrEmpty(request.UserAgent)) {
try {
var info = userAgentParser.Parse(request.UserAgent);
if (!String.Equals(info.UserAgent.Family, "Other")) {
request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family;
if (!String.IsNullOrEmpty(info.UserAgent.Major)) {
request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major;
}
}
if (!String.Equals(info.Device.Family, "Other"))
request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family;
if (!String.Equals(info.OS.Family, "Other")) {
request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family;
if (!String.IsNullOrEmpty(info.OS.Major)) {
request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major;
}
}
request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider;
} catch (Exception ex) {
Log.Warn().Project(e.ProjectId).Message("Unable to parse user agent {0}. Exception: {1}", request.UserAgent, ex.Message).Write();
}
}
e.AddRequestInfo(request);
}
foreach (var ip in GetIpAddresses(e, request)) {
var location = await _geoIpResolver.ResolveIpAsync(ip, token);
if (location == null || !location.IsValid())
continue;
e.Geo = location.ToString();
break;
}
if (e.Type == Event.KnownTypes.NotFound && request != null) {
if (String.IsNullOrWhiteSpace(e.Source)) {
e.Message = null;
e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false);
}
return;
}
var error = e.GetError();
if (error == null) {
Debugger.Break();
Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write();
return;
}
var stackingTarget = error.GetStackingTarget();
if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName()))
e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000);
var signature = new ErrorSignature(error);
if (signature.SignatureInfo.Count <= 0)
return;
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message"))
targetInfo["Message"] = stackingTarget.Error.Message;
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
});
Log.Info().Message("Saving events {0}-{1} {2:N0} total", errors.First().Id, errors.Last().Id, upgradedEvents.Count).Write();
try {
_eventRepository.Add(upgradedEvents, sendNotification: false);
} catch (Exception) {
foreach (var persistentEvent in upgradedEvents) {
try {
_eventRepository.Add(persistentEvent, sendNotification: false);
} catch (Exception ex) {
//Debugger.Break();
Log.Error().Exception(ex).Project(persistentEvent.ProjectId).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write();
}
}
}
batch++;
total += upgradedEvents.Count;
Log.Info().Message("Getting next batch of events").Write();
var sw = new Stopwatch();
sw.Start();
errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).SetSkip(_batchSize * batch).ToList();
sw.Stop();
Log.Info().Message("Finished getting next batch of events in {0}ms", sw.ElapsedMilliseconds).Write();
}
Log.Info().Message("Finished processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
_cache.Set("migration-completedperiod", queueEntry.Value.EndTicks);
queueEntry.Complete();
return JobResult.Success;
}
protected override async Task <JobResult> RunInternalAsync(CancellationToken token)
{
OutputPublicIp();
QueueEntry <EventMigrationBatch> queueEntry = null;
try {
queueEntry = _queue.Dequeue();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
int total = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
var errorCollection = GetErrorCollection();
var knownStackIds = new List <string>();
var serializerSettings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore
};
serializerSettings.AddModelConverters();
var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks));
var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList();
while (errors.Count > 0)
{
Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write();
var upgradedErrors = JArray.FromObject(errors);
var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true);
_eventUpgraderPluginManager.Upgrade(ctx);
var upgradedEvents = upgradedErrors.FromJson <PersistentEvent>(serializerSettings);
var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray();
if (stackIdsToCheck.Length > 0)
{
knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck));
}
upgradedEvents.ForEach(e => {
if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1))
{
e.Date = DateTimeOffset.Now;
}
e.CreatedUtc = e.Date.ToUniversalTime().DateTime;
if (!knownStackIds.Contains(e.StackId))
{
// We haven't processed this stack id yet in this run. Check to see if this stack has already been imported..
e.IsFirstOccurrence = true;
knownStackIds.Add(e.StackId);
}
var request = e.GetRequestInfo();
if (request != null)
{
e.AddRequestInfo(request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH));
}
foreach (var ip in GetIpAddresses(e, request))
{
var location = _geoIpResolver.ResolveIp(ip);
if (location == null || !location.IsValid())
{
continue;
}
e.Geo = location.ToString();
break;
}
if (e.Type == Event.KnownTypes.NotFound && request != null)
{
if (String.IsNullOrWhiteSpace(e.Source))
{
e.Message = null;
e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false);
}
return;
}
var error = e.GetError();
if (error == null)
{
Debugger.Break();
Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write();
return;
}
var stackingTarget = error.GetStackingTarget();
if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName()))
{
e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000);
}
var signature = new ErrorSignature(error);
if (signature.SignatureInfo.Count <= 0)
{
return;
}
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message"))
{
targetInfo["Message"] = error.GetStackingTarget().Error.Message;
}
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
});
try {
_eventRepository.Add(upgradedEvents, sendNotification: false);
} catch (Exception) {
foreach (var persistentEvent in upgradedEvents)
{
try {
_eventRepository.Add(persistentEvent, sendNotification: false);
} catch (Exception ex) {
//Debugger.Break();
Log.Error().Exception(ex).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write();
}
}
}
total += upgradedEvents.Count;
var lastId = upgradedEvents.Last().Id;
_cache.Set("migration-errorid", lastId);
errors = errorCollection.Find(Query.And(Query.GT(ErrorFieldNames.Id, ObjectId.Parse(lastId)), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks)))
.SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC))
.SetLimit(_batchSize).ToList();
}
_cache.Set("migration-completedday", queueEntry.Value.EndTicks);
queueEntry.Complete();
return(JobResult.Success);
}
private void LoadSettings()
{
Command = (SettingsDictionary.ContainsKey("Command") ? SettingsDictionary["Command"] : "");
ArgumentFields = (SettingsDictionary.ContainsKey("ArgumentFields") ? SettingsDictionary["ArgumentFields"] : "");
}
protected override async Task <JobResult> RunInternalAsync(CancellationToken token)
{
OutputPublicIp();
QueueEntry <EventMigrationBatch> queueEntry = null;
try {
queueEntry = _queue.Dequeue(TimeSpan.FromSeconds(1));
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return(JobResult.FromException(ex));
}
}
if (queueEntry == null)
{
return(JobResult.Success);
}
Log.Info().Message("Processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
int total = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
var errorCollection = GetErrorCollection();
var knownStackIds = new List <string>();
var userAgentParser = Parser.GetDefault();
var query = Query.And(Query.GTE(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.StartTicks), Query.LT(ErrorFieldNames.OccurrenceDate_UTC, queueEntry.Value.EndTicks));
var errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).ToList();
int batch = 0;
while (errors.Count > 0)
{
Log.Info().Message("Migrating events {0}-{1} {2:N0} total {3:N0}/s...", errors.First().Id, errors.Last().Id, total, total > 0 ? total / stopwatch.Elapsed.TotalSeconds : 0).Write();
var upgradedErrors = JArray.FromObject(errors);
var ctx = new EventUpgraderContext(upgradedErrors, new Version(1, 5), true);
_eventUpgraderPluginManager.Upgrade(ctx);
var upgradedEvents = upgradedErrors.FromJson <PersistentEvent>(_settings);
var stackIdsToCheck = upgradedEvents.Where(e => !knownStackIds.Contains(e.StackId)).Select(e => e.StackId).Distinct().ToArray();
if (stackIdsToCheck.Length > 0)
{
knownStackIds.AddRange(_eventRepository.ExistsByStackIds(stackIdsToCheck));
}
upgradedEvents.ForEach(async e => {
if (e.Date.UtcDateTime > DateTimeOffset.UtcNow.AddHours(1))
{
e.Date = DateTimeOffset.Now;
}
e.CreatedUtc = e.Date.ToUniversalTime().DateTime;
// Truncate really large fields
if (e.Message != null && e.Message.Length > 2000)
{
Log.Error().Project(e.ProjectId).Message("Event: {0} Message is Too Big: {1}", e.Id, e.Message.Length).Write();
e.Message = e.Message.Truncate(2000);
}
if (e.Source != null && e.Source.Length > 2000)
{
Log.Error().Project(e.ProjectId).Message("Event: {0} Source is Too Big: {1}", e.Id, e.Source.Length).Write();
e.Source = e.Source.Truncate(2000);
}
if (!knownStackIds.Contains(e.StackId))
{
// We haven't processed this stack id yet in this run. Check to see if this stack has already been imported..
e.IsFirstOccurrence = true;
knownStackIds.Add(e.StackId);
}
var request = e.GetRequestInfo();
if (request != null)
{
request = request.ApplyDataExclusions(RequestInfoPlugin.DefaultExclusions, RequestInfoPlugin.MAX_VALUE_LENGTH);
if (!String.IsNullOrEmpty(request.UserAgent))
{
try {
var info = userAgentParser.Parse(request.UserAgent);
if (!String.Equals(info.UserAgent.Family, "Other"))
{
request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family;
if (!String.IsNullOrEmpty(info.UserAgent.Major))
{
request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major;
}
}
if (!String.Equals(info.Device.Family, "Other"))
{
request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family;
}
if (!String.Equals(info.OS.Family, "Other"))
{
request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family;
if (!String.IsNullOrEmpty(info.OS.Major))
{
request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v)));
request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major;
}
}
request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider;
} catch (Exception ex) {
Log.Warn().Project(e.ProjectId).Message("Unable to parse user agent {0}. Exception: {1}", request.UserAgent, ex.Message).Write();
}
}
e.AddRequestInfo(request);
}
foreach (var ip in GetIpAddresses(e, request))
{
var location = await _geoIpResolver.ResolveIpAsync(ip, token);
if (location == null || !location.IsValid())
{
continue;
}
e.Geo = location.ToString();
break;
}
if (e.Type == Event.KnownTypes.NotFound && request != null)
{
if (String.IsNullOrWhiteSpace(e.Source))
{
e.Message = null;
e.Source = request.GetFullPath(includeHttpMethod: true, includeHost: false, includeQueryString: false);
}
return;
}
var error = e.GetError();
if (error == null)
{
Debugger.Break();
Log.Error().Project(e.ProjectId).Message("Unable to get parse error model: {0}", e.Id).Write();
return;
}
var stackingTarget = error.GetStackingTarget();
if (stackingTarget != null && stackingTarget.Method != null && !String.IsNullOrEmpty(stackingTarget.Method.GetDeclaringTypeFullName()))
{
e.Source = stackingTarget.Method.GetDeclaringTypeFullName().Truncate(2000);
}
var signature = new ErrorSignature(error);
if (signature.SignatureInfo.Count <= 0)
{
return;
}
var targetInfo = new SettingsDictionary(signature.SignatureInfo);
if (stackingTarget != null && stackingTarget.Error != null && !targetInfo.ContainsKey("Message"))
{
targetInfo["Message"] = stackingTarget.Error.Message;
}
error.Data[Error.KnownDataKeys.TargetInfo] = targetInfo;
});
Log.Info().Message("Saving events {0}-{1} {2:N0} total", errors.First().Id, errors.Last().Id, upgradedEvents.Count).Write();
try {
_eventRepository.Add(upgradedEvents, sendNotification: false);
} catch (Exception) {
foreach (var persistentEvent in upgradedEvents)
{
try {
_eventRepository.Add(persistentEvent, sendNotification: false);
} catch (Exception ex) {
//Debugger.Break();
Log.Error().Exception(ex).Project(persistentEvent.ProjectId).Message("An error occurred while migrating event '{0}': {1}", persistentEvent.Id, ex.Message).Write();
}
}
}
batch++;
total += upgradedEvents.Count;
Log.Info().Message("Getting next batch of events").Write();
var sw = new Stopwatch();
sw.Start();
errors = errorCollection.Find(query).SetSortOrder(SortBy.Ascending(ErrorFieldNames.OccurrenceDate_UTC)).SetLimit(_batchSize).SetSkip(_batchSize * batch).ToList();
sw.Stop();
Log.Info().Message("Finished getting next batch of events in {0}ms", sw.ElapsedMilliseconds).Write();
}
Log.Info().Message("Finished processing event migration jobs for date range: {0}-{1}", new DateTimeOffset(queueEntry.Value.StartTicks, TimeSpan.Zero).ToString("O"), new DateTimeOffset(queueEntry.Value.EndTicks, TimeSpan.Zero).ToString("O")).Write();
_cache.Set("migration-completedperiod", queueEntry.Value.EndTicks);
queueEntry.Complete();
return(JobResult.Success);
}
public bool ContainsKey <T>(string key)
{
Initialize();
return(_dictionary.ContainsKey <T>(key));
}
public bool ContainsKey(string key)
{
return(SettingsDictionary.ContainsKey(key));
}
/// <summary>
/// Stub function, this does the actual file loading. Run LoadFile(), not this function directly.
/// </summary>
private void LoadFile_()
{
CurrentGroup = "";
SettingsDictionary.Clear();
using (var sr = new StreamReader(Filename)) {
while (!sr.EndOfStream)
{
var thisLine = sr.ReadLine();
if (thisLine == null || thisLine.StartsWith(";")) // -- Comment
{
continue;
}
if (thisLine.Contains(";")) // -- Deal with mid-line comments.
{
thisLine = thisLine.Substring(0, thisLine.IndexOf(";"));
}
if (!thisLine.Contains("=") && (!thisLine.Contains("[") && !thisLine.Contains("]")))
{
continue;
}
if ((thisLine.StartsWith("[") && thisLine.EndsWith("]"))) // -- Group.
{
var grpName = thisLine.Substring(1, thisLine.Length - 2);
Dictionary <string, string> group;
while (SettingsDictionary.TryGetValue(grpName, out group))
{
grpName += "§";
continue;
}
SettingsDictionary.Add(grpName, new Dictionary <string, string>());
CurrentGroup = grpName;
continue;
}
if (!thisLine.Contains("="))
{
continue;
}
var key = thisLine.Substring(0, thisLine.IndexOf("=")).Trim(' ');
var value =
thisLine.Substring(thisLine.IndexOf("=") + 1, thisLine.Length - (thisLine.IndexOf("=") + 1))
.Trim(' ');
// -- Setting.
if (CurrentGroup == "" && !SettingsDictionary.ContainsKey(""))
{
SettingsDictionary.Add("", new Dictionary <string, string>());
}
if (SettingsDictionary[CurrentGroup].ContainsKey(key))
{
SettingsDictionary[CurrentGroup][key] = value;
}
else
{
SettingsDictionary[CurrentGroup].Add(key, value);
}
}
}
}
private void LoadSettings()
{
DataFields = (SettingsDictionary.ContainsKey("DataFields") ? SettingsDictionary["DataFields"] : "");
}
public override void LoadSettings()
{
base.LoadSettings();
AutoSend = (SettingsDictionary.ContainsKey("AutoSend") ? Convert.ToBoolean(SettingsDictionary["AutoSend"]) : false);
}
public static string Get(this SettingsDictionary settings, string key)
{
return(settings.ContainsKey(key) ? settings[key] : null);
}
public override bool IsFieldSettingsComplete()
{
return(SettingsDictionary.ContainsKey("FieldName") && SettingsDictionary.ContainsKey("CurrentFieldValue") && SettingsDictionary.ContainsKey("NewFieldValue") && !String.IsNullOrWhiteSpace(SettingsDictionary["FieldName"]) && (SettingsDictionary["CurrentFieldValue"] != SettingsDictionary["NewFieldValue"]));
}