public void Persist(TrackerData trackerData)
{
// setup the client
// - with a custome handler to approve all server certificates
var handler = new WebRequestHandler
{
ServerCertificateValidationCallback =
(sender, certificate, chain, errors) => true
};
try
{
using (var client = new HttpClient(handler))
{
// post it
HttpResponseMessage httpResponseMessage = client.PostAsJsonAsync(_serviceUrl, trackerData).Result;
if (!httpResponseMessage.IsSuccessStatusCode)
{
Configurator.Configuration.Logger.Warn(string.Format("Data not persisted with status {0}",
httpResponseMessage.StatusCode));
}
}
}
catch (Exception ex)
{
Configurator.Configuration.Logger.Error(ex.Message, ex);
}
}
public void Persist(TrackerData trackerData)
{
try
{
UpdateBuilder update2 = Update.Inc("Measurement._Total", trackerData.Measurement._Total)
.Inc("Measurement._Occurrence", trackerData.Measurement._Occurrence)
.SetOnInsert("TypeName", trackerData.TypeName)
.SetOnInsert("KeyFilter", trackerData.KeyFilter)
.SetOnInsert("Name", trackerData.Name)
.SetOnInsert("SearchFilters", new BsonArray(trackerData.SearchFilters))
.SetOnInsert("TimeSlot", trackerData.TimeSlot);
foreach (var nm in trackerData.Measurement.NamedMetrics)
{
update2.Inc("Measurement." + nm.Key, nm.Value);
}
IMongoQuery query = Query.EQ("_id", trackerData._id);
_mongoCollection.Update(query, update2, UpdateFlags.Upsert);
}
catch (Exception ex)
{
Configurator.Configuration.Logger.Error(ex.Message, ex);
}
}
public void Persist(TrackerData trackerData)
{
try
{
Persist(trackerData, 0);
}
catch (Exception ex)
{
_logger.Error(ex.Message, ex);
}
}
private static void trackerWriter(TrackerData tc)
{
try
{
_lastPersistanceComplete = false;
Configurator.Configuration.Persister.Persist(tc);
_lastPersistanceComplete = true;
lastPersistTime = DateTime.UtcNow;
}
catch (Exception ex)
{
Configurator.Configuration.Logger.Error(ex.Message, ex);
_lastPersistanceComplete = true;
}
}
private static void addToMeasurementTable(TrackerData trackerData, DataTable table, int measurementResolutionMinutes, string metricName, long metricValue)
{
if (trackerData.Measurement.CoveredResolutions != null)
{
if (_persistPreAggregatedBuckets)
{
table.Rows.Add(metricName, metricValue, measurementResolutionMinutes
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.Minute)
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.FiveMinute)
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.FifteenMinute)
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.ThirtyMinute)
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.Hour)
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.Day)
, trackerData.Measurement.CoveredResolutions.Contains(Resolution.Month));
}
else
{
table.Rows.Add(metricName, metricValue, measurementResolutionMinutes, 0, 0, 0, 0, 0, 0, 0);
}
}
}
public async void Persist(TrackerData trackerData)
{
}
public void Start(bool deleteRecordAfterMigration, int startAt, int stopAfter, int docsImported)
{
var docNumber = startAt;
try
{
var trackerDocuments = _mongoDatabase.GetCollection("TrackerData");
//var sqlPersister = new PersistToSQLServer(_sqlConnectionString, new ConsoleLogger());
foreach (var trackerDocument in trackerDocuments.FindAll().Skip(startAt))
{
Debug.Write(trackerDocument);
TrackerData trackerData = null;
ITrackable tracker = null;
string trackerTypeName = null;
DateTime? timeSlot = null;
foreach (var element in trackerDocument.Elements)
{
if (element.Name == "TypeName")
{
trackerTypeName = element.Value.ToString();
}
}
if (trackerTypeName != null)
{
trackerData = new TrackerData(trackerTypeName, Resolution.Hour);
foreach (var element in trackerDocument.Elements)
{
switch (element.Name)
{
case "Name":
trackerData.Name = element.Value.ToString();
break;
case "KeyFilter":
trackerData.KeyFilter = element.Value.ToString();
break;
case "TimeSlot":
timeSlot = Convert.ToDateTime(element.Value.ToString());
break;
//case "SearchFilters":
// var filters =
// element.Value.AsBsonArray.Select(p => p.AsString).ToArray();
// for (var i = 0; i < filters.Length; i++)
// {
// filters[i] = filters[i].Split(new [] {",,"}, StringSplitOptions.None).OrderBy(f => f).Aggregate((x, z) => string.Concat(x, ",,", z));//this is to fix a bug that was caused by the missing sort in the previous version.
// }
// trakerData.SearchFilters = filters;
// break;
case "Measurement":
var mesurementDoc = (BsonDocument) element.Value;
trackerData.Measurement = new Measure();
trackerData.Measurement.NamedMetrics = new ConcurrentDictionary<string, long>();
foreach (var measure in mesurementDoc.Elements)
{
//switch (measure.Name)
//{
// case "_Total":
// trakerData.Measurement._Total = Convert.ToInt32(measure.Value.ToString());
// break;
// case "_Occurrence":
// trakerData.Measurement._Occurrence =
// Convert.ToInt32(measure.Value.ToString());
// break;
// case "_Min":
// trakerData.Measurement._Min = Convert.ToInt32(measure.Value.ToString());
// break;
// case "_Max":
// trakerData.Measurement._Max = Convert.ToInt32(measure.Value.ToString());
// break;
// default:
var val = Convert.ToInt32(measure.Value);
trackerData.Measurement.NamedMetrics.AddOrUpdate(measure.Name, val,
(i, t) => t + val);
// break;
//}
}
break;
}
}
tracker = TrackerBuilder.CreateTrackerFromTypeName(trackerTypeName, trackerData.Name);
object filter = null;
if (!string.IsNullOrWhiteSpace(trackerData.KeyFilter))
filter = TrackerBuilder.CreateFilterObject(trackerData.KeyFilter);
var grapheneContainer = typeof(Container<>).MakeGenericType(tracker.GetType());
var grapheneContainerInstance = Activator.CreateInstance(grapheneContainer);
// var grapheneSetTrackerType =
// grapheneContainer.GetMethod("SetTrackerType", BindingFlags.Instance | BindingFlags.NonPublic);
// grapheneSetTrackerType.Invoke(grapheneContainerInstance, new[] { tracker });
var grapheneIncrement =
grapheneContainer.GetMethod("Increment", BindingFlags.Instance | BindingFlags.NonPublic, Type.DefaultBinder,
new[] { typeof(ITrackable), typeof(Resolution), typeof(DateTime), typeof(string), typeof(long), typeof(object) }, null);
foreach (var metric in trackerData.Measurement.NamedMetrics)
{
grapheneIncrement.Invoke(grapheneContainerInstance, new[] {tracker, Resolution.Day, timeSlot, metric.Key, metric.Value, filter });
}
if (deleteRecordAfterMigration)
trackerDocuments.Remove(Query<TrackerData>.EQ(td => td._id, trackerData._id));
if (_logger != null)
_logger.Info(string.Format("Migrated Document with ID:{0}, Number{1}", trackerData._id,
docNumber));
docNumber++;
docsImported++;
if (_stopCalled || (stopAfter!=-1 && docsImported >= stopAfter))
break;
}
}
}
catch (Exception exception)
{
_logger.Error(exception.Message, exception);
if (exception is MongoDB.Driver.MongoQueryException && exception.Message == "Cursor not found.")//todo: find out the exception type.
{
Start(deleteRecordAfterMigration, docNumber, stopAfter, docsImported);
}
}
}
private void persitTracker(TrackerData trackerData)
{
if (trackerData.MinResolution == Resolution.NA || trackerData.TimeSlot.Kind == DateTimeKind.Local)
trackerData.TimeSlot = trackerData.TimeSlot.ToUniversalTime();
using (var connection = new SqlConnection(_connectionString))
{
connection.Open();
using (var command = connection.CreateCommand())
{
command.CommandText = "dbo.UpdateTracker";
command.CommandType = CommandType.StoredProcedure;
command.Parameters.Add("@TrackerID", SqlDbType.NVarChar);
command.Parameters["@TrackerID"].Value = String.Concat(trackerData.TypeName, "-", trackerData.KeyFilter).Replace(" ", "");
command.Parameters.Add("@Name", SqlDbType.NVarChar);
command.Parameters["@Name"].Value = trackerData.Name;
command.Parameters.Add("@TypeName", SqlDbType.NVarChar);
command.Parameters["@TypeName"].Value = trackerData.TypeName;
command.Parameters.Add("@MinResolution", SqlDbType.Int);
command.Parameters["@MinResolution"].Value = bucketResolutionToMinutes(trackerData.MinResolution);
command.Parameters.Add("@KeyFilter", SqlDbType.NVarChar);
command.Parameters["@KeyFilter"].Value = trackerData.KeyFilter;
command.Parameters.Add("@TimeSlot", SqlDbType.DateTime);
command.Parameters["@TimeSlot"].Value = _convertToUTC ? trackerData.TimeSlot.ToUniversalTime() : trackerData.TimeSlot;
SqlParameter flParameter;
flParameter = command.Parameters.AddWithValue("@FilterList", createFilterDataTable(trackerData));
flParameter.SqlDbType = SqlDbType.Structured;
flParameter.TypeName = "dbo.FilterList";
SqlParameter mParameter;
mParameter = command.Parameters.AddWithValue("@Measurement", createMeasurementDataTable(trackerData));
mParameter.SqlDbType = SqlDbType.Structured;
mParameter.TypeName = "dbo.Measurement";
using (var transaction = connection.BeginTransaction())
{
command.Transaction = transaction;
command.CommandTimeout = command.CommandTimeout * 2;
command.ExecuteNonQuery();
transaction.Commit();
}
}
}
}
private void Persist(TrackerData trackerData, int retryCount)
{
try
{
persitTracker(trackerData);
}
catch (DbException ex)
{
if (retryCount < _maxRetries)
{
_logger.Warn(ex.Message + ". Retrying.");
Persist(trackerData, retryCount+1);
}
else
{
_logger.Error(ex.Message, ex);
throw;
}
}
}
private DataTable createFilterDataTable(TrackerData trackerData)
{
var table = new DataTable();
table.Columns.Add("Filter", typeof(string));
foreach (string filter in trackerData.SearchFilters)
{
table.Rows.Add(filter);
}
return table;
}
private static DataTable createMeasurementDataTable(TrackerData trackerData)
{
var table = new DataTable();
table.Columns.Add("Name", typeof(string));
table.Columns.Add("Value", typeof(long));
table.Columns.Add("BucketResolution", typeof(int));
table.Columns.Add("CoversMinuteBucket", typeof(bool));
table.Columns.Add("CoversFiveMinuteBucket", typeof(bool));
table.Columns.Add("CoversFifteenMinuteBucket", typeof(bool));
table.Columns.Add("CoversThirtyMinuteBucket", typeof(bool));
table.Columns.Add("CoversHourBucket", typeof(bool));
table.Columns.Add("CoversDayBucket", typeof(bool));
table.Columns.Add("CoversMonthBucket", typeof(bool));
foreach (var metrics in trackerData.Measurement.NamedMetrics)
{
addToMeasurementTable(trackerData, table, bucketResolutionToMinutes(trackerData.Measurement.BucketResolution), metrics.Key, metrics.Value);
}
addToMeasurementTable(trackerData, table, bucketResolutionToMinutes(trackerData.Measurement.BucketResolution), "_Occurrence", trackerData.Measurement._Occurrence);
addToMeasurementTable(trackerData, table, bucketResolutionToMinutes(trackerData.Measurement.BucketResolution), "_Total", trackerData.Measurement._Total);
return table;
}