//public static bool PromoteMaster()
//{
// var tryCount = 0;
// do
// {
// try
// {
// using (var context = new DatastoreEntities())
// {
// var item = context.ServiceInstance.FirstOrDefault();
// if (item == null)
// {
// //There is no item in the table so create one
// context.AddItem(new ServiceInstance
// {
// FirstCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew),
// LastCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew),
// InstanceId = RepositoryManager.InstanceId,
// });
// context.SaveChanges();
// return true;
// }
// else if (item.InstanceId == RepositoryManager.InstanceId)
// {
// //Nothing to do. This service is already the master
// return true;
// }
// else
// {
// //The instance was successfully change to this service instance
// item.InstanceId = RepositoryManager.InstanceId;
// item.FirstCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew);
// item.LastCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew);
// context.SaveChanges();
// return true;
// }
// }
// }
// catch (Exception)
// {
// //If the record could not be created OR not updated try again
// tryCount++;
// }
// } while (tryCount < 3);
// return false;
//}
private static DateTime GetDatabaseTime()
{
try
{
using (var connection = new SqlConnection(ConnectionString))
{
connection.Open();
using (var command = connection.CreateCommand())
{
command.Connection = connection;
command.CommandText = "select GETUTCDATE() as [Time]";
command.CommandType = CommandType.Text;
var da = new SqlDataAdapter {
SelectCommand = command
};
var ds = new DataSet();
da.Fill(ds);
if (ds.Tables.Count == 1 && ds.Tables[0].Rows.Count == 1)
{
return((DateTime)ds.Tables[0].Rows[0][0]);
}
}
}
return(DateTime.MinValue);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(DateTime.MinValue);
}
}
private static void TimerHeartBeatElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
//Send the heart beat to the DB to coordinate all data store instances
try
{
_timerHeartBeat.Stop();
using (var context = new DatastoreEntities())
{
var item = context.ServiceInstance.FirstOrDefault();
if (item != null && item.InstanceId == RepositoryManager.InstanceId)
{
item.LastCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew);
context.SaveChanges();
CurrentMaster = item.InstanceId;
}
else if (item != null)
{
CurrentMaster = item.InstanceId;
}
else
{
CurrentMaster = Guid.Empty;
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
finally
{
_timerHeartBeat.Start();
}
}
public static void StartFullIndex()
{
if (!ConfigHelper.EnabledDataManager)
{
return;
}
try
{
//If need a full index then add all repositories to the list
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
var list = context.Repository
.Where(x => x.ParentId == null)
.Select(x => x.UniqueKey)
.ToList();
list.ForEach(x => _highPriority.Add(x));
}
InFullIndex = true;
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public Task Execute()
{
//Console.WriteLine("CountBuilder:Execute:Start");
return(Task.Factory.StartNew(() =>
{
try
{
//Nothing to do
if (_configuration.query.ExcludeCount)
{
return;
}
if (!_configuration.UseGroupingSets)
{
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, $"CountBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
//Nothing to do
if (!_configuration.query.IncludeDimensions)
{
return;
}
if (_doExecute)
{
var timer = Stopwatch.StartNew();
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
//Log long running
timer.Stop();
if (timer.ElapsedMilliseconds > 10000)
{
LoggerCQ.LogWarning($"NormalDimensionBuilderDelay: ID={_configuration.schema.ID}, Elapsed={timer.ElapsedMilliseconds}, Query=\"{_configuration.query.ToString()}\"");
}
}
catch (Exception ex)
{
RepositoryHealthMonitor.HealthCheck(_configuration.schema.ID);
DataManager.AddSkipItem(_configuration.schema.ID);
LoggerCQ.LogError(ex, $"NormalDimensionBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}
}));
}
public static void Log(RealtimeStats item)
{
if (!_ready)
{
return;
}
try
{
//TODO: Log Stat
//using (var context = new DataCoreEntities())
//{
// var newItem = new ServerStat()
// {
// MemoryUsageTotal = item.MemoryUsageTotal,
// MemoryUsageAvailable = item.MemoryUsageAvailable,
// MemoryUsageProcess = item.MemoryUsageProcess,
// RepositoryInMem = item.RepositoryInMem,
// RepositoryLoadDelta = item.RepositoryLoadDelta,
// RepositoryUnloadDelta = item.RepositoryUnloadDelta,
// RepositoryTotal = item.RepositoryTotal,
// RepositoryCreateDelta = item.RepositoryCreateDelta,
// RepositoryDeleteDelta = item.RepositoryDeleteDelta,
// ProcessorUsage = item.ProcessorUsage,
// };
// context.AddItem(newItem);
// context.SaveChanges();
//}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
//Nothing to do
if (!_configuration.query.IncludeDimensions && !_configuration.query.IncludeRecords)
{
return;
}
var timer = Stopwatch.StartNew();
try
{
_dsList = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _listParameters);
//Log long running
timer.Stop();
if (timer.ElapsedMilliseconds > 10000)
{
LoggerCQ.LogWarning($"ListDimensionBuilderDelay: ID={_configuration.schema.ID}, DIdx={_newDimension?.DIdx}, Elapsed={timer.ElapsedMilliseconds}, Query=\"{_configuration.query.ToString()}\"");
}
}
catch (Exception ex)
{
RepositoryHealthMonitor.HealthCheck(_configuration.schema.ID);
DataManager.AddSkipItem(_configuration.schema.ID);
var message = ex.Message;
if (message.Contains("Timeout Expired"))
{
message = "Timeout Expired"; //Do not show whole message, no value
}
LoggerCQ.LogError($"ListDimensionBuilder: ID={_configuration.schema.ID}, DIdx={_newDimension?.DIdx}, Elapsed={timer.ElapsedMilliseconds}, Query=\"{_configuration.query.ToString()}\", Error={message}");
}
}));
}
public Task Load()
{
return(Task.Factory.StartNew(() =>
{
try
{
//Nothing to do
if (_configuration.query.ExcludeCount)
{
return;
}
if (!_configuration.UseGroupingSets)
{
var v = (int)_datset.Tables[0].Rows[0][0];
lock (_configuration.retval)
{
_configuration.retval.TotalRecordCount = v;
}
_configuration.PerfLoadCount = true;
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, $"CountBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
public Task GenerateSql()
{
return(Task.Factory.StartNew(() =>
{
try
{
//Do Nothing
if (_configuration.query.DerivedFieldList == null)
{
return;
}
var aggList = _configuration.query.DerivedFieldList
.Where(x => _configuration.schema.FieldList.Select(z => z.Name).Contains(x.Field))
.ToList();
if (aggList.Count > 0)
{
var sb = new StringBuilder();
sb.AppendLine($"--MARKER 18" + _configuration.QueryPlanDebug);
sb.Append("SELECT ");
foreach (var field in aggList)
{
switch (field.Action)
{
case AggregateOperationConstants.Count:
sb.Append($"COUNT([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Max:
sb.Append($"MAX([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Min:
sb.Append($"MIN([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Sum:
sb.Append($"SUM([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Distinct:
sb.Append($"COUNT(DISTINCT [Z].[{field.Field}]), ");
break;
}
}
sb.AppendLine("0");
sb.AppendLine($"FROM [{_configuration.dataTable}] Z {SqlHelper.NoLockText()}{_configuration.innerJoinClause}");
sb.AppendLine($"WHERE {_configuration.whereClause}");
_sql = sb.ToString();
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, $"AggregateBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
private bool ParseExpression(MemberExpression memberExpr, Expression valueExpr, ComparisonConstants comparison)
{
try
{
if (memberExpr == null)
{
return(false);
}
var constantExpr = valueExpr as ConstantExpression;
if (constantExpr != null)
{
AddFieldFilter(memberExpr.Member.Name, constantExpr.Value, comparison);
return(true);
}
// This is kinda of weird the compiler creates an object for the variables in a where conditions
// For example, Where(x => x.Category != f)
var memberAccessValueExpr = valueExpr as MemberExpression;
if (memberAccessValueExpr != null)
{
var theValue = ExpressionHelper.GetMemberExpressionValue(memberAccessValueExpr);
AddFieldFilter(memberExpr.Member.Name, theValue, comparison);
return(true);
}
var unaryExpressionExpr = valueExpr as UnaryExpression;
if (unaryExpressionExpr != null)
{
var theValue = ExpressionHelper.GetMemberExpressionValue(unaryExpressionExpr);
AddFieldFilter(memberExpr.Member.Name, theValue, comparison);
return(true);
}
var methodExpressionExpr = valueExpr as MethodCallExpression;
if (methodExpressionExpr != null)
{
//Parse DateTime functions like AddDays, etc.
//if (methodExpressionExpr.Method.DeclaringType.FullName == "System.DateTime")
{
var objectMember = Expression.Convert(methodExpressionExpr, typeof(object));
var getterLambda = Expression.Lambda <Func <object> >(objectMember);
var getter = getterLambda.Compile();
var theValue = getter();
AddFieldFilter(memberExpr.Member.Name, theValue, comparison);
return(true);
}
}
return(false);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(false);
}
}
public void Cleanup()
{
try
{
_core.ShutDown();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public static RepositorySummmaryStats QueryRepositoryStats(Guid repositoryId, DateTime start, DateTime end)
{
if (!_ready)
{
return(null);
}
try
{
var retval = new RepositorySummmaryStats()
{
ActionType = RepositoryActionConstants.Query
};
//TODO: Query stats
//using (var context = new DataCoreEntities())
//{
// var repository = context.RepositoryDefinition.FirstOrDefault(x => x.UniqueKey == repositoryId);
// if (repository == null)
// throw new Exception("Unknown Repository");
// var actionId = (int)RepositoryActionConstants.Query;
// var lambda = context.RepositoryStat.Where(x => start <= x.CreatedDate &&
// x.CreatedDate < end &&
// (repositoryId == Guid.Empty || repository.RepositoryId == x.RepositoryId) &&
// x.RepositoryActionTypeId == actionId)
// .OrderBy(x => x.CreatedDate);
// long count = 0;
// long totalElapsed = 0;
// if (lambda.Any())
// {
// count = lambda.Sum(x => x.Count);
// totalElapsed = lambda.Sum(x => x.Elapsed);
// }
// var elapsedPer = 0.0;
// if (count > 0) elapsedPer = ((totalElapsed * 1.0) / count);
// retval.ItemCount = (int)count;
// retval.Elapsed = (int)elapsedPer;
// retval.RepositoryId = repositoryId;
//}
return(retval);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(null);
}
}
public static void Sync(List <DataItem> list, RepositorySchema schema)
{
try
{
Task.Factory.StartNew(() =>
{
SyncInternal(list, schema);
});
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
protected override void OnShutdown()
{
try
{
base.OnShutdown();
_core.ShutDown();
LoggerCQ.LogInfo("Services ShutDown");
//KillTimer();
}
catch (Exception ex)
{
LoggerCQ.LogError("Error 0x2401: Shutdown Failed");
throw;
}
}
private static int RepositoryItemCount(Guid id)
{
try
{
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
return(context.Repository.FirstOrDefault(x => x.UniqueKey == id)?.ItemCount ?? 0);
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(0);
}
}
public void Start()
{
try
{
//Do this to avoid an infinite hang if the firewall has blocked the port
//You cannot shut down the service if blocked because it never finishes startup
var t = new System.Threading.Thread(StartupEndpoint);
t.Start();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
throw;
}
}
public DimensionCache()
{
try
{
//Cull cache every minute
_timer = new System.Timers.Timer(60000);
_timer.Elapsed += _timer_Elapsed;
_timer.Start();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
//throw;
}
}
private static void TimerTick(object sender, System.Timers.ElapsedEventArgs e)
{
if (!_ready)
{
return;
}
_timer.Stop();
try
{
//Lock the stats list and build queries
List <LockInfoItem> copyCache = null;
lock (_cache)
{
copyCache = _cache.ToList();
_cache.Clear();
}
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
foreach (var item in copyCache)
{
var newItem = new LockStat()
{
CurrentReadCount = item.CurrentReadCount,
Elapsed = item.Elapsed,
Failure = item.Failure,
IsWriteLockHeld = item.IsWriteLockHeld,
ThreadId = item.ThreadId,
WaitingReadCount = item.WaitingReadCount,
WaitingWriteCount = item.WaitingWriteCount,
DateStamp = item.DateStamp,
TraceInfo = item.TraceInfo,
};
context.AddItem(newItem);
}
context.SaveChanges();
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
finally
{
_timer.Start();
}
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
}
catch (Exception ex)
{
RepositoryHealthMonitor.HealthCheck(_configuration.schema.ID);
DataManager.AddSkipItem(_configuration.schema.ID);
LoggerCQ.LogError(ex, $"RecordBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
public static void Initialize()
{
try
{
//Log repository stats every N seconds
_timer = new System.Timers.Timer(LOG_TIMER_INTERVAL);
_timer.AutoReset = false;
_timer.Elapsed += TimerTick;
_timer.Start();
_ready = true;
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public QueryCache()
{
try
{
_maxItems = ConfigHelper.QueryCacheCount;
//Cull cache every minute
_timer = new System.Timers.Timer(TIMECHECK);
_timer.Elapsed += TimerElapsed;
_timer.Start();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
//throw;
}
}
public static void Log(RepositorySummmaryStats item)
{
try
{
lock (_statCache)
{
if (!_statCache.ContainsKey(item.RepositoryId))
{
_statCache.Add(item.RepositoryId, new List <RepositorySummmaryStats>());
}
_statCache[item.RepositoryId].Add(item);
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
protected override void OnStop()
{
//KillTimer();
try
{
ConfigHelper.ShutDown();
if (_core != null)
{
_core.ShutDown();
}
}
catch (Exception ex)
{
LoggerCQ.LogError("Error 0x2400: Shutdown Failed");
}
LoggerCQ.LogInfo("Services Stopped");
}
public static void Log(LockInfoItem item)
{
try
{
if (!ConfigHelper.AllowLockStats)
{
return;
}
lock (_cache)
{
_cache.Add(item);
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public static List <RealtimeStats> QueryServerStats(DateTime start, DateTime end)
{
if (!_ready)
{
return(null);
}
try
{
var retval = new List <RealtimeStats>();
//TODO: Query stats
//using (var context = new DataCoreEntities())
//{
// var list = context.ServerStat
// .Where(x => start <= x.AddedDate && x.AddedDate < end).OrderBy(x => x.AddedDate)
// .ToList();
// foreach (var item in list)
// {
// retval.Add(new RealtimeStats
// {
// Timestamp = item.AddedDate,
// MemoryUsageAvailable = item.MemoryUsageAvailable,
// MemoryUsageProcess = item.MemoryUsageProcess,
// MemoryUsageTotal = item.MemoryUsageTotal,
// ProcessorUsage = item.ProcessorUsage,
// RepositoryCreateDelta = (int)item.RepositoryCreateDelta,
// RepositoryDeleteDelta = (int)item.RepositoryDeleteDelta,
// RepositoryInMem = item.RepositoryInMem,
// RepositoryLoadDelta = item.RepositoryLoadDelta,
// RepositoryTotal = item.RepositoryTotal,
// RepositoryUnloadDelta = item.RepositoryUnloadDelta,
// });
// }
//}
return(retval);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(null);
}
}
private static void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
_timer.Stop();
var timer = Stopwatch.StartNew();
var itemCount = 0;
var removed = 0;
var hitCount = 0;
try
{
foreach (var repositoryId in _cache.Keys)
{
lock (_cache)
{
var rLookup = _cache.GetOrAdd(repositoryId, key => new ConcurrentDictionary <string, LDCacheItem>());
foreach (var key in rLookup.Keys)
{
var cacheContainer = rLookup[key];
if (DateTime.Now.Subtract(cacheContainer.Timestamp).TotalMinutes >= MaxCacheTime)
{
removed += cacheContainer.Cache.Count;
LDCacheItem v;
var b = rLookup.TryRemove(key, out v);
}
else
{
itemCount += cacheContainer.Cache.Count;
hitCount += cacheContainer.HitCount;
}
}
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, "ListDimensionCache housekeeping failed");
}
finally
{
timer.Stop();
_timer.Start();
}
}
private static void Refresh()
{
//Do not rethrow error. If error just skip this and do NOT reset the settings object to null
try
{
lock (_syncObject)
{
var v = GetSettings();
if (v != null)
{
_settings = v;
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
//Do Nothing
if (_configuration.query.DerivedFieldList == null || string.IsNullOrEmpty(_sql))
{
return;
}
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}));
}
internal void Populate()
{
try
{
using (var context = new DatastoreEntities())
{
var list = context.Repository
.Select(x => new { x.RepositoryId, x.ParentId })
.ToList();
foreach (var item in list)
{
_schemaParentCache.TryAdd(item.RepositoryId, item.ParentId);
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, "Populate schema parent ID cache failed");
}
}
private static Dictionary <string, string> GetSettings()
{
try
{
using (var context = new DatastoreEntities(ConnectionString))
{
return(context.ConfigurationSetting.ToList().ToDictionary(x => x.Name.ToLower(), x => x.Value));
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("Timeout expired"))
{
LoggerCQ.LogWarning(ex, "ConfigHelper.GetSettings timeout expired");
}
else
{
LoggerCQ.LogError(ex);
}
return(null);
}
}