public static void StartFullIndex()
{
if (!ConfigHelper.EnabledDataManager)
{
return;
}
try
{
//If need a full index then add all repositories to the list
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
var list = context.Repository
.Where(x => x.ParentId == null)
.Select(x => x.UniqueKey)
.ToList();
list.ForEach(x => _highPriority.Add(x));
}
InFullIndex = true;
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
private void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
List <Gravitybox.Datastore.EFDAL.Entity.RepositoryLog> work;
try
{
//Copy the cache list and empty so other threads can continue to use it
using (var q = new AcquireReaderLock(QueryLogID, "QueryLog"))
{
work = _cache.ToList();
_cache.Clear();
}
//Save all of these to disk
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
foreach (var item in work)
{
context.AddItem(item);
}
context.SaveChanges();
}
}
catch (Exception ex)
{
LoggerCQ.LogWarning($"QueryLogManager: Error={ex.Message}");
}
}
/// <summary />
protected virtual void Dispose(bool disposing)
{
if (!m_Disposed)
{
if (disposing && m_Lock != null)
{
var traceInfo = m_Lock.TraceInfo;
var elapsed = (int)DateTime.Now.Subtract(_initTime).TotalMilliseconds;
if (!_inError)
{
if (_lockIndex != 0)
{
DateTime dt;
if (!m_Lock.HeldReads.TryRemove(_lockIndex, out dt))
{
LoggerCQ.LogWarning($"HeldReads was not released. ObjectId={m_Lock.ObjectId}, Index={_lockIndex}, TraceInfo={m_Lock.TraceInfo}, Elapsed={elapsed}");
}
_lockIndex = 0;
}
m_Lock.TraceInfo = null;
m_Lock.HoldingThreadId = null;
}
if (ConfigHelper.AllowReadLocking)
{
m_Lock.ExitReadLock();
if (elapsed > 60000)
{
LoggerCQ.LogWarning($"ReaderLock Long: Elapsed={elapsed}, ID={_id}");
}
}
}
}
m_Disposed = true;
}
public static void Log(RealtimeStats item)
{
if (!_ready)
{
return;
}
try
{
//TODO: Log Stat
//using (var context = new DataCoreEntities())
//{
// var newItem = new ServerStat()
// {
// MemoryUsageTotal = item.MemoryUsageTotal,
// MemoryUsageAvailable = item.MemoryUsageAvailable,
// MemoryUsageProcess = item.MemoryUsageProcess,
// RepositoryInMem = item.RepositoryInMem,
// RepositoryLoadDelta = item.RepositoryLoadDelta,
// RepositoryUnloadDelta = item.RepositoryUnloadDelta,
// RepositoryTotal = item.RepositoryTotal,
// RepositoryCreateDelta = item.RepositoryCreateDelta,
// RepositoryDeleteDelta = item.RepositoryDeleteDelta,
// ProcessorUsage = item.ProcessorUsage,
// };
// context.AddItem(newItem);
// context.SaveChanges();
//}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
private static void TimerHeartBeatElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
//Send the heart beat to the DB to coordinate all data store instances
try
{
_timerHeartBeat.Stop();
using (var context = new DatastoreEntities())
{
var item = context.ServiceInstance.FirstOrDefault();
if (item != null && item.InstanceId == RepositoryManager.InstanceId)
{
item.LastCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew);
context.SaveChanges();
CurrentMaster = item.InstanceId;
}
else if (item != null)
{
CurrentMaster = item.InstanceId;
}
else
{
CurrentMaster = Guid.Empty;
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
finally
{
_timerHeartBeat.Start();
}
}
private void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
_timer.Stop();
try
{
lock (_cache)
{
//Keep only the last N items
var l = _cache
.OrderByDescending(x => x.Timestamp)
.Skip(MAXITEMS)
.ToList();
l.ForEach(x => _cache.Remove(x));
//Purge anything not used in an hour
l.Where(x => DateTime.Now.Subtract(x.Timestamp).TotalMinutes >= 60)
.ToList()
.ForEach(x => _cache.Remove(x));
}
}
catch (Exception ex)
{
LoggerCQ.LogWarning(ex, "DimensionCache housekeeping failed");
}
finally
{
_timer.Start();
}
}
//public static bool PromoteMaster()
//{
// var tryCount = 0;
// do
// {
// try
// {
// using (var context = new DatastoreEntities())
// {
// var item = context.ServiceInstance.FirstOrDefault();
// if (item == null)
// {
// //There is no item in the table so create one
// context.AddItem(new ServiceInstance
// {
// FirstCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew),
// LastCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew),
// InstanceId = RepositoryManager.InstanceId,
// });
// context.SaveChanges();
// return true;
// }
// else if (item.InstanceId == RepositoryManager.InstanceId)
// {
// //Nothing to do. This service is already the master
// return true;
// }
// else
// {
// //The instance was successfully change to this service instance
// item.InstanceId = RepositoryManager.InstanceId;
// item.FirstCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew);
// item.LastCommunication = DateTime.UtcNow.AddSeconds(-_serverTimeSkew);
// context.SaveChanges();
// return true;
// }
// }
// }
// catch (Exception)
// {
// //If the record could not be created OR not updated try again
// tryCount++;
// }
// } while (tryCount < 3);
// return false;
//}
private static DateTime GetDatabaseTime()
{
try
{
using (var connection = new SqlConnection(ConnectionString))
{
connection.Open();
using (var command = connection.CreateCommand())
{
command.Connection = connection;
command.CommandText = "select GETUTCDATE() as [Time]";
command.CommandType = CommandType.Text;
var da = new SqlDataAdapter {
SelectCommand = command
};
var ds = new DataSet();
da.Fill(ds);
if (ds.Tables.Count == 1 && ds.Tables[0].Rows.Count == 1)
{
return((DateTime)ds.Tables[0].Rows[0][0]);
}
}
}
return(DateTime.MinValue);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(DateTime.MinValue);
}
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
//Nothing to do
if (!_configuration.query.IncludeDimensions)
{
return;
}
if (_doExecute)
{
var timer = Stopwatch.StartNew();
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
//Log long running
timer.Stop();
if (timer.ElapsedMilliseconds > 10000)
{
LoggerCQ.LogWarning($"NormalDimensionBuilderDelay: ID={_configuration.schema.ID}, Elapsed={timer.ElapsedMilliseconds}, Query=\"{_configuration.query.ToString()}\"");
}
}
catch (Exception ex)
{
RepositoryHealthMonitor.HealthCheck(_configuration.schema.ID);
DataManager.AddSkipItem(_configuration.schema.ID);
LoggerCQ.LogError(ex, $"NormalDimensionBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}
}));
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
//Nothing to do
if (!_configuration.query.IncludeDimensions && !_configuration.query.IncludeRecords)
{
return;
}
var timer = Stopwatch.StartNew();
try
{
_dsList = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _listParameters);
//Log long running
timer.Stop();
if (timer.ElapsedMilliseconds > 10000)
{
LoggerCQ.LogWarning($"ListDimensionBuilderDelay: ID={_configuration.schema.ID}, DIdx={_newDimension?.DIdx}, Elapsed={timer.ElapsedMilliseconds}, Query=\"{_configuration.query.ToString()}\"");
}
}
catch (Exception ex)
{
RepositoryHealthMonitor.HealthCheck(_configuration.schema.ID);
DataManager.AddSkipItem(_configuration.schema.ID);
var message = ex.Message;
if (message.Contains("Timeout Expired"))
{
message = "Timeout Expired"; //Do not show whole message, no value
}
LoggerCQ.LogError($"ListDimensionBuilder: ID={_configuration.schema.ID}, DIdx={_newDimension?.DIdx}, Elapsed={timer.ElapsedMilliseconds}, Query=\"{_configuration.query.ToString()}\", Error={message}");
}
}));
}
public Task Execute()
{
//Console.WriteLine("CountBuilder:Execute:Start");
return(Task.Factory.StartNew(() =>
{
try
{
//Nothing to do
if (_configuration.query.ExcludeCount)
{
return;
}
if (!_configuration.UseGroupingSets)
{
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, $"CountBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
public Task Load()
{
return(Task.Factory.StartNew(() =>
{
try
{
//Nothing to do
if (_configuration.query.ExcludeCount)
{
return;
}
if (!_configuration.UseGroupingSets)
{
var v = (int)_datset.Tables[0].Rows[0][0];
lock (_configuration.retval)
{
_configuration.retval.TotalRecordCount = v;
}
_configuration.PerfLoadCount = true;
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, $"CountBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
private void _timerStats_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
try
{
if (DateTime.Now.Subtract(_lastTimerStats).TotalHours <= 12)
{
return;
}
_lastTimerStats = DateTime.Now;
HashSet <Guid> copy = null;
lock (_statisticsCache)
{
copy = new HashSet <Guid>(_statisticsCache);
_statisticsCache.Clear();
}
var timer = Stopwatch.StartNew();
foreach (var g in copy)
{
SqlHelper.UpdateStatistics(g);
}
timer.Stop();
if (copy.Count > 0)
{
LoggerCQ.LogDebug($"Update table statistics: Count={copy.Count}, Elapsed={timer.ElapsedMilliseconds}");
}
}
catch (Exception ex)
{
LoggerCQ.LogWarning(ex);
}
}
private void TimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
var lockTime = 0;
var cacheCount = 0;
_timer.Stop();
try
{
var timer = Stopwatch.StartNew();
var count = 0;
var allCaches = RepositoryCacheManager.All;
cacheCount = allCaches.Count;
foreach (var cache in allCaches)
{
using (var q = new AcquireWriterLock(ServerUtilities.RandomizeGuid(cache.ID, RSeed), "QueryCache"))
{
lockTime += q.LockTime;
_maxItems = System.Math.Max(0, ConfigHelper.QueryCacheCount);
//Purge anything not used in the last N minutes
count += cache.RemoveAll(x => DateTime.Now.Subtract(x.Timestamp).TotalMinutes >= CacheExpireMinutes);
//Keep only the last N items
cache.OrderByDescending(x => x.Timestamp)
.Skip(_maxItems)
.ToList()
.ForEach(x => { cache.Remove(x); count++; });
}
}
#region Now do the Slices
//Keep only the last N items
_cacheSlice.OrderByDescending(x => x.Timestamp)
.Skip(_maxItems)
.ToList()
.ForEach(x => { _cacheSlice.Remove(x); count++; });
//Purge anything not used in the last N minutes
count += _cacheSlice.RemoveAll(x => DateTime.Now.Subtract(x.Timestamp).TotalMinutes >= CacheExpireMinutes);
#endregion
timer.Stop();
//Log it if too long
if (timer.ElapsedMilliseconds > 2000)
{
LoggerCQ.LogWarning($"QueryCache housekeeping: Elapsed={timer.ElapsedMilliseconds}, LockTime={lockTime}, ItemsRemoved={count}, CacheCount={cacheCount}");
}
}
catch (Exception ex)
{
LoggerCQ.LogWarning(ex, "QueryCache housekeeping failed");
}
finally
{
_timer.Start();
}
}
private static void SyncInternal(List <DataItem> list, RepositorySchema schema)
{
var timer = Stopwatch.StartNew();
var count = 0;
var processed = 0;
long lastRIdx = 0;
try
{
if (list == null)
{
return;
}
if (schema == null)
{
return;
}
var l = list.Where(x => x.__Hash == 0 && x.__RecordIndex > 0).ToList();
if (!l.Any())
{
return;
}
count = l.Count;
var dataTable = SqlHelper.GetTableName(schema.ID);
foreach (var item in l)
{
var sb = new StringBuilder();
var parameters = new List <SqlParameter>();
parameters.Add(new SqlParameter
{
DbType = DbType.Int64,
IsNullable = false,
ParameterName = $"@{SqlHelper.HashField}",
Value = item.Hash(),
});
parameters.Add(new SqlParameter
{
DbType = DbType.Int64,
IsNullable = false,
ParameterName = $"@{SqlHelper.RecordIdxField}",
Value = item.__RecordIndex,
});
lastRIdx = item.__RecordIndex;
sb.AppendLine($"UPDATE [{dataTable}] SET [{SqlHelper.HashField}] = @{SqlHelper.HashField} WHERE [{SqlHelper.RecordIdxField}] = @{SqlHelper.RecordIdxField} AND [{SqlHelper.HashField}] = 0");
SqlHelper.ExecuteSql(ConfigHelper.ConnectionString, sb.ToString(), parameters, false, false, 5);
Interlocked.Increment(ref _counter);
processed++;
}
}
catch (Exception ex)
{
LoggerCQ.LogWarning($"DataManager.SyncInternal: ID={schema.ID}, Count={count}, Processed={processed}, RIdx={lastRIdx}, Elapsed={timer.ElapsedMilliseconds}");
}
}
private bool ParseExpression(MemberExpression memberExpr, Expression valueExpr, ComparisonConstants comparison)
{
try
{
if (memberExpr == null)
{
return(false);
}
var constantExpr = valueExpr as ConstantExpression;
if (constantExpr != null)
{
AddFieldFilter(memberExpr.Member.Name, constantExpr.Value, comparison);
return(true);
}
// This is kinda of weird the compiler creates an object for the variables in a where conditions
// For example, Where(x => x.Category != f)
var memberAccessValueExpr = valueExpr as MemberExpression;
if (memberAccessValueExpr != null)
{
var theValue = ExpressionHelper.GetMemberExpressionValue(memberAccessValueExpr);
AddFieldFilter(memberExpr.Member.Name, theValue, comparison);
return(true);
}
var unaryExpressionExpr = valueExpr as UnaryExpression;
if (unaryExpressionExpr != null)
{
var theValue = ExpressionHelper.GetMemberExpressionValue(unaryExpressionExpr);
AddFieldFilter(memberExpr.Member.Name, theValue, comparison);
return(true);
}
var methodExpressionExpr = valueExpr as MethodCallExpression;
if (methodExpressionExpr != null)
{
//Parse DateTime functions like AddDays, etc.
//if (methodExpressionExpr.Method.DeclaringType.FullName == "System.DateTime")
{
var objectMember = Expression.Convert(methodExpressionExpr, typeof(object));
var getterLambda = Expression.Lambda <Func <object> >(objectMember);
var getter = getterLambda.Compile();
var theValue = getter();
AddFieldFilter(memberExpr.Member.Name, theValue, comparison);
return(true);
}
}
return(false);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(false);
}
}
public Task GenerateSql()
{
return(Task.Factory.StartNew(() =>
{
try
{
//Do Nothing
if (_configuration.query.DerivedFieldList == null)
{
return;
}
var aggList = _configuration.query.DerivedFieldList
.Where(x => _configuration.schema.FieldList.Select(z => z.Name).Contains(x.Field))
.ToList();
if (aggList.Count > 0)
{
var sb = new StringBuilder();
sb.AppendLine($"--MARKER 18" + _configuration.QueryPlanDebug);
sb.Append("SELECT ");
foreach (var field in aggList)
{
switch (field.Action)
{
case AggregateOperationConstants.Count:
sb.Append($"COUNT([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Max:
sb.Append($"MAX([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Min:
sb.Append($"MIN([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Sum:
sb.Append($"SUM([Z].[{field.Field}]), ");
break;
case AggregateOperationConstants.Distinct:
sb.Append($"COUNT(DISTINCT [Z].[{field.Field}]), ");
break;
}
}
sb.AppendLine("0");
sb.AppendLine($"FROM [{_configuration.dataTable}] Z {SqlHelper.NoLockText()}{_configuration.innerJoinClause}");
sb.AppendLine($"WHERE {_configuration.whereClause}");
_sql = sb.ToString();
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex, $"AggregateBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
public static void HealthCheck(Guid id)
{
lock (_repositoryList)
{
if (!_repositoryList.Contains(id))
{
_repositoryList.Add(id);
LoggerCQ.LogTrace($"HealthCheck queued: RepositoryId={id}");
}
}
}
public void MarkRefreshStats(Guid repositoryId)
{
lock (_statisticsCache)
{
if (!_statisticsCache.Contains(repositoryId))
{
_statisticsCache.Add(repositoryId);
LoggerCQ.LogInfo($"TableStatsMaintenace MarkRefreshStats: ID={repositoryId}");
}
}
}
public void Cleanup()
{
try
{
_core.ShutDown();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public static RepositorySummmaryStats QueryRepositoryStats(Guid repositoryId, DateTime start, DateTime end)
{
if (!_ready)
{
return(null);
}
try
{
var retval = new RepositorySummmaryStats()
{
ActionType = RepositoryActionConstants.Query
};
//TODO: Query stats
//using (var context = new DataCoreEntities())
//{
// var repository = context.RepositoryDefinition.FirstOrDefault(x => x.UniqueKey == repositoryId);
// if (repository == null)
// throw new Exception("Unknown Repository");
// var actionId = (int)RepositoryActionConstants.Query;
// var lambda = context.RepositoryStat.Where(x => start <= x.CreatedDate &&
// x.CreatedDate < end &&
// (repositoryId == Guid.Empty || repository.RepositoryId == x.RepositoryId) &&
// x.RepositoryActionTypeId == actionId)
// .OrderBy(x => x.CreatedDate);
// long count = 0;
// long totalElapsed = 0;
// if (lambda.Any())
// {
// count = lambda.Sum(x => x.Count);
// totalElapsed = lambda.Sum(x => x.Elapsed);
// }
// var elapsedPer = 0.0;
// if (count > 0) elapsedPer = ((totalElapsed * 1.0) / count);
// retval.ItemCount = (int)count;
// retval.Elapsed = (int)elapsedPer;
// retval.RepositoryId = repositoryId;
//}
return(retval);
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(null);
}
}
public static void Sync(List <DataItem> list, RepositorySchema schema)
{
try
{
Task.Factory.StartNew(() =>
{
SyncInternal(list, schema);
});
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
protected override void OnShutdown()
{
try
{
base.OnShutdown();
_core.ShutDown();
LoggerCQ.LogInfo("Services ShutDown");
//KillTimer();
}
catch (Exception ex)
{
LoggerCQ.LogError("Error 0x2401: Shutdown Failed");
throw;
}
}
public void Start()
{
try
{
//Do this to avoid an infinite hang if the firewall has blocked the port
//You cannot shut down the service if blocked because it never finishes startup
var t = new System.Threading.Thread(StartupEndpoint);
t.Start();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
throw;
}
}
private static int RepositoryItemCount(Guid id)
{
try
{
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
return(context.Repository.FirstOrDefault(x => x.UniqueKey == id)?.ItemCount ?? 0);
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
return(0);
}
}
public DimensionCache()
{
try
{
//Cull cache every minute
_timer = new System.Timers.Timer(60000);
_timer.Elapsed += _timer_Elapsed;
_timer.Start();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
//throw;
}
}
private static void TimerTick(object sender, System.Timers.ElapsedEventArgs e)
{
if (!_ready)
{
return;
}
_timer.Stop();
try
{
//Lock the stats list and build queries
List <LockInfoItem> copyCache = null;
lock (_cache)
{
copyCache = _cache.ToList();
_cache.Clear();
}
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
foreach (var item in copyCache)
{
var newItem = new LockStat()
{
CurrentReadCount = item.CurrentReadCount,
Elapsed = item.Elapsed,
Failure = item.Failure,
IsWriteLockHeld = item.IsWriteLockHeld,
ThreadId = item.ThreadId,
WaitingReadCount = item.WaitingReadCount,
WaitingWriteCount = item.WaitingWriteCount,
DateStamp = item.DateStamp,
TraceInfo = item.TraceInfo,
};
context.AddItem(newItem);
}
context.SaveChanges();
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
finally
{
_timer.Start();
}
}
public Task Execute()
{
return(Task.Factory.StartNew(() =>
{
try
{
_datset = SqlHelper.GetDataset(ConfigHelper.ConnectionString, _sql, _configuration.parameters);
}
catch (Exception ex)
{
RepositoryHealthMonitor.HealthCheck(_configuration.schema.ID);
DataManager.AddSkipItem(_configuration.schema.ID);
LoggerCQ.LogError(ex, $"RecordBuilder: ID={_configuration.schema.ID}, Query=\"{_configuration.query.ToString()}\", Error={ex.Message}");
}
}));
}
private static void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
if (!ConfigHelper.EnabledDataManager)
{
return;
}
try
{
var value = Interlocked.Read(ref _counter);
Interlocked.Exchange(ref _counter, 0);
if (value > 0)
{
LoggerCQ.LogDebug($"DataManager: ResetCount={value}");
}
}
catch { }
}
public static void Initialize()
{
try
{
//Log repository stats every N seconds
_timer = new System.Timers.Timer(LOG_TIMER_INTERVAL);
_timer.AutoReset = false;
_timer.Elapsed += TimerTick;
_timer.Start();
_ready = true;
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
}
public QueryCache()
{
try
{
_maxItems = ConfigHelper.QueryCacheCount;
//Cull cache every minute
_timer = new System.Timers.Timer(TIMECHECK);
_timer.Elapsed += TimerElapsed;
_timer.Start();
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
//throw;
}
}
