public BiDefinition Get(string resourceType, string id) { try { var rt = this.GetResourceType(resourceType); var retVal = this.m_metadataRepository.GetType().GetGenericMethod(nameof(IBiMetadataRepository.Get), new Type[] { rt }, new Type[] { typeof(String) }).Invoke(this.m_metadataRepository, new object[] { id }) as BiDefinition; // Resolve any refs in the object if (retVal == null) { throw new KeyNotFoundException(id); } else { retVal = BiUtils.ResolveRefs(retVal); (retVal as BiReportDefinition)?.Views?.ForEach(o => o.IncludeBody = true); return(retVal); } } catch (Exception e) { this.m_tracer.TraceError("Error executing BIS Get: {0}", e); throw; } }
/// <summary> /// Run the specified job /// </summary> public void Run(object sender, EventArgs e, object[] parameters) { var biProvider = ApplicationServiceContext.Current.GetService <IBiDataSource>(); // Global default var biRepository = ApplicationServiceContext.Current.GetService <IBiMetadataRepository>(); // Global default var serviceManager = ApplicationServiceContext.Current.GetService <IServiceManager>(); // try { this.m_tracer.TraceInfo("Starting refresh of defined BI materialized views"); this.m_cancel = false; this.m_stateManager.SetState(this, JobStateType.Running); // TODO: Refactor on new enhanced persistence layer definition using (AuthenticationContext.EnterSystemContext()) { var definitions = biRepository.Query <BiQueryDefinition>(o => o.MetaData.Status != BiDefinitionStatus.Deprecated && o.MetaData.Status != BiDefinitionStatus.Obsolete, 0, 100).ToArray(); int i = 0; foreach (var itm in definitions) { if (parameters.Length > 0 && !String.IsNullOrEmpty(parameters[0]?.ToString()) && parameters[0]?.Equals(itm.Id) != true) { continue; } this.m_stateManager.SetProgress(this, $"Refreshing {itm.Name ?? itm.Id}", ((float)i++ / (float)definitions.Length)); var dataSource = biProvider; var queryDefinition = BiUtils.ResolveRefs(itm) as BiQueryDefinition; var providerType = queryDefinition.DataSources.FirstOrDefault()?.ProviderType; if (providerType != null) { dataSource = serviceManager.CreateInjected(providerType) as IBiDataSource; } dataSource.RefreshMaterializedView(itm); if (this.m_cancel) { this.m_stateManager.SetState(this, JobStateType.Cancelled); return; } } } this.m_stateManager.SetState(this, JobStateType.Completed); } catch (Exception ex) { this.m_stateManager.SetState(this, JobStateType.Aborted); this.m_stateManager.SetProgress(this, ex.Message, 0.0f); this.m_cancel = false; this.m_tracer.TraceError("Error processing BI materialized views: {0}", ex.Message); throw new Exception("Error running BI refresh job", ex); } finally { this.m_cancel = false; } }
/// <summary> /// Executes the specified view /// </summary> public BisResultContext ExecuteView(BiViewDefinition viewDef, IDictionary <string, object> parameters, int offset, int?count) { viewDef = BiUtils.ResolveRefs(viewDef) as BiViewDefinition; var retVal = this.ExecuteQuery(viewDef.Query, parameters, viewDef.AggregationDefinitions?.ToArray(), offset, count); if (viewDef.Pivot != null) { retVal = ApplicationServiceContext.Current.GetService <IBiPivotProvider>().Pivot(retVal, viewDef.Pivot); } return(retVal); }
/// <summary> /// Refresh materialized view /// </summary> public void RefreshMaterializedView(BiQueryDefinition materializeDefinition) { if (materializeDefinition == null) { throw new ArgumentNullException(nameof(materializeDefinition)); } materializeDefinition = BiUtils.ResolveRefs(materializeDefinition); // The ADO.NET provider only allows one connection to one db at a time, so verify the connections are appropriate if (materializeDefinition.DataSources?.Count != 1) { throw new InvalidOperationException($"ADO.NET BI queries can only source data from 1 connection source, query {materializeDefinition.Name} has {materializeDefinition.DataSources?.Count}"); } // We want to open the specified connection var provider = this.GetProvider(materializeDefinition); // Query definition var rdbmsQueryDefinition = this.GetSqlDefinition(materializeDefinition, provider); if (rdbmsQueryDefinition.Materialize == null) { return; // no materialized view } else if (String.IsNullOrEmpty(rdbmsQueryDefinition.Materialize.Name)) { throw new InvalidOperationException($"Materialization on {materializeDefinition.Id} must have a unique name"); } // Get connection and execute if (provider.Features.HasFlag(SqlEngineFeatures.MaterializedViews)) { using (var context = provider.GetWriteConnection()) { try { context.Open(); context.CommandTimeout = 360000; context.ExecuteNonQuery(new SqlStatement(provider, provider.CreateSqlKeyword(SqlKeyword.RefreshMaterializedView)) .Append(rdbmsQueryDefinition.Materialize.Name)); } catch (Exception e) { throw new DataPersistenceException($"Error refreshing materialized view for {materializeDefinition.Id}", e); } } } }
/// <summary> /// DI constructor /// </summary> public LocalBiRenderService(IServiceManager serviceManager, IJobManagerService jobManager, IBiMetadataRepository metadataRepository, IBiDataSource defaultDataSource = null) { this.m_serviceManager = serviceManager; var job = serviceManager.CreateInjected <BiMaterializeJob>(); jobManager.AddJob(job, JobStartType.TimerOnly); // Set default job if (jobManager.GetJobSchedules(job)?.Any() != true) { jobManager.SetJobSchedule(job, new DayOfWeek[] { DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday, DayOfWeek.Saturday, DayOfWeek.Sunday }, new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0)); // First run for tomorrow } // Scan and initialize all BI materialized views ApplicationServiceContext.Current.Started += (o, e) => { foreach (var itm in metadataRepository.Query <BiQueryDefinition>(x => x.MetaData.Status == BiDefinitionStatus.Active, 0, 100)) { try { IBiDataSource dataSource = null; var queryDefinition = BiUtils.ResolveRefs(itm) as BiQueryDefinition; var providerType = queryDefinition.DataSources.FirstOrDefault()?.ProviderType; if (providerType != null) { dataSource = this.m_serviceManager.CreateInjected(providerType) as IBiDataSource; } else { dataSource = defaultDataSource; } this.m_tracer.TraceInfo("Materializing views for {0}", queryDefinition.Id); dataSource.CreateMaterializedView(queryDefinition); } catch (Exception ex) { this.m_tracer.TraceWarning("Could not initialize materialized views for {0} - {1}", itm.Id, ex.Message); } } }; }
/// <summary> /// Executes the query /// </summary> public BisResultContext ExecuteQuery(BiQueryDefinition queryDefinition, IDictionary <string, object> parameters, BiAggregationDefinition[] aggregation, int offset, int?count) { if (queryDefinition == null) { throw new ArgumentNullException(nameof(queryDefinition)); } // First we want to grab the connection strings used by this object var filledQuery = BiUtils.ResolveRefs(queryDefinition); // The ADO.NET provider only allows one connection to one db at a time, so verify the connections are appropriate if (queryDefinition.DataSources?.Count != 1) { throw new InvalidOperationException($"ADO.NET BI queries can only source data from 1 connection source, query {queryDefinition.Name} has {queryDefinition.DataSources?.Count}"); } // Ensure we have sufficient priviledge var demandList = queryDefinition.DataSources.SelectMany(o => o?.MetaData.Demands); if (queryDefinition.MetaData?.Demands != null) { demandList = demandList.Union(queryDefinition.MetaData?.Demands); } foreach (var pol in demandList) { ApplicationServiceContext.Current.GetService <IPolicyEnforcementService>().Demand(pol); } // Apply defaults where possible foreach (var defaultParm in queryDefinition.Parameters.Where(p => !String.IsNullOrEmpty(p.DefaultValue) && !parameters.ContainsKey(p.Name))) { parameters.Add(defaultParm.Name, defaultParm.DefaultValue); } // Next we validate parameters if (!queryDefinition.Parameters.Where(p => p.Required == true).All(p => parameters.ContainsKey(p.Name))) { throw new InvalidOperationException("Missing required parameter"); } // Validate parameter values foreach (var kv in parameters.ToArray()) { var parmDef = queryDefinition.Parameters.FirstOrDefault(p => p.Name == kv.Key); if (parmDef == null) { continue; // skip } else { switch (parmDef.Type) { case BiDataType.Boolean: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else if (parmDef.Multiple && parameters[kv.Key] is IEnumerable <String> arr) { parameters[kv.Key] = arr.Select(o => Boolean.Parse(o)).ToArray(); } else { parameters[kv.Key] = Boolean.Parse(kv.Value.ToString()); } break; case BiDataType.Date: case BiDataType.DateTime: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else if (parmDef.Multiple && parameters[kv.Key] is IEnumerable <String> arr) { parameters[kv.Key] = arr.Select(o => DateTime.Parse(o)).ToArray(); } else { parameters[kv.Key] = DateTime.Parse(kv.Value.ToString()); } break; case BiDataType.Integer: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else if (parmDef.Multiple && parameters[kv.Key] is IEnumerable <String> arr) { parameters[kv.Key] = arr.Select(o => Int32.Parse(o)).ToArray(); } else { parameters[kv.Key] = Int32.Parse(kv.Value.ToString()); } break; case BiDataType.String: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else if (parmDef.Multiple && parameters[kv.Key] is IEnumerable <String> arr) { parameters[kv.Key] = arr.ToArray(); } else { parameters[kv.Key] = kv.Value; } break; case BiDataType.Uuid: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else if (parmDef.Multiple && parameters[kv.Key] is IEnumerable <String> arr) { parameters[kv.Key] = arr.Select(o => Guid.Parse(o)).ToArray(); } else { parameters[kv.Key] = Guid.Parse(kv.Value.ToString()); } break; default: throw new InvalidOperationException($"Cannot determine how to parse {parmDef.Type}"); } } } // We want to open the specified connection var provider = "sqlite"; var connectionString = ApplicationServiceContext.Current.GetService <IConfigurationManager>().GetSection <DataConfigurationSection>().ConnectionString.FirstOrDefault(o => o.Name == queryDefinition.DataSources.First().ConnectionString); // Query definition var rdbmsQueryDefinition = queryDefinition.QueryDefinitions.FirstOrDefault(o => o.Invariants.Contains(provider)); if (rdbmsQueryDefinition == null) { throw new InvalidOperationException($"Could not find a query definition for invariant {provider}"); } // Prepare the templated SQL var parmRegex = new Regex(@"\$\{([\w_][\-\d\w\._]*?)\}"); List <Object> values = new List <object>(); var stmt = parmRegex.Replace(rdbmsQueryDefinition.Sql, (m) => { object pValue = null; parameters.TryGetValue(m.Groups[1].Value, out pValue); if (pValue is Array arr) { values.AddRange(arr.OfType <Object>()); return(string.Join(",", arr.OfType <Object>().Select(o => "?"))); } else { values.Add(pValue); return("?"); } }); // Aggregation definitions if (aggregation?.Length > 0) { var agg = aggregation.FirstOrDefault(o => o.Invariants?.Contains(provider) == true) ?? aggregation.FirstOrDefault(o => o.Invariants?.Count == 0) ?? aggregation.FirstOrDefault(o => o.Invariants == null); // Aggregation found if (agg == null) { throw new InvalidOperationException($"No provided aggregation can be found for {provider}"); } var selector = agg.Columns?.Select(c => { switch (c.Aggregation) { case BiAggregateFunction.Average: return($"AVG({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Count: return($"COUNT({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.CountDistinct: return($"COUNT(DISTINCT {c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.First: return($"FIRST({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Last: return($"LAST({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Max: return($"MAX({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Min: return($"MIN({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Sum: return($"SUM({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Value: return($"{c.ColumnSelector} AS {c.Name}"); default: throw new InvalidOperationException("Cannot apply aggregation function"); } }).ToArray() ?? new string[] { "*" }; String[] groupings = agg.Groupings.Select(g => g.ColumnSelector).ToArray(), colGroupings = agg.Groupings.Select(g => $"{g.ColumnSelector} AS {g.Name}").ToArray(); // Aggregate stmt = $"SELECT {String.Join(",", colGroupings.Union(selector))} " + $"FROM ({stmt}) AS _inner " + $"GROUP BY {String.Join(",", groupings)}"; } // Get a readonly connection var connParts = new SqliteConnectionStringBuilder(connectionString.Value); var file = connParts["dbfile"]; var enc = connParts["encrypt"]; using (SQLiteConnectionManager.Current.ExternLock(connectionString.Name)) using (var conn = new SqliteConnection($"Data Source=\"{file}\"")) { try { // Decrypt database var securityKey = ApplicationContext.Current.GetCurrentContextSecurityKey(); if (securityKey != null && (enc ?? "true").Equals("true")) { conn.SetPassword(securityKey); } // Open the database conn.Open(); // Attach any other connection sources foreach (var itm in queryDefinition.DataSources.Skip(1)) { using (var attcmd = conn.CreateCommand()) { var cstr = ApplicationContext.Current.ConfigurationManager.GetConnectionString(itm.ConnectionString); if (cstr.GetComponent("encrypt") == "true") { attcmd.CommandText = $"ATTACH DATABASE '{cstr.GetComponent("dbfile")}' AS {itm.Identifier} KEY ''"; } else { attcmd.CommandText = $"ATTACH DATABASE '{cstr.GetComponent("dbfile")}' AS {itm.Identifier} KEY X'{BitConverter.ToString(ApplicationContext.Current.GetCurrentContextSecurityKey()).Replace("-", "")}'"; } attcmd.CommandType = System.Data.CommandType.Text; attcmd.ExecuteNonQuery(); } } // Start time DateTime startTime = DateTime.Now; var sqlStmt = new SqlStatement(stmt, values.ToArray()).Limit(count ?? 10000).Offset(offset).Build(); this.m_tracer.TraceInfo("Executing BI Query: {0}", sqlStmt.Build().SQL); // Create command for execution using (var cmd = this.CreateCommand(conn, sqlStmt.SQL, sqlStmt.Arguments.ToArray())) { var results = new List <ExpandoObject>(); using (var rdr = cmd.ExecuteReader()) while (rdr.Read()) { results.Add(this.MapExpando(rdr)); } return(new BisResultContext( queryDefinition, parameters, this, results, startTime)); } } catch (Exception e) { this.m_tracer.TraceError("Error executing BIS data query: {0}", e); throw new DataPersistenceException($"Error executing BIS data query", e); } } }
/// <summary> /// Executes the query /// </summary> public BisResultContext ExecuteQuery(BiQueryDefinition queryDefinition, IDictionary <string, object> parameters, BiAggregationDefinition[] aggregation, int offset, int?count) { if (queryDefinition == null) { throw new ArgumentNullException(nameof(queryDefinition)); } queryDefinition = BiUtils.ResolveRefs(queryDefinition); // The ADO.NET provider only allows one connection to one db at a time, so verify the connections are appropriate if (queryDefinition.DataSources?.Count != 1) { throw new InvalidOperationException($"ADO.NET BI queries can only source data from 1 connection source, query {queryDefinition.Name} has {queryDefinition.DataSources?.Count}"); } // Ensure we have sufficient priviledge this.AclCheck(queryDefinition); // Apply defaults where possible foreach (var defaultParm in queryDefinition.Parameters.Where(p => !String.IsNullOrEmpty(p.DefaultValue) && !parameters.ContainsKey(p.Name))) { parameters.Add(defaultParm.Name, defaultParm.DefaultValue); } // Next we validate parameters if (!queryDefinition.Parameters.Where(p => p.Required == true).All(p => parameters.ContainsKey(p.Name))) { throw new InvalidOperationException("Missing required parameter"); } // Validate parameter values foreach (var kv in parameters.ToArray()) { var parmDef = queryDefinition.Parameters.FirstOrDefault(p => p.Name == kv.Key); if (parmDef == null) { continue; // skip } else { switch (parmDef.Type) { case BiDataType.Boolean: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else { parameters[kv.Key] = Boolean.Parse(kv.Value.ToString()); } break; case BiDataType.Date: case BiDataType.DateTime: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else { parameters[kv.Key] = DateTime.Parse(kv.Value.ToString()); } break; case BiDataType.Integer: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else { parameters[kv.Key] = Int32.Parse(kv.Value.ToString()); } break; case BiDataType.String: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else { parameters[kv.Key] = kv.Value.ToString(); } break; case BiDataType.Uuid: if (string.IsNullOrEmpty(kv.Value?.ToString())) { parameters[kv.Key] = DBNull.Value; } else { parameters[kv.Key] = Guid.Parse(kv.Value.ToString()); } break; default: throw new InvalidOperationException($"Cannot determine how to parse {parmDef.Type}"); } } } // We want to open the specified connection var provider = this.GetProvider(queryDefinition); // Query definition var rdbmsQueryDefinition = this.GetSqlDefinition(queryDefinition, provider); // Prepare the templated SQL List <Object> values = new List <object>(); var stmt = this.m_parmRegex.Replace(rdbmsQueryDefinition.Sql, (m) => { object pValue = null; parameters.TryGetValue(m.Groups[1].Value, out pValue); values.Add(pValue); return("?"); }); // Aggregation definitions if (aggregation?.Length > 0) { var agg = aggregation.FirstOrDefault(o => o.Invariants?.Contains(provider.Invariant) == true) ?? aggregation.FirstOrDefault(o => o.Invariants?.Count == 0) ?? aggregation.FirstOrDefault(o => o.Invariants == null); // Aggregation found if (agg == null) { throw new InvalidOperationException($"No provided aggregation can be found for {provider.Invariant}"); } var selector = agg.Columns?.Select(c => { switch (c.Aggregation) { case BiAggregateFunction.Average: return($"AVG({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Count: return($"COUNT({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.CountDistinct: return($"COUNT(DISTINCT {c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.First: return($"FIRST({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Last: return($"LAST({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Max: return($"MAX({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Min: return($"MIN({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Sum: return($"SUM({c.ColumnSelector}) AS {c.Name}"); case BiAggregateFunction.Value: return($"{c.ColumnSelector} AS {c.Name}"); default: throw new InvalidOperationException("Cannot apply aggregation function"); } }).ToArray() ?? new string[] { "*" }; String[] groupings = agg.Groupings.Select(g => g.ColumnSelector).ToArray(), colGroupings = agg.Groupings.Select(g => $"{g.ColumnSelector} AS {g.Name}").ToArray(); // Aggregate stmt = $"SELECT {String.Join(",", colGroupings.Concat(selector))} " + $"FROM ({stmt}) {(provider.Features.HasFlag(SqlEngineFeatures.MustNameSubQuery) ? " AS _inner" : "")} " + $"GROUP BY {String.Join(",", groupings)}"; } // Get a readonly context using (var context = provider.GetReadonlyConnection()) { try { context.Open(); DateTime startTime = DateTime.Now; var sqlStmt = new SqlStatement(provider, stmt, values.ToArray()); this.m_tracer.TraceInfo("Executing BI Query: {0}", context.GetQueryLiteral(sqlStmt.Build())); var results = context.Query <ExpandoObject>(sqlStmt).Skip(offset).Take(count ?? 10000).ToArray(); return(new BisResultContext( queryDefinition, parameters, this, results, startTime)); } catch (Exception e) { this.m_tracer.TraceError("Error executing BIS data query {1} \r\n SQL: {2}\r\n Error: {0}", e, queryDefinition.Id, stmt); throw new DataPersistenceException($"Error executing BIS data query", e); } } }
/// <summary> /// Gets or executes teh result context /// </summary> public BisResultContext GetOrExecuteQuery(string name) { BisResultContext retVal = null; if (!this.m_dataSources.TryGetValue(name, out retVal)) { var viewDef = this.m_report.DataSource.FirstOrDefault(o => o.Name == name); if (viewDef == null) { throw new KeyNotFoundException($"Datasource {name} not found"); } viewDef = BiUtils.ResolveRefs(viewDef); // Get the datasource for the execution engine var dsource = (viewDef as BiViewDefinition)?.Query?.DataSources.FirstOrDefault(o => o.Name == "main") ?? (viewDef as BiViewDefinition)?.Query?.DataSources.FirstOrDefault() ?? (viewDef as BiQueryDefinition)?.DataSources.FirstOrDefault(o => o.Name == "main") ?? (viewDef as BiQueryDefinition)?.DataSources.FirstOrDefault(); IBiDataSource providerImplementation = null; if (dsource.ProviderType != null) { providerImplementation = ApplicationServiceContext.Current.GetService <IServiceManager>().CreateInjected(dsource.ProviderType) as IBiDataSource; } else { providerImplementation = ApplicationServiceContext.Current.GetService <IBiDataSource>(); // Global default } // Load from cache instead of DB? var cacheService = ApplicationServiceContext.Current.GetService <IAdhocCacheService>(); var key = $"{name}?{String.Join("&", this.Parameters.Select(o => $"{o.Key}={o.Value}"))}"; var cacheResult = cacheService?.Get <IEnumerable <dynamic> >(key); int count = 10000; if (this.m_maxResultSetSize.HasValue) { count = this.m_maxResultSetSize.Value; } else if (this.Parameters.TryGetValue("_count", out var parameterCount) && Int32.TryParse(parameterCount.ToString(), out var tCount)) { count = tCount; } if (cacheResult != null) { return(new BisResultContext(null, this.Parameters, providerImplementation, cacheResult, DateTime.Now)); } else if (viewDef is BiViewDefinition) { retVal = providerImplementation.ExecuteView(viewDef as BiViewDefinition, this.Parameters, 0, count); } else if (viewDef is BiQueryDefinition) { retVal = providerImplementation.ExecuteQuery(viewDef as BiQueryDefinition, this.Parameters, null, 0, count); } else { throw new InvalidOperationException($"Cannot determine data source type of {name}"); } cacheService?.Add(key, retVal.Dataset, new TimeSpan(0, 1, 0)); this.m_dataSources.Add(name, retVal); } return(retVal); }