/// <summary> /// Gets the schema from a sample data generated within 30 seconds as default /// </summary> /// <param name="storageConnectionString">storageConnectionString</param> /// <param name="samplesPath">samplesPath</param> /// <param name="userId">userId</param> /// <param name="flowId">flowId</param> /// <param name="seconds">seconds is the duration of time</param> /// <returns>Returns ApiResult with schema Result object</returns> public async Task <SchemaResult> GetSchemaAsync(string storageConnectionString, string samplesPath, string userId, string flowId, int seconds = _DefaultDuration) { // Get sample events EventsData eventsData = await GetSampleEvents(seconds); List <EventRaw> events = eventsData.Events; _logger.LogInformation($"Event count = {events.Count}"); if (events.Count < 1) { throw new Exception("Can't capture any data from the data source."); } // Save sample events SaveSample(storageConnectionString, samplesPath, userId, flowId, eventsData.EventsJson); _logger.LogInformation($"successful save to {samplesPath}"); // Get schema //log schemaResult.schema.length and count of errors SchemaResult schemaResult = GetSchemaHelper(events); _logger.LogInformation($"schemaResult.schema.length: {schemaResult.Schema.Length} and count of errors: {schemaResult.Errors.Count}"); return(schemaResult); }
public async Task <ApiResult> GetInputSchema(JObject jObject) { var diag = jObject.ToObject <InteractiveQueryObject>(); var eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames); if (_engineEnvironment.EngineFlowConfig == null) { await _engineEnvironment.GetEnvironmentVariables(); } if (string.IsNullOrEmpty(diag.Name)) { diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName); } var connectionString = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString); if (diag.InputType == Constants.InputType_EventHub) { string ehName = Helper.ParseEventHub(connectionString); eventHubNames = new List <string>() { ehName }; } string eventHubNamespace = Helper.ParseEventHubNamespace(connectionString); //Fixing a bug where '_' checkpoint container name cannot have '_' var checkPointContainerName = $"{_engineEnvironment.CheckPointContainerNameHelper(diag.Name)}-checkpoints"; // ResourceCreation is one of the environment variables. // If you don't want to create resource, you can set this to false. if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub)) { var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId); var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup); // Create consumer group if it doesn't exist foreach (string ehName in eventHubNames) { var result = EventHub.CreateConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, eventHubNamespace, ehName, _engineEnvironment.EngineFlowConfig.ConsumerGroup, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix); if (result.Error.HasValue && result.Error.Value) { _logger.LogError(result.Message); return(ApiResult.CreateError(result.Message)); } } } var bootstrapServers = Helper.TryGetBootstrapServers(connectionString); // Sample events and generate schema SchemaGenerator sg = new SchemaGenerator(bootstrapServers, connectionString, eventHubNames, _engineEnvironment.EngineFlowConfig.ConsumerGroup, _engineEnvironment.OpsBlobConnectionString, checkPointContainerName, _engineEnvironment.EngineFlowConfig.OpsBlobDirectory, diag.InputType, diag.InputMode, diag.BatchInputs, _logger); SchemaResult schema = await sg.GetSchemaAsync(_engineEnvironment.OpsBlobConnectionString, OpsSamplePath, diag.UserName, diag.Name, diag.Seconds); return(ApiResult.CreateSuccess(JObject.FromObject(schema))); }
/// <summary> /// GetSchema Helper /// </summary> /// <param name="events">events</param> /// <returns>Returns SchemaResult</returns> private SchemaResult GetSchemaHelper(List <EventRaw> events) { Engine engine = new Engine(); List <string> eventStrings = new List <string>(); foreach (EventRaw e in events) { eventStrings.Add(e.Raw); } SchemaResult output = engine.GetSchema(eventStrings); return(output); }
/// <summary> /// Gets schema of the event message. It will return the union schema across all events of the array /// </summary> /// <param name="events">JSON array whose each item is an event</param> /// <returns></returns> public SchemaResult GetSchema(string events) { JArray array = JArray.Parse(events); if (array == null || array.Count <= 0) { _errors.Add("No data to evaluate schema for."); return(new SchemaResult() { Errors = _errors }); } if (array.First.GetType() == typeof(JObject)) { _result = new StructObject(); foreach (JObject jObject in array) { GetSchemaStruct(jObject, _result as StructObject, null); } } else if (array.First.GetType() == typeof(JArray)) { _result = new Type(); foreach (JArray jArray in array) { GetSchemaArray(jArray, _result as Type, null); } } string output = JsonConvert.SerializeObject(_result); SchemaResult schemaResult = new SchemaResult { Schema = output, Errors = _errors }; return(schemaResult); }
/// <summary> /// Gets the schema of the event message. It will return the union schema of all events in the list /// </summary> /// <param name="events">List of events whose schema needs to be determined. Each event is a JSON string</param> /// <returns></returns> public SchemaResult GetSchema(List <string> events) { if (events == null || events.Count <= 0) { return(new SchemaResult()); } if (events[0].Trim().StartsWith('{')) { _result = new StructObject(); foreach (string str in events) { JObject jObject = JObject.Parse(str); GetSchemaStruct(jObject, _result as StructObject, null); } } else if (events[0].Trim().StartsWith('[')) { _result = new Type(); foreach (string str in events) { JArray jArray = JArray.Parse(str); GetSchemaArray(jArray, _result as Type, null); } } string output = JsonConvert.SerializeObject(_result); SchemaResult schemaResult = new SchemaResult { Schema = output, Errors = _errors }; return(schemaResult); }