/// <summary>
/// Gets the schema from a sample data generated within 30 seconds as default
/// </summary>
/// <param name="storageConnectionString">storageConnectionString</param>
/// <param name="samplesPath">samplesPath</param>
/// <param name="userId">userId</param>
/// <param name="flowId">flowId</param>
/// <param name="seconds">seconds is the duration of time</param>
/// <returns>Returns ApiResult with schema Result object</returns>
public async Task <SchemaResult> GetSchemaAsync(string storageConnectionString, string samplesPath, string userId, string flowId, int seconds = _DefaultDuration)
{
// Get sample events
EventsData eventsData = await GetSampleEvents(seconds);
List <EventRaw> events = eventsData.Events;
_logger.LogInformation($"Event count = {events.Count}");
if (events.Count < 1)
{
throw new Exception("Can't capture any data from the data source.");
}
// Save sample events
SaveSample(storageConnectionString, samplesPath, userId, flowId, eventsData.EventsJson);
_logger.LogInformation($"successful save to {samplesPath}");
// Get schema
//log schemaResult.schema.length and count of errors
SchemaResult schemaResult = GetSchemaHelper(events);
_logger.LogInformation($"schemaResult.schema.length: {schemaResult.Schema.Length} and count of errors: {schemaResult.Errors.Count}");
return(schemaResult);
}
public async Task <ApiResult> GetInputSchema(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
var eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
if (_engineEnvironment.EngineFlowConfig == null)
{
await _engineEnvironment.GetEnvironmentVariables();
}
if (string.IsNullOrEmpty(diag.Name))
{
diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName);
}
var connectionString = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
if (diag.InputType == Constants.InputType_EventHub)
{
string ehName = Helper.ParseEventHub(connectionString);
eventHubNames = new List <string>()
{
ehName
};
}
string eventHubNamespace = Helper.ParseEventHubNamespace(connectionString);
//Fixing a bug where '_' checkpoint container name cannot have '_'
var checkPointContainerName = $"{_engineEnvironment.CheckPointContainerNameHelper(diag.Name)}-checkpoints";
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
// Create consumer group if it doesn't exist
foreach (string ehName in eventHubNames)
{
var result = EventHub.CreateConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, eventHubNamespace, ehName, _engineEnvironment.EngineFlowConfig.ConsumerGroup, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
return(ApiResult.CreateError(result.Message));
}
}
}
var bootstrapServers = Helper.TryGetBootstrapServers(connectionString);
// Sample events and generate schema
SchemaGenerator sg = new SchemaGenerator(bootstrapServers, connectionString, eventHubNames, _engineEnvironment.EngineFlowConfig.ConsumerGroup, _engineEnvironment.OpsBlobConnectionString, checkPointContainerName, _engineEnvironment.EngineFlowConfig.OpsBlobDirectory, diag.InputType, diag.InputMode, diag.BatchInputs, _logger);
SchemaResult schema = await sg.GetSchemaAsync(_engineEnvironment.OpsBlobConnectionString, OpsSamplePath, diag.UserName, diag.Name, diag.Seconds);
return(ApiResult.CreateSuccess(JObject.FromObject(schema)));
}
/// <summary>
/// GetSchema Helper
/// </summary>
/// <param name="events">events</param>
/// <returns>Returns SchemaResult</returns>
private SchemaResult GetSchemaHelper(List <EventRaw> events)
{
Engine engine = new Engine();
List <string> eventStrings = new List <string>();
foreach (EventRaw e in events)
{
eventStrings.Add(e.Raw);
}
SchemaResult output = engine.GetSchema(eventStrings);
return(output);
}
/// <summary>
/// Gets schema of the event message. It will return the union schema across all events of the array
/// </summary>
/// <param name="events">JSON array whose each item is an event</param>
/// <returns></returns>
public SchemaResult GetSchema(string events)
{
JArray array = JArray.Parse(events);
if (array == null || array.Count <= 0)
{
_errors.Add("No data to evaluate schema for.");
return(new SchemaResult()
{
Errors = _errors
});
}
if (array.First.GetType() == typeof(JObject))
{
_result = new StructObject();
foreach (JObject jObject in array)
{
GetSchemaStruct(jObject, _result as StructObject, null);
}
}
else if (array.First.GetType() == typeof(JArray))
{
_result = new Type();
foreach (JArray jArray in array)
{
GetSchemaArray(jArray, _result as Type, null);
}
}
string output = JsonConvert.SerializeObject(_result);
SchemaResult schemaResult = new SchemaResult
{
Schema = output,
Errors = _errors
};
return(schemaResult);
}
/// <summary>
/// Gets the schema of the event message. It will return the union schema of all events in the list
/// </summary>
/// <param name="events">List of events whose schema needs to be determined. Each event is a JSON string</param>
/// <returns></returns>
public SchemaResult GetSchema(List <string> events)
{
if (events == null || events.Count <= 0)
{
return(new SchemaResult());
}
if (events[0].Trim().StartsWith('{'))
{
_result = new StructObject();
foreach (string str in events)
{
JObject jObject = JObject.Parse(str);
GetSchemaStruct(jObject, _result as StructObject, null);
}
}
else if (events[0].Trim().StartsWith('['))
{
_result = new Type();
foreach (string str in events)
{
JArray jArray = JArray.Parse(str);
GetSchemaArray(jArray, _result as Type, null);
}
}
string output = JsonConvert.SerializeObject(_result);
SchemaResult schemaResult = new SchemaResult
{
Schema = output,
Errors = _errors
};
return(schemaResult);
}
