public void FailBadCollection()
{
var cdb = new CosmosDB();
Assert.DoesNotThrow(() => cdb.OpenConnection());
Assert.Catch(() => cdb.UpsertDocument(new object(), "badCollection").Wait());
}
public FourChanAzurePipeline() : base()
{
Info("Initializing 4chan Azure pipeline.");
Aggregator = new FourChan();
Db = new CosmosDB("socialnews");
Initialized = Aggregator.Initialized && Db.Initialized;
}
public void FailSavingNull()
{
var cdb = new CosmosDB();
Assert.DoesNotThrow(() => cdb.OpenConnection());
Assert.Catch(() => cdb.UpsertDocument(null, "flight").Wait());
}
static void Main(string[] args)
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
IConfigurationRoot configuration = builder.Build();
// gets sites from configuration
var chartSites = new ChartSites();
configuration.GetSection("ChartSites").Bind(chartSites);
// gets cosmos db configuration
var cosmosDb = new CosmosDB();
configuration.GetSection("CosmosDB").Bind(cosmosDb);
// collects data from shazam
var shazamCollector = new ShazamDataCollector(chartSites.Shazam, cosmosDb);
Console.WriteLine("Collecting Shazam data");
shazamCollector.CollectAndPublish().Wait();
Console.WriteLine("Collecting Shazam data completed");
//collects data from youtube
//Not really, yet to be implemneted
var youtubeCollector = new YoutubeDataCollector(chartSites.Youtube);
Console.WriteLine("Collecting YouTube data");
youtubeCollector.CollectAndPublish().Wait();
Console.WriteLine("Collecting YouTube data completed");
Console.ReadLine();
}
public async Task <IActionResult> Index()
{
ViewData["Message"] = "Estamos en Index.";
await CosmosDB.Initialize(configuration);
return(View());
}
public GabAzurePipeline() : base()
{
Aggregator = new Gab();
Db = new CosmosDB("socialnews");
NLU = new AzureLUIS();
Initialized = Aggregator.Initialized && Db.Initialized;
}
public void BadCollection()
{
var cdb = new CosmosDB();
var act = ActorOf(CosmosSaveActor.Props(cdb));
act.Tell(new CosmosSaveActor.SaveRequest(new { id = 1 }, null), TestActor);
ExpectMsg <CosmosSaveActor.BadRequest>(TimeSpan.FromSeconds(10));
}
public void BadObjectToSave()
{
var cdb = new CosmosDB();
var act = ActorOf(CosmosSaveActor.Props(cdb));
act.Tell(new CosmosSaveActor.SaveRequest(null, null), TestActor);
ExpectMsg <CosmosSaveActor.BadRequest>(TimeSpan.FromSeconds(5));
}
public CosmosSaveActor(CosmosDB cosmos)
{
cdb = cosmos;
Receive <CosmosSaveRequest>(r =>
{
cdb.UpsertDocument(r.SaveObject, "flights").Wait();
});
}
public void StartUpActor()
{
var cdb = new CosmosDB();
var save = ActorOf(CosmosSaveActor.Props(cdb));
var icao = ActorOf <ICAOLookupActor>();
var a = ActorOf(FlightActor.Props());
a.Tell(new FlightActor.FlightActorInit(save, "123", icao), TestActor);
ExpectNoMsg(TimeSpan.FromSeconds(5));
}
public CosmosDBMgmtClient(
string subscriptionId,
RestClient restClient
)
{
_cosmosDBManagementClient = new CosmosDB(restClient)
{
SubscriptionId = subscriptionId
};
}
public void SubmitMany()
{
// resources for actor
var cdb = new CosmosDB();
cdb.OpenConnection();
var save = ActorOf(CosmosSaveActor.Props(cdb));
var icao = ActorOf <ICAOLookupActor>();
// test data
var data = getTestData();
DeviceReading dr = data[0];
FlightData fd = dr.aircraft.First(z => z.flight != null && z.flight.Trim() == "JBU238");
var flight = fd.flight.Trim();
var str = "activeSnap:" + flight.Trim();
var a = ActorOf(FlightActor.Props());
a.Tell(new FlightActor.FlightActorInit(save, flight.Trim(), icao), TestActor);
System.Threading.Thread.Sleep(1000);
data = data.OrderBy(z => z.now).ToList();
foreach (var d in data.Take(2))
{
if (d.aircraft.Any(z => z.flight != null && z.flight.Trim() == flight))
{
fd = d.aircraft.First(z => z.flight != null && z.flight.Trim() == flight);
dr = d;
a.Tell(new FlightActor.FlightDataRequest()
{
deviceId = d.deviceId,
flightData = fd,
now = d.now
}, TestActor);
}
}
ExpectNoMsg(TimeSpan.FromSeconds(5));
var res = cdb.GetDocumentQuery <FlightDataSnapshot>("flights")
.Where(z => z.id == str && z.flight == flight.Trim())
.OrderByDescending(z => z.now)
.ToList();
Assert.Greater(res.Count, 0);
var m = data.Where(z => z.now == res[0].now);
Assert.AreEqual(res.First().now, dr.now);
Assert.AreEqual(res.First().lat, fd.lat);
}
/// <summary>
/// Moving the method that sets the various environment variables
/// </summary>
/// <returns>This returns the success or failure as understood by the frontend</returns>
public async Task <ApiResult> GetEnvironmentVariables()
{
CosmosDBDatabaseName = "production";
var response = ServiceFabricUtil.GetServiceKeyVaultName();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
string serviceKeyvaultName = response.Result.ToString();
var cosmosCon = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigConnectionString").Result.ToString());
CosmosDBDatabaseName = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigDatabaseName").Result.ToString());
var namePassword = Helper.ParseCosmosDBUserNamePassword(cosmosCon);
if (string.IsNullOrEmpty(cosmosCon) || string.IsNullOrEmpty(namePassword) || namePassword.Split(new char[] { ':' }).Count() != 2)
{
return(ApiResult.CreateError("Can't get UserName or Password from CosmosDB connection string"));
}
CosmosDBEndPoint = Helper.ParseCosmosDBEndPoint(cosmosCon);
CosmosDBUserName = namePassword.Split(new char[] { ':' })[0];
CosmosDBPassword = namePassword.Split(new char[] { ':' })[1];
response = await CosmosDB.DownloadConfigFromDocumentDB(CosmosDBDatabaseName, CosmosDBEndPoint, CosmosDBUserName, CosmosDBPassword, ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigCollectionName").Result.ToString());
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
var flowConfigObj = response.Result.ToObject <FlowConfigObject>();
if (flowConfigObj != null)
{
EngineFlowConfig = flowConfigObj;
ResourceCreation = flowConfigObj.ResourceCreation.ToLower().Equals("true") ? true : false;
FlowBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.StorageAccountName + "-blobconnectionstring");
OpsBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.OpsStorageAccountName + "-blobconnectionstring");
SparkConnInfo = Helper.ParseConnectionString(Helper.PathResolver(flowConfigObj.SparkConnectionString));
return(ApiResult.CreateSuccess(""));
}
return(ApiResult.CreateError("Failed to get environment variables"));
}
/// <summary>
/// Controller action hit when the user searches from the search partial for a location.
/// </summary>
/// <param name="location">The location to search providers for</param>
/// <returns>The results view populated with supplier information</returns>
public IActionResult Search(string location)
{
// Creates a task to go and log to the CosmosDB. We wont await it as we can fire and forget.
_ = new CosmosDB().LogToCosmosDB(location);
// Initialise list of weather information and append from each available source.
List <SupplierInformation> weatherInformation = new List <SupplierInformation>
{
_accuWeather.GetWeatherInformation(location),
_google.GetWeatherInformation(location),
_openWeather.GetWeatherInformation(location)
};
return(View("~/Views/Search/Results.cshtml", weatherInformation));
}
public void GoodSave()
{
var cdb = new CosmosDB();
// clear out test document
cdb.DeleteDocument("system", "1", "1");
// do test
var act = ActorOf(CosmosSaveActor.Props(cdb));
act.Tell(new CosmosSaveActor.SaveRequest(new { id = "1", name = "test" }, "system"), TestActor);
ExpectNoMsg(TimeSpan.FromSeconds(10));
// check the DB
var vals = cdb
.GetDocumentQuery("system", "select * from c where c.id = '1'")
.ToList();
Assert.IsTrue(vals.Any());
}
public ControlTowerActor()
{
// CosmosDb methods
var cdb = new CosmosDB();
// get info about the flight
var icao = Context.ActorOf <ICAOLookupActor>();
// register actor type as a sharded entity
region = ClusterSharding.Get(Context.System).Start(
typeName: "FlightActor",
entityProps: Props.Create <FlightActor>(),
settings: ClusterShardingSettings.Create(Context.System),
messageExtractor: new MessageExtractor());
// get a set of data readings
Receive <DeviceReading>(r =>
{
foreach (var a in r.aircraft.Where(z => !string.IsNullOrWhiteSpace(z.flight)))
{
if (!initedActors.ContainsKey(a.flight))
{
initedActors.Add(a.flight, DateTime.Now);
var cos = Context.ActorOf(CosmosSaveActor.Props(cdb));
region.Tell(new ShardEnvelope(shardId: "1", entityId: a.flight, message: new FlightActor.FlightActorInit(cos, a.flight, icao)));
}
// create message for flight actor
var req = new FlightActor.FlightDataRequest()
{
deviceId = r.deviceId,
flightData = a,
now = r.now
};
// send message to entity through shard region
region.Tell(new ShardEnvelope(shardId: "1", entityId: a.flight, message: req));
initedActors[a.flight] = DateTime.Now;
}
});
}
public void SubmitNullFlightData()
{
// resources for actor
var cdb = new CosmosDB();
cdb.OpenConnection();
var save = ActorOf(CosmosSaveActor.Props(cdb));
var icao = ActorOf <ICAOLookupActor>();
var a = ActorOf(FlightActor.Props());
a.Tell(new FlightActor.FlightActorInit(save, "TEST", icao), TestActor);
Watch(a);
a.Tell(new FlightActor.FlightDataRequest()
{
deviceId = "TEST",
flightData = null,
now = DateTimeOffset.Now.ToUnixTimeMilliseconds()
}, TestActor);
ExpectTerminated(a, TimeSpan.FromSeconds(5));
}
public void SubmitOne()
{
// resources for actor
var cdb = new CosmosDB();
cdb.OpenConnection();
var save = ActorOf(CosmosSaveActor.Props(cdb));
var icao = ActorOf <ICAOLookupActor>();
// test data
var data = getTestData();
var info = data[0];
var tf = info.aircraft.First(z => z.flight != null);
var str = "activeSnap:" + tf.flight.Trim();
var a = ActorOf(FlightActor.Props());
a.Tell(new FlightActor.FlightActorInit(save, tf.flight, icao), TestActor);
a.Tell(new FlightActor.FlightDataRequest()
{
deviceId = info.deviceId,
flightData = tf,
now = info.now
}, TestActor);
ExpectNoMsg(TimeSpan.FromSeconds(5));
var res = cdb.GetDocumentQuery <FlightDataSnapshot>("flights")
.Where(z => z.id == str)
.ToList();
Assert.Greater(res.Count, 0);
Assert.AreEqual(res[0].now, info.now);
Assert.AreEqual(res[0].lat, tf.lat);
}
public static async Task ProcessChannel(IConfigurationRoot config, ILogger log)
{
//Read config values
BotFrameworkDirectLine.directLineSecret = config["DirectLineSecret"];
BotFrameworkDirectLine.botId = config["BotId"];
CosmosDB.cosmosDBEndpointUri = config["CosmosDBEndpointUri"];
CosmosDB.cosmosDBPrimaryKey = config["CosmosDBPrimaryKey"];
CosmosDB.cosmosDBDatabaseId = config["CosmosDBDatabaseId"];
CosmosDB.cosmosDBContainerId = config["CosmosDBContainerId"];
FreshDeskClient.freshDeskClientUrl = config["FreshDeskClientUrl"];
FreshDeskClient.freshDeskAPIKey = config["FreshDeskAPIKey"];
preProcessingExtensibility = config["PreProcessingExtensibility"];
postProcessingExtensibility = config["PostProcessingExtensibility"];
// Set last run time for differentials
DateTime lastRun = await CosmosDB.UpdateLastRun(log); //TODO: this doesn't work well when bot crashes halfway (batch will not be reprocessed)
// Read all updated FreshDesk tickets
List <FreshDeskTicket> listUpdatedFreshDeskTickets = await FreshDeskClient.GetUpdatedFreshDeskTicketsAsync(lastRun, log);
foreach (FreshDeskTicket freshDeskTicket in listUpdatedFreshDeskTickets)
{
await ProcessTicket(freshDeskTicket, lastRun, log);
}
// TODO: Read all other open tickets to process for delayed bot responses
// TODO: Close conversation for any resolved tickets?
}
public CosmosSaveActor(CosmosDB cdb)
{
cosmosDB = cdb;
Receive <SaveRequest>(r =>
{
if (r.ToSave == null)
{
Sender.Tell(new BadRequest()
{
BadToSave = true
});
}
else if (string.IsNullOrWhiteSpace(r.Collection))
{
Sender.Tell(new BadRequest()
{
BadCollection = true, ToSave = r.ToSave
});
}
else
{
try
{
cosmosDB.UpsertDocument(r.ToSave, r.Collection);
}
catch (Exception ex)
{
Sender.Tell(new FailedToSave()
{
Exception = ex, ToSave = r.ToSave
});
}
}
});
}
//constructor
public ShazamDataCollector(string chartUrl, CosmosDB cosmos)
{
_chartUrl = chartUrl;
_cosmosDB = cosmos;
}
public CosmosDbStorageTests() : base()
{
db = new CosmosDB("socialnews");
}
public BingNewsAzure(string topic, string dbEndpointUrl, string dbAuthKey, string searchApiKey)
{
Aggregator = new BingNews(searchApiKey);
Db = new CosmosDB(dbEndpointUrl, dbAuthKey, "articles", CancellationToken);
}
public BingNewsAzure(string topic)
{
Aggregator = new BingNews();
Db = new CosmosDB("tradnews");
}
public SubCoordinatorActor()
{
// CosmosDb methods
var cdb = new CosmosDB();
// get info about the flight
var icao = Context.ActorOf <ICAOLookupActor>();
// flight code => Actor
Dictionary <string, IActorRef> flightActors = new Dictionary <string, IActorRef>();
// flight code => last message processed
Dictionary <string, DateTime> flightExpiry = new Dictionary <string, DateTime>();
// clear out old actors
Context.System.Scheduler.ScheduleTellRepeatedly(TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(15), Self, new ExpireActors(), Self);
// group of readings
Receive <DeviceReading>(r =>
{
// foreach plane in current reading -- ignoring those w/o flight code
foreach (var dr in r.aircraft.Where(z => !string.IsNullOrWhiteSpace(z.flight)))
{
// create message for flight actor
var req = new FlightActor.FlightDataRequest()
{
deviceId = r.deviceId,
flightData = dr,
now = r.now
};
// if not started up, then do so now
if (!flightActors.ContainsKey(dr.flight))
{
var cos = Context.ActorOf(CosmosSaveActor.Props(cdb));
// flightActors.Add(dr.flight, Context.ActorOf(FlightActor.Props(cos, dr.flight, icao)));
flightActors.Add(dr.flight, Context.ActorOf(FlightActor.Props()));
flightActors[dr.flight].Tell(new FlightActor.FlightActorInit(cos, dr.flight, icao));
flightExpiry.Add(dr.flight, DateTime.Now);
}
// send message
flightActors[dr.flight].Tell(req);
// update timestamp
flightExpiry[dr.flight] = DateTime.Now;
}
});
// clear out actors that haven't recently processed a message
Receive <ExpireActors>(r =>
{
// get idle processor list
var toCleanup = flightExpiry
.Where(z => z.Value.AddHours(1) < DateTime.Now)
.Select(z => z.Key).ToList();
foreach (var t in toCleanup)
{
// shut down the actor
var actor = flightActors[t];
actor.GracefulStop(TimeSpan.FromSeconds(10));
// clear out from lists
flightActors.Remove(t);
flightExpiry.Remove(t);
}
});
}
public void TestOpenConnection()
{
var cdb = new CosmosDB();
Assert.DoesNotThrow(() => cdb.OpenConnection());
}
public static async Task ProcessTicket(FreshDeskTicket freshDeskTicket, DateTime lastRun, ILogger log)
{
//Start or continue conversation for ticketId
botConversationState = await CosmosDB.ReadItemAsync(freshDeskTicket.Id.ToString(), log);
if (botConversationState == null)
{
//start bot conversation
Conversation conversation = await BotFrameworkDirectLine.StartBotConversation(log);
log.LogInformation("Starting conversation ID: " + conversation.ConversationId);
//record new Bot conversation in CosmosDB, keeping track of TicketId<>BotConversationId
botConversationState = new BotConversationState
{
FreshDeskId = freshDeskTicket.Id.ToString(),
BotConversationId = conversation.ConversationId,
BotWatermark = "0",
Status = (BotConversationState.FreshDeskTicketStatus)freshDeskTicket.Status
};
await CosmosDB.AddItemsToContainerAsync(botConversationState, log);
}
else
{
//continue bot conversation
Conversation conversation = await BotFrameworkDirectLine.ContinueBotConveration(botConversationState.BotConversationId, log);
log.LogInformation("Continuing conversation ID: " + conversation.ConversationId);
}
// List new customer messages in FreshDesk
List <FreshDeskChannelData> listCustomerMessagesToProcess = new List <FreshDeskChannelData>();
//Original ticket description to process?
if (freshDeskTicket.Created_at > lastRun)
{
FreshDeskChannelData customerInitialMessage = new FreshDeskChannelData()
{
TicketId = freshDeskTicket.Id,
Subject = freshDeskTicket.Subject,
Message = freshDeskTicket.Description_text,
Group_id = freshDeskTicket.Group_id,
Responder_id = freshDeskTicket.Responder_id,
Source = freshDeskTicket.Source,
Company_id = freshDeskTicket.Company_id,
Status = (FreshDeskChannelData.FreshDeskTicketStatus)freshDeskTicket.Status,
Product_id = freshDeskTicket.Product_id,
Due_by = freshDeskTicket.Due_by,
MessageType = "initial_message",
Private = false,
FromEmail = freshDeskTicket.Requester.Email,
RequesterName = freshDeskTicket.Requester.Name,
Mobile = freshDeskTicket.Requester.Mobile,
Phone = freshDeskTicket.Requester.Phone
};
listCustomerMessagesToProcess.Add(customerInitialMessage);
}
//Any new incoming conversations in this ticket to process?
List <FreshDeskConversation> listTicketConversations = await FreshDeskClient.GetFreshDeskTicketConversationsAsync(freshDeskTicket.Id, log);
List <FreshDeskConversation> listIncomingConversationsSinceLastRun = (from c in listTicketConversations
where c.Incoming == true && c.Updated_at > lastRun
orderby c.Updated_at ascending
select c).ToList();
foreach (FreshDeskConversation incomingConversation in listIncomingConversationsSinceLastRun)
{
FreshDeskChannelData customerConversationMessage = new FreshDeskChannelData()
{
TicketId = freshDeskTicket.Id,
Subject = freshDeskTicket.Subject,
Message = incomingConversation.Body_text,
Group_id = freshDeskTicket.Group_id,
Responder_id = freshDeskTicket.Responder_id,
Source = freshDeskTicket.Source,
Company_id = freshDeskTicket.Company_id,
Status = (FreshDeskChannelData.FreshDeskTicketStatus)freshDeskTicket.Status,
Product_id = freshDeskTicket.Product_id,
Due_by = freshDeskTicket.Due_by,
MessageType = "continued_conversation",
Private = incomingConversation.Private,
FromEmail = incomingConversation.From_email,
RequesterName = freshDeskTicket.Requester.Name,
Mobile = freshDeskTicket.Requester.Mobile,
Phone = freshDeskTicket.Requester.Phone
};
listCustomerMessagesToProcess.Add(customerConversationMessage);
}
// Send new customer messages to Bot Framework for processing
foreach (FreshDeskChannelData freshDeskChannelData in listCustomerMessagesToProcess)
{
// Run Pre-processing Extensibility
FreshDeskChannelData processedFreshDeskChannelData = freshDeskChannelData;
if (!String.IsNullOrEmpty(preProcessingExtensibility))
{
processedFreshDeskChannelData = await PreProcessingExtensibility(preProcessingExtensibility, freshDeskChannelData, log);
}
await BotFrameworkDirectLine.SendMessagesAsync(botConversationState.BotConversationId, processedFreshDeskChannelData, log);
}
// Read any new Bot Framework responses on this ticket
ActivitySet activitySet = await BotFrameworkDirectLine.ReadBotMessagesAsync(botConversationState.BotConversationId, botConversationState.BotWatermark, log);
//Update the bot watermark in CosmosDB, to keep track which Bot Framework conversations we have already read
botConversationState.BotWatermark = activitySet?.Watermark;
await CosmosDB.ReplaceFreshDeskBotStateAsync(botConversationState, log);
//Send Bot Framework responses back to FreshDesk as ticket responses to the customer
foreach (Activity activity in activitySet.Activities)
{
//If there is specific ChannelData add it to the message, otherwise default to standard customer reply message
BotResponseChannelData botResponseChannelData;
if (activity.ChannelData != null)
{
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
};
botResponseChannelData = JsonSerializer.Deserialize <BotResponseChannelData>(activity.ChannelData.ToString(), options);
}
else
{
// Default to a standard reply message
botResponseChannelData = new BotResponseChannelData
{
Message = activity.Text,
MessageType = "reply",
Status = BotResponseChannelData.FreshDeskTicketStatus.Pending
};
}
// Run post-processing Extensibility
BotResponseChannelData processedBotResponseChannelData = botResponseChannelData;
if (!String.IsNullOrEmpty(postProcessingExtensibility))
{
processedBotResponseChannelData = await PostProcessingExtensibility(postProcessingExtensibility, botResponseChannelData, log);
}
// Send the bot response to FreshDesk in the chosen messageType (current allowed values: note, reply)
switch (processedBotResponseChannelData.MessageType)
{
case "note":
await FreshDeskClient.SendFreshDeskNote(freshDeskTicket.Id.ToString(), processedBotResponseChannelData.Message, processedBotResponseChannelData.Private, processedBotResponseChannelData.NotifyEmails, log);
break;
case "reply":
await FreshDeskClient.SendFreshDeskTicketReply(freshDeskTicket.Id.ToString(), processedBotResponseChannelData.Message, log);
await FreshDeskClient.SetTicketStatus(freshDeskTicket.Id.ToString(), processedBotResponseChannelData.Status, log);
break;
}
}
}
public static Props Props(CosmosDB cdb) => Akka.Actor.Props.Create(() => new CosmosSaveActor(cdb));
/// This is the function that can be called for deleting all assets created on save of a flow: consumergroup, secrets in Key Vault, cosmosDB products document such that this flow stops showing up in the UI under Flows and blobs
/// Please note if a a new asset is created on Azure as part of saving the flow or any other action taken by user in the UI, this function will need to be updated so that the asset can be deleted on delete of the flow.
/// </summary>
/// <param name="jObject">jObject requires: Subscription; Name; EventhubConnectionString; IsIotHub;EventhubName;userID</param>
/// <returns>Returns result - success or failure as the case maybe</returns>
public async Task <ApiResult> DeleteFlow(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
ConfigName = diag.Name;
bool errorExists = false;
var response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
///Delete consumer group
_logger.LogInformation($"For FlowId: {ConfigName} Deleting flow specific consumer group.. ");
var inputEventhubConnection = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
_inputEventhubConnectionStringRef = Helper.IsKeyVault(diag.EventhubConnectionString) ? diag.EventhubConnectionString : Helper.GenerateNewSecret(_keySecretList, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName + "-input-eventhubconnectionstring", diag.EventhubConnectionString, false);
diag.EventhubConnectionString = _inputEventhubConnectionStringRef;
if (diag.InputType == Constants.InputType_EventHub)
{
var ehName = Helper.ParseEventHub(inputEventhubConnection);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (string.IsNullOrWhiteSpace(ehName) || string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The connection string for Event Hub input type must contain Endpoint, SharedAccessKeyName, SharedAccessKey, and EntityPath";
_logger.LogError(error);
errorExists = true;
}
_eventHubNames = new List <string>()
{
ehName
};
}
else
{
_eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (_eventHubNames.Count < 1)
{
string error = "The event hub-compatible name for IoT Hub input type must be defined";
_logger.LogError(error);
errorExists = true;
}
if (string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The event hub-compatible endpoint for IoT Hub input type must contain Endpoint, SharedAccessKeyName, and SharedAccessKey";
_logger.LogError(error);
errorExists = true;
}
}
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
foreach (string ehName in _eventHubNames)
{
var result = EventHub.DeleteConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, _eventHubNamespace, ehName, ConsumerGroupName, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully deleted flow specific consumer group");
}
}
}
///Delete cosmosDB document related to a flow
response = await CosmosDB.DeleteConfigFromDocumentDB(_engineEnvironment.CosmosDBDatabaseName, _engineEnvironment.CosmosDBEndPoint, _engineEnvironment.CosmosDBUserName, _engineEnvironment.CosmosDBPassword, "flows", ConfigName);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific cosmosDB entry");
}
///Delete configs stored in blobs
// ruleDefinitions
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(RuleDefinitionPath, RuleDefinitionFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific rules definition blob");
}
// outputTemplates
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(OutputTemplatePath, OutputTemplateFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific output template blob");
}
string resourceGroupLocation = _engineEnvironment.EngineFlowConfig.ResourceGroupLocation;
string resourceGroupName = _engineEnvironment.EngineFlowConfig.ResourceGroupName;
string storageAccountName = _engineEnvironment.EngineFlowConfig.StorageAccountName;
string containerPath = Path.Combine(_flowContainerName, _engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName);
string subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific blobs under the folder {ConfigName} under container {_Centralprocessing}");
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _flowContainerName, Path.Combine(_engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific productconfig: {ProductConfigName} and {JobConfigName} for {ConfigName}.");
/// Delete sample data and the checkpoints folder if it exists for that flow
var hashValue = Helper.GetHashCode(diag.UserName);
await BlobHelper.DeleteBlob(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsSamplePath, $"{ConfigName}-{hashValue}.json"));
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific sampledata file: {ConfigName}-{ hashValue}.json");
await BlobHelper.DeleteAllBlobsInAContainer(_engineEnvironment.OpsBlobConnectionString, $"{_engineEnvironment.CheckPointContainerNameHelper(ConfigName)}-checkpoints", _engineEnvironment.EngineFlowConfig.OpsBlobDirectory);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific checkpoints for {ConfigName}.");
_logger.LogInformation("Deleting flow specific secrets..");
///Delete secrets specific to a flow from KeyVault
KeyVault.GetSecretsAndDeleteFromKeyvault(_engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific secrets");
if (!errorExists)
{
return(ApiResult.CreateSuccess("Deleted!"));
}
else
{
return(ApiResult.CreateError("Deleted but with some error. Please check logs for details"));
}
}
