public async Task TestFailIfDrops()
{
var toFile = "FailIfDataLoss/target.kql";
var targetDbName = await InitializeDbAsync();
var overrides = ImmutableArray <(string path, string value)>
.Empty
.Add(("jobs.main.target.adx.clusterUri", ClusterUri.ToString()))
.Add(("jobs.main.target.adx.database", targetDbName))
.Append(("failIfDataLoss", "true"));
await PrepareDbAsync(toFile, targetDbName);
try
{
// The "Main" will return non-zero which will throw an exception
var parameters =
await RunParametersAsync("FailIfDataLoss/no-fail.json", overrides);
Assert.True(parameters.FailIfDataLoss);
Assert.False(true, "Should have thrown by now");
}
catch (InvalidOperationException)
{
}
}
private void StopStatusChecks()
{
if (log != null && log.IsInfoEnabled)
{
log.Info("StopStatusChecks() Cluster: " + ClusterUri.ToString());
}
if (statusCheckTimer != null)
{
statusCheckTimer.Change(Timeout.Infinite, Timeout.Infinite);
}
}
private void BeginStatusChecks()
{
if (!RouteRegistered)
{
return;
}
if (log != null && log.IsInfoEnabled)
{
log.Info("BeginStatusChecks() Cluster: " + ClusterUri.ToString());
}
// Start status process -- if started already, this will cause a recheck at statusInitialDueTime
statusCheckTimer.Change((int)ClientConfig.StatusInitialDueTime.TotalMilliseconds, (int)ClientConfig.StatusPeriod.TotalMilliseconds);
}
public string Ping()
{
// TODO: can we do this if not registered?
if (!RouteRegistered)
{
return(null);
}
McmpResponse result = ClientConfig.ExecuteCommand(ClusterUri, Constants.Commands.Ping);
if (log != null && log.IsDebugEnabled)
{
log.Debug("Ping(" + ClusterUri.ToString() + ") " + result.ToString());
}
return(result.ResponseBody);
}
public async Task TestFailIfDropsNoDrop()
{
var toFile = "FailIfDataLoss/target.kql";
var targetDbName = await InitializeDbAsync();
var overrides = ImmutableArray <(string path, string value)>
.Empty
.Add(("jobs.main.target.adx.clusterUri", ClusterUri.ToString()))
.Add(("jobs.main.target.adx.database", targetDbName));
await PrepareDbAsync(toFile, targetDbName);
await RunParametersAsync("FailIfDataLoss/no-fail.json", overrides);
// We just test that this doesn't fail
}
protected async Task TestAdxToFile(string statesFolderPath)
{
await LoopThroughStateFilesAsync(
Path.Combine(statesFolderPath, "States"),
async (fromFile, toFile) =>
{
var testDbName = await InitializeDbAsync();
var currentDbName = await InitializeDbAsync();
await PrepareDbAsync(fromFile, currentDbName);
var outputPath = Path.Combine("outputs", statesFolderPath, "adx-to-file/")
+ Path.GetFileNameWithoutExtension(fromFile)
+ "_2_"
+ Path.GetFileNameWithoutExtension(toFile)
+ ".kql";
var overrides = ImmutableArray <(string path, string value)>
.Empty
.Add(("jobs.main.current.adx.clusterUri", ClusterUri.ToString()))
.Add(("jobs.main.current.adx.database", currentDbName))
.Add(("jobs.main.target.scripts[0].filePath", toFile))
.Add(("jobs.main.action.filePath", outputPath));
var parameters = await RunParametersAsync(
"adx-to-file-params.json",
overrides);
var outputCommands = await LoadScriptAsync("", outputPath);
var targetFileCommands = CommandBase.FromScript(
await File.ReadAllTextAsync(toFile));
await Task.WhenAll(
ApplyCommandsAsync(outputCommands, currentDbName),
ApplyCommandsAsync(targetFileCommands, testDbName));
var finalCommandsTask = FetchDbCommandsAsync(currentDbName);
var targetAdxCommands = await FetchDbCommandsAsync(testDbName);
var finalCommands = await finalCommandsTask;
var targetModel = DatabaseModel.FromCommands(targetAdxCommands);
var finalModel = DatabaseModel.FromCommands(finalCommands);
var finalScript = string.Join(";\n\n", finalCommands.Select(c => c.ToScript()));
var targetScript = string.Join(";\n\n", targetFileCommands.Select(c => c.ToScript()));
Assert.True(
finalModel.Equals(targetModel),
$"From {fromFile} to {toFile}:\n\n{finalScript}\nvs\n\n{targetScript}");
AdxDbTestHelper.Instance.ReleaseDb(currentDbName);
});
}
public async Task TestGivenToken()
{
var authenticationContext =
new AuthenticationContext("https://login.windows.net/" + TenantId);
var authenticationResult = await authenticationContext.AcquireTokenAsync(
ClusterUri.ToString(),
new ClientCredential(ServicePrincipalId, ServicePrincipalSecret));
var token = authenticationResult.AccessToken;
var targetDbName = await InitializeDbAsync();
var overrides = ImmutableArray <(string path, string value)>
.Empty
.Add(("jobs.main.target.adx.clusterUri", ClusterUri.ToString()))
.Add(("jobs.main.target.adx.database", targetDbName))
.Add(("tokenProvider.tokens.myToken.clusterUri", ClusterUri.ToString()))
.Add(("tokenProvider.tokens.myToken.token", token));
await PrepareDbAsync("GivenTokenProvider/target.kql", targetDbName);
await RunParametersAsync("GivenTokenProvider/given-token.yaml", overrides);
// We just test that this doesn't fail
}
internal void RegisterRoute(IEnumerable <string> contexts)
{
if (RouteRegistered)
{
return;
}
// Stop timers so they dont screw us up
StopRegistrationTimer();
// Register our route
try
{
// Query if this route/node exists already only if we're going to not register if we find it
if (!registerRouteAnywayIfNodeExists)
{
try
{
var dump = Dump();
if (dump.ContainsKey("node"))
{
var node = dump["node"];
foreach (var item in node)
{
if (item["JVMRoute"] == ClientConfig.JvmRoute)
{
log.Warn("Found our node already registered: " + string.Join(",", item.Select(i => i.Key + "=" + i.Value)));
RouteRegistered = true;
break;
}
}
}
}
catch (Exceptions.CommandFailureException ex)
{
if (ex.McmpResponse.ErrorMessage == "MEM: Can't read node")
{
// This is ok, it means the node doesnt really exist, which would be the case if we were just registering
RouteRegistered = false;
}
else
{
throw;
}
}
}
// If we arent registered, do it now
if (!RouteRegistered)
{
if (log != null && log.IsInfoEnabled)
{
log.Info("RegisterRoute() Cluster:" + ClusterUri.ToString() + " Contexts:" + string.Join(",", contexts.Select(c => GetRootedContextPath(c))));
}
McmpResponse result = ClientConfig.ExecuteCommand(ClusterUri, Constants.Commands.Config, "Context", string.Join(",", contexts.Select(c => GetRootedContextPath(c))));
if (log != null && log.IsDebugEnabled)
{
log.Debug("RegisterRoute() Result: " + result.ToString());
}
// Mark as registered
RouteRegistered = true;
// Add contexts to our list of contexts
//RegisteredContexts.AddRange(contexts.Select(c => GetRootedContextPath(c)));
}
foreach (string context in contexts)
{
RegisterContextPath(context);
}
}
catch (Exceptions.CommandFailureException ex)
{
if (ex.InnerException is WebException && (
(ex.InnerException as WebException).Status == WebExceptionStatus.ConnectFailure ||
(ex.InnerException as WebException).Status == WebExceptionStatus.Timeout
))
{
if (log != null && log.IsErrorEnabled && ex.McmpResponse != null)
{
try { log.Error("PerformStatusUpdate() Could not connect to mod_cluster for " + ex.McmpResponse.Command + " command - " + ex.Message + " on " + ClusterUri); }
catch { }
}
else
if (log != null && log.IsErrorEnabled)
{
try { log.Error("PerformStatusUpdate() CommandFailureException on " + ClusterUri, ex); }
catch { }
}
}
// Failure, append list of contexts to try again later
AppendContextsToRegister(contexts);
return;
}
catch (Exception ex)
{
if (log != null && log.IsErrorEnabled)
{
log.Error("RegisterRoute(" + string.Join(",", contexts) + ") fail", ex);
}
// Failure, append list of contexts to try again later
AppendContextsToRegister(contexts);
}
finally
{
// Start timers
StartRegistrationTimer();
// Start status processes -- if started already, this will cause a recheck at statusInitialDueTime
BeginStatusChecks();
}
}
public Dictionary <string, List <Dictionary <string, string> > > Dump()
{
Dictionary <string, List <Dictionary <string, string> > > dump = new Dictionary <string, List <Dictionary <string, string> > >();
McmpResponse result = ClientConfig.ExecuteCommand(ClusterUri, Constants.Commands.Dump);
if (log != null && log.IsInfoEnabled)
{
log.Info("Dump(" + ClusterUri.ToString() + ") " + result.ToString());
}
if (result.ResponseBody == null)
{
return(dump);
}
//balancer: [1] Name: mybalancer Sticky: 0 [NSESSION]/[nsession] remove: 0 force: 0 Timeout: 0 maxAttempts: 1
//node: [1:1],Balancer: mybalancer,JVMRoute: MyIISRoute,LBGroup: [],Host: localhost,Port: 17508,Type: http,flushpackets: 0,flushwait: 10,ping: 5,smax: 1,ttl: 60,timeout: 0
//host: 1 [localhost] vhost: 1 node: 1
//context: 1 [/] vhost: 1 node: 1 status: 1
foreach (string line in result.ResponseBody.Split('\n'))
{
// Get our TYPE
if (string.IsNullOrEmpty(line))
{
continue;
}
string type = line.Substring(0, line.IndexOf(':'));
if (!dump.ContainsKey(type))
{
dump.Add(type, new List <Dictionary <string, string> >());
}
switch (type)
{
case "node":
dump[type].Add(line.ToDictionary(separator: ',', equals: ": "));
break;
case "balancer":
Dictionary <string, string> balancer = new Dictionary <string, string>();
balancer.Add("RAW", line);
Queue <string> balanceritems = new Queue <string>(line.Split(' '));
balancer.Add(balanceritems.Dequeue().TrimEnd(':'), balanceritems.Dequeue());
balancer.Add(balanceritems.Dequeue().TrimEnd(':'), balanceritems.Dequeue());
balancer.Add(balanceritems.Dequeue().TrimEnd(':'), balanceritems.Dequeue() + " " + balanceritems.Dequeue());
while (balanceritems.Count > 0)
{
balancer.Add(balanceritems.Dequeue().TrimEnd(':'), balanceritems.Dequeue());
}
dump[type].Add(balancer);
break;
case "host":
case "context":
Dictionary <string, string> item = new Dictionary <string, string>();
item.Add("RAW", line);
Queue <string> dataitems = new Queue <string>(line.Split(' '));
item.Add(dataitems.Dequeue().TrimEnd(':'), dataitems.Dequeue() + " " + dataitems.Dequeue());
while (dataitems.Count > 0)
{
item.Add(dataitems.Dequeue().TrimEnd(':'), dataitems.Dequeue());
}
dump[type].Add(item);
break;
}
}
return(dump);
}
public override string ToString()
{
return(ClusterUri.ToString());
}