public void ValidateMultiDatabaseData_EmptyOverrideSequence()
{
MultiDbData dbData = new MultiDbData();
ServerData srv1 = new ServerData();
srv1.ServerName = "server1";
dbData["server1"] = srv1;
ServerData srv2 = new ServerData();
srv2.ServerName = "server2";
dbData["server2"] = srv2;
DbOverrideSequence ovr = new DbOverrideSequence();
ovr.Add("1", new DatabaseOverride("default", "target"));
ovr.Add("2", new DatabaseOverride("default2", "target2"));
dbData["server1"].OverrideSequence = ovr;
DbOverrideSequence ovr2 = new DbOverrideSequence();
ovr.Add("3", new DatabaseOverride("default", "target"));
ovr.Add("4", new DatabaseOverride("", ""));
dbData["server2"].OverrideSequence = ovr2;
bool expected = false;
bool actual = MultiDbHelper.ValidateMultiDatabaseData(dbData);
Assert.AreEqual(expected, actual);
}
public void ValidateMultiDatabaseData_BadConfuguration()
{
string[] fileContents = new string[] { "SERVER:default,target;default2,target2", "SERVER2:,;default2,target2" };
MultiDbData dbData = MultiDbHelper.ImportMultiDbTextConfig(fileContents);
bool expected = false;
bool actual = MultiDbHelper.ValidateMultiDatabaseData(dbData);
Assert.AreEqual(expected, actual);
}
public void DeserializeMultiDbConfigurationTest_EmptyFileName()
{
string fileName = string.Empty;
MultiDbData expected = null;
MultiDbData actual;
actual = MultiDbHelper.DeserializeMultiDbConfiguration(fileName);
Assert.AreEqual(expected, actual);
}
public void ImportMultiDbTextConfigTest_EmptyFileName()
{
string fileName = string.Empty;
MultiDbData expected = null;
MultiDbData actual;
actual = MultiDbHelper.ImportMultiDbTextConfig(fileName);
Assert.AreEqual(expected, actual);
}
public void BuildDataTest()
{
MultiDbData target = new MultiDbData();
SqlSyncBuildData expected = new SqlSyncBuildData();
SqlSyncBuildData actual;
target.BuildData = expected;
actual = target.BuildData;
Assert.AreEqual(expected, actual);
}
public void ProjectFileNameTest()
{
MultiDbData target = new MultiDbData();
string expected = "MyNewProjectName";
string actual;
target.ProjectFileName = expected;
actual = target.ProjectFileName;
Assert.AreEqual(expected, actual);
}
public void MultiRunIdTest()
{
MultiDbData target = new MultiDbData();
string expected = System.Guid.NewGuid().ToString();
string actual;
target.MultiRunId = expected;
actual = target.MultiRunId;
Assert.AreEqual(expected, actual);
}
public static List <(string, List <DatabaseOverride>)> FlattenOverride(MultiDbData multiData)
{
var flattened = new List <(string, List <DatabaseOverride>)>();
foreach (ServerData srv in multiData)
{
foreach (List <DatabaseOverride> ovr in srv.OverrideSequence.Values)
{
flattened.Add((srv.ServerName, ovr));
}
}
return(flattened);
}
public void ConvertMultiDbDataToTextConfigTest()
{
DbOverrideSequence sequenceA = new DbOverrideSequence();
sequenceA.Add("1", new DatabaseOverride("default1", "override1"));
sequenceA.Add("2", new DatabaseOverride("default2", "override2"));
sequenceA.Add("0", new DatabaseOverride("default0", "override0"));
DatabaseOverride ovrX = new DatabaseOverride("defaultX", "overrideX");
DatabaseOverride ovrY = new DatabaseOverride("defaultY", "overrideY");
List <DatabaseOverride> lstOvr = new List <DatabaseOverride>();
lstOvr.Add(ovrX);
lstOvr.Add(ovrY);
sequenceA.Add("M", lstOvr);
ServerData serverA = new ServerData();
serverA.OverrideSequence = sequenceA;
serverA.ServerName = "ServerA";
DbOverrideSequence sequenceB = new DbOverrideSequence();
sequenceB.Add("6", new DatabaseOverride("default6", "override6"));
sequenceB.Add("7", new DatabaseOverride("default7", "override7"));
sequenceB.Add("5", new DatabaseOverride("default5", "override5"));
ServerData serverB = new ServerData();
serverB.OverrideSequence = sequenceB;
serverB.ServerName = "ServerB";
MultiDbData cfg = new MultiDbData();
cfg.Add(serverA);
cfg.Add(serverB);
string expected =
@"ServerA:default1,override1
ServerA:default2,override2
ServerA:default0,override0
ServerA:defaultX,overrideX;defaultY,overrideY
ServerB:default6,override6
ServerB:default7,override7
ServerB:default5,override5
";
string actual;
actual = MultiDbHelper.ConvertMultiDbDataToTextConfig(cfg);
Assert.AreEqual(expected, actual);
}
public void IsTransactionalTest()
{
MultiDbData target = new MultiDbData();
bool expected = false;
bool actual;
target.IsTransactional = expected;
actual = target.IsTransactional;
Assert.AreEqual(expected, actual);
target.IsTransactional = true;
actual = target.IsTransactional;
Assert.AreEqual(true, actual);
}
public void RunAsTrialTest()
{
MultiDbData target = new MultiDbData();
bool expected = true;
bool actual;
target.RunAsTrial = expected;
actual = target.RunAsTrial;
Assert.AreEqual(expected, actual);
target.RunAsTrial = false;
actual = target.RunAsTrial;
Assert.AreEqual(false, actual);
}
public void ValidateAndLoadMultiDbDataTest_MissingCfgFile()
{
string multiDbOverrideSettingFileName = @"C:\temp\test_not_here.multidbq";
MultiDbData multiData = null;
string[] errorMessages = null;
int expected = (int)ExecutionReturn.NullMultiDbConfig;
int actual;
actual = Validation.ValidateAndLoadMultiDbData(multiDbOverrideSettingFileName, null, out multiData, out errorMessages);
Assert.IsNull(multiData);
Assert.AreEqual(2, errorMessages.Length);
Assert.IsTrue(errorMessages[0].IndexOf("Unable to read in configuration file") > -1);
Assert.AreEqual(expected, actual);
}
public void ValidateAndLoadMultiDbDataTest_WellFormedMultiDbData()
{
string cfgContents = @"Server1\Instance_1:Default,MyDatabaseOverride";
string multiDbOverrideSettingFileName = Path.GetTempPath() + System.Guid.NewGuid().ToString() + ".cfg";
File.WriteAllText(multiDbOverrideSettingFileName,cfgContents);
MultiDbData multiData = null;
string[] errorMessages = null;
int expected = 0;
int actual;
actual = Validation.ValidateAndLoadMultiDbData(multiDbOverrideSettingFileName, null, out multiData, out errorMessages);
File.Delete(multiDbOverrideSettingFileName);
Assert.IsNotNull(multiData);
Assert.AreEqual(0, errorMessages.Length);
Assert.AreEqual(expected, actual);
}
/// <summary>
/// Divides the targets into "concurrency" number of lists per database server. The lists would be run in parallel, with the items in each list run in serial
/// Would be used to control the number of threads targeting a single server
/// </summary>
/// <param name="multiData"></param>
/// <param name="concurrency"></param>
/// <returns>"Concurrency" number of lists per Server</returns>
public static List <IEnumerable <(string, List <DatabaseOverride>)> > MaxConcurrencyByServer(MultiDbData multiData, int concurrency)
{
List <IEnumerable <(string, List <DatabaseOverride>)> > tmp = new List <IEnumerable <(string, List <DatabaseOverride>)> >();
var serverChunks = ConcurrencyByServer(multiData);
foreach (var sC in serverChunks)
{
var subChunks = sC.SplitIntoChunks(concurrency);
tmp.AddRange(subChunks);
}
return(tmp);
}
/// <summary>
/// Divides the targets into a list per database server. The lists would be run in parallel, with the items in each list run in serial
/// Would be used to "single thread" per server while parallel across servers
/// </summary>
/// <param name="multiData">The database orveride object</param>
/// <returns>One list per Server</returns>
public static List <IEnumerable <(string, List <DatabaseOverride>)> > ConcurrencyByServer(MultiDbData multiData)
{
List <IEnumerable <(string, List <DatabaseOverride>)> > tmp = new List <IEnumerable <(string, List <DatabaseOverride>)> >();
foreach (ServerData srv in multiData)
{
var lstSrv = new List <(string, List <DatabaseOverride>)>();
foreach (List <DatabaseOverride> ovr in srv.OverrideSequence.Values)
{
lstSrv.Add((srv.ServerName, ovr));
}
tmp.Add(lstSrv);
}
return(tmp);
}
/// <summary>
/// Divides the targets into the "concurrency" count of Lists. The lists would be run in parallel, with the items in each list run in serial
/// Fully parallel to the extent of the concurrency number
/// </summary>
/// <param name="multiData">The database orveride object</param>
/// <param name="concurrency">The number of lists to return</param>
/// <returns>"Concurrency" number of target lists </returns>
public static List <IEnumerable <(string, List <DatabaseOverride>)> > ConcurrencyByInt(MultiDbData multiData, int concurrency)
{
var flattened = FlattenOverride(multiData);
var batches = flattened.SplitIntoChunks(concurrency).ToList();;
return(batches);
}
public static List <IEnumerable <(string, List <DatabaseOverride>)> > ConcurrencyByType(MultiDbData multiData, int concurrency, ConcurrencyType concurrencyType)
{
switch (concurrencyType)
{
case ConcurrencyType.Server:
return(ConcurrencyByServer(multiData));
case ConcurrencyType.MaxPerServer:
return(MaxConcurrencyByServer(multiData, concurrency));
case ConcurrencyType.Count:
default:
return(ConcurrencyByInt(multiData, concurrency));
}
}
public HashCollector(MultiDbData multiDbData)
{
this.multiDbData = multiDbData;
}
public static DacpacDeltasStatus GetSbmFromDacPac(string rootLoggingPath, string platinumDacPac, string database, AuthenticationType authType, string server, string username, string password, string buildRevision, int defaultScriptTimeout, MultiDbData multiDb, out string sbmName, bool batchScripts, bool allowObjectDelete)
{
return(GetSbmFromDacPac(rootLoggingPath, platinumDacPac, string.Empty, database, server, authType, username, password, buildRevision, defaultScriptTimeout, multiDb, out sbmName, batchScripts, allowObjectDelete));
}
public void MultiDbDataConstructorTest()
{
MultiDbData target = new MultiDbData();
Assert.AreEqual(0, target.Count);
}
/// <summary>
/// Execute method that is used inherently from the Execute()
/// </summary>
/// <returns></returns>
private int ExecuteFromOverrideFile(string buildZipFileName, string platinumDacPacFileName, MultiDbData multiData, string rootLoggingPath, string description, string buildRequestedBy, bool forceCustomDacpac, int concurrency = 20, ConcurrencyType concurrencyType = ConcurrencyType.Count)
{
try
{
var tasks = new List <Task <int> >();
int targetTotal = 0;
try
{
startTime = DateTime.Now;
log.LogInformation($"Starting Threaded processing at {startTime.ToString()}");
var concurrencyBuckets = Concurrency.ConcurrencyByType(multiData, concurrency, concurrencyType);
targetTotal = concurrencyBuckets.Sum(c => c.Count());
foreach (var bucket in concurrencyBuckets)
{
tasks.Add(ProcessConcurrencyBucket(bucket, forceCustomDacpac));
}
}
catch (Exception exe)
{
WriteToLog(new LogMsg()
{
Message = exe.ToString(), LogType = LogType.Error
});
}
//Wait for all of the tasks to finish
Task.WaitAll(tasks.ToArray());
TimeSpan interval = DateTime.Now - startTime;
var finalMsg = new LogMsg()
{
RunId = ThreadedExecution.RunID, Message = $"Ending threaded processing at {DateTime.Now.ToUniversalTime()}", LogType = LogType.Message
};
WriteToLog(finalMsg);
finalMsg.Message = $"Execution Duration: {interval.ToString()}";
WriteToLog(finalMsg);
finalMsg.Message = $"Total number of targets: {targetTotal.ToString()}";
WriteToLog(finalMsg);
WriteToLog(new LogMsg()
{
LogType = LogType.SuccessDatabases
});
if (this.hasError)
{
WriteToLog(new LogMsg()
{
LogType = LogType.FailureDatabases
});
finalMsg.Message = "Finishing with Errors";
finalMsg.LogType = LogType.Error;
WriteToLog(finalMsg);
finalMsg.LogType = LogType.Message;
WriteToLog(finalMsg);
return((int)ExecutionReturn.FinishingWithErrors);
}
else
{
log.LogInformation("Successful");
return((int)ExecutionReturn.Successful);
}
}
catch (Exception bigExe)
{
log.LogCritical($"Big problem running the threaded build...{bigExe.ToString()}");
return((int)ExecutionReturn.NullBuildData);
}
}
public static DacpacDeltasStatus GetSbmFromDacPac(string rootLoggingPath, string platinumDacPac, string targetDacpac, string database, string server, AuthenticationType authType, string username, string password, string buildRevision, int defaultScriptTimeout, MultiDbData multiDb, out string sbmName, bool batchScripts, bool allowObjectDelete)
{
string workingFolder = (!string.IsNullOrEmpty(rootLoggingPath) ? rootLoggingPath : Path.GetTempPath());
workingFolder = Path.Combine(workingFolder, "Dacpac");
if (!Directory.Exists(workingFolder))
{
Directory.CreateDirectory(workingFolder);
}
log.LogInformation("Starting process: create SBM build file from dacpac settings");
DacpacDeltasStatus stat = DacpacDeltasStatus.Processing;
sbmName = string.Empty;
if (!String.IsNullOrEmpty(targetDacpac))
{
stat = DacPacHelper.CreateSbmFromDacPacDifferences(platinumDacPac, targetDacpac, batchScripts, buildRevision, defaultScriptTimeout, allowObjectDelete, out sbmName);
}
else if (!string.IsNullOrEmpty(database) && !string.IsNullOrEmpty(server))
{
string targetDacPac = Path.Combine(workingFolder, database + ".dacpac");
if (!DacPacHelper.ExtractDacPac(database, server, authType, username, password, targetDacPac))
{
log.LogError($"Error extracting dacpac from {database} : {server}");
return(DacpacDeltasStatus.ExtractionFailure);
}
stat = DacPacHelper.CreateSbmFromDacPacDifferences(platinumDacPac, targetDacPac, batchScripts, buildRevision, defaultScriptTimeout, allowObjectDelete, out sbmName);
}
if (stat == DacpacDeltasStatus.Processing && multiDb != null)
{
foreach (var serv in multiDb)
{
server = serv.ServerName;
for (int i = 0; i < serv.OverrideSequence.Count; i++)
{
database = serv.OverrideSequence.ElementAt(i).Value[0].OverrideDbTarget;
string targetDacPac = Path.Combine(workingFolder, database + ".dacpac");
if (!DacPacHelper.ExtractDacPac(database, server, authType, username, password, targetDacPac))
{
log.LogError($"Error extracting dacpac from {server} : {database}");
return(DacpacDeltasStatus.ExtractionFailure);
}
stat = DacPacHelper.CreateSbmFromDacPacDifferences(platinumDacPac, targetDacPac, batchScripts, buildRevision, defaultScriptTimeout, allowObjectDelete, out sbmName);
if (stat == DacpacDeltasStatus.InSync)
{
log.LogInformation($"{Path.GetFileName(platinumDacPac)} and {Path.GetFileName(targetDacPac)} are already in sync. Looping to next database.");
stat = DacpacDeltasStatus.Processing;
}
else if (stat == DacpacDeltasStatus.OnlyPostDeployment)
{
log.LogInformation($"{Path.GetFileName(platinumDacPac)} to { Path.GetFileName(targetDacPac)} appears to have only Post-Deployment steps. Will not be used as a source - looping to next database.");
stat = DacpacDeltasStatus.Processing;
}
else
{
break;
}
}
if (stat != DacpacDeltasStatus.Processing)
{
break;
}
}
}
switch (stat)
{
case DacpacDeltasStatus.Success:
log.LogInformation("Successfully created SBM from two dacpacs");
break;
case DacpacDeltasStatus.InSync:
case DacpacDeltasStatus.OnlyPostDeployment:
log.LogInformation("The database is already in sync with platinum dacpac");
break;
case DacpacDeltasStatus.Processing: //we've looped through them all and they are in sync!
stat = DacpacDeltasStatus.InSync;
log.LogInformation("All databases are already in sync with platinum dacpac");
break;
default:
log.LogError("Error creating build package from supplied Platinum and Target dacpac files");
break;
}
return(stat);
}
/// <summary>
/// Constructor that takes int the MultiDbData object that contains the list of target databases and their servers.
/// </summary>
/// <param name="multiDbData">MultiDbData containing the target databases</param>
/// <param name="resultsFilePath">The directory where the ultimate results file will be created. Will be used as the root for the temp files</param>
public QueryCollector(MultiDbData multiDbData, ConnectionData connData)
{
this.multiDbData = multiDbData;
this.connData = connData;
}
public static (int, CommandLineArgs) ValidateAndLoadPlatinumDacpac(CommandLineArgs cmdLine, MultiDbData multiDb)
{
//DacPac settings validation
if (!String.IsNullOrEmpty(cmdLine.DacPacArgs.PlatinumDacpac))
{
if (!File.Exists(cmdLine.DacPacArgs.PlatinumDacpac))
{
string err = String.Format("A Platinum Dacpac file was specified but could not be located at '{0}'", cmdLine.DacPacArgs.PlatinumDacpac);
log.LogError(err);
return(-729, cmdLine);
}
if (!String.IsNullOrEmpty(cmdLine.DacPacArgs.TargetDacpac) && !File.Exists(cmdLine.DacPacArgs.TargetDacpac))
{
string err = String.Format("A Target Dacpac file was specified but could not be located at '{0}'", cmdLine.DacPacArgs.TargetDacpac);
log.LogError(err);
return(-728, cmdLine);
}
}
//If there are Dacpac settings... we will need to create the SBM automatically..
if (!string.IsNullOrEmpty(cmdLine.DacPacArgs.PlatinumDacpac) && string.IsNullOrEmpty(cmdLine.BuildFileName))
{
if (cmdLine.DacPacArgs.ForceCustomDacPac == false)
{
string sbmName;
var stat = Worker.GetSbmFromDacPac(cmdLine, multiDb, out sbmName, true);
if (stat == DacpacDeltasStatus.Success)
{
cmdLine.BuildFileName = sbmName;
return((int)ExecutionReturn.Successful, cmdLine);
}
else if (stat == DacpacDeltasStatus.InSync)
{
return((int)ExecutionReturn.DacpacDatabasesInSync, cmdLine);
}
else
{
log.LogError("Error creating SBM package from Platinum dacpac");
return(-5120, cmdLine);
}
}
else
{
log.LogInformation("Found --forcecustomdacpac setting. Skipping the creation of the single platinum SBM package. Individual dacpacs and SBMs will be created");
}
}
return(0, cmdLine);
}
/// <summary>
/// Accepts a Multi-Database configuration file, processes it and outputs a populated MultiDbData object
/// </summary>
/// <param name="multiDbOverrideSettingFileName">Valid Multi-database file (.multiDb, .multiDbQ, .cfg)</param>
/// <param name="multiData">Out parameter of populated MultiDbData object</param>
/// <param name="errorMessages">Out parameter or error messages (if any)</param>
/// <returns>Zero (0) if no errors, otherwise an error code</returns>
public static int ValidateAndLoadMultiDbData(string multiDbOverrideSettingFileName, CommandLineArgs cmdLine, out MultiDbData multiData, out string[] errorMessages)
{
log.LogInformation("Validating target database settings");
string message = string.Empty;
string error;
errorMessages = new string[0];
multiData = null;
string extension = Path.GetExtension(multiDbOverrideSettingFileName).ToLowerInvariant();
switch (extension)
{
case ".multidb":
multiData = MultiDbHelper.DeserializeMultiDbConfiguration(multiDbOverrideSettingFileName);
break;
case ".multidbq":
multiData = MultiDbHelper.CreateMultiDbConfigFromQueryFile(multiDbOverrideSettingFileName, out message);
break;
case ".sql":
if (cmdLine != null)
{
ConnectionData connData = new ConnectionData()
{
DatabaseName = cmdLine.Database,
SQLServerName = cmdLine.Server,
UserId = cmdLine.AuthenticationArgs.UserName,
Password = cmdLine.AuthenticationArgs.Password,
AuthenticationType = cmdLine.AuthenticationArgs.AuthenticationType
};
multiData = MultiDbHelper.CreateMultiDbConfigFromQuery(connData, File.ReadAllText(cmdLine.MultiDbRunConfigFileName), out message);
}
break;
case ".cfg":
default:
multiData = MultiDbHelper.ImportMultiDbTextConfig(multiDbOverrideSettingFileName);
break;
}
//if (multiDbOverrideSettingFileName.EndsWith(".multidb", StringComparison.InvariantCultureIgnoreCase))
// multiData = MultiDbHelper.DeserializeMultiDbConfiguration(multiDbOverrideSettingFileName);
//else if (multiDbOverrideSettingFileName.EndsWith(".multidbq", StringComparison.InvariantCultureIgnoreCase))
// multiData = MultiDbHelper.CreateMultiDbConfigFromQueryFile(multiDbOverrideSettingFileName, out message);
//else if (multiDbOverrideSettingFileName.EndsWith(".cfg", StringComparison.InvariantCultureIgnoreCase))
// multiData = MultiDbHelper.ImportMultiDbTextConfig(multiDbOverrideSettingFileName);
if (multiData == null || multiData.Count() == 0)
{
error = "Unable to read in configuration file " + multiDbOverrideSettingFileName + ((message.Length > 0) ? " :: " + message : "");
errorMessages = new string[] { error, "Returning error code: " + (int)ExecutionReturn.NullMultiDbConfig };
log.LogError(error);
return((int)ExecutionReturn.NullMultiDbConfig);
}
if (!MultiDbHelper.ValidateMultiDatabaseData(multiData))
{
error = "One or more scripts is missing a default or target override database setting. Run has been halted. Please correct the error and try again";
errorMessages = new string[] { error, "Returning error code: " + (int)ExecutionReturn.MissingTargetDbOverrideSetting };
log.LogError(error);
return((int)ExecutionReturn.MissingTargetDbOverrideSetting);
}
return(0);
}
public static List <IEnumerable <(string, List <DatabaseOverride>)> > RecombineServersToFixedBucketCount(MultiDbData multiData, int fixedBucketCount)
{
List <IEnumerable <(string, List <DatabaseOverride>)> > consolidated = new List <IEnumerable <(string, List <DatabaseOverride>)> >();
//Get a bucket per server
var buckets = ConcurrencyByServer(multiData);
int itemCheckSum = buckets.Sum(b => b.Count());
//get the ideal number of items per bucket
double idealBucketSize = Math.Ceiling((double)buckets.Sum(b => b.Count()) / (double)fixedBucketCount);
//Get any that are already over the ideal size
var over = buckets.Where(b => b.Count() >= idealBucketSize);
if (over.Count() > 0)
{
consolidated.AddRange(over);
over.ToList().ForEach(o => buckets.Remove(o));
}
//If we have taken some oversize items out, we need to re-calc the ideal bucket size for the remaining buckets
idealBucketSize = Math.Ceiling((double)buckets.Sum(b => b.Count()) / ((double)fixedBucketCount - consolidated.Count()));
//Start creating -- fill as best to start, but not exceeding the ideal size and equalling the bucket count
while (buckets.Count() > 0)
{
int bucketSum = 0;
//get the next group -- note this will actually return a collection that is just over the ideal size
var nextSet = buckets.OrderByDescending(b => b.Count()).TakeWhile(p =>
{
bool exceeded = bucketSum > idealBucketSize;
bucketSum += p.Count();
return(!exceeded);
});
var nextTmp = nextSet.ToList();
if (nextTmp.Count() > 0)
{
//trim the next group so that it's not too large
while (nextTmp.Sum(n => n.Count()) > idealBucketSize && nextTmp.Count() > 1)
{
nextTmp.RemoveAt(nextTmp.Count() - 1);
}
//If doing this combining would put us under the number of buckets needed, then add individual buckets to the desired count and let the clean up take care of the rest.
if (consolidated.Count + buckets.Count - nextTmp.Count < fixedBucketCount)
{
while (consolidated.Count() < fixedBucketCount && buckets.Count != 0)
{
consolidated.Add(buckets.First());
buckets.RemoveAt(0);
}
break;
}
//Combine the set into a single entry
var tmp = new List <(string, List <DatabaseOverride>)>();
foreach (var n in nextTmp)
{
tmp.AddRange(n);
}
//Remove those selected items from the original set of buckets
nextTmp.ForEach(o => buckets.Remove(o));
//Add the combined set into the consolidated list
consolidated.Add(tmp);
//if any further grouping will result in too few buckets, then add the remaining buckets individually
if (consolidated.Count() + buckets.Count() == fixedBucketCount && buckets.Count() > 0)
{
buckets.ForEach(b => consolidated.Add(b));
buckets.Clear();
break;
}
}
//did we hit the max number of buckets? if so, break out. We'll deal with any left overs next
if (consolidated.Count() == fixedBucketCount)
{
break;
}
}
//Capture any left over buckets, loop to add the largest remaining buckets to the smallest buckets in the consolidated list
if (buckets.Count() > 0)
{
buckets = buckets.OrderByDescending(c => c.Count()).ToList();
consolidated = consolidated.OrderBy(c => c.Count()).ToList();
while (buckets.Count() > 0)
{
for (int i = 0; i < consolidated.Count(); i++)
{
if (buckets.Count() == 0)
{
break;
}
var t = consolidated[i].ToList();
t.AddRange(buckets.First());
consolidated[i] = t;
buckets.RemoveAt(0);
}
}
}
//Make sure we didn't lose anything!!
if (itemCheckSum != consolidated.Sum(c => c.Count()))
{
throw new Exception($"While filling concurrency buckets, the end count of {consolidated.Sum(c => c.Count())} does not equal the start count of {itemCheckSum}");
}
return(consolidated);
}
public async Task <int> SendTargetsToQueue(MultiDbData multiDb, ConcurrencyType cType)
{
try
{
log.LogInformation($"Setting up Topic Subscription with Job filter name '{this.jobName}'");
await RemoveDefaultFilters();
await CleanUpCustomFilters();
await CreateBatchJobFilter(cType == ConcurrencyType.Count?false : true);
var sender = this.Client.CreateSender(topicName);
//Use bucketing to 1 bucket to get flattened list of targest
var concurrencyBuckets = Concurrency.ConcurrencyByType(multiDb, 1, ConcurrencyType.Count);
var messages = CreateMessages(concurrencyBuckets, jobName);
int count = messages.Count();
int sentCount = 0;
//because of partitiioning, can't batch across session Id, so group by SessionId first, then batch
var bySessionId = messages.GroupBy(s => s.SessionId);
foreach (var sessionSet in bySessionId)
{
var msgBatch = sessionSet.Batch(20); //send in batches of 20
foreach (var b in msgBatch)
{
var sbb = await sender.CreateMessageBatchAsync();
foreach (var msg in b)
{
if (!sbb.TryAddMessage(msg))
{
log.LogError($"Failed to add message to Service Bus batch.{Environment.NewLine}{msg.Body}");
}
else
{
sentCount++;
}
}
await sender.SendMessagesAsync(sbb);
}
}
if (sentCount != count)
{
log.LogError($"Only {sentCount} out of {count} database targets were sent to the Service Bus. Before running your workload, please run a 'dequeue' command and try again");
return(-1);
}
//Confirm message count in Queue
int retry = 0;
var activeMessages = await MonitorServiceBustopic(cType);
while (activeMessages != count && retry < 4)
{
Thread.Sleep(1000);
activeMessages = await MonitorServiceBustopic(cType);
}
if (activeMessages != count)
{
log.LogError($"After attempting to queue messages, there are only {activeMessages} out of {count} messages in the Service Bus Subscription. Before running your workload, please run a 'dequeue' command and try again");
return(-1);
}
else
{
log.LogInformation($"Validated {activeMessages} of {count} active messages in Service Bus Subscription {this.topicName}:{this.topicSessionSubscriptionName}");
}
return(count);
}
catch (Exception exe)
{
log.LogError(exe, "Failed to send database override targets to Service Bus Queue");
return(-1);
}
}
