/// <summary>
/// File Server failover
/// For windows, simulated by disabling current access node of clustered file server
/// </summary>
/// <param name="server">Name of clustered file server</param>
/// <param name="fsType">Type of clustered file server</param>
protected void FailoverServer(IPAddress currentAccessIp, string server, FileServerType fsType)
{
isAccessNodeFailovered = true;
// Non Windows
if (!TestConfig.IsWindowsPlatform)
{
sutController.TriggerFailover(currentAccessIp.ToString());
return;
}
// Windows, before failover, we need to restore all nodes to be enabled.
RestoreClusterNodes(TestConfig.ClusterNode01, TestConfig.ClusterNode02);
#region Failover accessed node
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Failover node {0}", currentAccessNode);
FailoverClusterNode(currentAccessNode);
#endregion
#region Wait for available server
if (fsType == FileServerType.GeneralFileServer)
{
string newOwnerNode = null;
//Wait the file server back online
DoUntilSucceed(() =>
{
newOwnerNode = sutController.GetClusterResourceOwner(server);
if (!string.IsNullOrEmpty(newOwnerNode))
{
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"currentAccessNode: {0}, newOwnerNode: {1}", currentAccessNode.ToUpper(), newOwnerNode.ToUpper());
}
else
{
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"currentAccessNode: {0}, newOwnerNode: {1}", currentAccessNode.ToUpper(), "null or empty!");
}
return(!string.IsNullOrEmpty(newOwnerNode) && newOwnerNode.ToUpper() != currentAccessNode.ToUpper());
}, TestConfig.FailoverTimeout, "Retry to get cluster owner node until succeed within timeout span.");
BaseTestSite.Assert.AreNotEqual(
null,
newOwnerNode,
"New owner node should not be null.");
BaseTestSite.Assert.AreNotEqual(
currentAccessNode.ToUpper(),
newOwnerNode.ToUpper(),
"New owner node should not be the same as the old one.");
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Current owner node changed to {0}", newOwnerNode);
}
#endregion
}
protected void AssignCurrentAccessNode(string server, FileServerType fsType, IPAddress currentAccessIpAddr)
{
if (fsType == FileServerType.GeneralFileServer)
{
currentAccessNode = sutController.GetClusterResourceOwner(server);
}
else
{
currentAccessNode = Dns.GetHostEntry(currentAccessIpAddr).HostName;
}
BaseTestSite.Assert.AreNotEqual(
true,
string.IsNullOrEmpty(currentAccessNode),
"Current owner node is NOT expected to be Null or Empty");
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Current owner node is {0}", currentAccessNode);
}
protected void AssignCurrentAccessNode(string server, FileServerType fsType, IPAddress currentAccessIpAddr)
{
if (fsType == FileServerType.GeneralFileServer)
{
currentAccessNode = sutController.GetClusterResourceOwner(server);
}
else
{
currentAccessNode = Dns.GetHostEntry(currentAccessIpAddr).HostName;
}
BaseTestSite.Assert.AreNotEqual(
true,
string.IsNullOrEmpty(currentAccessNode),
"Current owner node is NOT expected to be Null or Empty");
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Current owner node is {0}", currentAccessNode);
}
private void FileServerFailoverTest(string server, FileServerType fsType, bool reconnectWithoutFailover = false)
{
int ret = 0;
uint callId = 0;
IPAddress currentAccessIpAddr = null;
WITNESS_INTERFACE_INFO registerInterface = new WITNESS_INTERFACE_INFO();
WITNESS_INTERFACE_LIST interfaceList = new WITNESS_INTERFACE_LIST();
currentAccessIpAddr = SWNTestUtility.GetCurrentAccessIP(server);
BaseTestSite.Log.Add(LogEntryKind.Debug, "Get current file server IP: {0}.", currentAccessIpAddr);
#region Register SWN witness
if (witnessType == WitnessType.SwnWitness)
{
if (TestConfig.IsWindowsPlatform && fsType == FileServerType.ScaleOutFileServer)
{
// Windows Server: when stopping a non-owner node of ScaleOutFS, no notication will be sent by SMB witness.
// So get one IP of the owner node of ScaleOutFS to access.
string resourceOwnerNode = sutController.GetClusterResourceOwner(server);
IPAddress[] ownerIpList = Dns.GetHostEntry(resourceOwnerNode).AddressList;
foreach (var ip in ownerIpList)
{
BaseTestSite.Log.Add(LogEntryKind.Debug, "Owner IP: {0}", ip);
}
if (!ownerIpList.Contains(currentAccessIpAddr))
{
currentAccessIpAddr = null;
IPAddress[] accessIpList = Dns.GetHostEntry(server).AddressList;
foreach (var ip in accessIpList)
{
if (ownerIpList.Contains(ip))
{
currentAccessIpAddr = ip;
break;
}
}
BaseTestSite.Assert.IsNotNull(currentAccessIpAddr, "IP should not be null.");
BaseTestSite.Log.Add(LogEntryKind.Debug, "Get the owner IP {0} as file server IP.", currentAccessIpAddr);
}
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, resourceOwnerNode), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
}
else
{
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
}
DoUntilSucceed(() =>
{
ret = swnClientForInterface.WitnessrGetInterfaceList(out interfaceList);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"WitnessrGetInterfaceList returns with result code = 0x{0:x8}", ret);
return(SWNTestUtility.VerifyInterfaceList(interfaceList, TestConfig.Platform));
}, TestConfig.FailoverTimeout, "Retry to call WitnessrGetInterfaceList until succeed within timeout span.");
SWNTestUtility.GetRegisterInterface(interfaceList, out registerInterface);
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForWitness,
(registerInterface.Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(registerInterface.IPV4) : SWNTestUtility.ConvertIPV6(registerInterface.IPV6),
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, registerInterface.InterfaceGroupName), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
string clientName = Guid.NewGuid().ToString();
BaseTestSite.Log.Add(LogEntryKind.Debug, "Register witness:");
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tNetName: {0}", SWNTestUtility.GetPrincipleName(TestConfig.DomainName, server));
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tIPAddress: {0}", currentAccessIpAddr.ToString());
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tClient Name: {0}", clientName);
ret = swnClientForWitness.WitnessrRegister(SwnVersion.SWN_VERSION_1, SWNTestUtility.GetPrincipleName(TestConfig.DomainName, server),
currentAccessIpAddr.ToString(), clientName, out pContext);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"WitnessrRegister returns with result code = 0x{0:x8}", ret);
BaseTestSite.Assert.IsNotNull(
pContext,
"Expect pContext is not null.");
callId = swnClientForWitness.WitnessrAsyncNotify(pContext);
BaseTestSite.Assert.AreNotEqual <uint>(
0,
callId,
"WitnessrAsyncNotify returns callId = {0}", callId);
}
#endregion
#region Create a file and write content
string uncSharePath = Smb2Utility.GetUncPath(server, TestConfig.ClusteredFileShare);
string content = Smb2Utility.CreateRandomString(TestConfig.WriteBufferLengthInKb);
string testDirectory = CreateTestDirectory(uncSharePath);
string file = Path.Combine(testDirectory, Guid.NewGuid().ToString());
Guid clientGuid = Guid.NewGuid();
Guid createGuid = Guid.NewGuid();
DoUntilSucceed(() => WriteContentBeforeFailover(fsType, server, currentAccessIpAddr, uncSharePath, file, content, clientGuid, createGuid),
TestConfig.FailoverTimeout,
"Before failover, retry Write content until succeed within timeout span.");
#endregion
#region Disable accessed node
if (TestConfig.IsWindowsPlatform)
{
AssignCurrentAccessNode(server, fsType, currentAccessIpAddr);
}
if (!reconnectWithoutFailover)
{
BaseTestSite.Log.Add(
LogEntryKind.TestStep,
"Disable owner node for general file server or the node currently provides the access for scale-out file server.");
FailoverServer(currentAccessIpAddr, server, fsType);
}
#endregion
#region Wait for available server
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Wait for available server.");
if (witnessType == WitnessType.None)
{
if (fsType == FileServerType.GeneralFileServer)
{
currentAccessIpAddr = null;
IPAddress[] accessIpList = Dns.GetHostEntry(server).AddressList;
DoUntilSucceed(() =>
{
foreach (IPAddress ipAddress in accessIpList)
{
Smb2FunctionalClient pingClient = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
try
{
pingClient.ConnectToServerOverTCP(ipAddress);
pingClient.Disconnect();
pingClient = null;
currentAccessIpAddr = ipAddress;
return(true);
}
catch
{
}
}
return(false);
}, TestConfig.FailoverTimeout, "Retry to ping to server until succeed within timeout span");
}
else
{
currentAccessIpAddr = null;
IPAddress[] accessIpList = Dns.GetHostEntry(server).AddressList;
foreach (IPAddress ipAddress in accessIpList)
{
if (TestConfig.IsWindowsPlatform)
{
// When setting failover mode to StopNodeService for Windows, SMB2 servers on two nodes can still be accessed by the client.
// So the client needs to get the new node to access it after failover by comparing host name.
if (string.Compare(currentAccessNode, Dns.GetHostEntry(ipAddress).HostName, true) == 0)
{
continue;
}
}
Smb2FunctionalClient pingClient = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
try
{
pingClient.ConnectToServerOverTCP(ipAddress);
pingClient.Disconnect();
pingClient = null;
currentAccessIpAddr = ipAddress;
break;
}
catch
{
}
}
}
}
else if (witnessType == WitnessType.SwnWitness)
{
// Verifying for notification
RESP_ASYNC_NOTIFY respNotify;
do
{
// Wait the notification
ret = swnClientForWitness.ExpectWitnessrAsyncNotify(callId, out respNotify);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"ExpectWitnessrAsyncNotify returns with result code = 0x{0:x8}", ret);
SWNTestUtility.PrintNotification(respNotify);
RESOURCE_CHANGE[] resourceChangeList;
SwnUtility.ParseResourceChange(respNotify, out resourceChangeList);
BaseTestSite.Assert.AreEqual <uint>(0x00000001, respNotify.NumberOfMessages, "Expect NumberOfMessages is set to 1.");
if (resourceChangeList[0].ChangeType == (uint)SwnResourceChangeType.RESOURCE_STATE_AVAILABLE)
{
// Verify RESP_ASYNC_NOTIFY, the resource is available
SWNTestUtility.VerifyResourceChange(respNotify, SwnResourceChangeType.RESOURCE_STATE_AVAILABLE);
break;
}
// Verify RESP_ASYNC_NOTIFY, the resource is unavailable
SWNTestUtility.VerifyResourceChange(respNotify, SwnResourceChangeType.RESOURCE_STATE_UNAVAILABLE);
callId = swnClientForWitness.WitnessrAsyncNotify(pContext);
BaseTestSite.Assert.AreNotEqual <uint>(0, callId, "WitnessrAsyncNotify returns callId = {0}", callId);
} while (true);
ret = swnClientForWitness.WitnessrUnRegister(pContext);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"WitnessrUnRegister returns with result code = 0x{0:x8}", ret);
pContext = IntPtr.Zero;
swnClientForWitness.SwnUnbind(TestConfig.Timeout);
if (fsType == FileServerType.ScaleOutFileServer)
{
// For scale-out file server case, retrieve and use another access IP for connection
currentAccessIpAddr =
(registerInterface.Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(registerInterface.IPV4) : SWNTestUtility.ConvertIPV6(registerInterface.IPV6);
}
}
#endregion
#region Read content and close the file
DoUntilSucceed(() => ReadContentAfterFailover(server, currentAccessIpAddr, uncSharePath, file, content, clientGuid, createGuid),
TestConfig.FailoverTimeout,
"After failover, retry Read content until succeed within timeout span.");
#endregion
}
private void FileServerFailoverTest(string server, FileServerType fsType, bool reconnectWithoutFailover = false)
{
int ret = 0;
uint callId = 0;
IPAddress currentAccessIpAddr = null;
WITNESS_INTERFACE_INFO registerInterface = new WITNESS_INTERFACE_INFO();
WITNESS_INTERFACE_LIST interfaceList = new WITNESS_INTERFACE_LIST();
currentAccessIpAddr = SWNTestUtility.GetCurrentAccessIP(server);
BaseTestSite.Log.Add(LogEntryKind.Debug, "Get current file server IP: {0}.", currentAccessIpAddr);
#region Register SWN witness
if (witnessType == WitnessType.SwnWitness)
{
if (TestConfig.IsWindowsPlatform && fsType == FileServerType.ScaleOutFileServer)
{
// Windows Server: when stopping a non-owner node of ScaleOutFS, no notication will be sent by SMB witness.
// So get one IP of the owner node of ScaleOutFS to access.
string resourceOwnerNode = sutController.GetClusterResourceOwner(server);
IPAddress[] ownerIpList = Dns.GetHostEntry(resourceOwnerNode).AddressList;
foreach (var ip in ownerIpList)
{
BaseTestSite.Log.Add(LogEntryKind.Debug, "Owner IP: {0}", ip);
}
if (!ownerIpList.Contains(currentAccessIpAddr))
{
currentAccessIpAddr = null;
IPAddress[] accessIpList = Dns.GetHostEntry(server).AddressList;
foreach (var ip in accessIpList)
{
if (ownerIpList.Contains(ip))
{
currentAccessIpAddr = ip;
break;
}
}
BaseTestSite.Assert.IsNotNull(currentAccessIpAddr, "IP should not be null.");
BaseTestSite.Log.Add(LogEntryKind.Debug, "Get the owner IP {0} as file server IP.", currentAccessIpAddr);
}
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, resourceOwnerNode), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
}
else
{
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
}
DoUntilSucceed(() =>
{
ret = swnClientForInterface.WitnessrGetInterfaceList(out interfaceList);
BaseTestSite.Assert.AreEqual<SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"WitnessrGetInterfaceList returns with result code = 0x{0:x8}", ret);
return SWNTestUtility.VerifyInterfaceList(interfaceList, TestConfig.Platform);
}, TestConfig.FailoverTimeout, "Retry to call WitnessrGetInterfaceList until succeed within timeout span.");
SWNTestUtility.GetRegisterInterface(interfaceList, out registerInterface);
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForWitness,
(registerInterface.Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(registerInterface.IPV4) : SWNTestUtility.ConvertIPV6(registerInterface.IPV6),
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, registerInterface.InterfaceGroupName), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
string clientName = Guid.NewGuid().ToString();
BaseTestSite.Log.Add(LogEntryKind.Debug, "Register witness:");
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tNetName: {0}", SWNTestUtility.GetPrincipleName(TestConfig.DomainName, server));
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tIPAddress: {0}", currentAccessIpAddr.ToString());
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tClient Name: {0}", clientName);
ret = swnClientForWitness.WitnessrRegister(SwnVersion.SWN_VERSION_1, SWNTestUtility.GetPrincipleName(TestConfig.DomainName, server),
currentAccessIpAddr.ToString(), clientName, out pContext);
BaseTestSite.Assert.AreEqual<SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"WitnessrRegister returns with result code = 0x{0:x8}", ret);
BaseTestSite.Assert.IsNotNull(
pContext,
"Expect pContext is not null.");
callId = swnClientForWitness.WitnessrAsyncNotify(pContext);
BaseTestSite.Assert.AreNotEqual<uint>(
0,
callId,
"WitnessrAsyncNotify returns callId = {0}", callId);
}
#endregion
#region Create a file and write content
string uncSharePath = Smb2Utility.GetUncPath(server, TestConfig.ClusteredFileShare);
string content = Smb2Utility.CreateRandomString(TestConfig.WriteBufferLengthInKb);
string testDirectory = CreateTestDirectory(uncSharePath);
string file = Path.Combine(testDirectory, Guid.NewGuid().ToString());
Guid clientGuid = Guid.NewGuid();
Guid createGuid = Guid.NewGuid();
DoUntilSucceed(() => WriteContentBeforeFailover(fsType, server, currentAccessIpAddr, uncSharePath, file, content, clientGuid, createGuid),
TestConfig.FailoverTimeout,
"Before failover, retry Write content until succeed within timeout span.");
#endregion
#region Disable accessed node
if (TestConfig.IsWindowsPlatform)
{
AssignCurrentAccessNode(server, fsType, currentAccessIpAddr);
}
if (!reconnectWithoutFailover)
{
BaseTestSite.Log.Add(
LogEntryKind.TestStep,
"Disable owner node for general file server or the node currently provides the access for scale-out file server.");
FailoverServer(currentAccessIpAddr, server, fsType);
}
#endregion
#region Wait for available server
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Wait for available server.");
if (witnessType == WitnessType.None)
{
if (fsType == FileServerType.GeneralFileServer)
{
currentAccessIpAddr = null;
IPAddress[] accessIpList = Dns.GetHostEntry(server).AddressList;
DoUntilSucceed(() =>
{
foreach (IPAddress ipAddress in accessIpList)
{
Smb2FunctionalClient pingClient = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
try
{
pingClient.ConnectToServerOverTCP(ipAddress);
pingClient.Disconnect();
pingClient = null;
currentAccessIpAddr = ipAddress;
return true;
}
catch
{
}
}
return false;
}, TestConfig.FailoverTimeout, "Retry to ping to server until succeed within timeout span");
}
else
{
currentAccessIpAddr = null;
IPAddress[] accessIpList = Dns.GetHostEntry(server).AddressList;
foreach (IPAddress ipAddress in accessIpList)
{
if (TestConfig.IsWindowsPlatform)
{
// When setting failover mode to StopNodeService for Windows, SMB2 servers on two nodes can still be accessed by the client.
// So the client needs to get the new node to access it after failover by comparing host name.
if (string.Compare(currentAccessNode, Dns.GetHostEntry(ipAddress).HostName, true) == 0)
{
continue;
}
}
Smb2FunctionalClient pingClient = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
try
{
pingClient.ConnectToServerOverTCP(ipAddress);
pingClient.Disconnect();
pingClient = null;
currentAccessIpAddr = ipAddress;
break;
}
catch
{
}
}
}
}
else if (witnessType == WitnessType.SwnWitness)
{
// Verifying for notification
RESP_ASYNC_NOTIFY respNotify;
do
{
// Wait the notification
ret = swnClientForWitness.ExpectWitnessrAsyncNotify(callId, out respNotify);
BaseTestSite.Assert.AreEqual<SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"ExpectWitnessrAsyncNotify returns with result code = 0x{0:x8}", ret);
SWNTestUtility.PrintNotification(respNotify);
RESOURCE_CHANGE[] resourceChangeList;
SwnUtility.ParseResourceChange(respNotify, out resourceChangeList);
BaseTestSite.Assert.AreEqual<uint>(0x00000001, respNotify.NumberOfMessages, "Expect NumberOfMessages is set to 1.");
if (resourceChangeList[0].ChangeType == (uint)SwnResourceChangeType.RESOURCE_STATE_AVAILABLE)
{
// Verify RESP_ASYNC_NOTIFY, the resource is available
SWNTestUtility.VerifyResourceChange(respNotify, SwnResourceChangeType.RESOURCE_STATE_AVAILABLE);
break;
}
// Verify RESP_ASYNC_NOTIFY, the resource is unavailable
SWNTestUtility.VerifyResourceChange(respNotify, SwnResourceChangeType.RESOURCE_STATE_UNAVAILABLE);
callId = swnClientForWitness.WitnessrAsyncNotify(pContext);
BaseTestSite.Assert.AreNotEqual<uint>(0, callId, "WitnessrAsyncNotify returns callId = {0}", callId);
} while (true);
ret = swnClientForWitness.WitnessrUnRegister(pContext);
BaseTestSite.Assert.AreEqual<SwnErrorCode>(
SwnErrorCode.ERROR_SUCCESS,
(SwnErrorCode)ret,
"WitnessrUnRegister returns with result code = 0x{0:x8}", ret);
pContext = IntPtr.Zero;
swnClientForWitness.SwnUnbind(TestConfig.Timeout);
if (fsType == FileServerType.ScaleOutFileServer)
{
// For scale-out file server case, retrieve and use another access IP for connection
currentAccessIpAddr =
(registerInterface.Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(registerInterface.IPV4) : SWNTestUtility.ConvertIPV6(registerInterface.IPV6);
}
}
#endregion
#region Read content and close the file
DoUntilSucceed(() => ReadContentAfterFailover(server, currentAccessIpAddr, uncSharePath, file, content, clientGuid, createGuid),
TestConfig.FailoverTimeout,
"After failover, retry Read content until succeed within timeout span.");
#endregion
}
/// <summary>
/// Write content before failover
/// </summary>
/// <param name="fsType">FileServerType</param>
/// <param name="server">File Server name.</param>
/// <param name="serverAccessIp">File Server Access IP.</param>
/// <param name="uncSharePath">The share path to write the file.</param>
/// <param name="file">The file name for writing content.</param>
/// <param name="content">The content to write.</param>
/// <param name="clientGuid">Smb2 client Guid.</param>
/// <param name="createGuid">The Guid for smb2 create request.</param>
/// <returns></returns>
protected bool WriteContentBeforeFailover(
FileServerType fsType,
string server,
IPAddress serverAccessIp,
string uncSharePath,
string file,
string content,
Guid clientGuid,
Guid createGuid)
{
uint status = 0;
beforeFailover = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Start a client by sending the following requests: NEGOTIATE; SESSION_SETUP; TREE_CONNECT to {0}", uncSharePath);
beforeFailover.ConnectToServer(TestConfig.UnderlyingTransport, server, serverAccessIp);
Capabilities_Values requestCapabilities = Capabilities_Values.GLOBAL_CAP_DFS | Capabilities_Values.GLOBAL_CAP_DIRECTORY_LEASING | Capabilities_Values.GLOBAL_CAP_LARGE_MTU | Capabilities_Values.GLOBAL_CAP_LEASING | Capabilities_Values.GLOBAL_CAP_MULTI_CHANNEL | Capabilities_Values.GLOBAL_CAP_PERSISTENT_HANDLES;
status = beforeFailover.Negotiate(
TestConfig.RequestDialects,
TestConfig.IsSMB1NegotiateEnabled,
capabilityValue: requestCapabilities,
clientGuid: clientGuid,
checker: (header, response) =>
{
TestConfig.CheckNegotiateDialect(DialectRevision.Smb30, response);
});
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Negotiate failed with {0}.", Smb2Status.GetStatusCode(status));
return(false);
}
status = beforeFailover.SessionSetup(
TestConfig.DefaultSecurityPackage,
server,
TestConfig.AccountCredential,
TestConfig.UseServerGssToken);
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "SessionSetup failed with {0}.", Smb2Status.GetStatusCode(status));
return(false);
}
uint treeId = 0;
Share_Capabilities_Values shareCapabilities = Share_Capabilities_Values.NONE;
status = DoUntilSucceed(
() => beforeFailover.TreeConnect(uncSharePath, out treeId, (header, response) =>
{
shareCapabilities = response.Capabilities;
}),
TestConfig.FailoverTimeout,
"Retry TreeConnect until succeed within timeout span");
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "TreeConnect failed with {0}.", Smb2Status.GetStatusCode(status));
return(false);
}
BaseTestSite.Assert.IsTrue(shareCapabilities.HasFlag(Share_Capabilities_Values.SHARE_CAP_CONTINUOUS_AVAILABILITY),
"CA Share should have SHARE_CAP_CONTINUOUS_AVAILABILITY bit set for Capabilities in TreeConnect response.");
if (fsType == FileServerType.ScaleOutFileServer)
{
BaseTestSite.Assert.IsTrue(shareCapabilities.HasFlag(Share_Capabilities_Values.SHARE_CAP_SCALEOUT),
"ScaleOut FS should have SHARE_CAP_SCALEOUT bit set for Capabilities in TreeConnect response.");
}
FILEID fileId;
Smb2CreateContextResponse[] serverCreateContexts;
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Client sends CREATE request with SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 with PERSISTENT flag set.");
status = beforeFailover.Create(
treeId,
file,
CreateOptions_Values.FILE_NON_DIRECTORY_FILE,
out fileId,
out serverCreateContexts,
RequestedOplockLevel_Values.OPLOCK_LEVEL_NONE,
new Smb2CreateContextRequest[] {
new Smb2CreateDurableHandleRequestV2
{
CreateGuid = createGuid,
Flags = CREATE_DURABLE_HANDLE_REQUEST_V2_Flags.DHANDLE_FLAG_PERSISTENT,
Timeout = 3600000,
},
});
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Create failed with {0}.", Smb2Status.GetStatusCode(status));
return(false);
}
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Client sends WRITE request to write content to the file.");
status = beforeFailover.Write(treeId, fileId, content);
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Write content failed with {0}.", Smb2Status.GetStatusCode(status));
return(false);
}
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Client sends FLUSH request.");
status = beforeFailover.Flush(treeId, fileId);
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Flush failed with {0}.", Smb2Status.GetStatusCode(status));
return(false);
}
return(true);
}
/// <summary>
/// Test CLIENT_MOVE_NOTIFICATION.
/// </summary>
/// <param name="expectedVersion">SwnVersion.SWN_VERSION_2 indicates that register with WitnessrRegisterEx, SwnVersion.SWN_VERSION_1 indicates that register with WitnessrRegister.</param>
private void SWNAsyncNotification_ClientMove(SwnVersion expectedVersion)
{
int ret = 0;
uint callId = 0;
WITNESS_INTERFACE_INFO registerInterface;
string server = TestConfig.ClusteredScaleOutFileServerName;
#region Get the file server to access it through SMB2
IPAddress currentAccessIp = SWNTestUtility.GetCurrentAccessIP(server);
BaseTestSite.Assert.AreNotEqual(null, currentAccessIp, "IP address of the file server should NOT be empty");
BaseTestSite.Log.Add(LogEntryKind.Debug, "Got the IP {0} to access the file server", currentAccessIp.ToString());
#endregion
#region Get register interface
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIp,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
WITNESS_INTERFACE_LIST interfaceList = new WITNESS_INTERFACE_LIST();
DoUntilSucceed(() =>
{
ret = swnClientForInterface.WitnessrGetInterfaceList(out interfaceList);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(SwnErrorCode.ERROR_SUCCESS, (SwnErrorCode)ret, "WitnessrGetInterfaceList returns with result code = 0x{0:x8}", ret);
return(SWNTestUtility.VerifyInterfaceList(interfaceList, TestConfig.Platform));
}, TestConfig.FailoverTimeout, "Retry to call WitnessrGetInterfaceList until succeed within timeout span");
swnClientForInterface.SwnUnbind(TestConfig.Timeout);
SWNTestUtility.GetRegisterInterface(interfaceList, out registerInterface);
SWNTestUtility.CheckVersion(expectedVersion, (SwnVersion)registerInterface.Version);
#endregion
#region Register SWN witness
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForWitness,
(registerInterface.Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(registerInterface.IPV4) : SWNTestUtility.ConvertIPV6(registerInterface.IPV6),
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
string clientName = Guid.NewGuid().ToString();
string netName = SWNTestUtility.GetPrincipleName(TestConfig.DomainName, server);
BaseTestSite.Log.Add(LogEntryKind.Debug, "Register witness:");
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tNetName: {0}", netName);
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tIPAddress: {0}", currentAccessIp.ToString());
BaseTestSite.Log.Add(LogEntryKind.Debug, "\tClient Name: {0}", clientName);
if (SwnVersion.SWN_VERSION_2 == expectedVersion)
{
ret = swnClientForWitness.WitnessrRegisterEx(SwnVersion.SWN_VERSION_2,
netName,
null,
currentAccessIp.ToString(),
clientName,
WitnessrRegisterExFlagsValue.WITNESS_REGISTER_IP_NOTIFICATION,
120,
out pContext);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(SwnErrorCode.ERROR_SUCCESS, (SwnErrorCode)ret, "WitnessrRegisterEx returns with result code = 0x{0:x8}", ret);
}
else
{
ret = swnClientForWitness.WitnessrRegister(SwnVersion.SWN_VERSION_1,
netName,
currentAccessIp.ToString(),
clientName,
out pContext);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(SwnErrorCode.ERROR_SUCCESS, (SwnErrorCode)ret, "WitnessrRegister returns with result code = 0x{0:x8}", ret);
}
BaseTestSite.Assert.IsNotNull(pContext, "Expect pContext is not null.");
callId = swnClientForWitness.WitnessrAsyncNotify(pContext);
BaseTestSite.Assert.AreNotEqual <uint>(0, callId, "WitnessrAsyncNotify returns callId = {0}", callId);
#endregion
#region Create a file and write content
string uncSharePath = Smb2Utility.GetUncPath(server, TestConfig.ClusteredFileShare);
string content = Smb2Utility.CreateRandomString(TestConfig.WriteBufferLengthInKb);
string testDirectory = CreateTestDirectory(uncSharePath);
string file = Path.Combine(testDirectory, Guid.NewGuid().ToString());
Guid clientGuid = Guid.NewGuid();
Guid createGuid = Guid.NewGuid();
FileServerType fsType = FileServerType.ScaleOutFileServer;
DoUntilSucceed(() => WriteContentBeforeFailover(fsType, server, currentAccessIp, uncSharePath, file, content, clientGuid, createGuid),
TestConfig.FailoverTimeout,
"Before failover, retry Write content until succeed within timeout span.");
#endregion
#region Move resource node
// Move resource node to trigger CLIENT_MOVE_NOTIFICATION
BaseTestSite.Log.Add(LogEntryKind.Debug, "Move resource to interface {0} to trigger CLIENT_MOVE_NOTIFICATION", registerInterface.InterfaceGroupName);
sutController.MoveSmbWitnessClient(clientName,
SWNTestUtility.GetPrincipleName(TestConfig.DomainName, registerInterface.InterfaceGroupName));
#endregion
#region Wait CLIENT_MOVE_NOTIFICATION
RESP_ASYNC_NOTIFY respNotify;
// Wait the CLIENT_MOVE_NOTIFICATION
ret = swnClientForWitness.ExpectWitnessrAsyncNotify(callId, out respNotify);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(SwnErrorCode.ERROR_SUCCESS, (SwnErrorCode)ret, "WitnessrAsyncNotify returns with result code = 0x{0:x8}", ret);
SWNTestUtility.PrintNotification(respNotify);
// Verify RESP_ASYNC_NOTIFY
SWNTestUtility.VerifyClientMoveShareMoveAndIpChange(respNotify, SwnMessageType.CLIENT_MOVE_NOTIFICATION, (uint)SwnIPAddrInfoFlags.IPADDR_V4, TestConfig.Platform);
#endregion
#region Get the new IpAddr
IPADDR_INFO_LIST ipAddrInfoList;
SwnUtility.ParseIPAddrInfoList(respNotify, out ipAddrInfoList);
currentAccessIp = (ipAddrInfoList.IPAddrList[0].Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(ipAddrInfoList.IPAddrList[0].IPV4) : SWNTestUtility.ConvertIPV6(ipAddrInfoList.IPAddrList[0].IPV6);
#endregion
#region Unregister SWN Witness
ret = swnClientForWitness.WitnessrUnRegister(pContext);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(SwnErrorCode.ERROR_SUCCESS, (SwnErrorCode)ret, "WitnessrUnRegister returns with result code = 0x{0:x8}", ret);
pContext = IntPtr.Zero;
swnClientForWitness.SwnUnbind(TestConfig.Timeout);
#endregion
#region Make sure Cluster Share is available
DoUntilSucceed(() => sutProtocolController.CheckIfShareIsAvailable(uncSharePath),
testConfig.Timeout,
"Make sure cluster share is available.");
#endregion
#region Read content and close the file
DoUntilSucceed(() => ReadContentAfterFailover(server, currentAccessIp, uncSharePath, file, content, clientGuid, createGuid),
TestConfig.FailoverTimeout,
"Retry Read content until succeed within timeout span.");
#endregion
}
private void SWNGetInterfaceList_SingleNode(string server, FileServerType fsType)
{
int ret;
WITNESS_INTERFACE_LIST interfaceList = new WITNESS_INTERFACE_LIST();
IPAddress currentAccessIpAddr;
WITNESS_INTERFACE_INFO registerInterface;
#region Test Sequence
currentAccessIpAddr = SWNTestUtility.GetCurrentAccessIP(server);
#region Get interface list to register.
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Get interface list to register.");
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
DoUntilSucceed(() =>
{
ret = swnClientForInterface.WitnessrGetInterfaceList(out interfaceList);
BaseTestSite.Assert.AreEqual <SwnErrorCode>(SwnErrorCode.ERROR_SUCCESS, (SwnErrorCode)ret, "WitnessrGetInterfaceList returns with result code = 0x{0:x8}", ret);
return(SWNTestUtility.VerifyInterfaceList(interfaceList, TestConfig.Platform));
}, TestConfig.FailoverTimeout, "Retry to call WitnessrGetInterfaceList until succeed within timeout span");
swnClientForInterface.SwnUnbind(TestConfig.Timeout);
SWNTestUtility.GetRegisterInterface(interfaceList, out registerInterface);
#endregion
#region Disable register interface
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Disable interface {0}.", registerInterface.InterfaceGroupName);
disabledNode = SWNTestUtility.GetPrincipleName(TestConfig.DomainName, registerInterface.InterfaceGroupName);
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Disable node {0} to trigger cluster failover.", disabledNode);
FailoverClusterNode(disabledNode);
// Wait to sync information between witness services
System.Threading.Thread.Sleep(TestConfig.swnWitnessSyncTimeout);
if (TestConfig.IsWindowsPlatform)
{
//Wait the file server back online
string newOwnerNode = null;
DoUntilSucceed(() =>
{
newOwnerNode = sutController.GetClusterResourceOwner(server);
return(!string.IsNullOrEmpty(newOwnerNode));
}, TestConfig.FailoverTimeout, "Retry to get cluster owner node until succeed within timeout span");
BaseTestSite.Assert.IsNotNull(newOwnerNode, "The new owner of cluster is {0}", newOwnerNode);
}
#endregion
#region Test that WitnessrGetInterfaceList is blocked
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Test that WitnessrGetInterfaceList is blocked.");
if (fsType == FileServerType.ScaleOutFileServer)
{
WITNESS_INTERFACE_INFO accessInterface = new WITNESS_INTERFACE_INFO();
SWNTestUtility.GetAccessInterface(interfaceList, out accessInterface);
currentAccessIpAddr =
(accessInterface.Flags & (uint)SwnNodeFlagsValue.IPv4) != 0 ? new IPAddress(accessInterface.IPV4) : SWNTestUtility.ConvertIPV6(accessInterface.IPV6);
server = accessInterface.InterfaceGroupName;
}
swnClientForInterface = new SwnClient();
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
try
{
ret = swnClientForInterface.WitnessrGetInterfaceList(out interfaceList);
SWNTestUtility.PrintInterfaceList(interfaceList);
BaseTestSite.Assert.Fail("Expect WitnessrGetInterfaceList throws TimeoutException because there is no available node.");
}
catch (TimeoutException)
{
BaseTestSite.Assert.Pass("Expect WitnessrGetInterfaceList throws TimeoutException because there is no available node.");
}
swnClientForInterface.SwnUnbind(TestConfig.Timeout);
#endregion
#region Enable register interface
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Restore disabled node {0}.", disabledNode);
RestoreClusterNodes(disabledNode);
disabledNode = null;
#endregion
#region Call WitnessrGetInterfaceList successfully
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Call WitnessrGetInterfaceList successfully.");
DoUntilSucceed(() => SWNTestUtility.BindServer(swnClientForInterface, currentAccessIpAddr,
TestConfig.DomainName, TestConfig.UserName, TestConfig.UserPassword, TestConfig.DefaultSecurityPackage,
TestConfig.DefaultRpceAuthenticationLevel, TestConfig.Timeout, server), TestConfig.FailoverTimeout,
"Retry BindServer until succeed within timeout span");
DoUntilSucceed(() =>
{
ret = swnClientForInterface.WitnessrGetInterfaceList(out interfaceList);
BaseTestSite.Assert.AreEqual <int>(0, ret, "WitnessrGetInterfaceList returns with result code = 0x{0:x8}", ret);
return(SWNTestUtility.VerifyInterfaceList(interfaceList, TestConfig.Platform));
}, TestConfig.FailoverTimeout, "Retry to call WitnessrGetInterfaceList until succeed within timeout span");
swnClientForInterface.SwnUnbind(TestConfig.Timeout);
#endregion
#endregion
}
/// <summary>
/// Reconnect to share on clustered file server after failover happens, include NEGOTIATE, SESSION_SETUP, TREE_CONNECT
/// </summary>
/// <param name="server">Name of clustered file server</param>
/// <param name="fsType">Type of clustered file server</param>
/// <param name="treeId">Out param for tree Id that connected</param>
/// <param name="enableEncryptionPerShare">Set true if enable encryption on share, otherwise set false</param>
private void ReconnectServerAfterFailover(string server, FileServerType fsType, out uint treeId, bool enableEncryptionPerShare = false)
{
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"AfterFailover: Connect share {0} on server {1} with following steps.", uncSharePath, server);
if (fsType == FileServerType.ScaleOutFileServer)
{
// For scale-out file server case, retrieve and use another access IP for connection
IPAddress oldAccessIp = currentAccessIp;
currentAccessIp = null;
foreach (var ip in accessIpList)
{
if (!ip.Equals(oldAccessIp))
{
currentAccessIp = ip;
break;
}
}
}
else if (fsType == FileServerType.GeneralFileServer)
{
currentAccessIp = null;
DoUntilSucceed(() =>
{
foreach (IPAddress ipAddress in accessIpList)
{
Smb2FunctionalClient pingClient = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
try
{
pingClient.ConnectToServerOverTCP(ipAddress);
pingClient.Disconnect();
pingClient = null;
currentAccessIp = ipAddress;
return(true);
}
catch
{
}
}
return(false);
}, TestConfig.FailoverTimeout, "Retry to ping to server until succeed within timeout span");
}
BaseTestSite.Assert.AreNotEqual(
null,
currentAccessIp,
"Access IP to the file server should not be empty");
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Got IP {0} to access the file server",
currentAccessIp.ToString());
DoUntilSucceed(() => clientAfterFailover.ConnectToServer(TestConfig.UnderlyingTransport, server, currentAccessIp), TestConfig.FailoverTimeout,
"Retry to connect to server until succeed within timeout span");
#region Negotiate
Capabilities_Values clientCapabilitiesAfterFailover = Capabilities_Values.GLOBAL_CAP_DFS | Capabilities_Values.GLOBAL_CAP_DIRECTORY_LEASING | Capabilities_Values.GLOBAL_CAP_LARGE_MTU | Capabilities_Values.GLOBAL_CAP_LEASING | Capabilities_Values.GLOBAL_CAP_MULTI_CHANNEL | Capabilities_Values.GLOBAL_CAP_PERSISTENT_HANDLES | Capabilities_Values.GLOBAL_CAP_ENCRYPTION;
status = clientAfterFailover.Negotiate(
TestConfig.RequestDialects,
TestConfig.IsSMB1NegotiateEnabled,
capabilityValue: clientCapabilitiesAfterFailover,
clientGuid: clientGuid,
checker: (Packet_Header header, NEGOTIATE_Response response) =>
{
TestConfig.CheckNegotiateDialect(DialectRevision.Smb30, response);
});
#endregion
#region SESSION_SETUP
status = clientAfterFailover.ReconnectSessionSetup(
clientBeforeFailover,
TestConfig.DefaultSecurityPackage,
server,
TestConfig.AccountCredential,
TestConfig.UseServerGssToken);
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Global encryption disabled");
#endregion
#region TREE_CONNECT to share
uint innerTreeId = 0;
//Retry TreeConnect until succeed within timeout span
status = DoUntilSucceed(
() => clientAfterFailover.TreeConnect(uncSharePath, out innerTreeId, (header, response) => { }),
TestConfig.FailoverTimeout,
"Retry TreeConnect until succeed within timeout span");
BaseTestSite.Assert.AreEqual(Smb2Status.STATUS_SUCCESS, status, "TreeConnect to {0} should succeed, actual status is {1}", uncSharePath, Smb2Status.GetStatusCode(status));
treeId = innerTreeId;
clientAfterFailover.SetTreeEncryption(innerTreeId, enableEncryptionPerShare);
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Per share encryption for TreeId=0x{0:x} : {1}", treeId, enableEncryptionPerShare);
#endregion
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"AfterFailover: Finish connecting to share {0} on server {1}", uncSharePath, server);
}
/// <summary>
/// File Server failover
/// For windows, simulated by disabling current access node of clustered file server
/// </summary>
/// <param name="server">Name of clustered file server</param>
/// <param name="fsType">Type of clustered file server</param>
protected void FailoverServer(IPAddress currentAccessIp, string server, FileServerType fsType)
{
isAccessNodeFailovered = true;
// Non Windows
if (!TestConfig.IsWindowsPlatform)
{
sutController.TriggerFailover(currentAccessIp.ToString());
return;
}
// Windows
#region Failover accessed node
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Failover node {0}", currentAccessNode);
FailoverClusterNode(currentAccessNode);
#endregion
#region Wait for available server
if (fsType == FileServerType.GeneralFileServer)
{
string newOwnerNode = null;
//Wait the file server back online
DoUntilSucceed(() =>
{
newOwnerNode = sutController.GetClusterResourceOwner(server);
return (!string.IsNullOrEmpty(newOwnerNode) && newOwnerNode.ToUpper() != currentAccessNode.ToUpper());
}, TestConfig.FailoverTimeout, "Retry to get cluster owner node until succeed within timeout span.");
BaseTestSite.Assert.AreNotEqual(
null,
newOwnerNode,
"New owner node should not be null.");
BaseTestSite.Assert.AreNotEqual(
currentAccessNode.ToUpper(),
newOwnerNode.ToUpper(),
"New owner node should not be the same as the old one.");
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Current owner node changed to {0}", newOwnerNode);
}
#endregion
}
/// <summary>
/// Write content before failover
/// </summary>
/// <param name="fsType">FileServerType</param>
/// <param name="server">File Server name.</param>
/// <param name="serverAccessIp">File Server Access IP.</param>
/// <param name="uncSharePath">The share path to write the file.</param>
/// <param name="file">The file name for writing content.</param>
/// <param name="content">The content to write.</param>
/// <param name="clientGuid">Smb2 client Guid.</param>
/// <param name="createGuid">The Guid for smb2 create request.</param>
/// <returns></returns>
protected bool WriteContentBeforeFailover(
FileServerType fsType,
string server,
IPAddress serverAccessIp,
string uncSharePath,
string file,
string content,
Guid clientGuid,
Guid createGuid)
{
uint status = 0;
beforeFailover = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Start a client by sending the following requests: NEGOTIATE; SESSION_SETUP; TREE_CONNECT to {0}", uncSharePath);
beforeFailover.ConnectToServer(TestConfig.UnderlyingTransport, server, serverAccessIp);
Capabilities_Values requestCapabilities = Capabilities_Values.GLOBAL_CAP_DFS | Capabilities_Values.GLOBAL_CAP_DIRECTORY_LEASING | Capabilities_Values.GLOBAL_CAP_LARGE_MTU | Capabilities_Values.GLOBAL_CAP_LEASING | Capabilities_Values.GLOBAL_CAP_MULTI_CHANNEL | Capabilities_Values.GLOBAL_CAP_PERSISTENT_HANDLES;
status = beforeFailover.Negotiate(
TestConfig.RequestDialects,
TestConfig.IsSMB1NegotiateEnabled,
capabilityValue: requestCapabilities,
clientGuid: clientGuid,
checker: (header, response) =>
{
TestConfig.CheckNegotiateDialect(DialectRevision.Smb30, response);
});
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Negotiate failed with {0}.", Smb2Status.GetStatusCode(status));
return false;
}
status = beforeFailover.SessionSetup(
TestConfig.DefaultSecurityPackage,
server,
TestConfig.AccountCredential,
TestConfig.UseServerGssToken);
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "SessionSetup failed with {0}.", Smb2Status.GetStatusCode(status));
return false;
}
uint treeId = 0;
Share_Capabilities_Values shareCapabilities = Share_Capabilities_Values.NONE;
status = DoUntilSucceed(
() => beforeFailover.TreeConnect(uncSharePath, out treeId, (header, response) =>
{
shareCapabilities = response.Capabilities;
}),
TestConfig.FailoverTimeout,
"Retry TreeConnect until succeed within timeout span");
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "TreeConnect failed with {0}.", Smb2Status.GetStatusCode(status));
return false;
}
BaseTestSite.Assert.IsTrue(shareCapabilities.HasFlag(Share_Capabilities_Values.SHARE_CAP_CONTINUOUS_AVAILABILITY),
"CA Share should have SHARE_CAP_CONTINUOUS_AVAILABILITY bit set for Capabilities in TreeConnect response.");
if (fsType == FileServerType.ScaleOutFileServer)
{
BaseTestSite.Assert.IsTrue(shareCapabilities.HasFlag(Share_Capabilities_Values.SHARE_CAP_SCALEOUT),
"ScaleOut FS should have SHARE_CAP_SCALEOUT bit set for Capabilities in TreeConnect response.");
}
FILEID fileId;
Smb2CreateContextResponse[] serverCreateContexts;
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Client sends CREATE request with SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 with PERSISTENT flag set.");
status = beforeFailover.Create(
treeId,
file,
CreateOptions_Values.FILE_NON_DIRECTORY_FILE,
out fileId,
out serverCreateContexts,
RequestedOplockLevel_Values.OPLOCK_LEVEL_NONE,
new Smb2CreateContextRequest[] {
new Smb2CreateDurableHandleRequestV2
{
CreateGuid = createGuid,
Flags = CREATE_DURABLE_HANDLE_REQUEST_V2_Flags.DHANDLE_FLAG_PERSISTENT,
Timeout = 3600000,
},
});
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Create failed with {0}.", Smb2Status.GetStatusCode(status));
return false;
}
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Client sends WRITE request to write content to the file.");
status = beforeFailover.Write(treeId, fileId, content);
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Write content failed with {0}.", Smb2Status.GetStatusCode(status));
return false;
}
BaseTestSite.Log.Add(LogEntryKind.TestStep, "Client sends FLUSH request.");
status = beforeFailover.Flush(treeId, fileId);
if (status != Smb2Status.STATUS_SUCCESS)
{
BaseTestSite.Log.Add(LogEntryKind.Warning, "Flush failed with {0}.", Smb2Status.GetStatusCode(status));
return false;
}
return true;
}
public void ReqGetVersion(LighthouseConfig.Server serverData, string appVersion, string lighthouseId, string channel, FileServerType fromTo, Action <GetVersionResponseInfo> getVersionResponseInfoAction)
{
Clear();
UrlList = new string[serverData.FallbackUrlList.Count + 1];
UrlList[0] = serverData.Url;
serverData.FallbackUrlList.ForCall((x, index) => { UrlList[index + 1] = x; });
s_mLogger.Debug($"[appVersion : {appVersion}] [lighthouseId : {lighthouseId}] [channel : {channel}] [fromTo : {fromTo}] .");
mAppVersion = appVersion;
mChannel = channel;
mGetVersionInfoEvent = getVersionResponseInfoAction;
mCurRequest = new EntryPointRequest();
mCurRequest.AppVersion = appVersion;
mCurRequest.LighthouseId = lighthouseId;
mCurRequest.From = (fromTo == FileServerType.CDN) ? EntryPointFromType.Cdn : EntryPointFromType.Oss;
mInnerState = InnerState.StartRequest;
}
/// <summary>
/// Reconnect to share on clustered file server after failover happens, include NEGOTIATE, SESSION_SETUP, TREE_CONNECT
/// </summary>
/// <param name="server">Name of clustered file server</param>
/// <param name="fsType">Type of clustered file server</param>
/// <param name="treeId">Out param for tree Id that connected</param>
/// <param name="enableEncryptionPerShare">Set true if enable encryption on share, otherwise set false</param>
private void ReconnectServerAfterFailover(string server, FileServerType fsType, out uint treeId, bool enableEncryptionPerShare = false)
{
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"AfterFailover: Connect share {0} on server {1} with following steps.", uncSharePath, server);
if (fsType == FileServerType.ScaleOutFileServer)
{
// For scale-out file server case, retrieve and use another access IP for connection
IPAddress oldAccessIp = currentAccessIp;
currentAccessIp = null;
foreach (var ip in accessIpList)
{
if (!ip.Equals(oldAccessIp))
{
currentAccessIp = ip;
break;
}
}
}
else if (fsType == FileServerType.GeneralFileServer)
{
currentAccessIp = null;
DoUntilSucceed(() =>
{
foreach (IPAddress ipAddress in accessIpList)
{
Smb2FunctionalClient pingClient = new Smb2FunctionalClient(TestConfig.FailoverTimeout, TestConfig, BaseTestSite);
try
{
pingClient.ConnectToServerOverTCP(ipAddress);
pingClient.Disconnect();
pingClient = null;
currentAccessIp = ipAddress;
return true;
}
catch
{
}
}
return false;
}, TestConfig.FailoverTimeout, "Retry to ping to server until succeed within timeout span");
}
BaseTestSite.Assert.AreNotEqual(
null,
currentAccessIp,
"Access IP to the file server should not be empty");
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Got IP {0} to access the file server",
currentAccessIp.ToString());
DoUntilSucceed(() => clientAfterFailover.ConnectToServer(TestConfig.UnderlyingTransport, server, currentAccessIp), TestConfig.FailoverTimeout,
"Retry to connect to server until succeed within timeout span");
#region Negotiate
Capabilities_Values clientCapabilitiesAfterFailover = Capabilities_Values.GLOBAL_CAP_DFS | Capabilities_Values.GLOBAL_CAP_DIRECTORY_LEASING | Capabilities_Values.GLOBAL_CAP_LARGE_MTU | Capabilities_Values.GLOBAL_CAP_LEASING | Capabilities_Values.GLOBAL_CAP_MULTI_CHANNEL | Capabilities_Values.GLOBAL_CAP_PERSISTENT_HANDLES | Capabilities_Values.GLOBAL_CAP_ENCRYPTION;
status = clientAfterFailover.Negotiate(
TestConfig.RequestDialects,
TestConfig.IsSMB1NegotiateEnabled,
capabilityValue: clientCapabilitiesAfterFailover,
clientGuid: clientGuid,
checker: (Packet_Header header, NEGOTIATE_Response response) =>
{
TestConfig.CheckNegotiateDialect(DialectRevision.Smb30, response);
});
#endregion
#region SESSION_SETUP
status = clientAfterFailover.ReconnectSessionSetup(
clientBeforeFailover,
TestConfig.DefaultSecurityPackage,
server,
TestConfig.AccountCredential,
TestConfig.UseServerGssToken);
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Global encryption disabled");
#endregion
#region TREE_CONNECT to share
uint innerTreeId = 0;
//Retry TreeConnect until succeed within timeout span
status = DoUntilSucceed(
() => clientAfterFailover.TreeConnect(uncSharePath, out innerTreeId, (header, response) => { }),
TestConfig.FailoverTimeout,
"Retry TreeConnect until succeed within timeout span");
BaseTestSite.Assert.AreEqual(Smb2Status.STATUS_SUCCESS, status, "TreeConnect to {0} should succeed, actual status is {1}", uncSharePath, Smb2Status.GetStatusCode(status));
treeId = innerTreeId;
clientAfterFailover.SetTreeEncryption(innerTreeId, enableEncryptionPerShare);
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"Per share encryption for TreeId=0x{0:x} : {1}", treeId, enableEncryptionPerShare);
#endregion
BaseTestSite.Log.Add(
LogEntryKind.Debug,
"AfterFailover: Finish connecting to share {0} on server {1}", uncSharePath, server);
}
