/// <summary>
/// This method executes the test with the specified parameter set passed into the constructor.
/// </summary>
/// <param name="token">the Cancellation token for the async operation.</param>
/// <exception cref= "System.Fabric.FabricValidationException" >If any service does not stabilize within the specified timeout.</exception>
/// <returns>Returns a Task which represents the asynchronous operation.</returns>
public async Task ExecuteAsync(CancellationToken token)
{
DateTime testStartTime = DateTime.UtcNow;
try
{
this.stopwatch = new Stopwatch();
this.stopwatch.Start();
this.TestContext = this.FabricClient.FaultManager.TestContext;
// Call derived class method OnExecuteAsync()
await this.OnExecuteAsync(token);
this.ReportProgress("Scenario complete");
await this.ValidateScenarioAtExitAsync(token);
Log.WriteInfo(TraceType, "ExecuteAsync(): Scenario completed.");
}
catch (OperationCanceledException)
{
Log.WriteWarning(TraceType, "ExecuteAsync(): Cancellation was requested while running the scenario.");
throw;
}
catch (Exception e)
{
Log.WriteError(TraceType, "ExecuteAsync(): Exception was thrown while running the scenario: {0}", e);
throw;
}
}
public void WriteInfo(TraceType traceType, string format, params object[] args)
{
if (!string.IsNullOrEmpty(this.TraceId))
{
TraceSource.WriteInfoWithId(traceType.Name, this.TraceId, format, args);
}
else
{
TraceSource.WriteInfo(traceType.Name, format, args);
}
}
internal static ITelemetryWriter CreateTelemetryWriter(
ConfigReader configReader,
string clusterId,
string nodeName,
TelemetryCollection.LogErrorDelegate logErrorDelegate)
{
#if !DotNetCoreClrLinux
bool createFileShareWriter = false;
#endif
FabricEvents.ExtensionsEvents traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
traceSource.WriteInfo("TelemetryWriterFactory", "Entering TelemetryWriterFactory");
#if !DotNetCoreClrLinux
string fabricLogsShare;
bool isFabricLogsLocalFolder;
if (FileShareCommon.GetDestinationPath(
traceSource,
string.Concat(ConfigReader.FileShareWinFabSectionName),
configReader,
ConfigReader.FileShareWinFabSectionName,
out isFabricLogsLocalFolder,
out fabricLogsShare))
{
traceSource.WriteInfo("TelemetryWriterFactory", "isFabricLogsLocalFolder: {0}, fabricLogsShare: {1}", isFabricLogsLocalFolder, fabricLogsShare);
string customizedGoalStateFileUrl = configReader.GetUnencryptedConfigValue(
ConfigReader.UpgradeOrchestrationServiceConfigSectionName,
ConfigReader.GoalStateFileUrlParamName,
string.Empty);
if (customizedGoalStateFileUrl != string.Empty)
{
createFileShareWriter = true;
}
}
if (createFileShareWriter)
{
string fileShareDirectory = Path.Combine(Path.GetDirectoryName(fabricLogsShare), "fabrictelemetries-" + clusterId, nodeName);
traceSource.WriteInfo("TelemetryWriterFactory", "return fileShare {0}", fileShareDirectory);
return(new FileShareTelemetryWriter(fileShareDirectory, logErrorDelegate));
}
else
#endif
{
traceSource.WriteInfo("TelemetryWriterFactory", "return appinsight");
return(new AppInsightsTelemetryWriter());
}
}
private dSTSClientImpl(
string metadataEndpoint,
string[] serverCommonNames,
string[] serverThumbprints,
bool interactive,
string cloudServiceName,
string[] cloudServiceDnsNames)
{
this.metadataUrl = new Uri("https://" + metadataEndpoint.TrimEnd('/') + metadataSuffix + metadataVersion);
this.serverCommonNames = serverCommonNames;
this.serverThumbprints = serverThumbprints;
this.interactive = interactive;
this.cloudServiceName = cloudServiceName;
this.cloudServiceDnsNames = cloudServiceDnsNames;
TraceSource.WriteInfo(
TraceType,
"Creating DSTSClient with metadataurl: {0}, ServerCommonNames: {1}, ServerThumbprints: {2}, CloudServiceName: {3}, CloudServiceDNSNames: {4}, Interactive: {5}",
metadataUrl,
serverCommonNames != null ? string.Join(",", serverCommonNames) : "",
serverThumbprints != null ? string.Join(",", serverThumbprints) : "",
cloudServiceName != null ? cloudServiceName : "",
cloudServiceDnsNames != null? string.Join(",", cloudServiceDnsNames) : "",
interactive.ToString());
}
private FabricTestAction GetStartNodeAction(NodeStateTransitionAction ragAction)
{
ChaosUtility.ThrowOrAssertIfTrue(
ChaosConstants.StartNode_InvalidActionType_TelemetryId,
ragAction == null || ragAction.ActionType != StateTransitionActionType.NodeUp,
StringResources.ChaosEngineError_StartNode_InvalidActionType);
ChaosUtility.ThrowOrAssertIfTrue(
ChaosConstants.StartNode_InvalidNodeName_TelemetryId,
string.IsNullOrEmpty(ragAction.NodeName),
StringResources.ChaosEngineError_StartNode_InvalidNodeName);
string report = StringHelper.Format("Generating Random Action: {0}\n\t\tNodeName: {1}\n\t\tNodeInstanceId: {2}", ragAction.ActionType, ragAction.NodeName, ragAction.NodeInstanceId);
Log.WriteInfo(TraceType, report);
if (this.reportFunction != null)
{
this.reportFunction(report);
}
string nodeName = ragAction.NodeName;
BigInteger nodeInstanceId = ragAction.NodeInstanceId;
StartNodeAction startNodeAction = new StartNodeAction(nodeName, nodeInstanceId);
return(startNodeAction);
}
public IStatelessServiceInstance CreateInstance(string serviceTypeName, Uri serviceName, byte[] initializationData, Guid partitionId, long instanceId)
{
string configSection = Encoding.Unicode.GetString(initializationData);
TraceSource.WriteInfo(TraceType,
"Creating TokenValidationService instanceId {0}, partition id {1} and config section {2}",
instanceId,
partitionId,
configSection);
return(new TokenValidationService(
this.serviceAgent,
TokenValidationProviderFactory.Create(configSection)));
}
public bool ReadUnencryptedBool(string sectionName, string keyName, FabricEvents.ExtensionsEvents traceSource, string traceType, bool throwIfInvalid)
{
NativeConfigStore configStore = NativeConfigStore.FabricGetConfigStore();
string propertyValueStr = configStore.ReadUnencryptedString(sectionName, keyName);
bool propertyValue = false;
traceSource.WriteInfo(traceType, "{0} {1}", keyName, propertyValueStr);
if (!bool.TryParse(propertyValueStr, out propertyValue))
{
string message = string.Format("{0} could not parse: {1}", keyName, propertyValueStr);
traceSource.WriteError(traceType, message);
if (throwIfInvalid)
{
throw new InvalidDataException(message);
}
else
{
propertyValue = false;
}
}
return(propertyValue);
}
public static void WriteInfo(string format, params object[] args)
{
traceSource.WriteInfo(DeployerTrace.TraceType, format, args);
}
public static void WriteInfo(TraceType traceType, string format, params object[] args)
{
traceSource.WriteInfo(traceType.Name, format, args);
}
#pragma warning restore 1998
#endif
private static void WriteInfo(string format, params object[] args)
{
TraceSource.WriteInfo(TraceType, format, args);
}
public static void ReserveUrl(string networkUrl, string securityDescriptor)
{
TraceSource.WriteInfo(
PortAclUtility.TraceType,
"Started to Reserve Url. networkUrl: {0}, securityDescriptor: {1}.",
networkUrl,
securityDescriptor);
uint retVal = (uint)NOERROR; // NOERROR = 0
HTTPAPI_VERSION httpApiVersion = new HTTPAPI_VERSION(1, 0);
retVal = HttpInitialize(httpApiVersion, HTTP_INITIALIZE_CONFIG, IntPtr.Zero);
TraceSource.WriteInfo(
PortAclUtility.TraceType,
"HttpInitialize completed with return value: {0}.",
retVal);
if ((uint)NOERROR == retVal)
{
HTTP_SERVICE_CONFIG_URLACL_KEY keyDesc = new HTTP_SERVICE_CONFIG_URLACL_KEY(networkUrl);
HTTP_SERVICE_CONFIG_URLACL_PARAM paramDesc = new HTTP_SERVICE_CONFIG_URLACL_PARAM(securityDescriptor);
HTTP_SERVICE_CONFIG_URLACL_SET inputConfigInfoSet = new HTTP_SERVICE_CONFIG_URLACL_SET
{
KeyDesc = keyDesc,
ParamDesc = paramDesc
};
IntPtr pInputConfigInfo = Marshal.AllocCoTaskMem(Marshal.SizeOf(typeof(HTTP_SERVICE_CONFIG_URLACL_SET)));
Marshal.StructureToPtr(inputConfigInfoSet, pInputConfigInfo, false);
retVal = HttpSetServiceConfiguration(IntPtr.Zero,
HTTP_SERVICE_CONFIG_ID.HttpServiceConfigUrlAclInfo,
pInputConfigInfo,
Marshal.SizeOf(inputConfigInfoSet),
IntPtr.Zero);
TraceSource.WriteInfo(
PortAclUtility.TraceType,
"HttpSetServiceConfiguration completed with return value: {0}.",
retVal);
if ((uint)ERROR_ALREADY_EXISTS == retVal) // ERROR_ALREADY_EXISTS = 183
{
retVal = HttpDeleteServiceConfiguration(IntPtr.Zero,
HTTP_SERVICE_CONFIG_ID.HttpServiceConfigUrlAclInfo,
pInputConfigInfo,
Marshal.SizeOf(inputConfigInfoSet),
IntPtr.Zero);
TraceSource.WriteInfo(
PortAclUtility.TraceType,
"HttpDeleteServiceConfiguration(outer) completed with return value: {0}.",
retVal);
if ((uint)ERROR_FILE_NOT_FOUND == retVal)
{
// This means its possible that the URL is ACLed for the same port but different protocol (http/https)
// Try deleting ACL using the other protocol
var networkUrlInner = networkUrl.ToLower();
if (networkUrlInner.Contains("https"))
{
networkUrlInner = networkUrlInner.Replace("https", "http");
}
else
{
networkUrlInner = networkUrlInner.Replace("http", "https");
}
HTTP_SERVICE_CONFIG_URLACL_KEY keyDescInner = new HTTP_SERVICE_CONFIG_URLACL_KEY(networkUrlInner);
HTTP_SERVICE_CONFIG_URLACL_SET inputConfigInfoSetInner = new HTTP_SERVICE_CONFIG_URLACL_SET
{
KeyDesc = keyDescInner
};
IntPtr pInputConfigInfoInner = IntPtr.Zero;
try
{
pInputConfigInfoInner = Marshal.AllocCoTaskMem(Marshal.SizeOf(typeof(HTTP_SERVICE_CONFIG_URLACL_SET)));
Marshal.StructureToPtr(inputConfigInfoSetInner, pInputConfigInfoInner, false);
retVal = HttpDeleteServiceConfiguration(IntPtr.Zero,
HTTP_SERVICE_CONFIG_ID.HttpServiceConfigUrlAclInfo,
pInputConfigInfoInner,
Marshal.SizeOf(inputConfigInfoSetInner),
IntPtr.Zero);
TraceSource.WriteWarning(
PortAclUtility.TraceType,
"HttpDeleteServiceConfiguration(Inner) completed with return value: {0}.",
retVal);
}
finally
{
if (IntPtr.Zero != pInputConfigInfoInner)
{
Marshal.FreeCoTaskMem(pInputConfigInfoInner);
}
}
}
if ((uint)NOERROR == retVal)
{
retVal = HttpSetServiceConfiguration(IntPtr.Zero,
HTTP_SERVICE_CONFIG_ID.HttpServiceConfigUrlAclInfo,
pInputConfigInfo,
Marshal.SizeOf(inputConfigInfoSet),
IntPtr.Zero);
TraceSource.WriteInfo(
PortAclUtility.TraceType,
"HttpSetServiceConfiguration completed with return value: {0}.",
retVal);
}
}
Marshal.FreeCoTaskMem(pInputConfigInfo);
HttpTerminate(HTTP_INITIALIZE_CONFIG, IntPtr.Zero);
}
if ((uint)NOERROR != retVal)
{
TraceSource.WriteError(
PortAclUtility.TraceType,
"ReserveUrl failed with error: {0}.",
retVal);
throw new Win32Exception(Convert.ToInt32(retVal));
}
}
public static int Main(string[] args)
{
FabricEvents.ExtensionsEvents traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.BackupRestoreService);
int status = 0;
var codePackageActivationContext = FabricRuntime.GetActivationContext();
var configStore = NativeConfigStore.FabricGetConfigStore();
var serverAuthCredentialTypeName = configStore.ReadUnencryptedString(BackupRestoreContants.SecurityConfigSectionName, BackupRestoreContants.ServerAuthCredentialTypeName);
var serverAuthCredentialType = CredentialType.None;
EndpointResourceDescription endpoint = null;
try
{
endpoint = codePackageActivationContext.GetEndpoint(BackupRestoreContants.RestEndPointName);
}
catch (FabricElementNotFoundException)
{
traceSource.WriteWarning(TraceType, "Endpoint not found for EndpointName{0}", BackupRestoreContants.RestEndPointName);
throw;
}
if (!string.IsNullOrEmpty(serverAuthCredentialTypeName) && !Enum.TryParse(serverAuthCredentialTypeName, out serverAuthCredentialType))
{
traceSource.WriteWarning(TraceType, "Unable to convert configuration value serverAuthCredentialTypeName {0} for {1} type.", serverAuthCredentialTypeName, serverAuthCredentialType);
}
if (serverAuthCredentialType == CredentialType.X509 || serverAuthCredentialType == CredentialType.Claims)
{
// Get server auth details
var aclServerAuthX509StoreName = configStore.ReadUnencryptedString(BackupRestoreContants.FabricNodeConfigSectionName, BackupRestoreContants.ServerAuthX509StoreName);
var aclCertValueFindTypeName = configStore.ReadUnencryptedString(BackupRestoreContants.FabricNodeConfigSectionName, BackupRestoreContants.ServerAuthX509FindType);
var aclCertValue = configStore.ReadUnencryptedString(BackupRestoreContants.FabricNodeConfigSectionName, BackupRestoreContants.ServerAuthX509FindValue);
var aclCertValueSecondary = configStore.ReadUnencryptedString(BackupRestoreContants.FabricNodeConfigSectionName, BackupRestoreContants.ServerAuthX509FindValueSecondary);
var storeName = StoreName.My;
var x509FindType = X509FindType.FindByThumbprint;
X509Certificate2 certificate = null;
if (!string.IsNullOrEmpty(aclServerAuthX509StoreName) && !Enum.TryParse(aclServerAuthX509StoreName, out storeName))
{
traceSource.WriteWarning(TraceType, "Unable to Convert configuration aclServerAuthX509StoreName value {0} for {1} type.", aclServerAuthX509StoreName, storeName.GetType().ToString());
throw new InvalidEnumArgumentException(string.Format("Unable to Convert configuration aclCertValueFindTypeName value {0} for {1} type.", aclServerAuthX509StoreName, x509FindType.GetType()));
}
if (!string.IsNullOrEmpty(aclCertValueFindTypeName) && !Enum.TryParse(aclCertValueFindTypeName, out x509FindType))
{
traceSource.WriteWarning(TraceType, "Unable to Convert configuration aclCertValueFindTypeName value {0} for {1} type.", aclServerAuthX509StoreName, x509FindType.GetType().ToString());
throw new InvalidEnumArgumentException(string.Format("Unable to Convert configuration aclCertValueFindTypeName value {0} for {1} type.", aclServerAuthX509StoreName, x509FindType.GetType()));
}
if (!string.IsNullOrWhiteSpace(aclCertValue))
{
// Get certificate.
var store = new X509Store(storeName, StoreLocation.LocalMachine);
try
{
store.Open(OpenFlags.ReadOnly);
traceSource.WriteInfo(
TraceType,
"Finding applicable certificate for Acling. StoreName: {0}, StoreLocation: LocalMachine, X509FindType: {1}, X509FindValue: {2}.",
storeName,
x509FindType,
aclCertValue);
var certCollections = store.Certificates.Find(x509FindType, aclCertValue, false /*load self-signed cert*/);
if (certCollections.Count > 0)
{
certificate = certCollections[0];
}
else if (!string.IsNullOrWhiteSpace(aclCertValueSecondary))
{
traceSource.WriteInfo(
TraceType,
"Finding applicable certificate for Acling using Secondary cert config. StoreName: {0}, StoreLocation: LocalMachine, X509FindType: {1}, X509FindValue: {2}.",
storeName,
x509FindType,
aclCertValueSecondary);
var certCollectionsSecondary = store.Certificates.Find(x509FindType, aclCertValueSecondary, false /*load self-signed cert*/);
if (certCollectionsSecondary.Count > 0)
{
certificate = certCollectionsSecondary[0];
}
}
else
{
traceSource.WriteWarning(TraceType, "No matching certificate found. Thumbprint value: {0}, StoreName: {1}, StoreLocation: {2}", aclCertValue, aclServerAuthX509StoreName, StoreLocation.LocalMachine);
}
}
finally
{
store.Close();
}
}
else
{
traceSource.WriteWarning(TraceType, "Invalid configuration for certificate. Thumbprint value: {0}, StoreName: {1}, StoreLocation: {2}", aclCertValue, aclServerAuthX509StoreName, StoreLocation.LocalMachine);
}
if (certificate != null)
{
PortAclUtility.BindCertificate(endpoint.IpAddressOrFqdn, endpoint.Port, certificate.GetCertHash());
}
}
// Do URL ACLing
CodePackage codePackage = codePackageActivationContext.GetCodePackageObject("Code");
string daclString = "D:(A;;GX;;;NS)";
try
{
var runAsAccountName = configStore.ReadUnencryptedString(BackupRestoreContants.FabricNodeRunAsSectionName, BackupRestoreContants.RunAsAccountNameConfig);
if (!string.IsNullOrEmpty(runAsAccountName))
{
traceSource.WriteInfo(TraceType, "runAsAccountName for ACLing: {0} for CredentialType Windows", runAsAccountName);
daclString = GetAllowDaclFromUserName(runAsAccountName);
}
}
catch (IdentityNotMappedException ex)
{
traceSource.WriteWarning(TraceType, "Failed to resolve NTAccount: {0}, Exception: {1}", codePackage.EntryPointRunAsPolicy.UserName, ex);
throw;
}
catch (SystemException ex)
{
traceSource.WriteWarning(TraceType, "Failed to resolve NTAccount: {0}, Exception: {1}", codePackage.EntryPointRunAsPolicy.UserName, ex);
throw;
}
var fabricBrsSecuritySetting = SecuritySetting.GetClusterSecurityDetails();
var listeningAddress = String.Format(CultureInfo.InvariantCulture, "{0}://+:{1}/", fabricBrsSecuritySetting.EndpointProtocol, endpoint.Port);
traceSource.WriteInfo(TraceType, "ListeningAddress: {0} ,DaclString: {1} for ACLing", listeningAddress, daclString);
PortAclUtility.ReserveUrl(listeningAddress, daclString);
return(status);
}
/// <summary>
/// Worker thread to perform the file operations
/// </summary>
private void ThreadRunImpl()
{
// Initialize xstore blobcontainer and connection in the thread
// This is to prevent we block the dispatcher
// Connect to azure blob storage
string connectionString = this.storeParams.ConnectionString;
var storageAccount = CloudStorageAccount.Parse(connectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
blobClient.DefaultRequestOptions = new BlobRequestOptions();
blobClient.DefaultRequestOptions.MaximumExecutionTime = new TimeSpan(0, 5, 0);
// For large file copies set up a custom timeout period; and using parallel settings
blobClient.DefaultRequestOptions.ParallelOperationThreadCount = XStoreCommon.ParallelOperationThreadCount;
// Get and create the container
CloudBlobContainer blobContainer = blobClient.GetContainerReference(this.storeParams.Container);
while (WaitHandle.WaitAny(this.syncEvents.EventArray) != 1)
{
XStoreFileOperationTask task = this.xstoreTaskPool.GetTaskFromTaskQueue();
if (null != task)
{
// We have got valid task
if (task.RetryCount > 0)
{
// Sleep random time to prevent all worker threads do the next try at the same time
Random rand = new Random(DateTime.Now.Millisecond + this.workerID);
int timeoutInSeconds = rand.Next(MinRetryInSeconds, MaxRetryInSeconds);
traceSource.WriteError(
ClassName,
string.Format(CultureInfo.InvariantCulture, "[RETRY] sleep {0} seconds to work on {1}", timeoutInSeconds, task.SrcUri));
Thread.Sleep(timeoutInSeconds * 1000);
}
if (task.IsFolder &&
task.OpType != XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB &&
task.OpType != XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore &&
task.OpType != XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore)
{
try
{
this.HandleFolder(blobContainer, task);
}
catch (Exception e)
{
traceSource.WriteError(
ClassName,
string.Format(CultureInfo.InvariantCulture, "[{0}] [{1}] EXCEPTION = {2}", Thread.CurrentThread.ManagedThreadId, DateTime.Now, e.Message));
if (e is TimeoutException)
{
throw;
}
if (e is StorageException && e.InnerException is TimeoutException)
{
throw new TimeoutException(e.InnerException.Message);
}
if (ExceptionHandler.IsFatalException(e))
{
throw;
}
this.RetryTask(task);
}
finally
{
this.xstoreTaskPool.DecrementTaskCount();
}
}
else
{
try
{
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
this.TransferFileFromXStoreToSMB(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
this.TransferFileFromSMBToXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
this.TransferFileFromXStoreToXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.RemoveFromXStore:
DeleteFileFromXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore:
EndCopyFromSMBToXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB:
EndCopyFromXStoreToSMB(task);
break;
case XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore:
EndCopyFromXStoreToXStore(blobContainer, task);
break;
default:
break;
}
}
catch (Exception e)
{
traceSource.WriteError(
ClassName,
string.Format(CultureInfo.InvariantCulture, "[{0}] [{1}] EXCEPTION = {2}", Thread.CurrentThread.ManagedThreadId, DateTime.Now, e.Message));
string extraTracing = task.GetExtraTracing();
if (!string.IsNullOrEmpty(extraTracing))
{
traceSource.WriteInfo(ClassName, extraTracing);
}
if (e is TimeoutException)
{
throw;
}
if (e is StorageException && e.InnerException is TimeoutException)
{
throw new TimeoutException(e.InnerException.Message);
}
if (ExceptionHandler.IsFatalException(e))
{
throw;
}
this.RetryTask(task);
}
finally
{
this.xstoreTaskPool.DecrementTaskCount();
}
}
}
}
}
