void RoleEnvironment_Changed(object sender, RoleEnvironmentChangedEventArgs e) { WorkerTrace.TraceEvent(TraceEventType.Information, 18, "Role configuration has been changed. Requesting instance recycle"); if (current != null) { current.Abandon(); } Thread.Sleep(25); //waiting for logs transfer RoleEnvironment.RequestRecycle(); }
void RoleEnvironment_Stopping(object sender, RoleEnvironmentStoppingEventArgs e) { WorkerTrace.TraceEvent(TraceEventType.Information, 16, "Role is stopping"); stopRequested.Set(); // Wait until processing stops if (current != null) { current.Abandon(); } WorkerTrace.TraceEvent(TraceEventType.Information, 17, "Stopping FetchWorker"); Thread.Sleep(25); //waiting for logs transfer }
public void Initialize() { if (!DataSetFactory.ContainsProvider("memory")) { DataSetFactory.Register(typeof(MemoryDataSet)); } if (!DataSetFactory.ContainsProvider("ab")) { DataSetFactory.Register(typeof(AzureBlobDataSet)); } string customTempLocalResourcePath = RoleEnvironment.GetLocalResource("localStorage1").RootPath; Environment.SetEnvironmentVariable("TMP", customTempLocalResourcePath); Environment.SetEnvironmentVariable("TEMP", customTempLocalResourcePath); string jobsDatabaseConnectionString = RoleEnvironment.GetConfigurationSettingValue("FetchClimate.JobsDatabaseConnectionString"); string jobsStorageConnectionString = RoleEnvironment.GetConfigurationSettingValue("FetchClimate.JobsStorageConnectionString"); foreach (TraceListener item in Trace.Listeners) { if (!(item is DefaultTraceListener)) // The default trace listener is always in any TraceSource.Listeners collection. { AutoRegistratingTraceSource.RegisterTraceListener(item); WorkerTrace.TraceEvent(TraceEventType.Information, 19, string.Format("TraceListener \"{0}\" registered for accepting data from all AutoRegistratingTraceSources", item.ToString())); } } JobManager.InitializeJobTable(jobsDatabaseConnectionString, false); configProvider = new SqlExtendedConfigurationProvider( RoleEnvironment.GetConfigurationSettingValue("ConfigurationDatabaseConnectionString")); WorkerTrace.TraceEvent(TraceEventType.Information, 6, string.Format("Connected to configuration database. Latest timestamp {0}", configProvider.GetConfiguration(DateTime.MaxValue).TimeStamp)); azureGAC = new AssemblyStore(RoleEnvironment.GetConfigurationSettingValue("FetchWorker.AssemblyStoreConnectionString")); //overriding bug-containing default azure provider with the fixed one Type t = typeof(DataSetFactory); var dict = (System.Collections.IDictionary)t.InvokeMember("providersByName", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.GetField, null, null, null); dict.Remove("az"); DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.Azure.AzureDataSet)); WorkerTrace.TraceEvent(TraceEventType.Verbose, 9, "Available Scientific DataSet providers:\n" + DataSetFactory.RegisteredToString()); if (int.TryParse(RoleEnvironment.GetConfigurationSettingValue("JobTouchPeriod"), out touchPeriodInSeconds)) { WorkerTrace.TraceEvent(TraceEventType.Verbose, 10, string.Format("Touch period for processing job is set to {0}", touchPeriodInSeconds)); } else { WorkerTrace.TraceEvent(TraceEventType.Warning, 11, string.Format("Failed to read touch period from config. Parsing of value failed. touch period is set to {0}", touchPeriodInSeconds)); } manager = new JobManager(jobsDatabaseConnectionString, jobsStorageConnectionString); //Scheduling cleanup string instanceId = RoleEnvironment.CurrentRoleInstance.Id; int.TryParse(instanceId.Substring(instanceId.LastIndexOf(".") + 1), out instanceIndex); if (instanceIndex == 0) { double cleanPeriod = 0; if (!double.TryParse(RoleEnvironment.GetConfigurationSettingValue("HoursBetweenCleanup"), out cleanPeriod)) { cleanPeriod = 23; WorkerTrace.TraceEvent(TraceEventType.Warning, 12, "Failed to parse period between clean-ups from configuration. Setting it to default 23 hours."); } cleanupTimeSpan = TimeSpan.FromHours(cleanPeriod); lastCleanUpTime = manager.LastCleanUpTime; DateTime now = DateTime.UtcNow; if (now - lastCleanUpTime >= cleanupTimeSpan) { manager.SubmitCleanUp(); lastCleanUpTime = now; } } WorkerTrace.TraceEvent(TraceEventType.Verbose, 13, string.Format("starting Allocated memory: {0}Mb", GC.GetTotalMemory(false) / 1024 / 1024)); }
public override void Run() { WorkerTrace.TraceEvent(TraceEventType.Information, 0, "Running"); WorkerTrace.TraceEvent(TraceEventType.Information, 1, string.Format("Worker assembly: {0}", Assembly.GetExecutingAssembly().FullName)); WorkerTrace.TraceEvent(TraceEventType.Start, 2, "Initializing FetchWorker..."); try { Initialize(); } catch (Exception exc) { WorkerTrace.TraceEvent(TraceEventType.Critical, 2, string.Format("FetchWorker initialization failed: {0}", exc.ToString())); throw; } WorkerTrace.TraceEvent(TraceEventType.Stop, 2, "FetchWorker initialization complete"); int heavyJobsPermitedCount = int.Parse(RoleEnvironment.GetConfigurationSettingValue("HeavyJobsPermitedCount")); int lightJobExecutionPermitedTimeSec = int.Parse(RoleEnvironment.GetConfigurationSettingValue("LightJobExecutionTimeLimitSec")); TimeSpan daysBeforeJobDeletion; { double days = 0; if (!double.TryParse(RoleEnvironment.GetConfigurationSettingValue("DaysBeforeJobDeletion"), out days)) { days = 60; WorkerTrace.TraceEvent(TraceEventType.Warning, 3, string.Format("{0}: Bad \"DaysBeforeJobDeletion\" setting! Default value (60) is used.", JobManager.CleanUpJobHash)); } daysBeforeJobDeletion = TimeSpan.FromDays(days); } while (true) { //Scheduling cleanup if (instanceIndex == 0) { var utcnow = DateTime.UtcNow; if (utcnow - lastCleanUpTime >= cleanupTimeSpan) { manager.SubmitCleanUp(); lastCleanUpTime = utcnow; WorkerTrace.TraceEvent(TraceEventType.Information, 4, "Clean up job submited"); } } current = manager.PeekLockJob(stopRequested, int.Parse(RoleEnvironment.GetConfigurationSettingValue("JobQueuePollingMilisec")), heavyJobsPermitedCount, (job, context) => { if (job.Hash.Trim() == JobManager.CleanUpJobHash) { return(new CleanUpJob(job, context, manager, daysBeforeJobDeletion, new JobSettings() { PermitedHeavyPartWorkers = heavyJobsPermitedCount, LightJobExecutionPermitedTimeSec = lightJobExecutionPermitedTimeSec, TouchPeriodInSeconds = touchPeriodInSeconds })); } else { return(new ComputationJob(job, context, configProvider, new JobSettings() { PermitedHeavyPartWorkers = heavyJobsPermitedCount, LightJobExecutionPermitedTimeSec = lightJobExecutionPermitedTimeSec, TouchPeriodInSeconds = touchPeriodInSeconds }, manager.ResultDataSetUri(job.Hash, true))); } }); if (current == null) { break; } try { WorkerTrace.TraceEvent(TraceEventType.Start, 5, string.Format("{0}: start processing", current.ToShortString())); current.Perform(); } catch (Exception exc) { WorkerTrace.TraceEvent(TraceEventType.Error, 14, string.Format("{0}: error processing: {1}", current.ToShortString(), exc.ToString())); current.Complete(false); Exception toCheckForOutOfMemory = exc; do { if (toCheckForOutOfMemory.GetType() == typeof(OutOfMemoryException)) { WorkerTrace.TraceEvent(TraceEventType.Critical, 15, string.Format("{0}:Requesting role instance recycling due to OutOfMemory exception during calculation", current.ToShortString())); if (current != null) { current.Abandon(); } Thread.Sleep(25); RoleEnvironment.RequestRecycle(); } toCheckForOutOfMemory = toCheckForOutOfMemory.InnerException; } while (toCheckForOutOfMemory != null); } finally { if (current != null) { current.Dispose(); //dispose stops TouchThread } current = null; } } }