/// <summary>
 /// In startup load services which are already executed from DB for recovery purpose
 /// in order not to execute already executed services
 /// </summary>
 private void LoadRecovery()
 {
     if (Environment == null)
     {
         return;
     }
     try
     {
         var instanceList = Environment.GetServiceInstanceActiveList();
         foreach (var instance in instanceList)
         {
             _scheduledRequests.Add(instance);
         }
     }
     catch (Exception ex)
     {
         WriteLog(String.Format("Failed in LoadRecovery(), ex: {0}", ex.Message), ex);
     }
 }
        //==================================
        #endregion

        #region Scheduling algorithms
        //==================================
        /// <summary>
        /// The main method of creating scheduler
        /// </summary>
        private void Schedule(bool reschedule = false)
        {
            lock (_unscheduledRequests)
            {
                lock (_scheduledRequests)
                {
                    // Set need reschedule to false in order to avoid more schedule from other threads
                    _needReschedule = false;

                    #region Manage history and find services to schedule
                    // ------------------------------------

                    // Move pending uninitialized services to the unscheduled list so they can be rescheduled
                    foreach (ServiceInstance request in _scheduledRequests.RemoveNotActivated())
                    {
                        if (request.SchedulingInfo.RequestedTime + request.SchedulingInfo.MaxDeviationAfter > DateTime.Now)
                        {
                            _unscheduledRequests.Add(request);
                        }
                        else
                        {
                            AsLockable(request).Unlock(_instanceLock);
                            request.SchedulingInfo.SchedulingStatus = SchedulingStatus.CouldNotBeScheduled;
                            AsLockable(request).Lock(_instanceLock);
                        }
                    }

                    // Get Services for next time line
                    foreach (ServiceInstance request in GetServicesForTimeLine(reschedule))
                    {
                        _unscheduledRequests.Add(request);
                    }

                    // Copy unscheduled requests to an ordered list
                    var servicesForNextTimeLine = new List <ServiceInstance>(_unscheduledRequests
                                                                             .OrderBy(schedulingdata => schedulingdata.SchedulingInfo.RequestedTime));

                    // ------------------------------------
                    #endregion

                    #region Find Match services
                    // ------------------------------------

                    //Same services or same services with same profile
                    foreach (ServiceInstance serviceInstance in servicesForNextTimeLine)
                    {
                        //Get all services with same configurationID
                        var requestsWithSameConfiguration = _scheduledRequests.GetWithSameConfiguration(serviceInstance);

                        //Get all services with same profileID
                        var requestsWithSameProfile = _scheduledRequests.GetWithSameProfile(serviceInstance);

                        //Find the first available time this service with specific service and profile
                        TimeSpan avgExecutionTime = GetAverageExecutionTime(serviceInstance.Configuration.ServiceName, Convert.ToInt32(serviceInstance.Configuration.Profile.Parameters["AccountID"]), _percentile);

                        DateTime baseStartTime       = (serviceInstance.SchedulingInfo.RequestedTime < DateTime.Now) ? DateTime.Now : serviceInstance.SchedulingInfo.RequestedTime;
                        DateTime baseEndTime         = baseStartTime.Add(avgExecutionTime);
                        DateTime calculatedStartTime = baseStartTime;
                        DateTime calculatedEndTime   = baseEndTime;

                        bool found = false;
                        while (!found)
                        {
                            IOrderedEnumerable <ServiceInstance> whereToLookNext = null;

                            int countedPerConfiguration = requestsWithSameConfiguration.Count(s => (calculatedStartTime >= s.SchedulingInfo.ExpectedStartTime && calculatedStartTime <= s.SchedulingInfo.ExpectedEndTime) || (calculatedEndTime >= s.SchedulingInfo.ExpectedStartTime && calculatedEndTime <= s.SchedulingInfo.ExpectedEndTime));
                            if (countedPerConfiguration < serviceInstance.Configuration.Limits.MaxConcurrentGlobal)
                            {
                                int countedPerProfile = requestsWithSameProfile.Count(s => (calculatedStartTime >= s.SchedulingInfo.ExpectedStartTime && calculatedStartTime <= s.SchedulingInfo.ExpectedEndTime) || (calculatedEndTime >= s.SchedulingInfo.ExpectedStartTime && calculatedEndTime <= s.SchedulingInfo.ExpectedEndTime));
                                if (countedPerProfile < serviceInstance.Configuration.Limits.MaxConcurrentPerProfile)
                                {
                                    AsLockable(serviceInstance).Unlock(_instanceLock);
                                    serviceInstance.SchedulingInfo.ExpectedStartTime = calculatedStartTime;
                                    serviceInstance.SchedulingInfo.ExpectedEndTime   = calculatedEndTime;
                                    serviceInstance.SchedulingInfo.SchedulingStatus  = Services.SchedulingStatus.Scheduled;
                                    AsLockable(serviceInstance).Lock(_instanceLock);

                                    found = true;
                                }
                                else
                                {
                                    whereToLookNext = requestsWithSameProfile;
                                }
                            }
                            else
                            {
                                whereToLookNext = requestsWithSameConfiguration;
                            }

                            if (!found)
                            {
                                if (whereToLookNext == null)
                                {
                                    throw new Exception("This should not have happened.");
                                }

                                calculatedStartTime = whereToLookNext.Where(s => s.SchedulingInfo.ExpectedEndTime >= calculatedStartTime).Min(s => s.SchedulingInfo.ExpectedEndTime);
                                if (calculatedStartTime < DateTime.Now)
                                {
                                    calculatedStartTime = DateTime.Now;
                                }

                                //Get end time
                                calculatedEndTime = calculatedStartTime.Add(avgExecutionTime);

                                ////remove unfree time from servicePerConfiguration and servicePerProfile
                                if (calculatedStartTime <= _timeLineTo)
                                {
                                    requestsWithSameConfiguration = from s in requestsWithSameConfiguration
                                                                    where s.SchedulingInfo.ExpectedEndTime > calculatedStartTime
                                                                    orderby s.SchedulingInfo.ExpectedStartTime
                                                                    select s;

                                    requestsWithSameProfile = from s in requestsWithSameProfile
                                                              where s.SchedulingInfo.ExpectedEndTime > calculatedStartTime
                                                              orderby s.SchedulingInfo.ExpectedStartTime
                                                              select s;
                                }
                            }
                        }

                        if (serviceInstance.SchedulingInfo.ExpectedDeviation <= serviceInstance.SchedulingInfo.MaxDeviationAfter || serviceInstance.SchedulingInfo.MaxDeviationAfter == TimeSpan.Zero)
                        {
                            AsLockable(serviceInstance).Unlock(_instanceLock);
                            serviceInstance.SchedulingInfo.SchedulingStatus = SchedulingStatus.Scheduled;
                            serviceInstance.StateChanged += new EventHandler(Instance_StateChanged);
                            AsLockable(serviceInstance).Lock(_instanceLock);

                            // Legacy stuff
                            //TODO: MAXEXECUTIONTIME
                            TimeSpan maxExecutionTime = TimeSpan.FromMilliseconds(avgExecutionTime.TotalMilliseconds * double.Parse(AppSettings.Get(this, "MaxExecutionTimeProduct")));
                            serviceInstance.Configuration.Limits.MaxExecutionTime = maxExecutionTime;
                            _scheduledRequests.Add(serviceInstance);
                            _unscheduledRequests.Remove(serviceInstance);
                        }
                    }
                    #endregion
                }


                SchedulingInformationEventArgs args = new SchedulingInformationEventArgs();
                args.ScheduleInformation = new List <ServiceInstance>();
                foreach (var scheduleService in _scheduledRequests)
                {
                    args.ScheduleInformation.Add(scheduleService);
                }
                OnNewScheduleCreated(args);
                NotifyServicesToRun();
            }
        }
        /// <summary>
        /// The main method of scheduler calculation
        /// </summary>
        public void Schedule(bool reschedule = false)
        {
            WriteLog(String.Format("Start scheduling: scheduled={0} (uninitialized={1}, ended={2}), unscheduled={3}",
                                   _scheduledRequests.Count, _scheduledRequests.Count(s => s.State == ServiceState.Uninitialized),
                                   _scheduledRequests.Count(s => s.State == ServiceState.Ended), _unscheduledRequests.Count));
            try
            {
                lock (_unscheduledRequests)
                {
                    lock (_scheduledRequests)
                    {
                        // Set need reschedule to false in order to avoid more schedule from other threads
                        _needReschedule = false;

                        #region Manage history and find services to schedule
                        // ------------------------------------

                        RemoveNotRelevantRequests();

                        // Move pending uninitialized services to the unscheduled list so they can be rescheduled
                        foreach (ServiceInstance request in _scheduledRequests.RemoveNotActivated())
                        {
                            if (request.SchedulingInfo.RequestedTime + request.SchedulingInfo.MaxDeviationAfter > DateTime.Now)
                            {
                                WriteLog(String.Format("Move scheduled request to unscheduled list '{0}'", request.DebugInfo()));
                                _unscheduledRequests.Add(request);
                            }
                            else
                            {
                                // mark service as could not be scheduled because of deviation
                                ServiceCannotBeScheduled(request);
                            }
                        }

                        // Get Services for next timeframe
                        foreach (ServiceInstance request in GetServicesInTimeframe(reschedule))
                        {
                            WriteLog(String.Format("Add request to unscheduled list '{0}'", request.DebugInfo()));
                            _unscheduledRequests.Add(request);
                        }

                        // Copy unscheduled requests to an ordered list by request time + max deviation after
                        var servicesForNextTimeframe = new List <ServiceInstance>(_unscheduledRequests.OrderBy(
                                                                                      schedulingData =>
                                                                                      schedulingData.SchedulingInfo.RequestedTime +
                                                                                      schedulingData.SchedulingInfo.MaxDeviationAfter));
                        // ------------------------------------
                        #endregion

                        #region Find Match services
                        // ------------------------------------

                        //Same services or same services with same profile
                        foreach (ServiceInstance serviceInstance in servicesForNextTimeframe)
                        {
                            //Get all services with same configurationID (concurrent per Template)
                            var requestsWithSameTemplate = _scheduledRequests.GetWithSameTemplate(serviceInstance);

                            //Get all services with same profileID (concurrent per Profile)
                            var requestsWithSameProfile = _scheduledRequests.GetWithSameProfile(serviceInstance);

                            //take execution time by service configuration
                            TimeSpan avgExecutionTime = GetExecutionStatisticsForService(serviceInstance.Configuration);

                            DateTime baseStartTime       = serviceInstance.SchedulingInfo.RequestedTime < DateTime.Now ? DateTime.Now : serviceInstance.SchedulingInfo.RequestedTime;
                            DateTime baseEndTime         = baseStartTime.Add(avgExecutionTime);
                            DateTime calculatedStartTime = baseStartTime;
                            DateTime calculatedEndTime   = baseEndTime;

                            while (true)
                            {
                                IOrderedEnumerable <ServiceInstance> whereToLookNext;

                                // check concurren per Template
                                var countPerTemplate = CountConcurrent(requestsWithSameTemplate, calculatedStartTime, calculatedEndTime);
                                if (countPerTemplate < serviceInstance.Configuration.Limits.MaxConcurrentPerTemplate)
                                {
                                    // check concurrent per Profile
                                    var countPerProfile = CountConcurrent(requestsWithSameProfile, calculatedStartTime, calculatedEndTime);
                                    if (countPerProfile < serviceInstance.Configuration.Limits.MaxConcurrentPerProfile)
                                    {
                                        // if no concurrents set scheduling time (found time for scheduling)
                                        AsLockable(serviceInstance).Unlock(_instanceLock);
                                        serviceInstance.SchedulingInfo.ExpectedStartTime = calculatedStartTime;
                                        serviceInstance.SchedulingInfo.ExpectedEndTime   = calculatedEndTime;
                                        AsLockable(serviceInstance).Lock(_instanceLock);
                                        break;
                                    }
                                    whereToLookNext = requestsWithSameProfile;
                                }
                                else
                                {
                                    whereToLookNext = requestsWithSameTemplate;
                                }

                                if (whereToLookNext == null)
                                {
                                    throw new Exception("This should not have happened.");
                                }

                                // try to take the next available time to schedule the service (end time of concurrent service)
                                calculatedStartTime = !whereToLookNext.Any() ? calculatedStartTime :
                                                      whereToLookNext.Where(s => s.SchedulingInfo.ExpectedEndTime >= calculatedStartTime).Min(s => s.SchedulingInfo.ExpectedEndTime);

                                if (calculatedStartTime < DateTime.Now)
                                {
                                    calculatedStartTime = DateTime.Now;
                                }
                                calculatedEndTime = calculatedStartTime.Add(avgExecutionTime);

                                // check if a new calculated time is still valid according to max deviation after
                                if (calculatedStartTime <= serviceInstance.SchedulingInfo.RequestedTime + serviceInstance.SchedulingInfo.MaxDeviationAfter)
                                {
                                    requestsWithSameTemplate = GetNextConcurrentList(requestsWithSameTemplate, calculatedStartTime);
                                    requestsWithSameProfile  = GetNextConcurrentList(requestsWithSameProfile, calculatedStartTime);
                                }
                                else
                                {
                                    // found that request cannot be scheduled - set expected start and end, after the while loop it will be signed as CouldNotBeScheduled
                                    AsLockable(serviceInstance).Unlock(_instanceLock);
                                    serviceInstance.SchedulingInfo.ExpectedStartTime = calculatedStartTime;
                                    serviceInstance.SchedulingInfo.ExpectedEndTime   = calculatedEndTime;
                                    serviceInstance.SchedulingInfo.SchedulingStatus  = SchedulingStatus.CouldNotBeScheduled;
                                    AsLockable(serviceInstance).Lock(_instanceLock);
                                    break;
                                }
                            }

                            // check if scheduled inside max deviation frame
                            if (serviceInstance.SchedulingInfo.ExpectedDeviation <= serviceInstance.SchedulingInfo.MaxDeviationAfter)
                            {
                                ScheduleServiceInstance(serviceInstance, avgExecutionTime);
                            }
                        }
                        //------------------------------
                        #endregion

                        WriteLog(String.Format("Finish scheduling: scheduled={0} (uninitialized={1}, ended={2}), unscheduled={3}",
                                               _scheduledRequests.Count,
                                               _scheduledRequests.Count(s => s.State == ServiceState.Uninitialized),
                                               _scheduledRequests.Count(s => s.State == ServiceState.Ended), _unscheduledRequests.Count));

                        // send the current list of scheduled services
                        SendScheduledServicesUpdate();
                    }
                }
                // no need to run shcheduled services - another thread is doing this
                //ExecuteScheduledRequests();
            }
            catch (Exception ex)
            {
                WriteLog(String.Format("Failed in Schedule(), ex: {0}", ex.Message), ex);
            }
        }