private int CreateSpotPlacementData(SalesArea salesArea, Mutex sharedResourceMutex, DateTime fromDate, DateTime toDate, DateTime modifiedTime) { int countSpotsDone = 0; // Process spots in date range batches foreach (DateTime[] dateRange in DateHelper.GetDateRanges(fromDate, toDate, 7)) { using (var scope = _repositoryFactory.BeginRepositoryScope()) { sharedResourceMutex.WaitOne(); var spotRepository = scope.CreateRepository <ISpotRepository>(); sharedResourceMutex.ReleaseMutex(); // Get spots for date range var spots = spotRepository.Search(dateRange[0], dateRange[1], salesArea.Name).OrderBy(s => s.ExternalSpotRef); HashSet <string> spotsDone = new HashSet <string>(); // Create spot placements, saved in batches List <SpotPlacement> spotPlacementBatch = new List <SpotPlacement>(); foreach (var spot in spots) { countSpotsDone++; SpotPlacement spotPlacement = new SpotPlacement() { ModifiedTime = modifiedTime, ExternalSpotRef = spot.ExternalSpotRef, ExternalBreakRef = spot.ExternalBreakNo }; if (!spotsDone.Contains(spotPlacement.ExternalSpotRef)) // Handle duplicates { spotsDone.Add(spotPlacement.ExternalSpotRef); spotPlacementBatch.Add(spotPlacement); } // Save batch if ((spotPlacementBatch.Count >= 10000) || ((spot == spots.Last() && spotPlacementBatch.Count > 0))) { using (var scope2 = scope.BeginRepositoryScope()) { sharedResourceMutex.WaitOne(); var spotPlacementRepository = scope2.CreateRepository <ISpotPlacementRepository>(); sharedResourceMutex.ReleaseMutex(); spotPlacementRepository.Insert(spotPlacementBatch); spotPlacementRepository.SaveChanges(); spotPlacementBatch.Clear(); } } if (countSpotsDone % 10000 == 0) { System.Diagnostics.Debug.Write(string.Format("Created {0} SpotPlacement documents for {1}", countSpotsDone, salesArea.Name)); } } } } return(countSpotsDone); }
public void Update(SpotPlacement spotPlacement) { var entity = _dbContext.Find <Entities.Tenant.SpotPlacement>( (spotPlacement ?? throw new ArgumentNullException(nameof(spotPlacement))).Id); if (entity != null) { _mapper.Map(spotPlacement, entity); _dbContext.Update(entity, post => post.MapTo(spotPlacement), _mapper); } }
/// <summary> /// This function creates a spot instance request inside a VPC. It returns the request ID if successful, or sets the error /// code and message otherwise /// </summary> /// <param name="AvailabilityZone">Name of the Availability Zone where the instances will be launched</param> /// <param name="AMI_ID">Id of the AMI that will be used as a base for the instances</param> /// <param name="SecurityGroupId">The name of the security group to be assigned to the instance(s)</param> /// <param name="KeyPairName">The name of the keypair to be assigned to the instance(s)</param> /// <param name="InstanceType">The type of the instance(s)</param> /// <param name="InstancePrice">The max price to pay for the instance(s)</param> /// <param name="InstanceCount">The number of instances to be launched</param> /// <param name="UserData">The user-data script that will be run as the instance(s) is(are) initialized</param> /// <returns>The list of Request Ids if successful</returns> public List <string> RequestClassicSpotInstances(string AvailabilityZone, string AMI_ID, string SecurityGroupId, string KeyPairName, string InstanceType, double InstancePrice, int InstanceCount = 1, string UserData = "") { List <string> RequestIds = new List <string> (); // Initialize error values ErrorCode = 0; ErrorMessage = ""; // Create the list with security groups List <string> SecurityGroups = new List <string> () { SecurityGroupId }; // Create placement object SpotPlacement spotPlacement = new SpotPlacement() { AvailabilityZone = AvailabilityZone }; // Create the launch specification LaunchSpecification launchSpecification = new LaunchSpecification() { ImageId = AMI_ID, InstanceType = InstanceType, KeyName = KeyPairName, SecurityGroups = SecurityGroups, Placement = spotPlacement, UserData = Gadgets.Base64Encode(UserData) }; // Create the request object RequestSpotInstancesRequest spotRequest = new RequestSpotInstancesRequest() { SpotPrice = InstancePrice.ToString(), InstanceCount = InstanceCount, LaunchSpecification = launchSpecification }; // Request the instances try { var spotResponse = EC2client.RequestSpotInstances(spotRequest); // Check response for errors if (spotResponse.HttpStatusCode != HttpStatusCode.OK) { ErrorCode = Convert.ToInt32(spotResponse.HttpStatusCode); ErrorMessage = "Http Error [" + spotResponse.HttpStatusCode.ToString() + "]"; } else { foreach (SpotInstanceRequest request in spotResponse.SpotInstanceRequests) { RequestIds.Add(request.SpotInstanceRequestId); } } } catch (Exception ex) { ErrorCode = -1; ErrorMessage = ex.Message + "::" + ex.InnerException; } return(RequestIds); }
public void SaveSpotPlacements( DateTime processorDateTime, IReadOnlyCollection <Spot> spotsToSave, IReadOnlyDictionary <string, SpotPlacement> spotPlacementsByExternalRef, IReadOnlyDictionary <Guid, SpotInfo> spotInfos, string batchStartEndDateForLogging ) { if (spotsToSave is null || spotInfos is null) { return; } if (spotsToSave.Count == 0) { return; } RaiseInfo( "Starting to save SpotPlacements " + $"for {Log(spotsToSave.Count)} spots " + $"for {batchStartEndDateForLogging}"); int pageNumber = 0; int newCount = 0; int updatedCount = 0; var batch = spotsToSave.Take(DefaultBatchSize); using var scope = RepositoryFactory.BeginRepositoryScope(); do { var spotPlacementRepository = scope.CreateRepository <ISpotPlacementRepository>(); foreach (Spot spot in batch) { if (spotPlacementsByExternalRef.TryGetValue( spot.ExternalSpotRef, out SpotPlacement spotPlacement) ) { updatedCount++; // Before updating ExternalBreakRef record // previous value so we can reset schedule data spotPlacement.ResetExternalBreakRef = spotPlacement.ExternalBreakRef; spotPlacement.ModifiedTime = processorDateTime; spotPlacement.ExternalBreakRef = ExternalBreakNumberOrDefault(spot); spotPlacementRepository.Update(spotPlacement); } else { newCount++; SpotInfo spotInfo = spotInfos[spot.Uid]; // A subsequent schedule data reset needs to // reset SpotPlacement.ExternalBreakRef to the // break that the spot was in before this run started spotPlacement = new SpotPlacement() { ModifiedTime = processorDateTime, ExternalSpotRef = spot.ExternalSpotRef, ResetExternalBreakRef = spotInfo.ExternalBreakRefAtRunStart, ExternalBreakRef = ExternalBreakNumberOrDefault(spot), }; spotPlacementRepository.Add(spotPlacement); } try { spotPlacementRepository.SaveChanges(); } catch (Exception exception) { throw new Exception( "Error saving Smooth spot placements for " + $"spot {spotPlacement.ExternalSpotRef} for " + $"batch {batchStartEndDateForLogging}", exception ); } } batch = spotsToSave .Skip(DefaultBatchSize * ++pageNumber) .Take(DefaultBatchSize); } while (batch.Any()); RaiseInfo( $"Finished saving SpotPlacements for {Log(spotsToSave.Count)} spots " + $"(New={Log(newCount)}, Updated={Log(updatedCount)}) " + $"for {batchStartEndDateForLogging}" );
public IRequest Marshall(RequestSpotInstancesRequest requestSpotInstancesRequest) { IRequest request = new DefaultRequest(requestSpotInstancesRequest, "AmazonEC2"); request.Parameters.Add("Action", "RequestSpotInstances"); request.Parameters.Add("Version", "2014-02-01"); if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetSpotPrice()) { request.Parameters.Add("SpotPrice", StringUtils.FromString(requestSpotInstancesRequest.SpotPrice)); } if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetInstanceCount()) { request.Parameters.Add("InstanceCount", StringUtils.FromInt(requestSpotInstancesRequest.InstanceCount)); } if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetType()) { request.Parameters.Add("Type", StringUtils.FromString(requestSpotInstancesRequest.Type)); } if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetValidFrom()) { request.Parameters.Add("ValidFrom", StringUtils.FromDateTime(requestSpotInstancesRequest.ValidFrom)); } if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetValidUntil()) { request.Parameters.Add("ValidUntil", StringUtils.FromDateTime(requestSpotInstancesRequest.ValidUntil)); } if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetLaunchGroup()) { request.Parameters.Add("LaunchGroup", StringUtils.FromString(requestSpotInstancesRequest.LaunchGroup)); } if (requestSpotInstancesRequest != null && requestSpotInstancesRequest.IsSetAvailabilityZoneGroup()) { request.Parameters.Add("AvailabilityZoneGroup", StringUtils.FromString(requestSpotInstancesRequest.AvailabilityZoneGroup)); } if (requestSpotInstancesRequest != null) { LaunchSpecification launchSpecification = requestSpotInstancesRequest.LaunchSpecification; if (launchSpecification != null && launchSpecification.IsSetImageId()) { request.Parameters.Add("LaunchSpecification.ImageId", StringUtils.FromString(launchSpecification.ImageId)); } if (launchSpecification != null && launchSpecification.IsSetKeyName()) { request.Parameters.Add("LaunchSpecification.KeyName", StringUtils.FromString(launchSpecification.KeyName)); } if (launchSpecification != null) { List <GroupIdentifier> allSecurityGroupsList = launchSpecification.AllSecurityGroups; int allSecurityGroupsListIndex = 1; foreach (GroupIdentifier allSecurityGroupsListValue in allSecurityGroupsList) { if (allSecurityGroupsListValue != null && allSecurityGroupsListValue.IsSetGroupName()) { request.Parameters.Add("LaunchSpecification.GroupSet." + allSecurityGroupsListIndex + ".GroupName", StringUtils.FromString(allSecurityGroupsListValue.GroupName)); } if (allSecurityGroupsListValue != null && allSecurityGroupsListValue.IsSetGroupId()) { request.Parameters.Add("LaunchSpecification.GroupSet." + allSecurityGroupsListIndex + ".GroupId", StringUtils.FromString(allSecurityGroupsListValue.GroupId)); } allSecurityGroupsListIndex++; } } if (launchSpecification != null) { List <string> securityGroupsList = launchSpecification.SecurityGroups; int securityGroupsListIndex = 1; foreach (string securityGroupsListValue in securityGroupsList) { request.Parameters.Add("LaunchSpecification.SecurityGroup." + securityGroupsListIndex, StringUtils.FromString(securityGroupsListValue)); securityGroupsListIndex++; } } if (launchSpecification != null && launchSpecification.IsSetUserData()) { request.Parameters.Add("LaunchSpecification.UserData", StringUtils.FromString(launchSpecification.UserData)); } if (launchSpecification != null && launchSpecification.IsSetAddressingType()) { request.Parameters.Add("LaunchSpecification.AddressingType", StringUtils.FromString(launchSpecification.AddressingType)); } if (launchSpecification != null && launchSpecification.IsSetInstanceType()) { request.Parameters.Add("LaunchSpecification.InstanceType", StringUtils.FromString(launchSpecification.InstanceType)); } if (launchSpecification != null) { SpotPlacement placement = launchSpecification.Placement; if (placement != null && placement.IsSetAvailabilityZone()) { request.Parameters.Add("LaunchSpecification.Placement.AvailabilityZone", StringUtils.FromString(placement.AvailabilityZone)); } if (placement != null && placement.IsSetGroupName()) { request.Parameters.Add("LaunchSpecification.Placement.GroupName", StringUtils.FromString(placement.GroupName)); } } if (launchSpecification != null && launchSpecification.IsSetKernelId()) { request.Parameters.Add("LaunchSpecification.KernelId", StringUtils.FromString(launchSpecification.KernelId)); } if (launchSpecification != null && launchSpecification.IsSetRamdiskId()) { request.Parameters.Add("LaunchSpecification.RamdiskId", StringUtils.FromString(launchSpecification.RamdiskId)); } if (launchSpecification != null) { List <BlockDeviceMapping> blockDeviceMappingsList = launchSpecification.BlockDeviceMappings; int blockDeviceMappingsListIndex = 1; foreach (BlockDeviceMapping blockDeviceMappingsListValue in blockDeviceMappingsList) { if (blockDeviceMappingsListValue != null && blockDeviceMappingsListValue.IsSetVirtualName()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".VirtualName", StringUtils.FromString(blockDeviceMappingsListValue.VirtualName)); } if (blockDeviceMappingsListValue != null && blockDeviceMappingsListValue.IsSetDeviceName()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".DeviceName", StringUtils.FromString(blockDeviceMappingsListValue.DeviceName)); } if (blockDeviceMappingsListValue != null) { EbsBlockDevice ebs = blockDeviceMappingsListValue.Ebs; if (ebs != null && ebs.IsSetSnapshotId()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".Ebs.SnapshotId", StringUtils.FromString(ebs.SnapshotId)); } if (ebs != null && ebs.IsSetVolumeSize()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".Ebs.VolumeSize", StringUtils.FromInt(ebs.VolumeSize)); } if (ebs != null && ebs.IsSetDeleteOnTermination()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".Ebs.DeleteOnTermination", StringUtils.FromBool(ebs.DeleteOnTermination)); } if (ebs != null && ebs.IsSetVolumeType()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".Ebs.VolumeType", StringUtils.FromString(ebs.VolumeType)); } if (ebs != null && ebs.IsSetIops()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".Ebs.Iops", StringUtils.FromInt(ebs.Iops)); } } if (blockDeviceMappingsListValue != null && blockDeviceMappingsListValue.IsSetNoDevice()) { request.Parameters.Add("LaunchSpecification.BlockDeviceMapping." + blockDeviceMappingsListIndex + ".NoDevice", StringUtils.FromString(blockDeviceMappingsListValue.NoDevice)); } blockDeviceMappingsListIndex++; } } if (launchSpecification != null && launchSpecification.IsSetMonitoringEnabled()) { request.Parameters.Add("LaunchSpecification.Monitoring.Enabled", StringUtils.FromBool(launchSpecification.MonitoringEnabled)); } if (launchSpecification != null && launchSpecification.IsSetSubnetId()) { request.Parameters.Add("LaunchSpecification.SubnetId", StringUtils.FromString(launchSpecification.SubnetId)); } if (launchSpecification != null) { List <InstanceNetworkInterfaceSpecification> networkInterfacesList = launchSpecification.NetworkInterfaces; int networkInterfacesListIndex = 1; foreach (InstanceNetworkInterfaceSpecification networkInterfacesListValue in networkInterfacesList) { if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetNetworkInterfaceId()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".NetworkInterfaceId", StringUtils.FromString(networkInterfacesListValue.NetworkInterfaceId)); } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetDeviceIndex()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".DeviceIndex", StringUtils.FromInt(networkInterfacesListValue.DeviceIndex)); } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetSubnetId()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".SubnetId", StringUtils.FromString(networkInterfacesListValue.SubnetId)); } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetDescription()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".Description", StringUtils.FromString(networkInterfacesListValue.Description)); } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetPrivateIpAddress()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".PrivateIpAddress", StringUtils.FromString(networkInterfacesListValue.PrivateIpAddress)); } if (networkInterfacesListValue != null) { List <string> groupsList = networkInterfacesListValue.Groups; int groupsListIndex = 1; foreach (string groupsListValue in groupsList) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".SecurityGroupId." + groupsListIndex, StringUtils.FromString(groupsListValue)); groupsListIndex++; } } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetDeleteOnTermination()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".DeleteOnTermination", StringUtils.FromBool(networkInterfacesListValue.DeleteOnTermination)); } if (networkInterfacesListValue != null) { List <PrivateIpAddressSpecification> privateIpAddressesList = networkInterfacesListValue.PrivateIpAddresses; int privateIpAddressesListIndex = 1; foreach (PrivateIpAddressSpecification privateIpAddressesListValue in privateIpAddressesList) { if (privateIpAddressesListValue != null && privateIpAddressesListValue.IsSetPrivateIpAddress()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".PrivateIpAddresses." + privateIpAddressesListIndex + ".PrivateIpAddress", StringUtils.FromString(privateIpAddressesListValue.PrivateIpAddress)); } if (privateIpAddressesListValue != null && privateIpAddressesListValue.IsSetPrimary()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".PrivateIpAddresses." + privateIpAddressesListIndex + ".Primary", StringUtils.FromBool(privateIpAddressesListValue.Primary)); } privateIpAddressesListIndex++; } } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetSecondaryPrivateIpAddressCount()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".SecondaryPrivateIpAddressCount", StringUtils.FromInt(networkInterfacesListValue.SecondaryPrivateIpAddressCount)); } if (networkInterfacesListValue != null && networkInterfacesListValue.IsSetAssociatePublicIpAddress()) { request.Parameters.Add("LaunchSpecification.NetworkInterface." + networkInterfacesListIndex + ".AssociatePublicIpAddress", StringUtils.FromBool(networkInterfacesListValue.AssociatePublicIpAddress)); } networkInterfacesListIndex++; } } if (launchSpecification != null) { IamInstanceProfileSpecification iamInstanceProfile = launchSpecification.IamInstanceProfile; if (iamInstanceProfile != null && iamInstanceProfile.IsSetArn()) { request.Parameters.Add("LaunchSpecification.IamInstanceProfile.Arn", StringUtils.FromString(iamInstanceProfile.Arn)); } if (iamInstanceProfile != null && iamInstanceProfile.IsSetName()) { request.Parameters.Add("LaunchSpecification.IamInstanceProfile.Name", StringUtils.FromString(iamInstanceProfile.Name)); } } if (launchSpecification != null && launchSpecification.IsSetEbsOptimized()) { request.Parameters.Add("LaunchSpecification.EbsOptimized", StringUtils.FromBool(launchSpecification.EbsOptimized)); } } return(request); }
private int UpdateSpotPlacementData(SalesArea salesArea, Mutex sharedResourceMutex, DateTime fromDate, DateTime toDate, DateTime modifiedTime) { int countSpotsDone = 0; // Process spots in date range batches foreach (DateTime[] dateRange in DateHelper.GetDateRanges(fromDate, toDate, 7)) { using (var scope = _repositoryFactory.BeginRepositoryScope()) { sharedResourceMutex.WaitOne(); var spotRepository = scope.CreateRepository <ISpotRepository>(); sharedResourceMutex.ReleaseMutex(); // Get spots for date range var spots = spotRepository.Search(dateRange[0], dateRange[1], salesArea.Name).OrderBy(s => s.ExternalSpotRef); HashSet <string> spotsDone = new HashSet <string>(); // Create spot placements, saved in batches List <Spot> spotBatch = new List <Spot>(); foreach (var spot in spots) { countSpotsDone++; spotBatch.Add(spot); if ((spotBatch.Count >= 1000) || (spot == spots.Last())) { using (var scope2 = scope.BeginRepositoryScope()) { sharedResourceMutex.WaitOne(); var spotPlacementRepository = scope2.CreateRepository <ISpotPlacementRepository>(); sharedResourceMutex.ReleaseMutex(); // Get spot placements Dictionary <string, SpotPlacement> spotPlacementedByExternalSpotRef = SpotPlacement.IndexListByExternalSpotRef(spotPlacementRepository.GetByExternalSpotRefs(spotBatch.Select(s => s.ExternalSpotRef))); // Create/update spot placements for each spot List <SpotPlacement> spotPlacementBatch = new List <SpotPlacement>(); foreach (var spotInBatch in spotBatch) { SpotPlacement spotPlacement = spotPlacementedByExternalSpotRef.ContainsKey(spotInBatch.ExternalSpotRef) ? spotPlacementedByExternalSpotRef[spotInBatch.ExternalSpotRef] : null; if (spotPlacement == null) // Newly encountered spot { spotPlacement = new SpotPlacement() { ModifiedTime = modifiedTime, ExternalSpotRef = spotInBatch.ExternalSpotRef, ExternalBreakRef = spot.ExternalBreakNo }; spotPlacementBatch.Add(spotPlacement); // spotPlacementRepository.Insert(new List<SpotPlacement>() { spotPlacement }); } else // Existing spot { spotPlacement.ModifiedTime = modifiedTime; spotPlacement.ExternalBreakRef = spot.ExternalBreakNo; spotPlacementRepository.Update(spotPlacement); } } if (spotPlacementBatch.Any()) { spotPlacementRepository.Insert(spotPlacementBatch); } spotPlacementRepository.SaveChanges(); // Clear batch spotBatch.Clear(); } } if (countSpotsDone % 10000 == 0) { System.Diagnostics.Debug.Write(string.Format("Created {0} SpotPlacement documents for {1}", countSpotsDone, salesArea.Name)); } } } } return(countSpotsDone); }
public void Add(SpotPlacement item) { UpdateOrInsertItem(_folder, _type, item, item.Id.ToString()); }
public void Update(SpotPlacement spotPlacement) { UpdateOrInsertItem(_folder, _type, spotPlacement, spotPlacement.Id.ToString()); }
public void Add(SpotPlacement item) { _dbContext.Add(_mapper.Map <Entities.Tenant.SpotPlacement>(item), post => post.MapTo(item), _mapper); }