/// <summary> /// Returns the number of available devices of the provided type. This includes unprovisioned devices but not reserved ones /// Note: unprovisioned devices are currently only returned when device is not constrained /// </summary> public int GetTotalDeviceCount(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Validate = null) { lock (LockObject) { return(AvailableDevices.Union(ReservedDevices).Where(D => Validate == null ? Constraint.Check(D) : Validate(D)).Count() + UnprovisionedDevices.Where(D => Constraint.Check(D)).Count()); } }
/// <summary> /// Returns the number of available devices of the provided type. This includes unprovisioned devices but not reserved ones. /// Note: unprovisioned devices are currently only returned when device is not constrained /// </summary> public int GetAvailableDeviceCount(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Validate = null) { lock (LockObject) { return(AvailableDevices.Where(D => Validate == null ? Constraint.Check(D) : Validate(D)).Count() + UnprovisionedDevices.Where(D => D.Platform == Constraint.Platform && Constraint.IsIdentity()).Count()); } }
/// <summary> /// Registers the provided device for availability /// </summary> /// <param name="Device"></param> public void RegisterDevice(ITargetDevice Device, UnrealTargetConstraint Constraint = null) { lock (LockObject) { if (AvailableDevices.Contains(Device)) { throw new Exception("Device already registered!"); } InitialConnectState[Device] = Device.IsConnected; Constraints[Device] = Constraint ?? new UnrealTargetConstraint(Device.Platform.Value); AvailableDevices.Add(Device); if (Log.IsVerbose) { Device.RunOptions = Device.RunOptions & ~CommandUtils.ERunOptions.NoLoggingOfRunCommand; } } }
/// <summary> /// Helper that reserves and returns a list of available devices based on the passed in roles /// </summary> /// <param name="Configs"></param> /// <returns></returns> public bool TryReserveDevices() { List <ITargetDevice> AcquiredDevices = new List <ITargetDevice>(); List <ITargetDevice> SkippedDevices = new List <ITargetDevice>(); ReleaseDevices(); // figure out how many of each device we need Dictionary <UnrealTargetConstraint, int> RequiredDeviceTypes = new Dictionary <UnrealTargetConstraint, int>(); IEnumerable <UnrealSessionRole> RolesNeedingDevices = SessionRoles.Where(R => !R.IsNullRole()); // Get a count of the number of devices required for each platform RolesNeedingDevices.ToList().ForEach(C => { if (!RequiredDeviceTypes.ContainsKey(C.Constraint)) { RequiredDeviceTypes[C.Constraint] = 0; } RequiredDeviceTypes[C.Constraint]++; }); // check whether pool can accommodate devices if (!DevicePool.Instance.CheckAvailableDevices(RequiredDeviceTypes, ProblemDevices)) { return(false); } // AG - this needs more thought as it can either be good (lots of devices under contention, temp failure) or // just burn cycles spinning on something that won't ever work... // If we had problem devices last time see if there are enough of that type that we can exclude them //if (ProblemDevices.Count > 0) //{ // Dictionary<UnrealTargetPlatform, int> ProblemPlatforms = new Dictionary<UnrealTargetPlatform, int>(); // // count how many problems for each platform // ProblemDevices.ForEach(Device => // { // if (ProblemPlatforms.ContainsKey(Device.Platform) == false) // { // ProblemPlatforms[Device.Platform] = 0; // } // ProblemPlatforms[Device.Platform]++; // }); // List<ITargetDevice> ProblemsDevicesToRelease = new List<ITargetDevice>(); // // foreach device, see if we have enough others that we can ignore it // ProblemDevices.ForEach(Device => // { // if (ProblemPlatforms[Device.Platform] < AvailableDeviceTypes[Device.Platform]) // { // Log.Verbose("Had problem with device {0} last time, will now ignore", Device.Name); // } // else // { // Log.Verbose("Would like to ignore device {0} but not enough of this type in pool", Device.Name); // ProblemsDevicesToRelease.Add(Device); // } // }); // // remove any // ProblemDevices = ProblemDevices.Where(D => ProblemsDevicesToRelease.Contains(D) == false).ToList(); //} // nothing acquired yet... AcquiredDevices.Clear(); // for each platform, enumerate and select from the available devices foreach (var PlatformReqKP in RequiredDeviceTypes) { UnrealTargetConstraint Constraint = PlatformReqKP.Key; UnrealTargetPlatform? Platform = Constraint.Platform; int NeedOfThisType = RequiredDeviceTypes[Constraint]; DevicePool.Instance.EnumerateDevices(Constraint, Device => { int HaveOfThisType = AcquiredDevices.Where(D => D.Platform == Device.Platform && Constraint.Check(Device)).Count(); bool WeWant = NeedOfThisType > HaveOfThisType; if (WeWant) { bool Available = Device.IsAvailable; bool Have = AcquiredDevices.Contains(Device); bool Problem = ProblemDevices.Where(D => D.Name == Device.Name && D.Platform == Device.Platform).Count() > 0; Log.Verbose("Device {0}: Available:{1}, Have:{2}, HasProblem:{3}", Device.Name, Available, Have, Problem); if (Available && Have == false && Problem == false) { Log.Info("Acquiring device {0}", Device.Name); AcquiredDevices.Add(Device); HaveOfThisType++; } else { Log.Info("Skipping device {0}", Device.Name); SkippedDevices.Add(Device); } } // continue if we need more of this platform type return(HaveOfThisType < NeedOfThisType); }); } // If we got enough devices, go to step2 where we provision and try to connect them if (AcquiredDevices.Count == RolesNeedingDevices.Count()) { // actually acquire them DevicePool.Instance.ReserveDevices(AcquiredDevices); Log.Info("Selected devices {0} for client(s). Prepping", string.Join(", ", AcquiredDevices)); foreach (ITargetDevice Device in AcquiredDevices) { if (Device.IsOn == false) { Log.Info("Powering on {0}", Device); Device.PowerOn(); } else if (Globals.Params.ParseParam("reboot")) { Log.Info("Rebooting {0}", Device); Device.Reboot(); } if (Device.IsConnected == false) { Log.Verbose("Connecting to {0}", Device); Device.Connect(); } } // Step 3: Verify we actually connected to them var LostDevices = AcquiredDevices.Where(D => !D.IsConnected); if (LostDevices.Count() > 0) { Log.Info("Lost connection to devices {0} for client(s). ", string.Join(", ", LostDevices)); // mark these as problems. Could be something grabbed them before us, could be that they are // unresponsive in some way LostDevices.ToList().ForEach(D => MarkProblemDevice(D)); AcquiredDevices.ToList().ForEach(D => D.Disconnect()); DevicePool.Instance.ReleaseDevices(AcquiredDevices); AcquiredDevices.Clear(); } } if (AcquiredDevices.Count() != RolesNeedingDevices.Count()) { Log.Info("Failed to resolve all devices. Releasing the ones we have "); DevicePool.Instance.ReleaseDevices(AcquiredDevices); } else { ReservedDevices = AcquiredDevices; } // release devices that were skipped DevicePool.Instance.ReleaseDevices(SkippedDevices); return(ReservedDevices.Count() == RolesNeedingDevices.Count()); }
/// <summary> /// Checks whether device pool can accommodate requirements, optionally add service devices to meet demand /// </summary> internal bool CheckAvailableDevices(Dictionary <UnrealTargetConstraint, int> RequiredDevices, List <ProblemDevice> ProblemDevices = null, bool UseServiceDevices = true) { Dictionary <UnrealTargetConstraint, int> AvailableDeviceTypes = new Dictionary <UnrealTargetConstraint, int>(); Dictionary <UnrealTargetConstraint, int> TotalDeviceTypes = new Dictionary <UnrealTargetConstraint, int>(); // Do these "how many available" checks every time because the DevicePool provisions on demand so while it may think it has N machines, // some of them may fail to be provisioned and we could end up with none! // See how many of these types are in device pool (mostly to supply informative info if we can't meet these) foreach (var PlatformRequirement in RequiredDevices) { UnrealTargetConstraint Constraint = PlatformRequirement.Key; Func <ITargetDevice, bool> Validate = (ITargetDevice Device) => { if (!Constraint.Check(Device)) { return(false); } if (ProblemDevices == null) { return(true); } foreach (ProblemDevice PDevice in ProblemDevices) { if (PDevice.Platform == Device.Platform && PDevice.Name == Device.Name) { return(false); } } return(true); }; AvailableDeviceTypes[Constraint] = DevicePool.Instance.GetAvailableDeviceCount(Constraint, Validate); TotalDeviceTypes[Constraint] = DevicePool.Instance.GetTotalDeviceCount(Constraint, Validate); Log.Verbose("{0}: {1} devices required. Total:{2}, Available:{3}", Constraint, PlatformRequirement.Value, TotalDeviceTypes[PlatformRequirement.Key], AvailableDeviceTypes[PlatformRequirement.Key]); } // get a list of any platforms where we don't have enough var TooFewTotalDevices = RequiredDevices.Where(KP => TotalDeviceTypes[KP.Key] < RequiredDevices[KP.Key]).Select(KP => KP.Key); var TooFewCurrentDevices = RequiredDevices.Where(KP => AvailableDeviceTypes[KP.Key] < RequiredDevices[KP.Key]).Select(KP => KP.Key); List <UnrealTargetPlatform> ServicePlatforms = new List <UnrealTargetPlatform>() { UnrealTargetPlatform.PS4, UnrealTargetPlatform.XboxOne, UnrealTargetPlatform.Switch }; // support Android over wifi, though not on workers if (!Globals.IsWorker) { ServicePlatforms.Add(UnrealTargetPlatform.Android); } var Devices = TooFewTotalDevices.Concat(TooFewCurrentDevices); var UnsupportedPlatforms = Devices.Where(D => !ServicePlatforms.Contains(D.Platform.Value) && (!D.Platform.Value.ToString().StartsWith("XboxOne")) && (D.Platform.Value.ToString() != "XSX") && (D.Platform.Value.ToString() != "PS5")); // Request devices from the service if we need them if (UseServiceDevices && !String.IsNullOrEmpty(DeviceURL) && UnsupportedPlatforms.Count() == 0 && (TooFewTotalDevices.Count() > 0 || TooFewCurrentDevices.Count() > 0)) { Dictionary <UnrealTargetConstraint, int> DeviceCounts = new Dictionary <UnrealTargetConstraint, int>(); Devices.ToList().ForEach(Platform => DeviceCounts[Platform] = RequiredDevices[Platform]); Log.Info("Requesting devices from service at {0}", DeviceURL); // Acquire necessary devices from service if (!ReserveDevicesFromService(DeviceURL, DeviceCounts)) { return(false); } } else { // if we can't ever run then throw an exception if (TooFewTotalDevices.Count() > 0) { var MissingDeviceStrings = TooFewTotalDevices.Select(D => string.Format("Not enough devices of type {0} exist for test. ({1} required, {2} available)", D, RequiredDevices[D], AvailableDeviceTypes[D])); Log.Error("{0}", string.Join("\n", MissingDeviceStrings)); throw new AutomationException("Not enough devices available"); } // if we can't run now then return false if (TooFewCurrentDevices.Count() > 0) { var MissingDeviceStrings = TooFewCurrentDevices.Select(D => string.Format("Not enough devices of type {0} available for test. ({1} required, {2} available)", D, RequiredDevices[D], AvailableDeviceTypes[D])); Log.Verbose("{0}", string.Join("\n", MissingDeviceStrings)); return(false); } } return(true); }
public void EnumerateDevices(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Predicate) { lock (LockObject) { List <ITargetDevice> Selection = new List <ITargetDevice>(); // randomize the order of all devices that are of this platform var MatchingProvisionedDevices = AvailableDevices.Where(D => Constraint.Check(D)).ToList(); var MatchingUnprovisionedDevices = UnprovisionedDevices.Where(D => Constraint.Check(D)).ToList(); bool OutOfDevices = false; bool ContinuePredicate = true; do { // Go through all our provisioned devices to see if these fulfill the predicates // requirements ITargetDevice NextDevice = MatchingProvisionedDevices.FirstOrDefault(); while (NextDevice != null && ContinuePredicate) { Log.Verbose("Checking {0} against predicate", NextDevice.Name); MatchingProvisionedDevices.Remove(NextDevice); ContinuePredicate = Predicate(NextDevice); NextDevice = MatchingProvisionedDevices.FirstOrDefault(); } if (ContinuePredicate) { // add more devices if possible OutOfDevices = MatchingUnprovisionedDevices.Count() == 0; DeviceDefinition NextDeviceDef = MatchingUnprovisionedDevices.FirstOrDefault(); if (NextDeviceDef != null) { Log.Verbose("Provisioning device {0} for the pool", NextDeviceDef.Name); // try to create a device. This can fail, but if so we'll just end up back here // on the next iteration ITargetDevice NewDevice = CreateAndRegisterDeviceFromDefinition(NextDeviceDef); MatchingUnprovisionedDevices.Remove(NextDeviceDef); UnprovisionedDevices.Remove(NextDeviceDef); if (NewDevice != null) { MatchingProvisionedDevices.Add(NewDevice); Log.Verbose("Added device {0} to pool", NewDevice.Name); } else { Log.Info("Failed to provision {0}", NextDeviceDef.Name); // track this if (FailedProvisions.Contains(NextDeviceDef) == false) { FailedProvisions.Add(NextDeviceDef); } } } else { Log.Info("Pool ran out of devices of type {0}!", Constraint); OutOfDevices = true; } } } while (OutOfDevices == false && ContinuePredicate); } }