/// <summary>
/// Returns the number of available devices of the provided type. This includes unprovisioned devices but not reserved ones
/// Note: unprovisioned devices are currently only returned when device is not constrained
/// </summary>
public int GetTotalDeviceCount(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Validate = null)
{
lock (LockObject)
{
return(AvailableDevices.Union(ReservedDevices).Where(D => Validate == null ? Constraint.Check(D) : Validate(D)).Count() +
UnprovisionedDevices.Where(D => Constraint.Check(D)).Count());
}
}
/// <summary>
/// Returns the number of available devices of the provided type. This includes unprovisioned devices but not reserved ones.
/// Note: unprovisioned devices are currently only returned when device is not constrained
/// </summary>
public int GetAvailableDeviceCount(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Validate = null)
{
lock (LockObject)
{
return(AvailableDevices.Where(D => Validate == null ? Constraint.Check(D) : Validate(D)).Count() +
UnprovisionedDevices.Where(D => D.Platform == Constraint.Platform && Constraint.IsIdentity()).Count());
}
}
/// <summary>
/// Registers the provided device for availability
/// </summary>
/// <param name="Device"></param>
public void RegisterDevice(ITargetDevice Device, UnrealTargetConstraint Constraint = null)
{
lock (LockObject)
{
if (AvailableDevices.Contains(Device))
{
throw new Exception("Device already registered!");
}
InitialConnectState[Device] = Device.IsConnected;
Constraints[Device] = Constraint ?? new UnrealTargetConstraint(Device.Platform.Value);
AvailableDevices.Add(Device);
if (Log.IsVerbose)
{
Device.RunOptions = Device.RunOptions & ~CommandUtils.ERunOptions.NoLoggingOfRunCommand;
}
}
}
/// <summary>
/// Helper that reserves and returns a list of available devices based on the passed in roles
/// </summary>
/// <param name="Configs"></param>
/// <returns></returns>
public bool TryReserveDevices()
{
List <ITargetDevice> AcquiredDevices = new List <ITargetDevice>();
List <ITargetDevice> SkippedDevices = new List <ITargetDevice>();
ReleaseDevices();
// figure out how many of each device we need
Dictionary <UnrealTargetConstraint, int> RequiredDeviceTypes = new Dictionary <UnrealTargetConstraint, int>();
IEnumerable <UnrealSessionRole> RolesNeedingDevices = SessionRoles.Where(R => !R.IsNullRole());
// Get a count of the number of devices required for each platform
RolesNeedingDevices.ToList().ForEach(C =>
{
if (!RequiredDeviceTypes.ContainsKey(C.Constraint))
{
RequiredDeviceTypes[C.Constraint] = 0;
}
RequiredDeviceTypes[C.Constraint]++;
});
// check whether pool can accommodate devices
if (!DevicePool.Instance.CheckAvailableDevices(RequiredDeviceTypes, ProblemDevices))
{
return(false);
}
// AG - this needs more thought as it can either be good (lots of devices under contention, temp failure) or
// just burn cycles spinning on something that won't ever work...
// If we had problem devices last time see if there are enough of that type that we can exclude them
//if (ProblemDevices.Count > 0)
//{
// Dictionary<UnrealTargetPlatform, int> ProblemPlatforms = new Dictionary<UnrealTargetPlatform, int>();
// // count how many problems for each platform
// ProblemDevices.ForEach(Device =>
// {
// if (ProblemPlatforms.ContainsKey(Device.Platform) == false)
// {
// ProblemPlatforms[Device.Platform] = 0;
// }
// ProblemPlatforms[Device.Platform]++;
// });
// List<ITargetDevice> ProblemsDevicesToRelease = new List<ITargetDevice>();
// // foreach device, see if we have enough others that we can ignore it
// ProblemDevices.ForEach(Device =>
// {
// if (ProblemPlatforms[Device.Platform] < AvailableDeviceTypes[Device.Platform])
// {
// Log.Verbose("Had problem with device {0} last time, will now ignore", Device.Name);
// }
// else
// {
// Log.Verbose("Would like to ignore device {0} but not enough of this type in pool", Device.Name);
// ProblemsDevicesToRelease.Add(Device);
// }
// });
// // remove any
// ProblemDevices = ProblemDevices.Where(D => ProblemsDevicesToRelease.Contains(D) == false).ToList();
//}
// nothing acquired yet...
AcquiredDevices.Clear();
// for each platform, enumerate and select from the available devices
foreach (var PlatformReqKP in RequiredDeviceTypes)
{
UnrealTargetConstraint Constraint = PlatformReqKP.Key;
UnrealTargetPlatform? Platform = Constraint.Platform;
int NeedOfThisType = RequiredDeviceTypes[Constraint];
DevicePool.Instance.EnumerateDevices(Constraint, Device =>
{
int HaveOfThisType = AcquiredDevices.Where(D => D.Platform == Device.Platform && Constraint.Check(Device)).Count();
bool WeWant = NeedOfThisType > HaveOfThisType;
if (WeWant)
{
bool Available = Device.IsAvailable;
bool Have = AcquiredDevices.Contains(Device);
bool Problem = ProblemDevices.Where(D => D.Name == Device.Name && D.Platform == Device.Platform).Count() > 0;
Log.Verbose("Device {0}: Available:{1}, Have:{2}, HasProblem:{3}", Device.Name, Available, Have, Problem);
if (Available &&
Have == false &&
Problem == false)
{
Log.Info("Acquiring device {0}", Device.Name);
AcquiredDevices.Add(Device);
HaveOfThisType++;
}
else
{
Log.Info("Skipping device {0}", Device.Name);
SkippedDevices.Add(Device);
}
}
// continue if we need more of this platform type
return(HaveOfThisType < NeedOfThisType);
});
}
// If we got enough devices, go to step2 where we provision and try to connect them
if (AcquiredDevices.Count == RolesNeedingDevices.Count())
{
// actually acquire them
DevicePool.Instance.ReserveDevices(AcquiredDevices);
Log.Info("Selected devices {0} for client(s). Prepping", string.Join(", ", AcquiredDevices));
foreach (ITargetDevice Device in AcquiredDevices)
{
if (Device.IsOn == false)
{
Log.Info("Powering on {0}", Device);
Device.PowerOn();
}
else if (Globals.Params.ParseParam("reboot"))
{
Log.Info("Rebooting {0}", Device);
Device.Reboot();
}
if (Device.IsConnected == false)
{
Log.Verbose("Connecting to {0}", Device);
Device.Connect();
}
}
// Step 3: Verify we actually connected to them
var LostDevices = AcquiredDevices.Where(D => !D.IsConnected);
if (LostDevices.Count() > 0)
{
Log.Info("Lost connection to devices {0} for client(s). ", string.Join(", ", LostDevices));
// mark these as problems. Could be something grabbed them before us, could be that they are
// unresponsive in some way
LostDevices.ToList().ForEach(D => MarkProblemDevice(D));
AcquiredDevices.ToList().ForEach(D => D.Disconnect());
DevicePool.Instance.ReleaseDevices(AcquiredDevices);
AcquiredDevices.Clear();
}
}
if (AcquiredDevices.Count() != RolesNeedingDevices.Count())
{
Log.Info("Failed to resolve all devices. Releasing the ones we have ");
DevicePool.Instance.ReleaseDevices(AcquiredDevices);
}
else
{
ReservedDevices = AcquiredDevices;
}
// release devices that were skipped
DevicePool.Instance.ReleaseDevices(SkippedDevices);
return(ReservedDevices.Count() == RolesNeedingDevices.Count());
}
/// <summary>
/// Checks whether device pool can accommodate requirements, optionally add service devices to meet demand
/// </summary>
internal bool CheckAvailableDevices(Dictionary <UnrealTargetConstraint, int> RequiredDevices, List <ProblemDevice> ProblemDevices = null, bool UseServiceDevices = true)
{
Dictionary <UnrealTargetConstraint, int> AvailableDeviceTypes = new Dictionary <UnrealTargetConstraint, int>();
Dictionary <UnrealTargetConstraint, int> TotalDeviceTypes = new Dictionary <UnrealTargetConstraint, int>();
// Do these "how many available" checks every time because the DevicePool provisions on demand so while it may think it has N machines,
// some of them may fail to be provisioned and we could end up with none!
// See how many of these types are in device pool (mostly to supply informative info if we can't meet these)
foreach (var PlatformRequirement in RequiredDevices)
{
UnrealTargetConstraint Constraint = PlatformRequirement.Key;
Func <ITargetDevice, bool> Validate = (ITargetDevice Device) =>
{
if (!Constraint.Check(Device))
{
return(false);
}
if (ProblemDevices == null)
{
return(true);
}
foreach (ProblemDevice PDevice in ProblemDevices)
{
if (PDevice.Platform == Device.Platform && PDevice.Name == Device.Name)
{
return(false);
}
}
return(true);
};
AvailableDeviceTypes[Constraint] = DevicePool.Instance.GetAvailableDeviceCount(Constraint, Validate);
TotalDeviceTypes[Constraint] = DevicePool.Instance.GetTotalDeviceCount(Constraint, Validate);
Log.Verbose("{0}: {1} devices required. Total:{2}, Available:{3}",
Constraint, PlatformRequirement.Value,
TotalDeviceTypes[PlatformRequirement.Key], AvailableDeviceTypes[PlatformRequirement.Key]);
}
// get a list of any platforms where we don't have enough
var TooFewTotalDevices = RequiredDevices.Where(KP => TotalDeviceTypes[KP.Key] < RequiredDevices[KP.Key]).Select(KP => KP.Key);
var TooFewCurrentDevices = RequiredDevices.Where(KP => AvailableDeviceTypes[KP.Key] < RequiredDevices[KP.Key]).Select(KP => KP.Key);
List <UnrealTargetPlatform> ServicePlatforms = new List <UnrealTargetPlatform>()
{
UnrealTargetPlatform.PS4, UnrealTargetPlatform.XboxOne, UnrealTargetPlatform.Switch
};
// support Android over wifi, though not on workers
if (!Globals.IsWorker)
{
ServicePlatforms.Add(UnrealTargetPlatform.Android);
}
var Devices = TooFewTotalDevices.Concat(TooFewCurrentDevices);
var UnsupportedPlatforms = Devices.Where(D => !ServicePlatforms.Contains(D.Platform.Value) && (!D.Platform.Value.ToString().StartsWith("XboxOne")) && (D.Platform.Value.ToString() != "XSX") && (D.Platform.Value.ToString() != "PS5"));
// Request devices from the service if we need them
if (UseServiceDevices && !String.IsNullOrEmpty(DeviceURL) && UnsupportedPlatforms.Count() == 0 && (TooFewTotalDevices.Count() > 0 || TooFewCurrentDevices.Count() > 0))
{
Dictionary <UnrealTargetConstraint, int> DeviceCounts = new Dictionary <UnrealTargetConstraint, int>();
Devices.ToList().ForEach(Platform => DeviceCounts[Platform] = RequiredDevices[Platform]);
Log.Info("Requesting devices from service at {0}", DeviceURL);
// Acquire necessary devices from service
if (!ReserveDevicesFromService(DeviceURL, DeviceCounts))
{
return(false);
}
}
else
{
// if we can't ever run then throw an exception
if (TooFewTotalDevices.Count() > 0)
{
var MissingDeviceStrings = TooFewTotalDevices.Select(D => string.Format("Not enough devices of type {0} exist for test. ({1} required, {2} available)", D, RequiredDevices[D], AvailableDeviceTypes[D]));
Log.Error("{0}", string.Join("\n", MissingDeviceStrings));
throw new AutomationException("Not enough devices available");
}
// if we can't run now then return false
if (TooFewCurrentDevices.Count() > 0)
{
var MissingDeviceStrings = TooFewCurrentDevices.Select(D => string.Format("Not enough devices of type {0} available for test. ({1} required, {2} available)", D, RequiredDevices[D], AvailableDeviceTypes[D]));
Log.Verbose("{0}", string.Join("\n", MissingDeviceStrings));
return(false);
}
}
return(true);
}
public void EnumerateDevices(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Predicate)
{
lock (LockObject)
{
List <ITargetDevice> Selection = new List <ITargetDevice>();
// randomize the order of all devices that are of this platform
var MatchingProvisionedDevices = AvailableDevices.Where(D => Constraint.Check(D)).ToList();
var MatchingUnprovisionedDevices = UnprovisionedDevices.Where(D => Constraint.Check(D)).ToList();
bool OutOfDevices = false;
bool ContinuePredicate = true;
do
{
// Go through all our provisioned devices to see if these fulfill the predicates
// requirements
ITargetDevice NextDevice = MatchingProvisionedDevices.FirstOrDefault();
while (NextDevice != null && ContinuePredicate)
{
Log.Verbose("Checking {0} against predicate", NextDevice.Name);
MatchingProvisionedDevices.Remove(NextDevice);
ContinuePredicate = Predicate(NextDevice);
NextDevice = MatchingProvisionedDevices.FirstOrDefault();
}
if (ContinuePredicate)
{
// add more devices if possible
OutOfDevices = MatchingUnprovisionedDevices.Count() == 0;
DeviceDefinition NextDeviceDef = MatchingUnprovisionedDevices.FirstOrDefault();
if (NextDeviceDef != null)
{
Log.Verbose("Provisioning device {0} for the pool", NextDeviceDef.Name);
// try to create a device. This can fail, but if so we'll just end up back here
// on the next iteration
ITargetDevice NewDevice = CreateAndRegisterDeviceFromDefinition(NextDeviceDef);
MatchingUnprovisionedDevices.Remove(NextDeviceDef);
UnprovisionedDevices.Remove(NextDeviceDef);
if (NewDevice != null)
{
MatchingProvisionedDevices.Add(NewDevice);
Log.Verbose("Added device {0} to pool", NewDevice.Name);
}
else
{
Log.Info("Failed to provision {0}", NextDeviceDef.Name);
// track this
if (FailedProvisions.Contains(NextDeviceDef) == false)
{
FailedProvisions.Add(NextDeviceDef);
}
}
}
else
{
Log.Info("Pool ran out of devices of type {0}!", Constraint);
OutOfDevices = true;
}
}
} while (OutOfDevices == false && ContinuePredicate);
}
}