/// <summary>
/// Report target device issue to service with given error message
/// </summary>
public void ReportDeviceError(ITargetDevice Device, string ErrorMessage)
{
DeviceDefinition Def = null;
if (!ServiceDeviceInfo.TryGetValue(Device, out Def))
{
return;
}
ReportDeviceError(Def.Name, ErrorMessage);
}
public bool Check(DeviceDefinition DeviceDef)
{
if (Platform != DeviceDef.Platform)
{
return(false);
}
if (IsIdentity())
{
return(true);
}
bool ModelMatch = Model == string.Empty ? true : Model.Equals(DeviceDef.Model, StringComparison.InvariantCultureIgnoreCase);
bool PerfMatch = (PerfSpec == EPerfSpec.Unspecified) ? true : PerfSpec == DeviceDef.PerfSpec;
return(ModelMatch && PerfMatch);
}
public void AddLocalDevices(int Count)
{
// todo - default per platform?
UnrealTargetPlatform LocalPlat = UnrealTargetPlatform.Win64;
int NumDevices = GetAvailableDeviceCount(new UnrealTargetConstraint(LocalPlat));
// add local PCs (todo - some max number?)
for (int i = NumDevices; i < Count + NumDevices; i++)
{
DeviceDefinition Def = new DeviceDefinition();
Def.Name = string.Format("LocalDevice{0}", i);
Def.Platform = BuildHostPlatform.Current.Platform;
UnprovisionedDevices.Add(Def);
}
}
/// <summary>
/// Created and registered a device from the provided definition
/// </summary>
/// <param name="Def"></param>
/// <returns></returns>
protected ITargetDevice CreateAndRegisterDeviceFromDefinition(DeviceDefinition Def)
{
ITargetDevice NewDevice = null;
IDeviceFactory Factory = Gauntlet.Utils.InterfaceHelpers.FindImplementations <IDeviceFactory>()
.Where(F => F.CanSupportPlatform(Def.Platform))
.FirstOrDefault();
if (Factory == null)
{
throw new AutomationException("No IDeviceFactory implementation that supports {0}", Def.Platform);
}
try
{
bool IsDesktop = Def.Platform != null && UnrealBuildTool.Utils.GetPlatformsInClass(UnrealPlatformClass.Desktop).Contains(Def.Platform.Value);
if (IsDesktop)
{
string ClientTempDir = Path.Combine(LocalTempDir, "DeviceCache", Def.Platform.ToString());
int DeviceCount = AvailableDevices.Union(ReservedDevices).Where(D => D.Platform == Def.Platform).Count();
NewDevice = Factory.CreateDevice(Def.Name, ClientTempDir);
}
else
{
NewDevice = Factory.CreateDevice(Def.Address, Def.DeviceData);
}
lock (LockObject)
{
if (NewDevice != null)
{
RegisterDevice(NewDevice, new UnrealTargetConstraint(NewDevice.Platform, Def.PerfSpec, Def.Model));
}
}
}
catch (Exception Ex)
{
Log.Info("Failed to create device {0}. {1}", Def.ToString(), Ex.Message);
}
return(NewDevice);
}
/// <summary>
/// Created a list of device definitions from the passed in reference. Needs work....
/// </summary>
/// <param name="InputReference"></param>
/// <param name="InLocalTempDir"></param>
/// <param name="ObeyConstraints"></param>
public void AddDevices(UnrealTargetPlatform DefaultPlatform, string InputReference, bool ObeyConstraints = true)
{
lock (LockObject)
{
List <ITargetDevice> NewDevices = new List <ITargetDevice>();
int SlashIndex = InputReference.IndexOf("\\") >= 0 ? InputReference.IndexOf("\\") : InputReference.IndexOf("/");
bool PossibleFileName = InputReference.IndexOfAny(Path.GetInvalidPathChars()) < 0 &&
(InputReference.IndexOf(":") == -1 || (InputReference.IndexOf(":") == SlashIndex - 1));
// Did they specify a file?
if (PossibleFileName && File.Exists(InputReference))
{
Gauntlet.Log.Info("Adding devices from {0}", InputReference);
List <DeviceDefinition> DeviceDefinitions = JsonConvert.DeserializeObject <List <DeviceDefinition> >(File.ReadAllText(InputReference));
foreach (DeviceDefinition Def in DeviceDefinitions)
{
Gauntlet.Log.Info("Adding {0}", Def);
// use Legacy field if it exists
if (Def.Platform == null)
{
Def.Platform = Def.Type;
}
// check for an availability constraint
if (string.IsNullOrEmpty(Def.Available) == false && ObeyConstraints)
{
// check whether disabled
if (String.Compare(Def.Available, "disabled", true) == 0)
{
Gauntlet.Log.Info("Skipping {0} due to being disabled", Def.Name);
continue;
}
// availability is specified as a range, e.g 21:00-09:00.
Match M = Regex.Match(Def.Available, @"(\d{1,2}:\d\d)\s*-\s*(\d{1,2}:\d\d)");
if (M.Success)
{
DateTime From, To;
if (DateTime.TryParse(M.Groups[1].Value, out From) && DateTime.TryParse(M.Groups[2].Value, out To))
{
// these are just times so when parsed will have todays date. If the To time is less than
// From (22:00-06:00) time it spans midnight so move it to the next day
if (To < From)
{
To = To.AddDays(1);
}
// if From is in the future (e.g. it's 01:00 and range is 22:00-08:00) we may be in the previous days window,
// so move them both back a day
if (From > DateTime.Now)
{
From = From.AddDays(-1);
To = To.AddDays(-1);
}
if (DateTime.Now < From || DateTime.Now > To)
{
Gauntlet.Log.Info("Skipping {0} due to availability constraint {1}", Def.Name, Def.Available);
continue;
}
}
else
{
Gauntlet.Log.Warning("Failed to parse availability {0} for {1}", Def.Available, Def.Name);
}
}
}
Def.RemoveOnShutdown = true;
if (Def.Platform == null)
{
Def.Platform = DefaultPlatform;
}
UnprovisionedDevices.Add(Def);
}
// randomize devices so if there's a bad device st the start so we don't always hit it (or we do if its later)
UnprovisionedDevices = UnprovisionedDevices.OrderBy(D => Guid.NewGuid()).ToList();
}
else
{
if (string.IsNullOrEmpty(InputReference) == false)
{
string[] DevicesList = InputReference.Split(',');
foreach (string DeviceRef in DevicesList)
{
// check for <platform>:<address>:<port>|<model>. We pass address:port to device constructor
Match M = Regex.Match(DeviceRef, @"(.+?):(.+)");
UnrealTargetPlatform DevicePlatform = DefaultPlatform;
string DeviceAddress = DeviceRef;
string Model = string.Empty;
// when using device service, skip adding local non-desktop devices to pool
bool IsDesktop = UnrealBuildTool.Utils.GetPlatformsInClass(UnrealPlatformClass.Desktop).Contains(DevicePlatform);
if (!IsDesktop && DeviceRef.Equals("default", StringComparison.OrdinalIgnoreCase) && !String.IsNullOrEmpty(DeviceURL))
{
continue;
}
if (M.Success)
{
if (!UnrealTargetPlatform.TryParse(M.Groups[1].ToString(), out DevicePlatform))
{
throw new AutomationException("platform {0} is not a recognized device type", M.Groups[1].ToString());
}
DeviceAddress = M.Groups[2].ToString();
// parse device model
if (DeviceAddress.Contains("|"))
{
string[] Components = DeviceAddress.Split(new char[] { '|' });
DeviceAddress = Components[0];
Model = Components[1];
}
}
Log.Info("Added device {0}:{1} to pool", DevicePlatform, DeviceAddress);
DeviceDefinition Def = new DeviceDefinition();
Def.Address = DeviceAddress;
Def.Name = DeviceAddress;
Def.Platform = DevicePlatform;
Def.Model = Model;
UnprovisionedDevices.Add(Def);
}
}
}
}
}
/// <summary>
/// Reserve devices from service
/// </summary>
public bool ReserveDevicesFromService(string DeviceURL, Dictionary <UnrealTargetConstraint, int> DeviceTypes)
{
if (String.IsNullOrEmpty(DeviceURL))
{
return(false);
}
Dictionary <UnrealTargetPlatform, string> DeviceMap = new Dictionary <UnrealTargetPlatform, string>();
foreach (string Platform in UnrealTargetPlatform.GetValidPlatformNames())
{
if (Platform == "PS4" || Platform == "XboxOne")
{
DeviceMap.Add(UnrealTargetPlatform.Parse(Platform), string.Format("{0}-DevKit", Platform));
}
else
{
DeviceMap.Add(UnrealTargetPlatform.Parse(Platform), Platform);
}
}
List <string> Devices = new List <string>();
// convert devices to request list
foreach (KeyValuePair <UnrealTargetConstraint, int> Entry in DeviceTypes)
{
if (Entry.Key.Platform == null)
{
continue;
}
if (!DeviceMap.ContainsKey(Entry.Key.Platform.Value))
{
// if an unsupported device, we can't reserve it
Log.Info("Unable to reserve service device of type: {0}", Entry.Key);
return(false);
}
if (!string.IsNullOrEmpty(Entry.Key.Model))
{
// if specific device model, we can't currently reserve it from (legacy) service
if (DeviceURL.ToLower().Contains("deviceservice.epicgames.net"))
{
Log.Info("Unable to reserve service device of model: {0} on legacy service", Entry.Key.Model);
return(false);
}
}
for (int i = 0; i < Entry.Value; i++)
{
// @todo: if any additional reservation requirements, encode constraint into json
string Constraint = Entry.Key.PerfSpec.ToString();
if (!string.IsNullOrEmpty(Entry.Key.Model))
{
Constraint = Entry.Key.Model;
}
Devices.Add(DeviceMap[Entry.Key.Platform.Value] + ":" + Constraint);
}
}
// reserve devices
Uri ReservationServerUri;
if (Uri.TryCreate(DeviceURL, UriKind.Absolute, out ReservationServerUri))
{
DeviceReservationAutoRenew DeviceReservation = null;
string PoolID = Globals.WorkerPoolID != -1 ? Globals.WorkerPoolID.ToString() : "";
try
{
DeviceReservation = new DeviceReservationAutoRenew(DeviceURL, 0, PoolID, Devices.ToArray());
}
catch (Exception Ex)
{
Log.Info("Unable to make device registration: {0}", Ex.Message);
return(false);
}
if (DeviceReservation == null || DeviceReservation.Devices.Count != Devices.Count())
{
return(false);
}
// Add target devices from reservation
List <ITargetDevice> ReservedDevices = new List <ITargetDevice>();
foreach (var Device in DeviceReservation.Devices)
{
DeviceDefinition Def = new DeviceDefinition();
Def.Address = Device.IPOrHostName;
Def.Name = Device.Name;
Def.Platform = DeviceMap.FirstOrDefault(Entry => Entry.Value == Device.Type).Key;
Def.DeviceData = Device.DeviceData;
Def.Model = string.Empty;
if (!String.IsNullOrEmpty(Device.PerfSpec) && !Enum.TryParse <EPerfSpec>(Device.PerfSpec, true, out Def.PerfSpec))
{
throw new AutomationException("Unable to convert perfspec '{0}' into an EPerfSpec", Device.PerfSpec);
}
ITargetDevice TargetDevice = CreateAndRegisterDeviceFromDefinition(Def);
// If a device from service can't be added, fail reservation and cleanup devices
// @todo: device problem reporting, requesting additional devices
if (TargetDevice == null)
{
ReportDeviceError(Device.Name, "CreateDeviceError");
// If some devices from reservation have been created, release them which will also dispose of reservation
if (ReservedDevices.Count > 0)
{
ReleaseDevices(ReservedDevices);
}
else
{
// otherwise, no devices have been creation so just cancel this reservation
DeviceReservation.Dispose();
}
Log.Info("Unable to make device registration: device registration failed for {0}:{1}", Def.Platform, Def.Name);
return(false);
}
else
{
ReservedDevices.Add(TargetDevice);
}
ServiceDeviceInfo[TargetDevice] = Def;
ServiceReservations[TargetDevice] = DeviceReservation;
}
Log.Info("Successfully reserved service devices");
Devices.ForEach(Device => Log.Verbose(" Device: {0}", Device));
return(true);
}
Log.Info("Unable to reserve service devices:");
Devices.ForEach(Device => Log.Info(" Device: {0}", Device));
return(false);
}
public void EnumerateDevices(UnrealTargetConstraint Constraint, Func <ITargetDevice, bool> Predicate)
{
lock (LockObject)
{
List <ITargetDevice> Selection = new List <ITargetDevice>();
// randomize the order of all devices that are of this platform
var MatchingProvisionedDevices = AvailableDevices.Where(D => Constraint.Check(D)).ToList();
var MatchingUnprovisionedDevices = UnprovisionedDevices.Where(D => Constraint.Check(D)).ToList();
bool OutOfDevices = false;
bool ContinuePredicate = true;
do
{
// Go through all our provisioned devices to see if these fulfill the predicates
// requirements
ITargetDevice NextDevice = MatchingProvisionedDevices.FirstOrDefault();
while (NextDevice != null && ContinuePredicate)
{
Log.Verbose("Checking {0} against predicate", NextDevice.Name);
MatchingProvisionedDevices.Remove(NextDevice);
ContinuePredicate = Predicate(NextDevice);
NextDevice = MatchingProvisionedDevices.FirstOrDefault();
}
if (ContinuePredicate)
{
// add more devices if possible
OutOfDevices = MatchingUnprovisionedDevices.Count() == 0;
DeviceDefinition NextDeviceDef = MatchingUnprovisionedDevices.FirstOrDefault();
if (NextDeviceDef != null)
{
Log.Verbose("Provisioning device {0} for the pool", NextDeviceDef.Name);
// try to create a device. This can fail, but if so we'll just end up back here
// on the next iteration
ITargetDevice NewDevice = CreateAndRegisterDeviceFromDefinition(NextDeviceDef);
MatchingUnprovisionedDevices.Remove(NextDeviceDef);
UnprovisionedDevices.Remove(NextDeviceDef);
if (NewDevice != null)
{
MatchingProvisionedDevices.Add(NewDevice);
Log.Verbose("Added device {0} to pool", NewDevice.Name);
}
else
{
Log.Info("Failed to provision {0}", NextDeviceDef.Name);
// track this
if (FailedProvisions.Contains(NextDeviceDef) == false)
{
FailedProvisions.Add(NextDeviceDef);
}
}
}
else
{
Log.Info("Pool ran out of devices of type {0}!", Constraint);
OutOfDevices = true;
}
}
} while (OutOfDevices == false && ContinuePredicate);
}
}
/// <summary>
/// Reserve devices from service
/// </summary>
public bool ReserveDevicesFromService(string DeviceURL, Dictionary <UnrealTargetConstraint, int> DeviceTypes)
{
if (String.IsNullOrEmpty(DeviceURL))
{
return(false);
}
Dictionary <UnrealTargetPlatform, string> DeviceMap = new Dictionary <UnrealTargetPlatform, string>()
{
// todo: add other platforms and externalize this mapping
{ UnrealTargetPlatform.PS4, "PS4-DevKit" },
{ UnrealTargetPlatform.XboxOne, "XboxOne-DevKit" },
{ UnrealTargetPlatform.Android, "Android" },
{ UnrealTargetPlatform.Switch, "Switch" }
};
List <string> Devices = new List <string>();
// convert devices to request list
foreach (KeyValuePair <UnrealTargetConstraint, int> Entry in DeviceTypes)
{
if (!DeviceMap.ContainsKey(Entry.Key.Platform))
{
// if an unsupported device, we can't reserve it
Log.Error("Unable to reserve service device of type: {0}", Entry.Key);
return(false);
}
for (int i = 0; i < Entry.Value; i++)
{
// @todo: if any additional reservation requirements, encode constraint into json
Devices.Add(DeviceMap[Entry.Key.Platform] + ":" + Entry.Key.PerfSpec);
}
}
// reserve devices
Uri ReservationServerUri;
if (Uri.TryCreate(DeviceURL, UriKind.Absolute, out ReservationServerUri))
{
DeviceReservationAutoRenew DeviceReservation = null;
try
{
DeviceReservation = new DeviceReservationAutoRenew(DeviceURL, 0, Devices.ToArray());
}
catch (Exception Ex)
{
Log.Info("Unable to make device registration: {0}", Ex.Message);
return(false);
}
if (DeviceReservation == null || DeviceReservation.Devices.Count != Devices.Count())
{
return(false);
}
// Add target devices from reservation
List <ITargetDevice> ReservedDevices = new List <ITargetDevice>();
foreach (var Device in DeviceReservation.Devices)
{
DeviceDefinition Def = new DeviceDefinition();
Def.Address = Device.IPOrHostName;
Def.Name = Device.Name;
Def.Platform = DeviceMap.FirstOrDefault(Entry => Entry.Value == Device.Type).Key;
Def.DeviceData = Device.DeviceData;
if (!String.IsNullOrEmpty(Device.PerfSpec) && !Enum.TryParse <EPerfSpec>(Device.PerfSpec, true, out Def.PerfSpec))
{
throw new AutomationException("Unable to convert perfspec '{0}' into an EPerfSpec", Device.PerfSpec);
}
ITargetDevice TargetDevice = CreateAndRegisterDeviceFromDefinition(Def);
// If a device from service can't be added, fail reservation and cleanup devices
// @todo: device problem reporting, requesting additional devices
if (TargetDevice == null)
{
ReportDeviceError(Device.Name, "CreateDeviceError");
// If some devices from reservation have been created, release them which will also dispose of reservation
if (ReservedDevices.Count > 0)
{
ReleaseDevices(ReservedDevices);
}
else
{
// otherwise, no devices have been creation so just cancel this reservation
DeviceReservation.Dispose();
}
Log.Info("Unable to make device registration: device registration failed for {0}:{1}", Def.Platform, Def.Name);
return(false);
}
else
{
ReservedDevices.Add(TargetDevice);
}
ServiceDeviceInfo[TargetDevice] = Def;
ServiceReservations[TargetDevice] = DeviceReservation;
}
Log.Info("Successfully reserved service devices");
Devices.ForEach(Device => Log.Verbose(" Device: {0}", Device));
return(true);
}
Log.Error("Unable to reserve service devices:");
Devices.ForEach(Device => Log.Error(" Device: {0}", Device));
return(false);
}
public bool Check(DeviceDefinition DeviceDef)
{
bool Match = (PerfSpec == EPerfSpec.Unspecified) ? DeviceDef.Model == Model : PerfSpec == DeviceDef.PerfSpec;
return(Platform == DeviceDef.Platform && (IsIdentity() || Match));
}
