public NativeYarnSubmission(
string rmNode, int wsPort, DfsClient dfsClient, string queueName, Uri stagingUri, Uri jobUri, string launcherNode, int launcherPort,
int amMemoryInMB, int numberOfProcesses, int workerMemoryInMB, string[] args)
: base(new NativeYarnClient(rmNode, wsPort, dfsClient, jobUri, launcherNode, launcherPort),
stagingUri, queueName, amMemoryInMB, numberOfProcesses, workerMemoryInMB, args)
{
}
private static void Main(string[] args)
{
var cert = new X509Certificate2(@"D:\cert\openssl\ace.realm\device.pfx", "acerealm");
var cl = new TcpClient();
while (true)
{
try
{
cl.ConnectAsync("localhost", 0xAceF).GetAwaiter().GetResult();
break;
}
catch
{
Thread.Sleep(500);
}
}
var cfg = DfsProtocolConfiguration.Instance;
var ssl = cfg.GetClientSslFactory("ace.dfs", cert);
client = new DfsClient(ssl, cl, "D:\\Ace-Dfs-Client-Cache");
client.Initialize();
Task.Factory.StartNew(Test);
Console.ReadKey();
}
static public Uri UploadResourceIfNeeded(DfsClient dfs, string localPath, Uri dfsDirectory,
out long modificationTime, out long size, ILogger logger = null)
{
ConsoleLogger.EnsureLogger(ref logger);
string suffix = MakeHash(localPath);
string localFileName = Path.GetFileName(localPath);
string dfsFileName = localFileName + "." + suffix;
Uri dfsPath = dfs.Combine(dfsDirectory, dfsFileName);
if (dfs.GetFileStatus(dfsPath, out modificationTime, out size))
{
return(dfsPath);
}
logger.Log("Uploading " + dfsPath.AbsoluteUri);
dfs.PutDfsFile(dfsPath, localPath);
if (dfs.GetFileStatus(dfsPath, out modificationTime, out size))
{
return(dfsPath);
}
throw new ApplicationException("Failed to upload resource " + localPath + " to " + dfsPath);
}
public NativeYarnSubmission(
string rmNode, int wsPort, DfsClient dfsClient, string queueName, Uri stagingUri, Uri jobUri, string peloponneseDirectory,
int amMemoryInMB, int numberOfProcesses, int workerMemoryInMB, string[] args)
: base(
new NativeYarnClient(rmNode, wsPort, dfsClient, jobUri, LauncherJarFile(peloponneseDirectory), YarnDirectory()),
stagingUri, queueName, amMemoryInMB, numberOfProcesses, workerMemoryInMB, args)
{
}
public static XElement MakeResourceElement(string localName, long modificationTime, long size,
DfsClient dfsClient, Uri dfsPath)
{
XElement resourceElement = new XElement("Resource");
resourceElement.SetAttributeValue("localName", localName);
resourceElement.SetAttributeValue("timestamp", modificationTime);
resourceElement.SetAttributeValue("size", size);
resourceElement.Value = dfsClient.GetClusterInternalUri(dfsPath).AbsoluteUri;
return resourceElement;
}
public static Task<XElement> MakePeloponneseResourceGroupAsync(DfsClient dfsClient, Uri stagingRoot, string ppmHome, ILogger logger = null)
{
if (dfsClient.IsThreadSafe)
{
return Task.Run<XElement>(() => { return MakePeloponneseResourceGroup(dfsClient, stagingRoot, ppmHome, logger); });
}
else
{
return Task.FromResult<XElement>(MakePeloponneseResourceGroup(dfsClient, stagingRoot, ppmHome, logger));
}
}
public static XElement MakeConfigResourceGroup(DfsClient dfsClient, Uri dfsDirectory,
XDocument config, string configName, ILogger logger = null)
{
dfsClient.EnsureDirectory(dfsDirectory, false);
XElement resourceElement = MakeConfigResource(dfsClient, dfsDirectory, config, configName, logger);
XElement groupElement = MakeDfsResourceGroupElement(dfsClient, dfsDirectory, false);
groupElement.Add(resourceElement);
return groupElement;
}
public static XElement MakePeloponneseWorkerResourceGroup(DfsClient dfsClient, Uri stagingRoot, string ppmHome, ILogger logger = null)
{
ConsoleLogger.EnsureLogger(ref logger);
if (!IsValidPpmLocation(ppmHome))
{
throw new ApplicationException("Specified Peloponnese location " + ppmHome + " is missing some required files");
}
IEnumerable<string> ppmResourcePaths = EnumeratePeloponneseWorkerFiles(ppmHome);
return MakeResourceGroup(dfsClient, dfsClient.Combine(stagingRoot, "peloponnese"), true, ppmResourcePaths, logger);
}
public NativeYarnClient(string headNode, int wsPort, DfsClient fsClient)
{
this.rmNode = headNode;
this.wsPort = wsPort;
this.fsClient = fsClient;
this.baseUri = null;
this.launcherNode = null;
this.launcherPort = -1;
this.launcherJarFile = null;
this.yarnDirectory = null;
}
public static Task<XElement> MakeResourceGroupAsync(DfsClient dfsClient, Uri dfsDirectory,
bool isPublic, IEnumerable<string> localFiles,
ILogger logger = null)
{
if (dfsClient.IsThreadSafe)
{
return Task.Run<XElement>(() => MakeResourceGroup(dfsClient, dfsDirectory, isPublic, localFiles, logger));
}
else
{
return Task.FromResult<XElement>(MakeResourceGroup(dfsClient, dfsDirectory, isPublic, localFiles, logger));
}
}
public static XElement MakeDfsResourceGroupElement(DfsClient dfsClient, Uri dfsDirectory, bool isPublic)
{
XElement groupElement = new XElement("ResourceGroup");
groupElement.SetAttributeValue("type", "hdfs");
groupElement.SetAttributeValue("location", dfsClient.GetClusterInternalUri(dfsDirectory));
if (isPublic)
{
groupElement.SetAttributeValue("public", "true");
}
return groupElement;
}
public static XElement MakeResourceGroup(DfsClient dfsClient, Uri dfsDirectory,
bool isPublic, IEnumerable<string> localFiles,
ILogger logger = null)
{
ConsoleLogger.EnsureLogger(ref logger);
dfsClient.EnsureDirectory(dfsDirectory, isPublic);
XElement groupElement = MakeDfsResourceGroupElement(dfsClient, dfsDirectory, isPublic);
if (dfsClient.IsThreadSafe)
{
List<Task<XElement>> waiters = new List<Task<XElement>>();
foreach (string filePath in localFiles)
{
Task<XElement> waiter = Task.Run(() => MakeDfsResourceFromFile(filePath, dfsClient, dfsDirectory, logger));
waiters.Add(waiter);
}
try
{
Task.WaitAll(waiters.ToArray());
}
catch (Exception e)
{
throw new ApplicationException("Dfs resource make failed", e);
}
foreach (Task<XElement> t in waiters)
{
groupElement.Add(t.Result);
}
}
else
{
try
{
foreach (string filePath in localFiles)
{
groupElement.Add(MakeDfsResourceFromFile(filePath, dfsClient, dfsDirectory, logger));
}
}
catch (Exception e)
{
throw new ApplicationException("Dfs resource make failed", e);
}
}
return groupElement;
}
public static XElement MakeDfsResourceFromFile(string localPath, DfsClient dfsClient, Uri dfsDirectory,
ILogger logger = null)
{
ConsoleLogger.EnsureLogger(ref logger);
long modificationTime;
long size;
Uri dfsPath = ResourceFile.UploadResourceIfNeeded(dfsClient, localPath, dfsDirectory,
out modificationTime, out size, logger);
string fileName = Path.GetFileName(localPath);
return MakeResourceElement(fileName, modificationTime, size, dfsClient, dfsPath);
}
public static XElement MakeConfigResource(DfsClient dfsClient, Uri dfsDirectory,
XDocument config, string localName, ILogger logger = null)
{
byte[] configBytes;
using (MemoryStream ms = new MemoryStream())
{
using (XmlWriter writer = XmlWriter.Create(ms))
{
config.WriteTo(writer);
}
configBytes = ms.ToArray();
}
return MakeDfsResourceFromBuffer(localName, configBytes, dfsClient, dfsDirectory, logger);
}
public static XElement MakeDfsResourceFromRemote(DfsClient dfsClient, Uri resource, ILogger logger = null)
{
ConsoleLogger.EnsureLogger(ref logger);
long modificationTime;
long size;
bool b = dfsClient.GetFileStatus(resource, out modificationTime, out size);
if (!b)
{
throw new ApplicationException("Unknown resource " + resource.AbsoluteUri);
}
string fileName = resource.AbsoluteUri.Substring(resource.AbsoluteUri.LastIndexOf('/') + 1);
return MakeResourceElement(fileName, modificationTime, size, dfsClient, resource);
}
public static XElement MakeDfsResourceFromBuffer(string localName, byte[] buffer, DfsClient dfsClient, Uri dfsDirectory,
ILogger logger = null)
{
long modificationTime;
long size;
Uri dfsPath = ResourceFile.UploadResourceIfNeeded(dfsClient, localName, buffer, dfsDirectory,
out modificationTime, out size, logger);
return MakeResourceElement(localName, modificationTime, size, dfsClient, dfsPath);
}