/// <summary>
/// Releases all resources used by the HpcLinqContext.
/// </summary>
public void Dispose()
{
_configuration = null;
if (_runtime != null)
{
_runtime.Dispose();
_runtime = null;
}
if (_dscService != null)
{
_dscService.Close();
_dscService = null;
}
}
/// <summary>
/// Initializes a new instance of the HpcLinqConfiguration class.
/// </summary>
/// <param name="configuration">Configuration information.</param>
/// <remarks>
/// Connections will be opened to DSC and HPC Server using configuration.HeadNode.
/// The connections will be opened regardless of whether DSC is used and/or whether
/// configuration.LocalDebug is true
/// </remarks>
public HpcLinqContext(HpcLinqConfiguration configuration)
{
// Verify that the head node is set
if (configuration.HeadNode == null)
{
throw new DryadLinqException(HpcLinqErrorCode.ClusterNameMustBeSpecified,
SR.ClusterNameMustBeSpecified);
}
_configuration = configuration.MakeImmutableCopy();
_runtime = new HpcQueryRuntime(_configuration.HeadNode);
_dscService = new DscService(_configuration.HeadNode);
_hdfsServiceNode = _configuration.HdfsNameNode;
}
private string BuildExpandedClasspath(HpcLinqConfiguration config)
{
String classPathString = System.Environment.GetEnvironmentVariable("classpath");
//Console.WriteLine(classPathString);
var fields = classPathString.Split(';');
StringBuilder sb = new StringBuilder(16384);
var jarFiles = Directory.GetFiles(config.DryadHomeDirectory, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
foreach (String file in jarFiles)
{
//Console.WriteLine("\t{0}", file);
sb.Append(file);
sb.Append(";");
}
foreach (String field in fields)
{
//Console.WriteLine(field);
if (!field.EndsWith("*"))
{
sb.Append(field);
sb.Append(";");
}
else
{
var dirField = field.Substring(0, field.Length - 1); // trim the trailing *
jarFiles = Directory.GetFiles(dirField, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
foreach (String file in jarFiles)
{
//Console.WriteLine(file);
sb.Append(file);
sb.Append(";");
}
}
}
return(sb.ToString());
}
internal HpcLinqConfiguration MakeImmutableCopy()
{
HpcLinqConfiguration newConfig = new HpcLinqConfiguration();
newConfig._isReadOnly = true;
newConfig._jobEnvironmentVariables = this._jobEnvironmentVariables.GetImmutableClone();
newConfig._resourcesToAdd = this._resourcesToAdd.GetImmutableClone();
newConfig._resourcesToRemove = this._resourcesToRemove.GetImmutableClone();
newConfig._intermediateDataCompressionScheme = this._intermediateDataCompressionScheme;
newConfig._outputCompressionScheme = this._outputCompressionScheme;
newConfig._compileForVertexDebugging = this._compileForVertexDebugging;
newConfig._headNode = this._headNode;
newConfig._hdfsNameNode = this._hdfsNameNode;
newConfig._hdfsNameNodeHttpPort = this._hdfsNameNodeHttpPort;
newConfig._jobFriendlyName = this._jobFriendlyName;
newConfig._jobMinNodes = this._jobMinNodes;
newConfig._jobMaxNodes = this._jobMaxNodes;
newConfig._nodeGroup = this._nodeGroup;
newConfig._jobRuntimeLimit = this._jobRuntimeLimit;
newConfig._localDebug = this._localDebug;
newConfig._orderPreserving = this._orderPreserving;
newConfig._jobUsername = this._jobUsername;
newConfig._jobPassword = this.JobPassword;
newConfig._runtimeTraceLevel = this._runtimeTraceLevel;
newConfig._graphManagerNode = this._graphManagerNode;
newConfig._selectAndWherePreserveOrder = this._selectAndWherePreserveOrder;
newConfig._matchClientNetFrameworkVersion = this._matchClientNetFrameworkVersion;
newConfig._enableSpeculativeDuplication = this._enableSpeculativeDuplication;
newConfig._multiThreading = this._multiThreading;
newConfig._dryadHome = this._dryadHome;
newConfig._yarnHome = this._yarnHome;
return(newConfig);
}
internal HpcLinqConfiguration MakeImmutableCopy()
{
HpcLinqConfiguration newConfig = new HpcLinqConfiguration();
newConfig._isReadOnly = true;
newConfig._jobEnvironmentVariables = this._jobEnvironmentVariables.GetImmutableClone();
newConfig._resourcesToAdd = this._resourcesToAdd.GetImmutableClone();
newConfig._resourcesToRemove = this._resourcesToRemove.GetImmutableClone();
newConfig._intermediateDataCompressionScheme = this._intermediateDataCompressionScheme;
newConfig._outputCompressionScheme = this._outputCompressionScheme;
newConfig._compileForVertexDebugging = this._compileForVertexDebugging;
newConfig._headNode = this._headNode;
newConfig._hdfsNameNode = this._hdfsNameNode;
newConfig._hdfsNameNodeHttpPort = this._hdfsNameNodeHttpPort;
newConfig._jobFriendlyName = this._jobFriendlyName;
newConfig._jobMinNodes = this._jobMinNodes;
newConfig._jobMaxNodes = this._jobMaxNodes;
newConfig._nodeGroup = this._nodeGroup;
newConfig._jobRuntimeLimit = this._jobRuntimeLimit;
newConfig._localDebug = this._localDebug;
newConfig._orderPreserving = this._orderPreserving;
newConfig._jobUsername = this._jobUsername;
newConfig._jobPassword = this.JobPassword;
newConfig._runtimeTraceLevel = this._runtimeTraceLevel;
newConfig._graphManagerNode = this._graphManagerNode;
newConfig._selectAndWherePreserveOrder = this._selectAndWherePreserveOrder;
newConfig._matchClientNetFrameworkVersion = this._matchClientNetFrameworkVersion;
newConfig._enableSpeculativeDuplication = this._enableSpeculativeDuplication;
newConfig._multiThreading = this._multiThreading;
newConfig._dryadHome = this._dryadHome;
newConfig._yarnHome = this._yarnHome;
return newConfig;
}
private string BuildExpandedClasspath(HpcLinqConfiguration config)
{
String classPathString = System.Environment.GetEnvironmentVariable("classpath");
//Console.WriteLine(classPathString);
var fields = classPathString.Split(';');
StringBuilder sb = new StringBuilder(16384);
var jarFiles = Directory.GetFiles(config.DryadHomeDirectory, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
foreach (String file in jarFiles)
{
//Console.WriteLine("\t{0}", file);
sb.Append(file);
sb.Append(";");
}
foreach (String field in fields)
{
//Console.WriteLine(field);
if (!field.EndsWith("*"))
{
sb.Append(field);
sb.Append(";");
}
else
{
var dirField = field.Substring(0, field.Length - 1); // trim the trailing *
jarFiles = Directory.GetFiles(dirField, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
foreach (String file in jarFiles)
{
//Console.WriteLine(file);
sb.Append(file);
sb.Append(";");
}
}
}
return sb.ToString();
}