/// <summary> /// Releases all resources used by the HpcLinqContext. /// </summary> public void Dispose() { _configuration = null; if (_runtime != null) { _runtime.Dispose(); _runtime = null; } if (_dscService != null) { _dscService.Close(); _dscService = null; } }
/// <summary> /// Initializes a new instance of the HpcLinqConfiguration class. /// </summary> /// <param name="configuration">Configuration information.</param> /// <remarks> /// Connections will be opened to DSC and HPC Server using configuration.HeadNode. /// The connections will be opened regardless of whether DSC is used and/or whether /// configuration.LocalDebug is true /// </remarks> public HpcLinqContext(HpcLinqConfiguration configuration) { // Verify that the head node is set if (configuration.HeadNode == null) { throw new DryadLinqException(HpcLinqErrorCode.ClusterNameMustBeSpecified, SR.ClusterNameMustBeSpecified); } _configuration = configuration.MakeImmutableCopy(); _runtime = new HpcQueryRuntime(_configuration.HeadNode); _dscService = new DscService(_configuration.HeadNode); _hdfsServiceNode = _configuration.HdfsNameNode; }
private string BuildExpandedClasspath(HpcLinqConfiguration config) { String classPathString = System.Environment.GetEnvironmentVariable("classpath"); //Console.WriteLine(classPathString); var fields = classPathString.Split(';'); StringBuilder sb = new StringBuilder(16384); var jarFiles = Directory.GetFiles(config.DryadHomeDirectory, "*.jar", System.IO.SearchOption.TopDirectoryOnly); foreach (String file in jarFiles) { //Console.WriteLine("\t{0}", file); sb.Append(file); sb.Append(";"); } foreach (String field in fields) { //Console.WriteLine(field); if (!field.EndsWith("*")) { sb.Append(field); sb.Append(";"); } else { var dirField = field.Substring(0, field.Length - 1); // trim the trailing * jarFiles = Directory.GetFiles(dirField, "*.jar", System.IO.SearchOption.TopDirectoryOnly); foreach (String file in jarFiles) { //Console.WriteLine(file); sb.Append(file); sb.Append(";"); } } } return(sb.ToString()); }
internal HpcLinqConfiguration MakeImmutableCopy() { HpcLinqConfiguration newConfig = new HpcLinqConfiguration(); newConfig._isReadOnly = true; newConfig._jobEnvironmentVariables = this._jobEnvironmentVariables.GetImmutableClone(); newConfig._resourcesToAdd = this._resourcesToAdd.GetImmutableClone(); newConfig._resourcesToRemove = this._resourcesToRemove.GetImmutableClone(); newConfig._intermediateDataCompressionScheme = this._intermediateDataCompressionScheme; newConfig._outputCompressionScheme = this._outputCompressionScheme; newConfig._compileForVertexDebugging = this._compileForVertexDebugging; newConfig._headNode = this._headNode; newConfig._hdfsNameNode = this._hdfsNameNode; newConfig._hdfsNameNodeHttpPort = this._hdfsNameNodeHttpPort; newConfig._jobFriendlyName = this._jobFriendlyName; newConfig._jobMinNodes = this._jobMinNodes; newConfig._jobMaxNodes = this._jobMaxNodes; newConfig._nodeGroup = this._nodeGroup; newConfig._jobRuntimeLimit = this._jobRuntimeLimit; newConfig._localDebug = this._localDebug; newConfig._orderPreserving = this._orderPreserving; newConfig._jobUsername = this._jobUsername; newConfig._jobPassword = this.JobPassword; newConfig._runtimeTraceLevel = this._runtimeTraceLevel; newConfig._graphManagerNode = this._graphManagerNode; newConfig._selectAndWherePreserveOrder = this._selectAndWherePreserveOrder; newConfig._matchClientNetFrameworkVersion = this._matchClientNetFrameworkVersion; newConfig._enableSpeculativeDuplication = this._enableSpeculativeDuplication; newConfig._multiThreading = this._multiThreading; newConfig._dryadHome = this._dryadHome; newConfig._yarnHome = this._yarnHome; return(newConfig); }
internal HpcLinqConfiguration MakeImmutableCopy() { HpcLinqConfiguration newConfig = new HpcLinqConfiguration(); newConfig._isReadOnly = true; newConfig._jobEnvironmentVariables = this._jobEnvironmentVariables.GetImmutableClone(); newConfig._resourcesToAdd = this._resourcesToAdd.GetImmutableClone(); newConfig._resourcesToRemove = this._resourcesToRemove.GetImmutableClone(); newConfig._intermediateDataCompressionScheme = this._intermediateDataCompressionScheme; newConfig._outputCompressionScheme = this._outputCompressionScheme; newConfig._compileForVertexDebugging = this._compileForVertexDebugging; newConfig._headNode = this._headNode; newConfig._hdfsNameNode = this._hdfsNameNode; newConfig._hdfsNameNodeHttpPort = this._hdfsNameNodeHttpPort; newConfig._jobFriendlyName = this._jobFriendlyName; newConfig._jobMinNodes = this._jobMinNodes; newConfig._jobMaxNodes = this._jobMaxNodes; newConfig._nodeGroup = this._nodeGroup; newConfig._jobRuntimeLimit = this._jobRuntimeLimit; newConfig._localDebug = this._localDebug; newConfig._orderPreserving = this._orderPreserving; newConfig._jobUsername = this._jobUsername; newConfig._jobPassword = this.JobPassword; newConfig._runtimeTraceLevel = this._runtimeTraceLevel; newConfig._graphManagerNode = this._graphManagerNode; newConfig._selectAndWherePreserveOrder = this._selectAndWherePreserveOrder; newConfig._matchClientNetFrameworkVersion = this._matchClientNetFrameworkVersion; newConfig._enableSpeculativeDuplication = this._enableSpeculativeDuplication; newConfig._multiThreading = this._multiThreading; newConfig._dryadHome = this._dryadHome; newConfig._yarnHome = this._yarnHome; return newConfig; }
private string BuildExpandedClasspath(HpcLinqConfiguration config) { String classPathString = System.Environment.GetEnvironmentVariable("classpath"); //Console.WriteLine(classPathString); var fields = classPathString.Split(';'); StringBuilder sb = new StringBuilder(16384); var jarFiles = Directory.GetFiles(config.DryadHomeDirectory, "*.jar", System.IO.SearchOption.TopDirectoryOnly); foreach (String file in jarFiles) { //Console.WriteLine("\t{0}", file); sb.Append(file); sb.Append(";"); } foreach (String field in fields) { //Console.WriteLine(field); if (!field.EndsWith("*")) { sb.Append(field); sb.Append(";"); } else { var dirField = field.Substring(0, field.Length - 1); // trim the trailing * jarFiles = Directory.GetFiles(dirField, "*.jar", System.IO.SearchOption.TopDirectoryOnly); foreach (String file in jarFiles) { //Console.WriteLine(file); sb.Append(file); sb.Append(";"); } } } return sb.ToString(); }