Exemplo n.º 1
0
 /// <summary>
 /// Releases all resources used by the HpcLinqContext.
 /// </summary>
 public void Dispose()
 {
     _configuration = null;
     if (_runtime != null)
     {
         _runtime.Dispose();
         _runtime = null;
     }
     if (_dscService != null)
     {
         _dscService.Close();
         _dscService = null;
     }
 }
Exemplo n.º 2
0
        /// <summary>
        /// Initializes a new instance of the HpcLinqConfiguration class.
        /// </summary>
        /// <param name="configuration">Configuration information.</param>
        /// <remarks>
        /// Connections will be opened to DSC and HPC Server using configuration.HeadNode.
        /// The connections will be opened regardless of whether DSC is used and/or whether
        /// configuration.LocalDebug is true
        /// </remarks>
        public HpcLinqContext(HpcLinqConfiguration configuration)
        {
            // Verify that the head node is set
            if (configuration.HeadNode == null)
            {
                throw new DryadLinqException(HpcLinqErrorCode.ClusterNameMustBeSpecified,
                                             SR.ClusterNameMustBeSpecified);
            }

            _configuration   = configuration.MakeImmutableCopy();
            _runtime         = new HpcQueryRuntime(_configuration.HeadNode);
            _dscService      = new DscService(_configuration.HeadNode);
            _hdfsServiceNode = _configuration.HdfsNameNode;
        }
Exemplo n.º 3
0
        private string BuildExpandedClasspath(HpcLinqConfiguration config)
        {
            String classPathString = System.Environment.GetEnvironmentVariable("classpath");
            //Console.WriteLine(classPathString);

            var fields = classPathString.Split(';');

            StringBuilder sb = new StringBuilder(16384);

            var jarFiles = Directory.GetFiles(config.DryadHomeDirectory, "*.jar", System.IO.SearchOption.TopDirectoryOnly);

            foreach (String file in jarFiles)
            {
                //Console.WriteLine("\t{0}", file);
                sb.Append(file);
                sb.Append(";");
            }

            foreach (String field in fields)
            {
                //Console.WriteLine(field);
                if (!field.EndsWith("*"))
                {
                    sb.Append(field);
                    sb.Append(";");
                }
                else
                {
                    var dirField = field.Substring(0, field.Length - 1); // trim the trailing *
                    jarFiles = Directory.GetFiles(dirField, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
                    foreach (String file in jarFiles)
                    {
                        //Console.WriteLine(file);
                        sb.Append(file);
                        sb.Append(";");
                    }
                }
            }
            return(sb.ToString());
        }
Exemplo n.º 4
0
        internal HpcLinqConfiguration MakeImmutableCopy()
        {
            HpcLinqConfiguration newConfig = new HpcLinqConfiguration();

            newConfig._isReadOnly = true;

            newConfig._jobEnvironmentVariables = this._jobEnvironmentVariables.GetImmutableClone();

            newConfig._resourcesToAdd    = this._resourcesToAdd.GetImmutableClone();
            newConfig._resourcesToRemove = this._resourcesToRemove.GetImmutableClone();

            newConfig._intermediateDataCompressionScheme = this._intermediateDataCompressionScheme;
            newConfig._outputCompressionScheme           = this._outputCompressionScheme;
            newConfig._compileForVertexDebugging         = this._compileForVertexDebugging;
            newConfig._headNode                       = this._headNode;
            newConfig._hdfsNameNode                   = this._hdfsNameNode;
            newConfig._hdfsNameNodeHttpPort           = this._hdfsNameNodeHttpPort;
            newConfig._jobFriendlyName                = this._jobFriendlyName;
            newConfig._jobMinNodes                    = this._jobMinNodes;
            newConfig._jobMaxNodes                    = this._jobMaxNodes;
            newConfig._nodeGroup                      = this._nodeGroup;
            newConfig._jobRuntimeLimit                = this._jobRuntimeLimit;
            newConfig._localDebug                     = this._localDebug;
            newConfig._orderPreserving                = this._orderPreserving;
            newConfig._jobUsername                    = this._jobUsername;
            newConfig._jobPassword                    = this.JobPassword;
            newConfig._runtimeTraceLevel              = this._runtimeTraceLevel;
            newConfig._graphManagerNode               = this._graphManagerNode;
            newConfig._selectAndWherePreserveOrder    = this._selectAndWherePreserveOrder;
            newConfig._matchClientNetFrameworkVersion = this._matchClientNetFrameworkVersion;
            newConfig._enableSpeculativeDuplication   = this._enableSpeculativeDuplication;
            newConfig._multiThreading                 = this._multiThreading;

            newConfig._dryadHome = this._dryadHome;
            newConfig._yarnHome  = this._yarnHome;

            return(newConfig);
        }
Exemplo n.º 5
0
        internal HpcLinqConfiguration MakeImmutableCopy()
        {
            HpcLinqConfiguration newConfig = new HpcLinqConfiguration();

            newConfig._isReadOnly = true;

            newConfig._jobEnvironmentVariables = this._jobEnvironmentVariables.GetImmutableClone();

            newConfig._resourcesToAdd = this._resourcesToAdd.GetImmutableClone();
            newConfig._resourcesToRemove = this._resourcesToRemove.GetImmutableClone();

            newConfig._intermediateDataCompressionScheme = this._intermediateDataCompressionScheme;
            newConfig._outputCompressionScheme = this._outputCompressionScheme;
            newConfig._compileForVertexDebugging = this._compileForVertexDebugging;
            newConfig._headNode = this._headNode;
            newConfig._hdfsNameNode = this._hdfsNameNode;
            newConfig._hdfsNameNodeHttpPort = this._hdfsNameNodeHttpPort;
            newConfig._jobFriendlyName = this._jobFriendlyName;
            newConfig._jobMinNodes = this._jobMinNodes;
            newConfig._jobMaxNodes = this._jobMaxNodes;
            newConfig._nodeGroup = this._nodeGroup;
            newConfig._jobRuntimeLimit = this._jobRuntimeLimit;
            newConfig._localDebug = this._localDebug;
            newConfig._orderPreserving = this._orderPreserving;
            newConfig._jobUsername = this._jobUsername;
            newConfig._jobPassword = this.JobPassword;
            newConfig._runtimeTraceLevel = this._runtimeTraceLevel;
            newConfig._graphManagerNode = this._graphManagerNode;
            newConfig._selectAndWherePreserveOrder = this._selectAndWherePreserveOrder;
            newConfig._matchClientNetFrameworkVersion = this._matchClientNetFrameworkVersion;
            newConfig._enableSpeculativeDuplication = this._enableSpeculativeDuplication;
            newConfig._multiThreading = this._multiThreading;

            newConfig._dryadHome = this._dryadHome;
            newConfig._yarnHome = this._yarnHome;

            return newConfig;
        }
Exemplo n.º 6
0
        private string BuildExpandedClasspath(HpcLinqConfiguration config)
        {
            String classPathString = System.Environment.GetEnvironmentVariable("classpath");
            //Console.WriteLine(classPathString);

            var fields = classPathString.Split(';');

            StringBuilder sb = new StringBuilder(16384);

            var jarFiles = Directory.GetFiles(config.DryadHomeDirectory, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
            foreach (String file in jarFiles)
            {
                //Console.WriteLine("\t{0}", file);
                sb.Append(file);
                sb.Append(";");
            }

            foreach (String field in fields)
            {
                //Console.WriteLine(field);
                if (!field.EndsWith("*"))
                {
                    sb.Append(field);
                    sb.Append(";");
                }
                else
                {
                    var dirField = field.Substring(0, field.Length - 1); // trim the trailing *
                    jarFiles = Directory.GetFiles(dirField, "*.jar", System.IO.SearchOption.TopDirectoryOnly);
                    foreach (String file in jarFiles)
                    {
                        //Console.WriteLine(file);
                        sb.Append(file);
                        sb.Append(";");
                    }
                }
            }
            return sb.ToString();
        }