private Task AddClusters(ITransaction tx, IReliableDictionary<int, Cluster> dictionary, int count, ClusterStatus status) { return this.AddClusters(tx, dictionary, count, () => this.CreateCluster(status)); }
/// <summary> /// Converts the <see cref="sourceValue" /> parameter to the <see cref="destinationType" /> parameter using <see cref="formatProvider" /// /> and <see cref="ignoreCase" /> /// </summary> /// <param name="sourceValue">the value to convert into an instance of <see cref="ClusterStatus" />.</param> /// <returns> /// an instance of <see cref="ClusterStatus" />, or <c>null</c> if there is no suitable conversion. /// </returns> public static object ConvertFrom(dynamic sourceValue) { if (null == sourceValue) { return(null); } try { ClusterStatus.FromJsonString(typeof(string) == sourceValue.GetType() ? sourceValue : sourceValue.ToJsonString()); } catch { // Unable to use JSON pattern } try { return(new ClusterStatus { ApiVersion = sourceValue.ApiVersion, Code = sourceValue.Code, Kind = sourceValue.Kind, MessageList = sourceValue.MessageList, State = sourceValue.State, }); } catch { } return(null); }
private void AdvanceTerm() { Term++; Status = ClusterStatus.Turmoil; Tock = 0; InboundAssignments.Clear(); }
public void SetStatus() { if (!Nodes.Any()) { Status = ClusterStatus.NoThresholdsSet; return; } Status = (ClusterStatus)Nodes.SelectMany(n => n.Metrics).Select(m => m.Status).Max(); }
/// <summary>This is the main routine for launching a distributed random write job.</summary> /// <remarks> /// This is the main routine for launching a distributed random write job. /// It runs 10 maps/node and each node writes 1 gig of data to a DFS file. /// The reduce doesn't do anything. /// </remarks> /// <exception cref="System.IO.IOException"></exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { if (args.Length == 0) { System.Console.Out.WriteLine("Usage: writer <out-dir>"); ToolRunner.PrintGenericCommandUsage(System.Console.Out); return(2); } Path outDir = new Path(args[0]); Configuration conf = GetConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.GetClusterStatus(); int numMapsPerHost = conf.GetInt(MapsPerHost, 10); long numBytesToWritePerMap = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024); if (numBytesToWritePerMap == 0) { System.Console.Error.WriteLine("Cannot have" + BytesPerMap + " set to 0"); return(-2); } long totalBytesToWrite = conf.GetLong(TotalBytes, numMapsPerHost * numBytesToWritePerMap * cluster.GetTaskTrackers()); int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap); if (numMaps == 0 && totalBytesToWrite > 0) { numMaps = 1; conf.SetLong(BytesPerMap, totalBytesToWrite); } conf.SetInt(MRJobConfig.NumMaps, numMaps); Job job = Job.GetInstance(conf); job.SetJarByClass(typeof(RandomWriter)); job.SetJobName("random-writer"); FileOutputFormat.SetOutputPath(job, outDir); job.SetOutputKeyClass(typeof(BytesWritable)); job.SetOutputValueClass(typeof(BytesWritable)); job.SetInputFormatClass(typeof(RandomWriter.RandomInputFormat)); job.SetMapperClass(typeof(RandomWriter.RandomMapper)); job.SetReducerClass(typeof(Reducer)); job.SetOutputFormatClass(typeof(SequenceFileOutputFormat)); System.Console.Out.WriteLine("Running " + numMaps + " maps."); // reducer NONE job.SetNumReduceTasks(0); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); int ret = job.WaitForCompletion(true) ? 0 : 1; DateTime endTime = new DateTime(); System.Console.Out.WriteLine("Job ended: " + endTime); System.Console.Out.WriteLine("The job took " + (endTime.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(ret); }
/// <summary> /// Create a new instance copied from the given Cluster with the given status. /// </summary> /// <param name="status"></param> /// <param name="copyFrom"></param> public Cluster(ClusterStatus status, Cluster copyFrom) : this(copyFrom.InternalName, status, copyFrom.AppCount, copyFrom.ServiceCount, copyFrom.Address, new List <int>(copyFrom.Ports), new List <ClusterUser>(copyFrom.Users), copyFrom.CreatedOn) { }
public Cluster(ClusterStatus status, Cluster copyFrom) : this(copyFrom.InternalName, status, copyFrom.AppCount, copyFrom.ServiceCount, copyFrom.Address, new List<int>(copyFrom.Ports), new List<ClusterUser>(copyFrom.Users), copyFrom.CreatedOn) { }
public void TestClusterStatusRefresh() { ClusterStatus clusterStatus = new ClusterStatus("er-clust1-c07"); clusterStatus.Cluster = "er-clust1-c07"; bool ActualResult = clusterStatus.Refresh(); string ActualString = clusterStatus.ToString(); string ExpectedString = "er-clust1-c07: -1/-2"; bool expectedResult = true; Assert.AreEqual(expectedResult, ActualResult); Assert.AreEqual(ExpectedString, ActualString); }
public void TestClusterStatusRefresh() { ClusterStatus clusterStatus = new ClusterStatus("er-clust1-c07") { Cluster = "er-clust1-c07" }; bool actualResult = clusterStatus.Refresh(); Assert.AreEqual(true, actualResult); const string expectedString = "er-clust1-c07: -1/-2"; string actualString = clusterStatus.ToString(); Assert.AreEqual(expectedString, actualString); }
public void ClusterStatus_Serialize_ClusterStatus() { Dictionary <string, string> clusterProps = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); EnhancedMemoryStream ms = new EnhancedMemoryStream(); ClusterMemberSettings settings = new ClusterMemberSettings((MsgEP)"logical://test"); ClusterMemberStatus memberStatus; ClusterStatus status; status = new ClusterStatus((MsgEP)"logical://test/master"); status.LoadProperties(clusterProps); status.MasterProtocolCaps = unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF); memberStatus = new ClusterMemberStatus("logical://test/master", ClusterMemberState.Master, settings); memberStatus["setting1"] = "value1"; memberStatus["setting2"] = "value2"; status.Members.Add(memberStatus); memberStatus = new ClusterMemberStatus("logical://test/bar", ClusterMemberState.Slave, settings); memberStatus["setting3"] = "value3"; memberStatus["setting4"] = "value4"; status.Members.Add(memberStatus); status.Write(ms); ms.Position = 0; status = new ClusterStatus(ms); Assert.AreEqual((MsgEP)"logical://test/master", status.MasterEP); Assert.AreEqual(Helper.MachineName, status.MasterMachine); Assert.AreEqual(unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF), status.MasterProtocolCaps); Assert.AreEqual(2, status.Members.Count); memberStatus = status.Members[0]; Assert.AreEqual((MsgEP)"logical://test/master", memberStatus.InstanceEP); Assert.AreEqual(Helper.MachineName, memberStatus.MachineName); Assert.AreEqual(ClusterMemberState.Master, memberStatus.State); Assert.AreEqual(settings, memberStatus.Settings); Assert.AreEqual("value1", memberStatus["setting1"]); Assert.AreEqual("value2", memberStatus["setting2"]); memberStatus = status.Members[1]; Assert.AreEqual((MsgEP)"logical://test/bar", memberStatus.InstanceEP); Assert.AreEqual(Helper.MachineName, memberStatus.MachineName); Assert.AreEqual(ClusterMemberState.Slave, memberStatus.State); Assert.AreEqual(settings, memberStatus.Settings); Assert.AreEqual("value3", memberStatus["setting3"]); Assert.AreEqual("value4", memberStatus["setting4"]); }
public void ClusterStatus_Clone_ClusterStatus() { ClusterMemberSettings settings = new ClusterMemberSettings((MsgEP)"logical://test"); ClusterMemberStatus memberStatus; ClusterStatus input, output; input = new ClusterStatus((MsgEP)"logical://test/master"); input.ClusterTime = DateTime.UtcNow; input.MasterProtocolCaps = unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF); memberStatus = new ClusterMemberStatus("logical://test/master", ClusterMemberState.Master, settings); memberStatus["setting1"] = "value1"; memberStatus["setting2"] = "value2"; input.Members.Add(memberStatus); memberStatus = new ClusterMemberStatus("logical://test/bar", ClusterMemberState.Slave, settings); memberStatus["setting3"] = "value3"; memberStatus["setting4"] = "value4"; input.Members.Add(memberStatus); output = input.Clone(); Assert.AreNotSame(input, output); Assert.AreEqual((MsgEP)"logical://test/master", output.MasterEP); Assert.AreEqual(Helper.MachineName, output.MasterMachine); Assert.AreEqual(input.ClusterTime, output.ClusterTime); Assert.AreEqual(input.ReceiveTime, output.ReceiveTime); Assert.AreEqual(unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF), output.MasterProtocolCaps); Assert.AreEqual(2, output.Members.Count); memberStatus = output.Members[0]; Assert.AreEqual((MsgEP)"logical://test/master", memberStatus.InstanceEP); Assert.AreEqual(Helper.MachineName, memberStatus.MachineName); Assert.AreEqual(ClusterMemberState.Master, memberStatus.State); Assert.AreEqual(settings, memberStatus.Settings); Assert.AreEqual("value1", memberStatus["setting1"]); Assert.AreEqual("value2", memberStatus["setting2"]); memberStatus = output.Members[1]; Assert.AreEqual((MsgEP)"logical://test/bar", memberStatus.InstanceEP); Assert.AreEqual(Helper.MachineName, memberStatus.MachineName); Assert.AreEqual(ClusterMemberState.Slave, memberStatus.State); Assert.AreEqual(settings, memberStatus.Settings); Assert.AreEqual("value3", memberStatus["setting3"]); Assert.AreEqual("value4", memberStatus["setting4"]); }
public void CommandFollowers(List <Followership> followers) { Tock++; followers = followers .Where(IsObedientFollower) .OrderBy(x => x.FollowerId) .ToList(); var differences = new HashSet <string>(followers.Select(x => x.FollowerId)); differences.SymmetricExceptWith(InboundAssignments.Select(x => x.FollowerId)); if (differences.Any() && Status == ClusterStatus.Harmony) { AdvanceTerm(); } if (!followers.Any()) { return; } if (differences.Any()) { var range = byte.MaxValue / followers.Count; //match MaxValue to TransportMessage ClaimTicket var remainder = byte.MaxValue - (range * followers.Count); InboundAssignments = followers .Select((follower, i) => new FollowerAssignment { FollowerId = follower.FollowerId, LowerBound = i * range + (i == 0 ? 0 : 1), UpperBound = (i * range) + range + (i == followers.Count - 1 ? remainder : 0) }) .ToList(); OutboundAssignment = followers.First().FollowerId; } Status = followers.All(x => x.Term == Term) ? ClusterStatus.Harmony : ClusterStatus.Turmoil; }
/// <summary> /// Translates a Cluster Status Value into a character /// </summary> /// <param name="C">Cluster Status</param> /// <returns>Character</returns> private static char ToConsoleChar(ClusterStatus C) { switch (C) { case ClusterStatus.Damaged: return('B'); case ClusterStatus.EOF: return('░'); case ClusterStatus.Occupied: return('█'); case ClusterStatus.Reserved: return('R'); } return('.'); }
/// <summary> /// Translates a Cluster Status Value into a Console color /// </summary> /// <param name="C">Cluster Status</param> /// <returns>Console Color Value</returns> private static ConsoleColor ToConsoleColor(ClusterStatus C) { switch (C) { case ClusterStatus.Damaged: return(ConsoleColor.Red); case ClusterStatus.EOF: return(ConsoleColor.Green); case ClusterStatus.Occupied: return(ConsoleColor.Blue); case ClusterStatus.Reserved: return(ConsoleColor.DarkCyan); } return(ConsoleColor.White); }
/// <summary> /// Translates a Cluster Status Value into an RGB bitmap Color /// </summary> /// <param name="C">Cluster Status</param> /// <returns>Bitmap Color Value</returns> private static Color ToBitmapColor(ClusterStatus C) { switch (C) { case ClusterStatus.Damaged: return(Color.Red); case ClusterStatus.EOF: return(Color.Green); case ClusterStatus.Occupied: return(Color.Blue); case ClusterStatus.Reserved: return(Color.Purple); } return(Color.White); }
public Cluster( string internalName, ClusterStatus status, int appCount, int serviceCount, string address, IEnumerable<int> ports, IEnumerable<ClusterUser> users, DateTimeOffset createdOn) { this.InternalName = internalName; this.Status = status; this.AppCount = appCount; this.ServiceCount = serviceCount; this.Address = address; this.Ports = ports; this.Users = users; this.CreatedOn = createdOn; }
/// <summary> /// Query the cluster again for the list of jobs. /// </summary> /// <returns>True if the cluster could be found.</returns> private void RefreshClusterJobList() { if (this.cluster == null) { return; } string clus = this.cluster; if (!string.IsNullOrEmpty(this.SelectedVirtualCluster)) { clus += "/" + this.SelectedVirtualCluster; } this.Status("Querying cluster " + clus, StatusKind.LongOp); try { this.openFromURLToolStripMenuItem.Visible = false; if ( this.Configuration is CacheClusterConfiguration) { this.filteredDataGridView.DataGridView.Columns["VirtualCluster"].Visible = true; } else { this.filteredDataGridView.DataGridView.Columns["VirtualCluster"].Visible = false; } this.clusterStatus = this.Configuration.CreateClusterStatus(); this.toolStripMenuItem_job.Enabled = true; var item = new BackgroundWorkItem <List <ClusterJobInformation> >( m => BuildClusterJobList(m, this.clusterStatus, this.SelectedVirtualCluster), this.JobListRetrieved, "getJobs"); this.Queue(item); } catch (Exception ex) { this.Status("Cannot retrieve information from cluster " + cluster + ": " + ex.Message, StatusKind.Error); Trace.TraceInformation(ex.ToString()); this.comboBox_virtualCluster.Text = ""; } }
/// <summary> /// Creates a new instance. /// </summary> /// <param name="internalName"></param> /// <param name="status"></param> /// <param name="appCount"></param> /// <param name="serviceCount"></param> /// <param name="address"></param> /// <param name="ports"></param> /// <param name="users"></param> /// <param name="createdOn"></param> public Cluster( string internalName, ClusterStatus status, int appCount, int serviceCount, string address, IEnumerable <int> ports, IEnumerable <ClusterUser> users, DateTimeOffset createdOn) { this.InternalName = internalName; this.Status = status; this.AppCount = appCount; this.ServiceCount = serviceCount; this.Address = address; this.Ports = ports; this.Users = users; this.CreatedOn = createdOn; }
/// <summary> /// Cancel a job. /// </summary> /// <param name="jobs">Jobs to cancel.</param> /// <param name="cluster">Cluster where the jobs are running.</param> /// <returns>True if all cancellations succeed.</returns> /// <param name="manager">Communicatoni manager.</param> // ReSharper disable once UnusedParameter.Global public static bool CancelJobs(IEnumerable <DryadLinqJobSummary> jobs, ClusterStatus cluster, CommManager manager) { bool done = true; foreach (DryadLinqJobSummary job in jobs) { manager.Token.ThrowIfCancellationRequested(); if (job.Status != ClusterJobInformation.ClusterJobStatus.Running) { manager.Status("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error); } bool success; string reason = ""; try { success = cluster.CancelJob(job); } catch (Exception ex) { success = false; reason = ex.Message; Trace.TraceInformation(ex.ToString()); } if (success) { manager.Status("Job " + job.Name + " cancelled", StatusKind.OK); } else { manager.Status("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error); } done &= success; } return(done); }
public ESEndpoint(ImportEngine engine, XmlNode node) : base(engine, node) { Connection = new ESConnection(node.ReadStr("@url")); CacheSize = node.ReadInt("@cache", -1); MaxParallel = node.ReadInt("@maxparallel", 0); ReadOnly = node.ReadBool("@readonly", false); XmlNode root = node.SelectSingleNode("indexes"); if (root == null) { root = node; } Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig); if (Indexes.Count == 0) { throw new BMNodeException(node, "At least 1 index+type is required!"); } WaitFor = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt); WaitForTimeout = node.ReadInt("waitfor/@timeout", 30); WaitForMustExcept = node.ReadBool("waitfor/@except", true); }
public void SetStatus_Sets_Appropriate_Status( MetricStatus status1, MetricStatus status2, MetricStatus status3, MetricStatus status4, ClusterStatus expectedStatus) { var cluster = new Cluster { Nodes = new List <ClusterNode> { new ClusterNode { Metrics = new List <Metric> { new Metric { Status = status1 }, new Metric { Status = status2 }, new Metric { Status = status3 }, new Metric { Status = status4 }, } } } }; cluster.SetStatus(); cluster.Status.Should().Be(expectedStatus); }
private Task AddClusters(ITransaction tx, IReliableDictionary <int, Cluster> dictionary, int count, ClusterStatus status) { return(this.AddClusters(tx, dictionary, count, () => new Cluster() { Status = status })); }
private Cluster CreateCluster(ClusterStatus status) { return new Cluster( status, new Cluster("test")); }
private Cluster CreateCluster(ClusterStatus status, IEnumerable<ClusterUser> users) { return new Cluster( "test", status, 0, 0, String.Empty, new int[0], users, DateTimeOffset.MaxValue); }
/// <summary> /// Talk to the web server and build the list of clustr jobs; used it to populate the upper panel. /// </summary> /// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param> /// <param name="manager">Communication manager.</param> /// <param name="status">Cluster to scan.</param> private static List <ClusterJobInformation> BuildClusterJobList(CommManager manager, ClusterStatus status, string virtualCluster) { return(status.GetClusterJobList(virtualCluster, manager).ToList()); }
/// <summary> /// Creates a new Cluster Entry from the given Value /// </summary> /// <param name="Raw">Value</param> public ClusterEntry(ushort Raw) { RawValue = Raw; Status = GetFat12Status(Raw); }
/// <summary>The main driver for sort program.</summary> /// <remarks> /// The main driver for sort program. /// Invoke this method to submit the map/reduce job. /// </remarks> /// <exception cref="System.IO.IOException"> /// When there is communication problems with the /// job tracker. /// </exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { Configuration conf = GetConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.GetClusterStatus(); int num_reduces = (int)(cluster.GetMaxReduceTasks() * 0.9); string sort_reduces = conf.Get(ReducesPerHost); if (sort_reduces != null) { num_reduces = cluster.GetTaskTrackers() * System.Convert.ToInt32(sort_reduces); } Type inputFormatClass = typeof(SequenceFileInputFormat); Type outputFormatClass = typeof(SequenceFileOutputFormat); Type outputKeyClass = typeof(BytesWritable); Type outputValueClass = typeof(BytesWritable); IList <string> otherArgs = new AList <string>(); InputSampler.Sampler <K, V> sampler = null; for (int i = 0; i < args.Length; ++i) { try { if ("-r".Equals(args[i])) { num_reduces = System.Convert.ToInt32(args[++i]); } else { if ("-inFormat".Equals(args[i])) { inputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <InputFormat>(); } else { if ("-outFormat".Equals(args[i])) { outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>(); } else { if ("-outKey".Equals(args[i])) { outputKeyClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <WritableComparable >(); } else { if ("-outValue".Equals(args[i])) { outputValueClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <Writable>(); } else { if ("-totalOrder".Equals(args[i])) { double pcnt = double.ParseDouble(args[++i]); int numSamples = System.Convert.ToInt32(args[++i]); int maxSplits = System.Convert.ToInt32(args[++i]); if (0 >= maxSplits) { maxSplits = int.MaxValue; } sampler = new InputSampler.RandomSampler <K, V>(pcnt, numSamples, maxSplits); } else { otherArgs.AddItem(args[i]); } } } } } } } catch (FormatException) { System.Console.Out.WriteLine("ERROR: Integer expected instead of " + args[i]); return(PrintUsage()); } catch (IndexOutOfRangeException) { System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i - 1]); return(PrintUsage()); } } // exits // Set user-supplied (possibly default) job configs job = Job.GetInstance(conf); job.SetJobName("sorter"); job.SetJarByClass(typeof(Sort)); job.SetMapperClass(typeof(Mapper)); job.SetReducerClass(typeof(Reducer)); job.SetNumReduceTasks(num_reduces); job.SetInputFormatClass(inputFormatClass); job.SetOutputFormatClass(outputFormatClass); job.SetOutputKeyClass(outputKeyClass); job.SetOutputValueClass(outputValueClass); // Make sure there are exactly 2 parameters left. if (otherArgs.Count != 2) { System.Console.Out.WriteLine("ERROR: Wrong number of parameters: " + otherArgs.Count + " instead of 2."); return(PrintUsage()); } FileInputFormat.SetInputPaths(job, otherArgs[0]); FileOutputFormat.SetOutputPath(job, new Path(otherArgs[1])); if (sampler != null) { System.Console.Out.WriteLine("Sampling input to effect total-order sort..."); job.SetPartitionerClass(typeof(TotalOrderPartitioner)); Path inputDir = FileInputFormat.GetInputPaths(job)[0]; inputDir = inputDir.MakeQualified(inputDir.GetFileSystem(conf)); Path partitionFile = new Path(inputDir, "_sortPartitioning"); TotalOrderPartitioner.SetPartitionFile(conf, partitionFile); InputSampler.WritePartitionFile <K, V>(job, sampler); URI partitionUri = new URI(partitionFile.ToString() + "#" + "_sortPartitioning"); DistributedCache.AddCacheFile(partitionUri, conf); } System.Console.Out.WriteLine("Running on " + cluster.GetTaskTrackers() + " nodes to sort from " + FileInputFormat.GetInputPaths(job)[0] + " into " + FileOutputFormat.GetOutputPath (job) + " with " + num_reduces + " reduces."); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); int ret = job.WaitForCompletion(true) ? 0 : 1; DateTime end_time = new DateTime(); System.Console.Out.WriteLine("Job ended: " + end_time); System.Console.Out.WriteLine("The job took " + (end_time.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(ret); }
private Task AddClusters(ITransaction tx, IReliableDictionary<int, Cluster> dictionary, int count, ClusterStatus status) { return this.AddClusters(tx, dictionary, count, () => new Cluster() { Status = status }); }
/// <summary>This is the main routine for launching a distributed random write job.</summary> /// <remarks> /// This is the main routine for launching a distributed random write job. /// It runs 10 maps/node and each node writes 1 gig of data to a DFS file. /// The reduce doesn't do anything. /// </remarks> /// <exception cref="System.IO.IOException"></exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { if (args.Length == 0) { return(PrintUsage()); } Configuration conf = GetConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.GetClusterStatus(); int numMapsPerHost = conf.GetInt(MapsPerHost, 10); long numBytesToWritePerMap = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024); if (numBytesToWritePerMap == 0) { System.Console.Error.WriteLine("Cannot have " + BytesPerMap + " set to 0"); return(-2); } long totalBytesToWrite = conf.GetLong(TotalBytes, numMapsPerHost * numBytesToWritePerMap * cluster.GetTaskTrackers()); int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap); if (numMaps == 0 && totalBytesToWrite > 0) { numMaps = 1; conf.SetLong(BytesPerMap, totalBytesToWrite); } conf.SetInt(MRJobConfig.NumMaps, numMaps); Job job = Job.GetInstance(conf); job.SetJarByClass(typeof(RandomTextWriter)); job.SetJobName("random-text-writer"); job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text)); job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text)); job.SetInputFormatClass(typeof(RandomWriter.RandomInputFormat)); job.SetMapperClass(typeof(RandomTextWriter.RandomTextMapper)); Type outputFormatClass = typeof(SequenceFileOutputFormat); IList <string> otherArgs = new AList <string>(); for (int i = 0; i < args.Length; ++i) { try { if ("-outFormat".Equals(args[i])) { outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>(); } else { otherArgs.AddItem(args[i]); } } catch (IndexOutOfRangeException) { System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i - 1]); return(PrintUsage()); } } // exits job.SetOutputFormatClass(outputFormatClass); FileOutputFormat.SetOutputPath(job, new Path(otherArgs[0])); System.Console.Out.WriteLine("Running " + numMaps + " maps."); // reducer NONE job.SetNumReduceTasks(0); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); int ret = job.WaitForCompletion(true) ? 0 : 1; DateTime endTime = new DateTime(); System.Console.Out.WriteLine("Job ended: " + endTime); System.Console.Out.WriteLine("The job took " + (endTime.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(ret); }
/// <summary>The main driver for sort program.</summary> /// <remarks> /// The main driver for sort program. /// Invoke this method to submit the map/reduce job. /// </remarks> /// <exception cref="System.IO.IOException"> /// When there is communication problems with the /// job tracker. /// </exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { Configuration conf = GetConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.GetClusterStatus(); int num_reduces = (int)(cluster.GetMaxReduceTasks() * 0.9); string join_reduces = conf.Get(ReducesPerHost); if (join_reduces != null) { num_reduces = cluster.GetTaskTrackers() * System.Convert.ToInt32(join_reduces); } Job job = Job.GetInstance(conf); job.SetJobName("join"); job.SetJarByClass(typeof(Sort)); job.SetMapperClass(typeof(Mapper)); job.SetReducerClass(typeof(Reducer)); Type inputFormatClass = typeof(SequenceFileInputFormat); Type outputFormatClass = typeof(SequenceFileOutputFormat); Type outputKeyClass = typeof(BytesWritable); Type outputValueClass = typeof(TupleWritable); string op = "inner"; IList <string> otherArgs = new AList <string>(); for (int i = 0; i < args.Length; ++i) { try { if ("-r".Equals(args[i])) { num_reduces = System.Convert.ToInt32(args[++i]); } else { if ("-inFormat".Equals(args[i])) { inputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <InputFormat>(); } else { if ("-outFormat".Equals(args[i])) { outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>(); } else { if ("-outKey".Equals(args[i])) { outputKeyClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <WritableComparable >(); } else { if ("-outValue".Equals(args[i])) { outputValueClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <Writable>(); } else { if ("-joinOp".Equals(args[i])) { op = args[++i]; } else { otherArgs.AddItem(args[i]); } } } } } } } catch (FormatException) { System.Console.Out.WriteLine("ERROR: Integer expected instead of " + args[i]); return(PrintUsage()); } catch (IndexOutOfRangeException) { System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i - 1]); return(PrintUsage()); } } // exits // Set user-supplied (possibly default) job configs job.SetNumReduceTasks(num_reduces); if (otherArgs.Count < 2) { System.Console.Out.WriteLine("ERROR: Wrong number of parameters: "); return(PrintUsage()); } FileOutputFormat.SetOutputPath(job, new Path(otherArgs.Remove(otherArgs.Count - 1 ))); IList <Path> plist = new AList <Path>(otherArgs.Count); foreach (string s in otherArgs) { plist.AddItem(new Path(s)); } job.SetInputFormatClass(typeof(CompositeInputFormat)); job.GetConfiguration().Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose (op, inputFormatClass, Sharpen.Collections.ToArray(plist, new Path[0]))); job.SetOutputFormatClass(outputFormatClass); job.SetOutputKeyClass(outputKeyClass); job.SetOutputValueClass(outputValueClass); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); int ret = job.WaitForCompletion(true) ? 0 : 1; DateTime end_time = new DateTime(); System.Console.Out.WriteLine("Job ended: " + end_time); System.Console.Out.WriteLine("The job took " + (end_time.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(ret); }