private Task AddClusters(ITransaction tx, IReliableDictionary<int, Cluster> dictionary, int count, ClusterStatus status)
{
return this.AddClusters(tx, dictionary, count, () => this.CreateCluster(status));
}
/// <summary>
/// Converts the <see cref="sourceValue" /> parameter to the <see cref="destinationType" /> parameter using <see cref="formatProvider"
/// /> and <see cref="ignoreCase" />
/// </summary>
/// <param name="sourceValue">the value to convert into an instance of <see cref="ClusterStatus" />.</param>
/// <returns>
/// an instance of <see cref="ClusterStatus" />, or <c>null</c> if there is no suitable conversion.
/// </returns>
public static object ConvertFrom(dynamic sourceValue)
{
if (null == sourceValue)
{
return(null);
}
try
{
ClusterStatus.FromJsonString(typeof(string) == sourceValue.GetType() ? sourceValue : sourceValue.ToJsonString());
}
catch
{
// Unable to use JSON pattern
}
try
{
return(new ClusterStatus
{
ApiVersion = sourceValue.ApiVersion,
Code = sourceValue.Code,
Kind = sourceValue.Kind,
MessageList = sourceValue.MessageList,
State = sourceValue.State,
});
}
catch
{
}
return(null);
}
private void AdvanceTerm()
{
Term++;
Status = ClusterStatus.Turmoil;
Tock = 0;
InboundAssignments.Clear();
}
public void SetStatus()
{
if (!Nodes.Any())
{
Status = ClusterStatus.NoThresholdsSet;
return;
}
Status = (ClusterStatus)Nodes.SelectMany(n => n.Metrics).Select(m => m.Status).Max();
}
/// <summary>This is the main routine for launching a distributed random write job.</summary>
/// <remarks>
/// This is the main routine for launching a distributed random write job.
/// It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
/// The reduce doesn't do anything.
/// </remarks>
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
if (args.Length == 0)
{
System.Console.Out.WriteLine("Usage: writer <out-dir>");
ToolRunner.PrintGenericCommandUsage(System.Console.Out);
return(2);
}
Path outDir = new Path(args[0]);
Configuration conf = GetConf();
JobClient client = new JobClient(conf);
ClusterStatus cluster = client.GetClusterStatus();
int numMapsPerHost = conf.GetInt(MapsPerHost, 10);
long numBytesToWritePerMap = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
if (numBytesToWritePerMap == 0)
{
System.Console.Error.WriteLine("Cannot have" + BytesPerMap + " set to 0");
return(-2);
}
long totalBytesToWrite = conf.GetLong(TotalBytes, numMapsPerHost * numBytesToWritePerMap
* cluster.GetTaskTrackers());
int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
if (numMaps == 0 && totalBytesToWrite > 0)
{
numMaps = 1;
conf.SetLong(BytesPerMap, totalBytesToWrite);
}
conf.SetInt(MRJobConfig.NumMaps, numMaps);
Job job = Job.GetInstance(conf);
job.SetJarByClass(typeof(RandomWriter));
job.SetJobName("random-writer");
FileOutputFormat.SetOutputPath(job, outDir);
job.SetOutputKeyClass(typeof(BytesWritable));
job.SetOutputValueClass(typeof(BytesWritable));
job.SetInputFormatClass(typeof(RandomWriter.RandomInputFormat));
job.SetMapperClass(typeof(RandomWriter.RandomMapper));
job.SetReducerClass(typeof(Reducer));
job.SetOutputFormatClass(typeof(SequenceFileOutputFormat));
System.Console.Out.WriteLine("Running " + numMaps + " maps.");
// reducer NONE
job.SetNumReduceTasks(0);
DateTime startTime = new DateTime();
System.Console.Out.WriteLine("Job started: " + startTime);
int ret = job.WaitForCompletion(true) ? 0 : 1;
DateTime endTime = new DateTime();
System.Console.Out.WriteLine("Job ended: " + endTime);
System.Console.Out.WriteLine("The job took " + (endTime.GetTime() - startTime.GetTime
()) / 1000 + " seconds.");
return(ret);
}
/// <summary>
/// Create a new instance copied from the given Cluster with the given status.
/// </summary>
/// <param name="status"></param>
/// <param name="copyFrom"></param>
public Cluster(ClusterStatus status, Cluster copyFrom)
: this(copyFrom.InternalName,
status,
copyFrom.AppCount,
copyFrom.ServiceCount,
copyFrom.Address,
new List <int>(copyFrom.Ports),
new List <ClusterUser>(copyFrom.Users),
copyFrom.CreatedOn)
{
}
public Cluster(ClusterStatus status, Cluster copyFrom)
: this(copyFrom.InternalName,
status,
copyFrom.AppCount,
copyFrom.ServiceCount,
copyFrom.Address,
new List<int>(copyFrom.Ports),
new List<ClusterUser>(copyFrom.Users),
copyFrom.CreatedOn)
{
}
public void TestClusterStatusRefresh()
{
ClusterStatus clusterStatus = new ClusterStatus("er-clust1-c07");
clusterStatus.Cluster = "er-clust1-c07";
bool ActualResult = clusterStatus.Refresh();
string ActualString = clusterStatus.ToString();
string ExpectedString = "er-clust1-c07: -1/-2";
bool expectedResult = true;
Assert.AreEqual(expectedResult, ActualResult);
Assert.AreEqual(ExpectedString, ActualString);
}
public void TestClusterStatusRefresh()
{
ClusterStatus clusterStatus = new ClusterStatus("er-clust1-c07")
{
Cluster = "er-clust1-c07"
};
bool actualResult = clusterStatus.Refresh();
Assert.AreEqual(true, actualResult);
const string expectedString = "er-clust1-c07: -1/-2";
string actualString = clusterStatus.ToString();
Assert.AreEqual(expectedString, actualString);
}
public void ClusterStatus_Serialize_ClusterStatus()
{
Dictionary <string, string> clusterProps = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
EnhancedMemoryStream ms = new EnhancedMemoryStream();
ClusterMemberSettings settings = new ClusterMemberSettings((MsgEP)"logical://test");
ClusterMemberStatus memberStatus;
ClusterStatus status;
status = new ClusterStatus((MsgEP)"logical://test/master");
status.LoadProperties(clusterProps);
status.MasterProtocolCaps = unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF);
memberStatus = new ClusterMemberStatus("logical://test/master", ClusterMemberState.Master, settings);
memberStatus["setting1"] = "value1";
memberStatus["setting2"] = "value2";
status.Members.Add(memberStatus);
memberStatus = new ClusterMemberStatus("logical://test/bar", ClusterMemberState.Slave, settings);
memberStatus["setting3"] = "value3";
memberStatus["setting4"] = "value4";
status.Members.Add(memberStatus);
status.Write(ms);
ms.Position = 0;
status = new ClusterStatus(ms);
Assert.AreEqual((MsgEP)"logical://test/master", status.MasterEP);
Assert.AreEqual(Helper.MachineName, status.MasterMachine);
Assert.AreEqual(unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF), status.MasterProtocolCaps);
Assert.AreEqual(2, status.Members.Count);
memberStatus = status.Members[0];
Assert.AreEqual((MsgEP)"logical://test/master", memberStatus.InstanceEP);
Assert.AreEqual(Helper.MachineName, memberStatus.MachineName);
Assert.AreEqual(ClusterMemberState.Master, memberStatus.State);
Assert.AreEqual(settings, memberStatus.Settings);
Assert.AreEqual("value1", memberStatus["setting1"]);
Assert.AreEqual("value2", memberStatus["setting2"]);
memberStatus = status.Members[1];
Assert.AreEqual((MsgEP)"logical://test/bar", memberStatus.InstanceEP);
Assert.AreEqual(Helper.MachineName, memberStatus.MachineName);
Assert.AreEqual(ClusterMemberState.Slave, memberStatus.State);
Assert.AreEqual(settings, memberStatus.Settings);
Assert.AreEqual("value3", memberStatus["setting3"]);
Assert.AreEqual("value4", memberStatus["setting4"]);
}
public void ClusterStatus_Clone_ClusterStatus()
{
ClusterMemberSettings settings = new ClusterMemberSettings((MsgEP)"logical://test");
ClusterMemberStatus memberStatus;
ClusterStatus input, output;
input = new ClusterStatus((MsgEP)"logical://test/master");
input.ClusterTime = DateTime.UtcNow;
input.MasterProtocolCaps = unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF);
memberStatus = new ClusterMemberStatus("logical://test/master", ClusterMemberState.Master, settings);
memberStatus["setting1"] = "value1";
memberStatus["setting2"] = "value2";
input.Members.Add(memberStatus);
memberStatus = new ClusterMemberStatus("logical://test/bar", ClusterMemberState.Slave, settings);
memberStatus["setting3"] = "value3";
memberStatus["setting4"] = "value4";
input.Members.Add(memberStatus);
output = input.Clone();
Assert.AreNotSame(input, output);
Assert.AreEqual((MsgEP)"logical://test/master", output.MasterEP);
Assert.AreEqual(Helper.MachineName, output.MasterMachine);
Assert.AreEqual(input.ClusterTime, output.ClusterTime);
Assert.AreEqual(input.ReceiveTime, output.ReceiveTime);
Assert.AreEqual(unchecked ((ClusterMemberProtocolCaps)0xFFFFFFFF), output.MasterProtocolCaps);
Assert.AreEqual(2, output.Members.Count);
memberStatus = output.Members[0];
Assert.AreEqual((MsgEP)"logical://test/master", memberStatus.InstanceEP);
Assert.AreEqual(Helper.MachineName, memberStatus.MachineName);
Assert.AreEqual(ClusterMemberState.Master, memberStatus.State);
Assert.AreEqual(settings, memberStatus.Settings);
Assert.AreEqual("value1", memberStatus["setting1"]);
Assert.AreEqual("value2", memberStatus["setting2"]);
memberStatus = output.Members[1];
Assert.AreEqual((MsgEP)"logical://test/bar", memberStatus.InstanceEP);
Assert.AreEqual(Helper.MachineName, memberStatus.MachineName);
Assert.AreEqual(ClusterMemberState.Slave, memberStatus.State);
Assert.AreEqual(settings, memberStatus.Settings);
Assert.AreEqual("value3", memberStatus["setting3"]);
Assert.AreEqual("value4", memberStatus["setting4"]);
}
public void CommandFollowers(List <Followership> followers)
{
Tock++;
followers = followers
.Where(IsObedientFollower)
.OrderBy(x => x.FollowerId)
.ToList();
var differences = new HashSet <string>(followers.Select(x => x.FollowerId));
differences.SymmetricExceptWith(InboundAssignments.Select(x => x.FollowerId));
if (differences.Any() && Status == ClusterStatus.Harmony)
{
AdvanceTerm();
}
if (!followers.Any())
{
return;
}
if (differences.Any())
{
var range = byte.MaxValue / followers.Count; //match MaxValue to TransportMessage ClaimTicket
var remainder = byte.MaxValue - (range * followers.Count);
InboundAssignments =
followers
.Select((follower, i) => new FollowerAssignment
{
FollowerId = follower.FollowerId,
LowerBound = i * range + (i == 0 ? 0 : 1),
UpperBound = (i * range) + range + (i == followers.Count - 1 ? remainder : 0)
})
.ToList();
OutboundAssignment = followers.First().FollowerId;
}
Status = followers.All(x => x.Term == Term)
? ClusterStatus.Harmony
: ClusterStatus.Turmoil;
}
/// <summary>
/// Translates a Cluster Status Value into a character
/// </summary>
/// <param name="C">Cluster Status</param>
/// <returns>Character</returns>
private static char ToConsoleChar(ClusterStatus C)
{
switch (C)
{
case ClusterStatus.Damaged:
return('B');
case ClusterStatus.EOF:
return('â–‘');
case ClusterStatus.Occupied:
return('â–ˆ');
case ClusterStatus.Reserved:
return('R');
}
return('.');
}
/// <summary>
/// Translates a Cluster Status Value into a Console color
/// </summary>
/// <param name="C">Cluster Status</param>
/// <returns>Console Color Value</returns>
private static ConsoleColor ToConsoleColor(ClusterStatus C)
{
switch (C)
{
case ClusterStatus.Damaged:
return(ConsoleColor.Red);
case ClusterStatus.EOF:
return(ConsoleColor.Green);
case ClusterStatus.Occupied:
return(ConsoleColor.Blue);
case ClusterStatus.Reserved:
return(ConsoleColor.DarkCyan);
}
return(ConsoleColor.White);
}
/// <summary>
/// Translates a Cluster Status Value into an RGB bitmap Color
/// </summary>
/// <param name="C">Cluster Status</param>
/// <returns>Bitmap Color Value</returns>
private static Color ToBitmapColor(ClusterStatus C)
{
switch (C)
{
case ClusterStatus.Damaged:
return(Color.Red);
case ClusterStatus.EOF:
return(Color.Green);
case ClusterStatus.Occupied:
return(Color.Blue);
case ClusterStatus.Reserved:
return(Color.Purple);
}
return(Color.White);
}
public Cluster(
string internalName,
ClusterStatus status,
int appCount,
int serviceCount,
string address,
IEnumerable<int> ports,
IEnumerable<ClusterUser> users,
DateTimeOffset createdOn)
{
this.InternalName = internalName;
this.Status = status;
this.AppCount = appCount;
this.ServiceCount = serviceCount;
this.Address = address;
this.Ports = ports;
this.Users = users;
this.CreatedOn = createdOn;
}
/// <summary>
/// Query the cluster again for the list of jobs.
/// </summary>
/// <returns>True if the cluster could be found.</returns>
private void RefreshClusterJobList()
{
if (this.cluster == null)
{
return;
}
string clus = this.cluster;
if (!string.IsNullOrEmpty(this.SelectedVirtualCluster))
{
clus += "/" + this.SelectedVirtualCluster;
}
this.Status("Querying cluster " + clus, StatusKind.LongOp);
try
{
this.openFromURLToolStripMenuItem.Visible = false;
if (
this.Configuration is CacheClusterConfiguration)
{
this.filteredDataGridView.DataGridView.Columns["VirtualCluster"].Visible = true;
}
else
{
this.filteredDataGridView.DataGridView.Columns["VirtualCluster"].Visible = false;
}
this.clusterStatus = this.Configuration.CreateClusterStatus();
this.toolStripMenuItem_job.Enabled = true;
var item = new BackgroundWorkItem <List <ClusterJobInformation> >(
m => BuildClusterJobList(m, this.clusterStatus, this.SelectedVirtualCluster),
this.JobListRetrieved,
"getJobs");
this.Queue(item);
}
catch (Exception ex)
{
this.Status("Cannot retrieve information from cluster " + cluster + ": " + ex.Message, StatusKind.Error);
Trace.TraceInformation(ex.ToString());
this.comboBox_virtualCluster.Text = "";
}
}
/// <summary>
/// Creates a new instance.
/// </summary>
/// <param name="internalName"></param>
/// <param name="status"></param>
/// <param name="appCount"></param>
/// <param name="serviceCount"></param>
/// <param name="address"></param>
/// <param name="ports"></param>
/// <param name="users"></param>
/// <param name="createdOn"></param>
public Cluster(
string internalName,
ClusterStatus status,
int appCount,
int serviceCount,
string address,
IEnumerable <int> ports,
IEnumerable <ClusterUser> users,
DateTimeOffset createdOn)
{
this.InternalName = internalName;
this.Status = status;
this.AppCount = appCount;
this.ServiceCount = serviceCount;
this.Address = address;
this.Ports = ports;
this.Users = users;
this.CreatedOn = createdOn;
}
/// <summary>
/// Cancel a job.
/// </summary>
/// <param name="jobs">Jobs to cancel.</param>
/// <param name="cluster">Cluster where the jobs are running.</param>
/// <returns>True if all cancellations succeed.</returns>
/// <param name="manager">Communicatoni manager.</param>
// ReSharper disable once UnusedParameter.Global
public static bool CancelJobs(IEnumerable <DryadLinqJobSummary> jobs, ClusterStatus cluster, CommManager manager)
{
bool done = true;
foreach (DryadLinqJobSummary job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
if (job.Status != ClusterJobInformation.ClusterJobStatus.Running)
{
manager.Status("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
}
bool success;
string reason = "";
try
{
success = cluster.CancelJob(job);
}
catch (Exception ex)
{
success = false;
reason = ex.Message;
Trace.TraceInformation(ex.ToString());
}
if (success)
{
manager.Status("Job " + job.Name + " cancelled", StatusKind.OK);
}
else
{
manager.Status("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
}
done &= success;
}
return(done);
}
public ESEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node)
{
Connection = new ESConnection(node.ReadStr("@url"));
CacheSize = node.ReadInt("@cache", -1);
MaxParallel = node.ReadInt("@maxparallel", 0);
ReadOnly = node.ReadBool("@readonly", false);
XmlNode root = node.SelectSingleNode("indexes");
if (root == null)
{
root = node;
}
Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig);
if (Indexes.Count == 0)
{
throw new BMNodeException(node, "At least 1 index+type is required!");
}
WaitFor = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt);
WaitForTimeout = node.ReadInt("waitfor/@timeout", 30);
WaitForMustExcept = node.ReadBool("waitfor/@except", true);
}
public void SetStatus_Sets_Appropriate_Status(
MetricStatus status1,
MetricStatus status2,
MetricStatus status3,
MetricStatus status4,
ClusterStatus expectedStatus)
{
var cluster = new Cluster
{
Nodes = new List <ClusterNode>
{
new ClusterNode
{
Metrics = new List <Metric>
{
new Metric {
Status = status1
},
new Metric {
Status = status2
},
new Metric {
Status = status3
},
new Metric {
Status = status4
},
}
}
}
};
cluster.SetStatus();
cluster.Status.Should().Be(expectedStatus);
}
private Task AddClusters(ITransaction tx, IReliableDictionary <int, Cluster> dictionary, int count, ClusterStatus status)
{
return(this.AddClusters(tx, dictionary, count, () => new Cluster()
{
Status = status
}));
}
private Cluster CreateCluster(ClusterStatus status)
{
return new Cluster(
status,
new Cluster("test"));
}
private Cluster CreateCluster(ClusterStatus status, IEnumerable<ClusterUser> users)
{
return new Cluster(
"test",
status,
0,
0,
String.Empty,
new int[0],
users,
DateTimeOffset.MaxValue);
}
/// <summary>
/// Talk to the web server and build the list of clustr jobs; used it to populate the upper panel.
/// </summary>
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
/// <param name="manager">Communication manager.</param>
/// <param name="status">Cluster to scan.</param>
private static List <ClusterJobInformation> BuildClusterJobList(CommManager manager, ClusterStatus status, string virtualCluster)
{
return(status.GetClusterJobList(virtualCluster, manager).ToList());
}
/// <summary>
/// Creates a new Cluster Entry from the given Value
/// </summary>
/// <param name="Raw">Value</param>
public ClusterEntry(ushort Raw)
{
RawValue = Raw;
Status = GetFat12Status(Raw);
}
/// <summary>The main driver for sort program.</summary>
/// <remarks>
/// The main driver for sort program.
/// Invoke this method to submit the map/reduce job.
/// </remarks>
/// <exception cref="System.IO.IOException">
/// When there is communication problems with the
/// job tracker.
/// </exception>
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
Configuration conf = GetConf();
JobClient client = new JobClient(conf);
ClusterStatus cluster = client.GetClusterStatus();
int num_reduces = (int)(cluster.GetMaxReduceTasks() * 0.9);
string sort_reduces = conf.Get(ReducesPerHost);
if (sort_reduces != null)
{
num_reduces = cluster.GetTaskTrackers() * System.Convert.ToInt32(sort_reduces);
}
Type inputFormatClass = typeof(SequenceFileInputFormat);
Type outputFormatClass = typeof(SequenceFileOutputFormat);
Type outputKeyClass = typeof(BytesWritable);
Type outputValueClass = typeof(BytesWritable);
IList <string> otherArgs = new AList <string>();
InputSampler.Sampler <K, V> sampler = null;
for (int i = 0; i < args.Length; ++i)
{
try
{
if ("-r".Equals(args[i]))
{
num_reduces = System.Convert.ToInt32(args[++i]);
}
else
{
if ("-inFormat".Equals(args[i]))
{
inputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <InputFormat>();
}
else
{
if ("-outFormat".Equals(args[i]))
{
outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>();
}
else
{
if ("-outKey".Equals(args[i]))
{
outputKeyClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <WritableComparable
>();
}
else
{
if ("-outValue".Equals(args[i]))
{
outputValueClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <Writable>();
}
else
{
if ("-totalOrder".Equals(args[i]))
{
double pcnt = double.ParseDouble(args[++i]);
int numSamples = System.Convert.ToInt32(args[++i]);
int maxSplits = System.Convert.ToInt32(args[++i]);
if (0 >= maxSplits)
{
maxSplits = int.MaxValue;
}
sampler = new InputSampler.RandomSampler <K, V>(pcnt, numSamples, maxSplits);
}
else
{
otherArgs.AddItem(args[i]);
}
}
}
}
}
}
}
catch (FormatException)
{
System.Console.Out.WriteLine("ERROR: Integer expected instead of " + args[i]);
return(PrintUsage());
}
catch (IndexOutOfRangeException)
{
System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i -
1]);
return(PrintUsage());
}
}
// exits
// Set user-supplied (possibly default) job configs
job = Job.GetInstance(conf);
job.SetJobName("sorter");
job.SetJarByClass(typeof(Sort));
job.SetMapperClass(typeof(Mapper));
job.SetReducerClass(typeof(Reducer));
job.SetNumReduceTasks(num_reduces);
job.SetInputFormatClass(inputFormatClass);
job.SetOutputFormatClass(outputFormatClass);
job.SetOutputKeyClass(outputKeyClass);
job.SetOutputValueClass(outputValueClass);
// Make sure there are exactly 2 parameters left.
if (otherArgs.Count != 2)
{
System.Console.Out.WriteLine("ERROR: Wrong number of parameters: " + otherArgs.Count
+ " instead of 2.");
return(PrintUsage());
}
FileInputFormat.SetInputPaths(job, otherArgs[0]);
FileOutputFormat.SetOutputPath(job, new Path(otherArgs[1]));
if (sampler != null)
{
System.Console.Out.WriteLine("Sampling input to effect total-order sort...");
job.SetPartitionerClass(typeof(TotalOrderPartitioner));
Path inputDir = FileInputFormat.GetInputPaths(job)[0];
inputDir = inputDir.MakeQualified(inputDir.GetFileSystem(conf));
Path partitionFile = new Path(inputDir, "_sortPartitioning");
TotalOrderPartitioner.SetPartitionFile(conf, partitionFile);
InputSampler.WritePartitionFile <K, V>(job, sampler);
URI partitionUri = new URI(partitionFile.ToString() + "#" + "_sortPartitioning");
DistributedCache.AddCacheFile(partitionUri, conf);
}
System.Console.Out.WriteLine("Running on " + cluster.GetTaskTrackers() + " nodes to sort from "
+ FileInputFormat.GetInputPaths(job)[0] + " into " + FileOutputFormat.GetOutputPath
(job) + " with " + num_reduces + " reduces.");
DateTime startTime = new DateTime();
System.Console.Out.WriteLine("Job started: " + startTime);
int ret = job.WaitForCompletion(true) ? 0 : 1;
DateTime end_time = new DateTime();
System.Console.Out.WriteLine("Job ended: " + end_time);
System.Console.Out.WriteLine("The job took " + (end_time.GetTime() - startTime.GetTime
()) / 1000 + " seconds.");
return(ret);
}
private Task AddClusters(ITransaction tx, IReliableDictionary<int, Cluster> dictionary, int count, ClusterStatus status)
{
return this.AddClusters(tx, dictionary, count, () => new Cluster() { Status = status });
}
/// <summary>This is the main routine for launching a distributed random write job.</summary>
/// <remarks>
/// This is the main routine for launching a distributed random write job.
/// It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
/// The reduce doesn't do anything.
/// </remarks>
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
if (args.Length == 0)
{
return(PrintUsage());
}
Configuration conf = GetConf();
JobClient client = new JobClient(conf);
ClusterStatus cluster = client.GetClusterStatus();
int numMapsPerHost = conf.GetInt(MapsPerHost, 10);
long numBytesToWritePerMap = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
if (numBytesToWritePerMap == 0)
{
System.Console.Error.WriteLine("Cannot have " + BytesPerMap + " set to 0");
return(-2);
}
long totalBytesToWrite = conf.GetLong(TotalBytes, numMapsPerHost * numBytesToWritePerMap
* cluster.GetTaskTrackers());
int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
if (numMaps == 0 && totalBytesToWrite > 0)
{
numMaps = 1;
conf.SetLong(BytesPerMap, totalBytesToWrite);
}
conf.SetInt(MRJobConfig.NumMaps, numMaps);
Job job = Job.GetInstance(conf);
job.SetJarByClass(typeof(RandomTextWriter));
job.SetJobName("random-text-writer");
job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
job.SetInputFormatClass(typeof(RandomWriter.RandomInputFormat));
job.SetMapperClass(typeof(RandomTextWriter.RandomTextMapper));
Type outputFormatClass = typeof(SequenceFileOutputFormat);
IList <string> otherArgs = new AList <string>();
for (int i = 0; i < args.Length; ++i)
{
try
{
if ("-outFormat".Equals(args[i]))
{
outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>();
}
else
{
otherArgs.AddItem(args[i]);
}
}
catch (IndexOutOfRangeException)
{
System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i -
1]);
return(PrintUsage());
}
}
// exits
job.SetOutputFormatClass(outputFormatClass);
FileOutputFormat.SetOutputPath(job, new Path(otherArgs[0]));
System.Console.Out.WriteLine("Running " + numMaps + " maps.");
// reducer NONE
job.SetNumReduceTasks(0);
DateTime startTime = new DateTime();
System.Console.Out.WriteLine("Job started: " + startTime);
int ret = job.WaitForCompletion(true) ? 0 : 1;
DateTime endTime = new DateTime();
System.Console.Out.WriteLine("Job ended: " + endTime);
System.Console.Out.WriteLine("The job took " + (endTime.GetTime() - startTime.GetTime
()) / 1000 + " seconds.");
return(ret);
}
/// <summary>The main driver for sort program.</summary>
/// <remarks>
/// The main driver for sort program.
/// Invoke this method to submit the map/reduce job.
/// </remarks>
/// <exception cref="System.IO.IOException">
/// When there is communication problems with the
/// job tracker.
/// </exception>
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
Configuration conf = GetConf();
JobClient client = new JobClient(conf);
ClusterStatus cluster = client.GetClusterStatus();
int num_reduces = (int)(cluster.GetMaxReduceTasks() * 0.9);
string join_reduces = conf.Get(ReducesPerHost);
if (join_reduces != null)
{
num_reduces = cluster.GetTaskTrackers() * System.Convert.ToInt32(join_reduces);
}
Job job = Job.GetInstance(conf);
job.SetJobName("join");
job.SetJarByClass(typeof(Sort));
job.SetMapperClass(typeof(Mapper));
job.SetReducerClass(typeof(Reducer));
Type inputFormatClass = typeof(SequenceFileInputFormat);
Type outputFormatClass = typeof(SequenceFileOutputFormat);
Type outputKeyClass = typeof(BytesWritable);
Type outputValueClass = typeof(TupleWritable);
string op = "inner";
IList <string> otherArgs = new AList <string>();
for (int i = 0; i < args.Length; ++i)
{
try
{
if ("-r".Equals(args[i]))
{
num_reduces = System.Convert.ToInt32(args[++i]);
}
else
{
if ("-inFormat".Equals(args[i]))
{
inputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <InputFormat>();
}
else
{
if ("-outFormat".Equals(args[i]))
{
outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>();
}
else
{
if ("-outKey".Equals(args[i]))
{
outputKeyClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <WritableComparable
>();
}
else
{
if ("-outValue".Equals(args[i]))
{
outputValueClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <Writable>();
}
else
{
if ("-joinOp".Equals(args[i]))
{
op = args[++i];
}
else
{
otherArgs.AddItem(args[i]);
}
}
}
}
}
}
}
catch (FormatException)
{
System.Console.Out.WriteLine("ERROR: Integer expected instead of " + args[i]);
return(PrintUsage());
}
catch (IndexOutOfRangeException)
{
System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i -
1]);
return(PrintUsage());
}
}
// exits
// Set user-supplied (possibly default) job configs
job.SetNumReduceTasks(num_reduces);
if (otherArgs.Count < 2)
{
System.Console.Out.WriteLine("ERROR: Wrong number of parameters: ");
return(PrintUsage());
}
FileOutputFormat.SetOutputPath(job, new Path(otherArgs.Remove(otherArgs.Count - 1
)));
IList <Path> plist = new AList <Path>(otherArgs.Count);
foreach (string s in otherArgs)
{
plist.AddItem(new Path(s));
}
job.SetInputFormatClass(typeof(CompositeInputFormat));
job.GetConfiguration().Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose
(op, inputFormatClass, Sharpen.Collections.ToArray(plist, new Path[0])));
job.SetOutputFormatClass(outputFormatClass);
job.SetOutputKeyClass(outputKeyClass);
job.SetOutputValueClass(outputValueClass);
DateTime startTime = new DateTime();
System.Console.Out.WriteLine("Job started: " + startTime);
int ret = job.WaitForCompletion(true) ? 0 : 1;
DateTime end_time = new DateTime();
System.Console.Out.WriteLine("Job ended: " + end_time);
System.Console.Out.WriteLine("The job took " + (end_time.GetTime() - startTime.GetTime
()) / 1000 + " seconds.");
return(ret);
}
