public virtual void TestMerge()
{
string TestMethodKey = "testMerge.mapfile";
int Size = 10;
int Iterations = 5;
Path[] @in = new Path[5];
IList <int> expected = new AList <int>();
for (int j = 0; j < 5; j++)
{
using (MapFile.Writer writer = CreateWriter(TestMethodKey + "." + j, typeof(IntWritable
), typeof(Text)))
{
@in[j] = new Path(TestDir, TestMethodKey + "." + j);
for (int i = 0; i < Size; i++)
{
expected.AddItem(i + j);
writer.Append(new IntWritable(i + j), new Text("Value:" + (i + j)));
}
}
}
// Sort expected values
expected.Sort();
// Merge all 5 files
MapFile.Merger merger = new MapFile.Merger(conf);
merger.Merge(@in, true, new Path(TestDir, TestMethodKey));
using (MapFile.Reader reader = CreateReader(TestMethodKey, typeof(IntWritable)))
{
int start = 0;
// test iteration
Text startValue = new Text("Value:" + start);
int i = 0;
while (i++ < Iterations)
{
IEnumerator <int> expectedIterator = expected.GetEnumerator();
IntWritable key = new IntWritable(start);
Text value = startValue;
IntWritable prev = new IntWritable(start);
while (reader.Next(key, value))
{
Assert.True("Next key should be always equal or more", prev.Get
() <= key.Get());
Assert.Equal(expectedIterator.Next(), key.Get());
prev.Set(key.Get());
}
reader.Reset();
}
}
// inputs should be deleted
for (int j_1 = 0; j_1 < @in.Length; j_1++)
{
Path path = @in[j_1];
NUnit.Framework.Assert.IsFalse("inputs should be deleted", path.GetFileSystem(conf
).Exists(path));
}
}
/// <summary>Generate Java code for records.</summary>
/// <remarks>
/// Generate Java code for records. This method is only a front-end to
/// JRecord, since one file is generated for each record.
/// </remarks>
/// <param name="name">possibly full pathname to the file</param>
/// <param name="ilist">included files (as JFile)</param>
/// <param name="rlist">List of records defined within this file</param>
/// <param name="destDir">output directory</param>
/// <exception cref="System.IO.IOException"/>
internal override void GenCode(string name, AList <JFile> ilist, AList <JRecord> rlist
, string destDir, AList <string> options)
{
for (IEnumerator <JRecord> iter = rlist.GetEnumerator(); iter.HasNext();)
{
JRecord rec = iter.Next();
rec.GenJavaCode(destDir, options);
}
}
/// <summary>Prints the node report for node id.</summary>
/// <param name="nodeIdStr"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void PrintNodeStatus(string nodeIdStr)
{
NodeId nodeId = ConverterUtils.ToNodeId(nodeIdStr);
IList <NodeReport> nodesReport = client.GetNodeReports();
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter nodeReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
NodeReport nodeReport = null;
foreach (NodeReport report in nodesReport)
{
if (!report.GetNodeId().Equals(nodeId))
{
continue;
}
nodeReport = report;
nodeReportStr.WriteLine("Node Report : ");
nodeReportStr.Write("\tNode-Id : ");
nodeReportStr.WriteLine(nodeReport.GetNodeId());
nodeReportStr.Write("\tRack : ");
nodeReportStr.WriteLine(nodeReport.GetRackName());
nodeReportStr.Write("\tNode-State : ");
nodeReportStr.WriteLine(nodeReport.GetNodeState());
nodeReportStr.Write("\tNode-Http-Address : ");
nodeReportStr.WriteLine(nodeReport.GetHttpAddress());
nodeReportStr.Write("\tLast-Health-Update : ");
nodeReportStr.WriteLine(DateFormatUtils.Format(Sharpen.Extensions.CreateDate(nodeReport
.GetLastHealthReportTime()), "E dd/MMM/yy hh:mm:ss:SSzz"));
nodeReportStr.Write("\tHealth-Report : ");
nodeReportStr.WriteLine(nodeReport.GetHealthReport());
nodeReportStr.Write("\tContainers : ");
nodeReportStr.WriteLine(nodeReport.GetNumContainers());
nodeReportStr.Write("\tMemory-Used : ");
nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0MB" : (nodeReport.GetUsed
().GetMemory() + "MB"));
nodeReportStr.Write("\tMemory-Capacity : ");
nodeReportStr.WriteLine(nodeReport.GetCapability().GetMemory() + "MB");
nodeReportStr.Write("\tCPU-Used : ");
nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0 vcores" : (nodeReport
.GetUsed().GetVirtualCores() + " vcores"));
nodeReportStr.Write("\tCPU-Capacity : ");
nodeReportStr.WriteLine(nodeReport.GetCapability().GetVirtualCores() + " vcores");
nodeReportStr.Write("\tNode-Labels : ");
// Create a List for node labels since we need it get sorted
IList <string> nodeLabelsList = new AList <string>(report.GetNodeLabels());
nodeLabelsList.Sort();
nodeReportStr.WriteLine(StringUtils.Join(nodeLabelsList.GetEnumerator(), ','));
}
if (nodeReport == null)
{
nodeReportStr.Write("Could not find the node report for node id : " + nodeIdStr);
}
nodeReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
}
public virtual void TestPipesReduser()
{
FilePath[] psw = CleanTokenPasswordFile();
JobConf conf = new JobConf();
try
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service"));
TokenCache.SetJobToken(token, conf.GetCredentials());
FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeReducerStub"
);
conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath());
PipesReducer <BooleanWritable, Text, IntWritable, Text> reducer = new PipesReducer
<BooleanWritable, Text, IntWritable, Text>();
reducer.Configure(conf);
BooleanWritable bw = new BooleanWritable(true);
conf.Set(MRJobConfig.TaskAttemptId, taskName);
InitStdOut(conf);
conf.SetBoolean(MRJobConfig.SkipRecords, true);
TestPipeApplication.CombineOutputCollector <IntWritable, Text> output = new TestPipeApplication.CombineOutputCollector
<IntWritable, Text>(this, new Counters.Counter(), new TestPipeApplication.Progress
(this));
Reporter reporter = new TestPipeApplication.TestTaskReporter(this);
IList <Text> texts = new AList <Text>();
texts.AddItem(new Text("first"));
texts.AddItem(new Text("second"));
texts.AddItem(new Text("third"));
reducer.Reduce(bw, texts.GetEnumerator(), output, reporter);
reducer.Close();
string stdOut = ReadStdOut(conf);
// test data: key
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reducer key :true"));
// and values
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :first"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :second"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :third"));
}
finally
{
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
}
internal CppRecord(JRecord _enclosing, string name, AList <JField <JType> > flist)
: base(_enclosing)
{
this._enclosing = _enclosing;
this.fullName = name.ReplaceAll("\\.", "::");
int idx = name.LastIndexOf('.');
this.name = Runtime.Substring(name, idx + 1);
this.module = Runtime.Substring(name, 0, idx).ReplaceAll("\\.", "::");
for (IEnumerator <JField <JType> > iter = flist.GetEnumerator(); iter.HasNext();)
{
JField <JType> f = iter.Next();
this.fields.AddItem(new JField <JType.CppType>(f.GetName(), f.GetType().GetCppType
()));
}
}
/// <summary>Creates a new instance of JRecord</summary>
public JRecord(string name, AList <JField <JType> > flist)
{
SetJavaType(new JRecord.JavaRecord(this, name, flist));
SetCppType(new JRecord.CppRecord(this, name, flist));
SetCType(new JRecord.CRecord(this));
// precompute signature
int idx = name.LastIndexOf('.');
string recName = Runtime.Substring(name, idx + 1);
StringBuilder sb = new StringBuilder();
sb.Append("L").Append(recName).Append("(");
for (IEnumerator <JField <JType> > i = flist.GetEnumerator(); i.HasNext();)
{
string s = i.Next().GetType().GetSignature();
sb.Append(s);
}
sb.Append(")");
signature = sb.ToString();
}
/// <summary>Chooses datanode locations for caching from a list of valid possibilities.
/// </summary>
/// <remarks>
/// Chooses datanode locations for caching from a list of valid possibilities.
/// Non-stale nodes are chosen before stale nodes.
/// </remarks>
/// <param name="possibilities">List of candidate datanodes</param>
/// <param name="neededCached">Number of replicas needed</param>
/// <param name="staleInterval">Age of a stale datanode</param>
/// <returns>A list of chosen datanodes</returns>
private static IList <DatanodeDescriptor> ChooseDatanodesForCaching(IList <DatanodeDescriptor
> possibilities, int neededCached, long staleInterval)
{
// Make a copy that we can modify
IList <DatanodeDescriptor> targets = new AList <DatanodeDescriptor>(possibilities);
// Selected targets
IList <DatanodeDescriptor> chosen = new List <DatanodeDescriptor>();
// Filter out stale datanodes
IList <DatanodeDescriptor> stale = new List <DatanodeDescriptor>();
IEnumerator <DatanodeDescriptor> it = targets.GetEnumerator();
while (it.HasNext())
{
DatanodeDescriptor d = it.Next();
if (d.IsStale(staleInterval))
{
it.Remove();
stale.AddItem(d);
}
}
// Select targets
while (chosen.Count < neededCached)
{
// Try to use stale nodes if we're out of non-stale nodes, else we're done
if (targets.IsEmpty())
{
if (!stale.IsEmpty())
{
targets = stale;
}
else
{
break;
}
}
// Select a random target
DatanodeDescriptor target = ChooseRandomDatanodeByRemainingCapacity(targets);
chosen.AddItem(target);
targets.Remove(target);
}
return(chosen);
}
/// <exception cref="System.IO.IOException"/>
private static void CreateStorageDirs(DataStorage storage, Configuration conf, int
numDirs)
{
IList <Storage.StorageDirectory> dirs = new AList <Storage.StorageDirectory>();
IList <string> dirStrings = new AList <string>();
for (int i = 0; i < numDirs; i++)
{
FilePath loc = new FilePath(BaseDir + "/data" + i);
dirStrings.AddItem(new Path(loc.ToString()).ToUri().ToString());
loc.Mkdirs();
dirs.AddItem(CreateStorageDirectory(loc));
Org.Mockito.Mockito.When(storage.GetStorageDir(i)).ThenReturn(dirs[i]);
}
string dataDir = StringUtils.Join(",", dirStrings);
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dataDir);
Org.Mockito.Mockito.When(storage.DirIterator()).ThenReturn(dirs.GetEnumerator());
Org.Mockito.Mockito.When(storage.GetNumStorageDirs()).ThenReturn(numDirs);
}
/// <exception cref="System.IO.IOException"/>
private void WriteConfigFile(Path name, AList <string> nodes)
{
// delete if it already exists
if (localFileSys.Exists(name))
{
localFileSys.Delete(name, true);
}
FSDataOutputStream stm = localFileSys.Create(name);
if (nodes != null)
{
for (IEnumerator <string> it = nodes.GetEnumerator(); it.HasNext();)
{
string node = it.Next();
stm.WriteBytes(node);
stm.WriteBytes("\n");
}
}
stm.Close();
}
/// <summary>Generate C++ code.</summary>
/// <remarks>
/// Generate C++ code. This method only creates the requested file(s)
/// and spits-out file-level elements (such as include statements etc.)
/// record-level code is generated by JRecord.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal override void GenCode(string name, AList <JFile> ilist, AList <JRecord> rlist
, string destDir, AList <string> options)
{
name = new FilePath(destDir, (new FilePath(name)).GetName()).GetAbsolutePath();
FileWriter cc = new FileWriter(name + ".cc");
try
{
FileWriter hh = new FileWriter(name + ".hh");
try
{
string fileName = (new FilePath(name)).GetName();
hh.Write("#ifndef __" + StringUtils.ToUpperCase(fileName).Replace('.', '_') + "__\n"
);
hh.Write("#define __" + StringUtils.ToUpperCase(fileName).Replace('.', '_') + "__\n"
);
hh.Write("#include \"recordio.hh\"\n");
hh.Write("#include \"recordTypeInfo.hh\"\n");
for (IEnumerator <JFile> iter = ilist.GetEnumerator(); iter.HasNext();)
{
hh.Write("#include \"" + iter.Next().GetName() + ".hh\"\n");
}
cc.Write("#include \"" + fileName + ".hh\"\n");
cc.Write("#include \"utils.hh\"\n");
for (IEnumerator <JRecord> iter_1 = rlist.GetEnumerator(); iter_1.HasNext();)
{
iter_1.Next().GenCppCode(hh, cc, options);
}
hh.Write("#endif //" + StringUtils.ToUpperCase(fileName).Replace('.', '_') + "__\n"
);
}
finally
{
hh.Close();
}
}
finally
{
cc.Close();
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override IList <ContainerReport> GetContainers(ApplicationAttemptId applicationAttemptId
)
{
IList <ContainerReport> containersForAttempt = new AList <ContainerReport>();
bool appNotFoundInRM = false;
try
{
GetContainersRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainersRequest
>();
request.SetApplicationAttemptId(applicationAttemptId);
GetContainersResponse response = rmClient.GetContainers(request);
Sharpen.Collections.AddAll(containersForAttempt, response.GetContainerList());
}
catch (YarnException e)
{
if (e.GetType() != typeof(ApplicationNotFoundException) || !historyServiceEnabled)
{
// If Application is not in RM and history service is enabled then we
// need to check with history service else throw exception.
throw;
}
appNotFoundInRM = true;
}
if (historyServiceEnabled)
{
// Check with AHS even if found in RM because to capture info of finished
// containers also
IList <ContainerReport> containersListFromAHS = null;
try
{
containersListFromAHS = historyClient.GetContainers(applicationAttemptId);
}
catch (IOException e)
{
// History service access might be enabled but system metrics publisher
// is disabled hence app not found exception is possible
if (appNotFoundInRM)
{
// app not found in bothM and RM then propagate the exception.
throw;
}
}
if (null != containersListFromAHS && containersListFromAHS.Count > 0)
{
// remove duplicates
ICollection <ContainerId> containerIdsToBeKeptFromAHS = new HashSet <ContainerId>();
IEnumerator <ContainerReport> tmpItr = containersListFromAHS.GetEnumerator();
while (tmpItr.HasNext())
{
containerIdsToBeKeptFromAHS.AddItem(tmpItr.Next().GetContainerId());
}
IEnumerator <ContainerReport> rmContainers = containersForAttempt.GetEnumerator();
while (rmContainers.HasNext())
{
ContainerReport tmp = rmContainers.Next();
containerIdsToBeKeptFromAHS.Remove(tmp.GetContainerId());
}
// Remove containers from AHS as container from RM will have latest
// information
if (containerIdsToBeKeptFromAHS.Count > 0 && containersListFromAHS.Count != containerIdsToBeKeptFromAHS
.Count)
{
IEnumerator <ContainerReport> containersFromHS = containersListFromAHS.GetEnumerator
();
while (containersFromHS.HasNext())
{
ContainerReport containerReport = containersFromHS.Next();
if (containerIdsToBeKeptFromAHS.Contains(containerReport.GetContainerId()))
{
containersForAttempt.AddItem(containerReport);
}
}
}
else
{
if (containersListFromAHS.Count == containerIdsToBeKeptFromAHS.Count)
{
Sharpen.Collections.AddAll(containersForAttempt, containersListFromAHS);
}
}
}
}
return(containersForAttempt);
}
public override void Reset()
{
iter = data.GetEnumerator();
}
private static IList <E> Apply2Previous <K, E>(IList <E> previous, IList <E> clist, IList
<E> dlist)
where E : Diff.Element <K>
{
// Assumptions:
// (A1) All lists are sorted.
// (A2) All elements in dlist must be in previous.
// (A3) All elements in clist must be not in tmp = previous - dlist.
IList <E> tmp = new AList <E>(previous.Count - dlist.Count);
{
// tmp = previous - dlist
IEnumerator <E> i = previous.GetEnumerator();
foreach (E deleted in dlist)
{
E e = i.Next();
//since dlist is non-empty, e must exist by (A2).
int cmp = 0;
for (; (cmp = e.CompareTo(deleted.GetKey())) < 0; e = i.Next())
{
tmp.AddItem(e);
}
Preconditions.CheckState(cmp == 0);
}
// check (A2)
for (; i.HasNext();)
{
tmp.AddItem(i.Next());
}
}
IList <E> current = new AList <E>(tmp.Count + clist.Count);
{
// current = tmp + clist
IEnumerator <E> tmpIterator = tmp.GetEnumerator();
IEnumerator <E> cIterator = clist.GetEnumerator();
E t = tmpIterator.HasNext() ? tmpIterator.Next() : null;
E c = cIterator.HasNext() ? cIterator.Next() : null;
for (; t != null || c != null;)
{
int cmp = c == null ? 1 : t == null ? -1 : c.CompareTo(t.GetKey());
if (cmp < 0)
{
current.AddItem(c);
c = cIterator.HasNext() ? cIterator.Next() : null;
}
else
{
if (cmp > 0)
{
current.AddItem(t);
t = tmpIterator.HasNext() ? tmpIterator.Next() : null;
}
else
{
throw new Exception("Violated assumption (A3).");
}
}
}
}
return(current);
}
