public virtual void TestMerge() { string TestMethodKey = "testMerge.mapfile"; int Size = 10; int Iterations = 5; Path[] @in = new Path[5]; IList <int> expected = new AList <int>(); for (int j = 0; j < 5; j++) { using (MapFile.Writer writer = CreateWriter(TestMethodKey + "." + j, typeof(IntWritable ), typeof(Text))) { @in[j] = new Path(TestDir, TestMethodKey + "." + j); for (int i = 0; i < Size; i++) { expected.AddItem(i + j); writer.Append(new IntWritable(i + j), new Text("Value:" + (i + j))); } } } // Sort expected values expected.Sort(); // Merge all 5 files MapFile.Merger merger = new MapFile.Merger(conf); merger.Merge(@in, true, new Path(TestDir, TestMethodKey)); using (MapFile.Reader reader = CreateReader(TestMethodKey, typeof(IntWritable))) { int start = 0; // test iteration Text startValue = new Text("Value:" + start); int i = 0; while (i++ < Iterations) { IEnumerator <int> expectedIterator = expected.GetEnumerator(); IntWritable key = new IntWritable(start); Text value = startValue; IntWritable prev = new IntWritable(start); while (reader.Next(key, value)) { Assert.True("Next key should be always equal or more", prev.Get () <= key.Get()); Assert.Equal(expectedIterator.Next(), key.Get()); prev.Set(key.Get()); } reader.Reset(); } } // inputs should be deleted for (int j_1 = 0; j_1 < @in.Length; j_1++) { Path path = @in[j_1]; NUnit.Framework.Assert.IsFalse("inputs should be deleted", path.GetFileSystem(conf ).Exists(path)); } }
/// <summary>Generate Java code for records.</summary> /// <remarks> /// Generate Java code for records. This method is only a front-end to /// JRecord, since one file is generated for each record. /// </remarks> /// <param name="name">possibly full pathname to the file</param> /// <param name="ilist">included files (as JFile)</param> /// <param name="rlist">List of records defined within this file</param> /// <param name="destDir">output directory</param> /// <exception cref="System.IO.IOException"/> internal override void GenCode(string name, AList <JFile> ilist, AList <JRecord> rlist , string destDir, AList <string> options) { for (IEnumerator <JRecord> iter = rlist.GetEnumerator(); iter.HasNext();) { JRecord rec = iter.Next(); rec.GenJavaCode(destDir, options); } }
/// <summary>Prints the node report for node id.</summary> /// <param name="nodeIdStr"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private void PrintNodeStatus(string nodeIdStr) { NodeId nodeId = ConverterUtils.ToNodeId(nodeIdStr); IList <NodeReport> nodesReport = client.GetNodeReports(); // Use PrintWriter.println, which uses correct platform line ending. ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter nodeReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding ("UTF-8"))); NodeReport nodeReport = null; foreach (NodeReport report in nodesReport) { if (!report.GetNodeId().Equals(nodeId)) { continue; } nodeReport = report; nodeReportStr.WriteLine("Node Report : "); nodeReportStr.Write("\tNode-Id : "); nodeReportStr.WriteLine(nodeReport.GetNodeId()); nodeReportStr.Write("\tRack : "); nodeReportStr.WriteLine(nodeReport.GetRackName()); nodeReportStr.Write("\tNode-State : "); nodeReportStr.WriteLine(nodeReport.GetNodeState()); nodeReportStr.Write("\tNode-Http-Address : "); nodeReportStr.WriteLine(nodeReport.GetHttpAddress()); nodeReportStr.Write("\tLast-Health-Update : "); nodeReportStr.WriteLine(DateFormatUtils.Format(Sharpen.Extensions.CreateDate(nodeReport .GetLastHealthReportTime()), "E dd/MMM/yy hh:mm:ss:SSzz")); nodeReportStr.Write("\tHealth-Report : "); nodeReportStr.WriteLine(nodeReport.GetHealthReport()); nodeReportStr.Write("\tContainers : "); nodeReportStr.WriteLine(nodeReport.GetNumContainers()); nodeReportStr.Write("\tMemory-Used : "); nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0MB" : (nodeReport.GetUsed ().GetMemory() + "MB")); nodeReportStr.Write("\tMemory-Capacity : "); nodeReportStr.WriteLine(nodeReport.GetCapability().GetMemory() + "MB"); nodeReportStr.Write("\tCPU-Used : "); nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0 vcores" : (nodeReport .GetUsed().GetVirtualCores() + " vcores")); nodeReportStr.Write("\tCPU-Capacity : "); nodeReportStr.WriteLine(nodeReport.GetCapability().GetVirtualCores() + " vcores"); nodeReportStr.Write("\tNode-Labels : "); // Create a List for node labels since we need it get sorted IList <string> nodeLabelsList = new AList <string>(report.GetNodeLabels()); nodeLabelsList.Sort(); nodeReportStr.WriteLine(StringUtils.Join(nodeLabelsList.GetEnumerator(), ',')); } if (nodeReport == null) { nodeReportStr.Write("Could not find the node report for node id : " + nodeIdStr); } nodeReportStr.Close(); sysout.WriteLine(baos.ToString("UTF-8")); }
public virtual void TestPipesReduser() { FilePath[] psw = CleanTokenPasswordFile(); JobConf conf = new JobConf(); try { Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString ("password"), new Text("kind"), new Text("service")); TokenCache.SetJobToken(token, conf.GetCredentials()); FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeReducerStub" ); conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath()); PipesReducer <BooleanWritable, Text, IntWritable, Text> reducer = new PipesReducer <BooleanWritable, Text, IntWritable, Text>(); reducer.Configure(conf); BooleanWritable bw = new BooleanWritable(true); conf.Set(MRJobConfig.TaskAttemptId, taskName); InitStdOut(conf); conf.SetBoolean(MRJobConfig.SkipRecords, true); TestPipeApplication.CombineOutputCollector <IntWritable, Text> output = new TestPipeApplication.CombineOutputCollector <IntWritable, Text>(this, new Counters.Counter(), new TestPipeApplication.Progress (this)); Reporter reporter = new TestPipeApplication.TestTaskReporter(this); IList <Text> texts = new AList <Text>(); texts.AddItem(new Text("first")); texts.AddItem(new Text("second")); texts.AddItem(new Text("third")); reducer.Reduce(bw, texts.GetEnumerator(), output, reporter); reducer.Close(); string stdOut = ReadStdOut(conf); // test data: key NUnit.Framework.Assert.IsTrue(stdOut.Contains("reducer key :true")); // and values NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :first")); NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :second")); NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :third")); } finally { if (psw != null) { // remove password files foreach (FilePath file in psw) { file.DeleteOnExit(); } } } }
internal CppRecord(JRecord _enclosing, string name, AList <JField <JType> > flist) : base(_enclosing) { this._enclosing = _enclosing; this.fullName = name.ReplaceAll("\\.", "::"); int idx = name.LastIndexOf('.'); this.name = Runtime.Substring(name, idx + 1); this.module = Runtime.Substring(name, 0, idx).ReplaceAll("\\.", "::"); for (IEnumerator <JField <JType> > iter = flist.GetEnumerator(); iter.HasNext();) { JField <JType> f = iter.Next(); this.fields.AddItem(new JField <JType.CppType>(f.GetName(), f.GetType().GetCppType ())); } }
/// <summary>Creates a new instance of JRecord</summary> public JRecord(string name, AList <JField <JType> > flist) { SetJavaType(new JRecord.JavaRecord(this, name, flist)); SetCppType(new JRecord.CppRecord(this, name, flist)); SetCType(new JRecord.CRecord(this)); // precompute signature int idx = name.LastIndexOf('.'); string recName = Runtime.Substring(name, idx + 1); StringBuilder sb = new StringBuilder(); sb.Append("L").Append(recName).Append("("); for (IEnumerator <JField <JType> > i = flist.GetEnumerator(); i.HasNext();) { string s = i.Next().GetType().GetSignature(); sb.Append(s); } sb.Append(")"); signature = sb.ToString(); }
/// <summary>Chooses datanode locations for caching from a list of valid possibilities. /// </summary> /// <remarks> /// Chooses datanode locations for caching from a list of valid possibilities. /// Non-stale nodes are chosen before stale nodes. /// </remarks> /// <param name="possibilities">List of candidate datanodes</param> /// <param name="neededCached">Number of replicas needed</param> /// <param name="staleInterval">Age of a stale datanode</param> /// <returns>A list of chosen datanodes</returns> private static IList <DatanodeDescriptor> ChooseDatanodesForCaching(IList <DatanodeDescriptor > possibilities, int neededCached, long staleInterval) { // Make a copy that we can modify IList <DatanodeDescriptor> targets = new AList <DatanodeDescriptor>(possibilities); // Selected targets IList <DatanodeDescriptor> chosen = new List <DatanodeDescriptor>(); // Filter out stale datanodes IList <DatanodeDescriptor> stale = new List <DatanodeDescriptor>(); IEnumerator <DatanodeDescriptor> it = targets.GetEnumerator(); while (it.HasNext()) { DatanodeDescriptor d = it.Next(); if (d.IsStale(staleInterval)) { it.Remove(); stale.AddItem(d); } } // Select targets while (chosen.Count < neededCached) { // Try to use stale nodes if we're out of non-stale nodes, else we're done if (targets.IsEmpty()) { if (!stale.IsEmpty()) { targets = stale; } else { break; } } // Select a random target DatanodeDescriptor target = ChooseRandomDatanodeByRemainingCapacity(targets); chosen.AddItem(target); targets.Remove(target); } return(chosen); }
/// <exception cref="System.IO.IOException"/> private static void CreateStorageDirs(DataStorage storage, Configuration conf, int numDirs) { IList <Storage.StorageDirectory> dirs = new AList <Storage.StorageDirectory>(); IList <string> dirStrings = new AList <string>(); for (int i = 0; i < numDirs; i++) { FilePath loc = new FilePath(BaseDir + "/data" + i); dirStrings.AddItem(new Path(loc.ToString()).ToUri().ToString()); loc.Mkdirs(); dirs.AddItem(CreateStorageDirectory(loc)); Org.Mockito.Mockito.When(storage.GetStorageDir(i)).ThenReturn(dirs[i]); } string dataDir = StringUtils.Join(",", dirStrings); conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dataDir); Org.Mockito.Mockito.When(storage.DirIterator()).ThenReturn(dirs.GetEnumerator()); Org.Mockito.Mockito.When(storage.GetNumStorageDirs()).ThenReturn(numDirs); }
/// <exception cref="System.IO.IOException"/> private void WriteConfigFile(Path name, AList <string> nodes) { // delete if it already exists if (localFileSys.Exists(name)) { localFileSys.Delete(name, true); } FSDataOutputStream stm = localFileSys.Create(name); if (nodes != null) { for (IEnumerator <string> it = nodes.GetEnumerator(); it.HasNext();) { string node = it.Next(); stm.WriteBytes(node); stm.WriteBytes("\n"); } } stm.Close(); }
/// <summary>Generate C++ code.</summary> /// <remarks> /// Generate C++ code. This method only creates the requested file(s) /// and spits-out file-level elements (such as include statements etc.) /// record-level code is generated by JRecord. /// </remarks> /// <exception cref="System.IO.IOException"/> internal override void GenCode(string name, AList <JFile> ilist, AList <JRecord> rlist , string destDir, AList <string> options) { name = new FilePath(destDir, (new FilePath(name)).GetName()).GetAbsolutePath(); FileWriter cc = new FileWriter(name + ".cc"); try { FileWriter hh = new FileWriter(name + ".hh"); try { string fileName = (new FilePath(name)).GetName(); hh.Write("#ifndef __" + StringUtils.ToUpperCase(fileName).Replace('.', '_') + "__\n" ); hh.Write("#define __" + StringUtils.ToUpperCase(fileName).Replace('.', '_') + "__\n" ); hh.Write("#include \"recordio.hh\"\n"); hh.Write("#include \"recordTypeInfo.hh\"\n"); for (IEnumerator <JFile> iter = ilist.GetEnumerator(); iter.HasNext();) { hh.Write("#include \"" + iter.Next().GetName() + ".hh\"\n"); } cc.Write("#include \"" + fileName + ".hh\"\n"); cc.Write("#include \"utils.hh\"\n"); for (IEnumerator <JRecord> iter_1 = rlist.GetEnumerator(); iter_1.HasNext();) { iter_1.Next().GenCppCode(hh, cc, options); } hh.Write("#endif //" + StringUtils.ToUpperCase(fileName).Replace('.', '_') + "__\n" ); } finally { hh.Close(); } } finally { cc.Close(); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override IList <ContainerReport> GetContainers(ApplicationAttemptId applicationAttemptId ) { IList <ContainerReport> containersForAttempt = new AList <ContainerReport>(); bool appNotFoundInRM = false; try { GetContainersRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainersRequest >(); request.SetApplicationAttemptId(applicationAttemptId); GetContainersResponse response = rmClient.GetContainers(request); Sharpen.Collections.AddAll(containersForAttempt, response.GetContainerList()); } catch (YarnException e) { if (e.GetType() != typeof(ApplicationNotFoundException) || !historyServiceEnabled) { // If Application is not in RM and history service is enabled then we // need to check with history service else throw exception. throw; } appNotFoundInRM = true; } if (historyServiceEnabled) { // Check with AHS even if found in RM because to capture info of finished // containers also IList <ContainerReport> containersListFromAHS = null; try { containersListFromAHS = historyClient.GetContainers(applicationAttemptId); } catch (IOException e) { // History service access might be enabled but system metrics publisher // is disabled hence app not found exception is possible if (appNotFoundInRM) { // app not found in bothM and RM then propagate the exception. throw; } } if (null != containersListFromAHS && containersListFromAHS.Count > 0) { // remove duplicates ICollection <ContainerId> containerIdsToBeKeptFromAHS = new HashSet <ContainerId>(); IEnumerator <ContainerReport> tmpItr = containersListFromAHS.GetEnumerator(); while (tmpItr.HasNext()) { containerIdsToBeKeptFromAHS.AddItem(tmpItr.Next().GetContainerId()); } IEnumerator <ContainerReport> rmContainers = containersForAttempt.GetEnumerator(); while (rmContainers.HasNext()) { ContainerReport tmp = rmContainers.Next(); containerIdsToBeKeptFromAHS.Remove(tmp.GetContainerId()); } // Remove containers from AHS as container from RM will have latest // information if (containerIdsToBeKeptFromAHS.Count > 0 && containersListFromAHS.Count != containerIdsToBeKeptFromAHS .Count) { IEnumerator <ContainerReport> containersFromHS = containersListFromAHS.GetEnumerator (); while (containersFromHS.HasNext()) { ContainerReport containerReport = containersFromHS.Next(); if (containerIdsToBeKeptFromAHS.Contains(containerReport.GetContainerId())) { containersForAttempt.AddItem(containerReport); } } } else { if (containersListFromAHS.Count == containerIdsToBeKeptFromAHS.Count) { Sharpen.Collections.AddAll(containersForAttempt, containersListFromAHS); } } } } return(containersForAttempt); }
public override void Reset() { iter = data.GetEnumerator(); }
private static IList <E> Apply2Previous <K, E>(IList <E> previous, IList <E> clist, IList <E> dlist) where E : Diff.Element <K> { // Assumptions: // (A1) All lists are sorted. // (A2) All elements in dlist must be in previous. // (A3) All elements in clist must be not in tmp = previous - dlist. IList <E> tmp = new AList <E>(previous.Count - dlist.Count); { // tmp = previous - dlist IEnumerator <E> i = previous.GetEnumerator(); foreach (E deleted in dlist) { E e = i.Next(); //since dlist is non-empty, e must exist by (A2). int cmp = 0; for (; (cmp = e.CompareTo(deleted.GetKey())) < 0; e = i.Next()) { tmp.AddItem(e); } Preconditions.CheckState(cmp == 0); } // check (A2) for (; i.HasNext();) { tmp.AddItem(i.Next()); } } IList <E> current = new AList <E>(tmp.Count + clist.Count); { // current = tmp + clist IEnumerator <E> tmpIterator = tmp.GetEnumerator(); IEnumerator <E> cIterator = clist.GetEnumerator(); E t = tmpIterator.HasNext() ? tmpIterator.Next() : null; E c = cIterator.HasNext() ? cIterator.Next() : null; for (; t != null || c != null;) { int cmp = c == null ? 1 : t == null ? -1 : c.CompareTo(t.GetKey()); if (cmp < 0) { current.AddItem(c); c = cIterator.HasNext() ? cIterator.Next() : null; } else { if (cmp > 0) { current.AddItem(t); t = tmpIterator.HasNext() ? tmpIterator.Next() : null; } else { throw new Exception("Violated assumption (A3)."); } } } } return(current); }