/// <exception cref="System.Exception"/> public virtual void TestInvalidReduceNumberOrLength() { fs.Delete(p, true); conf.SetInt(TTConfig.TtIndexCache, 1); int partsPerMap = 1000; int bytesPerFile = partsPerMap * 24; IndexCache cache = new IndexCache(conf); // fill cache Path feq = new Path(p, "invalidReduceOrPartsPerMap"); WriteFile(fs, feq, bytesPerFile, partsPerMap); // Number of reducers should always be less than partsPerMap as reducer // numbers start from 0 and there cannot be more reducer than parts try { // Number of reducers equal to partsPerMap cache.GetIndexInformation("reduceEqualPartsPerMap", partsPerMap, feq, UserGroupInformation .GetCurrentUser().GetShortUserName()); // reduce number == partsPerMap Fail("Number of reducers equal to partsPerMap did not fail"); } catch (Exception e) { if (!(e is IOException)) { throw; } } try { // Number of reducers more than partsPerMap cache.GetIndexInformation("reduceMorePartsPerMap", partsPerMap + 1, feq, UserGroupInformation .GetCurrentUser().GetShortUserName()); // reduce number > partsPerMap Fail("Number of reducers more than partsPerMap did not fail"); } catch (Exception e) { if (!(e is IOException)) { throw; } } }
/// <exception cref="System.Exception"/> public virtual void TestBadIndex() { int parts = 30; fs.Delete(p, true); conf.SetInt(TTConfig.TtIndexCache, 1); IndexCache cache = new IndexCache(conf); Path f = new Path(p, "badindex"); FSDataOutputStream @out = fs.Create(f, false); CheckedOutputStream iout = new CheckedOutputStream(@out, new CRC32()); DataOutputStream dout = new DataOutputStream(iout); for (int i = 0; i < parts; ++i) { for (int j = 0; j < MapTask.MapOutputIndexRecordLength / 8; ++j) { if (0 == (i % 3)) { dout.WriteLong(i); } else { @out.WriteLong(i); } } } @out.WriteLong(iout.GetChecksum().GetValue()); dout.Close(); try { cache.GetIndexInformation("badindex", 7, f, UserGroupInformation.GetCurrentUser() .GetShortUserName()); Fail("Did not detect bad checksum"); } catch (IOException e) { if (!(e.InnerException is ChecksumException)) { throw; } } }
/// <exception cref="System.Exception"/> public virtual void TestLRCPolicy() { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); System.Console.Out.WriteLine("seed: " + seed); fs.Delete(p, true); conf.SetInt(TTConfig.TtIndexCache, 1); int partsPerMap = 1000; int bytesPerFile = partsPerMap * 24; IndexCache cache = new IndexCache(conf); // fill cache int totalsize = bytesPerFile; for (; totalsize < 1024 * 1024; totalsize += bytesPerFile) { Path f = new Path(p, Sharpen.Extensions.ToString(totalsize, 36)); WriteFile(fs, f, totalsize, partsPerMap); IndexRecord rec = cache.GetIndexInformation(Sharpen.Extensions.ToString(totalsize , 36), r.Next(partsPerMap), f, UserGroupInformation.GetCurrentUser().GetShortUserName ()); CheckRecord(rec, totalsize); } // delete files, ensure cache retains all elem foreach (FileStatus stat in fs.ListStatus(p)) { fs.Delete(stat.GetPath(), true); } for (int i = bytesPerFile; i < 1024 * 1024; i += bytesPerFile) { Path f = new Path(p, Sharpen.Extensions.ToString(i, 36)); IndexRecord rec = cache.GetIndexInformation(Sharpen.Extensions.ToString(i, 36), r .Next(partsPerMap), f, UserGroupInformation.GetCurrentUser().GetShortUserName()); CheckRecord(rec, i); } // push oldest (bytesPerFile) out of cache Path f_1 = new Path(p, Sharpen.Extensions.ToString(totalsize, 36)); WriteFile(fs, f_1, totalsize, partsPerMap); cache.GetIndexInformation(Sharpen.Extensions.ToString(totalsize, 36), r.Next(partsPerMap ), f_1, UserGroupInformation.GetCurrentUser().GetShortUserName()); fs.Delete(f_1, false); // oldest fails to read, or error bool fnf = false; try { cache.GetIndexInformation(Sharpen.Extensions.ToString(bytesPerFile, 36), r.Next(partsPerMap ), new Path(p, Sharpen.Extensions.ToString(bytesPerFile)), UserGroupInformation. GetCurrentUser().GetShortUserName()); } catch (IOException e) { if (e.InnerException == null || !(e.InnerException is FileNotFoundException)) { throw; } else { fnf = true; } } if (!fnf) { Fail("Failed to push out last entry"); } // should find all the other entries for (int i_1 = bytesPerFile << 1; i_1 < 1024 * 1024; i_1 += bytesPerFile) { IndexRecord rec = cache.GetIndexInformation(Sharpen.Extensions.ToString(i_1, 36), r.Next(partsPerMap), new Path(p, Sharpen.Extensions.ToString(i_1, 36)), UserGroupInformation .GetCurrentUser().GetShortUserName()); CheckRecord(rec, i_1); } IndexRecord rec_1 = cache.GetIndexInformation(Sharpen.Extensions.ToString(totalsize , 36), r.Next(partsPerMap), f_1, UserGroupInformation.GetCurrentUser().GetShortUserName ()); CheckRecord(rec_1, totalsize); }