/// <exception cref="System.IO.IOException"/> public static void Main(params string[] args) { if (args.Length < 2) { System.Console.Error.WriteLine("Usage HDFSConcat target srcs.."); System.Environment.Exit(0); } Configuration conf = new Configuration(); string uri = conf.Get("fs.default.name", def_uri); Path path = new Path(uri); DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.Get(path.ToUri(), conf ); Path[] srcs = new Path[args.Length - 1]; for (int i = 1; i < args.Length; i++) { srcs[i - 1] = new Path(args[i]); } dfs.Concat(new Path(args[0]), srcs); }
public virtual void TestConcat() { int numFiles = 10; long fileLen = blockSize * 3; HdfsFileStatus fStatus; FSDataInputStream stm; string trg = "/trg"; Path trgPath = new Path(trg); DFSTestUtil.CreateFile(dfs, trgPath, fileLen, ReplFactor, 1); fStatus = nn.GetFileInfo(trg); long trgLen = fStatus.GetLen(); long trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount(); Path[] files = new Path[numFiles]; byte[][] bytes = new byte[][] { new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int )fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen ], new byte[(int)fileLen], new byte[(int)fileLen] }; LocatedBlocks[] lblocks = new LocatedBlocks[numFiles]; long[] lens = new long[numFiles]; stm = dfs.Open(trgPath); stm.ReadFully(0, bytes[0]); stm.Close(); int i; for (i = 0; i < files.Length; i++) { files[i] = new Path("/file" + i); Path path = files[i]; System.Console.Out.WriteLine("Creating file " + path); // make files with different content DFSTestUtil.CreateFile(dfs, path, fileLen, ReplFactor, i); fStatus = nn.GetFileInfo(path.ToUri().GetPath()); lens[i] = fStatus.GetLen(); NUnit.Framework.Assert.AreEqual(trgLen, lens[i]); // file of the same length. lblocks[i] = nn.GetBlockLocations(path.ToUri().GetPath(), 0, lens[i]); //read the file stm = dfs.Open(path); stm.ReadFully(0, bytes[i + 1]); //bytes[i][10] = 10; stm.Close(); } // check permissions -try the operation with the "wrong" user UserGroupInformation user1 = UserGroupInformation.CreateUserForTesting("theDoctor" , new string[] { "tardis" }); DistributedFileSystem hdfs = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(user1 , conf); try { hdfs.Concat(trgPath, files); NUnit.Framework.Assert.Fail("Permission exception expected"); } catch (IOException ie) { System.Console.Out.WriteLine("Got expected exception for permissions:" + ie.GetLocalizedMessage ()); } // expected // check count update ContentSummary cBefore = dfs.GetContentSummary(trgPath.GetParent()); // resort file array, make INode id not sorted. for (int j = 0; j < files.Length / 2; j++) { Path tempPath = files[j]; files[j] = files[files.Length - 1 - j]; files[files.Length - 1 - j] = tempPath; byte[] tempBytes = bytes[1 + j]; bytes[1 + j] = bytes[files.Length - 1 - j + 1]; bytes[files.Length - 1 - j + 1] = tempBytes; } // now concatenate dfs.Concat(trgPath, files); // verify count ContentSummary cAfter = dfs.GetContentSummary(trgPath.GetParent()); NUnit.Framework.Assert.AreEqual(cBefore.GetFileCount(), cAfter.GetFileCount() + files .Length); // verify other stuff long totalLen = trgLen; long totalBlocks = trgBlocks; for (i = 0; i < files.Length; i++) { totalLen += lens[i]; totalBlocks += lblocks[i].LocatedBlockCount(); } System.Console.Out.WriteLine("total len=" + totalLen + "; totalBlocks=" + totalBlocks ); fStatus = nn.GetFileInfo(trg); trgLen = fStatus.GetLen(); // new length // read the resulting file stm = dfs.Open(trgPath); byte[] byteFileConcat = new byte[(int)trgLen]; stm.ReadFully(0, byteFileConcat); stm.Close(); trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount(); //verifications // 1. number of blocks NUnit.Framework.Assert.AreEqual(trgBlocks, totalBlocks); // 2. file lengths NUnit.Framework.Assert.AreEqual(trgLen, totalLen); // 3. removal of the src file foreach (Path p in files) { fStatus = nn.GetFileInfo(p.ToUri().GetPath()); NUnit.Framework.Assert.IsNull("File " + p + " still exists", fStatus); // file shouldn't exist // try to create fie with the same name DFSTestUtil.CreateFile(dfs, p, fileLen, ReplFactor, 1); } // 4. content CheckFileContent(byteFileConcat, bytes); // add a small file (less then a block) Path smallFile = new Path("/sfile"); int sFileLen = 10; DFSTestUtil.CreateFile(dfs, smallFile, sFileLen, ReplFactor, 1); dfs.Concat(trgPath, new Path[] { smallFile }); fStatus = nn.GetFileInfo(trg); trgLen = fStatus.GetLen(); // new length // check number of blocks trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount(); NUnit.Framework.Assert.AreEqual(trgBlocks, totalBlocks + 1); // and length NUnit.Framework.Assert.AreEqual(trgLen, totalLen + sFileLen); }