/// <exception cref="System.IO.IOException"/>
public static void Main(params string[] args)
{
if (args.Length < 2)
{
System.Console.Error.WriteLine("Usage HDFSConcat target srcs..");
System.Environment.Exit(0);
}
Configuration conf = new Configuration();
string uri = conf.Get("fs.default.name", def_uri);
Path path = new Path(uri);
DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.Get(path.ToUri(), conf
);
Path[] srcs = new Path[args.Length - 1];
for (int i = 1; i < args.Length; i++)
{
srcs[i - 1] = new Path(args[i]);
}
dfs.Concat(new Path(args[0]), srcs);
}
public virtual void TestConcat()
{
int numFiles = 10;
long fileLen = blockSize * 3;
HdfsFileStatus fStatus;
FSDataInputStream stm;
string trg = "/trg";
Path trgPath = new Path(trg);
DFSTestUtil.CreateFile(dfs, trgPath, fileLen, ReplFactor, 1);
fStatus = nn.GetFileInfo(trg);
long trgLen = fStatus.GetLen();
long trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount();
Path[] files = new Path[numFiles];
byte[][] bytes = new byte[][] { new byte[(int)fileLen], new byte[(int)fileLen], new
byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int
)fileLen], new byte[(int)fileLen], new byte[(int)fileLen], new byte[(int)fileLen
], new byte[(int)fileLen], new byte[(int)fileLen] };
LocatedBlocks[] lblocks = new LocatedBlocks[numFiles];
long[] lens = new long[numFiles];
stm = dfs.Open(trgPath);
stm.ReadFully(0, bytes[0]);
stm.Close();
int i;
for (i = 0; i < files.Length; i++)
{
files[i] = new Path("/file" + i);
Path path = files[i];
System.Console.Out.WriteLine("Creating file " + path);
// make files with different content
DFSTestUtil.CreateFile(dfs, path, fileLen, ReplFactor, i);
fStatus = nn.GetFileInfo(path.ToUri().GetPath());
lens[i] = fStatus.GetLen();
NUnit.Framework.Assert.AreEqual(trgLen, lens[i]);
// file of the same length.
lblocks[i] = nn.GetBlockLocations(path.ToUri().GetPath(), 0, lens[i]);
//read the file
stm = dfs.Open(path);
stm.ReadFully(0, bytes[i + 1]);
//bytes[i][10] = 10;
stm.Close();
}
// check permissions -try the operation with the "wrong" user
UserGroupInformation user1 = UserGroupInformation.CreateUserForTesting("theDoctor"
, new string[] { "tardis" });
DistributedFileSystem hdfs = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(user1
, conf);
try
{
hdfs.Concat(trgPath, files);
NUnit.Framework.Assert.Fail("Permission exception expected");
}
catch (IOException ie)
{
System.Console.Out.WriteLine("Got expected exception for permissions:" + ie.GetLocalizedMessage
());
}
// expected
// check count update
ContentSummary cBefore = dfs.GetContentSummary(trgPath.GetParent());
// resort file array, make INode id not sorted.
for (int j = 0; j < files.Length / 2; j++)
{
Path tempPath = files[j];
files[j] = files[files.Length - 1 - j];
files[files.Length - 1 - j] = tempPath;
byte[] tempBytes = bytes[1 + j];
bytes[1 + j] = bytes[files.Length - 1 - j + 1];
bytes[files.Length - 1 - j + 1] = tempBytes;
}
// now concatenate
dfs.Concat(trgPath, files);
// verify count
ContentSummary cAfter = dfs.GetContentSummary(trgPath.GetParent());
NUnit.Framework.Assert.AreEqual(cBefore.GetFileCount(), cAfter.GetFileCount() + files
.Length);
// verify other stuff
long totalLen = trgLen;
long totalBlocks = trgBlocks;
for (i = 0; i < files.Length; i++)
{
totalLen += lens[i];
totalBlocks += lblocks[i].LocatedBlockCount();
}
System.Console.Out.WriteLine("total len=" + totalLen + "; totalBlocks=" + totalBlocks
);
fStatus = nn.GetFileInfo(trg);
trgLen = fStatus.GetLen();
// new length
// read the resulting file
stm = dfs.Open(trgPath);
byte[] byteFileConcat = new byte[(int)trgLen];
stm.ReadFully(0, byteFileConcat);
stm.Close();
trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount();
//verifications
// 1. number of blocks
NUnit.Framework.Assert.AreEqual(trgBlocks, totalBlocks);
// 2. file lengths
NUnit.Framework.Assert.AreEqual(trgLen, totalLen);
// 3. removal of the src file
foreach (Path p in files)
{
fStatus = nn.GetFileInfo(p.ToUri().GetPath());
NUnit.Framework.Assert.IsNull("File " + p + " still exists", fStatus);
// file shouldn't exist
// try to create fie with the same name
DFSTestUtil.CreateFile(dfs, p, fileLen, ReplFactor, 1);
}
// 4. content
CheckFileContent(byteFileConcat, bytes);
// add a small file (less then a block)
Path smallFile = new Path("/sfile");
int sFileLen = 10;
DFSTestUtil.CreateFile(dfs, smallFile, sFileLen, ReplFactor, 1);
dfs.Concat(trgPath, new Path[] { smallFile });
fStatus = nn.GetFileInfo(trg);
trgLen = fStatus.GetLen();
// new length
// check number of blocks
trgBlocks = nn.GetBlockLocations(trg, 0, trgLen).LocatedBlockCount();
NUnit.Framework.Assert.AreEqual(trgBlocks, totalBlocks + 1);
// and length
NUnit.Framework.Assert.AreEqual(trgLen, totalLen + sFileLen);
}