public virtual void TestListLocatedStatus()
{
string testHarPath = this.GetType().GetResource("/test.har").AbsolutePath;
URI uri = new URI("har://" + testHarPath);
HarFileSystem hfs = new HarFileSystem(localFileSystem);
hfs.Initialize(uri, new Configuration());
// test.har has the following contents:
// dir1/1.txt
// dir1/2.txt
ICollection <string> expectedFileNames = new HashSet <string>();
expectedFileNames.AddItem("1.txt");
expectedFileNames.AddItem("2.txt");
// List contents of dir, and ensure we find all expected files
Path path = new Path("dir1");
RemoteIterator <LocatedFileStatus> fileList = hfs.ListLocatedStatus(path);
while (fileList.HasNext())
{
string fileName = fileList.Next().GetPath().GetName();
Assert.True(fileName + " not in expected files list", expectedFileNames
.Contains(fileName));
expectedFileNames.Remove(fileName);
}
Assert.Equal("Didn't find all of the expected file names: " +
expectedFileNames, 0, expectedFileNames.Count);
}
public virtual void TestNegativeInitWithAnUnsupportedVersion()
{
// NB: should wait at least 1 second to ensure the timestamp of the master
// index will change upon the writing, because Linux seems to update the
// file modification
// time with 1 second accuracy:
Thread.Sleep(1000);
// write an unsupported version:
WriteVersionToMasterIndexImpl(7777, new Path(harPath, "_masterindex"));
// init the Har:
HarFileSystem hfs = new HarFileSystem(localFileSystem);
// the metadata should *not* be reused from cache:
NUnit.Framework.Assert.IsFalse(hfs.GetMetadata() == harFileSystem.GetMetadata());
URI uri = new URI("har://" + harPath.ToString());
try
{
hfs.Initialize(uri, new Configuration());
NUnit.Framework.Assert.Fail("IOException expected.");
}
catch (IOException)
{
}
}
public virtual void TestPositiveInitWithoutUnderlyingFS()
{
// Init HarFS with no constructor arg, so that the underlying FS object
// is created on demand or got from cache in #initialize() method.
HarFileSystem hfs = new HarFileSystem();
URI uri = new URI("har://" + harPath.ToString());
hfs.Initialize(uri, new Configuration());
}
public virtual void TestPositiveNewHarFsOnTheSameUnderlyingFs()
{
// Init 2nd har file system on the same underlying FS, so the
// metadata gets reused:
HarFileSystem hfs = new HarFileSystem(localFileSystem);
URI uri = new URI("har://" + harPath.ToString());
hfs.Initialize(uri, new Configuration());
// the metadata should be reused from cache:
Assert.True(hfs.GetMetadata() == harFileSystem.GetMetadata());
}
public virtual void TestNegativeGetHarVersionOnNotInitializedFS()
{
HarFileSystem hfs = new HarFileSystem(localFileSystem);
try
{
int version = hfs.GetHarVersion();
NUnit.Framework.Assert.Fail("Exception expected, but got a Har version " + version
+ ".");
}
catch (IOException)
{
}
}
public virtual void Before()
{
FilePath rootDirIoFile = new FilePath(rootPath.ToUri().GetPath());
rootDirIoFile.Mkdirs();
if (!rootDirIoFile.Exists())
{
throw new IOException("Failed to create temp directory [" + rootDirIoFile.GetAbsolutePath
() + "]");
}
// create Har to test:
conf = new Configuration();
harFileSystem = CreateHarFileSystem(conf);
}
/// <exception cref="System.Exception"/>
private HarFileSystem CreateHarFileSystem(Configuration conf, Path aHarPath)
{
localFileSystem.Mkdirs(aHarPath);
Path indexPath = new Path(aHarPath, "_index");
Path masterIndexPath = new Path(aHarPath, "_masterindex");
localFileSystem.CreateNewFile(indexPath);
Assert.True(localFileSystem.Exists(indexPath));
localFileSystem.CreateNewFile(masterIndexPath);
Assert.True(localFileSystem.Exists(masterIndexPath));
WriteVersionToMasterIndexImpl(HarFileSystem.Version, masterIndexPath);
HarFileSystem harFileSystem = new HarFileSystem(localFileSystem);
URI uri = new URI("har://" + aHarPath.ToString());
harFileSystem.Initialize(uri, conf);
return(harFileSystem);
}
public virtual void TestFileChecksum()
{
Path p = new Path("har://file-localhost/foo.har/file1");
HarFileSystem harfs = new HarFileSystem();
try
{
Assert.Equal(null, harfs.GetFileChecksum(p));
}
finally
{
if (harfs != null)
{
harfs.Close();
}
}
}
public virtual void TestNegativeInitWithoutIndex()
{
// delete the index file:
Path indexPath = new Path(harPath, "_index");
localFileSystem.Delete(indexPath, false);
// now init the HarFs:
HarFileSystem hfs = new HarFileSystem(localFileSystem);
URI uri = new URI("har://" + harPath.ToString());
try
{
hfs.Initialize(uri, new Configuration());
NUnit.Framework.Assert.Fail("Exception expected.");
}
catch (IOException)
{
}
}
public virtual void TestPositiveLruMetadataCacheFs()
{
// Init 2nd har file system on the same underlying FS, so the
// metadata gets reused:
HarFileSystem hfs = new HarFileSystem(localFileSystem);
URI uri = new URI("har://" + harPath.ToString());
hfs.Initialize(uri, new Configuration());
// the metadata should be reused from cache:
Assert.True(hfs.GetMetadata() == harFileSystem.GetMetadata());
// Create more hars, until the cache is full + 1; the last creation should evict the first entry from the cache
for (int i = 0; i <= hfs.MetadataCacheEntriesDefault; i++)
{
Path p = new Path(rootPath, "path1/path2/my" + i + ".har");
CreateHarFileSystem(conf, p);
}
// The first entry should not be in the cache anymore:
hfs = new HarFileSystem(localFileSystem);
uri = new URI("har://" + harPath.ToString());
hfs.Initialize(uri, new Configuration());
Assert.True(hfs.GetMetadata() != harFileSystem.GetMetadata());
}
public virtual void After()
{
// close Har FS:
FileSystem harFS = harFileSystem;
if (harFS != null)
{
harFS.Close();
harFileSystem = null;
}
// cleanup: delete all the temporary files:
FilePath rootDirIoFile = new FilePath(rootPath.ToUri().GetPath());
if (rootDirIoFile.Exists())
{
FileUtil.FullyDelete(rootDirIoFile);
}
if (rootDirIoFile.Exists())
{
throw new IOException("Failed to delete temp directory [" + rootDirIoFile.GetAbsolutePath
() + "]");
}
}
public virtual void TestFixBlockLocations()
{
{
// do some tests where start == 0
// case 1: range starts before current har block and ends after
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 0, 20, 5);
Assert.Equal(b[0].GetOffset(), 5);
Assert.Equal(b[0].GetLength(), 10);
}
{
// case 2: range starts in current har block and ends after
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 0, 20, 15);
Assert.Equal(b[0].GetOffset(), 0);
Assert.Equal(b[0].GetLength(), 5);
}
{
// case 3: range starts before current har block and ends in
// current har block
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 0, 10, 5);
Assert.Equal(b[0].GetOffset(), 5);
Assert.Equal(b[0].GetLength(), 5);
}
{
// case 4: range starts and ends in current har block
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 0, 6, 12);
Assert.Equal(b[0].GetOffset(), 0);
Assert.Equal(b[0].GetLength(), 6);
}
{
// now try a range where start == 3
// case 5: range starts before current har block and ends after
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 3, 20, 5);
Assert.Equal(b[0].GetOffset(), 5);
Assert.Equal(b[0].GetLength(), 10);
}
{
// case 6: range starts in current har block and ends after
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 3, 20, 15);
Assert.Equal(b[0].GetOffset(), 3);
Assert.Equal(b[0].GetLength(), 2);
}
{
// case 7: range starts before current har block and ends in
// current har block
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 3, 7, 5);
Assert.Equal(b[0].GetOffset(), 5);
Assert.Equal(b[0].GetLength(), 5);
}
{
// case 8: range starts and ends in current har block
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 10, 10) };
HarFileSystem.FixBlockLocations(b, 3, 3, 12);
Assert.Equal(b[0].GetOffset(), 3);
Assert.Equal(b[0].GetLength(), 3);
}
{
// test case from JIRA MAPREDUCE-1752
BlockLocation[] b = new BlockLocation[] { new BlockLocation(null, null, 512, 512)
, new BlockLocation(null, null, 1024, 512) };
HarFileSystem.FixBlockLocations(b, 0, 512, 896);
Assert.Equal(b[0].GetOffset(), 0);
Assert.Equal(b[0].GetLength(), 128);
Assert.Equal(b[1].GetOffset(), 128);
Assert.Equal(b[1].GetLength(), 384);
}
}
