internal static void CheckEqual(LocalResourceRequest a, LocalResourceRequest b)
{
NUnit.Framework.Assert.AreEqual(a, b);
NUnit.Framework.Assert.AreEqual(a.GetHashCode(), b.GetHashCode());
NUnit.Framework.Assert.AreEqual(0, a.CompareTo(b));
NUnit.Framework.Assert.AreEqual(0, b.CompareTo(a));
}
public _LocalizedResource_93(long rsrcSize, Path p, long ts, LocalResourceRequest
baseArg1, Dispatcher baseArg2)
: base(baseArg1, baseArg2)
{
this.rsrcSize = rsrcSize;
this.p = p;
this.ts = ts;
}
internal static void CheckNotEqual(LocalResourceRequest a, LocalResourceRequest b
)
{
NUnit.Framework.Assert.IsFalse(a.Equals(b));
NUnit.Framework.Assert.IsFalse(b.Equals(a));
NUnit.Framework.Assert.IsFalse(a.GetHashCode() == b.GetHashCode());
NUnit.Framework.Assert.IsFalse(0 == a.CompareTo(b));
NUnit.Framework.Assert.IsFalse(0 == b.CompareTo(a));
}
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// absolute path for localization which includes local
/// directory path and the relative hierarchical path (if use local
/// cache directory manager is enabled)
/// </returns>
/// <?/>
/// <?/>
/// <?/>
public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
, DeletionService delService)
{
Path rPath = localDirPath;
if (useLocalCacheDirectoryManager && localDirPath != null)
{
if (!directoryManagers.Contains(localDirPath))
{
directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
}
LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
rPath = localDirPath;
string hierarchicalPath = dir.GetRelativePathForLocalization();
// For most of the scenarios we will get root path only which
// is an empty string
if (!hierarchicalPath.IsEmpty())
{
rPath = new Path(localDirPath, hierarchicalPath);
}
inProgressLocalResourcesMap[req] = rPath;
}
while (true)
{
Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
if (!file.Exists())
{
rPath = uniquePath;
break;
}
// If the directory already exists, delete it and move to next one.
Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
if (delService != null)
{
delService.Delete(GetUser(), uniquePath);
}
}
Path localPath = new Path(rPath, req.GetPath().GetName());
LocalizedResource rsrc = localrsrc[req];
rsrc.SetLocalPath(localPath);
LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
.GetVisibility(), req.GetSize(), req.GetTimestamp());
try
{
stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
(), localPath);
}
catch (IOException e)
{
Log.Error("Unable to record localization start for " + rsrc, e);
}
return(rPath);
}
public virtual void TestResourceEquality()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
long basetime = (long)(((ulong)r.NextLong()) >> 2);
LocalResource yA = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime
, LocalResourceType.File, LocalResourceVisibility.Public, null);
LocalResource yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime
, LocalResourceType.File, LocalResourceVisibility.Public, null);
LocalResourceRequest a = new LocalResourceRequest(yA);
LocalResourceRequest b = new LocalResourceRequest(yA);
CheckEqual(a, b);
b = new LocalResourceRequest(yB);
CheckEqual(a, b);
// ignore visibility
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType
.File, LocalResourceVisibility.Private, null);
b = new LocalResourceRequest(yB);
CheckEqual(a, b);
// ignore size
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime, LocalResourceType
.File, LocalResourceVisibility.Private, null);
b = new LocalResourceRequest(yB);
CheckEqual(a, b);
// note path
yB = GetYarnResource(new Path("hdfs://dingo.org:80/foobar"), 0, basetime, LocalResourceType
.Archive, LocalResourceVisibility.Public, null);
b = new LocalResourceRequest(yB);
CheckNotEqual(a, b);
// note type
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime, LocalResourceType
.Archive, LocalResourceVisibility.Public, null);
b = new LocalResourceRequest(yB);
CheckNotEqual(a, b);
// note timestamp
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime + 1, LocalResourceType
.File, LocalResourceVisibility.Public, null);
b = new LocalResourceRequest(yB);
CheckNotEqual(a, b);
// note pattern
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime + 1, LocalResourceType
.File, LocalResourceVisibility.Public, "^/foo/.*");
b = new LocalResourceRequest(yB);
CheckNotEqual(a, b);
}
/*
* Update the file-count statistics for a local cache-directory.
* This will retrieve the localized path for the resource from
* 1) inProgressRsrcMap if the resource was under localization and it
* failed.
* 2) LocalizedResource if the resource is already localized.
* From this path it will identify the local directory under which the
* resource was localized. Then rest of the path will be used to decrement
* file count for the HierarchicalSubDirectory pointing to this relative
* path.
*/
private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource
rsrc)
{
if (useLocalCacheDirectoryManager)
{
Path rsrcPath = null;
if (inProgressLocalResourcesMap.Contains(req))
{
// This happens when localization of a resource fails.
rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
else
{
if (rsrc != null && rsrc.GetLocalPath() != null)
{
rsrcPath = rsrc.GetLocalPath().GetParent().GetParent();
}
}
if (rsrcPath != null)
{
Path parentPath = new Path(rsrcPath.ToUri().GetRawPath());
while (!directoryManagers.Contains(parentPath))
{
parentPath = parentPath.GetParent();
if (parentPath == null)
{
return;
}
}
if (parentPath != null)
{
string parentDir = parentPath.ToUri().GetRawPath().ToString();
LocalCacheDirectoryManager dir = directoryManagers[parentPath];
string rsrcDir = rsrcPath.ToUri().GetRawPath();
if (rsrcDir.Equals(parentDir))
{
dir.DecrementFileCountForPath(string.Empty);
}
else
{
dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length
+ 1));
}
}
}
}
}
public LocalizedResource(LocalResourceRequest rsrc, Dispatcher dispatcher)
{
timestamp = new AtomicLong(CurrentTime());
// Queue of containers using this localized
// resource
// From INIT (ref == 0, awaiting req)
// From DOWNLOADING (ref > 0, may be localizing)
// TODO: Duplicate addition!!
// From LOCALIZED (ref >= 0, on disk)
this.rsrc = rsrc;
this.dispatcher = dispatcher;
this.@ref = new List <ContainerId>();
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.ReadLock();
this.writeLock = readWriteLock.WriteLock();
this.stateMachine = stateMachineFactory.Make(this);
}
private LocalizedResource RecoverResource(LocalResourceRequest req, ResourceRecoveredEvent
@event)
{
// unique number for a resource is the directory of the resource
Path localDir = @event.GetLocalPath().GetParent();
long rsrcId = long.Parse(localDir.GetName());
// update ID generator to avoid conflicts with existing resources
while (true)
{
long currentRsrcId = uniqueNumberGenerator.Get();
long nextRsrcId = Math.Max(currentRsrcId, rsrcId);
if (uniqueNumberGenerator.CompareAndSet(currentRsrcId, nextRsrcId))
{
break;
}
}
IncrementFileCountForLocalCacheDirectory(localDir.GetParent());
return(new LocalizedResource(req, dispatcher));
}
private void RemoveResource(LocalResourceRequest req)
{
LocalizedResource rsrc = Sharpen.Collections.Remove(localrsrc, req);
DecrementFileCountForLocalCacheDirectory(req, rsrc);
if (rsrc != null)
{
Path localPath = rsrc.GetLocalPath();
if (localPath != null)
{
try
{
stateStore.RemoveLocalizedResource(user, appId, localPath);
}
catch (IOException e)
{
Log.Error("Unable to remove resource " + rsrc + " from state store", e);
}
}
}
}
internal virtual LocalResourcesTracker CreateMockTracker(string user, long rsrcSize
, long nRsrcs, long timestamp, long tsstep)
{
Configuration conf = new Configuration();
ConcurrentMap <LocalResourceRequest, LocalizedResource> trackerResources = new ConcurrentHashMap
<LocalResourceRequest, LocalizedResource>();
LocalResourcesTracker ret = Org.Mockito.Mockito.Spy(new LocalResourcesTrackerImpl
(user, null, null, trackerResources, false, conf, new NMNullStateStoreService())
);
for (int i = 0; i < nRsrcs; ++i)
{
LocalResourceRequest req = new LocalResourceRequest(new Path("file:///" + user +
"/rsrc" + i), timestamp + i * tsstep, LocalResourceType.File, LocalResourceVisibility
.Public, null);
long ts = timestamp + i * tsstep;
Path p = new Path("file:///local/" + user + "/rsrc" + i);
LocalizedResource rsrc = new _LocalizedResource_93(rsrcSize, p, ts, req, null);
trackerResources[req] = rsrc;
}
return(ret);
}
public virtual void TestResourceOrder()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
long basetime = (long)(((ulong)r.NextLong()) >> 2);
LocalResource yA = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime
, LocalResourceType.File, LocalResourceVisibility.Public, "^/foo/.*");
LocalResourceRequest a = new LocalResourceRequest(yA);
// Path primary
LocalResource yB = GetYarnResource(new Path("http://yak.org:80/foobaz"), -1, basetime
, LocalResourceType.File, LocalResourceVisibility.Public, "^/foo/.*");
LocalResourceRequest b = new LocalResourceRequest(yB);
NUnit.Framework.Assert.IsTrue(0 > a.CompareTo(b));
// timestamp secondary
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime + 1, LocalResourceType
.File, LocalResourceVisibility.Public, "^/foo/.*");
b = new LocalResourceRequest(yB);
NUnit.Framework.Assert.IsTrue(0 > a.CompareTo(b));
// type tertiary
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType
.Archive, LocalResourceVisibility.Public, "^/foo/.*");
b = new LocalResourceRequest(yB);
NUnit.Framework.Assert.IsTrue(0 != a.CompareTo(b));
// don't care about order, just ne
// path 4th
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType
.Archive, LocalResourceVisibility.Public, "^/food/.*");
b = new LocalResourceRequest(yB);
NUnit.Framework.Assert.IsTrue(0 != a.CompareTo(b));
// don't care about order, just ne
yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType
.Archive, LocalResourceVisibility.Public, null);
b = new LocalResourceRequest(yB);
NUnit.Framework.Assert.IsTrue(0 != a.CompareTo(b));
}
public virtual void TestNotification()
{
// mocked generic
DrainDispatcher dispatcher = new DrainDispatcher();
dispatcher.Init(new Configuration());
try
{
dispatcher.Start();
EventHandler <ContainerEvent> containerBus = Org.Mockito.Mockito.Mock <EventHandler
>();
EventHandler <LocalizerEvent> localizerBus = Org.Mockito.Mockito.Mock <EventHandler
>();
dispatcher.Register(typeof(ContainerEventType), containerBus);
dispatcher.Register(typeof(LocalizerEventType), localizerBus);
// mock resource
LocalResource apiRsrc = CreateMockResource();
ContainerId container0 = GetMockContainer(0L);
Credentials creds0 = new Credentials();
LocalResourceVisibility vis0 = LocalResourceVisibility.Private;
LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
local.Handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
dispatcher.Await();
// Register C0, verify request event
TestLocalizedResource.LocalizerEventMatcher matchesL0Req = new TestLocalizedResource.LocalizerEventMatcher
(container0, creds0, vis0, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL0Req));
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Register C1, verify request event
Credentials creds1 = new Credentials();
ContainerId container1 = GetMockContainer(1L);
LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
LocalResourceVisibility vis1 = LocalResourceVisibility.Public;
local.Handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
dispatcher.Await();
TestLocalizedResource.LocalizerEventMatcher matchesL1Req = new TestLocalizedResource.LocalizerEventMatcher
(container1, creds1, vis1, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL1Req));
// Release C0 container localization, verify no notification
local.Handle(new ResourceReleaseEvent(rsrcA, container0));
dispatcher.Await();
Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA
<ContainerEvent>());
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Release C1 container localization, verify no notification
local.Handle(new ResourceReleaseEvent(rsrcA, container1));
dispatcher.Await();
Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA
<ContainerEvent>());
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Register C2, C3
ContainerId container2 = GetMockContainer(2L);
LocalResourceVisibility vis2 = LocalResourceVisibility.Private;
Credentials creds2 = new Credentials();
LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
ContainerId container3 = GetMockContainer(3L);
LocalResourceVisibility vis3 = LocalResourceVisibility.Private;
Credentials creds3 = new Credentials();
LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
local.Handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
local.Handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
dispatcher.Await();
TestLocalizedResource.LocalizerEventMatcher matchesL2Req = new TestLocalizedResource.LocalizerEventMatcher
(container2, creds2, vis2, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL2Req));
TestLocalizedResource.LocalizerEventMatcher matchesL3Req = new TestLocalizedResource.LocalizerEventMatcher
(container3, creds3, vis3, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL3Req));
// Successful localization. verify notification C2, C3
Path locA = new Path("file:///cache/rsrcA");
local.Handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
dispatcher.Await();
TestLocalizedResource.ContainerEventMatcher matchesC2Localized = new TestLocalizedResource.ContainerEventMatcher
(container2, ContainerEventType.ResourceLocalized);
TestLocalizedResource.ContainerEventMatcher matchesC3Localized = new TestLocalizedResource.ContainerEventMatcher
(container3, ContainerEventType.ResourceLocalized);
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC2Localized
));
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC3Localized
));
NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState());
// Register C4, verify notification
ContainerId container4 = GetMockContainer(4L);
Credentials creds4 = new Credentials();
LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
LocalResourceVisibility vis4 = LocalResourceVisibility.Private;
local.Handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
dispatcher.Await();
TestLocalizedResource.ContainerEventMatcher matchesC4Localized = new TestLocalizedResource.ContainerEventMatcher
(container4, ContainerEventType.ResourceLocalized);
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC4Localized
));
NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState());
}
finally
{
dispatcher.Stop();
}
}
public virtual LocalizedResource GetLocalizedResource(LocalResourceRequest request
)
{
return(localrsrc[request]);
}
/*
* Synchronizing this method for avoiding races due to multiple ResourceEvent's
* coming to LocalResourcesTracker from Public/Private localizer and
* Resource Localization Service.
*/
public virtual void Handle(ResourceEvent @event)
{
lock (this)
{
LocalResourceRequest req = @event.GetLocalResourceRequest();
LocalizedResource rsrc = localrsrc[req];
switch (@event.GetType())
{
case ResourceEventType.Localized:
{
if (useLocalCacheDirectoryManager)
{
Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
break;
}
case ResourceEventType.Request:
{
if (rsrc != null && (!IsResourcePresent(rsrc)))
{
Log.Info("Resource " + rsrc.GetLocalPath() + " is missing, localizing it again");
RemoveResource(req);
rsrc = null;
}
if (null == rsrc)
{
rsrc = new LocalizedResource(req, dispatcher);
localrsrc[req] = rsrc;
}
break;
}
case ResourceEventType.Release:
{
if (null == rsrc)
{
// The container sent a release event on a resource which
// 1) Failed
// 2) Removed for some reason (ex. disk is no longer accessible)
ResourceReleaseEvent relEvent = (ResourceReleaseEvent)@event;
Log.Info("Container " + relEvent.GetContainer() + " sent RELEASE event on a resource request "
+ req + " not present in cache.");
return;
}
break;
}
case ResourceEventType.LocalizationFailed:
{
/*
* If resource localization fails then Localized resource will be
* removed from local cache.
*/
RemoveResource(req);
break;
}
case ResourceEventType.Recovered:
{
if (rsrc != null)
{
Log.Warn("Ignoring attempt to recover existing resource " + rsrc);
return;
}
rsrc = RecoverResource(req, (ResourceRecoveredEvent)@event);
localrsrc[req] = rsrc;
break;
}
}
if (rsrc == null)
{
Log.Warn("Received " + @event.GetType() + " event for request " + req + " but localized resource is missing"
);
return;
}
rsrc.Handle(@event);
// Remove the resource if its downloading and its reference count has
// become 0 after RELEASE. This maybe because a container was killed while
// localizing and no other container is referring to the resource.
// NOTE: This should NOT be done for public resources since the
// download is not associated with a container-specific localizer.
if (@event.GetType() == ResourceEventType.Release)
{
if (rsrc.GetState() == ResourceState.Downloading && rsrc.GetRefCount() <= 0 && rsrc
.GetRequest().GetVisibility() != LocalResourceVisibility.Public)
{
RemoveResource(req);
}
}
if (@event.GetType() == ResourceEventType.Localized)
{
if (rsrc.GetLocalPath() != null)
{
try
{
stateStore.FinishResourceLocalization(user, appId, BuildLocalizedResourceProto(rsrc
));
}
catch (IOException ioe)
{
Log.Error("Error storing resource state for " + rsrc, ioe);
}
}
else
{
Log.Warn("Resource " + rsrc + " localized without a location");
}
}
}
}
