public override void Transition(LocalizedResource rsrc, ResourceEvent @event)
{
ResourceRecoveredEvent recoveredEvent = (ResourceRecoveredEvent)@event;
rsrc.localPath = recoveredEvent.GetLocalPath();
rsrc.size = recoveredEvent.GetSize();
}
public virtual void AddResources(LocalResourcesTracker newTracker)
{
foreach (LocalizedResource resource in newTracker)
{
currentSize += resource.GetSize();
if (resource.GetRefCount() > 0)
{
// always retain resources in use
continue;
}
retain[resource] = newTracker;
}
for (IEnumerator <KeyValuePair <LocalizedResource, LocalResourcesTracker> > i = retain
.GetEnumerator(); currentSize - delSize > targetSize && i.HasNext();)
{
KeyValuePair <LocalizedResource, LocalResourcesTracker> rsrc = i.Next();
LocalizedResource resource_1 = rsrc.Key;
LocalResourcesTracker tracker = rsrc.Value;
if (tracker.Remove(resource_1, delService))
{
delSize += resource_1.GetSize();
i.Remove();
}
}
}
public override void Transition(LocalizedResource rsrc, ResourceEvent @event)
{
// Note: assumes that localizing container must succeed or fail
ResourceReleaseEvent relEvent = (ResourceReleaseEvent)@event;
rsrc.Release(relEvent.GetContainer());
}
public virtual bool Remove(LocalizedResource rem, DeletionService delService)
{
// current synchronization guaranteed by crude RLS event for cleanup
LocalizedResource rsrc = localrsrc[rem.GetRequest()];
if (null == rsrc)
{
Log.Error("Attempt to remove absent resource: " + rem.GetRequest() + " from " + GetUser
());
return(true);
}
if (rsrc.GetRefCount() > 0 || ResourceState.Downloading.Equals(rsrc.GetState()) ||
rsrc != rem)
{
// internal error
Log.Error("Attempt to remove resource: " + rsrc + " with non-zero refcount");
return(false);
}
else
{
// ResourceState is LOCALIZED or INIT
if (ResourceState.Localized.Equals(rsrc.GetState()))
{
delService.Delete(GetUser(), GetPathToDelete(rsrc.GetLocalPath()));
}
RemoveResource(rem.GetRequest());
Log.Info("Removed " + rsrc.GetLocalPath() + " from localized cache");
return(true);
}
}
private YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto BuildLocalizedResourceProto
(LocalizedResource rsrc)
{
return((YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto)YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto
.NewBuilder().SetResource(BuildLocalResourceProto(rsrc.GetRequest())).SetLocalPath
(rsrc.GetLocalPath().ToString()).SetSize(rsrc.GetSize()).Build());
}
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// absolute path for localization which includes local
/// directory path and the relative hierarchical path (if use local
/// cache directory manager is enabled)
/// </returns>
/// <?/>
/// <?/>
/// <?/>
public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
, DeletionService delService)
{
Path rPath = localDirPath;
if (useLocalCacheDirectoryManager && localDirPath != null)
{
if (!directoryManagers.Contains(localDirPath))
{
directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
}
LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
rPath = localDirPath;
string hierarchicalPath = dir.GetRelativePathForLocalization();
// For most of the scenarios we will get root path only which
// is an empty string
if (!hierarchicalPath.IsEmpty())
{
rPath = new Path(localDirPath, hierarchicalPath);
}
inProgressLocalResourcesMap[req] = rPath;
}
while (true)
{
Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
if (!file.Exists())
{
rPath = uniquePath;
break;
}
// If the directory already exists, delete it and move to next one.
Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
if (delService != null)
{
delService.Delete(GetUser(), uniquePath);
}
}
Path localPath = new Path(rPath, req.GetPath().GetName());
LocalizedResource rsrc = localrsrc[req];
rsrc.SetLocalPath(localPath);
LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
.GetVisibility(), req.GetSize(), req.GetTimestamp());
try
{
stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
(), localPath);
}
catch (IOException e)
{
Log.Error("Unable to record localization start for " + rsrc, e);
}
return(rPath);
}
// dispatcher not typed
public override void Transition(LocalizedResource rsrc, ResourceEvent @event)
{
// notify waiting containers
ResourceRequestEvent reqEvent = (ResourceRequestEvent)@event;
ContainerId container = reqEvent.GetContext().GetContainerId();
[email protected](container);
rsrc.dispatcher.GetEventHandler().Handle(new ContainerResourceLocalizedEvent(container
, rsrc.rsrc, rsrc.localPath));
}
// dispatcher not typed
public override void Transition(LocalizedResource rsrc, ResourceEvent @event)
{
ResourceRequestEvent req = (ResourceRequestEvent)@event;
LocalizerContext ctxt = req.GetContext();
ContainerId container = ctxt.GetContainerId();
[email protected](container);
rsrc.dispatcher.GetEventHandler().Handle(new LocalizerResourceRequestEvent(rsrc,
req.GetVisibility(), ctxt, req.GetLocalResourceRequest().GetPattern()));
}
public override void Transition(LocalizedResource rsrc, ResourceEvent @event)
{
ResourceFailedLocalizationEvent failedEvent = (ResourceFailedLocalizationEvent)@event;
Queue <ContainerId> containers = rsrc.@ref;
foreach (ContainerId container in containers)
{
rsrc.dispatcher.GetEventHandler().Handle(new ContainerResourceFailedEvent(container
, failedEvent.GetLocalResourceRequest(), failedEvent.GetDiagnosticMessage()));
}
}
// dispatcher not typed
public override void Transition(LocalizedResource rsrc, ResourceEvent @event)
{
ResourceLocalizedEvent locEvent = (ResourceLocalizedEvent)@event;
rsrc.localPath = Path.GetPathWithoutSchemeAndAuthority(locEvent.GetLocation());
rsrc.size = locEvent.GetSize();
foreach (ContainerId container in rsrc.@ref)
{
rsrc.dispatcher.GetEventHandler().Handle(new ContainerResourceLocalizedEvent(container
, rsrc.rsrc, rsrc.localPath));
}
}
/// <summary>
/// This module checks if the resource which was localized is already present
/// or not
/// </summary>
/// <param name="rsrc"/>
/// <returns>true/false based on resource is present or not</returns>
public virtual bool IsResourcePresent(LocalizedResource rsrc)
{
bool ret = true;
if (rsrc.GetState() == ResourceState.Localized)
{
FilePath file = new FilePath(rsrc.GetLocalPath().ToUri().GetRawPath().ToString());
if (!file.Exists())
{
ret = false;
}
}
return(ret);
}
private void RemoveResource(LocalResourceRequest req)
{
LocalizedResource rsrc = Sharpen.Collections.Remove(localrsrc, req);
DecrementFileCountForLocalCacheDirectory(req, rsrc);
if (rsrc != null)
{
Path localPath = rsrc.GetLocalPath();
if (localPath != null)
{
try
{
stateStore.RemoveLocalizedResource(user, appId, localPath);
}
catch (IOException e)
{
Log.Error("Unable to remove resource " + rsrc + " from state store", e);
}
}
}
}
public virtual void TestNotification()
{
// mocked generic
DrainDispatcher dispatcher = new DrainDispatcher();
dispatcher.Init(new Configuration());
try
{
dispatcher.Start();
EventHandler <ContainerEvent> containerBus = Org.Mockito.Mockito.Mock <EventHandler
>();
EventHandler <LocalizerEvent> localizerBus = Org.Mockito.Mockito.Mock <EventHandler
>();
dispatcher.Register(typeof(ContainerEventType), containerBus);
dispatcher.Register(typeof(LocalizerEventType), localizerBus);
// mock resource
LocalResource apiRsrc = CreateMockResource();
ContainerId container0 = GetMockContainer(0L);
Credentials creds0 = new Credentials();
LocalResourceVisibility vis0 = LocalResourceVisibility.Private;
LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
local.Handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
dispatcher.Await();
// Register C0, verify request event
TestLocalizedResource.LocalizerEventMatcher matchesL0Req = new TestLocalizedResource.LocalizerEventMatcher
(container0, creds0, vis0, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL0Req));
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Register C1, verify request event
Credentials creds1 = new Credentials();
ContainerId container1 = GetMockContainer(1L);
LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
LocalResourceVisibility vis1 = LocalResourceVisibility.Public;
local.Handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
dispatcher.Await();
TestLocalizedResource.LocalizerEventMatcher matchesL1Req = new TestLocalizedResource.LocalizerEventMatcher
(container1, creds1, vis1, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL1Req));
// Release C0 container localization, verify no notification
local.Handle(new ResourceReleaseEvent(rsrcA, container0));
dispatcher.Await();
Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA
<ContainerEvent>());
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Release C1 container localization, verify no notification
local.Handle(new ResourceReleaseEvent(rsrcA, container1));
dispatcher.Await();
Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA
<ContainerEvent>());
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Register C2, C3
ContainerId container2 = GetMockContainer(2L);
LocalResourceVisibility vis2 = LocalResourceVisibility.Private;
Credentials creds2 = new Credentials();
LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
ContainerId container3 = GetMockContainer(3L);
LocalResourceVisibility vis3 = LocalResourceVisibility.Private;
Credentials creds3 = new Credentials();
LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
local.Handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
local.Handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
dispatcher.Await();
TestLocalizedResource.LocalizerEventMatcher matchesL2Req = new TestLocalizedResource.LocalizerEventMatcher
(container2, creds2, vis2, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL2Req));
TestLocalizedResource.LocalizerEventMatcher matchesL3Req = new TestLocalizedResource.LocalizerEventMatcher
(container3, creds3, vis3, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL3Req));
// Successful localization. verify notification C2, C3
Path locA = new Path("file:///cache/rsrcA");
local.Handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
dispatcher.Await();
TestLocalizedResource.ContainerEventMatcher matchesC2Localized = new TestLocalizedResource.ContainerEventMatcher
(container2, ContainerEventType.ResourceLocalized);
TestLocalizedResource.ContainerEventMatcher matchesC3Localized = new TestLocalizedResource.ContainerEventMatcher
(container3, ContainerEventType.ResourceLocalized);
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC2Localized
));
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC3Localized
));
NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState());
// Register C4, verify notification
ContainerId container4 = GetMockContainer(4L);
Credentials creds4 = new Credentials();
LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
LocalResourceVisibility vis4 = LocalResourceVisibility.Private;
local.Handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
dispatcher.Await();
TestLocalizedResource.ContainerEventMatcher matchesC4Localized = new TestLocalizedResource.ContainerEventMatcher
(container4, ContainerEventType.ResourceLocalized);
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC4Localized
));
NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState());
}
finally
{
dispatcher.Stop();
}
}
/*
* Update the file-count statistics for a local cache-directory.
* This will retrieve the localized path for the resource from
* 1) inProgressRsrcMap if the resource was under localization and it
* failed.
* 2) LocalizedResource if the resource is already localized.
* From this path it will identify the local directory under which the
* resource was localized. Then rest of the path will be used to decrement
* file count for the HierarchicalSubDirectory pointing to this relative
* path.
*/
private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource
rsrc)
{
if (useLocalCacheDirectoryManager)
{
Path rsrcPath = null;
if (inProgressLocalResourcesMap.Contains(req))
{
// This happens when localization of a resource fails.
rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
else
{
if (rsrc != null && rsrc.GetLocalPath() != null)
{
rsrcPath = rsrc.GetLocalPath().GetParent().GetParent();
}
}
if (rsrcPath != null)
{
Path parentPath = new Path(rsrcPath.ToUri().GetRawPath());
while (!directoryManagers.Contains(parentPath))
{
parentPath = parentPath.GetParent();
if (parentPath == null)
{
return;
}
}
if (parentPath != null)
{
string parentDir = parentPath.ToUri().GetRawPath().ToString();
LocalCacheDirectoryManager dir = directoryManagers[parentPath];
string rsrcDir = rsrcPath.ToUri().GetRawPath();
if (rsrcDir.Equals(parentDir))
{
dir.DecrementFileCountForPath(string.Empty);
}
else
{
dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length
+ 1));
}
}
}
}
}
/*
* Synchronizing this method for avoiding races due to multiple ResourceEvent's
* coming to LocalResourcesTracker from Public/Private localizer and
* Resource Localization Service.
*/
public virtual void Handle(ResourceEvent @event)
{
lock (this)
{
LocalResourceRequest req = @event.GetLocalResourceRequest();
LocalizedResource rsrc = localrsrc[req];
switch (@event.GetType())
{
case ResourceEventType.Localized:
{
if (useLocalCacheDirectoryManager)
{
Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
break;
}
case ResourceEventType.Request:
{
if (rsrc != null && (!IsResourcePresent(rsrc)))
{
Log.Info("Resource " + rsrc.GetLocalPath() + " is missing, localizing it again");
RemoveResource(req);
rsrc = null;
}
if (null == rsrc)
{
rsrc = new LocalizedResource(req, dispatcher);
localrsrc[req] = rsrc;
}
break;
}
case ResourceEventType.Release:
{
if (null == rsrc)
{
// The container sent a release event on a resource which
// 1) Failed
// 2) Removed for some reason (ex. disk is no longer accessible)
ResourceReleaseEvent relEvent = (ResourceReleaseEvent)@event;
Log.Info("Container " + relEvent.GetContainer() + " sent RELEASE event on a resource request "
+ req + " not present in cache.");
return;
}
break;
}
case ResourceEventType.LocalizationFailed:
{
/*
* If resource localization fails then Localized resource will be
* removed from local cache.
*/
RemoveResource(req);
break;
}
case ResourceEventType.Recovered:
{
if (rsrc != null)
{
Log.Warn("Ignoring attempt to recover existing resource " + rsrc);
return;
}
rsrc = RecoverResource(req, (ResourceRecoveredEvent)@event);
localrsrc[req] = rsrc;
break;
}
}
if (rsrc == null)
{
Log.Warn("Received " + @event.GetType() + " event for request " + req + " but localized resource is missing"
);
return;
}
rsrc.Handle(@event);
// Remove the resource if its downloading and its reference count has
// become 0 after RELEASE. This maybe because a container was killed while
// localizing and no other container is referring to the resource.
// NOTE: This should NOT be done for public resources since the
// download is not associated with a container-specific localizer.
if (@event.GetType() == ResourceEventType.Release)
{
if (rsrc.GetState() == ResourceState.Downloading && rsrc.GetRefCount() <= 0 && rsrc
.GetRequest().GetVisibility() != LocalResourceVisibility.Public)
{
RemoveResource(req);
}
}
if (@event.GetType() == ResourceEventType.Localized)
{
if (rsrc.GetLocalPath() != null)
{
try
{
stateStore.FinishResourceLocalization(user, appId, BuildLocalizedResourceProto(rsrc
));
}
catch (IOException ioe)
{
Log.Error("Error storing resource state for " + rsrc, ioe);
}
}
else
{
Log.Warn("Resource " + rsrc + " localized without a location");
}
}
}
}