public virtual bool Remove(LocalizedResource rem, DeletionService delService)
{
// current synchronization guaranteed by crude RLS event for cleanup
LocalizedResource rsrc = localrsrc[rem.GetRequest()];
if (null == rsrc)
{
Log.Error("Attempt to remove absent resource: " + rem.GetRequest() + " from " + GetUser
());
return(true);
}
if (rsrc.GetRefCount() > 0 || ResourceState.Downloading.Equals(rsrc.GetState()) ||
rsrc != rem)
{
// internal error
Log.Error("Attempt to remove resource: " + rsrc + " with non-zero refcount");
return(false);
}
else
{
// ResourceState is LOCALIZED or INIT
if (ResourceState.Localized.Equals(rsrc.GetState()))
{
delService.Delete(GetUser(), GetPathToDelete(rsrc.GetLocalPath()));
}
RemoveResource(rem.GetRequest());
Log.Info("Removed " + rsrc.GetLocalPath() + " from localized cache");
return(true);
}
}
private YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto BuildLocalizedResourceProto
(LocalizedResource rsrc)
{
return((YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto)YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto
.NewBuilder().SetResource(BuildLocalResourceProto(rsrc.GetRequest())).SetLocalPath
(rsrc.GetLocalPath().ToString()).SetSize(rsrc.GetSize()).Build());
}
/*
* Update the file-count statistics for a local cache-directory.
* This will retrieve the localized path for the resource from
* 1) inProgressRsrcMap if the resource was under localization and it
* failed.
* 2) LocalizedResource if the resource is already localized.
* From this path it will identify the local directory under which the
* resource was localized. Then rest of the path will be used to decrement
* file count for the HierarchicalSubDirectory pointing to this relative
* path.
*/
private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource
rsrc)
{
if (useLocalCacheDirectoryManager)
{
Path rsrcPath = null;
if (inProgressLocalResourcesMap.Contains(req))
{
// This happens when localization of a resource fails.
rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
else
{
if (rsrc != null && rsrc.GetLocalPath() != null)
{
rsrcPath = rsrc.GetLocalPath().GetParent().GetParent();
}
}
if (rsrcPath != null)
{
Path parentPath = new Path(rsrcPath.ToUri().GetRawPath());
while (!directoryManagers.Contains(parentPath))
{
parentPath = parentPath.GetParent();
if (parentPath == null)
{
return;
}
}
if (parentPath != null)
{
string parentDir = parentPath.ToUri().GetRawPath().ToString();
LocalCacheDirectoryManager dir = directoryManagers[parentPath];
string rsrcDir = rsrcPath.ToUri().GetRawPath();
if (rsrcDir.Equals(parentDir))
{
dir.DecrementFileCountForPath(string.Empty);
}
else
{
dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length
+ 1));
}
}
}
}
}
/// <summary>
/// This module checks if the resource which was localized is already present
/// or not
/// </summary>
/// <param name="rsrc"/>
/// <returns>true/false based on resource is present or not</returns>
public virtual bool IsResourcePresent(LocalizedResource rsrc)
{
bool ret = true;
if (rsrc.GetState() == ResourceState.Localized)
{
FilePath file = new FilePath(rsrc.GetLocalPath().ToUri().GetRawPath().ToString());
if (!file.Exists())
{
ret = false;
}
}
return(ret);
}
private void RemoveResource(LocalResourceRequest req)
{
LocalizedResource rsrc = Sharpen.Collections.Remove(localrsrc, req);
DecrementFileCountForLocalCacheDirectory(req, rsrc);
if (rsrc != null)
{
Path localPath = rsrc.GetLocalPath();
if (localPath != null)
{
try
{
stateStore.RemoveLocalizedResource(user, appId, localPath);
}
catch (IOException e)
{
Log.Error("Unable to remove resource " + rsrc + " from state store", e);
}
}
}
}
/*
* Synchronizing this method for avoiding races due to multiple ResourceEvent's
* coming to LocalResourcesTracker from Public/Private localizer and
* Resource Localization Service.
*/
public virtual void Handle(ResourceEvent @event)
{
lock (this)
{
LocalResourceRequest req = @event.GetLocalResourceRequest();
LocalizedResource rsrc = localrsrc[req];
switch (@event.GetType())
{
case ResourceEventType.Localized:
{
if (useLocalCacheDirectoryManager)
{
Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
break;
}
case ResourceEventType.Request:
{
if (rsrc != null && (!IsResourcePresent(rsrc)))
{
Log.Info("Resource " + rsrc.GetLocalPath() + " is missing, localizing it again");
RemoveResource(req);
rsrc = null;
}
if (null == rsrc)
{
rsrc = new LocalizedResource(req, dispatcher);
localrsrc[req] = rsrc;
}
break;
}
case ResourceEventType.Release:
{
if (null == rsrc)
{
// The container sent a release event on a resource which
// 1) Failed
// 2) Removed for some reason (ex. disk is no longer accessible)
ResourceReleaseEvent relEvent = (ResourceReleaseEvent)@event;
Log.Info("Container " + relEvent.GetContainer() + " sent RELEASE event on a resource request "
+ req + " not present in cache.");
return;
}
break;
}
case ResourceEventType.LocalizationFailed:
{
/*
* If resource localization fails then Localized resource will be
* removed from local cache.
*/
RemoveResource(req);
break;
}
case ResourceEventType.Recovered:
{
if (rsrc != null)
{
Log.Warn("Ignoring attempt to recover existing resource " + rsrc);
return;
}
rsrc = RecoverResource(req, (ResourceRecoveredEvent)@event);
localrsrc[req] = rsrc;
break;
}
}
if (rsrc == null)
{
Log.Warn("Received " + @event.GetType() + " event for request " + req + " but localized resource is missing"
);
return;
}
rsrc.Handle(@event);
// Remove the resource if its downloading and its reference count has
// become 0 after RELEASE. This maybe because a container was killed while
// localizing and no other container is referring to the resource.
// NOTE: This should NOT be done for public resources since the
// download is not associated with a container-specific localizer.
if (@event.GetType() == ResourceEventType.Release)
{
if (rsrc.GetState() == ResourceState.Downloading && rsrc.GetRefCount() <= 0 && rsrc
.GetRequest().GetVisibility() != LocalResourceVisibility.Public)
{
RemoveResource(req);
}
}
if (@event.GetType() == ResourceEventType.Localized)
{
if (rsrc.GetLocalPath() != null)
{
try
{
stateStore.FinishResourceLocalization(user, appId, BuildLocalizedResourceProto(rsrc
));
}
catch (IOException ioe)
{
Log.Error("Error storing resource state for " + rsrc, ioe);
}
}
else
{
Log.Warn("Resource " + rsrc + " localized without a location");
}
}
}
}
