public virtual bool Remove(LocalizedResource rem, DeletionService delService) { // current synchronization guaranteed by crude RLS event for cleanup LocalizedResource rsrc = localrsrc[rem.GetRequest()]; if (null == rsrc) { Log.Error("Attempt to remove absent resource: " + rem.GetRequest() + " from " + GetUser ()); return(true); } if (rsrc.GetRefCount() > 0 || ResourceState.Downloading.Equals(rsrc.GetState()) || rsrc != rem) { // internal error Log.Error("Attempt to remove resource: " + rsrc + " with non-zero refcount"); return(false); } else { // ResourceState is LOCALIZED or INIT if (ResourceState.Localized.Equals(rsrc.GetState())) { delService.Delete(GetUser(), GetPathToDelete(rsrc.GetLocalPath())); } RemoveResource(rem.GetRequest()); Log.Info("Removed " + rsrc.GetLocalPath() + " from localized cache"); return(true); } }
private YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto BuildLocalizedResourceProto (LocalizedResource rsrc) { return((YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto)YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto .NewBuilder().SetResource(BuildLocalResourceProto(rsrc.GetRequest())).SetLocalPath (rsrc.GetLocalPath().ToString()).SetSize(rsrc.GetSize()).Build()); }
/* * Update the file-count statistics for a local cache-directory. * This will retrieve the localized path for the resource from * 1) inProgressRsrcMap if the resource was under localization and it * failed. * 2) LocalizedResource if the resource is already localized. * From this path it will identify the local directory under which the * resource was localized. Then rest of the path will be used to decrement * file count for the HierarchicalSubDirectory pointing to this relative * path. */ private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource rsrc) { if (useLocalCacheDirectoryManager) { Path rsrcPath = null; if (inProgressLocalResourcesMap.Contains(req)) { // This happens when localization of a resource fails. rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req); } else { if (rsrc != null && rsrc.GetLocalPath() != null) { rsrcPath = rsrc.GetLocalPath().GetParent().GetParent(); } } if (rsrcPath != null) { Path parentPath = new Path(rsrcPath.ToUri().GetRawPath()); while (!directoryManagers.Contains(parentPath)) { parentPath = parentPath.GetParent(); if (parentPath == null) { return; } } if (parentPath != null) { string parentDir = parentPath.ToUri().GetRawPath().ToString(); LocalCacheDirectoryManager dir = directoryManagers[parentPath]; string rsrcDir = rsrcPath.ToUri().GetRawPath(); if (rsrcDir.Equals(parentDir)) { dir.DecrementFileCountForPath(string.Empty); } else { dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length + 1)); } } } } }
/// <summary> /// This module checks if the resource which was localized is already present /// or not /// </summary> /// <param name="rsrc"/> /// <returns>true/false based on resource is present or not</returns> public virtual bool IsResourcePresent(LocalizedResource rsrc) { bool ret = true; if (rsrc.GetState() == ResourceState.Localized) { FilePath file = new FilePath(rsrc.GetLocalPath().ToUri().GetRawPath().ToString()); if (!file.Exists()) { ret = false; } } return(ret); }
private void RemoveResource(LocalResourceRequest req) { LocalizedResource rsrc = Sharpen.Collections.Remove(localrsrc, req); DecrementFileCountForLocalCacheDirectory(req, rsrc); if (rsrc != null) { Path localPath = rsrc.GetLocalPath(); if (localPath != null) { try { stateStore.RemoveLocalizedResource(user, appId, localPath); } catch (IOException e) { Log.Error("Unable to remove resource " + rsrc + " from state store", e); } } } }
/* * Synchronizing this method for avoiding races due to multiple ResourceEvent's * coming to LocalResourcesTracker from Public/Private localizer and * Resource Localization Service. */ public virtual void Handle(ResourceEvent @event) { lock (this) { LocalResourceRequest req = @event.GetLocalResourceRequest(); LocalizedResource rsrc = localrsrc[req]; switch (@event.GetType()) { case ResourceEventType.Localized: { if (useLocalCacheDirectoryManager) { Sharpen.Collections.Remove(inProgressLocalResourcesMap, req); } break; } case ResourceEventType.Request: { if (rsrc != null && (!IsResourcePresent(rsrc))) { Log.Info("Resource " + rsrc.GetLocalPath() + " is missing, localizing it again"); RemoveResource(req); rsrc = null; } if (null == rsrc) { rsrc = new LocalizedResource(req, dispatcher); localrsrc[req] = rsrc; } break; } case ResourceEventType.Release: { if (null == rsrc) { // The container sent a release event on a resource which // 1) Failed // 2) Removed for some reason (ex. disk is no longer accessible) ResourceReleaseEvent relEvent = (ResourceReleaseEvent)@event; Log.Info("Container " + relEvent.GetContainer() + " sent RELEASE event on a resource request " + req + " not present in cache."); return; } break; } case ResourceEventType.LocalizationFailed: { /* * If resource localization fails then Localized resource will be * removed from local cache. */ RemoveResource(req); break; } case ResourceEventType.Recovered: { if (rsrc != null) { Log.Warn("Ignoring attempt to recover existing resource " + rsrc); return; } rsrc = RecoverResource(req, (ResourceRecoveredEvent)@event); localrsrc[req] = rsrc; break; } } if (rsrc == null) { Log.Warn("Received " + @event.GetType() + " event for request " + req + " but localized resource is missing" ); return; } rsrc.Handle(@event); // Remove the resource if its downloading and its reference count has // become 0 after RELEASE. This maybe because a container was killed while // localizing and no other container is referring to the resource. // NOTE: This should NOT be done for public resources since the // download is not associated with a container-specific localizer. if (@event.GetType() == ResourceEventType.Release) { if (rsrc.GetState() == ResourceState.Downloading && rsrc.GetRefCount() <= 0 && rsrc .GetRequest().GetVisibility() != LocalResourceVisibility.Public) { RemoveResource(req); } } if (@event.GetType() == ResourceEventType.Localized) { if (rsrc.GetLocalPath() != null) { try { stateStore.FinishResourceLocalization(user, appId, BuildLocalizedResourceProto(rsrc )); } catch (IOException ioe) { Log.Error("Error storing resource state for " + rsrc, ioe); } } else { Log.Warn("Resource " + rsrc + " localized without a location"); } } } }