internal static void CheckEqual(LocalResourceRequest a, LocalResourceRequest b) { NUnit.Framework.Assert.AreEqual(a, b); NUnit.Framework.Assert.AreEqual(a.GetHashCode(), b.GetHashCode()); NUnit.Framework.Assert.AreEqual(0, a.CompareTo(b)); NUnit.Framework.Assert.AreEqual(0, b.CompareTo(a)); }
public _LocalizedResource_93(long rsrcSize, Path p, long ts, LocalResourceRequest baseArg1, Dispatcher baseArg2) : base(baseArg1, baseArg2) { this.rsrcSize = rsrcSize; this.p = p; this.ts = ts; }
internal static void CheckNotEqual(LocalResourceRequest a, LocalResourceRequest b ) { NUnit.Framework.Assert.IsFalse(a.Equals(b)); NUnit.Framework.Assert.IsFalse(b.Equals(a)); NUnit.Framework.Assert.IsFalse(a.GetHashCode() == b.GetHashCode()); NUnit.Framework.Assert.IsFalse(0 == a.CompareTo(b)); NUnit.Framework.Assert.IsFalse(0 == b.CompareTo(a)); }
/// <returns> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// absolute path for localization which includes local /// directory path and the relative hierarchical path (if use local /// cache directory manager is enabled) /// </returns> /// <?/> /// <?/> /// <?/> public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath , DeletionService delService) { Path rPath = localDirPath; if (useLocalCacheDirectoryManager && localDirPath != null) { if (!directoryManagers.Contains(localDirPath)) { directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf)); } LocalCacheDirectoryManager dir = directoryManagers[localDirPath]; rPath = localDirPath; string hierarchicalPath = dir.GetRelativePathForLocalization(); // For most of the scenarios we will get root path only which // is an empty string if (!hierarchicalPath.IsEmpty()) { rPath = new Path(localDirPath, hierarchicalPath); } inProgressLocalResourcesMap[req] = rPath; } while (true) { Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FilePath file = new FilePath(uniquePath.ToUri().GetRawPath()); if (!file.Exists()) { rPath = uniquePath; break; } // If the directory already exists, delete it and move to next one. Log.Warn("Directory " + uniquePath + " already exists, " + "try next one."); if (delService != null) { delService.Delete(GetUser(), uniquePath); } } Path localPath = new Path(rPath, req.GetPath().GetName()); LocalizedResource rsrc = localrsrc[req]; rsrc.SetLocalPath(localPath); LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req .GetVisibility(), req.GetSize(), req.GetTimestamp()); try { stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto (), localPath); } catch (IOException e) { Log.Error("Unable to record localization start for " + rsrc, e); } return(rPath); }
public virtual void TestResourceEquality() { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); System.Console.Out.WriteLine("SEED: " + seed); long basetime = (long)(((ulong)r.NextLong()) >> 2); LocalResource yA = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime , LocalResourceType.File, LocalResourceVisibility.Public, null); LocalResource yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime , LocalResourceType.File, LocalResourceVisibility.Public, null); LocalResourceRequest a = new LocalResourceRequest(yA); LocalResourceRequest b = new LocalResourceRequest(yA); CheckEqual(a, b); b = new LocalResourceRequest(yB); CheckEqual(a, b); // ignore visibility yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType .File, LocalResourceVisibility.Private, null); b = new LocalResourceRequest(yB); CheckEqual(a, b); // ignore size yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime, LocalResourceType .File, LocalResourceVisibility.Private, null); b = new LocalResourceRequest(yB); CheckEqual(a, b); // note path yB = GetYarnResource(new Path("hdfs://dingo.org:80/foobar"), 0, basetime, LocalResourceType .Archive, LocalResourceVisibility.Public, null); b = new LocalResourceRequest(yB); CheckNotEqual(a, b); // note type yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime, LocalResourceType .Archive, LocalResourceVisibility.Public, null); b = new LocalResourceRequest(yB); CheckNotEqual(a, b); // note timestamp yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime + 1, LocalResourceType .File, LocalResourceVisibility.Public, null); b = new LocalResourceRequest(yB); CheckNotEqual(a, b); // note pattern yB = GetYarnResource(new Path("http://yak.org:80/foobar"), 0, basetime + 1, LocalResourceType .File, LocalResourceVisibility.Public, "^/foo/.*"); b = new LocalResourceRequest(yB); CheckNotEqual(a, b); }
/* * Update the file-count statistics for a local cache-directory. * This will retrieve the localized path for the resource from * 1) inProgressRsrcMap if the resource was under localization and it * failed. * 2) LocalizedResource if the resource is already localized. * From this path it will identify the local directory under which the * resource was localized. Then rest of the path will be used to decrement * file count for the HierarchicalSubDirectory pointing to this relative * path. */ private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource rsrc) { if (useLocalCacheDirectoryManager) { Path rsrcPath = null; if (inProgressLocalResourcesMap.Contains(req)) { // This happens when localization of a resource fails. rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req); } else { if (rsrc != null && rsrc.GetLocalPath() != null) { rsrcPath = rsrc.GetLocalPath().GetParent().GetParent(); } } if (rsrcPath != null) { Path parentPath = new Path(rsrcPath.ToUri().GetRawPath()); while (!directoryManagers.Contains(parentPath)) { parentPath = parentPath.GetParent(); if (parentPath == null) { return; } } if (parentPath != null) { string parentDir = parentPath.ToUri().GetRawPath().ToString(); LocalCacheDirectoryManager dir = directoryManagers[parentPath]; string rsrcDir = rsrcPath.ToUri().GetRawPath(); if (rsrcDir.Equals(parentDir)) { dir.DecrementFileCountForPath(string.Empty); } else { dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length + 1)); } } } } }
public LocalizedResource(LocalResourceRequest rsrc, Dispatcher dispatcher) { timestamp = new AtomicLong(CurrentTime()); // Queue of containers using this localized // resource // From INIT (ref == 0, awaiting req) // From DOWNLOADING (ref > 0, may be localizing) // TODO: Duplicate addition!! // From LOCALIZED (ref >= 0, on disk) this.rsrc = rsrc; this.dispatcher = dispatcher; this.@ref = new List <ContainerId>(); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.ReadLock(); this.writeLock = readWriteLock.WriteLock(); this.stateMachine = stateMachineFactory.Make(this); }
private LocalizedResource RecoverResource(LocalResourceRequest req, ResourceRecoveredEvent @event) { // unique number for a resource is the directory of the resource Path localDir = @event.GetLocalPath().GetParent(); long rsrcId = long.Parse(localDir.GetName()); // update ID generator to avoid conflicts with existing resources while (true) { long currentRsrcId = uniqueNumberGenerator.Get(); long nextRsrcId = Math.Max(currentRsrcId, rsrcId); if (uniqueNumberGenerator.CompareAndSet(currentRsrcId, nextRsrcId)) { break; } } IncrementFileCountForLocalCacheDirectory(localDir.GetParent()); return(new LocalizedResource(req, dispatcher)); }
private void RemoveResource(LocalResourceRequest req) { LocalizedResource rsrc = Sharpen.Collections.Remove(localrsrc, req); DecrementFileCountForLocalCacheDirectory(req, rsrc); if (rsrc != null) { Path localPath = rsrc.GetLocalPath(); if (localPath != null) { try { stateStore.RemoveLocalizedResource(user, appId, localPath); } catch (IOException e) { Log.Error("Unable to remove resource " + rsrc + " from state store", e); } } } }
internal virtual LocalResourcesTracker CreateMockTracker(string user, long rsrcSize , long nRsrcs, long timestamp, long tsstep) { Configuration conf = new Configuration(); ConcurrentMap <LocalResourceRequest, LocalizedResource> trackerResources = new ConcurrentHashMap <LocalResourceRequest, LocalizedResource>(); LocalResourcesTracker ret = Org.Mockito.Mockito.Spy(new LocalResourcesTrackerImpl (user, null, null, trackerResources, false, conf, new NMNullStateStoreService()) ); for (int i = 0; i < nRsrcs; ++i) { LocalResourceRequest req = new LocalResourceRequest(new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep, LocalResourceType.File, LocalResourceVisibility .Public, null); long ts = timestamp + i * tsstep; Path p = new Path("file:///local/" + user + "/rsrc" + i); LocalizedResource rsrc = new _LocalizedResource_93(rsrcSize, p, ts, req, null); trackerResources[req] = rsrc; } return(ret); }
public virtual void TestResourceOrder() { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); System.Console.Out.WriteLine("SEED: " + seed); long basetime = (long)(((ulong)r.NextLong()) >> 2); LocalResource yA = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime , LocalResourceType.File, LocalResourceVisibility.Public, "^/foo/.*"); LocalResourceRequest a = new LocalResourceRequest(yA); // Path primary LocalResource yB = GetYarnResource(new Path("http://yak.org:80/foobaz"), -1, basetime , LocalResourceType.File, LocalResourceVisibility.Public, "^/foo/.*"); LocalResourceRequest b = new LocalResourceRequest(yB); NUnit.Framework.Assert.IsTrue(0 > a.CompareTo(b)); // timestamp secondary yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime + 1, LocalResourceType .File, LocalResourceVisibility.Public, "^/foo/.*"); b = new LocalResourceRequest(yB); NUnit.Framework.Assert.IsTrue(0 > a.CompareTo(b)); // type tertiary yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType .Archive, LocalResourceVisibility.Public, "^/foo/.*"); b = new LocalResourceRequest(yB); NUnit.Framework.Assert.IsTrue(0 != a.CompareTo(b)); // don't care about order, just ne // path 4th yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType .Archive, LocalResourceVisibility.Public, "^/food/.*"); b = new LocalResourceRequest(yB); NUnit.Framework.Assert.IsTrue(0 != a.CompareTo(b)); // don't care about order, just ne yB = GetYarnResource(new Path("http://yak.org:80/foobar"), -1, basetime, LocalResourceType .Archive, LocalResourceVisibility.Public, null); b = new LocalResourceRequest(yB); NUnit.Framework.Assert.IsTrue(0 != a.CompareTo(b)); }
public virtual void TestNotification() { // mocked generic DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.Init(new Configuration()); try { dispatcher.Start(); EventHandler <ContainerEvent> containerBus = Org.Mockito.Mockito.Mock <EventHandler >(); EventHandler <LocalizerEvent> localizerBus = Org.Mockito.Mockito.Mock <EventHandler >(); dispatcher.Register(typeof(ContainerEventType), containerBus); dispatcher.Register(typeof(LocalizerEventType), localizerBus); // mock resource LocalResource apiRsrc = CreateMockResource(); ContainerId container0 = GetMockContainer(0L); Credentials creds0 = new Credentials(); LocalResourceVisibility vis0 = LocalResourceVisibility.Private; LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0); LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc); LocalizedResource local = new LocalizedResource(rsrcA, dispatcher); local.Handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0)); dispatcher.Await(); // Register C0, verify request event TestLocalizedResource.LocalizerEventMatcher matchesL0Req = new TestLocalizedResource.LocalizerEventMatcher (container0, creds0, vis0, LocalizerEventType.RequestResourceLocalization); Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL0Req)); NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState()); // Register C1, verify request event Credentials creds1 = new Credentials(); ContainerId container1 = GetMockContainer(1L); LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1); LocalResourceVisibility vis1 = LocalResourceVisibility.Public; local.Handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1)); dispatcher.Await(); TestLocalizedResource.LocalizerEventMatcher matchesL1Req = new TestLocalizedResource.LocalizerEventMatcher (container1, creds1, vis1, LocalizerEventType.RequestResourceLocalization); Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL1Req)); // Release C0 container localization, verify no notification local.Handle(new ResourceReleaseEvent(rsrcA, container0)); dispatcher.Await(); Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA <ContainerEvent>()); NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState()); // Release C1 container localization, verify no notification local.Handle(new ResourceReleaseEvent(rsrcA, container1)); dispatcher.Await(); Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA <ContainerEvent>()); NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState()); // Register C2, C3 ContainerId container2 = GetMockContainer(2L); LocalResourceVisibility vis2 = LocalResourceVisibility.Private; Credentials creds2 = new Credentials(); LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2); ContainerId container3 = GetMockContainer(3L); LocalResourceVisibility vis3 = LocalResourceVisibility.Private; Credentials creds3 = new Credentials(); LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3); local.Handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2)); local.Handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3)); dispatcher.Await(); TestLocalizedResource.LocalizerEventMatcher matchesL2Req = new TestLocalizedResource.LocalizerEventMatcher (container2, creds2, vis2, LocalizerEventType.RequestResourceLocalization); Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL2Req)); TestLocalizedResource.LocalizerEventMatcher matchesL3Req = new TestLocalizedResource.LocalizerEventMatcher (container3, creds3, vis3, LocalizerEventType.RequestResourceLocalization); Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL3Req)); // Successful localization. verify notification C2, C3 Path locA = new Path("file:///cache/rsrcA"); local.Handle(new ResourceLocalizedEvent(rsrcA, locA, 10)); dispatcher.Await(); TestLocalizedResource.ContainerEventMatcher matchesC2Localized = new TestLocalizedResource.ContainerEventMatcher (container2, ContainerEventType.ResourceLocalized); TestLocalizedResource.ContainerEventMatcher matchesC3Localized = new TestLocalizedResource.ContainerEventMatcher (container3, ContainerEventType.ResourceLocalized); Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC2Localized )); Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC3Localized )); NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState()); // Register C4, verify notification ContainerId container4 = GetMockContainer(4L); Credentials creds4 = new Credentials(); LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4); LocalResourceVisibility vis4 = LocalResourceVisibility.Private; local.Handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4)); dispatcher.Await(); TestLocalizedResource.ContainerEventMatcher matchesC4Localized = new TestLocalizedResource.ContainerEventMatcher (container4, ContainerEventType.ResourceLocalized); Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC4Localized )); NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState()); } finally { dispatcher.Stop(); } }
public virtual LocalizedResource GetLocalizedResource(LocalResourceRequest request ) { return(localrsrc[request]); }
/* * Synchronizing this method for avoiding races due to multiple ResourceEvent's * coming to LocalResourcesTracker from Public/Private localizer and * Resource Localization Service. */ public virtual void Handle(ResourceEvent @event) { lock (this) { LocalResourceRequest req = @event.GetLocalResourceRequest(); LocalizedResource rsrc = localrsrc[req]; switch (@event.GetType()) { case ResourceEventType.Localized: { if (useLocalCacheDirectoryManager) { Sharpen.Collections.Remove(inProgressLocalResourcesMap, req); } break; } case ResourceEventType.Request: { if (rsrc != null && (!IsResourcePresent(rsrc))) { Log.Info("Resource " + rsrc.GetLocalPath() + " is missing, localizing it again"); RemoveResource(req); rsrc = null; } if (null == rsrc) { rsrc = new LocalizedResource(req, dispatcher); localrsrc[req] = rsrc; } break; } case ResourceEventType.Release: { if (null == rsrc) { // The container sent a release event on a resource which // 1) Failed // 2) Removed for some reason (ex. disk is no longer accessible) ResourceReleaseEvent relEvent = (ResourceReleaseEvent)@event; Log.Info("Container " + relEvent.GetContainer() + " sent RELEASE event on a resource request " + req + " not present in cache."); return; } break; } case ResourceEventType.LocalizationFailed: { /* * If resource localization fails then Localized resource will be * removed from local cache. */ RemoveResource(req); break; } case ResourceEventType.Recovered: { if (rsrc != null) { Log.Warn("Ignoring attempt to recover existing resource " + rsrc); return; } rsrc = RecoverResource(req, (ResourceRecoveredEvent)@event); localrsrc[req] = rsrc; break; } } if (rsrc == null) { Log.Warn("Received " + @event.GetType() + " event for request " + req + " but localized resource is missing" ); return; } rsrc.Handle(@event); // Remove the resource if its downloading and its reference count has // become 0 after RELEASE. This maybe because a container was killed while // localizing and no other container is referring to the resource. // NOTE: This should NOT be done for public resources since the // download is not associated with a container-specific localizer. if (@event.GetType() == ResourceEventType.Release) { if (rsrc.GetState() == ResourceState.Downloading && rsrc.GetRefCount() <= 0 && rsrc .GetRequest().GetVisibility() != LocalResourceVisibility.Public) { RemoveResource(req); } } if (@event.GetType() == ResourceEventType.Localized) { if (rsrc.GetLocalPath() != null) { try { stateStore.FinishResourceLocalization(user, appId, BuildLocalizedResourceProto(rsrc )); } catch (IOException ioe) { Log.Error("Error storing resource state for " + rsrc, ioe); } } else { Log.Warn("Resource " + rsrc + " localized without a location"); } } } }