Esempio n. 1
0
        public virtual void TestProcessEvictableResource()
        {
            FileSystem     fs      = Org.Mockito.Mockito.Mock <FileSystem>();
            CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
            SCMStore       store   = Org.Mockito.Mockito.Mock <SCMStore>();
            CleanerTask    task    = CreateSpiedTask(fs, store, metrics, new ReentrantLock());

            // mock an evictable resource
            Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
                                                               <FileStatus>())).ThenReturn(true);
            FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();

            Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
                                                                           ));
            Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
                (true);
            // rename succeeds
            Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn
                (true);
            // delete returns true
            Org.Mockito.Mockito.When(fs.Delete(Matchers.IsA <Path>(), Matchers.AnyBoolean())).
            ThenReturn(true);
            // process the resource
            task.ProcessSingleResource(status);
            // the directory should be renamed
            Org.Mockito.Mockito.Verify(fs).Rename(Matchers.Eq(status.GetPath()), Matchers.IsA
                                                  <Path>());
            // metrics should record a deleted file
            Org.Mockito.Mockito.Verify(metrics).ReportAFileDelete();
            Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileProcess
                ();
        }
Esempio n. 2
0
        public virtual void TestProcessFreshResource()
        {
            FileSystem     fs      = Org.Mockito.Mockito.Mock <FileSystem>();
            CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
            SCMStore       store   = Org.Mockito.Mockito.Mock <SCMStore>();
            CleanerTask    task    = CreateSpiedTask(fs, store, metrics, new ReentrantLock());

            // mock a resource that is not evictable
            Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
                                                               <FileStatus>())).ThenReturn(false);
            FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();

            Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
                                                                           ));
            // process the resource
            task.ProcessSingleResource(status);
            // the directory should not be renamed
            Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Rename(Matchers.Eq(status
                                                                                           .GetPath()), Matchers.IsA <Path>());
            // metrics should record a processed file (but not delete)
            Org.Mockito.Mockito.Verify(metrics).ReportAFileProcess();
            Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileDelete
                ();
        }
Esempio n. 3
0
        public virtual void TestResourceIsInUseHasAnActiveApp()
        {
            FileSystem     fs       = Org.Mockito.Mockito.Mock <FileSystem>();
            CleanerMetrics metrics  = Org.Mockito.Mockito.Mock <CleanerMetrics>();
            SCMStore       store    = Org.Mockito.Mockito.Mock <SCMStore>();
            FileStatus     resource = Org.Mockito.Mockito.Mock <FileStatus>();

            Org.Mockito.Mockito.When(resource.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
                                                                             ));
            // resource is stale
            Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
                                                               <FileStatus>())).ThenReturn(true);
            // but still has appIds
            Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
                (false);
            CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());

            // process the resource
            task.ProcessSingleResource(resource);
            // metrics should record a processed file (but not delete)
            Org.Mockito.Mockito.Verify(metrics).ReportAFileProcess();
            Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileDelete
                ();
        }
Esempio n. 4
0
        /// <summary>Processes a single shared cache resource directory.</summary>
        internal virtual void ProcessSingleResource(FileStatus resource)
        {
            Path path = resource.GetPath();

            // indicates the processing status of the resource
            CleanerTask.ResourceStatus resourceStatus = CleanerTask.ResourceStatus.Init;
            // first, if the path ends with the renamed suffix, it indicates the
            // directory was moved (as stale) but somehow not deleted (probably due to
            // SCM failure); delete the directory
            if (path.ToString().EndsWith(RenamedSuffix))
            {
                Log.Info("Found a renamed directory that was left undeleted at " + path.ToString(
                             ) + ". Deleting.");
                try
                {
                    if (fs.Delete(path, true))
                    {
                        resourceStatus = CleanerTask.ResourceStatus.Deleted;
                    }
                }
                catch (IOException e)
                {
                    Log.Error("Error while processing a shared cache resource: " + path, e);
                }
            }
            else
            {
                // this is the path to the cache resource directory
                // the directory name is the resource key (i.e. a unique identifier)
                string key = path.GetName();
                try
                {
                    store.CleanResourceReferences(key);
                }
                catch (YarnException e)
                {
                    Log.Error("Exception thrown while removing dead appIds.", e);
                }
                if (store.IsResourceEvictable(key, resource))
                {
                    try
                    {
                        /*
                         * TODO See YARN-2663: There is a race condition between
                         * store.removeResource(key) and
                         * removeResourceFromCacheFileSystem(path) operations because they do
                         * not happen atomically and resources can be uploaded with different
                         * file names by the node managers.
                         */
                        // remove the resource from scm (checks for appIds as well)
                        if (store.RemoveResource(key))
                        {
                            // remove the resource from the file system
                            bool deleted = RemoveResourceFromCacheFileSystem(path);
                            if (deleted)
                            {
                                resourceStatus = CleanerTask.ResourceStatus.Deleted;
                            }
                            else
                            {
                                Log.Error("Failed to remove path from the file system." + " Skipping this resource: "
                                          + path);
                                resourceStatus = CleanerTask.ResourceStatus.Error;
                            }
                        }
                        else
                        {
                            // we did not delete the resource because it contained application
                            // ids
                            resourceStatus = CleanerTask.ResourceStatus.Processed;
                        }
                    }
                    catch (IOException e)
                    {
                        Log.Error("Failed to remove path from the file system. Skipping this resource: "
                                  + path, e);
                        resourceStatus = CleanerTask.ResourceStatus.Error;
                    }
                }
                else
                {
                    resourceStatus = CleanerTask.ResourceStatus.Processed;
                }
            }
            switch (resourceStatus)
            {
            case CleanerTask.ResourceStatus.Deleted:
            {
                // record the processing
                metrics.ReportAFileDelete();
                break;
            }

            case CleanerTask.ResourceStatus.Processed:
            {
                metrics.ReportAFileProcess();
                break;
            }

            case CleanerTask.ResourceStatus.Error:
            {
                metrics.ReportAFileError();
                break;
            }

            default:
            {
                Log.Error("Cleaner encountered an invalid status (" + resourceStatus + ") while processing resource: "
                          + path.GetName());
                break;
            }
            }
        }