コード例 #1
0
        public void GetFileListFromDir()
        {
            AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
            var dataStore = utils.BuildSampleDirectoryBeforeBuild("GetFileListFromDir", "ds1.1.1");
            var configInfo = utils.GetLocal(dataStore);

            var w = new WindowsDataset(dataStore);
            var list = w.ListOfDSFiles("ds1.1.1");
            Assert.AreEqual(5, list.Length);
        }
コード例 #2
0
        public void GetFileListFromDirWithoutFileList()
        {
            AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
            var dataStore = utils.BuildSampleDirectoryBeforeBuild("GetFileListFromDirWithoutFileList", "ds1.1.1");
            var f = new FileInfo(Path.Combine(dataStore.FullName, "ds1.1.1", "aa_dataset_complete_file_list.txt"));
            Assert.IsTrue(f.Exists);
            f.Delete();

            var configInfo = utils.GetLocal(dataStore);

            var w = new WindowsDataset(dataStore);
            var list = w.ListOfDSFiles("ds1.1.1");
            Assert.AreEqual(5, list.Length);
        }
コード例 #3
0
ファイル: LocalMachine.cs プロジェクト: LHCAtlas/AtlasSSH
        public static Location GetLocation(Dictionary<string, string> props)
        {
            var l = new Location();
            l.Name = props["Name"];

            var dirCacheLocations = props["Paths"].Split(',')
                .Select(dirname => new DirectoryInfo(dirname.Trim()))
                .ToArray();

            // We are always good - and we test for directory locations on the fly
            l.LocationTests.Add(() => true);

            l.GetDSInfo = name =>
            {
                var d = FindDataset(dirCacheLocations, name);
                if (d == null)
                {
                    return new DSInfo()
                    {
                        Name = name,
                        IsLocal = filter => false,
                        CanBeGeneratedAutomatically = false,
                        ListOfFiles = () => new string[0],
                        LocationProvider = l,
                    };
                }
                else
                {
                    var w = new WindowsDataset(d.Parent);
                    return new DSInfo()
                    {
                        Name = name,
                        IsLocal = filter => w.FindDSFiles(name, filter) != null,
                        CanBeGeneratedAutomatically = false,
                        ListOfFiles = () => w.ListOfDSFiles(name),
                        LocationProvider = l,
                    };
                }
            };

            // Even though we claim we can't download a data file locally - we can. It is just that we won't do it automatically.
            var linuxFinder = FetchToRemoteLinuxDirInstance.FetchRemoteLinuxInstance(props);
            l.GetDS = (dsinfo, status, filter, failNow, timeout) =>
                {
                    var d = FindDataset(dirCacheLocations, dsinfo.Name);
                    if (d == null)
                    {
                        var validLocalCache = FindLocalCache(dirCacheLocations);
                        if (validLocalCache == null)
                        {
                            throw new InvalidOperationException(string.Format("No local cache directory has been created; we can't copy any files locally until it has. See the {0}.Paths property in the configuration", l.Name));
                        }
                        var dsdir = new DirectoryInfo(Path.Combine(validLocalCache.FullName, dsinfo.Name.SantizeDSName()));
                        dsdir.Create();
                        return LoadDatasetFromOtherSource(new WindowsDataset(dsdir.Parent), dsinfo, status, filter, l.Name, linuxFinder, props["LinuxTempLocation"], failNow, timeout);
                    }
                    var w = new WindowsDataset(d.Parent);
                    var result = w.FindDSFiles(dsinfo.Name, filter);
                    if (result != null)
                    {
                        return result;
                    }
                    return LoadDatasetFromOtherSource(w, dsinfo, status, filter, l.Name, linuxFinder, string.Format("{0}/{1}", props["LinuxTempLocation"], dsinfo.Name.SantizeDSName()), failNow, timeout);
                };

            return l;
        }
コード例 #4
0
ファイル: LocalMachine.cs プロジェクト: LHCAtlas/AtlasSSH
        /// <summary>
        /// Attempt to load the dataset from another location.
        /// </summary>
        /// <param name="dsinfo"></param>
        /// <param name="status"></param>
        /// <param name="filter"></param>
        /// <returns></returns>
        private static Uri[] LoadDatasetFromOtherSource(WindowsDataset dsLocalLocation,
            DSInfo dsinfo,
            Action<string> status,
            Func<string[], string[]> filter, 
            string locName, 
            IFetchToRemoteLinuxDir fetcher,
            string linuxLocation,
            Func<bool> failNow,
            int timeout)
        {
            // First, attempt to find the Uri's from somewhere that we can see and easily copy.
            try
            {
                var files = GRIDDatasetLocator.FetchDatasetUris(dsinfo.Name, status, filter, locationFilter: locname => locname != locName, failNow: failNow);
                var dsinfoRemote = GRIDDatasetLocator.FetchDSInfo(dsinfo.Name, filter, locationFilter: locname => locname != locName);
                if (files != null)
                {
                    dsLocalLocation.MarkAsPartialDownload(dsinfo.Name);
                    var flookup = new HashSet<string>(dsLocalLocation.FindDSFiles(dsinfo.Name, returnWhatWeHave: true).Select(u => Path.GetFileName(u.LocalPath)));
                    foreach (var fremote in files)
                    {
                        if (!flookup.Contains(Path.GetFileName(fremote.LocalPath)))
                        {
                            var f = new FileInfo(fremote.LocalPath);
                            if (status != null)
                            {
                                status(string.Format("Copying file {0}.", f.Name));
                            }
                            f.CopyTo(Path.Combine(dsLocalLocation.LocationOfDataset(dsinfo.Name).FullName, f.Name));
                        }
                    }

                    // And update the meta data.
                    dsLocalLocation.SaveListOfDSFiles(dsinfo.Name, dsinfoRemote.ListOfFiles());
                    dsLocalLocation.RemovePartialDownloadMark(dsinfo.Name);

                    // The files are all local. So we are going to have to see if there is a backup to do the download for us.
                    return dsLocalLocation.FindDSFiles(dsinfo.Name, filter);
                }
            }
            catch (InvalidOperationException)
            {
                // No worries - we weren't able to run one of the data finders.
                // We go onto the next step.
            }

            // OK, nothing could get them. We need to fall back on our secondary copy method. So copy everything to a local Linux directory, and then
            // copy from there down to here.
            dsLocalLocation.MarkAsPartialDownload(dsinfo.Name);
            var allfiles = fetcher.GetListOfFiles(dsinfo.Name, status, failNow: failNow);
            var linuxLocationPerDS = $"{linuxLocation}/{dsinfo.Name}";
            fetcher.Fetch(dsinfo.Name, linuxLocationPerDS, status, filter, failNow: failNow, timeout: timeout);

            // Next, copy the files from there down to our location.
            dsLocalLocation.SaveListOfDSFiles(dsinfo.Name, allfiles);
            fetcher.CopyFromRemote(linuxLocationPerDS, dsLocalLocation.LocationOfDataset(dsinfo.Name), status, removeDirectoryWhenDone: true);
            dsLocalLocation.RemovePartialDownloadMark(dsinfo.Name);

            return dsLocalLocation.FindDSFiles(dsinfo.Name, filter);
        }
コード例 #5
0
 /// <summary>
 /// Initialized the fetch to Linux code, where the Linux stuff is visible on windows.
 /// </summary>
 /// <param name="windowsFilesLocation"></param>
 /// <param name="fetcher"></param>
 /// <param name="rootLinuxLocation"></param>
 public GRIDFetchToLinuxVisibleOnWindows(DirectoryInfo windowsFilesLocation, IFetchToRemoteLinuxDir fetcher, string rootLinuxLocation)
 {
     _winDataset = new WindowsDataset(windowsFilesLocation);
     LinuxFetcher = fetcher;
     LinuxRootDSDirectory = rootLinuxLocation;
 }