/// <summary> /// Attempt to load the dataset from another location. /// </summary> /// <param name="dsinfo"></param> /// <param name="status"></param> /// <param name="filter"></param> /// <returns></returns> private static Uri[] LoadDatasetFromOtherSource(WindowsDataset dsLocalLocation, DSInfo dsinfo, Action<string> status, Func<string[], string[]> filter, string locName, IFetchToRemoteLinuxDir fetcher, string linuxLocation, Func<bool> failNow, int timeout) { // First, attempt to find the Uri's from somewhere that we can see and easily copy. try { var files = GRIDDatasetLocator.FetchDatasetUris(dsinfo.Name, status, filter, locationFilter: locname => locname != locName, failNow: failNow); var dsinfoRemote = GRIDDatasetLocator.FetchDSInfo(dsinfo.Name, filter, locationFilter: locname => locname != locName); if (files != null) { dsLocalLocation.MarkAsPartialDownload(dsinfo.Name); var flookup = new HashSet<string>(dsLocalLocation.FindDSFiles(dsinfo.Name, returnWhatWeHave: true).Select(u => Path.GetFileName(u.LocalPath))); foreach (var fremote in files) { if (!flookup.Contains(Path.GetFileName(fremote.LocalPath))) { var f = new FileInfo(fremote.LocalPath); if (status != null) { status(string.Format("Copying file {0}.", f.Name)); } f.CopyTo(Path.Combine(dsLocalLocation.LocationOfDataset(dsinfo.Name).FullName, f.Name)); } } // And update the meta data. dsLocalLocation.SaveListOfDSFiles(dsinfo.Name, dsinfoRemote.ListOfFiles()); dsLocalLocation.RemovePartialDownloadMark(dsinfo.Name); // The files are all local. So we are going to have to see if there is a backup to do the download for us. return dsLocalLocation.FindDSFiles(dsinfo.Name, filter); } } catch (InvalidOperationException) { // No worries - we weren't able to run one of the data finders. // We go onto the next step. } // OK, nothing could get them. We need to fall back on our secondary copy method. So copy everything to a local Linux directory, and then // copy from there down to here. dsLocalLocation.MarkAsPartialDownload(dsinfo.Name); var allfiles = fetcher.GetListOfFiles(dsinfo.Name, status, failNow: failNow); var linuxLocationPerDS = $"{linuxLocation}/{dsinfo.Name}"; fetcher.Fetch(dsinfo.Name, linuxLocationPerDS, status, filter, failNow: failNow, timeout: timeout); // Next, copy the files from there down to our location. dsLocalLocation.SaveListOfDSFiles(dsinfo.Name, allfiles); fetcher.CopyFromRemote(linuxLocationPerDS, dsLocalLocation.LocationOfDataset(dsinfo.Name), status, removeDirectoryWhenDone: true); dsLocalLocation.RemovePartialDownloadMark(dsinfo.Name); return dsLocalLocation.FindDSFiles(dsinfo.Name, filter); }
/// <summary> /// Initialized the fetch to Linux code, where the Linux stuff is visible on windows. /// </summary> /// <param name="windowsFilesLocation"></param> /// <param name="fetcher"></param> /// <param name="rootLinuxLocation"></param> public GRIDFetchToLinuxVisibleOnWindows(DirectoryInfo windowsFilesLocation, IFetchToRemoteLinuxDir fetcher, string rootLinuxLocation) { _winDataset = new WindowsDataset(windowsFilesLocation); LinuxFetcher = fetcher; LinuxRootDSDirectory = rootLinuxLocation; }