/// <summary>
/// Create a cluster resident object corresponding to a given pathname.
/// </summary>
/// <param name="path">Path to the cluster-resident object.</param>
/// <param name="config">Cluster where the file resides.</param>
/// <param name="shouldCache">If true the file should be cached.</param>
/// <param name="job">Job who owns this file.</param>
public UNCFile(ClusterConfiguration config, DryadLinqJobSummary job, UNCPathname path, bool shouldCache) : base(config, job)
{
this.Pathname = path;
this.Exception = null;
this.ShouldCacheLocally = shouldCache;
//if (! this.RepresentsAFolder)
this.LocalCachePath = this.CachePath(this.Pathname);
}
/// <summary>
/// From the URL extract a Path to a filename in the local cache.
/// </summary>
/// <param name="path">Path that is to be cached.</param>
/// <returns>A local pathname, or null if file should not be cached.</returns>
private string CachePath(UNCPathname path)
{
if (CachedClusterResidentObject.CacheDirectory == null || !this.ShouldCacheLocally)
{
return(null);
}
{
return(null);
}
}
/// <summary>
/// For a folder object, returns the contained folder with the specified name.
/// </summary>
/// <param name="foldername">Folder name within the folder.</param>
/// <returns>The subfolder within the folder.</returns>
public override IClusterResidentObject GetFolder(string foldername)
{
if (!this.RepresentsAFolder)
{
throw new InvalidOperationException("Cannot find file within non-folder");
}
UNCPathname dirpath = new UNCPathname(this.Pathname);
dirpath.Directory = Path.Combine(dirpath.Directory, foldername);
return(new UNCFile(this.Config, this.Job, dirpath, this.ShouldCacheLocally));
}
/// <summary>
/// The contents of the folder.
/// </summary>
/// <param name="match">Pattern to match.</param>
/// <returns>The matching objects.</returns>
public override IEnumerable <IClusterResidentObject> GetFilesAndFolders(string match)
{
if (!this.RepresentsAFolder)
{
yield break;
}
string[] dirs = null, files = null;
Exception exception = null;
try
{
dirs = Directory.GetDirectories(this.Pathname.ToString(), match);
}
catch (Exception ex) {
exception = ex;
}
if (exception != null)
{
yield return(new UNCFile(exception));
yield break;
}
foreach (string dir in dirs)
{
UNCPathname dirpath = new UNCPathname(this.Pathname);
// ReSharper disable once AssignNullToNotNullAttribute
dirpath.Directory = Path.Combine(dirpath.Directory, Path.GetFileName(dir));
yield return(new UNCFile(this.Config, this.Job, dirpath, this.ShouldCacheLocally));
}
try
{
files = Directory.GetFiles(this.Pathname.ToString(), match);
}
catch (Exception ex) {
exception = ex;
}
if (exception != null)
{
yield return(new UNCFile(exception));
yield break;
}
foreach (string file in files)
{
UNCPathname dirpath = new UNCPathname(this.Pathname);
dirpath.Filename = Path.GetFileName(file);
yield return(new UNCFile(this.Config, this.Job, dirpath, this.ShouldCacheLocally));
}
}
/// <summary>
/// Create a PartitionedTable metadata file from a list of partitions.
/// </summary>
/// <param name="metadataFile">Pathname for file containing the metadata.</param>
/// <returns>The URI to use to read this metadata file.</returns>
public string CreateMetadataFile(UNCPathname metadataFile)
{
// compute the prefix of the first partition
string prefix = partitions.First().Prefix;
Utilities.EnsureDirectoryExistsForFile(metadataFile.ToString());
StreamWriter sw = new StreamWriter(metadataFile.ToString());
sw.WriteLine(prefix);
sw.WriteLine(this.partitions.Count);
foreach (var p in this.partitions)
{
sw.WriteLine(p.ToShortString());
}
sw.Close();
return(Uri.UriSchemeFile + @"://" + metadataFile);
}
/// <summary>
/// Read the metadata from a partitioned file.
/// </summary>
/// <param name="metadataFile">Metadata file to read.</param>
public PartitionedFileMetadata(UNCPathname metadataFile)
{
StreamReader sr = new StreamReader(metadataFile.ToString());
string defaultPrefix = sr.ReadLine();
string str = sr.ReadLine();
if (string.IsNullOrEmpty(str))
{
throw new InvalidDataException("Expected a partitioned count, found none");
}
int partitionCount = int.Parse(str);
partitions = new List <Partition>(partitionCount);
for (int i = 0; i < partitionCount; i++)
{
this.Add(new Partition(sr.ReadLine(), defaultPrefix));
}
}
/// <summary>
/// Creates a new instance of this class in preparation for debugging a vertex locally.
/// </summary>
/// <param name="guid">Guid for the vertex to debug.</param>
/// <param name="config">Cluster where job debugged is running.</param>
/// <param name="vertexWorkingDirPath">Path to the (remote) working directory of the vertex.</param>
/// <param name="statusWriter">Used to display status messages nicely.</param>
/// <param name="version">Vertex version to debug.</param>
/// <param name="managed">If true debug managed version.</param>
/// <param name="cpuSampling">If true perform cpu sampling based profiling.</param>
/// <param name="number">Vertex number.</param>
public LocalDebuggingAndProfiling(ClusterConfiguration config,
string guid,
int number,
int version,
IClusterResidentObject vertexWorkingDirPath,
bool managed,
bool cpuSampling,
StatusReporter statusWriter)
{
this.cluster = config;
this.workingDirPath = (vertexWorkingDirPath as UNCFile).Pathname;
this.guid = guid;
this.reporter = statusWriter;
this.cpuSampling = cpuSampling;
this.number = number;
this.version = version;
if (!managed)
throw new Exception("Unmanaged debugging not supported");
}
/// <summary>
/// Creates a new instance of this class in preparation for debugging a vertex locally.
/// </summary>
/// <param name="guid">Guid for the vertex to debug.</param>
/// <param name="config">Cluster where job debugged is running.</param>
/// <param name="vertexWorkingDirPath">Path to the (remote) working directory of the vertex.</param>
/// <param name="statusWriter">Used to display status messages nicely.</param>
/// <param name="version">Vertex version to debug.</param>
/// <param name="managed">If true debug managed version.</param>
/// <param name="cpuSampling">If true perform cpu sampling based profiling.</param>
/// <param name="number">Vertex number.</param>
public LocalDebuggingAndProfiling(ClusterConfiguration config,
string guid,
int number,
int version,
IClusterResidentObject vertexWorkingDirPath,
bool managed,
bool cpuSampling,
StatusReporter statusWriter)
{
this.cluster = config;
this.workingDirPath = (vertexWorkingDirPath as UNCFile).Pathname;
this.guid = guid;
this.reporter = statusWriter;
this.cpuSampling = cpuSampling;
this.number = number;
this.version = version;
if (!managed)
{
throw new Exception("Unmanaged debugging not supported");
}
}
/// <summary>
/// For a folder object, returns the contained folder with the specified name.
/// </summary>
/// <param name="foldername">Folder name within the folder.</param>
/// <returns>The subfolder within the folder.</returns>
public override IClusterResidentObject GetFolder(string foldername)
{
if (!this.RepresentsAFolder)
throw new InvalidOperationException("Cannot find file within non-folder");
UNCPathname dirpath = new UNCPathname(this.Pathname);
dirpath.Directory = Path.Combine(dirpath.Directory, foldername);
return new UNCFile(this.Config, this.Job, dirpath, this.ShouldCacheLocally);
}
/// <summary>
/// The contents of the folder.
/// </summary>
/// <param name="match">Pattern to match.</param>
/// <returns>The matching objects.</returns>
public override IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
{
if (!this.RepresentsAFolder)
yield break;
string[] dirs = null, files = null;
Exception exception = null;
try
{
dirs = Directory.GetDirectories(this.Pathname.ToString(), match);
}
catch (Exception ex) {
exception = ex;
}
if (exception != null)
{
yield return new UNCFile(exception);
yield break;
}
foreach (string dir in dirs)
{
UNCPathname dirpath = new UNCPathname(this.Pathname);
// ReSharper disable once AssignNullToNotNullAttribute
dirpath.Directory = Path.Combine(dirpath.Directory, Path.GetFileName(dir));
yield return new UNCFile(this.Config, this.Job, dirpath, this.ShouldCacheLocally);
}
try
{
files = Directory.GetFiles(this.Pathname.ToString(), match);
}
catch (Exception ex) {
exception = ex;
}
if (exception != null)
{
yield return new UNCFile(exception);
yield break;
}
foreach (string file in files)
{
UNCPathname dirpath = new UNCPathname(this.Pathname);
dirpath.Filename = Path.GetFileName(file);
yield return new UNCFile(this.Config, this.Job, dirpath, this.ShouldCacheLocally);
}
}
/// <summary>
/// From the URL extract a Path to a filename in the local cache.
/// </summary>
/// <param name="path">Path that is to be cached.</param>
/// <returns>A local pathname, or null if file should not be cached.</returns>
private string CachePath(UNCPathname path)
{
if (CachedClusterResidentObject.CacheDirectory == null || !this.ShouldCacheLocally)
return null;
{
return null;
}
}
/// <summary>
/// Create a cluster resident object corresponding to a given pathname.
/// </summary>
/// <param name="path">Path to the cluster-resident object.</param>
/// <param name="config">Cluster where the file resides.</param>
/// <param name="shouldCache">If true the file should be cached.</param>
/// <param name="job">Job who owns this file.</param>
public UNCFile(ClusterConfiguration config, DryadLinqJobSummary job, UNCPathname path, bool shouldCache) : base(config, job)
{
this.Pathname = path;
this.Exception = null;
this.ShouldCacheLocally = shouldCache;
//if (! this.RepresentsAFolder)
this.LocalCachePath = this.CachePath(this.Pathname);
}
/// <summary>
/// Add a new replica to a partition.
/// </summary>
/// <param name="path">Pathname to the replica to add.</param>
public void AddReplica(UNCPathname path)
{
replicas.Add(path);
}
/// <summary>
/// Read the metadata from a partitioned file.
/// </summary>
/// <param name="metadataFile">Metadata file to read.</param>
public PartitionedFileMetadata(UNCPathname metadataFile)
{
StreamReader sr = new StreamReader(metadataFile.ToString());
string defaultPrefix = sr.ReadLine();
string str = sr.ReadLine();
if (string.IsNullOrEmpty(str))
throw new InvalidDataException("Expected a partitioned count, found none");
int partitionCount = int.Parse(str);
partitions = new List<Partition>(partitionCount);
for (int i = 0; i < partitionCount; i++)
this.Add(new Partition(sr.ReadLine(), defaultPrefix));
}
/// <summary>
/// Create a PartitionedTable metadata file from a list of partitions.
/// </summary>
/// <param name="metadataFile">Pathname for file containing the metadata.</param>
/// <returns>The URI to use to read this metadata file.</returns>
public string CreateMetadataFile(UNCPathname metadataFile)
{
// compute the prefix of the first partition
string prefix = partitions.First().Prefix;
Utilities.EnsureDirectoryExistsForFile(metadataFile.ToString());
StreamWriter sw = new StreamWriter(metadataFile.ToString());
sw.WriteLine(prefix);
sw.WriteLine(this.partitions.Count);
foreach (var p in this.partitions)
sw.WriteLine(p.ToShortString());
sw.Close();
return Uri.UriSchemeFile + @"://" + metadataFile;
}
/// <summary>
/// Add a new replica to a partition.
/// </summary>
/// <param name="path">Pathname to the replica to add.</param>
public void AddReplica(UNCPathname path)
{
replicas.Add(path);
}
