/// <summary>
/// add the local file to the list of resources, adding it to HDFS decorated
/// with its hash if necessary
/// </summary>
/// <param name="fileName">leaf name of the local file</param>
/// <param name="localDirectory">local directory containing the file</param>
public void EnsureResource(string fileName, string localDirectory)
{
var filePath = Path.Combine(localDirectory, fileName);
if (!File.Exists(filePath))
{
throw new FileNotFoundException("Can't find resource", fileName);
}
var hash = ResourceTools.MakeHash(filePath);
var remoteName = fileName + "." + hash;
var hdfsFile = new Uri(directoryURI, remoteName).AbsoluteUri;
using (var hdfs = new Hdfs.HdfsInstance(directoryURI))
{
if (!hdfs.IsFileExists(hdfsFile))
{
hdfs.UploadAll(filePath, hdfsFile);
}
var info = hdfs.GetFileInfo(hdfsFile, false);
var file = new HdfsIdentifier();
file.remoteName = remoteName;
file.localName = fileName;
file.size = info.Size;
file.timestamp = info.LastModified;
files.Add(file);
}
}
/// <summary>
/// get the details of the file names and their location from an XML element
/// </summary>
/// <param name="config">the XML element containing config information</param>
public void ReadFromConfig(XElement config)
{
files = new List <HdfsIdentifier>();
// the relative or absolute path prefix that identifies the location of the
// files
directoryURI = new Uri(config.Attribute("location").Value);
isPublic = (config.Attribute("public") != null && config.Attribute("public").Value == "true");
bool needsLookup = false;
// the metadata for each file is stored in a separate Resource element.
foreach (var e in config.Descendants("Resource"))
{
var file = new HdfsIdentifier();
var lnAttribute = e.Attribute("localName");
if (lnAttribute == null)
{
file.localName = e.Value;
}
else
{
file.localName = lnAttribute.Value;
}
file.remoteName = e.Value;
if (e.Attribute("timestamp") == null || e.Attribute("size") == null)
{
needsLookup = true;
file.timestamp = -1;
file.size = -1;
}
else
{
file.timestamp = long.Parse(e.Attribute("timestamp").Value);
file.size = long.Parse(e.Attribute("size").Value);
}
files.Add(file);
}
if (needsLookup)
{
using (var instance = new Hdfs.HdfsInstance(directoryURI))
{
foreach (var r in files.Where(f => f.timestamp < 0 || f.size < 0))
{
var fileUri = new Uri(directoryURI, r.remoteName);
var info = instance.GetFileInfo(fileUri.AbsoluteUri, false);
r.timestamp = info.LastModified;
r.size = info.Size;
}
}
}
}
