/// <summary> /// add the local file to the list of resources, adding it to HDFS decorated /// with its hash if necessary /// </summary> /// <param name="fileName">leaf name of the local file</param> /// <param name="localDirectory">local directory containing the file</param> public void EnsureResource(string fileName, string localDirectory) { var filePath = Path.Combine(localDirectory, fileName); if (!File.Exists(filePath)) { throw new FileNotFoundException("Can't find resource", fileName); } var hash = ResourceTools.MakeHash(filePath); var remoteName = fileName + "." + hash; var hdfsFile = new Uri(directoryURI, remoteName).AbsoluteUri; using (var hdfs = new Hdfs.HdfsInstance(directoryURI)) { if (!hdfs.IsFileExists(hdfsFile)) { hdfs.UploadAll(filePath, hdfsFile); } var info = hdfs.GetFileInfo(hdfsFile, false); var file = new HdfsIdentifier(); file.remoteName = remoteName; file.localName = fileName; file.size = info.Size; file.timestamp = info.LastModified; files.Add(file); } }
/// <summary> /// get the details of the file names and their location from an XML element /// </summary> /// <param name="config">the XML element containing config information</param> public void ReadFromConfig(XElement config) { files = new List <HdfsIdentifier>(); // the relative or absolute path prefix that identifies the location of the // files directoryURI = new Uri(config.Attribute("location").Value); isPublic = (config.Attribute("public") != null && config.Attribute("public").Value == "true"); bool needsLookup = false; // the metadata for each file is stored in a separate Resource element. foreach (var e in config.Descendants("Resource")) { var file = new HdfsIdentifier(); var lnAttribute = e.Attribute("localName"); if (lnAttribute == null) { file.localName = e.Value; } else { file.localName = lnAttribute.Value; } file.remoteName = e.Value; if (e.Attribute("timestamp") == null || e.Attribute("size") == null) { needsLookup = true; file.timestamp = -1; file.size = -1; } else { file.timestamp = long.Parse(e.Attribute("timestamp").Value); file.size = long.Parse(e.Attribute("size").Value); } files.Add(file); } if (needsLookup) { using (var instance = new Hdfs.HdfsInstance(directoryURI)) { foreach (var r in files.Where(f => f.timestamp < 0 || f.size < 0)) { var fileUri = new Uri(directoryURI, r.remoteName); var info = instance.GetFileInfo(fileUri.AbsoluteUri, false); r.timestamp = info.LastModified; r.size = info.Size; } } } }