public void UploadMetadataDocument(Record record)
{
string resourceUrl = config.HttpRootUrl + "/" + WebificationUtility.GetUnrootedDataPath(record.Id, record.Path);
var metaXmlDoc = xmlHelper.GetMetadataDocument(record, resourceUrl);
string metaPath = String.Format("waf/{0}.xml", record.Id);
string metaFtpPath = config.FtpRootUrl + "/" + metaPath;
Logger.Info("Metadata file path: " + metaPath);
Logger.Info("Metadata FTP path: " + metaFtpPath);
ftpClient.UploadBytes(metaFtpPath, metaXmlDoc);
Logger.Info("Uploaded metadata document successfully");
}
public void UploadDataFile(Guid recordId, string filePath)
{
filePath = JnccDriveMappings.GetUncPath(filePath);
string unrootedDataPath = WebificationUtility.GetUnrootedDataPath(recordId, filePath);
string dataFtpPath = config.FtpRootUrl + "/" + unrootedDataPath;
Logger.Info("Data file path: " + filePath);
Logger.Info("Data FTP path: " + dataFtpPath);
ftpClient.UploadFile(dataFtpPath, filePath);
Logger.Info("Uploaded data file successfully");
}
private void ResourcesUpdatedCorrectly(string recordId, List <Resource> originalResources, List <Resource> updatedResources)
{
originalResources.Count.Should().Be(updatedResources.Count);
foreach (var originalResource in originalResources)
{
updatedResources.Count(r => r.Path.Equals(originalResource.Path)).Should().Be(1);
var updatedResource = updatedResources.Find(x => x.Path.Equals(originalResource.Path));
if (IsFileResource(updatedResource))
{
string dataPath = WebificationUtility.GetUnrootedDataPath(recordId, originalResource.Path);
updatedResource.PublishedUrl.Should().Be("http://data.jncc.gov.uk" + dataPath);
}
else
{
updatedResource.PublishedUrl.Should().BeNullOrEmpty();
}
}
}
public void UploadAlternativeResources(Record record)
{
// check no duplicate filenames after webifying
var fileNames = from r in record.Publication.OpenData.Resources
let fileName = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(r.Path))
group r by fileName;
if (fileNames.Count() != record.Publication.OpenData.Resources.Count)
{
var e = new Exception("There are duplicate resource file names (after webifying) for this record.");
e.LogAndThrow(Logger);
}
// upload the resources
foreach (var r in record.Publication.OpenData.Resources)
{
UploadDataFile(record.Id, r.Path);
}
}
XElement MakeResourceLocator(Metadata metadata, Guid id)
{
string fileName = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(metadata.ResourceLocator));
// but this doesn't work so well when the resource locator contains the guid like this
// http://data.jncc.gov.uk/data/d4ddd363-97eb-4ef6-9b8b-6f019f434103-ProcessedImages.zip
// so remove the proceeding guid
if (fileName.IsNotBlank() && fileName.StartsWith(id + "-"))
{
fileName = fileName.Replace(id + "-", String.Empty);
}
return(new XElement(gmd + "transferOptions",
new XElement(gmd + "MD_DigitalTransferOptions",
new XElement(gmd + "onLine",
new XElement(gmd + "CI_OnlineResource",
new XElement(gmd + "linkage",
new XElement(gmd + "URL", metadata.ResourceLocator)),
new XElement(gmd + "name",
new XElement(gco + "CharacterString", fileName)))))));
}
public void UploadDataFile(string recordId, string filePath)
{
filePath = JnccDriveMappings.GetUncPath(filePath);
var fileSize = fileHelper.GetFileSizeInBytes(filePath);
if (fileSize <= env.MAX_FILE_SIZE_IN_BYTES)
{
string dataFtpPath = WebificationUtility.GetUnrootedDataPath(recordId, filePath);
Logger.Info("Data file path: " + filePath);
Logger.Info("Data FTP path: " + dataFtpPath);
ftpClient.UploadFile(dataFtpPath, filePath);
Logger.Info("Uploaded data file successfully");
}
else
{
// force fail large files
throw new InvalidOperationException($"File at path {filePath} is too large to be uploaded by Topcat - manual upload required");
}
}
public byte[] GetMetadataDocument(Record record, string resourceUrl)
{
bool alternativeResources = record.Publication != null && record.Publication.OpenData != null && record.Publication.OpenData.Resources != null && record.Publication.OpenData.Resources.Any();
var doc = new Gemini.Encoding.XmlEncoder().Create(record.Id, record.Gemini);
if (alternativeResources)
{
// mung (mutate) the metadata doc so data.gov.uk knows about the resources
var onlineResources = record.Publication.OpenData.Resources
.Select(r => new OnlineResource
{
Name = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(r.Path)),
Url = resourceUrl
}).ToList();
Gemini.Encoding.XmlEncoder.ReplaceDigitalTransferOptions(doc, onlineResources);
}
var s = new MemoryStream();
doc.Save(s);
return(s.ToArray());
}
public string test_to_url_friendly_string(string s)
{
return(WebificationUtility.ToUrlFriendlyString(s));
}