public void try_ftp_client()
{
string password = "******";
var c = new FtpClient("topcat", password);
string s = c.DownloadString("ftp://data.jncc.gov.uk/waf/index.html");
Console.WriteLine(s);
string filePath = @"C:\Work\test-data.csv";
File.Exists(filePath).Should().BeTrue();
c.UploadFile("ftp://data.jncc.gov.uk/" + new FileInfo(filePath).Name, filePath);
}
static int RunPublishOpenData(PublishOpenDataOptions options)
{
InitDatabase();
// load the config
var configPath = Path.Combine(Environment.CurrentDirectory, "data-gov-uk-publisher-config.json");
if (!File.Exists(configPath))
throw new Exception("No data-gov-uk-publisher-config.json file in current directory.");
string configJson = File.ReadAllText(configPath);
var config = JsonConvert.DeserializeObject<OpenDataPublisherConfig>(configJson);
if (config.FtpRootUrl.IsBlank())
throw new Exception("No FtpRootUrl specified in data-gov-uk-publisher-config.json file.");
if (config.HttpRootUrl.IsBlank())
throw new Exception("No HttpRootUrl specified in data-gov-uk-publisher-config.json file.");
if (config.FtpUsername.IsBlank())
throw new Exception("No FtpUsername specified in data-gov-uk-publisher-config.json file.");
if (config.FtpPassword.IsBlank())
throw new Exception("No FtpPassword specified in data-gov-uk-publisher-config.json file.");
Console.WriteLine("Publishing to '{0}'", config.FtpRootUrl);
var ids = new List<Guid>();
using (var db = DocumentStore.OpenSession())
{
if (options.RecordId.IsNotBlank())
{
// publish the specified record
ids = new List<Guid> { Guid.Parse(options.RecordId) };
}
else
{
// get the records that are pending publication (ie not PublishedSinceLastUpdated)
ids = db.Query<RecordsWithOpenDataPublicationInfoIndex.Result, RecordsWithOpenDataPublicationInfoIndex>()
.Where(x => !x.PublishedSinceLastUpdated)
.Where(x => x.GeminiValidated) // all open data should be gemini-valid - this is a safety
.OfType<Record>() //.Select(r => r.Id) // this doesn't work in RavenDB, and doesn't throw!
.Take(1000) // so take 1000 which is enough for one run
.ToList() // so materialize the record first
.Where(r => !r.Publication.OpenData.Paused) // .Where(x => !x.PublishingIsPaused) on the server doesn't work on live - thanks, ravenDB
.Select(r => r.Id)
.ToList();
}
}
Console.WriteLine("Publishing {0} records...", ids.Count);
if (!options.WhatIf)
{
foreach (var id in ids)
{
var ftpClient = new FtpClient(config.FtpUsername, config.FtpPassword);
using (var db = DocumentStore.OpenSession())
{
new OpenDataRecordPublisher(db, config, options.MetadataOnly, ftpClient).PublishRecord(id);
}
}
}
Console.WriteLine("Published (or skipped) {0} records.", ids.Count);
if (options.WhatIf)
{
Console.WriteLine("Ran in 'what-if' mode. Nothing was really done.");
}
return 1;
}