private void ProcessOneBook(Book book)
{
try
{
_logger.TrackEvent("ProcessOneBook Start");
string message = $"Processing: {book.BaseUrl}";
Console.Out.WriteLine(message);
_logger.LogVerbose(message);
var initialUpdates = new UpdateOperation();
initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString());
initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
var startTime = new Parse.Model.Date(DateTime.UtcNow);
initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson());
// Process the book
var finalUpdates = new UpdateOperation();
var warnings = FindBookWarnings(book);
finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings));
// ENHANCE: Do more processing here
// Write the updates
finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson());
_logger.TrackEvent("ProcessOneBook End - Success");
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\"");
// Attempt to write to Parse that processing failed
if (!String.IsNullOrEmpty(book?.ObjectId))
{
try
{
var onErrorUpdates = new UpdateOperation();
onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\"");
onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson());
}
catch (Exception)
{
// If it fails, just let it be and throw the first exception rather than the nested exception.
}
}
throw;
}
}
public void TestIssueSubmission()
{
var issueId = YouTrackIssueConnector.SubmitToYouTrack("Harvester Test Issue",
"This is a test from Harvester, which apparently cannot be overemphasized.", "AUT");
Assert.That(issueId, Is.Not.Null);
Assert.That(issueId, Does.StartWith("AUT-"));
// Creating a new submitter seems a bit wasteful, and a bit too far into implementation details,
// but it's the only way to delete the newly created issue.
var submitter = new Bloom.YouTrackIssueSubmitter("AUT");
var deleted = submitter.DeleteIssue(issueId);
Assert.That(deleted, Is.True);
}
// Command line arguments sample: "harvestAll --environment=dev --parseDBEnvironment=prod"
//
// Some presets that you might copy and paste in:
// harvestAll --environment=dev --parseDBEnvironment=local --suppressLogs --count=2
// harvestWarnings --environment=dev --parseDBEnvironment=local --suppressLogs
// harvestAll --environment=dev --parseDBEnvironment=local --suppressLogs "--queryWhere={ \"objectId\":\"38WdeYJ0yF\"}"
// harvestAll --environment=dev --parseDBEnvironment=dev --suppressLogs "--queryWhere={ \"objectId\":\"JUCL9OMOza\"}"
// harvestAll --environment=dev --parseDBEnvironment=dev --suppressLogs "--queryWhere={ \"title\":{\"$in\":[\"Vaccinations\",\"Fox and Frog\",\"The Moon and the Cap\"]}}"
public static void Main(string[] args)
{
// See https://github.com/commandlineparser/commandline for documentation about CommandLine.Parser
var parser = new CommandLine.Parser((settings) =>
{
settings.CaseInsensitiveEnumValues = true;
settings.CaseSensitive = false;
settings.HelpWriter = Console.Error;
});
try
{
parser.ParseArguments <HarvestAllOptions, HarvestHighPriorityOptions, HarvestLowPriorityOptions, HarvestWarningsOptions>(args)
.WithParsed <HarvestAllOptions>(options =>
{
Harvester.RunHarvestAll(options);
})
.WithParsed <HarvestWarningsOptions>(options =>
{
Harvester.RunHarvestWarnings(options);
})
// TODO: Replace placeholders
.WithParsed <HarvestHighPriorityOptions>(options => { throw new NotImplementedException("HarvestHighPriority"); })
.WithParsed <HarvestLowPriorityOptions>(options => { throw new NotImplementedException("HarvestLowPriority"); })
.WithNotParsed(errors =>
{
Console.Out.WriteLine("Error parsing command line arguments.");
Environment.Exit(1);
});
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, "An exception was thrown which was not handled by the program.");
throw;
}
}
private void ProcessOneBook(Book book)
{
try
{
_logger.TrackEvent("ProcessOneBook Start");
string message = $"Processing: {book.BaseUrl}";
Console.Out.WriteLine(message);
_logger.LogVerbose(message);
var initialUpdates = new UpdateOperation();
initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString());
initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
var startTime = new Parse.Model.Date(DateTime.UtcNow);
initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson());
// Download the book
_logger.TrackEvent("Download Book");
string decodedUrl = HttpUtility.UrlDecode(book.BaseUrl);
string urlWithoutTitle = RemoveBookTitleFromBaseUrl(decodedUrl);
string downloadRootDir = Path.Combine(Path.GetTempPath(), Path.Combine("BloomHarvester", this.Identifier));
_logger.LogVerbose("Download Dir: {0}", downloadRootDir);
string downloadBookDir = _transfer.HandleDownloadWithoutProgress(urlWithoutTitle, downloadRootDir);
// Process the book
var finalUpdates = new UpdateOperation();
var warnings = FindBookWarnings(book);
finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings));
// ENHANCE: Do more processing here
_logger.TrackEvent("Upload Book");
UploadBook(decodedUrl, downloadBookDir);
// Write the updates
finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson());
_logger.TrackEvent("ProcessOneBook End - Success");
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\"");
// Attempt to write to Parse that processing failed
if (!String.IsNullOrEmpty(book?.ObjectId))
{
try
{
var onErrorUpdates = new UpdateOperation();
onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\"");
onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson());
}
catch (Exception)
{
// If it fails, just let it be and throw the first exception rather than the nested exception.
}
}
throw;
}
}
