private void ProcessOneBook(Book book)
{
try
{
_logger.TrackEvent("ProcessOneBook Start");
string message = $"Processing: {book.BaseUrl}";
Console.Out.WriteLine(message);
_logger.LogVerbose(message);
var initialUpdates = new UpdateOperation();
initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString());
initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
var startTime = new Parse.Model.Date(DateTime.UtcNow);
initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson());
// Process the book
var finalUpdates = new UpdateOperation();
var warnings = FindBookWarnings(book);
finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings));
// ENHANCE: Do more processing here
// Write the updates
finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson());
_logger.TrackEvent("ProcessOneBook End - Success");
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\"");
// Attempt to write to Parse that processing failed
if (!String.IsNullOrEmpty(book?.ObjectId))
{
try
{
var onErrorUpdates = new UpdateOperation();
onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\"");
onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson());
}
catch (Exception)
{
// If it fails, just let it be and throw the first exception rather than the nested exception.
}
}
throw;
}
}
public static void Main(string[] args)
{
if (Environment.OSVersion.Platform != PlatformID.Win32NT)
{
// See https://issues.bloomlibrary.org/youtrack/issue/BL-8475.
Console.WriteLine("Harvester cannot run on Linux until we verify that phash computations always yield the same result on both Windows and Linux!");
return;
}
// See https://github.com/commandlineparser/commandline for documentation about CommandLine.Parser
var parser = new CommandLine.Parser((settings) =>
{
settings.CaseInsensitiveEnumValues = true;
settings.CaseSensitive = false;
settings.HelpWriter = Console.Error;
});
try
{
parser.ParseArguments <HarvesterOptions, UpdateStateInParseOptions, BatchUpdateStateInParseOptions, GenerateProcessedFilesTSVOptions>(args)
.WithParsed <HarvesterOptions>(options =>
{
options.ValidateOptions();
Harvester.RunHarvest(options);
})
.WithParsed <UpdateStateInParseOptions>(options =>
{
HarvestStateUpdater.UpdateState(options.ParseDBEnvironment, options.ObjectId, options.NewState);
})
.WithParsed <BatchUpdateStateInParseOptions>(options =>
{
HarvestStateBatchUpdater.RunBatchUpdateStates(options);
})
.WithParsed <GenerateProcessedFilesTSVOptions>(options =>
{
ProcessedFilesInfoGenerator.GenerateTSV(options);
})
.WithNotParsed(errors =>
{
Console.Out.WriteLine("Error parsing command line arguments.");
Environment.Exit(1);
});
}
catch (Exception e)
{
YouTrackIssueConnector.GetInstance(EnvironmentSetting.Unknown).ReportException(e, "An exception was thrown which was not handled by the program.", null);
throw;
}
}
public void TestIssueSubmission()
{
var issueId = YouTrackIssueConnector.SubmitToYouTrack("Harvester Test Issue",
"This is a test from Harvester, which apparently cannot be overemphasized.", "AUT");
Assert.That(issueId, Is.Not.Null);
Assert.That(issueId, Does.StartWith("AUT-"));
// Creating a new submitter seems a bit wasteful, and a bit too far into implementation details,
// but it's the only way to delete the newly created issue.
var submitter = new Bloom.YouTrackIssueSubmitter("AUT");
var deleted = submitter.DeleteIssue(issueId);
Assert.That(deleted, Is.True);
}
// Command line arguments sample: "harvestAll --environment=dev --parseDBEnvironment=prod"
//
// Some presets that you might copy and paste in:
// harvestAll --environment=dev --parseDBEnvironment=local --suppressLogs --count=2
// harvestWarnings --environment=dev --parseDBEnvironment=local --suppressLogs
// harvestAll --environment=dev --parseDBEnvironment=local --suppressLogs "--queryWhere={ \"objectId\":\"38WdeYJ0yF\"}"
// harvestAll --environment=dev --parseDBEnvironment=dev --suppressLogs "--queryWhere={ \"objectId\":\"JUCL9OMOza\"}"
// harvestAll --environment=dev --parseDBEnvironment=dev --suppressLogs "--queryWhere={ \"title\":{\"$in\":[\"Vaccinations\",\"Fox and Frog\",\"The Moon and the Cap\"]}}"
public static void Main(string[] args)
{
// See https://github.com/commandlineparser/commandline for documentation about CommandLine.Parser
var parser = new CommandLine.Parser((settings) =>
{
settings.CaseInsensitiveEnumValues = true;
settings.CaseSensitive = false;
settings.HelpWriter = Console.Error;
});
try
{
parser.ParseArguments <HarvestAllOptions, HarvestHighPriorityOptions, HarvestLowPriorityOptions, HarvestWarningsOptions>(args)
.WithParsed <HarvestAllOptions>(options =>
{
Harvester.RunHarvestAll(options);
})
.WithParsed <HarvestWarningsOptions>(options =>
{
Harvester.RunHarvestWarnings(options);
})
// TODO: Replace placeholders
.WithParsed <HarvestHighPriorityOptions>(options => { throw new NotImplementedException("HarvestHighPriority"); })
.WithParsed <HarvestLowPriorityOptions>(options => { throw new NotImplementedException("HarvestLowPriority"); })
.WithNotParsed(errors =>
{
Console.Out.WriteLine("Error parsing command line arguments.");
Environment.Exit(1);
});
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, "An exception was thrown which was not handled by the program.");
throw;
}
}
public void InitializeYouTrackConnector()
{
_connector = YouTrackIssueConnector.GetInstance(EnvironmentSetting.Test, "SB");
}
private void ProcessOneBook(Book book)
{
try
{
_logger.TrackEvent("ProcessOneBook Start");
string message = $"Processing: {book.BaseUrl}";
Console.Out.WriteLine(message);
_logger.LogVerbose(message);
var initialUpdates = new UpdateOperation();
initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString());
initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
var startTime = new Parse.Model.Date(DateTime.UtcNow);
initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson());
// Download the book
_logger.TrackEvent("Download Book");
string decodedUrl = HttpUtility.UrlDecode(book.BaseUrl);
string urlWithoutTitle = RemoveBookTitleFromBaseUrl(decodedUrl);
string downloadRootDir = Path.Combine(Path.GetTempPath(), Path.Combine("BloomHarvester", this.Identifier));
_logger.LogVerbose("Download Dir: {0}", downloadRootDir);
string downloadBookDir = _transfer.HandleDownloadWithoutProgress(urlWithoutTitle, downloadRootDir);
// Process the book
var finalUpdates = new UpdateOperation();
var warnings = FindBookWarnings(book);
finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings));
// ENHANCE: Do more processing here
_logger.TrackEvent("Upload Book");
UploadBook(decodedUrl, downloadBookDir);
// Write the updates
finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson());
_logger.TrackEvent("ProcessOneBook End - Success");
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\"");
// Attempt to write to Parse that processing failed
if (!String.IsNullOrEmpty(book?.ObjectId))
{
try
{
var onErrorUpdates = new UpdateOperation();
onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\"");
onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson());
}
catch (Exception)
{
// If it fails, just let it be and throw the first exception rather than the nested exception.
}
}
throw;
}
}
public AzureMonitorLogger(EnvironmentSetting environment, string harvesterId)
{
_issueReporter = YouTrackIssueConnector.GetInstance(environment);
// Get the Instrumentation Key for Azure from an environment variable.
string environmentVarName = "BloomHarvesterAzureAppInsightsKeyDev";
if (environment == EnvironmentSetting.Test)
{
environmentVarName = "BloomHarvesterAzureAppInsightsKeyTest";
}
else if (environment == EnvironmentSetting.Prod)
{
environmentVarName = "BloomHarvesterAzureAppInsightsKeyProd";
}
string instrumentationKey = Environment.GetEnvironmentVariable(environmentVarName);
Debug.Assert(!String.IsNullOrWhiteSpace(instrumentationKey), "Azure Instrumentation Key is invalid. Azure logging probably won't work.");
try
{
Microsoft.ApplicationInsights.Extensibility.TelemetryConfiguration.Active.InstrumentationKey = instrumentationKey;
}
catch (ArgumentNullException e)
{
_issueReporter.ReportException(e, $"InstrumentationKey: {instrumentationKey ?? "null"}.\nenvironmentVarName: {environmentVarName}", null);
}
_telemetry.Context.User.Id = "BloomHarvester " + harvesterId;
_telemetry.Context.Session.Id = Guid.NewGuid().ToString();
_telemetry.Context.Device.OperatingSystem = Environment.OSVersion.ToString();
string logFilePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "BloomHarvester", "log.txt");
Console.Out.WriteLine("Creating log file at: " + logFilePath);
try
{
if (File.Exists(logFilePath))
{
// Check if the file is too big (~10 MB)
if (new FileInfo(logFilePath).Length > 10000000)
{
var oldPath = logFilePath + "-OLD";
// Preserve one previous log file for debugging help, and start over with
// an empty log file.
// (The data is in Azure too anyway, but having it local may speed things up.)
if (RobustFile.Exists(oldPath))
{
RobustFile.Delete(oldPath);
}
RobustFile.Move(logFilePath, oldPath);
}
}
}
catch
{
// Doesn't matter if there are any errors
}
try
{
_fileLogger = new FileLogger(logFilePath);
}
catch
{
// That's unfortunate that creating the logger failed, but I don't really want to throw an exception since the file logger isn't even the main purpose of this calss.
// Let's just replace it with something to get it to be quiet.
_fileLogger = new NullLogger();
}
}