private void ProcessOneBook(Book book) { try { _logger.TrackEvent("ProcessOneBook Start"); string message = $"Processing: {book.BaseUrl}"; Console.Out.WriteLine(message); _logger.LogVerbose(message); var initialUpdates = new UpdateOperation(); initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString()); initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier); var startTime = new Parse.Model.Date(DateTime.UtcNow); initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson()); _parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson()); // Process the book var finalUpdates = new UpdateOperation(); var warnings = FindBookWarnings(book); finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings)); // ENHANCE: Do more processing here // Write the updates finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString()); _parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson()); _logger.TrackEvent("ProcessOneBook End - Success"); } catch (Exception e) { YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\""); // Attempt to write to Parse that processing failed if (!String.IsNullOrEmpty(book?.ObjectId)) { try { var onErrorUpdates = new UpdateOperation(); onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\""); onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier); _parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson()); } catch (Exception) { // If it fails, just let it be and throw the first exception rather than the nested exception. } } throw; } }
public static void Main(string[] args) { if (Environment.OSVersion.Platform != PlatformID.Win32NT) { // See https://issues.bloomlibrary.org/youtrack/issue/BL-8475. Console.WriteLine("Harvester cannot run on Linux until we verify that phash computations always yield the same result on both Windows and Linux!"); return; } // See https://github.com/commandlineparser/commandline for documentation about CommandLine.Parser var parser = new CommandLine.Parser((settings) => { settings.CaseInsensitiveEnumValues = true; settings.CaseSensitive = false; settings.HelpWriter = Console.Error; }); try { parser.ParseArguments <HarvesterOptions, UpdateStateInParseOptions, BatchUpdateStateInParseOptions, GenerateProcessedFilesTSVOptions>(args) .WithParsed <HarvesterOptions>(options => { options.ValidateOptions(); Harvester.RunHarvest(options); }) .WithParsed <UpdateStateInParseOptions>(options => { HarvestStateUpdater.UpdateState(options.ParseDBEnvironment, options.ObjectId, options.NewState); }) .WithParsed <BatchUpdateStateInParseOptions>(options => { HarvestStateBatchUpdater.RunBatchUpdateStates(options); }) .WithParsed <GenerateProcessedFilesTSVOptions>(options => { ProcessedFilesInfoGenerator.GenerateTSV(options); }) .WithNotParsed(errors => { Console.Out.WriteLine("Error parsing command line arguments."); Environment.Exit(1); }); } catch (Exception e) { YouTrackIssueConnector.GetInstance(EnvironmentSetting.Unknown).ReportException(e, "An exception was thrown which was not handled by the program.", null); throw; } }
public void TestIssueSubmission() { var issueId = YouTrackIssueConnector.SubmitToYouTrack("Harvester Test Issue", "This is a test from Harvester, which apparently cannot be overemphasized.", "AUT"); Assert.That(issueId, Is.Not.Null); Assert.That(issueId, Does.StartWith("AUT-")); // Creating a new submitter seems a bit wasteful, and a bit too far into implementation details, // but it's the only way to delete the newly created issue. var submitter = new Bloom.YouTrackIssueSubmitter("AUT"); var deleted = submitter.DeleteIssue(issueId); Assert.That(deleted, Is.True); }
// Command line arguments sample: "harvestAll --environment=dev --parseDBEnvironment=prod" // // Some presets that you might copy and paste in: // harvestAll --environment=dev --parseDBEnvironment=local --suppressLogs --count=2 // harvestWarnings --environment=dev --parseDBEnvironment=local --suppressLogs // harvestAll --environment=dev --parseDBEnvironment=local --suppressLogs "--queryWhere={ \"objectId\":\"38WdeYJ0yF\"}" // harvestAll --environment=dev --parseDBEnvironment=dev --suppressLogs "--queryWhere={ \"objectId\":\"JUCL9OMOza\"}" // harvestAll --environment=dev --parseDBEnvironment=dev --suppressLogs "--queryWhere={ \"title\":{\"$in\":[\"Vaccinations\",\"Fox and Frog\",\"The Moon and the Cap\"]}}" public static void Main(string[] args) { // See https://github.com/commandlineparser/commandline for documentation about CommandLine.Parser var parser = new CommandLine.Parser((settings) => { settings.CaseInsensitiveEnumValues = true; settings.CaseSensitive = false; settings.HelpWriter = Console.Error; }); try { parser.ParseArguments <HarvestAllOptions, HarvestHighPriorityOptions, HarvestLowPriorityOptions, HarvestWarningsOptions>(args) .WithParsed <HarvestAllOptions>(options => { Harvester.RunHarvestAll(options); }) .WithParsed <HarvestWarningsOptions>(options => { Harvester.RunHarvestWarnings(options); }) // TODO: Replace placeholders .WithParsed <HarvestHighPriorityOptions>(options => { throw new NotImplementedException("HarvestHighPriority"); }) .WithParsed <HarvestLowPriorityOptions>(options => { throw new NotImplementedException("HarvestLowPriority"); }) .WithNotParsed(errors => { Console.Out.WriteLine("Error parsing command line arguments."); Environment.Exit(1); }); } catch (Exception e) { YouTrackIssueConnector.SubmitToYouTrack(e, "An exception was thrown which was not handled by the program."); throw; } }
public void InitializeYouTrackConnector() { _connector = YouTrackIssueConnector.GetInstance(EnvironmentSetting.Test, "SB"); }
private void ProcessOneBook(Book book) { try { _logger.TrackEvent("ProcessOneBook Start"); string message = $"Processing: {book.BaseUrl}"; Console.Out.WriteLine(message); _logger.LogVerbose(message); var initialUpdates = new UpdateOperation(); initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString()); initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier); var startTime = new Parse.Model.Date(DateTime.UtcNow); initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson()); _parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson()); // Download the book _logger.TrackEvent("Download Book"); string decodedUrl = HttpUtility.UrlDecode(book.BaseUrl); string urlWithoutTitle = RemoveBookTitleFromBaseUrl(decodedUrl); string downloadRootDir = Path.Combine(Path.GetTempPath(), Path.Combine("BloomHarvester", this.Identifier)); _logger.LogVerbose("Download Dir: {0}", downloadRootDir); string downloadBookDir = _transfer.HandleDownloadWithoutProgress(urlWithoutTitle, downloadRootDir); // Process the book var finalUpdates = new UpdateOperation(); var warnings = FindBookWarnings(book); finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings)); // ENHANCE: Do more processing here _logger.TrackEvent("Upload Book"); UploadBook(decodedUrl, downloadBookDir); // Write the updates finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString()); _parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson()); _logger.TrackEvent("ProcessOneBook End - Success"); } catch (Exception e) { YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\""); // Attempt to write to Parse that processing failed if (!String.IsNullOrEmpty(book?.ObjectId)) { try { var onErrorUpdates = new UpdateOperation(); onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\""); onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier); _parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson()); } catch (Exception) { // If it fails, just let it be and throw the first exception rather than the nested exception. } } throw; } }
public AzureMonitorLogger(EnvironmentSetting environment, string harvesterId) { _issueReporter = YouTrackIssueConnector.GetInstance(environment); // Get the Instrumentation Key for Azure from an environment variable. string environmentVarName = "BloomHarvesterAzureAppInsightsKeyDev"; if (environment == EnvironmentSetting.Test) { environmentVarName = "BloomHarvesterAzureAppInsightsKeyTest"; } else if (environment == EnvironmentSetting.Prod) { environmentVarName = "BloomHarvesterAzureAppInsightsKeyProd"; } string instrumentationKey = Environment.GetEnvironmentVariable(environmentVarName); Debug.Assert(!String.IsNullOrWhiteSpace(instrumentationKey), "Azure Instrumentation Key is invalid. Azure logging probably won't work."); try { Microsoft.ApplicationInsights.Extensibility.TelemetryConfiguration.Active.InstrumentationKey = instrumentationKey; } catch (ArgumentNullException e) { _issueReporter.ReportException(e, $"InstrumentationKey: {instrumentationKey ?? "null"}.\nenvironmentVarName: {environmentVarName}", null); } _telemetry.Context.User.Id = "BloomHarvester " + harvesterId; _telemetry.Context.Session.Id = Guid.NewGuid().ToString(); _telemetry.Context.Device.OperatingSystem = Environment.OSVersion.ToString(); string logFilePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "BloomHarvester", "log.txt"); Console.Out.WriteLine("Creating log file at: " + logFilePath); try { if (File.Exists(logFilePath)) { // Check if the file is too big (~10 MB) if (new FileInfo(logFilePath).Length > 10000000) { var oldPath = logFilePath + "-OLD"; // Preserve one previous log file for debugging help, and start over with // an empty log file. // (The data is in Azure too anyway, but having it local may speed things up.) if (RobustFile.Exists(oldPath)) { RobustFile.Delete(oldPath); } RobustFile.Move(logFilePath, oldPath); } } } catch { // Doesn't matter if there are any errors } try { _fileLogger = new FileLogger(logFilePath); } catch { // That's unfortunate that creating the logger failed, but I don't really want to throw an exception since the file logger isn't even the main purpose of this calss. // Let's just replace it with something to get it to be quiet. _fileLogger = new NullLogger(); } }