/// <summary>
/// Descarg los blogs
/// </summary>
private async System.Threading.Tasks.Task DownloadProcessAsync(bool includeDisabled, BlogsModelCollection blogs = null)
{
FeedProcessor processor = new FeedProcessor();
EntriesModelCollection entriesForDownload = new EntriesModelCollection();
// Lanza el evento de inicio
RaiseEvent(EventArguments.DownloadEventArgs.ActionType.StartDownload, "Start download process");
// Crea la colección de blogs si estaba vacía
if (blogs == null)
{
blogs = _blogManager.File.GetBlogsRecursive();
}
// Descarga los blogs
foreach (BlogModel blog in blogs)
{
if (blog.Enabled || (includeDisabled && !blog.Enabled))
{
AtomChannel atom;
// Lanza el evento
RaiseEvent(EventArguments.DownloadEventArgs.ActionType.StartDownloadBlog, $"Start download {blog.Name}");
// Descarga el archivo
try
{
// Descarga el archivo Atom / Rss
atom = await processor.DownloadAsync(blog.URL);
// Añade los mensajes
if (atom != null)
{
EntriesModelCollection downloaded = AddMessages(blog, atom);
if (downloaded.Count > 0 && blog.DownloadPodcast)
{
entriesForDownload.AddRange(downloaded);
}
}
// Modifica la fecha de última descarga
blog.DateLastDownload = DateTime.Now;
// Indica que en las entradas del blog se han hecho modificaciones (para el recálculo de elementos leídos)
blog.IsDirty = true;
// Lanza el evento
RaiseEvent(EventArguments.DownloadEventArgs.ActionType.EndDownloadBlog, $"End download {blog.Name}");
}
catch (Exception exception)
{
RaiseEvent(EventArguments.DownloadEventArgs.ActionType.ErrorDonwloadBlog, $"Error when download {blog.Name}. {exception.Message}");
}
}
}
// Graba los blogs
_blogManager.Save();
// Descarga los adjuntos
if (entriesForDownload.Count > 0)
{
await DownloadAttachmentsAsync(entriesForDownload);
}
// Lanza el evento de fin
RaiseEvent(EventArguments.DownloadEventArgs.ActionType.EndDownload, "End download blogs");
}
public PartitionSupervisorCore(DocumentServiceLease lease, ChangeFeedObserver <T> observer, FeedProcessor processor, LeaseRenewer renewer)
{
this.lease = lease;
this.observer = observer;
this.processor = processor;
this.renewer = renewer;
}
public void PerformanceTestForStructureProcessing()
{
var repositoryStub = new RepositoryStub();
var interestRate = new Mock <InterestRates>();
interestRate.Setup(ir => ir.PositiveInterestRate()).Returns(2.0);
interestRate.Setup(ir => ir.NegativeInterestRate()).Returns(3.0);
var feedProcessor = new FeedProcessor(repositoryStub, "../../../Pickup/testdata/feed.csv");
feedProcessor.Process();
var paymentInstructionService = new PaymentInstructionService();
var structures = repositoryStub.Accounts.GroupBy(a => a.GetClientId())
.Select(g => new
{
ClientAccount = new ClientAccounts(g),
Allocation = GetAllocation(g.ToList())
})
.Select(cs => new Structure(cs.ClientAccount, cs.Allocation, interestRate.Object, paymentInstructionService));
var structureList = structures.ToList();
var start = DateTime.Now.Ticks;
structureList.ForEach(s => s.GeneratePaymentInstruction());
var stop = DateTime.Now.Ticks;
Console.WriteLine((stop - start) / 10000);
}
public async Task AddLease_ShouldIgnorePartitionObserving_IfDuplicateLease()
{
await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false);
FeedProcessor processorDuplicate = MockPartitionProcessor();
Mock.Get(this.partitionSupervisorFactory)
.Setup(f => f.Create(this.lease))
.Returns(new PartitionSupervisorCore(this.lease, this.observer, processorDuplicate, this.leaseRenewer));
await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false);
Mock.Get(this.leaseManager)
.Verify(manager => manager.AcquireAsync(this.lease), Times.Once);
Mock.Get(this.leaseManager)
.Verify(manager => manager.UpdatePropertiesAsync(this.lease), Times.Once);
Mock.Get(this.leaseManager)
.Verify(manager => manager.ReleaseAsync(It.IsAny <DocumentServiceLease>()), Times.Never);
Mock.Get(this.partitionProcessor)
.Verify(p => p.RunAsync(It.IsAny <CancellationToken>()), Times.Once);
Mock.Get(processorDuplicate)
.Verify(p => p.RunAsync(It.IsAny <CancellationToken>()), Times.Never);
}
public PartitionControllerTests()
{
this.lease = Mock.Of <DocumentServiceLease>();
Mock.Get(this.lease)
.Setup(l => l.CurrentLeaseToken)
.Returns("partitionId");
this.partitionProcessor = MockPartitionProcessor();
this.leaseRenewer = MockRenewer();
this.observer = Mock.Of <ChangeFeedObserver>();
this.partitionSupervisorFactory = Mock.Of <PartitionSupervisorFactory>(f => f.Create(this.lease) == new PartitionSupervisorCore(this.lease, this.observer, this.partitionProcessor, this.leaseRenewer));
this.leaseManager = Mock.Of <DocumentServiceLeaseManager>();
Mock.Get(this.leaseManager).Reset(); // Reset implicit/by default setup of properties.
Mock.Get(this.leaseManager)
.Setup(manager => manager.AcquireAsync(this.lease))
.ReturnsAsync(this.lease);
Mock.Get(this.leaseManager)
.Setup(manager => manager.ReleaseAsync(this.lease))
.Returns(Task.CompletedTask);
DocumentServiceLeaseContainer leaseContainer = Mock.Of <DocumentServiceLeaseContainer>();
this.synchronizer = Mock.Of <PartitionSynchronizer>();
this.sut = new PartitionControllerCore(leaseContainer, this.leaseManager, this.partitionSupervisorFactory, this.synchronizer);
}
public void ShouldBeAbleToProcessTheFeed()
{
var account = new Account(new AccountId(87654123), new ClientId("ABC823"))
{
Balance = 4000, LastUpdatedDate = DateTime.Now
};
var mockRepository = new Mock <IRepository>();
var feedProcessor = new FeedProcessor(mockRepository.Object, "../../../Pickup/feed.csv");
mockRepository.Setup(repo => repo.Save(account));
feedProcessor.Process();
mockRepository.Verify(repo => repo.Save(account));
}
public override PartitionSupervisor Create(DocumentServiceLease lease)
{
if (lease == null)
{
throw new ArgumentNullException(nameof(lease));
}
ChangeFeedObserver changeFeedObserver = this.observerFactory.CreateObserver();
FeedProcessor processor = this.partitionProcessorFactory.Create(lease, changeFeedObserver);
LeaseRenewerCore renewer = new LeaseRenewerCore(lease, this.leaseManager, this.changeFeedLeaseOptions.LeaseRenewInterval);
return(new PartitionSupervisorCore(lease, changeFeedObserver, processor, renewer));
}
public PartitionSupervisorTests()
{
lease = Mock.Of <DocumentServiceLease>();
Mock.Get(lease)
.Setup(l => l.CurrentLeaseToken)
.Returns("partitionId");
leaseRenewer = Mock.Of <LeaseRenewer>();
partitionProcessor = Mock.Of <FeedProcessor>();
observer = Mock.Of <ChangeFeedObserver <dynamic> >();
sut = new PartitionSupervisorCore <dynamic>(lease, observer, partitionProcessor, leaseRenewer);
}
public PartitionSupervisorTests()
{
this.lease = Mock.Of <DocumentServiceLease>();
Mock.Get(this.lease)
.Setup(l => l.CurrentLeaseToken)
.Returns("partitionId");
this.leaseRenewer = Mock.Of <LeaseRenewer>();
this.partitionProcessor = Mock.Of <FeedProcessor>();
this.observer = Mock.Of <ChangeFeedObserver>();
this.sut = new PartitionSupervisorCore(this.lease, this.observer, this.partitionProcessor, this.leaseRenewer);
}
public static async Task ProcessAsync()
{
Console.WriteLine("Loading nuget package meta data...");
var fProcessor = new FeedProcessor();
var packageData = await fProcessor.ProcessAsync();
Console.WriteLine("Processing packages...");
var results = new List <ProcessResult>();
Parallel.ForEach <PackageData>(packageData, package =>
{
Console.WriteLine($"Processing: {package.Title}");
using (var aProc = new PackageProcessor(package))
{
var result = aProc.Process();
results.Add(result);
}
});
//
Console.WriteLine("Saving results to database...");
using (var aDb = new AuditorDbContext())
{
//empty the old results
aDb.Results.RemoveRange(aDb.Results);
aDb.Results.AddRange(results);
await aDb.SaveChangesAsync();
}
//check for failed packages, that haven't been signed but only after the cutoff date when signing was required
var failedSignCheck = results.Where(x => x.IsSigned.Equals(false) &&
x.DatePublished >= SettingsHelper.CutOffDateTime);
if (failedSignCheck.Any())
{
var packageIds = failedSignCheck.Select(x => x.PackageId);
SlackNotifier.Notify(packageIds.ToList());
}
Console.WriteLine($"Processed: {results.Count} Packages");
Console.ReadLine();
}
public void SetUp()
{
_options = new EasyMwsOptions();
_dbContext = new EasyMwsContext();
_actualFeedSubmissionReportContent = null;
_loggerMock = new Mock <IEasyMwsLogger>();
_mwsClientMock = new Mock <IMarketplaceWebServiceClient>();
var reportProcessor = new Mock <IReportQueueingProcessor>();
var feedProcessor = new FeedProcessor(_region, _merchantId, _mwsAuthToken, _options, _mwsClientMock.Object, _loggerMock.Object);
_easyMwsClient = new EasyMwsClient(AmazonRegion.Europe, _merchantId, _mwsAuthToken, "test", "test", reportProcessor.Object,
feedProcessor, _loggerMock.Object, _options);
}
public void SetUp()
{
var options = new EasyMwsOptions();
_feedSubmissionServiceMock = new Mock <IFeedSubmissionEntryService>();
_marketplaceWebServiceClientMock = new Mock <IMarketplaceWebServiceClient>();
_feedSubmissionProcessorMock = new Mock <IFeedSubmissionProcessor>();
_callbackActivatorMock = new Mock <ICallbackActivator>();
_loggerMock = new Mock <IEasyMwsLogger>();
_callbackActivatorMock.Setup(cam => cam.SerializeCallback(It.IsAny <Action <Stream, object> >(), It.IsAny <object>()))
.Returns(new Callback("", "", "", ""));
_feedProcessor = new FeedProcessor(_amazonRegion, _merchantId, _mwsAuthToken, options, _marketplaceWebServiceClientMock.Object,
_feedSubmissionProcessorMock.Object, _callbackActivatorMock.Object, _loggerMock.Object);
}
public async Task AddLease_ShouldRunObserver_IfSecondAdded()
{
DocumentServiceLease lease2 = Mock.Of <DocumentServiceLease>();
Mock.Get(lease2)
.Setup(l => l.CurrentLeaseToken)
.Returns("partitionId2");
FeedProcessor partitionProcessor2 = MockPartitionProcessor();
Mock.Get(this.partitionSupervisorFactory)
.Setup(f => f.Create(lease2))
.Returns(new PartitionSupervisorCore(lease2, this.observer, partitionProcessor2, this.leaseRenewer));
await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false);
await this.sut.AddOrUpdateLeaseAsync(lease2).ConfigureAwait(false);
Mock.Get(partitionProcessor2)
.Verify(p => p.RunAsync(It.IsAny <CancellationToken>()), Times.Once);
}
public async Task ExtractAndPopulateQueueAsync_SelfPage_TestAsync()
{
// Arrange
var dummyLastReadBookmarkId = Guid.NewGuid();
var dummyFeedPage = new Models.FeedPage
{
Entries = new[]
{
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = dummyLastReadBookmarkId
},
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
}
};
SetupMocks(dummyLastReadBookmarkId, dummyFeedPage);
Mock.Get(_mockFeedProcessorConfiguration)
.Setup(c => c.SetLastReadPage(It.IsAny <int>()))
.Returns(Task.CompletedTask);
// Act
var feedProcessor = new FeedProcessor(_mockFeedReader, _mockQueuePopulator, _mockFeedProcessorConfiguration, _mockLogger);
await feedProcessor.ExtractAndPopulateQueueAsync(_mockCollector);
// Assert
Mock.Get(_mockFeedProcessorConfiguration).VerifyAll();
Mock.Get(_mockFeedReader).VerifyAll();
Mock.Get(_mockQueuePopulator)
.Verify(q => q.PopulateSessionQueue(It.IsAny <IAsyncCollector <Message> >(), It.Is <IEnumerable <FeedEntry> >(m => m.Count() == 2)));
Mock.Get(_mockLogger)
.Verify(l => l.Log(LogLevel.Information, 0, It.IsAny <It.IsAnyType>(), null, It.IsAny <Func <It.IsAnyType, Exception, string> >()), Times.Exactly(2));
}
public void ExtractAndPopulateQueueAsync_ReadArchives_ThrowsException_TestAsync()
{
// Arrange
var dummyLastReadBookmarkId = Guid.NewGuid();
var dummyFeedPage = new Models.FeedPage
{
Entries = new[]
{
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
}
};
SetupMocks(dummyLastReadBookmarkId, dummyFeedPage);
Mock.Get(_mockFeedReader)
.Setup(r => r.ReadPageAsync(It.IsAny <int>()))
.ReturnsAsync(dummyFeedPage);
// Act
var feedProcessor = new FeedProcessor(_mockFeedReader, _mockQueuePopulator, _mockFeedProcessorConfiguration, _mockLogger);
Func <Task> act = async() => await feedProcessor.ExtractAndPopulateQueueAsync(_mockCollector);
// Assert
act.Should().Throw <InvalidOperationException>();
Mock.Get(_mockFeedProcessorConfiguration).VerifyAll();
Mock.Get(_mockFeedReader).VerifyAll();
Mock.Get(_mockQueuePopulator).VerifyNoOtherCalls();
Mock.Get(_mockLogger)
.Verify(l => l.Log(LogLevel.Information, 0, It.IsAny <It.IsAnyType>(), null, It.IsAny <Func <It.IsAnyType, Exception, string> >()), Times.Once);
}
public async Task ExtractAndPopulateQueueAsync_ReadArchives_TestAsync()
{
// Arrange
var dummyLastReadBookmarkId = Guid.NewGuid();
var dummySelfFeedPage = new Models.FeedPage
{
IsSelfPage = true,
Entries = new[]
{
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
},
PreviousPageNumber = 10
};
var missedPreviousSelfFeedPage = new Models.FeedPage
{
IsSelfPage = true,
Entries = new[]
{
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
},
PreviousPageNumber = 9
};
var dummyFeedPage = new Models.FeedPage
{
Entries = new[]
{
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = dummyLastReadBookmarkId
},
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
},
NextPageNumber = 12
};
var lastDummyFeedPage = new Models.FeedPage
{
Entries = new[]
{
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
new FeedEntry {
Id = Guid.NewGuid()
},
},
};
SetupMocks(dummyLastReadBookmarkId, dummySelfFeedPage);
Mock.Get(_mockFeedReader)
.Setup(r => r.ReadPageAsync(10))
.ReturnsAsync(dummyFeedPage);
Mock.Get(_mockFeedReader)
.Setup(r => r.ReadPageAsync(12))
.ReturnsAsync(missedPreviousSelfFeedPage);
Mock.Get(_mockFeedReader)
.Setup(r => r.ReadPageAsync(11))
.ReturnsAsync(lastDummyFeedPage);
Mock.Get(_mockFeedProcessorConfiguration)
.Setup(c => c.SetLastReadPage(It.IsAny <int>()))
.Returns(Task.CompletedTask);
// Act
var feedProcessor = new FeedProcessor(_mockFeedReader, _mockQueuePopulator, _mockFeedProcessorConfiguration, _mockLogger);
await feedProcessor.ExtractAndPopulateQueueAsync(_mockCollector);
// Assert
Mock.Get(_mockFeedProcessorConfiguration).VerifyAll();
Mock.Get(_mockFeedReader).VerifyAll();
Mock.Get(_mockQueuePopulator)
.Verify(q => q.PopulateSessionQueue(It.IsAny <IAsyncCollector <Message> >(), It.IsAny <IEnumerable <FeedEntry> >()), Times.Exactly(3));
Mock.Get(_mockLogger)
.Verify(l => l.Log(LogLevel.Information, 0, It.IsAny <It.IsAnyType>(), null, It.IsAny <Func <It.IsAnyType, Exception, string> >()), Times.Exactly(4));
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseWorker" /> class.
/// </summary>
/// <param name="user">The user.</param>
/// <param name="password">The password.</param>
/// <param name="database">The database.</param>
/// <param name="server">The server.</param>
/// <param name="gpsToken">The GPS token.</param>
/// <param name="statusToken">The status token.</param>
/// <param name="faultToken">The fault token.</param>
/// <param name="tripToken">The trip token.</param>
/// <param name="exceptionToken">The exception token.</param>
/// <param name="path">The path.</param>
public DatabaseWorker(string user, string password, string database, string server, long?gpsToken, long?statusToken, long?faultToken, long?tripToken, long?exceptionToken, string path)
: base(path)
{
feedParameters = new FeedParameters(gpsToken, statusToken, faultToken, tripToken, exceptionToken);
feedService = new FeedProcessor(server, database, user, password);
}
public void GetAllRacesOrderByPriceTest()
{
var feedMock1 = new Mock <IFeed>();
feedMock1.Setup(f => f.GetRaceTrackWithAllRaces()).Returns(
new RaceTrack
{
Name = "Caulfield",
Races = new List <Race>
{
new Race
{
Name = "race 1",
Horses = new List <Horse>
{
new Horse {
Name = "2", Price = 2
}, new Horse {
Name = "1", Price = 1
}
}
}
}
});
var feedMock2 = new Mock <IFeed>();
feedMock2.Setup(f => f.GetRaceTrackWithAllRaces()).Returns(
new RaceTrack
{
Name = "Wolferhampton",
Races = new List <Race>
{
new Race
{
Name = "race 3",
Horses = new List <Horse>
{
new Horse {
Name = "4", Price = 4
}, new Horse {
Name = "3", Price = 3
}
}
}
}
});
var processor = new FeedProcessor(new List <IFeed> {
feedMock1.Object, feedMock2.Object
});
var result = processor.GetAllRacesOrderByPrice();
var caulHorses = result.Where(t => t.Name == "Caulfield").SelectMany(t => t.Races[0].Horses).ToList();
var wolfHorses = result.Where(t => t.Name == "Wolferhampton").SelectMany(t => t.Races[0].Horses).ToList();
Assert.Equal(2, caulHorses.Count);
Assert.Equal("1", caulHorses[0].Name);
Assert.Equal("2", caulHorses[1].Name);
Assert.Equal(2, wolfHorses.Count);
Assert.Equal("3", wolfHorses[0].Name);
Assert.Equal("4", wolfHorses[1].Name);
}
public static async Task ProcessAsync()
{
try
{
LogHelper.WriteLine("Loading nuget package meta data...");
var fProcessor = new FeedProcessor();
var packageData = await fProcessor.ProcessQueryAsync();
var searchData = await fProcessor.ProcessSearchAsync();
LogHelper.WriteLine("Processing packages...");
var results = new List <ProcessResult>();
var options = new ParallelOptions()
{
MaxDegreeOfParallelism = 2
};
Parallel.ForEach <PackageData>(packageData, options, package =>
{
LogHelper.WriteLine($"Processing: {package.Title}");
var sPackage = searchData.FirstOrDefault(x => x.PackageRegistration.Id.Equals(package.PackageId));
using (var aProc = new PackageProcessor(package, sPackage))
{
var result = aProc.Process();
results.Add(result);
}
});
//
LogHelper.WriteLine("Saving results to database...");
using (var aDb = new AuditorDbContext())
{
//empty the old results
aDb.Results.RemoveRange(aDb.Results);
await aDb.SaveChangesAsync();
aDb.Results.AddRange(results);
await aDb.SaveChangesAsync();
}
//check for failed packages, that haven't been signed but only after the cutoff date when signing was required
var failedSignCheck = results.Where(x => x.IsSigned.Equals(false) &&
x.DatePublished >= SettingsHelper.CutOffDateTime);
if (failedSignCheck.Any())
{
var packageIds = failedSignCheck.Select(x => x.PackageId);
SlackNotifier.Notify(packageIds.ToList());
}
LogHelper.WriteLine($"Processed: {results.Count} Packages");
//Console.ReadLine();
}
catch (Exception ex)
{
var msg = ex.Message;
if (ex.InnerException != null)
{
msg += Environment.NewLine + Environment.NewLine + ex.InnerException.Message;
}
LogHelper.WriteLine(msg);
}
}
private void StartService()
{
try
{
m_NLog = LogManager.GetLogger("AppLogger");
m_NLog.Info("Starting Notification Provider.");
m_NotificationRequests = new BlockingCollection <string>();
m_NotificationMessages = new BlockingCollection <NotificationMessage>();
m_NotificationMessagesLog = new BlockingCollection <NotificationMessage>();
m_NotificationRemoveRequests = new BlockingCollection <string>();
m_FeedMessagesLog = new BlockingCollection <FeedMessage[]>();
m_dsMonitor = new dsMonitor();
//Start NotificationEngine proxy handler.
m_NotificaionEngineProxy = new NotificaionEngineProxy(m_NotificationRequests, m_NotificationMessages,
m_NotificationMessagesLog, m_NotificationRemoveRequests);
m_NotificaionEngineProxy.Initialize();
m_NotificaionEngineProxy.ConnectToNotificationEngine(); // subscribe to Notificaion Engine.
#region Sessions Management
m_SessionManager = new SessionsManager(m_NotificationRequests, m_NotificationRemoveRequests, m_FeedMessagesLog, m_dsMonitor);
#endregion
#region Initialize Router
m_Router = new Router(m_SessionManager);
#endregion
#region Start Process Received Feed
m_FeedProcessor = new FeedProcessor(m_NotificationMessages, m_Router);
m_FeedProcessor.StartHanldeReceivedFeed();
#endregion
#region Logs
if (bool.Parse(System.Configuration.ConfigurationManager.AppSettings["LogNotificationMsgs"]))
{
m_NotificaionLogHandler = new NotificationLogHandler(m_NotificationMessagesLog, m_FeedMessagesLog, m_dsMonitor);;
m_NotificaionLogHandler.StartLogging();
}
#endregion
#region test
// Test
// m_NotificationRequests.Add("Notify_User#1248");
//Task.Factory.StartNew(() =>
//{
// TestFeed();
//});
#endregion
#region Start Wcf Service
try
{
m_FeedService = new FeedService(m_SessionManager);
serviceHost = new ServiceHost(m_FeedService);
serviceHost.Open();
Console.BackgroundColor = ConsoleColor.Yellow;
m_NLog.Info("Notification Provider Service started and ready to handle dataAdapters requests.");
Console.ResetColor();
}
catch (Exception exp)
{
m_NLog.Error("Error in Start Notification Provider Service(). Error Details : {0}", exp.ToString());
}
#endregion
}
catch (Exception exp)
{
m_NLog.Error("Error in Service Startup. Error Details : ", exp.ToString());
}
}
