private async Task NotifyComponent(Guid jobId, string component, object notification)
{
var dataSource = await _componentRegistry.GetComponentByIdAsync(component);
if (dataSource == null)
{
var errorMessage =
$"Data acquisition component '{component}' was not registered";
_logger.TrackError(
"NotifyComponent",
errorMessage);
}
else
{
var metrics = $"Config pushed to: {component}";
_logger.TrackInfo(
"NotifyComponent",
metrics,
new
{
jobId,
config = notification
});
await _componentConfigUpdateNotifier.NotifyComponentAsync(
dataSource.UpdateChannelName,
notification);
}
}
public Task StartNewJobAsync(DataAcquirerJobConfig jobConfig)
{
lock (_dictionaryLock)
{
var jobId = jobConfig.JobId;
if (_isStopping)
{
_logger.TrackWarning(
"StartNewJob",
"Could not start job, because the component is stopping",
new { jobId = jobId });
return(Task.CompletedTask);
}
if (_runningJobsRecords.ContainsKey(jobId))
{
_logger.TrackWarning(
"StartNewJob",
"Job is with this id already running",
new { jobId = jobId });
return(Task.CompletedTask);
}
_logger.TrackInfo(
"StartNewJob",
"Config recieved",
new { config = jobConfig });
var cancellationTokenSource = new CancellationTokenSource();
var downloadingTask = RunJobAsync(jobConfig, cancellationTokenSource.Token).
ContinueWith(async r =>
{
try
{
await r;
}
catch (TaskCanceledException) { }
_runningJobsRecords.Remove(jobId, out _);
_logger.TrackInfo(
"StartNewJob",
"Job removed",
new { jobId = jobId.ToString() });
});
var jobManagerJobRecord = new JobManagerJobRecord
{
JobId = jobConfig.JobId,
JobTask = downloadingTask,
CancellationTokenSource = cancellationTokenSource
};
_runningJobsRecords.TryAdd(jobManagerJobRecord.JobId, jobManagerJobRecord);
return(Task.CompletedTask);
}
}
public ActionResult <string> HeartBeat()
{
_eventTracker.TrackInfo("HeartBeat", "Heart beat requested");
var status = new
{
TimeStamp = DateTime.Now.ToString("s")
};
return(Ok(status));
}
public async Task <ActionResult <LoginResponse> > Login([FromBody] LoginRequest login)
{
_eventTracker.TrackInfo("Login", "A user tried to login", new { User = login.Username });
var authenticatedUser = await _userService.Authenticate(login.Username, login.Password);
if (authenticatedUser == null)
{
_eventTracker.TrackInfo("Login", "Wrong credentials when trying to login");
return(Unauthorized(new { message = "Username or password is incorrect" }));
}
var loginResponse = LoginResponse.FromModel(authenticatedUser);
return(Ok(loginResponse));
}
public async Task <ActionResult <List <ChartDefinitionDto> > > GetJobCharts([FromRoute] Guid jobId)
{
if (!await _authorizationService.IsUserAuthorizedToSeeJob(User.Identity.Name, jobId))
{
_eventTracker.TrackInfo("GetJobCharts", $"User '{User.Identity.Name}' is unauthorized to see job '{jobId}'");
return(Unauthorized());
}
var jobCharts = await _chartsService.GetJobCharts(jobId);
var mappedChartDefinitions = jobCharts
.Select(ChartDefinitionDto.FromModel)
.ToList();
return(Ok(mappedChartDefinitions));
}
public async Task <string> GetString(string path)
{
var fullPath = GetFullPath(path);
_eventTracker.TrackInfo(EventTrackerEventName, $"GET /{path}");
try
{
var response = await _client.GetAsync(fullPath);
response.EnsureSuccessStatusCode();
return(await response.Content.ReadAsStringAsync());
}
catch (HttpRequestException e)
{
throw WrapHttpException(e);
}
}
public Task ListenAsync(CancellationToken token)
{
_logger.TrackInfo("StartNewJob", "Starting listening",
new { channelName = _updateChannelName });
while (true)
{
return(_messageBrokerConsumer.ConsumeAsync(
_updateChannelName,
ProcessJobConfigAsync,
token));
}
}
public async Task <ActionResult <List <JobDto> > > GetJobs()
{
var username = User.Identity.Name;
if (username == null)
{
_eventTracker.TrackInfo("GetJobs", "Unauthorized attempt to list all jobs");
return(Unauthorized());
}
var jobStatuses = await _jobService.GetJobsDetails(username);
var mappedJobStatuses = jobStatuses
.Select(JobDto.FromModel)
.ToList();
return(Ok(mappedJobStatuses));
}
public async IAsyncEnumerable <DataAcquirerPost> CreateBatchPostEnumerator(
ITwitterContext twitterContext,
TwitterMetadata defaultMetadata)
{
var includeRetweets = false;
var metadata = await _metadataContext.GetOrCreateAsync(defaultMetadata);
_logger.TrackInfo("QueryData", "Twitter - using metadata",
new
{
metadata,
jobId = _jobId
});
var query = defaultMetadata.Query;
while (true)
{
var twitterInputModel = new TwitterQueryInput(
query,
metadata.Language,
metadata.MaxId,
metadata.SinceId,
metadata.BatchSize);
var batches = QueryPastPostAsync(
twitterContext,
twitterInputModel);
await foreach (var batch in batches)
{
if (!batch.Any())
{
break;
}
foreach (var post in batch)
{
metadata.MaxId = Math.Min(post.StatusID - 1, metadata.MaxId);
metadata.SinceId = Math.Max(post.StatusID, metadata.SinceId);
if (post.CreatedAt > _mostRecentPost)
{
_mostRecentPost = post.CreatedAt;
}
var isRetweet = post.RetweetedStatus?.StatusID > 0;
if (isRetweet && !includeRetweets)
{
continue;
}
yield return(FromStatus(post, query));
}
_foundPosts += batch.Count;
_logger.TrackStatistics("QueryDataStatistics",
new
{
_foundPosts,
_mostRecentPost,
query = metadata.Query,
language = metadata.Language,
jobId = _jobId
});
await _metadataContext.UpdateAsync(metadata);
}
metadata.MaxId = ulong.MaxValue;
await _metadataContext.UpdateAsync(metadata);
_logger.TrackInfo("QueryData", $"Waiting {_options.NoPostWaitDelay} for new posts");
await Task.Delay(_options.NoPostWaitDelay);
}
}