private void PostProcess_Logs(DataFileHttpResult uploadResult)
{
var timestamp = DateTime.Now.ToString("yyyyMMddHHmmss");
if (uploadResult.Endpoint.LogEnabled ?? false)
{
// Create Log file
var destinationDir = uploadResult.Endpoint.LogFolderPath;
if (!Directory.Exists(destinationDir))
{
Directory.CreateDirectory(destinationDir);
}
var guid = Regex.Match(uploadResult.Content, "[0-9a-f]{32}", RegexOptions.IgnoreCase | RegexOptions.Multiline);
var logContent = uploadResult.Content;//guid.Success ? guid.Value : ;
var destinationPath = Path.Combine(destinationDir, $"Log_{timestamp}_{uploadResult.FileName}");
if (File.Exists(destinationPath))
{
File.Delete(destinationPath);
}
File.WriteAllText(destinationPath, logContent);
}
}
private void PostProcess_Processed(DataFileHttpResult uploadResult)
{
if (uploadResult.Endpoint.ProcessedBackupEnabled ?? false)
{
// Move file to processed folder
var destinationDir = uploadResult.Endpoint.ProcessedFolderPath;
if (!Directory.Exists(destinationDir))
{
Directory.CreateDirectory(destinationDir);
}
var destinationPath = Path.Combine(destinationDir, uploadResult.FileName);
if (File.Exists(destinationPath))
{
File.Delete(destinationPath);
}
File.Move(uploadResult.FilePath, destinationPath);
}
else
{
// Delete data file
File.Delete(uploadResult.FilePath);
}
}
private void ProcessDataSetStatus(DataFileHttpResult statusResult)
{
_log.Debug($"Will process log status {statusResult.FilePath}");
if (!Directory.Exists(_config.DataSetStatus_FolderPath))
{
Directory.CreateDirectory(_config.DataSetStatus_FolderPath);
}
if (!Directory.Exists(statusResult.Endpoint.DataSetStatus_FolderPath))
{
Directory.CreateDirectory(statusResult.Endpoint.DataSetStatus_FolderPath);
}
var statusFolderPath = statusResult.Endpoint.DataSetStatus_FolderPath;
var fileName = statusResult.FileName;
if (statusResult.StatusCode != System.Net.HttpStatusCode.OK)
{
if (!statusResult.Endpoint.DataConfig.DataSetStatusKeepNotOkResponses)
{
File.Delete(statusResult.FilePath);
_log.Info($"Delete log file. Data set status can not be queried. Http Status Code is '{statusResult.StatusCode}' for '{statusResult.FilePath}'");
}
else
{
File.Move(statusResult.FilePath, Path.Combine(statusFolderPath, fileName));
_log.Info($"Archive log file. Data set status can not be queried. Http Status Code is '{statusResult.StatusCode}' for '{statusResult.FilePath}'");
}
return;
}
XElement root = XElement.Parse(statusResult.Content);
var isCompleted = root.Elements("queuedCount").Any(el => (string)el.Value == "0");
var hasError = root.Elements("errorCount").Any(el => (string)el.Value != "0");
if (isCompleted)
{
if (hasError)
{
// Delete log file
File.Delete(statusResult.FilePath);
_log.Info($"Delete log file. Data set status reported error for '{statusResult.FilePath}'");
// Create new status file
var statusFilePath = Path.Combine(statusFolderPath, fileName);
File.WriteAllText(statusFilePath, statusResult.Content);
_log.Info($"Write data set status. Data set status reported error for '{statusFilePath}'");
_log.Fatal($"Data set has errors. Please inspect details '{statusFilePath}'");
}
else // Delete log file, no errors reported by Bb
{
_log.Info($"Delete log file. Data set status reported success with no errors for '{statusResult.FilePath}'");
File.Delete(statusResult.FilePath);
}
}
}
private void PostProcessUploadedFile(DataFileHttpResult uploadResult)
{
if (uploadResult.StatusCode != HttpStatusCode.OK)
{
return;
}
PostProcess_Processed(uploadResult);
PostProcess_Logs(uploadResult);
}
private DataFileHttpResult UploadFile(DataFile dataFile)
{
using (var client = new HttpClient())
{
var timeoutSecs = dataFile.Endpoint.DataConfig.UploadTimeoutInSecs ?? 0;
if (timeoutSecs > 100)
{
client.Timeout = TimeSpan.FromSeconds(timeoutSecs);
}
var username = dataFile.Endpoint.Username;
var password = dataFile.Endpoint.Password;
var url = dataFile.Endpoint.Url;
var uri = new Uri(url);
var request = new HttpRequestMessage(HttpMethod.Post, uri);
if (dataFile.Endpoint.DataConfig.UploadBinary)
{
request.Content = new ByteArrayContent(File.ReadAllBytes(dataFile.FilePath));
}
else
{
var utf8 = new UTF8Encoding(true);
var fileContent = File.ReadAllText(dataFile.FilePath, utf8);
request.Content = new StringContent(fileContent, utf8, "text/plain");
}
var authBasic = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", authBasic);
request.Headers.Add("Accept", "*/*");
var responseMessage = client.SendAsync(request).Result;
var result = new DataFileHttpResult
{
StatusCode = responseMessage.StatusCode,
Content = responseMessage.Content.ReadAsStringAsync().Result,
Url = uri,
Endpoint = dataFile.Endpoint,
FilePath = dataFile.FilePath,
FileName = dataFile.FileName
};
_log.Info($"Post data set http result [{result.StatusCode}] : {result.Url}");
return(result);
}
}
public DataFileHttpResult DryRunUpload(DataFile dataFile)
{
var url = dataFile.Endpoint.Url;
var uri = new Uri(url);
var result = new DataFileHttpResult
{
StatusCode = HttpStatusCode.OK,
Content = "Dry run upload content. No files uploaded to Blackboard",
Url = uri,
Endpoint = dataFile.Endpoint,
FilePath = dataFile.FilePath,
FileName = dataFile.FileName
};
_log.Info($"Dry run upload : {url}");
return(result);
}
private DataFileHttpResult GetDataSetStatus(DataFile dataFile)
{
if (!File.Exists(dataFile.FilePath))
{
return(null);
}
_log.Debug($"Will get log status {dataFile.FilePath}");
var logFileContent = File.ReadAllText(dataFile.FilePath, Encoding.UTF8);
var guid = Regex.Match(logFileContent, "[0-9a-f]{32}", RegexOptions.IgnoreCase | RegexOptions.Multiline);
if (!guid.Success)
{
return(null);
}
using (var client = new HttpClient())
{
var timeoutSecs = dataFile.Endpoint.DataConfig.DownloadTimeoutInSecs ?? 0;
if (timeoutSecs > 100)
{
client.Timeout = TimeSpan.FromSeconds(timeoutSecs);
}
var username = dataFile.Endpoint.Username;
var password = dataFile.Endpoint.Password;
var rootUrl = _config.DataSetStatusEndpointUrl;
if (String.IsNullOrWhiteSpace(rootUrl))
{
return(null);
}
if (!rootUrl.EndsWith("/"))
{
rootUrl += "/";
}
var url = rootUrl + guid.Value;
var uri = new Uri(url);
var request = new HttpRequestMessage(HttpMethod.Get, uri);
request.Headers.Add("Accept", "text/xml");
var authBasic = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", authBasic);
var responseMessage = client.SendAsync(request).Result;
var result = new DataFileHttpResult
{
StatusCode = responseMessage.StatusCode,
Content = responseMessage.Content.ReadAsStringAsync().Result,
Url = uri,
Endpoint = dataFile.Endpoint,
FilePath = dataFile.FilePath,
FileName = dataFile.FileName
};
_log.Info($"Get data set status http result [{result.StatusCode}] : {result.Url}");
return(result);
}
}
