public async Task ProcessAsync()
{
if (!_isInited)
{
await InitAsync();
}
List <string> blobs;
if (_allBlobsReprocessingRequired)
{
_log.WriteInfo("BlobProcessor.ProcessAsync", _instanceTag, $"All blobs will be reprocessed");
blobs = await _blobReader.GetBlobsForConversionAsync(null);
_allBlobsReprocessingRequired = false;
}
else
{
string lastBlob;
if (!string.IsNullOrWhiteSpace(_lastBlob))
{
lastBlob = _lastBlob;
_lastBlob = null;
}
else
{
lastBlob = await _blobSaver.GetLastSavedBlobAsync();
}
blobs = await _blobReader.GetBlobsForConversionAsync(lastBlob);
}
foreach (var blob in blobs)
{
try
{
_log.WriteInfo("BlobProcessor.ProcessAsync", _instanceTag, $"Processing {blob}");
_blobSaver.StartBlobProcessing();
_messageConverter.StartBlobProcessing((directory, messages) => _blobSaver.SaveToBlobAsync(messages, directory, blob));
await _blobReader.ReadAndProcessBlobAsync(blob);
await _messageConverter.FinishBlobProcessingAsync();
await _blobSaver.FinishBlobProcessingAsync(blob);
_log.WriteInfo("BlobProcessor.ProcessAsync", _instanceTag, $"Processed {blob}");
}
catch (Exception ex)
{
_log.WriteError("BlobProcessor.ProcessAsync", string.IsNullOrWhiteSpace(_instanceTag) ? blob : $"{_instanceTag}:{blob}", ex);
throw;
}
}
if (blobs.Count > 0)
{
_log.WriteInfo("BlobProcessor.ProcessAsync", _instanceTag, $"Processed {blobs.Count} blobs");
}
}
public async Task ProcessDirectoryAsync()
{
var dirs = Directory.GetDirectories(_directoryPath, "*", SearchOption.TopDirectoryOnly);
if (dirs.Length <= 1)
{
return;
}
var dirsToProcess = dirs.OrderBy(i => i).ToList();
try
{
for (int i = 0; i < dirsToProcess.Count - 1; ++i)
{
string dir = dirsToProcess[i];
var files = Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly);
var messages = new List <byte[]>();
int filesCount = 0;
DateTime minDate = DateTime.UtcNow;
foreach (var file in files)
{
DateTime fileTime = GetTimeFromFileName(file);
if (minDate > fileTime)
{
minDate = fileTime;
}
var fileBytes = await File.ReadAllBytesAsync(file);
messages.Add(fileBytes);
++filesCount;
}
string storagePath = Path.GetFileName(dir);
await _blobSaver.SaveToBlobAsync(messages, storagePath);
foreach (var file in files)
{
File.Delete(file);
}
await _log.WriteInfoAsync(
nameof(DirectoryProcessor),
nameof(ProcessDirectoryAsync),
$"Uploaded and deleted {filesCount} files for {_directory}/{storagePath}");
}
}
catch (Exception ex)
{
await _log.WriteErrorAsync(nameof(DirectoryProcessor), nameof(ProcessDirectoryAsync), ex);
}
}
public async Task <string> ProcessDirectoryAsync(string directoryPath)
{
var dirs = Directory.GetDirectories(directoryPath, "*", SearchOption.TopDirectoryOnly);
if (dirs.Length == 0)
{
return(null);
}
string container = Path.GetFileName(directoryPath);
string inProgressFilePath = Path.Combine(directoryPath, _inProgressMarkFile);
if (File.Exists(inProgressFilePath))
{
var creationDate = File.GetCreationTimeUtc(inProgressFilePath);
if (DateTime.UtcNow.Subtract(creationDate).TotalDays < 1)
{
return(null);
}
}
else
{
try
{
File.Create(inProgressFilePath).Dispose();
}
catch (Exception ex)
{
_log.WriteWarning("DirectoryProcessor.ProcessDirectoryAsync", container, "Error on in progress file creation", ex);
return(null);
}
}
var dirsToProcess = dirs.OrderBy(i => i).ToList();
int processedDirsCount = 0;
int dirsToPocessCount = dirsToProcess.Count - 1;
string lastDir = Path.GetFileName(dirsToProcess[dirsToProcess.Count - 1]);
if (DateTime.TryParseExact(lastDir, _timeFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime dirCreationTime) &&
DateTime.UtcNow.Subtract(dirCreationTime).TotalDays >= 1)
{
dirsToPocessCount = dirsToProcess.Count;
}
try
{
for (int i = 0; i < dirsToPocessCount; ++i)
{
string dir = dirsToProcess[i];
var files = Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly);
var messages = new List <string>();
int filesCount = 0;
foreach (var file in files)
{
using (var sr = File.OpenText(file))
{
do
{
var str = sr.ReadLine();
if (str == null)
{
break;
}
messages.Add(str);
} while (true);
}
++filesCount;
}
string storagePath = Path.GetFileName(dir);
try
{
await _blobSaver.SaveToBlobAsync(messages, container, storagePath);
}
catch (Exception ex)
{
throw new InvalidOperationException($"Couldn't save on {container} into {storagePath}", ex);
}
foreach (var file in files)
{
File.Delete(file);
}
try
{
Directory.Delete(dir);
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete {dir} folder", ex);
}
++processedDirsCount;
}
File.Delete(inProgressFilePath);
}
catch (Exception ex)
{
_log.WriteError("DirectoryProcessor.ProcessDirectoryAsync", container, ex);
}
return(processedDirsCount > 0 ? container : null);
}