public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(_logDirectoryArgs))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(_arguments.FilePattern);
foreach (DirectoryObjectMetadata file in source.ListFiles().Where(x => x.Path.Contains(".zip")))
{
string tempZipDirectoryPath = file.Path.Replace(".zip", "");
string tempZipDirectoryName = file.Name.Replace(".zip", "");
ZipFile.ExtractToDirectory(file.Path, tempZipDirectoryPath);
source.DeleteFile(file.Path);
foreach (String unzippedfile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => filePattern.IsMatch(x)))
{
string filename = unzippedfile.Split(new[] { "\\" }, StringSplitOptions.None).Last();
List <String> currentFiles = source.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(filename))
{
source.MoveFile(unzippedfile, filename);
}
}
foreach (String gzipFile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => x.Contains(".gz")))
{
string fileNameConcat = tempZipDirectoryName + ".log";
List <String> currentFiles = destination.ListFiles().Select(x => x.Name).ToList();
if (!currentFiles.Contains(fileNameConcat))
{
using (GZipStream gzipStream = new GZipStream(source.OpenFile(gzipFile), CompressionMode.Decompress))
{
using (Stream unzippedDestination = destination.CreateFile(fileNameConcat, Repository.Directory.FileCreationMode.ThrowIfFileExists))
{
gzipStream.CopyTo(unzippedDestination);
}
}
}
}
source.DeleteDirectory(tempZipDirectoryPath);
}
}
List <String> modified = new List <String>();
Int32 newCount = 0;
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
{
logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/");
Dictionary <String, DirectoryRecord> dictionary = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath);
Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);
if (OperationID == 0)
{
logMessage("Warning: OperationID was not set properly. Correcting this.");
OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
}
foreach (DirectoryObjectMetadata file in sourceFiles)
{
if (!dictionary.ContainsKey(file.Path))
{
modified.Add(file.Name);
newCount++;
harvester.DataContext.DirectoryRecords.InsertOnSubmit(new DirectoryRecord
{
OperationID = OperationID,
RepositoryID = repository.ID,
FilePath = file.Path,
FileModifiedDate = file.ModifiedDate,
CreationDate = DateTime.Now,
ModifiedDate = DateTime.Now
});
}
else
{
DirectoryRecord element = dictionary[file.Path];
if (file.ModifiedDate > element.FileModifiedDate)
{
modified.Add(file.Name);
element.FileModifiedDate = file.ModifiedDate;
element.ModifiedDate = DateTime.Now;
}
}
}
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
harvester.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
harvester.DataContext.SubmitChanges();
}
logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");
if (modified.Count == 0)
{
return;
}
using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs))
{
logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})");
StreamParser <EZProxyAudit> Parser = new StreamParser <EZProxyAudit>();
List <EZProxyAudit> records = modified.Select(file =>
{
logMessage($"Processing '{file}':");
int lineNumber = 0;
return(Parser.ParseStream(source.OpenFile(file)).Select(x => new EZProxyAudit
{
DateTime = x.DateTime,
Event = x.Event,
IP = x.IP,
Other = x.Other,
Session = x.Session,
Username = x.Username,
LineNumber = lineNumber++,
}));
}).SelectMany(x => x).ToList();
logMessage($"Records Found: {records.Count}");
destination.DataContext.BulkImportEZProxyAudit(records.ToDataReader(r => new object[] { r.DateTime, r.Event, r.IP, r.Username, r.Session, r.Other, r.LineNumber }));
}
}
}
public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
{
using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(sourceDirectory))
{
logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");
using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(destinationDirectory))
{
logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");
Regex filePattern = new Regex(arguments.FilePattern, RegexOptions.IgnoreCase);
List <DirectoryObjectMetadata> destinationFiles = destination.ListFiles("/").ToList();
int newCount = 0, modified = 0;
foreach (DirectoryObjectMetadata file in source.ListFiles("/").Where(x => filePattern.IsMatch(x.Name)))
{
try
{
if (!destinationFiles.Select(x => x.Name).Contains(file.Name))
{
logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");
using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
{
using (Stream sourceStream = source.OpenFile(file.Name))
{
sourceStream.CopyTo(destStream);
}
}
destination.CopyFile("TMP" + file.Name, file.Name);
destination.DeleteFile("TMP" + file.Name);
logMessage($"Added {file.Name} from {source.Name} to {destination.Name}");
newCount++;
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
}
else
{
DirectoryObjectMetadata destinationFile = destinationFiles.Find(x => x.Name == file.Name);
if (file.ModifiedDate > destinationFile.ModifiedDate)
{
logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");
using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
{
using (Stream sourceStream = source.OpenFile(file.Name))
{
sourceStream.CopyTo(destStream);
}
}
destination.DeleteFile(file.Name);
destination.MoveFile("TMP" + file.Name, file.Name);
logMessage($"Updated {file.Name} from {source.Name} to {destination.Name}");
modified++;
if (cancellationToken.IsCancellationRequested)
{
source.Dispose();
destination.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
}
}
}
catch (Exception)
{
destination.DeleteFile("TMP" + file.Name);
throw;
}
}
logMessage($"Discovered {newCount} new files and {modified} updated files).");
}
}
}