public override void Execute(DateTime runDate, Action <string> logMessage, System.Threading.CancellationToken cancellationToken)
        {
            using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(_directoryArgs))
            {
                logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");

                using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(_logDirectoryArgs))
                {
                    logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");

                    Regex filePattern = new Regex(_arguments.FilePattern);

                    foreach (DirectoryObjectMetadata file in source.ListFiles().Where(x => x.Path.Contains(".zip")))
                    {
                        string tempZipDirectoryPath = file.Path.Replace(".zip", "");
                        string tempZipDirectoryName = file.Name.Replace(".zip", "");
                        ZipFile.ExtractToDirectory(file.Path, tempZipDirectoryPath);
                        source.DeleteFile(file.Path);

                        foreach (String unzippedfile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => filePattern.IsMatch(x)))
                        {
                            string        filename     = unzippedfile.Split(new[] { "\\" }, StringSplitOptions.None).Last();
                            List <String> currentFiles = source.ListFiles().Select(x => x.Name).ToList();
                            if (!currentFiles.Contains(filename))
                            {
                                source.MoveFile(unzippedfile, filename);
                            }
                        }

                        foreach (String gzipFile in source.ListFiles(tempZipDirectoryName).Select(x => x.Path).Where(x => x.Contains(".gz")))
                        {
                            string        fileNameConcat = tempZipDirectoryName + ".log";
                            List <String> currentFiles   = destination.ListFiles().Select(x => x.Name).ToList();
                            if (!currentFiles.Contains(fileNameConcat))
                            {
                                using (GZipStream gzipStream = new GZipStream(source.OpenFile(gzipFile), CompressionMode.Decompress))
                                {
                                    using (Stream unzippedDestination = destination.CreateFile(fileNameConcat, Repository.Directory.FileCreationMode.ThrowIfFileExists))
                                    {
                                        gzipStream.CopyTo(unzippedDestination);
                                    }
                                }
                            }
                        }

                        source.DeleteDirectory(tempZipDirectoryPath);
                    }
                }

                List <String> modified = new List <String>();
                Int32         newCount = 0;

                using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(_harvesterArgs))
                {
                    logMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");

                    IEnumerable <DirectoryObjectMetadata> sourceFiles = source.ListFiles("/");
                    Dictionary <String, DirectoryRecord>  dictionary  = harvester.DataContext.DirectoryRecords.Where(d => d.Operation.Name == Name && d.Repository.Name == source.Name).ToDictionary(d => d.FilePath);

                    Entities.Repository repository = harvester.DataContext.Repositories.First(x => x.Name == source.Name);

                    if (OperationID == 0)
                    {
                        logMessage("Warning: OperationID was not set properly. Correcting this.");
                        OperationID = harvester.DataContext.Operations.First(d => d.Name == Name).ID;
                    }

                    foreach (DirectoryObjectMetadata file in sourceFiles)
                    {
                        if (!dictionary.ContainsKey(file.Path))
                        {
                            modified.Add(file.Name);
                            newCount++;

                            harvester.DataContext.DirectoryRecords.InsertOnSubmit(new DirectoryRecord
                            {
                                OperationID      = OperationID,
                                RepositoryID     = repository.ID,
                                FilePath         = file.Path,
                                FileModifiedDate = file.ModifiedDate,
                                CreationDate     = DateTime.Now,
                                ModifiedDate     = DateTime.Now
                            });
                        }
                        else
                        {
                            DirectoryRecord element = dictionary[file.Path];

                            if (file.ModifiedDate > element.FileModifiedDate)
                            {
                                modified.Add(file.Name);
                                element.FileModifiedDate = file.ModifiedDate;
                                element.ModifiedDate     = DateTime.Now;
                            }
                        }
                    }

                    if (cancellationToken.IsCancellationRequested)
                    {
                        source.Dispose();
                        harvester.Dispose();
                        cancellationToken.ThrowIfCancellationRequested();
                    }
                    harvester.DataContext.SubmitChanges();
                }

                logMessage($"Discovered {modified.Count} files to be processed ({newCount} new and {modified.Count - newCount} updated).");

                if (modified.Count == 0)
                {
                    return;
                }

                using (IDatabaseRepository <IStatisticsDataContext> destination = RepositoryFactory.CreateStatisticsRepository(_statisticsArgs))
                {
                    logMessage($"Connected to database '{destination.Name}' ({destination.ConnectionString})");

                    StreamParser <EZProxyAudit> Parser = new StreamParser <EZProxyAudit>();

                    List <EZProxyAudit> records = modified.Select(file =>
                    {
                        logMessage($"Processing '{file}':");

                        int lineNumber = 0;

                        return(Parser.ParseStream(source.OpenFile(file)).Select(x => new EZProxyAudit
                        {
                            DateTime = x.DateTime,
                            Event = x.Event,
                            IP = x.IP,
                            Other = x.Other,
                            Session = x.Session,
                            Username = x.Username,
                            LineNumber = lineNumber++,
                        }));
                    }).SelectMany(x => x).ToList();

                    logMessage($"Records Found: {records.Count}");

                    destination.DataContext.BulkImportEZProxyAudit(records.ToDataReader(r => new object[] { r.DateTime, r.Event, r.IP, r.Username, r.Session, r.Other, r.LineNumber }));
                }
            }
        }
Esempio n. 2
0
        public override void Execute(DateTime runDate, Action <string> logMessage, CancellationToken cancellationToken)
        {
            using (IDirectoryRepository source = RepositoryFactory.CreateDirectoryRepository(sourceDirectory))
            {
                logMessage($"Connected to source repository '{source.Name}' ({source.ConnectionString})");

                using (IDirectoryRepository destination = RepositoryFactory.CreateDirectoryRepository(destinationDirectory))
                {
                    logMessage($"Connected to destination repository '{destination.Name}' ({destination.ConnectionString})");

                    Regex filePattern = new Regex(arguments.FilePattern, RegexOptions.IgnoreCase);
                    List <DirectoryObjectMetadata> destinationFiles = destination.ListFiles("/").ToList();
                    int newCount = 0, modified = 0;
                    foreach (DirectoryObjectMetadata file in source.ListFiles("/").Where(x => filePattern.IsMatch(x.Name)))
                    {
                        try
                        {
                            if (!destinationFiles.Select(x => x.Name).Contains(file.Name))
                            {
                                logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");

                                using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
                                {
                                    using (Stream sourceStream = source.OpenFile(file.Name))
                                    {
                                        sourceStream.CopyTo(destStream);
                                    }
                                }

                                destination.CopyFile("TMP" + file.Name, file.Name);
                                destination.DeleteFile("TMP" + file.Name);

                                logMessage($"Added {file.Name} from {source.Name} to {destination.Name}");
                                newCount++;

                                if (cancellationToken.IsCancellationRequested)
                                {
                                    source.Dispose();
                                    destination.Dispose();
                                    cancellationToken.ThrowIfCancellationRequested();
                                }
                            }
                            else
                            {
                                DirectoryObjectMetadata destinationFile = destinationFiles.Find(x => x.Name == file.Name);
                                if (file.ModifiedDate > destinationFile.ModifiedDate)
                                {
                                    logMessage($"Processing {file.Name} from {source.Name} to {destination.Name}");

                                    using (Stream destStream = destination.CreateFile("TMP" + file.Name, FileCreationMode.Overwrite))
                                    {
                                        using (Stream sourceStream = source.OpenFile(file.Name))
                                        {
                                            sourceStream.CopyTo(destStream);
                                        }
                                    }

                                    destination.DeleteFile(file.Name);
                                    destination.MoveFile("TMP" + file.Name, file.Name);

                                    logMessage($"Updated {file.Name} from {source.Name} to {destination.Name}");
                                    modified++;

                                    if (cancellationToken.IsCancellationRequested)
                                    {
                                        source.Dispose();
                                        destination.Dispose();
                                        cancellationToken.ThrowIfCancellationRequested();
                                    }
                                }
                            }
                        }
                        catch (Exception)
                        {
                            destination.DeleteFile("TMP" + file.Name);
                            throw;
                        }
                    }

                    logMessage($"Discovered {newCount} new files and {modified} updated files).");
                }
            }
        }