Beispiel #1
0
 public FtpDownloadStream(FtpRawLogClient client, Uri uri)
 {
     _client = client;
     _uri = uri;
 }
Beispiel #2
0
        public override async Task<bool> Run()
        {
            try
            {
                var ftpClient = new FtpRawLogClient(JobEventSource.Log, _ftpUsername, _ftpPassword);
                var azureClient = new CloudBlobRawLogClient(JobEventSource.Log, _cloudStorageAccount);

                // Collect directory listing.
                IEnumerable<RawLogFileInfo> rawLogFiles = await ftpClient.GetRawLogFiles(_ftpServerUri);

                // Prepare cloud storage blob container.
                var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_cloudStorageContainerName);

                foreach (var rawLogFile in rawLogFiles)
                {
                    try
                    {
                        // Only process the raw log files matching the target CDN platform and account number.
                        if (_azureCdnPlatform == rawLogFile.AzureCdnPlatform && _azureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase))
                        {
                            bool alreadyUploaded = false;
                            bool uploadSucceeded = false;
                            Uri rawLogUri = rawLogFile.Uri;

                            // Check if this is an already renamed file.
                            if (rawLogFile.IsPendingDownload)
                            {
                                // Check if the file has already been uploaded to blob storage.
                                alreadyUploaded = await azureClient.CheckIfBlobExistsAsync(cloudBlobContainer, rawLogFile);
                            }
                            else
                            {
                                // Rename the file on the origin to ensure we're not locking a file that still can be written to.
                                rawLogUri = await ftpClient.RenameAsync(rawLogFile, rawLogFile.FileName + FileExtensions.Download);

                                if (rawLogUri == null)
                                {
                                    // Failed to rename the file. Leave it and try again later.
                                    continue;
                                }
                            }

                            if (!alreadyUploaded)
                            {
                                // open the raw log from FTP
                                using (var rawLogStream = await ftpClient.OpenReadAsync(rawLogUri))
                                using (var rawLogStreamInMemory = new MemoryStream())
                                {
                                    // copy the raw, compressed stream to memory - FTP does not like reading line by line
                                    await rawLogStream.CopyToAsync(rawLogStreamInMemory);
                                    rawLogStreamInMemory.Position = 0;

                                    // process the raw, compressed memory stream
                                    using (var rawGzipStream = new GZipInputStream(rawLogStreamInMemory))
                                    {
                                        try
                                        {
                                            // ensure the .download suffix is trimmed away
                                            var fileName = rawLogFile.FileName.Replace(".download", string.Empty);

                                            Trace.TraceInformation("Uploading file '{0}'.", fileName);
                                            JobEventSource.Log.BeginningBlobUpload(rawLogFile.Uri.ToString());

                                            // open the resulting cleaned blob and stream modified entries
                                            // note the missing using() statement so that we can skip committing if an exception occurs
                                            var resultLogStream = await azureClient.OpenBlobForWriteAsync(cloudBlobContainer, rawLogFile, fileName);
                                            try
                                            {
                                                using (var resultGzipStream = new GZipOutputStream(resultLogStream))
                                                {
                                                    resultGzipStream.IsStreamOwner = false;

                                                    ProcessLogStream(rawGzipStream, resultGzipStream);

                                                    resultGzipStream.Flush();
                                                }

                                                // commit to blob storage
                                                resultLogStream.Commit();

                                                uploadSucceeded = true;
                                            }
                                            catch
                                            {
                                                resultLogStream = null;
                                                uploadSucceeded = false;
                                                throw;
                                            }

                                            Trace.TraceInformation("Finished uploading file '{0}' to '{1}'.", fileName, rawLogFile.Uri.AbsoluteUri);
                                            JobEventSource.Log.FinishingBlobUpload(rawLogFile.Uri.ToString());
                                        }
                                        catch (Exception exception)
                                        {
                                            Trace.TraceError(exception.ToString());
                                            JobEventSource.Log.FailedToUploadFile(rawLogFile.Uri.ToString(), exception.ToString());
                                        }
                                    }
                                }
                            }

                            // Delete the renamed file from the origin.
                            if (alreadyUploaded || uploadSucceeded)
                            {
                                await ftpClient.DeleteAsync(rawLogUri);
                            }
                        }
                    }
                    catch (UnknownAzureCdnPlatformException exception)
                    {
                        // Trace, but ignore the failing file. Other files should go through just fine.
                        Trace.TraceWarning(exception.ToString());
                    }
                    catch (InvalidRawLogFileNameException exception)
                    {
                        // Trace, but ignore the failing file. Other files should go through just fine.
                        Trace.TraceWarning(exception.ToString());
                    }
                }

                return true;
            }
            catch (Exception exception)
            {
                Trace.TraceError(exception.ToString());
            }
            return false;
        }
 public FtpDownloadStream(FtpRawLogClient client, Uri uri)
 {
     _client = client;
     _uri    = uri;
 }