public FtpDownloadStream(FtpRawLogClient client, Uri uri) { _client = client; _uri = uri; }
public override async Task<bool> Run() { try { var ftpClient = new FtpRawLogClient(JobEventSource.Log, _ftpUsername, _ftpPassword); var azureClient = new CloudBlobRawLogClient(JobEventSource.Log, _cloudStorageAccount); // Collect directory listing. IEnumerable<RawLogFileInfo> rawLogFiles = await ftpClient.GetRawLogFiles(_ftpServerUri); // Prepare cloud storage blob container. var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_cloudStorageContainerName); foreach (var rawLogFile in rawLogFiles) { try { // Only process the raw log files matching the target CDN platform and account number. if (_azureCdnPlatform == rawLogFile.AzureCdnPlatform && _azureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase)) { bool alreadyUploaded = false; bool uploadSucceeded = false; Uri rawLogUri = rawLogFile.Uri; // Check if this is an already renamed file. if (rawLogFile.IsPendingDownload) { // Check if the file has already been uploaded to blob storage. alreadyUploaded = await azureClient.CheckIfBlobExistsAsync(cloudBlobContainer, rawLogFile); } else { // Rename the file on the origin to ensure we're not locking a file that still can be written to. rawLogUri = await ftpClient.RenameAsync(rawLogFile, rawLogFile.FileName + FileExtensions.Download); if (rawLogUri == null) { // Failed to rename the file. Leave it and try again later. continue; } } if (!alreadyUploaded) { // open the raw log from FTP using (var rawLogStream = await ftpClient.OpenReadAsync(rawLogUri)) using (var rawLogStreamInMemory = new MemoryStream()) { // copy the raw, compressed stream to memory - FTP does not like reading line by line await rawLogStream.CopyToAsync(rawLogStreamInMemory); rawLogStreamInMemory.Position = 0; // process the raw, compressed memory stream using (var rawGzipStream = new GZipInputStream(rawLogStreamInMemory)) { try { // ensure the .download suffix is trimmed away var fileName = rawLogFile.FileName.Replace(".download", string.Empty); Trace.TraceInformation("Uploading file '{0}'.", fileName); JobEventSource.Log.BeginningBlobUpload(rawLogFile.Uri.ToString()); // open the resulting cleaned blob and stream modified entries // note the missing using() statement so that we can skip committing if an exception occurs var resultLogStream = await azureClient.OpenBlobForWriteAsync(cloudBlobContainer, rawLogFile, fileName); try { using (var resultGzipStream = new GZipOutputStream(resultLogStream)) { resultGzipStream.IsStreamOwner = false; ProcessLogStream(rawGzipStream, resultGzipStream); resultGzipStream.Flush(); } // commit to blob storage resultLogStream.Commit(); uploadSucceeded = true; } catch { resultLogStream = null; uploadSucceeded = false; throw; } Trace.TraceInformation("Finished uploading file '{0}' to '{1}'.", fileName, rawLogFile.Uri.AbsoluteUri); JobEventSource.Log.FinishingBlobUpload(rawLogFile.Uri.ToString()); } catch (Exception exception) { Trace.TraceError(exception.ToString()); JobEventSource.Log.FailedToUploadFile(rawLogFile.Uri.ToString(), exception.ToString()); } } } } // Delete the renamed file from the origin. if (alreadyUploaded || uploadSucceeded) { await ftpClient.DeleteAsync(rawLogUri); } } } catch (UnknownAzureCdnPlatformException exception) { // Trace, but ignore the failing file. Other files should go through just fine. Trace.TraceWarning(exception.ToString()); } catch (InvalidRawLogFileNameException exception) { // Trace, but ignore the failing file. Other files should go through just fine. Trace.TraceWarning(exception.ToString()); } } return true; } catch (Exception exception) { Trace.TraceError(exception.ToString()); } return false; }
public FtpDownloadStream(FtpRawLogClient client, Uri uri) { _client = client; _uri = uri; }