private bool TryCheckForFilesToBeSynced(DateTime dueTime)
        {
            var syncCompletedSuccessfully = true;

            this.perfHelper.FileSyncPassBegin();
            this.TraceSource.WriteInfo(
                this.LogSourceId,
                "Starting to examine folder {0} for files to synchronize with the file share ...",
                this.folderName);

            // Do a breadth-first traversal of the folder from which files need
            // to be uploaded
            FolderInfo folderInfo = new FolderInfo
            {
                FullPath     = this.folderName,
                RelativePath = string.Empty,
                LocalMapPath = this.localMap
            };

            Queue <FolderInfo> folderQueue = new Queue <FolderInfo>();

            folderQueue.Enqueue(folderInfo);
            bool syncInterruptedDueToStop    = false;
            bool syncInterruptedDueToTimeout = false;

            while (folderQueue.Count != 0)
            {
                if (this.CheckForInterruptions(dueTime, ref syncInterruptedDueToStop, ref syncInterruptedDueToTimeout))
                {
                    break;
                }

                folderInfo = folderQueue.Dequeue();

                try
                {
                    // Create a DirectoryInfo object representing the current folder
                    DirectoryInfo dirInfo = new DirectoryInfo(folderInfo.FullPath);

                    // Add the current folder's sub-folders to the queue
                    IEnumerable <DirectoryInfo> subFolders = dirInfo.EnumerateDirectories();
                    foreach (DirectoryInfo subFolder in subFolders)
                    {
                        if (this.CheckForInterruptions(dueTime, ref syncInterruptedDueToStop, ref syncInterruptedDueToTimeout))
                        {
                            break;
                        }

                        FolderInfo subFolderInfo = new FolderInfo();
                        subFolderInfo.FullPath     = subFolder.FullName;
                        subFolderInfo.RelativePath = string.IsNullOrEmpty(folderInfo.RelativePath) ?
                                                     subFolder.Name :
                                                     Path.Combine(
                            folderInfo.RelativePath,
                            subFolder.Name);
                        subFolderInfo.LocalMapPath = Path.Combine(
                            folderInfo.LocalMapPath,
                            subFolder.Name);
                        folderQueue.Enqueue(subFolderInfo);
                    }

                    // Iterate over the files in the current folder and figure out which
                    // ones need to be synchronized
                    IEnumerable <FileInfo> files = dirInfo.EnumerateFiles();
                    foreach (FileInfo file in files)
                    {
                        try
                        {
                            if (this.CheckForInterruptions(dueTime, ref syncInterruptedDueToStop, ref syncInterruptedDueToTimeout))
                            {
                                break;
                            }

                            bool   needsSync    = false;
                            string localMapFile = Path.Combine(
                                folderInfo.LocalMapPath,
                                file.Name);

                            if (false == FabricFile.Exists(localMapFile))
                            {
                                // We don't have any record of this file in our local map.
                                // So we definitely need to synchronize this file.
                                needsSync = true;
                            }
                            else
                            {
                                // We have a record of this file in our local map. Compare
                                // the timestamp of the file with the timestamp of its
                                // counterpart in the local map to determine if the file
                                // has changed since we last copied it to the destination.
                                FileInfo localMapFileInfo = new FileInfo(localMapFile);
                                if (localMapFileInfo.LastWriteTime.CompareTo(file.LastWriteTime) < 0)
                                {
                                    // The file has changed since we last copied it to the
                                    // destination. We need to synchronize this file.
                                    needsSync = true;
                                }
                            }

                            if (needsSync)
                            {
                                // This file needs to be synchronized. Add it to the list
                                // of files to be copied to the file share.
                                this.AddFileToCopy(Path.Combine(
                                                       folderInfo.RelativePath,
                                                       file.Name));
                            }

                            this.perfHelper.FileCheckedForSync();
                        }
                        catch (IOException)
                        {
                            // Unable to read information about the file. Maybe
                            // it got deleted. Handle the exception and move on.
                            syncCompletedSuccessfully = false;
                        }
                    }
                }
                catch (IOException e)
                {
                    // Exception occurred during enumeration of directories or files.
                    // Handle the exception and move on.
                    this.TraceSource.WriteError(
                        this.LogSourceId,
                        "Failed to enumerate folder {0} while looking for files to sync. Exception information: {1}",
                        folderInfo.FullPath,
                        e);
                    syncCompletedSuccessfully = false;
                }
            }

            if (syncInterruptedDueToStop)
            {
                this.TraceSource.WriteInfo(
                    this.LogSourceId,
                    "The consumer is being stopped. Therefore, examination of folder {0} for files to synchronize with the file share was interrupted.",
                    this.folderName);
                syncCompletedSuccessfully = false;
            }
            else if (syncInterruptedDueToTimeout)
            {
                this.TraceSource.WriteInfo(
                    this.LogSourceId,
                    "Examination of folder {0} for files to synchronize with the file share was interrupted due to timeout.",
                    this.folderName);
                syncCompletedSuccessfully = false;
            }
            else
            {
                this.TraceSource.WriteInfo(
                    this.LogSourceId,
                    "Finished examining folder {0} for files to synchronize with the file share.",
                    this.folderName);
            }

            this.perfHelper.FileSyncPassEnd();
            return(syncCompletedSuccessfully);
        }
        /// <summary>
        /// Download file from XStore to SMB
        /// </summary>
        /// <param name="blobContainer">The blob container having the file.</param>
        /// <param name="task">The task to be performed.</param>
        private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
        {
            if (!this.ShouldCopy(task.SrcUri))
            {
                return;
            }

            // download the file
            if (task.PartialID >= 0)
            {
                // This is already a divided work; let's go ahead and download
                string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID);

                // Delete the file if it exists
                string directoryName = FabricPath.GetDirectoryName(dstSMBPath);
                if (!FabricDirectory.Exists(directoryName))
                {
                    FabricDirectory.CreateDirectory(directoryName);
                }

                if (FabricFile.Exists(dstSMBPath))
                {
                    FabricFile.Delete(dstSMBPath, deleteReadonly: true);
                }

                var blob        = blobContainer.GetBlockBlobReference(task.SrcUri);
                var blobWrapper = new XStoreBlobWrapper(blob);
                if (task.TimeoutHelper != null)
                {
                    //task.TimeoutHelper.ThrowIfExpired();
                    blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime());
                }
                else
                {
                    blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue);
                }
            }
            else
            {
                // we are going to download a file, which we don't know the size yet
                var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
                blob.FetchAttributes(null, this.defaultRequestOption);
#else
                blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait();
#endif
                int blobLength = (int)blob.Properties.Length;

                // Delete the file if it exists
                string directoryName = FabricPath.GetDirectoryName(task.DstUri);
                if (!FabricDirectory.Exists(directoryName))
                {
                    FabricDirectory.CreateDirectory(directoryName);
                }

                if (FabricFile.Exists(task.DstUri))
                {
                    FabricFile.Delete(task.DstUri, deleteReadonly: true);
                }

                if (blobLength < DownloadSizeThreshold)
                {
                    var blobWrapper = new XStoreBlobWrapper(blob);
                    if (task.TimeoutHelper != null)
                    {
                        blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext);
                    }
                    else
                    {
                        blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext);
                    }

                    return;
                }
                else
                {
                    // For large files we divided the work to couple threads.
                    int numThreads = Math.Min(
                        blobLength / ParrallelDownloadSize,
                        ParrallelDownloadThreadCount);

                    // Create new tasks to parallel download
                    Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
                    int offset = 0;
                    for (int i = 0; i < numThreads; i++)
                    {
                        int length;
                        if (i < numThreads - 1)
                        {
                            length = blobLength / numThreads;
                        }
                        else
                        {
                            length = blobLength - offset;
                        }

                        XStoreFileOperationTask newTask = new XStoreFileOperationTask(
                            XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB,
                            task.SrcUri,
                            task.DstUri,
                            false,
                            0,
                            task.TimeoutHelper);

                        newTask.FileCopyFlag = task.FileCopyFlag;
                        newTask.Offset       = offset;
                        newTask.Length       = length;
                        newTask.PartialID    = i;

                        tasks.Enqueue(newTask);
                        offset += length;
                    }

                    // enqueue all divided tasks
                    this.xstoreTaskPool.AddTaskToTaskQueue(tasks);

                    // Add an EndTask to the endTaskQueue as well; to combine all downloads
                    XStoreFileOperationTask endTask = new XStoreFileOperationTask(
                        XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles,
                        task.SrcUri,
                        task.DstUri,
                        false,
                        0,
                        task.TimeoutHelper);

                    endTask.IsSucceeded  = true;
                    endTask.FileCopyFlag = task.FileCopyFlag;
                    endTask.PartialID    = numThreads;
                    endTask.Content      = task.DstUri;
                    this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
                }
            }
        }
Exemplo n.º 3
0
 private static bool CopyFileToDestination(object context)
 {
     FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
     FabricFile.Copy(fileCopyInfo.Source, fileCopyInfo.Destination, true);
     return true;
 }
        public void ProvisionFabric(
            string localCodePath,
            string localConfigPath,
#if !DotNetCoreClrLinux && !DotNetCoreClrIOT
            string configurationCsvFilePath,
#endif
            string infrastructureManifestFilePath,
            bool validateOnly,
            TimeSpan timeout)
        {
            TimeoutHelper timeoutHelper = new TimeoutHelper(timeout);

#if DotNetCoreClrLinux || DotNetCoreClrIOT
            ImageBuilder.TraceSource.WriteInfo(
                TraceType,
                "Starting ProvisionFabric. CodePath:{0}, ConfigPath:{1}, InfrastructureManifestFilePath:{2}, Timeout:{3}",
                localCodePath,
                localConfigPath,
                infrastructureManifestFilePath,
                timeoutHelper.GetRemainingTime());
#else
            ImageBuilder.TraceSource.WriteInfo(
                TraceType,
                "Starting ProvisionFabric. CodePath:{0}, ConfigPath:{1}, ConfigurationCsvFilePath:{2}, InfrastructureManifestFilePath:{3}, Timeout:{4}",
                localCodePath,
                localConfigPath,
                configurationCsvFilePath,
                infrastructureManifestFilePath,
                timeoutHelper.GetRemainingTime());
#endif

            // TODO: Once FabricTest has been modified to generate InfrastructureManifest.xml file, we should not
            // allow InfrastructureManifest file to be optional
            if (string.IsNullOrEmpty(infrastructureManifestFilePath) || !FabricFile.Exists(infrastructureManifestFilePath))
            {
                infrastructureManifestFilePath = null;
                ImageBuilder.TraceSource.WriteWarning(
                    TraceType,
                    "The InfrastrucutreManifestFile:{0} is not found",
                    infrastructureManifestFilePath);
            }

            string codeVersion = string.Empty, clusterManifestVersion = string.Empty;

            if (!string.IsNullOrEmpty(localCodePath))
            {
                codeVersion = GetCodeVersion(localCodePath);
            }

            if (!string.IsNullOrEmpty(localConfigPath))
            {
                try
                {
                    var parameters = new Dictionary <string, dynamic>
                    {
                        { DeploymentParameters.ClusterManifestString, localConfigPath },
                        { DeploymentParameters.InfrastructureManifestString, infrastructureManifestFilePath }
                    };

                    var deploymentParameters = new DeploymentParameters();
                    deploymentParameters.SetParameters(parameters, DeploymentOperations.ValidateClusterManifest);
                    DeploymentOperation.ExecuteOperation(deploymentParameters);
                }
                catch (Exception e)
                {
                    ImageBuilderUtility.TraceAndThrowValidationError(
                        FabricProvisionOperation.TraceType,
                        StringResources.ImageBuilderError_ClusterManifestValidationFailed,
                        e.ToString(),
                        localConfigPath);
                }
            }

            if (!validateOnly)
            {
                timeoutHelper.ThrowIfExpired();

                WinFabStoreLayoutSpecification winFabLayout = WinFabStoreLayoutSpecification.Create();
                if (!string.IsNullOrEmpty(localCodePath))
                {
                    // Upload MSP file
                    string destinationCodePath;
                    if (ImageBuilderUtility.IsInstaller(localCodePath))
                    {
                        destinationCodePath = winFabLayout.GetPatchFile(codeVersion);
                    }
                    else if (ImageBuilderUtility.IsCabFile(localCodePath))
                    {
                        // Upload CAB file
                        destinationCodePath = winFabLayout.GetCabPatchFile(codeVersion);
                    }
                    else
                    {
                        destinationCodePath = winFabLayout.GetCodePackageFolder(codeVersion);
                    }

                    this.imageStoreWrapper.UploadContent(destinationCodePath, localCodePath, timeoutHelper.GetRemainingTime());
                }

                ClusterManifestType clusterManifest = null;
                if (!string.IsNullOrEmpty(localConfigPath))
                {
                    clusterManifest        = ImageBuilderUtility.ReadXml <ClusterManifestType>(localConfigPath, this.validatingXmlReaderSettings);
                    clusterManifestVersion = clusterManifest.Version;
                }

                if (clusterManifest != null)
                {
                    // Upload ClusterManifest file

                    ReplaceDefaultImageStoreConnectionString(clusterManifest);

                    string destinationConfigPath = winFabLayout.GetClusterManifestFile(clusterManifestVersion);
                    imageStoreWrapper.SetToStore <ClusterManifestType>(destinationConfigPath, clusterManifest, timeoutHelper.GetRemainingTime());
                }
            }

#if DotNetCoreClrLinux || DotNetCoreClrIOT
            ImageBuilder.TraceSource.WriteInfo(
                TraceType,
                "Completed ProvisionFabric. CodePath:{0}, ConfigPath:{1}",
                localCodePath,
                localConfigPath);
#else
            ImageBuilder.TraceSource.WriteInfo(
                TraceType,
                "Completed ProvisionFabric. CodePath:{0}, ConfigPath:{1}, ConfigurationCsvFilePath:{2}",
                localCodePath,
                localConfigPath,
                configurationCsvFilePath);
#endif
        }
        public void OnEtwEventProcessingPeriodStart()
        {
            this.streamWriter = null;
            this.perfHelper.EtlReadPassBegin();

            // Build the full path to our buffered event file
            string tempCacheFileName = string.Format(
                CultureInfo.InvariantCulture,
                "{0}{1}.{2}",
                TempCacheFileNamePrefix,
                DateTime.Now.Ticks,
                TempCacheFileExtension);
            string tempCacheFileFullPath = Path.Combine(this.etwEventCache, tempCacheFileName);

            // Open the file
            StreamWriter writer = null;

            try
            {
                Utility.PerformIOWithRetries(
                    ctx =>
                {
                    string fileName       = ctx;
                    FileStream fileStream = FabricFile.Open(fileName, FileMode.Create, FileAccess.Write);
#if !DotNetCoreClr
                    Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
#endif
                    writer = new StreamWriter(fileStream);
                },
                    tempCacheFileFullPath);
            }
            catch (Exception e)
            {
                // Log an error and move on. No events from this pass will be
                // written to the destination.
                this.TraceSource.WriteExceptionAsError(
                    this.LogSourceId,
                    e,
                    "Failed to create a new file in the buffered event directory. None of the events from this pass will be written to the event buffer.");
                return;
            }

            // Write the version number. This will help us read data from the file.
            try
            {
                Utility.PerformIOWithRetries(
                    () =>
                {
                    writer.WriteLine(EtlConsumerConstants.EtwEventCacheFormatVersionString);
                });
            }
            catch (Exception e)
            {
                // Log an error and move on. No events from this pass will be
                // written to the destination.
                this.TraceSource.WriteExceptionAsError(
                    this.LogSourceId,
                    e,
                    "Failed to write buffered event file format version number to file {0} in the buffered event file. None of the events from this pass will be written to the event buffer.",
                    tempCacheFileFullPath);
                return;
            }

            this.streamWriter = writer;
            this.TraceSource.WriteInfo(
                this.LogSourceId,
                "Filtered ETW events for consumer will be buffered in file {0}.",
                tempCacheFileFullPath);
        }
Exemplo n.º 6
0
        /// <summary>
        /// Remove tag from store and clear the data associated with tag.
        /// </summary>
        /// <param name="tag">Location (relative to RootUri) from where to delete the content.</param>
        /// <param name="timeout">The timeout for performing the delete operation.</param>
        public void DeleteContent(string tag, TimeSpan timeout)
        {
            TimeoutHelper helper = (timeout == TimeSpan.MaxValue) ? null : new TimeoutHelper(timeout);

#if !DotNetCoreClr
            using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
            {
                try
                {
                    string smbTag = this.ConvertTagToSMBPath(tag);
                    if ((!FabricFile.Exists(smbTag)) &&
                        (!FabricDirectory.Exists(smbTag)))
                    {
                        return;
                    }

                    using (FileWriterLock fileWriterLock = new FileWriterLock(smbTag))
                    {
                        if (!fileWriterLock.Acquire())
                        {
                            throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
                        }

                        if (helper != null)
                        {
                            helper.ThrowIfExpired();
                        }

                        if (FabricFile.Exists(smbTag))
                        {
                            FabricFile.Delete(smbTag, deleteReadonly: true);
                        }
                        else if (FabricDirectory.Exists(smbTag))
                        {
                            FabricDirectory.Delete(smbTag, recursive: true, deleteReadOnlyFiles: true);
                        }
                    }
                }
                catch (IOException exception)
                {
                    // HResult 0x80070020: THE PROCESS CANNOT ACCESS THE FILE BECAUSE IT IS BEING USED BY ANOTHER PROCESS
                    // This work-around is done here since when we read a file from a folder we only take a reader lock on the file and not the
                    // entire folder. When we delete a folder, we only take writer lock on the folder and hence we might run into scenarios
                    // where a folder is attempted to be deleted even when a file in it is being read.
                    // Returning FabricImageStoreException, causes CM to retry the delete operation.
                    int hrError = Marshal.GetHRForException(exception);
                    TraceSource.WriteWarning(
                        TraceType,
                        "Delete of {0} failed because of {1}, hrError {2:X}.",
                        tag,
                        exception.Message,
                        hrError);

                    if (hrError == unchecked ((int)0x80070020))
                    {
                        throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, exception, FabricErrorCode.ImageStoreAcquireFileLockFailed);
                    }

                    throw;
                }
            }
        }
Exemplo n.º 7
0
 internal static bool FileExists(string fileName)
 {
     return(FabricFile.Exists(fileName));
 }
Exemplo n.º 8
0
        /// <summary>
        /// Creates the bookmark file if it does not exist.
        /// </summary>
        /// <param name="bookmarkFolder"></param>
        /// <param name="bytesWritten"></param>
        /// <returns></returns>
        internal bool CreateBookmarkFile(
            string bookmarkFolder,
            out long bytesWritten)
        {
            FileStream   fs     = null;
            StreamWriter writer = null;

            bytesWritten = 0;
            string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);

            if (true == FabricFile.Exists(bookmarkFile))
            {
                return(true);
            }

            try
            {
                // Create the bookmark file
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        fs     = new FileStream(bookmarkFile, FileMode.Create);
                        writer = new StreamWriter(fs);
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to create bookmark file {0} while processing events.",
                        bookmarkFile);
                    return(false);
                }

                long localBytesWritten = 0;

                // Write the version information
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        writer.WriteLine(BookmarkFileFormatVersionString);
                        localBytesWritten += BookmarkFileFormatVersionString.Length + 2;
                        writer.Flush();
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to write version information in bookmark file {0} while processing events.",
                        bookmarkFile);
                    return(false);
                }

                // Write the timestamp and index of the last event read and close the file
                int    lastEventIndex      = -1;
                string lastEventReadString = string.Format("{0},{1},{2}", DateTime.MinValue.ToBinary(), DateTime.MinValue.ToString(), lastEventIndex.ToString());
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        writer.WriteLine(lastEventReadString);
                        localBytesWritten += lastEventReadString.Length + 2;
                        writer.Dispose();
                        fs.Dispose();
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to write last event read {0} in bookmark file {1} while processing events.",
                        lastEventReadString,
                        bookmarkFile);
                    return(false);
                }

                // record bytes written
                bytesWritten = localBytesWritten;
            }
            finally
            {
                if (null != writer)
                {
                    writer.Dispose();
                }

                if (null != fs)
                {
                    fs.Dispose();
                }
            }

            return(true);
        }
Exemplo n.º 9
0
        /// <summary>
        /// Gets the file position where last event index is recorded.
        /// </summary>
        /// <param name="bookmarkFolder"></param>
        /// <param name="lastEventReadPosition"></param>
        /// <param name="bytesRead"></param>
        /// <returns></returns>
        internal bool GetLastEventReadPosition(
            string bookmarkFolder,
            out long lastEventReadPosition,
            out long bytesRead)
        {
            lastEventReadPosition = 0;
            bytesRead             = 0;

            string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);

            if (false == FabricFile.Exists(bookmarkFile))
            {
                // Bookmark file doesn't exist
                return(false);
            }

            StreamReader reader = null;

            try
            {
                // Open the file
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        FileStream fileStream = FabricFile.Open(bookmarkFile, FileMode.Open, FileAccess.Read);
                        Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
                        reader = new StreamReader(fileStream);
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to open bookmark file {0}.",
                        bookmarkFile);
                    return(false);
                }

                long localBytesRead = 0;
                long streamPosition = 0;

                // Get the version
                string versionString = string.Empty;
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        versionString   = reader.ReadLine();
                        localBytesRead += versionString.Length + 2;
                        streamPosition  = localBytesRead;
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to read version information from bookmark file {0}.",
                        bookmarkFile);
                    return(false);
                }

                // Check the version
                if (false == versionString.Equals(BookmarkFileFormatVersionString, StringComparison.Ordinal))
                {
                    this.traceSource.WriteError(
                        this.logSourceId,
                        "Unexpected version string {0} encountered in bookmark file {1}.",
                        versionString,
                        bookmarkFile);
                    return(false);
                }

                // Get information about the last event that we read
                string infoLine = string.Empty;
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        infoLine        = reader.ReadLine();
                        localBytesRead += infoLine.Length + 2;
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to read information about last event read from bookmark file {0}.",
                        bookmarkFile);
                    return(false);
                }

                // record bytes read
                bytesRead = localBytesRead;

                lastEventReadPosition = streamPosition;
            }
            finally
            {
                if (null != reader)
                {
                    reader.Dispose();
                }
            }

            return(true);
        }
Exemplo n.º 10
0
        private void TestIntersectingCopyAndReplace(bool isLongPath)
        {
            var folderPath = this.testPath;

            if (true == isLongPath)
            {
                folderPath = this.ExtendPath(folderPath);
            }

            var filePath = Path.Combine(folderPath, this.testFileName);

            Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");

            var copyFilePath = Path.Combine(folderPath, this.testHardLinkedFileName);

            Assert.IsTrue(!isLongPath || copyFilePath.Length > 260, "hard linked file path must be greater than max path size.");

            var replaceFilePath = Path.Combine(folderPath, this.testReplaceFileName);

            Assert.IsTrue(!isLongPath || replaceFilePath.Length > 260, "replace file path must be greater than max path size.");

            var backupFilePath = Path.Combine(folderPath, this.testBackupFileName);

            Assert.IsTrue(!isLongPath || backupFilePath.Length > 260, "backup file path must be greater than max path size.");

            LogHelper.Log("FabricDirectory.Create {0}", folderPath);
            FabricDirectory.CreateDirectory(folderPath);

            LogHelper.Log("FabricFile.Create {0}", filePath);
            using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
            {
                LogHelper.Log("Write {0}", this.testString);
                streamWriter.WriteLine(this.testString);
                streamWriter.Flush();
            }

            using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(replaceFilePath)))
            {
                LogHelper.Log("Write {0}", this.testNewString);
                streamWriter.WriteLine(this.testNewString);
                streamWriter.Flush();
            }

            FabricFile.Copy(filePath, copyFilePath, false);

            using (StreamReader reader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read)))
            {
                var content = reader.ReadLine();
                Assert.AreEqual <string>(this.testString, content, "before replace current file must have the old content.");
            }

            using (StreamReader copyReader0 = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
            {
                var hardLinkContent0 = copyReader0.ReadLine();
                Assert.AreEqual <string>(this.testString, hardLinkContent0, "after replace hard link file must have the old content.");

                FabricFile.Replace(replaceFilePath, filePath, backupFilePath, false);

                using (StreamReader fileReader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read)))
                {
                    var content = fileReader.ReadLine();
                    Assert.AreEqual <string>(this.testNewString, content, "after replace current file must have the new content.");
                }

                using (StreamReader copyReader1 = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
                {
                    var content = copyReader1.ReadLine();
                    Assert.AreEqual <string>(this.testString, content, "after replace hard link file must have the old content.");
                }
            }
        }
Exemplo n.º 11
0
        internal override async Task RenameCopyLogAtomicallyAsync()
        {
            try
            {
                // Retrieve head and tail records of the copy log
                var tailRecord      = this.PhysicalLogWriter.CurrentLogTailRecord;
                var callbackManager = this.PhysicalLogWriter.CallbackManager;
                this.PhysicalLogWriter.ResetCallbackManager = true;
                var closedException = this.PhysicalLogWriter.ClosedException;
                var exitTcs         = this.PhysicalLogWriter.ExitTcs;

                // Now, close the copy log
                await this.CloseCurrentLogAsync().ConfigureAwait(false);

                var fullcurrentLogfileName = FileLogicalLog.GetFullPathToLog(
                    this.LogFileDirectoryPath,
                    this.CurrentLogFileAlias);
                var fullBaseLogFileName = FileLogicalLog.GetFullPathToLog(
                    this.LogFileDirectoryPath,
                    this.BaseLogFileAlias);
                var fullbackupLogFileName = FileLogicalLog.GetFullPathToLog(
                    this.LogFileDirectoryPath,
                    this.BaseLogFileAlias + BackupSuffix);

                FabricFile.Replace(fullcurrentLogfileName, fullBaseLogFileName, fullbackupLogFileName, false);

                // opens using currentLogFileAlias
                this.CurrentLogFileAlias = this.BaseLogFileAlias;
                this.LogicalLog          = await this.CreateLogFileAsync(false, CancellationToken.None).ConfigureAwait(false);

                // Write cursor is auto-position to eos
                this.PhysicalLogWriter = new PhysicalLogWriter(
                    this.LogicalLog,
                    tailRecord,
                    callbackManager,
                    closedException,
                    this.Tracer,
                    this.MaxWriteCacheSizeInMB,
                    this.IncomingBytesRateCounterWriter,
                    this.LogFlushBytesRateCounterWriter,
                    this.BytesPerFlushCounterWriter,
                    this.AvgFlushLatencyCounterWriter,
                    this.AvgSerializationLatencyCounterWriter)
                {
                    ExitTcs = exitTcs
                };

                var message =
                    string.Format(
                        CultureInfo.InvariantCulture,
                        "RenameCopyLogAtomically: Renamed log. Head record Position: {0}" + Environment.NewLine
                        + "             Tail record. Type: {1} LSN: {2} PSN: {3} Position: {4}",
                        this.LogHeadRecordPosition,
                        tailRecord.RecordType,
                        tailRecord.Lsn.LSN,
                        tailRecord.Psn.PSN,
                        tailRecord.RecordPosition);

                FabricEvents.Events.LogManager(this.Tracer.Type, message);
            }
            catch (Exception e)
            {
                Utility.ProcessUnexpectedException(
                    "LogManager::RenameCopyLogAtomically",
                    this.Tracer,
                    "renaming log",
                    e);
            }
        }
Exemplo n.º 12
0
        private void CopyTest(bool copyFileExists, bool isLongPath, bool overWrite)
        {
            var folderPath = this.testPath;

            if (true == isLongPath)
            {
                folderPath = this.ExtendPath(folderPath);
            }

            var filePath = Path.Combine(folderPath, this.testFileName);

            Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");

            var copyFilePath = Path.Combine(folderPath, this.testReplaceFileName);

            Assert.IsTrue(!isLongPath || copyFilePath.Length > 260, "replace file path must be greater than max path size.");

            LogHelper.Log("FabricDirectory.Create {0}", folderPath);
            FabricDirectory.CreateDirectory(folderPath);

            LogHelper.Log("FabricFile.Create {0}", filePath);
            using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
            {
                LogHelper.Log("Write {0}", this.testNewString);
                streamWriter.WriteLine(this.testNewString);
                streamWriter.Flush();
            }

            if (copyFileExists)
            {
                using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(copyFilePath)))
                {
                    LogHelper.Log("Write {0}", this.testString);
                    streamWriter.WriteLine(this.testString);
                    streamWriter.Flush();
                }

                using (StreamReader copyReader = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
                {
                    var content = copyReader.ReadLine();
                    Assert.AreEqual <string>(this.testString, content, "before copy state.");
                }
            }
            else
            {
                Assert.IsFalse(FabricFile.Exists(copyFilePath));
            }

            try
            {
                FabricFile.Copy(filePath, copyFilePath, overWrite);
                Assert.IsTrue(overWrite);
            }
            catch (FabricException e)
            {
                Assert.IsFalse(overWrite);
                LogHelper.Log("Exception thrown as expected {0}", e.GetType().ToString());
            }

            if (true == overWrite || false == copyFileExists)
            {
                using (StreamReader copyReader = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
                {
                    var content = copyReader.ReadLine();
                    Assert.AreEqual <string>(this.testNewString, content, "after copy file must have the new content.");
                    LogHelper.Log("Read {0} as expected", this.testNewString);
                }
            }
        }
Exemplo n.º 13
0
 public void FabricFile_ExistsNegative()
 {
     LogHelper.Log("FabricFile.Exists {0}", this.badPath);
     Assert.IsFalse(FabricFile.Exists(this.badPath));
 }
Exemplo n.º 14
0
        public void FabricFile_EndToEndPositive()
        {
            var folderPath = this.testPath;

            folderPath = this.ExtendPath(folderPath);

            var filePath = Path.Combine(folderPath, this.testFileName);

            Assert.IsTrue(filePath.Length > 260);

            LogHelper.Log("FabricDirectory.Create {0}", folderPath);
            FabricDirectory.CreateDirectory(folderPath);

            LogHelper.Log("FabricFile.Create {0}", filePath);
            using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
            {
                LogHelper.Log("Write {0}", this.testString);
                streamWriter.WriteLine(this.testString);
            }

            LogHelper.Log("FabricDirectory.GetDirectories {0}", this.testPath);
            var result = FabricDirectory.GetDirectories(this.testPath);

            Assert.AreEqual(1, result.Length);

            LogHelper.Log("FabricDirectory.GetFiles {0}", this.testPath);
            result = FabricDirectory.GetFiles(this.testPath);
            Assert.AreEqual(0, result.Length);

            LogHelper.Log("FabricDirectory.GetFiles {0}, AllDirectories", this.testPath);
            result = FabricDirectory.GetFiles(this.testPath, "*", SearchOption.AllDirectories);
            Assert.AreEqual(1, result.Length);

            LogHelper.Log("FabricDirectory.GetDirectories {0}", folderPath);
            result = FabricDirectory.GetDirectories(folderPath);
            Assert.AreEqual(0, result.Length);

            LogHelper.Log("FabricDirectory.GetFiles {0}", folderPath);
            result = FabricDirectory.GetFiles(folderPath);
            Assert.AreEqual(1, result.Length);

            LogHelper.Log("FabricFile.Open {0}", filePath);
            using (StreamReader streamReader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.None)))
            {
                string actual = streamReader.ReadLine();
                LogHelper.Log("Read {0}", actual);
                Assert.AreEqual(this.testString, actual);
            }

            LogHelper.Log("FabricFile.GetSize {0}", filePath);
            long size = FabricFile.GetSize(filePath);

            Assert.IsTrue(size > 0);

            LogHelper.Log("FabricPath.GetDirectoryName {0}", filePath);
            string directoryName = FabricPath.GetDirectoryName(filePath);

            Assert.AreEqual(folderPath, directoryName);

            LogHelper.Log("FabricFile.GetLastWriteTime {0}", filePath);
            DateTime oldTime = FabricFile.GetLastWriteTime(filePath);

            Thread.Sleep(TimeSpan.FromSeconds(1));
            using (StreamWriter streamWriter = new StreamWriter(FabricFile.Open(filePath, FileMode.Open, FileAccess.Write)))
            {
                LogHelper.Log("Write {0}", this.testString);
                streamWriter.WriteLine(this.testString);
            }

            DateTime newTime = FabricFile.GetLastWriteTime(filePath);

            Assert.IsTrue(newTime > oldTime);
        }
Exemplo n.º 15
0
        /// <summary>
        /// Uploads the local source content to the image store at the remote destination.
        /// </summary>
        /// <param name="storeSource"> Location (relative to RootUri) from where to download the content. </param>
        /// <param name="storeDestination"> Location relative to RootUri where the content needs to be uploaded. </param>
        /// <param name="timeout">The timeout for copy content operation.</param>
        /// <param name="skipFiles">Files that do not need to be copied.</param>
        /// <param name="copyFlag">The copying control information to specify how file can be overwritten</param>
        /// <param name="checkMarkFile">Flag the specified the checkmark file.</param>
        public void CopyContent(string storeSource, string storeDestination, TimeSpan timeout, string[] skipFiles, CopyFlag copyFlag, bool checkMarkFile)
        {
            TimeoutHelper  helper     = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
            FileReaderLock readerLock = null;

            try
            {
                string smbSourcePath      = this.ConvertTagToSMBPath(storeSource);
                string smbDestinationPath = this.ConvertTagToSMBPath(storeDestination);

                readerLock = new FileReaderLock(smbSourcePath);
                if (!readerLock.Acquire())
                {
                    throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
                }

                if (helper != null)
                {
                    helper.ThrowIfExpired();
                }

#if !DotNetCoreClr
                using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
                {
                    bool fabricDirectoryExists = FabricDirectory.Exists(smbSourcePath);
                    if (fabricDirectoryExists && skipFiles.Any())
                    {
                        string[] fullFileNames    = FabricDirectory.GetFiles(smbSourcePath, "*", true, SearchOption.AllDirectories);
                        var      relativeFilePath = from file in fullFileNames
                                                    select file.Replace(smbSourcePath, "").TrimStart(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar });

                        string[] filtered = relativeFilePath.Where(file => !skipFiles.Contains <string>(Path.GetFileName(file))).ToArray <string>();

                        if (filtered.Count() < fullFileNames.Count())
                        {
                            foreach (string file in filtered)
                            {
                                CopyCallerHoldsReaderLock(Path.Combine(smbSourcePath, file), Path.Combine(smbDestinationPath, file), copyFlag, helper);
                            }
                        }
                        else
                        {
                            CopyCallerHoldsReaderLock(smbSourcePath, smbDestinationPath, copyFlag, helper);
                        }
                    }
                    else
                    {
                        bool fabricFileExist = FabricFile.Exists(smbSourcePath);
                        if ((!FabricFile.Exists(smbSourcePath)) &&
                            (!FabricDirectory.Exists(smbSourcePath)))
                        {
                            throw new IOException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_DoesNotExistError, smbSourcePath));
                        }

                        CopyCallerHoldsReaderLock(smbSourcePath, smbDestinationPath, copyFlag, helper);
                    }
                }
            }
            catch (IOException exception)
            {
                if (exception.GetType() == typeof(IOException))
                {
                    throw new FabricImageStoreIOException(exception);
                }
                else
                {
                    throw;
                }
            }
            finally
            {
                if (readerLock != null)
                {
                    readerLock.Dispose();
                }
            }
        }
Exemplo n.º 16
0
        /// <summary>
        /// Retrieves the last event index processed.
        /// </summary>
        /// <param name="bookmarkFolder"></param>
        /// <param name="bytesRead"></param>
        /// <returns></returns>
        internal EventIndex ReadBookmarkFile(
            string bookmarkFolder,
            out long bytesRead)
        {
            bytesRead = 0;
            EventIndex lastEventIndex = new EventIndex();

            lastEventIndex.Set(DateTime.MinValue, -1);

            string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);

            if (false == FabricFile.Exists(bookmarkFile))
            {
                // Bookmark file doesn't exist
                return(lastEventIndex);
            }

            StreamReader reader = null;

            try
            {
                // Open the file
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        FileStream fileStream = FabricFile.Open(bookmarkFile, FileMode.Open, FileAccess.Read);
                        Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
                        reader = new StreamReader(fileStream);
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to open bookmark file {0}.",
                        bookmarkFile);
                    return(lastEventIndex);
                }

                long localBytesRead = 0;

                // Get the version
                string versionString = string.Empty;
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        versionString   = reader.ReadLine();
                        localBytesRead += versionString.Length + 2;
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to read version information from bookmark file {0}.",
                        bookmarkFile);
                    return(lastEventIndex);
                }

                // Check the version
                if (false == versionString.Equals(BookmarkFileFormatVersionString, StringComparison.Ordinal))
                {
                    this.traceSource.WriteError(
                        this.logSourceId,
                        "Unexpected version string {0} encountered in bookmark file {1}.",
                        versionString,
                        bookmarkFile);
                    return(lastEventIndex);
                }

                // Get information about the last event that we read
                string infoLine = string.Empty;
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        infoLine        = reader.ReadLine();
                        localBytesRead += infoLine.Length + 2;
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to read information about last event read from bookmark file {0}.",
                        bookmarkFile);
                    return(lastEventIndex);
                }

                string[] infoLineParts = infoLine.Split(',');
                if (infoLineParts.Length != (int)LastEventReadInfoParts.Count)
                {
                    this.traceSource.WriteError(
                        this.logSourceId,
                        "The information in bookmark file {0} about the last event read is not in the expected format. {1}",
                        bookmarkFile,
                        infoLine);
                    return(lastEventIndex);
                }

                string lastEventTimestampString = infoLineParts[(int)LastEventReadInfoParts.LastEventTimestampLong].Trim();
                long   lastEventTimestampBinary;
                if (false == long.TryParse(lastEventTimestampString, out lastEventTimestampBinary))
                {
                    this.traceSource.WriteError(
                        this.logSourceId,
                        "Unable to retrieve timestamp of last event from bookmark file {0}.",
                        bookmarkFile);
                    return(lastEventIndex);
                }

                DateTime lastEventTimestamp = DateTime.FromBinary(lastEventTimestampBinary);

                string lastEventTimestampDifferentiatorString = infoLineParts[(int)LastEventReadInfoParts.LastEventIndex].Trim();
                int    lastEventTimestampDifferentiator;
                if (false == int.TryParse(lastEventTimestampDifferentiatorString, out lastEventTimestampDifferentiator))
                {
                    this.traceSource.WriteError(
                        this.logSourceId,
                        "Unable to retrieve timestamp differentiator of last event from bookmark file {0}.",
                        bookmarkFile);
                    return(lastEventIndex);
                }

                // record bytes read
                bytesRead = localBytesRead;

                lastEventIndex.Set(lastEventTimestamp, lastEventTimestampDifferentiator);
            }
            finally
            {
                if (null != reader)
                {
                    reader.Dispose();
                }
            }

            return(lastEventIndex);
        }
Exemplo n.º 17
0
        /// <summary>
        /// Downloads content from the file image store to local destination.
        /// </summary>
        /// <param name="storeSource">The relative path of source file image store to be downloaded from.</param>
        /// <param name="localDestination">The local destination path to download the content.</param>
        /// <param name="handler">The image store progress handler which is not supported at the file image store.</param>
        /// <param name="timeout">The timeout for performing the downloading operation.</param>
        /// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
        public void DownloadContent(string storeSource, string localDestination, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
        {
            TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            localDestination = this.GetLocalPath(localDestination);
            string         tempDirectory = null;
            FileReaderLock readerLock    = null;

            try
            {
                string smbTag = this.ConvertTagToSMBPath(storeSource);
                if ((!FabricFile.Exists(smbTag)) &&
                    (!FabricDirectory.Exists(smbTag)))
                {
                    throw new IOException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_DoesNotExistError, smbTag));
                }

                readerLock = new FileReaderLock(smbTag);
                if (!readerLock.Acquire())
                {
                    throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
                }

                if (helper != null)
                {
                    helper.ThrowIfExpired();
                }

                if (accessDescription != null && !accessDescription.HasWriteAccess)
                {
                    // Copy the content from the remote Store to a temp location using the ImpersonationContext
                    string tempLocalResource = null;
#if !DotNetCoreClr
                    using (WindowsImpersonationContext impersonationContext = accessDescription.WindowsIdentity.Impersonate())
#endif
                    {
                        tempDirectory     = CreateAndAclDirectory();
                        tempLocalResource = Path.Combine(tempDirectory, Path.GetRandomFileName());
                        CopyCallerHoldsReaderLock(smbTag, tempLocalResource, CopyFlag.AtomicCopy, helper);
                    }

                    readerLock.Release();

                    // Copy the content from the temp location to the local destination outside the ImpersonationContext
                    CopyCallerHoldsReaderLock(tempLocalResource, localDestination, copyFlag, helper);
                }

#if !DotNetCoreClr
                using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
                {
                    CopyCallerHoldsReaderLock(smbTag, localDestination, copyFlag, helper);
                }
            }
            catch (IOException exception)
            {
                if (exception.GetType() == typeof(IOException))
                {
                    throw new FabricImageStoreException(StringResources.Error_ImageStoreIOException, exception);
                }
                else
                {
                    throw;
                }
            }
            finally
            {
                if (readerLock != null)
                {
                    readerLock.Dispose();
                }

                if (tempDirectory != null && FabricDirectory.Exists(tempDirectory))
                {
                    FabricDirectory.Delete(tempDirectory, recursive: true, deleteReadOnlyFiles: true);
                }
            }
        }
Exemplo n.º 18
0
        /// <summary>
        /// Updates the bookmark file with the last event read index.
        /// </summary>
        /// <param name="bookmarkFolder"></param>
        /// <param name="lastReadEventIndexPosition"></param>
        /// <param name="eventIndex"></param>
        /// <param name="bytesWritten"></param>
        /// <returns></returns>
        internal bool UpdateBookmarkFile(
            string bookmarkFolder,
            long lastReadEventIndexPosition,
            EventIndex eventIndex,
            out long bytesWritten)
        {
            bytesWritten = 0;
            FileStream   fs           = null;
            StreamWriter writer       = null;
            string       bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);

            if (false == FabricFile.Exists(bookmarkFile))
            {
                return(false);
            }

            try
            {
                // Open the bookmark file
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        fs     = new FileStream(bookmarkFile, FileMode.Open);
                        writer = new StreamWriter(fs);
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to open bookmark file {0} while processing events.",
                        bookmarkFile);
                    return(false);
                }

                // Move the stream position to the point where the event index needs to be written
                fs.Position = lastReadEventIndexPosition;

                long localBytesWritten = 0;

                // Write the timestamp and index of the last event read and close the file
                string lastEventReadString = string.Format("{0},{1},{2}", eventIndex.Timestamp.ToBinary(), eventIndex.Timestamp.ToString(), eventIndex.TimestampDifferentiator);
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                    {
                        writer.WriteLine(lastEventReadString);
                        localBytesWritten = lastEventReadString.Length + 2;
                        writer.Dispose();
                        fs.Dispose();
                    },
                        this.methodExecutionInitialRetryIntervalMs,
                        this.methodExecutionMaxRetryCount,
                        this.methodExecutionMaxRetryIntervalMs);
                }
                catch (Exception e)
                {
                    this.traceSource.WriteExceptionAsError(
                        this.logSourceId,
                        e,
                        "Unable to write last event read {0} in bookmark file {1} while processing events.",
                        lastEventReadString,
                        bookmarkFile);
                    return(false);
                }

                // record bytes written
                bytesWritten = localBytesWritten;
            }
            finally
            {
                if (null != writer)
                {
                    writer.Dispose();
                }

                if (null != fs)
                {
                    fs.Dispose();
                }
            }

            return(true);
        }
Exemplo n.º 19
0
        private void CopyCallerHoldsReaderLock(string source, string destination, CopyFlag copyFlag, TimeoutHelper helper)
        {
            string destinationDirectory = FabricPath.GetDirectoryName(destination);

            if (!string.IsNullOrEmpty(destinationDirectory) && !FabricDirectory.Exists(destinationDirectory))
            {
                FabricDirectory.CreateDirectory(destinationDirectory);
            }

            using (FileWriterLock writerLock = new FileWriterLock(destination))
            {
                if (!writerLock.Acquire())
                {
                    throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
                }

                if (helper != null)
                {
                    helper.ThrowIfExpired();
                }

                if (FabricFile.Exists(source))
                {
                    // This is a file copy
                    if (FabricFile.Exists(destination))
                    {
                        FabricFile.Delete(destination, deleteReadonly: true);
                    }

                    int retryCount = 0;
                    while (helper == null || !TimeoutHelper.HasExpired(helper))
                    {
                        try
                        {
                            bool shouldOverwrite = (copyFlag != CopyFlag.AtomicCopySkipIfExists);
                            FabricFile.Copy(source, destination, shouldOverwrite);
                            break;
                        }
                        catch (UnauthorizedAccessException)
                        {
                            TraceSource.WriteInfo(
                                TraceType,
                                "Uploading {0} to {1} caused UnauthorizedAccessException. RetryCount: {2}.",
                                source,
                                destination,
                                retryCount);

                            if (retryCount++ > 3)
                            {
                                throw;
                            }

                            // This could happen when a file is marked for delete and we try to delete
                            // it again or try to open the file. Retrying after sometime should fix the issue.
                            Thread.Sleep(TimeSpan.FromSeconds(retryCount));
                        }

                        if (helper != null)
                        {
                            helper.ThrowIfExpired();
                        }
                    }
                }
                else
                {
                    // This is a folder copy
                    using (FolderCopy fc = new FolderCopy(copyFlag, null))
                    {
                        fc.Copy(source, destination);
                    }
                }
            }
        }
Exemplo n.º 20
0
        private void ProcessEtlFilesWorker(
            string fileNamePattern,
            DateTime lastEndTime,
            string friendlyName,
            CancellationToken cancellationToken,
            out DateTime endTime)
        {
            // By default, initialize endTime to lastEndTime. If we run into a
            // situation where we don't have any ETL files to process, the
            // default value will not be updated.
            endTime = lastEndTime;

            if (!Directory.Exists(this.traceDirectory) &&
                this.traceDirectory.StartsWith(
                    ContainerEnvironment.ContainerLogRootDirectory,
                    StringComparison.InvariantCultureIgnoreCase))
            {
                // A temporary handling for the containers case, where DCA could be processing an instance's
                // folder with a different older structure (if traces collectors or DCA itself got updated).
                this.traceSource.WriteInfo(
                    this.logSourceId,
                    "Skipping a search cycle to match '{0}' as corresponding container instance directory '{1}' is not found.",
                    fileNamePattern,
                    this.traceDirectory);

                return;
            }

            // Get all files in the traces directory that match the pattern
            this.traceSource.WriteInfo(
                this.logSourceId,
                "Searching for ETL files whose names match '{0}' in directory '{1}'.",
                fileNamePattern,
                this.traceDirectory);

            var dirInfo  = new DirectoryInfo(this.traceDirectory);
            var etlFiles = dirInfo.GetFiles(fileNamePattern);

            this.traceSource.WriteInfo(
                this.logSourceId,
                "Found {0} ETL files whose names match '{1}' in directory '{2}'.",
                etlFiles.Length,
                fileNamePattern,
                this.traceDirectory);

            if (0 == etlFiles.Length)
            {
                // No ETL files found. Nothing more to do here.
                return;
            }

            // We ignore files that are so old that they need to be deleted
            var cutoffTime = DateTime.UtcNow.Add(-this.etlInMemoryProducerWorkerSettings.EtlDeletionAge);

            // Reading in chronological order. Sort the files such that the
            // file with the least recent creation time comes first.
            etlFiles = etlFiles.OrderBy(e => e.CreationTime).ToArray();

            // Ignore ETL files whose last-write time is older than the timestamp
            // up to which we have already read all events. However, we never
            // ignore the active ETL file because its timestamp is updated only
            // after it has reached its full size.
            var activeEtlFileName = etlFiles[etlFiles.Length - 1].Name;

            etlFiles = etlFiles
                       .Where(f => f.Name.Equals(activeEtlFileName, StringComparison.OrdinalIgnoreCase) ||
                              (f.LastWriteTimeUtc.CompareTo(lastEndTime) >= 0))
                       .ToArray();

            // Processing start time
            var processingStartTime = DateTime.Now;
            var processingEndTime   = processingStartTime.Add(this.perProviderProcessingTime);
            var filesProcessed      = 0;

            for (int i = 0; i < etlFiles.Length; i++)
            {
                if (etlFiles[i].LastWriteTimeUtc.CompareTo(cutoffTime) <= 0)
                {
                    this.traceSource.WriteInfo(
                        this.logSourceId,
                        "Skipping ETL file {0} in directory {1} because it is too old.",
                        etlFiles[i],
                        this.traceDirectory);
                    continue;
                }

                // If the file has already been fully processed, then ignore
                // that file.
                if (FabricFile.Exists(Path.Combine(this.markerFileDirectory, etlFiles[i].Name)))
                {
                    continue;
                }

                this.traceSource.WriteInfo(
                    this.logSourceId,
                    "Processing ETL file: {0}.",
                    etlFiles[i].Name);

                // If we are processing Windows Fabric ETL files from their default location, then
                // make sure that we have loaded the correct manifest for them.
                if (this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation())
                {
                    this.etlProcessor.EnsureCorrectWinFabManifestVersionLoaded(etlFiles[i].Name);
                }

                bool isActiveEtl = i == etlFiles.Length - 1;

                if (isActiveEtl)
                {
                    // process active etl file and then break so that we can process additional
                    // events in the next pass.
                    this.etlProcessor.ProcessActiveEtlFile(
                        etlFiles[i],
                        lastEndTime,
                        cancellationToken,
                        out endTime);

                    filesProcessed++;

                    if (false == this.checkpointManager.SetLastEndTime(etlFiles[i].Name, endTime))
                    {
                        if (0 != lastEndTime.CompareTo(endTime))
                        {
                            this.traceSource.WriteError(
                                this.logSourceId,
                                "Failed to write the last timestamp up to which {0} ETW events have been read to file {1}. Due to this error, although we have read events up to {2}, in our next pass we will start reading events from {3}.",
                                friendlyName,
                                etlFiles[i].Name,
                                endTime,
                                lastEndTime);
                        }
                    }

                    break;
                }
                else
                {
                    this.etlProcessor.ProcessInactiveEtlFile(
                        etlFiles[i],
                        lastEndTime,
                        cancellationToken,
                        out endTime);

                    filesProcessed++;

                    if (false == this.checkpointManager.SetLastEndTime(etlFiles[i].Name, endTime))
                    {
                        if (0 != lastEndTime.CompareTo(endTime))
                        {
                            this.traceSource.WriteError(
                                this.logSourceId,
                                "Failed to write the last timestamp up to which {0} ETW events have been read to file {1}. Due to this error, although we have read events up to {2}, in our next pass we will start reading events from {3}.",
                                friendlyName,
                                etlFiles[i].Name,
                                endTime,
                                lastEndTime);
                        }
                    }
                    else
                    {
                        // set last end time so that the next iteration can calculate start
                        // time accurately
                        lastEndTime = endTime;
                    }
                }

                // check if time for this provider is up
                if (0 > DateTime.Compare(processingEndTime, DateTime.Now))
                {
                    this.traceSource.WriteInfo(
                        this.logSourceId,
                        "Due to time limit on etl processing, no more {0} files will be processed in this pass.",
                        fileNamePattern);
                    break;
                }

                if (cancellationToken.IsCancellationRequested)
                {
                    this.traceSource.WriteInfo(
                        this.logSourceId,
                        "The producer is being stopped, so no more {0} files will be processed in this pass.",
                        fileNamePattern);
                    break;
                }
            }

            // Write the length of the DCA's ETL file backlog. The performance test consumes this information.
            this.perfHelper.RecordEtlPassBacklog(friendlyName, etlFiles.Length - filesProcessed);
        }
Exemplo n.º 21
0
        private static void CopyFileWorker(object context)
        {
            CopyFileParameters copyParam = (CopyFileParameters)context;

            FabricFile.Copy(copyParam.Source, copyParam.Destination, copyParam.Overwrite);
        }
Exemplo n.º 22
0
        internal EtlInMemoryProducerWorker(
            EtlInMemoryProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId             = initParam.ProducerInstanceId;
            this.traceSource             = initParam.TraceSource;
            this.cancellationTokenSource = new CancellationTokenSource();
            this.perfHelper       = new EtlInMemoryPerformance(this.traceSource);
            this.diskSpaceManager = diskSpaceManager;

            // Initialize the settings
            this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));

            // Figure out where the ETL files are located
            this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                this.providers,
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.traceSource,
                this.logSourceId);

            this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                // We need to collect bootstrap traces
                this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                    this.traceDirectory,
                    this.markerFileDirectory,
                    this.etlToInMemoryBufferWriters,
                    this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
                    this.traceSource,
                    this.logSourceId);
                this.bootstrapTraceProcessor.Start();

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    true,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.markerFileDirectory,
                    this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToInMemoryBufferWriters,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // Figure out how much processing time is available to each provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            // Disk manager set up for traces
            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                this.diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                    f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }

            // Disk manager set up for marker files
            this.diskSpaceManager.RegisterFolder(
                this.logSourceId,
                () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
        }
        private void DeliverEventsToConsumer(object state)
        {
            this.perfHelper.EventDeliveryPassBegin();

            this.eventDeliveryPeriodAborted = false;

            // Perform any tasks necessary at the beginning of the event delivery pass
            this.eventSink.OnEtwEventDeliveryStart();

            // Get the files from the buffered event directory
            // We ignore files that are so old that they need to be deleted
            DateTime cutoffTime = DateTime.UtcNow.Add(-this.eventDeletionAge);

            DirectoryInfo dirInfo = new DirectoryInfo(this.etwEventCache);

            FileInfo[] eventFilesInfo = dirInfo.GetFiles(CacheFileNameSearchPattern)
                                        .Where(file => file.LastWriteTimeUtc.CompareTo(cutoffTime) > 0)
                                        .ToArray();

            // Sort the files such that the file with the most recent last-write
            // time comes first. We'll process files in that order so that in
            // case of huge backlogs the most recent (and hence likely to be
            // most interesting) traces are processed first.
            Array.Sort(eventFilesInfo, CompareFileLastWriteTimes);

            // Process each of the files
            int           filesProcessed    = 0;
            DateTime      processingEndTime = DateTime.Now.Add(this.eventDeliveryPassLength);
            List <string> filesToDelete     = new List <string>();

            foreach (FileInfo eventFileInfo in eventFilesInfo)
            {
                // Process events from the current file
                string eventFile = eventFileInfo.FullName;
                this.ProcessEventsFromFile(eventFile);
                filesProcessed++;

                // If the event delivery pass is being aborted, then don't process
                // any more files. Also, don't delete the current file because
                // its processing may have been interrupted. This file will be
                // processed again in the next pass.
                if (this.eventDeliveryPeriodAborted)
                {
                    this.TraceSource.WriteInfo(
                        this.LogSourceId,
                        "The event delivery pass is being aborted. Therefore, no more files in the buffered event directory will be processed.");
                    break;
                }

                // If we are in the process of stopping, then don't process any
                // more files. Also, don't delete the current file because its
                // processing may have been interrupted. This file will be
                // processed again when we are restarted.
                if (this.Stopping)
                {
                    this.TraceSource.WriteInfo(
                        this.LogSourceId,
                        "The consumer is being stopped. Therefore, no more files in the buffered event directory will be processed.");
                    break;
                }

                // Add the file to the list of files that we delete at the end
                // of the event delivery pass
                filesToDelete.Add(eventFile);

                if (0 > DateTime.Compare(processingEndTime, DateTime.Now))
                {
                    this.TraceSource.WriteInfo(
                        this.LogSourceId,
                        "Due to time limit on backlog processing, no more files in the buffered event directory will be processed in this pass.");
                    break;
                }
            }

            // Perform any tasks necessary at the end of the ETL processing pass
            this.eventSink.OnEtwEventDeliveryStop();

            // Delete the files that we successfully processed in this pass
            foreach (string fileToDelete in filesToDelete)
            {
                try
                {
                    Utility.PerformIOWithRetries(
                        ctx =>
                    {
                        string fileName = ctx;
                        FabricFile.Delete(fileName);
                    },
                        fileToDelete);
                }
                catch (Exception e)
                {
                    this.TraceSource.WriteExceptionAsError(
                        this.LogSourceId,
                        e,
                        "Failed to delete file {0}.",
                        fileToDelete);
                }
            }

            // Write performance-related information.
            this.perfHelper.RecordEventDeliveryBacklog(eventFilesInfo.Length - filesProcessed);
            this.perfHelper.EventDeliveryPassEnd();

            // Schedule the next pass
            this.eventDeliveryTimer.Start();
        }
Exemplo n.º 24
0
        protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed)
        {
            // No compression
            fileCompressed = false;

            // Compute the destination path
            string destination = Path.Combine(this.destinationPath, sourceRelative);

            // Figure out the name of the directory at the destination and in the
            // local map.
            var destinationDir = FabricPath.GetDirectoryName(destination);

            // If the directory at the destination doesn't exist, then create it
            if ((false == string.IsNullOrEmpty(destinationDir)) &&
                (false == this.PerformDestinationOperation(DirectoryExistsAtDestination, destinationDir)))
            {
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                        {
                            PerformDestinationOperation(
                                CreateDirectoryAtDestination,
                                destinationDir);
                        },
                        retryCount);
                }
                catch (Exception e)
                {
                    this.TraceSource.WriteExceptionAsError(
                        this.LogSourceId,
                        e,
                        "Failed to create directory {0} for copying file {1}.",
                        destinationDir,
                        sourceRelative);
                    fileSkipped = true;
                    return false;
                }
            }

            // Determine whether we need to first copy the file to the staging folder
            string stagingFilePath = null;
            if (false == string.IsNullOrEmpty(this.stagingFolderPath))
            {
                stagingFilePath = Path.Combine(this.stagingFolderPath, Path.GetFileName(sourceRelative));
            }

            // Copy the file over to its destination
            DateTime sourceLastWriteTime = DateTime.MinValue;
            bool sourceFileNotFound = false;
            try
            {
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                        {
                            if (string.IsNullOrEmpty(stagingFilePath))
                            {
                                this.perfHelper.ExternalOperationBegin(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);
                            }

                            try
                            {
                                // Copy the file
                                if (false == string.IsNullOrEmpty(stagingFilePath))
                                {
                                    FabricFile.Copy(source, stagingFilePath, true);
                                }
                                else
                                {
                                    FileCopyInfo copyInfo = new FileCopyInfo
                                    {
                                        Source = source,
                                        Destination = destination
                                    };
                                    PerformDestinationOperation(CopyFileToDestination, copyInfo);
                                }
                            }
                            catch (FileNotFoundException)
                            {
                                sourceFileNotFound = true;
                            }

                            if (string.IsNullOrEmpty(stagingFilePath))
                            {
                                this.perfHelper.ExternalOperationEnd(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);
                                this.perfHelper.FileUploaded();
                            }
                        },
                        retryCount);
                }
                catch (Exception e)
                {
                    this.TraceSource.WriteExceptionAsError(
                        this.LogSourceId,
                        e,
                        "File copy failed. Source: {0}, destination: {1}",
                        source,
                        destination);
                    fileSkipped = true;
                    return false;
                }

                if (sourceFileNotFound)
                {
                    // The source file was not found. Maybe it
                    // got deleted before we had a chance to copy
                    // it. Handle the error and move on.
                    this.TraceSource.WriteWarning(
                        this.LogSourceId,
                        "File {0} could not be uploaded to {1} because it was not found.",
                        source,
                        destination);
                    fileSkipped = true;
                    return true;
                }

                if (false == string.IsNullOrEmpty(stagingFilePath))
                {
                    try
                    {
                        Utility.PerformIOWithRetries(
                            () =>
                            {
                                this.perfHelper.ExternalOperationBegin(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);

                                FileCopyInfo copyInfo = new FileCopyInfo
                                {
                                    Source = stagingFilePath,
                                    Destination = destination
                                };
                                PerformDestinationOperation(CopyFileToDestination, copyInfo);

                                this.perfHelper.ExternalOperationEnd(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);
                                this.perfHelper.FileUploaded();
                            },
                            retryCount);
                    }
                    catch (Exception e)
                    {
                        this.TraceSource.WriteExceptionAsError(
                            this.LogSourceId,
                            e,
                            "File copy failed. Source: {0}, destination: {1}",
                            source,
                            destination);
                        fileSkipped = true;
                        return false;
                    }
                }
            }
            finally
            {
                if (false == string.IsNullOrEmpty(stagingFilePath))
                {
                    try
                    {
                        Utility.PerformIOWithRetries(
                            () =>
                            {
                                FabricFile.Delete(stagingFilePath);
                            },
                            retryCount);
                    }
                    catch (Exception e)
                    {
                        this.TraceSource.WriteExceptionAsError(
                            this.LogSourceId,
                            e,
                            "Failed to delete file {0}",
                            stagingFilePath);
                    }
                }
            }

            fileSkipped = false;
            return true;
        }
        public void OnEtwEventProcessingPeriodStop()
        {
            if (null == this.streamWriter)
            {
                // An error occurred during the creation of the buffered event
                // file. So just return immediately without doing anything.
                return;
            }

            // Close the buffered event file that we are currently working on.
            try
            {
                this.streamWriter.Dispose();
            }
            catch (System.Text.EncoderFallbackException ex)
            {
                // This can happen if the manifest file does not match the binary.
                // Write an error message and move on.
                this.TraceSource.WriteError(
                    this.LogSourceId,
                    "Exception occurred while closing buffered event file. Exception information: {0}",
                    ex);
            }

            // Make our buffered event files available for delivery to the consumer.
            // This includes:
            //  - the buffered event file that we wrote in this pass
            //  - any old buffered event files that we wrote in previous passes
            //    (and got interrupted before we could rename them for delivery to
            //    the consumer).
            string[] cacheFiles = FabricDirectory.GetFiles(
                this.etwEventCache,
                TempCacheFileNameSearchPattern);
            foreach (string cacheFile in cacheFiles)
            {
                try
                {
                    Utility.PerformIOWithRetries(
                        ctx =>
                    {
                        string fileName = ctx;
                        FabricFile.Move(
                            fileName,
                            Path.ChangeExtension(fileName, CacheFileNameExtension));
                    },
                        cacheFile);

                    this.TraceSource.WriteInfo(
                        this.LogSourceId,
                        "File containing filtered ETW events was renamed for delivery to consumer. Old name: {0}, new name: {1}.",
                        cacheFile,
                        Path.ChangeExtension(cacheFile, CacheFileNameExtension));
                }
                catch (Exception e)
                {
                    this.TraceSource.WriteExceptionAsError(
                        this.LogSourceId,
                        e,
                        "Buffered event file {0} could not be renamed for delivery to consumer.",
                        cacheFile);
                }
            }

            this.perfHelper.EtlReadPassEnd();
        }
Exemplo n.º 26
0
        /// <summary>
        /// Open an existing <see cref="BackupMetadataFile"/> from the given file path.
        /// </summary>
        /// <param name="backupMetadataFilePath">Path of the backup meta data file.</param>
        /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
        /// <returns>The read <see cref="BackupMetadataFile"/>.</returns>
        public static async Task <BackupMetadataFile> OpenAsync(
            string backupMetadataFilePath,
            CancellationToken cancellationToken)
        {
            if (string.IsNullOrEmpty(backupMetadataFilePath))
            {
                throw new ArgumentException(
                          string.Format(CultureInfo.CurrentCulture,
                                        SR.Error_NullArgument_Formatted,
                                        "backupMetadataFilePath"));
            }

            if (!FabricFile.Exists(backupMetadataFilePath))
            {
                throw new ArgumentException(
                          string.Format(
                              CultureInfo.CurrentCulture,
                              SR.Error_FilePath_Null,
                              backupMetadataFilePath),
                          "backupMetadataFilePath");
            }

            // Open the file with asynchronous flag and 4096 cache size (C# default).
            // MCoskun: Default IoPriorityHint is used since this operation is called during restore.
            using (var filestream = FabricFile.Open(
                       backupMetadataFilePath,
                       FileMode.Open,
                       FileAccess.Read,
                       FileShare.Read,
                       4096,
                       FileOptions.Asynchronous))
            {
                var backupMetadataFile = new BackupMetadataFile(backupMetadataFilePath);

                // Read and validate the Footer section.  The footer is always at the end of the stream, minus space for the checksum.
                var footerHandle = new BlockHandle(
                    filestream.Length - FileFooter.SerializedSize - sizeof(ulong),
                    FileFooter.SerializedSize);
                backupMetadataFile.footer =
                    await
                    FileBlock.ReadBlockAsync(
                        filestream,
                        footerHandle,
                        (reader, handle) => FileFooter.Read(reader, handle)).ConfigureAwait(false);

                cancellationToken.ThrowIfCancellationRequested();

                // Verify we know how to deserialize this version of the backup log file.
                if (backupMetadataFile.footer.Version != Version)
                {
                    throw new InvalidDataException(
                              string.Format(
                                  CultureInfo.CurrentCulture,
                                  SR.Error_BackupMetadata_Deserialized,
                                  backupMetadataFile.footer.Version,
                                  Version));
                }

                // Read and validate the properties section.
                var propertiesHandle = backupMetadataFile.footer.PropertiesHandle;
                backupMetadataFile.properties =
                    await
                    FileBlock.ReadBlockAsync(
                        filestream,
                        propertiesHandle,
                        (reader, handle) => FileProperties.Read <BackupMetadataFileProperties>(reader, handle)).ConfigureAwait(false);

                cancellationToken.ThrowIfCancellationRequested();

                return(backupMetadataFile);
            }
        }