예제 #1
0
        private void GenerateEachFolderCatalog(IMailboxData mailboxData,
           IFolderData folderData,
           ICatalogDataAccess dataAccess,
           IDataConvert dataConvert,
           Stack<IFolderData> folderStack,
           IEwsAdapter EwsAdapter,
           IServiceContext serviceContext, MailboxDealInfo mailboxDealInfo,
           int level = 0)
        {
            var foldeHyPath = string.Join(" | ", (from f in folderStack select ((IFolderDataBase)f).DisplayName).Reverse().AsEnumerable());
            CurrentFolder = foldeHyPath;
            GenerateFolderStart(folderStack, mailboxData, folderData);
            bool hasError = true;
            try
            {
                if (level != 0) // root folder don't need save items;
                {
                    OnFolderProgressChanged(CatalogFolderProgressType.ChildItemStart, mailboxData, folderStack, folderData);
                    if (folderData.ChildItemCountInEx > 0)
                    {
                        OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemStart, mailboxData, folderStack, folderData);
                        List<IItemData> folderItems = EwsAdapter.GetFolderItems(folderData);
                        int itemCount = folderItems.Count;
                        OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemsEnd, mailboxData, folderStack, folderData, new Process(-1, itemCount));

                        if (itemCount > 0)
                        {
                            int itemDealedCount = 0;
                            Parallel.ForEach(source: folderItems,
                                parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = MaxConcurrentItemNumber },
                                localInit: () =>
                                {
                                    return 0;
                                },
                                body: (item, state, index, localValue) =>
                             {
                                 var dealedCountTemp = 0;
                                 long allItemIndex = 0;

                                 using (_lockObj.LockWhile(() =>
                                 {
                                     itemDealedCount++;
                                     AllItemIndex++;
                                     allItemIndex = AllItemIndex;
                                     dealedCountTemp = itemDealedCount;
                                     mailboxDealInfo.ItemCount++;
                                     ThreadData.Information = AllItemIndex.ToString("D8");
                                 }))
                                 { };

                                 CurrentFolder = foldeHyPath;
                                 ItemPercent = string.Format("{0}/{1}", itemDealedCount, itemCount);
                                 CurrentItem = item.DisplayName;
                                 //LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{1} Item {0} start.", item.Subject, index));
                                 DateTime itemStartTime = DateTime.Now;
                                 do
                                 {
                                     bool itemHasError = true;
                                     try
                                     {
                                         if (Filter.IsFilterItem(item, mailboxData, folderStack))
                                         {
                                             OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, item);
                                             break;
                                         }

                                         if (item.SizeInEx > ExportUploadHelper.MaxSupportItemSize)
                                         {
                                             LogFactory.LogInstance.WriteLog(LogLevel.WARN, "Item out of max size.", "{3:D8} {0:D8} item {1} out of max size {2}", dealedCountTemp, dealedCountTemp, ExportUploadHelper.MaxSupportItemSize, allItemIndex);
                                             break;
                                         }

                                         Interlocked.Increment(ref AllDealedItemIndex);
                                         Interlocked.Add(ref MailboxSize, item.SizeInEx);
                                         Interlocked.Add(ref mailboxDealInfo.ItemSize, item.SizeInEx);

                                         OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
                                         GenerateItemStart(item, mailboxData, folderStack, folderData);

                                         OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, item);
                                         dataAccess.SaveItem(item, mailboxData, folderData);

                                         using (_lockObj.LockWhile(() =>
                                         {
                                             folderData.ChildItemCount++;
                                         }))
                                         { };

                                         OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, item);

                                         var itemIsNew = EwsAdapter.IsItemNew(item, DateTime.MinValue, StartTime) && !dataAccess.IsItemContentExist(item.Id);
                                         if (itemIsNew)
                                         {
                                             OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, item);
                                             dataAccess.SaveItemContent(item, mailboxData.MailAddress, StartTime, true, !itemIsNew);
                                             OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, item);
                                             Interlocked.Add(ref ActualSize, item.ActualSize);
                                             Interlocked.Add(ref mailboxDealInfo.ItemActualSize, item.ActualSize);
                                         }
                                         else
                                         {
                                             OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, item);
                                         }
                                         itemHasError = false;

                                     }
                                     catch (Exception ex)
                                     {
                                         System.Diagnostics.Trace.TraceError(ex.GetExceptionDetail());
                                         var itemFailedMsg = string.Format("{3:D8} {0:D8} Item ItemId:{2} in {1} can't export.", dealedCountTemp, foldeHyPath, item.Id, allItemIndex);
                                         FailureItems.Add(itemFailedMsg);
                                         LogFactory.LogInstance.WriteException(LogInterface.LogLevel.ERR, itemFailedMsg
                                             , ex, ex.Message);
                                     }
                                     finally
                                     {
                                         GenerateItemEnd(item, mailboxData, folderStack, folderData, itemHasError);
                                         if (itemHasError)
                                             OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
                                         else
                                             OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
                                         LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{3:D8} {0:D8} item {1:D8} size {2:D16}b end", dealedCountTemp, dealedCountTemp, item.SizeInEx, allItemIndex),
                                             "TotalTime:{0}", (DateTime.Now - itemStartTime).TotalSeconds);
                                     }
                                 } while (false);

                                 return localValue;
                             }, localFinally: (localValue) =>
                             {
                             });

                            //foreach (var item in folderItems)
                            //{
                            //    itemIndex++;
                            //    DateTime itemStartTime = DateTime.Now;
                            //    IItemData itemData = dataConvert.Convert(item);
                            //    if (Filter.IsFilterItem(itemData, mailboxData, folderStack))
                            //    {
                            //        OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, itemData);
                            //        continue;
                            //    }

                            //    bool itemHasError = true;

                            //    OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
                            //    GenerateItemStart(itemData, mailboxData, folderStack, folderData);
                            //    try
                            //    {
                            //        OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, itemData);
                            //        dataAccess.SaveItem(itemData, mailboxData, folderData);
                            //        folderData.ChildItemCount++;
                            //        OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, itemData);

                            //        var itemIsNew = itemOperator.IsItemNew(item, DateTime.MinValue, StartTime);
                            //        if (itemIsNew)
                            //        {
                            //            OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, itemData);
                            //            dataAccess.SaveItemContent(itemData, mailboxData.MailAddress, StartTime, true, !itemIsNew);
                            //            OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, itemData);
                            //        }
                            //        else
                            //        {
                            //            OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, itemData);
                            //        }
                            //        itemHasError = false;
                            //    }
                            //    finally
                            //    {
                            //        GenerateItemEnd(itemData, mailboxData, folderStack, folderData, itemHasError);
                            //        if (itemHasError)
                            //            OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
                            //        else
                            //            OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);

                            //        LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{0} item end", itemIndex),
                            //            "ThreadId:{0}, TaskId:{1}, TotalTime:{2}", Thread.CurrentThread.ManagedThreadId, System.Threading.Tasks.Task.CurrentId, (DateTime.Now - itemStartTime).TotalSeconds);
                            //    }
                            //}
                        }
                        else
                        {
                            OnFolderProgressChanged(CatalogFolderProgressType.NoChildItem, mailboxData, folderStack, folderData);
                        }

                        dataAccess.SaveChanges();
                    }
                    OnFolderProgressChanged(CatalogFolderProgressType.ChildItemEnd, mailboxData, folderStack, folderData);
                }

                OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderStart, mailboxData, folderStack, folderData);
                if (folderData.ChildFolderCountInEx > 0)
                {
                    OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersStart, mailboxData, folderStack, folderData);
                    List<IFolderData> childFolders = EwsAdapter.GetChildFolder(folderData.FolderId);
                    int childFolderIndex = 0;
                    int childFolderCount = childFolders.Count;

                    CurrentFolder = foldeHyPath;

                    LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} has {1} folders:{2}.",
                        ((IFolderDataBase)folderData).DisplayName,
                        childFolderCount,
                        CurrentFolder));

                    OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersEnd, mailboxData, folderStack, folderData, null, null, new Process(-1, childFolderCount));

                    foreach (var childFolder in childFolders)
                    {
                        childFolderIndex++;

                        FolderPercent = string.Format("{0}/{1}", childFolderIndex, childFolders.Count);

                        IFolderData childFolderData = childFolder;
                        if (!EwsAdapter.IsFolderNeedGenerateCatalog(childFolder))
                        {
                            OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
                            continue;
                        }

                        if (Filter.IsFilterFolder(childFolderData, mailboxData, folderStack))
                        {
                            OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
                            continue;
                        }

                        Interlocked.Increment(ref AllDealedFolderIndex);
                        Interlocked.Increment(ref mailboxDealInfo.FolderCount);

                        folderStack.Push(childFolderData);
                        GenerateEachFolderCatalog(mailboxData, childFolderData, dataAccess, dataConvert, folderStack, EwsAdapter, serviceContext, mailboxDealInfo,level + 1);
                        folderStack.Pop();
                        //dataAccess.UpdateFolderChildFolderItemCount(childFolderData, StartTime);
                        //ServiceContext.DataAccessObj.SaveChanges();

                        OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderStart, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
                        dataAccess.SaveFolder(childFolderData, mailboxData, folderData);
                        dataAccess.SaveChanges();
                        OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderEnd, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));

                        folderData.ChildFolderCount++;
                    }

                }
                else
                {
                    OnFolderProgressChanged(CatalogFolderProgressType.NoChildFolder, mailboxData, folderStack, folderData);
                }
                OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderEnd, mailboxData, folderStack, folderData);

                hasError = false;
            }
            finally
            {
                dataAccess.SaveChanges();
                GenerateFolderEnd(folderStack, mailboxData, folderData, hasError);
            }
        }
예제 #2
0
        //private IEwsAdapter EwsAdapter { get; set; }
        /// <summary>
        /// 
        /// </summary>
        public void GenerateCatalog()
        {
            ThreadData.Information = (-1).ToString("D8");
            StartTime = DateTime.Now;
            if (Filter == null)
            {
                Filter = new NoFilter();
            }

            GenerateCatalogStart();
            bool isFinished = false;
            ICatalogDataAccess dbDataAccess = null;
            try
            {
                IDataConvert dataConvert = NewDataConvertInstance();
                dataConvert.StartTime = StartTime;
                dataConvert.OrganizationName = AdminInfo.OrganizationName;

                OnProgressChanged(CatalogProgressType.GetAllMailboxStart);
                List<IMailboxData> allUserMailbox = GetAllUserMailboxFromFilter();
                OnProgressChanged(CatalogProgressType.GetAllMailboxEnd, new Process(-1, allUserMailbox.Count));

                int mailboxIndex = 0;

                var validMailbox = new List<IMailboxData>();
                foreach (IMailboxData userMailbox in allUserMailbox)
                {
                    if (!Filter.IsFilterMailbox(userMailbox))
                    {
                        validMailbox.Add(userMailbox);
                    }
                }
                int totalMailboxCount = validMailbox.Count;

                Parallel.ForEach(validMailbox, new ParallelOptions() { MaxDegreeOfParallelism = MaxConcurrentMailboxNumber }, (userMailbox) =>
                {
                    Stack<IFolderData> folderStack = new Stack<IFolderData>(4);
                    mailboxIndex++;
                    MailboxGenerateStart(userMailbox, mailboxIndex, totalMailboxCount);

                    bool hasError = true;
                    IServiceContext serviceContext = null;
                    ICatalogDataAccess dataAccess = null;
                    var mailboxDealInfo = new MailboxDealInfo();
                    try
                    {
                        serviceContext = EwsFrame.ServiceContext.NewServiceContext(AdminInfo.UserName, AdminInfo.UserPassword, AdminInfo.UserDomain, AdminInfo.OrganizationName, TaskType.Catalog);
                        serviceContext.CurrentMailbox = userMailbox.DisplayName;
                        var ewsAdapter = CatalogFactory.Instance.NewEwsAdapter();
                        ewsAdapter.DataConvert = dataConvert;

                        dataAccess = CatalogFactory.Instance.NewCatalogDataAccessInternal(serviceContext.Argument, Organization);
                        dataAccess.OtherObj = ewsAdapter;
                        OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxStart, userMailbox);
                        serviceContext.CurrentMailbox = userMailbox.MailAddress;
                        ewsAdapter.ConnectMailbox(serviceContext.Argument, userMailbox.MailAddress);
                        OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxEnd, userMailbox);

                        OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderStart, userMailbox);
                        IFolderData rootFolder = ewsAdapter.GetRootFolder();

                        OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderEnd, userMailbox);

                        userMailbox.RootFolderId = rootFolder.Id;
                        IMailboxData mailboxData = dataConvert.Convert(userMailbox);

                        IFolderData rootFolderData = rootFolder;
                        //dataAccess.SaveFolder(rootFolderData, mailboxData, null); // root folder don't need save.

                        folderStack.Push(rootFolderData);
                        GenerateEachFolderCatalog(mailboxData,
                            rootFolderData,
                            dataAccess,
                            dataConvert,
                            folderStack,
                            ewsAdapter,
                            serviceContext, mailboxDealInfo
                            );
                        folderStack.Pop();

                        //mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
                        //dataAccess.UpdateMailboxChildFolderCount(mailboxData, StartTime);

                        OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxStart, userMailbox);
                        mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
                        dataAccess.SaveMailbox(mailboxData);
                        OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxEnd, userMailbox);

                        hasError = false;
                    }
                    catch(Exception ex)
                    {
                        LogFactory.LogInstance.WriteException(LogLevel.ERR, "maibox export error.", ex, ex.Message);
                        mailboxDealInfo.MailboxEndTime = DateTime.Now;
                        LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} catalog Job failed", userMailbox.DisplayName),
                    "Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes",
                    mailboxDealInfo.FolderCount, mailboxDealInfo.ItemCount, mailboxDealInfo.ItemSize, mailboxDealInfo.ItemActualSize, (mailboxDealInfo.MailboxEndTime - mailboxDealInfo.MailboxStartTime).TotalMinutes);
                    }

                    finally
                    {
                        dataAccess.SaveChanges();
                        MailboxGenerateEnd(userMailbox, hasError, mailboxIndex, totalMailboxCount);
                        dataAccess.Dispose();
                        serviceContext.Dispose();
                        mailboxDealInfo.MailboxEndTime = DateTime.Now;
                        LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} catalog Job completed", userMailbox.DisplayName),
                    "Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes",
                    mailboxDealInfo.FolderCount, mailboxDealInfo.ItemCount, mailboxDealInfo.ItemSize, mailboxDealInfo.ItemActualSize, (mailboxDealInfo.MailboxEndTime - mailboxDealInfo.MailboxStartTime).TotalMinutes);
                    }
                });

                //Stack<IFolderData> folderStack = new Stack<IFolderData>(4);
                //foreach (IMailboxData userMailbox in allUserMailbox)
                //{
                //    mailboxIndex++;
                //    MailboxGenerateStart(userMailbox, mailboxIndex, totalMailboxCount);

                //    if (Filter.IsFilterMailbox(userMailbox))
                //    {
                //        OnMailboxProgressChanged(CatalogMailboxProgressType.SkipMailbox, userMailbox);
                //        continue;
                //    }

                //    bool hasError = true;
                //    try
                //    {
                //        OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxStart, userMailbox);
                //        ServiceContext.CurrentMailbox = userMailbox.MailAddress;
                //        EwsAdapter.ConnectMailbox(ServiceContext.Argument, userMailbox.MailAddress);
                //        OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxEnd, userMailbox);

                //        OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderStart, userMailbox);
                //        IFolderData rootFolder = EwsAdapter.GetRootFolder();

                //        OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderEnd, userMailbox);

                //        userMailbox.RootFolderId = rootFolder.Id;
                //        IMailboxData mailboxData = dataConvert.Convert(userMailbox);

                //        IFolderData rootFolderData = rootFolder;
                //        //dataAccess.SaveFolder(rootFolderData, mailboxData, null); // root folder don't need save.

                //        folderStack.Push(rootFolderData);
                //        GenerateEachFolderCatalog(mailboxData,
                //            rootFolderData,
                //            dataAccess,
                //            dataConvert,
                //            folderStack);
                //        folderStack.Pop();

                //        //mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
                //        //dataAccess.UpdateMailboxChildFolderCount(mailboxData, StartTime);

                //        OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxStart, userMailbox);
                //        mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
                //        dataAccess.SaveMailbox(mailboxData);
                //        OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxEnd, userMailbox);

                //        hasError = false;
                //    }

                //    finally
                //    {
                //        ServiceContext.DataAccessObj.SaveChanges();
                //        MailboxGenerateEnd(userMailbox, hasError, mailboxIndex, totalMailboxCount);
                //    }
                //}
                dbDataAccess = CatalogFactory.Instance.NewCatalogDataAccessInternal(null, Organization);
                ICatalogJob job = dataConvert.Convert(this);
                dbDataAccess.SaveCatalogJob(job);

                isFinished = true;
                //SendMail("complete success", null);
            }
            catch (Exception e)
            {
                OnExceptionThrowed(e);

                System.Diagnostics.Trace.TraceError(e.GetExceptionDetail());
                var timeSpan = DateTime.Now - StartTime;
                LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, "Job failed",
                    "Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes, speed:{5} G/H.",
                    AllDealedFolderIndex, AllDealedItemIndex, MailboxSize, ActualSize, timeSpan.TotalMinutes, GetGitaEachHour(ActualSize, timeSpan));

                LogFactory.LogInstance.WriteException(LogInterface.LogLevel.ERR, "InCompleted", e, e.Message);

                //SendMail("complete with error", e);
                throw new CatalogException("Catalog Failure, please view inner Exception.", e);
            }
            finally
            {
                dbDataAccess.SaveChanges();
                var timeSpan = DateTime.Now - StartTime;
                LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, "Job completed",
                    "Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes, speed:{5} G/H.",
                    AllDealedFolderIndex, AllDealedItemIndex, MailboxSize, ActualSize, timeSpan.TotalMinutes, GetGitaEachHour(ActualSize, timeSpan));
                dbDataAccess.Dispose();
                GenerateCatalogEnd(isFinished);
            }
        }