private void GenerateEachFolderCatalog(IMailboxData mailboxData,
IFolderData folderData,
ICatalogDataAccess dataAccess,
IDataConvert dataConvert,
Stack<IFolderData> folderStack,
IEwsAdapter EwsAdapter,
IServiceContext serviceContext, MailboxDealInfo mailboxDealInfo,
int level = 0)
{
var foldeHyPath = string.Join(" | ", (from f in folderStack select ((IFolderDataBase)f).DisplayName).Reverse().AsEnumerable());
CurrentFolder = foldeHyPath;
GenerateFolderStart(folderStack, mailboxData, folderData);
bool hasError = true;
try
{
if (level != 0) // root folder don't need save items;
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildItemStart, mailboxData, folderStack, folderData);
if (folderData.ChildItemCountInEx > 0)
{
OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemStart, mailboxData, folderStack, folderData);
List<IItemData> folderItems = EwsAdapter.GetFolderItems(folderData);
int itemCount = folderItems.Count;
OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemsEnd, mailboxData, folderStack, folderData, new Process(-1, itemCount));
if (itemCount > 0)
{
int itemDealedCount = 0;
Parallel.ForEach(source: folderItems,
parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = MaxConcurrentItemNumber },
localInit: () =>
{
return 0;
},
body: (item, state, index, localValue) =>
{
var dealedCountTemp = 0;
long allItemIndex = 0;
using (_lockObj.LockWhile(() =>
{
itemDealedCount++;
AllItemIndex++;
allItemIndex = AllItemIndex;
dealedCountTemp = itemDealedCount;
mailboxDealInfo.ItemCount++;
ThreadData.Information = AllItemIndex.ToString("D8");
}))
{ };
CurrentFolder = foldeHyPath;
ItemPercent = string.Format("{0}/{1}", itemDealedCount, itemCount);
CurrentItem = item.DisplayName;
//LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{1} Item {0} start.", item.Subject, index));
DateTime itemStartTime = DateTime.Now;
do
{
bool itemHasError = true;
try
{
if (Filter.IsFilterItem(item, mailboxData, folderStack))
{
OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, item);
break;
}
if (item.SizeInEx > ExportUploadHelper.MaxSupportItemSize)
{
LogFactory.LogInstance.WriteLog(LogLevel.WARN, "Item out of max size.", "{3:D8} {0:D8} item {1} out of max size {2}", dealedCountTemp, dealedCountTemp, ExportUploadHelper.MaxSupportItemSize, allItemIndex);
break;
}
Interlocked.Increment(ref AllDealedItemIndex);
Interlocked.Add(ref MailboxSize, item.SizeInEx);
Interlocked.Add(ref mailboxDealInfo.ItemSize, item.SizeInEx);
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
GenerateItemStart(item, mailboxData, folderStack, folderData);
OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, item);
dataAccess.SaveItem(item, mailboxData, folderData);
using (_lockObj.LockWhile(() =>
{
folderData.ChildItemCount++;
}))
{ };
OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, item);
var itemIsNew = EwsAdapter.IsItemNew(item, DateTime.MinValue, StartTime) && !dataAccess.IsItemContentExist(item.Id);
if (itemIsNew)
{
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, item);
dataAccess.SaveItemContent(item, mailboxData.MailAddress, StartTime, true, !itemIsNew);
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, item);
Interlocked.Add(ref ActualSize, item.ActualSize);
Interlocked.Add(ref mailboxDealInfo.ItemActualSize, item.ActualSize);
}
else
{
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, item);
}
itemHasError = false;
}
catch (Exception ex)
{
System.Diagnostics.Trace.TraceError(ex.GetExceptionDetail());
var itemFailedMsg = string.Format("{3:D8} {0:D8} Item ItemId:{2} in {1} can't export.", dealedCountTemp, foldeHyPath, item.Id, allItemIndex);
FailureItems.Add(itemFailedMsg);
LogFactory.LogInstance.WriteException(LogInterface.LogLevel.ERR, itemFailedMsg
, ex, ex.Message);
}
finally
{
GenerateItemEnd(item, mailboxData, folderStack, folderData, itemHasError);
if (itemHasError)
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
else
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{3:D8} {0:D8} item {1:D8} size {2:D16}b end", dealedCountTemp, dealedCountTemp, item.SizeInEx, allItemIndex),
"TotalTime:{0}", (DateTime.Now - itemStartTime).TotalSeconds);
}
} while (false);
return localValue;
}, localFinally: (localValue) =>
{
});
//foreach (var item in folderItems)
//{
// itemIndex++;
// DateTime itemStartTime = DateTime.Now;
// IItemData itemData = dataConvert.Convert(item);
// if (Filter.IsFilterItem(itemData, mailboxData, folderStack))
// {
// OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, itemData);
// continue;
// }
// bool itemHasError = true;
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// GenerateItemStart(itemData, mailboxData, folderStack, folderData);
// try
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, itemData);
// dataAccess.SaveItem(itemData, mailboxData, folderData);
// folderData.ChildItemCount++;
// OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, itemData);
// var itemIsNew = itemOperator.IsItemNew(item, DateTime.MinValue, StartTime);
// if (itemIsNew)
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, itemData);
// dataAccess.SaveItemContent(itemData, mailboxData.MailAddress, StartTime, true, !itemIsNew);
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, itemData);
// }
// else
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, itemData);
// }
// itemHasError = false;
// }
// finally
// {
// GenerateItemEnd(itemData, mailboxData, folderStack, folderData, itemHasError);
// if (itemHasError)
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// else
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{0} item end", itemIndex),
// "ThreadId:{0}, TaskId:{1}, TotalTime:{2}", Thread.CurrentThread.ManagedThreadId, System.Threading.Tasks.Task.CurrentId, (DateTime.Now - itemStartTime).TotalSeconds);
// }
//}
}
else
{
OnFolderProgressChanged(CatalogFolderProgressType.NoChildItem, mailboxData, folderStack, folderData);
}
dataAccess.SaveChanges();
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildItemEnd, mailboxData, folderStack, folderData);
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderStart, mailboxData, folderStack, folderData);
if (folderData.ChildFolderCountInEx > 0)
{
OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersStart, mailboxData, folderStack, folderData);
List<IFolderData> childFolders = EwsAdapter.GetChildFolder(folderData.FolderId);
int childFolderIndex = 0;
int childFolderCount = childFolders.Count;
CurrentFolder = foldeHyPath;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} has {1} folders:{2}.",
((IFolderDataBase)folderData).DisplayName,
childFolderCount,
CurrentFolder));
OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersEnd, mailboxData, folderStack, folderData, null, null, new Process(-1, childFolderCount));
foreach (var childFolder in childFolders)
{
childFolderIndex++;
FolderPercent = string.Format("{0}/{1}", childFolderIndex, childFolders.Count);
IFolderData childFolderData = childFolder;
if (!EwsAdapter.IsFolderNeedGenerateCatalog(childFolder))
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
continue;
}
if (Filter.IsFilterFolder(childFolderData, mailboxData, folderStack))
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
continue;
}
Interlocked.Increment(ref AllDealedFolderIndex);
Interlocked.Increment(ref mailboxDealInfo.FolderCount);
folderStack.Push(childFolderData);
GenerateEachFolderCatalog(mailboxData, childFolderData, dataAccess, dataConvert, folderStack, EwsAdapter, serviceContext, mailboxDealInfo,level + 1);
folderStack.Pop();
//dataAccess.UpdateFolderChildFolderItemCount(childFolderData, StartTime);
//ServiceContext.DataAccessObj.SaveChanges();
OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderStart, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
dataAccess.SaveFolder(childFolderData, mailboxData, folderData);
dataAccess.SaveChanges();
OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderEnd, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
folderData.ChildFolderCount++;
}
}
else
{
OnFolderProgressChanged(CatalogFolderProgressType.NoChildFolder, mailboxData, folderStack, folderData);
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderEnd, mailboxData, folderStack, folderData);
hasError = false;
}
finally
{
dataAccess.SaveChanges();
GenerateFolderEnd(folderStack, mailboxData, folderData, hasError);
}
}
//private IEwsAdapter EwsAdapter { get; set; }
/// <summary>
///
/// </summary>
public void GenerateCatalog()
{
ThreadData.Information = (-1).ToString("D8");
StartTime = DateTime.Now;
if (Filter == null)
{
Filter = new NoFilter();
}
GenerateCatalogStart();
bool isFinished = false;
ICatalogDataAccess dbDataAccess = null;
try
{
IDataConvert dataConvert = NewDataConvertInstance();
dataConvert.StartTime = StartTime;
dataConvert.OrganizationName = AdminInfo.OrganizationName;
OnProgressChanged(CatalogProgressType.GetAllMailboxStart);
List<IMailboxData> allUserMailbox = GetAllUserMailboxFromFilter();
OnProgressChanged(CatalogProgressType.GetAllMailboxEnd, new Process(-1, allUserMailbox.Count));
int mailboxIndex = 0;
var validMailbox = new List<IMailboxData>();
foreach (IMailboxData userMailbox in allUserMailbox)
{
if (!Filter.IsFilterMailbox(userMailbox))
{
validMailbox.Add(userMailbox);
}
}
int totalMailboxCount = validMailbox.Count;
Parallel.ForEach(validMailbox, new ParallelOptions() { MaxDegreeOfParallelism = MaxConcurrentMailboxNumber }, (userMailbox) =>
{
Stack<IFolderData> folderStack = new Stack<IFolderData>(4);
mailboxIndex++;
MailboxGenerateStart(userMailbox, mailboxIndex, totalMailboxCount);
bool hasError = true;
IServiceContext serviceContext = null;
ICatalogDataAccess dataAccess = null;
var mailboxDealInfo = new MailboxDealInfo();
try
{
serviceContext = EwsFrame.ServiceContext.NewServiceContext(AdminInfo.UserName, AdminInfo.UserPassword, AdminInfo.UserDomain, AdminInfo.OrganizationName, TaskType.Catalog);
serviceContext.CurrentMailbox = userMailbox.DisplayName;
var ewsAdapter = CatalogFactory.Instance.NewEwsAdapter();
ewsAdapter.DataConvert = dataConvert;
dataAccess = CatalogFactory.Instance.NewCatalogDataAccessInternal(serviceContext.Argument, Organization);
dataAccess.OtherObj = ewsAdapter;
OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxStart, userMailbox);
serviceContext.CurrentMailbox = userMailbox.MailAddress;
ewsAdapter.ConnectMailbox(serviceContext.Argument, userMailbox.MailAddress);
OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxEnd, userMailbox);
OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderStart, userMailbox);
IFolderData rootFolder = ewsAdapter.GetRootFolder();
OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderEnd, userMailbox);
userMailbox.RootFolderId = rootFolder.Id;
IMailboxData mailboxData = dataConvert.Convert(userMailbox);
IFolderData rootFolderData = rootFolder;
//dataAccess.SaveFolder(rootFolderData, mailboxData, null); // root folder don't need save.
folderStack.Push(rootFolderData);
GenerateEachFolderCatalog(mailboxData,
rootFolderData,
dataAccess,
dataConvert,
folderStack,
ewsAdapter,
serviceContext, mailboxDealInfo
);
folderStack.Pop();
//mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
//dataAccess.UpdateMailboxChildFolderCount(mailboxData, StartTime);
OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxStart, userMailbox);
mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
dataAccess.SaveMailbox(mailboxData);
OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxEnd, userMailbox);
hasError = false;
}
catch(Exception ex)
{
LogFactory.LogInstance.WriteException(LogLevel.ERR, "maibox export error.", ex, ex.Message);
mailboxDealInfo.MailboxEndTime = DateTime.Now;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} catalog Job failed", userMailbox.DisplayName),
"Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes",
mailboxDealInfo.FolderCount, mailboxDealInfo.ItemCount, mailboxDealInfo.ItemSize, mailboxDealInfo.ItemActualSize, (mailboxDealInfo.MailboxEndTime - mailboxDealInfo.MailboxStartTime).TotalMinutes);
}
finally
{
dataAccess.SaveChanges();
MailboxGenerateEnd(userMailbox, hasError, mailboxIndex, totalMailboxCount);
dataAccess.Dispose();
serviceContext.Dispose();
mailboxDealInfo.MailboxEndTime = DateTime.Now;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} catalog Job completed", userMailbox.DisplayName),
"Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes",
mailboxDealInfo.FolderCount, mailboxDealInfo.ItemCount, mailboxDealInfo.ItemSize, mailboxDealInfo.ItemActualSize, (mailboxDealInfo.MailboxEndTime - mailboxDealInfo.MailboxStartTime).TotalMinutes);
}
});
//Stack<IFolderData> folderStack = new Stack<IFolderData>(4);
//foreach (IMailboxData userMailbox in allUserMailbox)
//{
// mailboxIndex++;
// MailboxGenerateStart(userMailbox, mailboxIndex, totalMailboxCount);
// if (Filter.IsFilterMailbox(userMailbox))
// {
// OnMailboxProgressChanged(CatalogMailboxProgressType.SkipMailbox, userMailbox);
// continue;
// }
// bool hasError = true;
// try
// {
// OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxStart, userMailbox);
// ServiceContext.CurrentMailbox = userMailbox.MailAddress;
// EwsAdapter.ConnectMailbox(ServiceContext.Argument, userMailbox.MailAddress);
// OnMailboxProgressChanged(CatalogMailboxProgressType.ConnectMailboxEnd, userMailbox);
// OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderStart, userMailbox);
// IFolderData rootFolder = EwsAdapter.GetRootFolder();
// OnMailboxProgressChanged(CatalogMailboxProgressType.GetRootFolderEnd, userMailbox);
// userMailbox.RootFolderId = rootFolder.Id;
// IMailboxData mailboxData = dataConvert.Convert(userMailbox);
// IFolderData rootFolderData = rootFolder;
// //dataAccess.SaveFolder(rootFolderData, mailboxData, null); // root folder don't need save.
// folderStack.Push(rootFolderData);
// GenerateEachFolderCatalog(mailboxData,
// rootFolderData,
// dataAccess,
// dataConvert,
// folderStack);
// folderStack.Pop();
// //mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
// //dataAccess.UpdateMailboxChildFolderCount(mailboxData, StartTime);
// OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxStart, userMailbox);
// mailboxData.ChildFolderCount = rootFolderData.ChildFolderCount;
// dataAccess.SaveMailbox(mailboxData);
// OnMailboxProgressChanged(CatalogMailboxProgressType.SaveMailboxEnd, userMailbox);
// hasError = false;
// }
// finally
// {
// ServiceContext.DataAccessObj.SaveChanges();
// MailboxGenerateEnd(userMailbox, hasError, mailboxIndex, totalMailboxCount);
// }
//}
dbDataAccess = CatalogFactory.Instance.NewCatalogDataAccessInternal(null, Organization);
ICatalogJob job = dataConvert.Convert(this);
dbDataAccess.SaveCatalogJob(job);
isFinished = true;
//SendMail("complete success", null);
}
catch (Exception e)
{
OnExceptionThrowed(e);
System.Diagnostics.Trace.TraceError(e.GetExceptionDetail());
var timeSpan = DateTime.Now - StartTime;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, "Job failed",
"Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes, speed:{5} G/H.",
AllDealedFolderIndex, AllDealedItemIndex, MailboxSize, ActualSize, timeSpan.TotalMinutes, GetGitaEachHour(ActualSize, timeSpan));
LogFactory.LogInstance.WriteException(LogInterface.LogLevel.ERR, "InCompleted", e, e.Message);
//SendMail("complete with error", e);
throw new CatalogException("Catalog Failure, please view inner Exception.", e);
}
finally
{
dbDataAccess.SaveChanges();
var timeSpan = DateTime.Now - StartTime;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, "Job completed",
"Total {0} folders {1} items's size {2} bytes actual size {3} bytes cost {4} minutes, speed:{5} G/H.",
AllDealedFolderIndex, AllDealedItemIndex, MailboxSize, ActualSize, timeSpan.TotalMinutes, GetGitaEachHour(ActualSize, timeSpan));
dbDataAccess.Dispose();
GenerateCatalogEnd(isFinished);
}
}
