public CatalogItemArgs(CatalogItemProgressType type, IMailboxData mailbox, Stack<IFolderData> folderStack, IFolderData folder, IItemData currentItem)
{
this.Type = type;
this.Mailbox = mailbox;
this.FolderStack = folderStack;
this.Folder = folder;
this.CurrentItem = currentItem;
}
public CatalogFolderArgs(CatalogFolderProgressType type, IMailboxData currentMailbox, Stack<IFolderData> folderStack, IFolderData currentFolder, Process itemProcess, IItemData currentItem, Process childFolderProcess)
{
this.Type = type;
this.CurrentMailbox = currentMailbox;
this.FolderStack = folderStack;
this.CurrentFolder = currentFolder;
this.ItemProcess = itemProcess;
this.CurrentItem = currentItem;
this.ChildFolderProcess = childFolderProcess;
}
public IMailboxData Convert(IMailboxData mailbox)
{
MailboxModel model = new MailboxModel();
model.DisplayName = mailbox.DisplayName;
model.MailAddress = mailbox.MailAddress;
model.RootFolderId = mailbox.RootFolderId;
model.StartTime = StartTime;
model.ChildFolderCount = 0;
return model;
}
public bool IsFilterFolder(IFolderData currentFolder, IMailboxData mailbox, Stack<IFolderData> folders)
{
var mailboxItem = _orgSelectedItemUtils.GetChild(mailbox.Id);
if (mailboxItem == null)
{
mailboxItem = _orgSelectedItemUtils.GetChildByDisplayName(mailbox.DisplayName);
if (mailboxItem == null)
return true;
}
if (mailboxItem.Item.Status == (int)SelectedItemStatus.Selected)
return false;
else if (mailboxItem.Item.Status == (int)SelectedItemStatus.UnSelected)
return true;
else
{
List<IFolderData> list = new List<IFolderData>(folders.Count + 1);
list.Add(currentFolder);
list.AddRange(folders);
Stack<IFolderData> stack = new Stack<IFolderData>(list);
int index = 0;
LoadedTreeItemUtil folderItemUtil = mailboxItem;
foreach (var folder in stack)
{
if (index++ == 0) // skip the rootfolder.
continue;
folderItemUtil = folderItemUtil.GetChild(folder.Id);
if (folderItemUtil == null)
{
folderItemUtil = folderItemUtil.GetChildByDisplayName(((IItemBase)folder).DisplayName);
}
if (folderItemUtil == null)
return true;
if (folderItemUtil.Item.Status == (int)SelectedItemStatus.Selected && folderItemUtil.Item.CanSelect == 1)
return false;
else if (folderItemUtil.Item.Status == (int)SelectedItemStatus.UnSelected)
return true;
else
continue;
}
}
return true;
}
public void SaveItem(IItemData item, IMailboxData mailboxData, IFolderData parentFolderData)
{
ItemEntity entity = item as ItemEntity;
if (entity == null)
throw new NullReferenceException("ItemEntity");
MailboxEntity mailboxEntity = mailboxData as MailboxEntity;
if (mailboxEntity == null)
throw new NullReferenceException("mailboxEntity");
string orignizeName = mailboxEntity.OrganizationName;
string tableName = ItemEntity.GetItemTableName(orignizeName, parentFolderData.FolderId, entity.StartTime);
tableName = TableDataAccess.ValidateTableName(tableName);
TableDataAccess tableDataAccess = new TableDataAccess(TableClient);
CloudTable table = tableDataAccess.CreateIfNotExist(tableName);
TableResult tableResult = tableDataAccess.InsertEntity(table, entity);
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, "Insert item to table", "Insert item {0} to table {1} result, Etag:{2}, HttpStatusCode:{3},",
item.DisplayName, tableName, tableResult.Etag, tableResult.HttpStatusCode);
}
private void AddToResult(IMailboxData data, List<Item> infos, int position = -1)
{
if (position == -1)
infos.Add(new Item()
{
Id = data.MailAddress,
DisplayName = data.DisplayName,
ChildCount = int.MaxValue,
ItemType = ItemTypeStr.Mailbox,
CanSelect = 0,
OtherInformation = data
});
else
{
infos.Insert(position, new Item()
{
Id = data.MailAddress,
DisplayName = data.DisplayName,
ChildCount = int.MaxValue,
ItemType = ItemTypeStr.Mailbox,
CanSelect = 1,
OtherInformation = data
});
}
}
public bool IsFilterItem(IItemData item, IMailboxData mailbox, Stack<IFolderData> folders)
{
return false;
}
private void GenerateItemStart(IItemData item, IMailboxData mailboxData, Stack<IFolderData> folderStack, IFolderData folderData)
{
OnItemProgressChanged(CatalogItemProgressType.Start, mailboxData, folderStack, folderData, item);
}
private void GenerateFolderStart(Stack<IFolderData> folderStack, IMailboxData mailboxData, IFolderData folderData)
{
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} folder start.", ((IFolderDataBase)folderData).DisplayName));
OnFolderProgressChanged(CatalogFolderProgressType.Start, mailboxData, folderStack, folderData);
//FolderPercent = "";
}
public void SaveMailbox(IMailboxData mailboxAddress)
{
throw new NotImplementedException();
}
public void SaveItem(IItemData item, IMailboxData mailboxData, IFolderData parentFolderData)
{
throw new NotImplementedException();
}
public void SaveFolder(IFolderData folder, IMailboxData mailboxData, IFolderData parentFolderData)
{
throw new NotImplementedException();
}
public void UpdateMailboxChildFolderCount(IMailboxData mailboxData, DateTime startTime)
{
throw new NotImplementedException();
}
public void SaveMailbox(IMailboxData mailboxAddress)
{
MailboxEntity entity = mailboxAddress as MailboxEntity;
if (entity == null)
throw new NullReferenceException("mailboxEntity");
string tableName = TableDataAccess.ValidateTableName(MailboxEntity.GetMailTableName(entity.OrganizationName, entity.StartTime));
TableDataAccess tableDataAccess = new TableDataAccess(TableClient);
CloudTable table = tableDataAccess.CreateIfNotExist(tableName);
TableResult tableResult = tableDataAccess.InsertEntity(table, entity);
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, "Insert mailbox to table", "Insert mailbox {0} to table {1} result, Etag:{2}, HttpStatusCode:{3},",
mailboxAddress.DisplayName, tableName, tableResult.Etag, tableResult.HttpStatusCode);
}
private void GenerateEachFolderCatalog(IMailboxData mailboxData,
IFolderData folderData,
ICatalogDataAccess dataAccess,
IDataConvert dataConvert,
Stack<IFolderData> folderStack,
IEwsAdapter EwsAdapter,
IServiceContext serviceContext, MailboxDealInfo mailboxDealInfo,
int level = 0)
{
var foldeHyPath = string.Join(" | ", (from f in folderStack select ((IFolderDataBase)f).DisplayName).Reverse().AsEnumerable());
CurrentFolder = foldeHyPath;
GenerateFolderStart(folderStack, mailboxData, folderData);
bool hasError = true;
try
{
if (level != 0) // root folder don't need save items;
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildItemStart, mailboxData, folderStack, folderData);
if (folderData.ChildItemCountInEx > 0)
{
OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemStart, mailboxData, folderStack, folderData);
List<IItemData> folderItems = EwsAdapter.GetFolderItems(folderData);
int itemCount = folderItems.Count;
OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemsEnd, mailboxData, folderStack, folderData, new Process(-1, itemCount));
if (itemCount > 0)
{
int itemDealedCount = 0;
Parallel.ForEach(source: folderItems,
parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = MaxConcurrentItemNumber },
localInit: () =>
{
return 0;
},
body: (item, state, index, localValue) =>
{
var dealedCountTemp = 0;
long allItemIndex = 0;
using (_lockObj.LockWhile(() =>
{
itemDealedCount++;
AllItemIndex++;
allItemIndex = AllItemIndex;
dealedCountTemp = itemDealedCount;
mailboxDealInfo.ItemCount++;
ThreadData.Information = AllItemIndex.ToString("D8");
}))
{ };
CurrentFolder = foldeHyPath;
ItemPercent = string.Format("{0}/{1}", itemDealedCount, itemCount);
CurrentItem = item.DisplayName;
//LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{1} Item {0} start.", item.Subject, index));
DateTime itemStartTime = DateTime.Now;
do
{
bool itemHasError = true;
try
{
if (Filter.IsFilterItem(item, mailboxData, folderStack))
{
OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, item);
break;
}
if (item.SizeInEx > ExportUploadHelper.MaxSupportItemSize)
{
LogFactory.LogInstance.WriteLog(LogLevel.WARN, "Item out of max size.", "{3:D8} {0:D8} item {1} out of max size {2}", dealedCountTemp, dealedCountTemp, ExportUploadHelper.MaxSupportItemSize, allItemIndex);
break;
}
Interlocked.Increment(ref AllDealedItemIndex);
Interlocked.Add(ref MailboxSize, item.SizeInEx);
Interlocked.Add(ref mailboxDealInfo.ItemSize, item.SizeInEx);
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
GenerateItemStart(item, mailboxData, folderStack, folderData);
OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, item);
dataAccess.SaveItem(item, mailboxData, folderData);
using (_lockObj.LockWhile(() =>
{
folderData.ChildItemCount++;
}))
{ };
OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, item);
var itemIsNew = EwsAdapter.IsItemNew(item, DateTime.MinValue, StartTime) && !dataAccess.IsItemContentExist(item.Id);
if (itemIsNew)
{
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, item);
dataAccess.SaveItemContent(item, mailboxData.MailAddress, StartTime, true, !itemIsNew);
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, item);
Interlocked.Add(ref ActualSize, item.ActualSize);
Interlocked.Add(ref mailboxDealInfo.ItemActualSize, item.ActualSize);
}
else
{
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, item);
}
itemHasError = false;
}
catch (Exception ex)
{
System.Diagnostics.Trace.TraceError(ex.GetExceptionDetail());
var itemFailedMsg = string.Format("{3:D8} {0:D8} Item ItemId:{2} in {1} can't export.", dealedCountTemp, foldeHyPath, item.Id, allItemIndex);
FailureItems.Add(itemFailedMsg);
LogFactory.LogInstance.WriteException(LogInterface.LogLevel.ERR, itemFailedMsg
, ex, ex.Message);
}
finally
{
GenerateItemEnd(item, mailboxData, folderStack, folderData, itemHasError);
if (itemHasError)
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
else
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{3:D8} {0:D8} item {1:D8} size {2:D16}b end", dealedCountTemp, dealedCountTemp, item.SizeInEx, allItemIndex),
"TotalTime:{0}", (DateTime.Now - itemStartTime).TotalSeconds);
}
} while (false);
return localValue;
}, localFinally: (localValue) =>
{
});
//foreach (var item in folderItems)
//{
// itemIndex++;
// DateTime itemStartTime = DateTime.Now;
// IItemData itemData = dataConvert.Convert(item);
// if (Filter.IsFilterItem(itemData, mailboxData, folderStack))
// {
// OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, itemData);
// continue;
// }
// bool itemHasError = true;
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// GenerateItemStart(itemData, mailboxData, folderStack, folderData);
// try
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, itemData);
// dataAccess.SaveItem(itemData, mailboxData, folderData);
// folderData.ChildItemCount++;
// OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, itemData);
// var itemIsNew = itemOperator.IsItemNew(item, DateTime.MinValue, StartTime);
// if (itemIsNew)
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, itemData);
// dataAccess.SaveItemContent(itemData, mailboxData.MailAddress, StartTime, true, !itemIsNew);
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, itemData);
// }
// else
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, itemData);
// }
// itemHasError = false;
// }
// finally
// {
// GenerateItemEnd(itemData, mailboxData, folderStack, folderData, itemHasError);
// if (itemHasError)
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// else
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{0} item end", itemIndex),
// "ThreadId:{0}, TaskId:{1}, TotalTime:{2}", Thread.CurrentThread.ManagedThreadId, System.Threading.Tasks.Task.CurrentId, (DateTime.Now - itemStartTime).TotalSeconds);
// }
//}
}
else
{
OnFolderProgressChanged(CatalogFolderProgressType.NoChildItem, mailboxData, folderStack, folderData);
}
dataAccess.SaveChanges();
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildItemEnd, mailboxData, folderStack, folderData);
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderStart, mailboxData, folderStack, folderData);
if (folderData.ChildFolderCountInEx > 0)
{
OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersStart, mailboxData, folderStack, folderData);
List<IFolderData> childFolders = EwsAdapter.GetChildFolder(folderData.FolderId);
int childFolderIndex = 0;
int childFolderCount = childFolders.Count;
CurrentFolder = foldeHyPath;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} has {1} folders:{2}.",
((IFolderDataBase)folderData).DisplayName,
childFolderCount,
CurrentFolder));
OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersEnd, mailboxData, folderStack, folderData, null, null, new Process(-1, childFolderCount));
foreach (var childFolder in childFolders)
{
childFolderIndex++;
FolderPercent = string.Format("{0}/{1}", childFolderIndex, childFolders.Count);
IFolderData childFolderData = childFolder;
if (!EwsAdapter.IsFolderNeedGenerateCatalog(childFolder))
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
continue;
}
if (Filter.IsFilterFolder(childFolderData, mailboxData, folderStack))
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
continue;
}
Interlocked.Increment(ref AllDealedFolderIndex);
Interlocked.Increment(ref mailboxDealInfo.FolderCount);
folderStack.Push(childFolderData);
GenerateEachFolderCatalog(mailboxData, childFolderData, dataAccess, dataConvert, folderStack, EwsAdapter, serviceContext, mailboxDealInfo,level + 1);
folderStack.Pop();
//dataAccess.UpdateFolderChildFolderItemCount(childFolderData, StartTime);
//ServiceContext.DataAccessObj.SaveChanges();
OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderStart, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
dataAccess.SaveFolder(childFolderData, mailboxData, folderData);
dataAccess.SaveChanges();
OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderEnd, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
folderData.ChildFolderCount++;
}
}
else
{
OnFolderProgressChanged(CatalogFolderProgressType.NoChildFolder, mailboxData, folderStack, folderData);
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderEnd, mailboxData, folderStack, folderData);
hasError = false;
}
finally
{
dataAccess.SaveChanges();
GenerateFolderEnd(folderStack, mailboxData, folderData, hasError);
}
}
private void GenerateFolderEnd(Stack<IFolderData> folderStack, IMailboxData mailboxData, IFolderData folderData, bool hasError)
{
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} folder end.", ((IFolderDataBase)folderData).DisplayName));
if (!hasError)
OnFolderProgressChanged(CatalogFolderProgressType.EndWithNoError, mailboxData, folderStack, folderData);
else
OnFolderProgressChanged(CatalogFolderProgressType.EndWithError, mailboxData, folderStack, folderData);
}
private void OnFolderProgressChanged(CatalogFolderProgressType type,
IMailboxData currentMailbox,
Stack<IFolderData> folderStack = null,
IFolderData currentFolder = null,
Process itemProcess = null,
IItemData currentItem = null,
Process childFolderProcess = null)
{
if (FolderProgressChanged != null)
{
try
{
FolderProgressChanged(this, new CatalogFolderArgs(type, currentMailbox, folderStack, currentFolder, itemProcess, currentItem, childFolderProcess));
}
catch (Exception e)
{
System.Diagnostics.Trace.TraceError(e.GetExceptionDetail());
OnExceptionThrowed(e);
}
}
}
private void GenerateItemEnd(IItemData item, IMailboxData mailboxData, Stack<IFolderData> folderStack, IFolderData folderData, bool itemHasError)
{
if (itemHasError)
OnItemProgressChanged(CatalogItemProgressType.EndWithError, mailboxData, folderStack, folderData, item);
else
OnItemProgressChanged(CatalogItemProgressType.EndWithNoError, mailboxData, folderStack, folderData, item);
}
private void OnItemProgressChanged(CatalogItemProgressType type, IMailboxData mailbox, Stack<IFolderData> folderStack, IFolderData folder, IItemData currentItem = null)
{
if (ItemProgressChanged != null)
{
try
{
ItemProgressChanged(this, new CatalogItemArgs(type, mailbox, folderStack, folder, currentItem));
}
catch (Exception e)
{
System.Diagnostics.Trace.TraceError(e.GetExceptionDetail());
OnExceptionThrowed(e);
}
}
}
public IMailboxData Convert(IMailboxData mailboxData)
{
return MailboxEntity.CreateMailboxEntityFromEws(OrganizationName, mailboxData.DisplayName, mailboxData.MailAddress, StartTime);
}
private void OnProgressChanged(CatalogProgressType type, Process mailboxProcess = null, IMailboxData mailbox = null)
{
if (ProgressChanged != null)
{
try
{
ProgressChanged(this, new CatalogProgressArgs(type, mailboxProcess, mailbox));
}
catch (Exception e)
{
System.Diagnostics.Trace.TraceError(e.GetExceptionDetail());
OnExceptionThrowed(e);
}
}
}
public static string GetMailboxPath(string sessionFolder, IMailboxData mailboxData)
{
throw new NotImplementedException();
}
public bool IsFilterFolder(IFolderData currentFolder, IMailboxData mailbox, Stack<IFolderData> folders)
{
return false;
}
public bool IsFilterMailbox(IMailboxData mailbox)
{
return false;
}
public void MailboxGenerateEnd(IMailboxData mailbox, bool hasError, int mailboxIndex, int mailboxCount)
{
if (!hasError)
{
MailboxCacheManager.CacheManager.ReleaseCache(true);
OnProgressChanged(CatalogProgressType.GRTForMailboxEndWithNoError, new Process(mailboxIndex, mailboxCount), mailbox);
OnMailboxProgressChanged(CatalogMailboxProgressType.EndWithNoError, mailbox);
}
else
{
MailboxCacheManager.CacheManager.ReleaseCache();
OnProgressChanged(CatalogProgressType.GRTForMailboxEndWithError, new Process(mailboxIndex, mailboxCount), mailbox);
OnMailboxProgressChanged(CatalogMailboxProgressType.EndWithError, mailbox);
}
}
public bool IsFilterMailbox(IMailboxData mailbox)
{
var id = mailbox.Id;
if (string.IsNullOrEmpty(id))
{
id = mailbox.MailAddress;
}
var mailboxItem = _orgSelectedItemUtils.GetChild(id);
if (mailboxItem == null)
{
mailboxItem = _orgSelectedItemUtils.GetChildByDisplayName(mailbox.DisplayName);
if (mailboxItem == null)
return true;
}
if (mailboxItem.Item.Status == (int)SelectedItemStatus.Selected && mailboxItem.Item.CanSelect == 1)
return false;
else if (mailboxItem.Item.Status == (int)SelectedItemStatus.UnSelected)
return true;
return false;
}
public void MailboxGenerateStart(IMailboxData mailbox, int mailboxIndex, int mailboxCount)
{
OnProgressChanged(CatalogProgressType.GRTForMailboxStart, new Process(mailboxIndex, mailboxCount), mailbox);
OnMailboxProgressChanged(CatalogMailboxProgressType.Start, mailbox);
CurrentMailbox = mailbox.MailAddress;
MailboxPercent = string.Format("{0}/{1}", mailboxIndex, mailboxCount);
MailboxCacheManager.CacheManager.ReleaseCache();
}
public CatalogMailboxArgs(CatalogMailboxProgressType type, IMailboxData mailbox)
{
this.Type = type;
this.Mailbox = mailbox;
}
public void SaveFolder(IFolderData folder, IMailboxData mailboxData, IFolderData parentFolder)
{
FolderEntity entity = folder as FolderEntity;
if (entity == null)
throw new NullReferenceException("FolderEntity");
MailboxEntity mailboxEntity = mailboxData as MailboxEntity;
if (mailboxEntity == null)
throw new NullReferenceException("mailboxEntity");
string orignizeName = mailboxEntity.OrganizationName;
string tableName = FolderEntity.GetFolderTableName(orignizeName, mailboxData.MailAddress, entity.StartTime);
tableName = TableDataAccess.ValidateTableName(tableName);
TableDataAccess tableDataAccess = new TableDataAccess(TableClient);
CloudTable table = tableDataAccess.CreateIfNotExist(tableName);
entity.Location = tableName;
TableResult tableResult = tableDataAccess.InsertEntity(table, entity);
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, "Insert folder to table", "Insert folder {0} to table {1} result, Etag:{2}, HttpStatusCode:{3},",
((IItemBase)folder).DisplayName, tableName, tableResult.Etag, tableResult.HttpStatusCode);
}
