/// <summary>
/// 获取导出对象的属性元数据
/// </summary>
/// <param name="context"></param>
/// <param name="data"></param>
/// <returns></returns>
public virtual IEnumerable <ExportProperty> Fetch(object data, IList <IExportFilter> filters)
{
//获取所有的导出属性过滤器
var propertyFilters = filters == null ? new List <IExportPropertyFilter>() : filters.Where(g => g is IExportPropertyFilter).Select(g => ((IExportPropertyFilter)g));
var props = GetExportProperties(data);
var filterProps = new List <ExportProperty>();
foreach (var prop in props)
{
//执行过滤器,如果filter返回值为true, 则不导出
if (propertyFilters.Any(g => g.Filter(prop)))
{
continue;
}
IDataConvert converter = null;
if (prop.DataConverter == null)
{
converter = ExportHelper.Converters.Get(prop.ValueType);
}
else
{
converter = Activator.CreateInstance(prop.DataConverter) as IDataConvert;
}
if (converter != null)
{
prop.Value = converter.Convert(prop.Value, data);
}
filterProps.Add(prop);
}
return(filterProps.OrderBy(g => g.Order));
}
public IDataConvert GetConvertChain(List <string> versions, Dictionary <string, string> convertInfo, string dllName)
{
IDataConvert result;
if (versions.Count != 0)
{
string text = versions[0];
object objectFromAssembly = AssemblyAssist.GetObjectFromAssembly(convertInfo[text], dllName);
IDataConvert dataConvert = objectFromAssembly as IDataConvert;
dataConvert.SetVersion(text);
versions.Remove(text);
convertInfo.Remove(text);
if (dataConvert.GetNext() == null)
{
IDataConvert convertChain = this.GetConvertChain(versions, convertInfo, dllName);
dataConvert.SetNext(convertChain);
result = dataConvert;
}
else
{
result = dataConvert;
}
}
else
{
result = null;
}
return(result);
}
public MNISTDataView(IDataConvert <double[], double> dataConverter)
{
this.dataConverter = dataConverter;
}
/// <summary>
/// constructor
/// </summary>
/// <param name="convert"></param>
public ConvertorAttribute(IDataConvert convert)
{
_c = convert;
}
public MNISTDataProcessing(IDataConvert <double[], double> dataConverter)
{
this.dataConverter = dataConverter;
}
public MNISTEvaluater(IDataConvert <double[], double> converter)
{
this.converter = converter;
}
private void GenerateEachFolderCatalog(IMailboxData mailboxData,
IFolderData folderData,
ICatalogDataAccess dataAccess,
IDataConvert dataConvert,
Stack<IFolderData> folderStack,
IEwsAdapter EwsAdapter,
IServiceContext serviceContext, MailboxDealInfo mailboxDealInfo,
int level = 0)
{
var foldeHyPath = string.Join(" | ", (from f in folderStack select ((IFolderDataBase)f).DisplayName).Reverse().AsEnumerable());
CurrentFolder = foldeHyPath;
GenerateFolderStart(folderStack, mailboxData, folderData);
bool hasError = true;
try
{
if (level != 0) // root folder don't need save items;
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildItemStart, mailboxData, folderStack, folderData);
if (folderData.ChildItemCountInEx > 0)
{
OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemStart, mailboxData, folderStack, folderData);
List<IItemData> folderItems = EwsAdapter.GetFolderItems(folderData);
int itemCount = folderItems.Count;
OnFolderProgressChanged(CatalogFolderProgressType.GetChildItemsEnd, mailboxData, folderStack, folderData, new Process(-1, itemCount));
if (itemCount > 0)
{
int itemDealedCount = 0;
Parallel.ForEach(source: folderItems,
parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = MaxConcurrentItemNumber },
localInit: () =>
{
return 0;
},
body: (item, state, index, localValue) =>
{
var dealedCountTemp = 0;
long allItemIndex = 0;
using (_lockObj.LockWhile(() =>
{
itemDealedCount++;
AllItemIndex++;
allItemIndex = AllItemIndex;
dealedCountTemp = itemDealedCount;
mailboxDealInfo.ItemCount++;
ThreadData.Information = AllItemIndex.ToString("D8");
}))
{ };
CurrentFolder = foldeHyPath;
ItemPercent = string.Format("{0}/{1}", itemDealedCount, itemCount);
CurrentItem = item.DisplayName;
//LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{1} Item {0} start.", item.Subject, index));
DateTime itemStartTime = DateTime.Now;
do
{
bool itemHasError = true;
try
{
if (Filter.IsFilterItem(item, mailboxData, folderStack))
{
OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, item);
break;
}
if (item.SizeInEx > ExportUploadHelper.MaxSupportItemSize)
{
LogFactory.LogInstance.WriteLog(LogLevel.WARN, "Item out of max size.", "{3:D8} {0:D8} item {1} out of max size {2}", dealedCountTemp, dealedCountTemp, ExportUploadHelper.MaxSupportItemSize, allItemIndex);
break;
}
Interlocked.Increment(ref AllDealedItemIndex);
Interlocked.Add(ref MailboxSize, item.SizeInEx);
Interlocked.Add(ref mailboxDealInfo.ItemSize, item.SizeInEx);
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
GenerateItemStart(item, mailboxData, folderStack, folderData);
OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, item);
dataAccess.SaveItem(item, mailboxData, folderData);
using (_lockObj.LockWhile(() =>
{
folderData.ChildItemCount++;
}))
{ };
OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, item);
var itemIsNew = EwsAdapter.IsItemNew(item, DateTime.MinValue, StartTime) && !dataAccess.IsItemContentExist(item.Id);
if (itemIsNew)
{
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, item);
dataAccess.SaveItemContent(item, mailboxData.MailAddress, StartTime, true, !itemIsNew);
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, item);
Interlocked.Add(ref ActualSize, item.ActualSize);
Interlocked.Add(ref mailboxDealInfo.ItemActualSize, item.ActualSize);
}
else
{
OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, item);
}
itemHasError = false;
}
catch (Exception ex)
{
System.Diagnostics.Trace.TraceError(ex.GetExceptionDetail());
var itemFailedMsg = string.Format("{3:D8} {0:D8} Item ItemId:{2} in {1} can't export.", dealedCountTemp, foldeHyPath, item.Id, allItemIndex);
FailureItems.Add(itemFailedMsg);
LogFactory.LogInstance.WriteException(LogInterface.LogLevel.ERR, itemFailedMsg
, ex, ex.Message);
}
finally
{
GenerateItemEnd(item, mailboxData, folderStack, folderData, itemHasError);
if (itemHasError)
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
else
OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process((int)dealedCountTemp, itemCount), item);
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{3:D8} {0:D8} item {1:D8} size {2:D16}b end", dealedCountTemp, dealedCountTemp, item.SizeInEx, allItemIndex),
"TotalTime:{0}", (DateTime.Now - itemStartTime).TotalSeconds);
}
} while (false);
return localValue;
}, localFinally: (localValue) =>
{
});
//foreach (var item in folderItems)
//{
// itemIndex++;
// DateTime itemStartTime = DateTime.Now;
// IItemData itemData = dataConvert.Convert(item);
// if (Filter.IsFilterItem(itemData, mailboxData, folderStack))
// {
// OnItemProgressChanged(CatalogItemProgressType.SkipItem, mailboxData, folderStack, folderData, itemData);
// continue;
// }
// bool itemHasError = true;
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemStart, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// GenerateItemStart(itemData, mailboxData, folderStack, folderData);
// try
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemStart, mailboxData, folderStack, folderData, itemData);
// dataAccess.SaveItem(itemData, mailboxData, folderData);
// folderData.ChildItemCount++;
// OnItemProgressChanged(CatalogItemProgressType.SaveItemEnd, mailboxData, folderStack, folderData, itemData);
// var itemIsNew = itemOperator.IsItemNew(item, DateTime.MinValue, StartTime);
// if (itemIsNew)
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentStart, mailboxData, folderStack, folderData, itemData);
// dataAccess.SaveItemContent(itemData, mailboxData.MailAddress, StartTime, true, !itemIsNew);
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEnd, mailboxData, folderStack, folderData, itemData);
// }
// else
// {
// OnItemProgressChanged(CatalogItemProgressType.SaveItemContentEndForExist, mailboxData, folderStack, folderData, itemData);
// }
// itemHasError = false;
// }
// finally
// {
// GenerateItemEnd(itemData, mailboxData, folderStack, folderData, itemHasError);
// if (itemHasError)
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndWithError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// else
// OnFolderProgressChanged(CatalogFolderProgressType.ProcessingItemEndNoError, mailboxData, folderStack, folderData, new Process(itemIndex, itemCount), itemData);
// LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.DEBUG, string.Format("{0} item end", itemIndex),
// "ThreadId:{0}, TaskId:{1}, TotalTime:{2}", Thread.CurrentThread.ManagedThreadId, System.Threading.Tasks.Task.CurrentId, (DateTime.Now - itemStartTime).TotalSeconds);
// }
//}
}
else
{
OnFolderProgressChanged(CatalogFolderProgressType.NoChildItem, mailboxData, folderStack, folderData);
}
dataAccess.SaveChanges();
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildItemEnd, mailboxData, folderStack, folderData);
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderStart, mailboxData, folderStack, folderData);
if (folderData.ChildFolderCountInEx > 0)
{
OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersStart, mailboxData, folderStack, folderData);
List<IFolderData> childFolders = EwsAdapter.GetChildFolder(folderData.FolderId);
int childFolderIndex = 0;
int childFolderCount = childFolders.Count;
CurrentFolder = foldeHyPath;
LogFactory.LogInstance.WriteLog(LogInterface.LogLevel.INFO, string.Format("{0} has {1} folders:{2}.",
((IFolderDataBase)folderData).DisplayName,
childFolderCount,
CurrentFolder));
OnFolderProgressChanged(CatalogFolderProgressType.GetChildFoldersEnd, mailboxData, folderStack, folderData, null, null, new Process(-1, childFolderCount));
foreach (var childFolder in childFolders)
{
childFolderIndex++;
FolderPercent = string.Format("{0}/{1}", childFolderIndex, childFolders.Count);
IFolderData childFolderData = childFolder;
if (!EwsAdapter.IsFolderNeedGenerateCatalog(childFolder))
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
continue;
}
if (Filter.IsFilterFolder(childFolderData, mailboxData, folderStack))
{
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderSkip, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
continue;
}
Interlocked.Increment(ref AllDealedFolderIndex);
Interlocked.Increment(ref mailboxDealInfo.FolderCount);
folderStack.Push(childFolderData);
GenerateEachFolderCatalog(mailboxData, childFolderData, dataAccess, dataConvert, folderStack, EwsAdapter, serviceContext, mailboxDealInfo,level + 1);
folderStack.Pop();
//dataAccess.UpdateFolderChildFolderItemCount(childFolderData, StartTime);
//ServiceContext.DataAccessObj.SaveChanges();
OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderStart, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
dataAccess.SaveFolder(childFolderData, mailboxData, folderData);
dataAccess.SaveChanges();
OnFolderProgressChanged(CatalogFolderProgressType.SaveFolderEnd, mailboxData, folderStack, folderData, null, null, new Process(childFolderIndex, childFolderCount));
folderData.ChildFolderCount++;
}
}
else
{
OnFolderProgressChanged(CatalogFolderProgressType.NoChildFolder, mailboxData, folderStack, folderData);
}
OnFolderProgressChanged(CatalogFolderProgressType.ChildFolderEnd, mailboxData, folderStack, folderData);
hasError = false;
}
finally
{
dataAccess.SaveChanges();
GenerateFolderEnd(folderStack, mailboxData, folderData, hasError);
}
}
public AvilaEvaluater(IDataConvert <double[], string> converter)
{
this.converter = converter;
}
public AvilaDataProcessing(IDataConvert <double[], string> dataConverter)
{
this.dataConverter = dataConverter;
}
