/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += Execution.GetWordCount();
wordCount += Pipe.GetWordCount();
wordCount += NumPackets.GetWordCount();
wordCount += PacketSize.GetWordCount();
wordCount += PacketAlignment.GetWordCount();
return(wordCount);
}
public void Generate_grows_success()
{
var g = new EasyGenerator(10);
IdResult oldIdResult = new IdResult();
for (int i = 0; i < 5000000; i++)
{
IdResult newIdResult = g.GetIdResult();
Assert.True(newIdResult.GenerateId() > oldIdResult.GenerateId(), $"{newIdResult}>{oldIdResult}???");
oldIdResult = newIdResult;
}
;
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += SrcImage.GetWordCount();
wordCount += FwdRefImage.GetWordCount();
wordCount += BwdRefImage.GetWordCount();
wordCount += Payload.GetWordCount();
wordCount += StreaminComponents.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += NDRange.GetWordCount();
wordCount += Invoke.GetWordCount();
wordCount += Param.GetWordCount();
wordCount += ParamSize.GetWordCount();
wordCount += ParamAlign.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += SampledImage.GetWordCount();
wordCount += Coordinate.GetWordCount();
wordCount += Granularity.GetWordCount();
wordCount += Coarse.GetWordCount();
wordCount += ImageOperands?.GetWordCount() ?? 0u;
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += Execution.GetWordCount();
wordCount += Destination.GetWordCount();
wordCount += Source.GetWordCount();
wordCount += NumElements.GetWordCount();
wordCount += Stride.GetWordCount();
wordCount += Event.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += Pipe.GetWordCount();
wordCount += ReserveId.GetWordCount();
wordCount += Index.GetWordCount();
wordCount += Pointer.GetWordCount();
wordCount += PacketSize.GetWordCount();
wordCount += PacketAlignment.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += SkipBlockPartitionType.GetWordCount();
wordCount += SkipMotionVectorMask.GetWordCount();
wordCount += MotionVectors.GetWordCount();
wordCount += BidirectionalWeight.GetWordCount();
wordCount += SadAdjustment.GetWordCount();
wordCount += Payload.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += Pointer.GetWordCount();
wordCount += Memory.GetWordCount();
wordCount += Equal.GetWordCount();
wordCount += Unequal.GetWordCount();
wordCount += Value.GetWordCount();
wordCount += Comparator.GetWordCount();
return(wordCount);
}
private async Task <Guid?> GetIdAsync(Type matchingType, MethodInfo matchingMethod)
{
foreach (IIdMapper idMapper in _idMappers)
{
IdResult result = await idMapper.MapAsync(matchingType, matchingMethod);
if (result.ResultType == IdResultType.IdMapped)
{
return(result.Id);
}
}
return(null);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResult.GetWordCount();
wordCount += SampledType.GetWordCount();
wordCount += Dim.GetWordCount();
wordCount += Depth.GetWordCount();
wordCount += Arrayed.GetWordCount();
wordCount += MS.GetWordCount();
wordCount += Sampled.GetWordCount();
wordCount += ImageFormat.GetWordCount();
wordCount += AccessQualifier?.GetWordCount() ?? 0u;
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += SrcCoord.GetWordCount();
wordCount += MotionVectors.GetWordCount();
wordCount += MajorShapes.GetWordCount();
wordCount += MinorShapes.GetWordCount();
wordCount += Direction.GetWordCount();
wordCount += PixelResolution.GetWordCount();
wordCount += SadAdjustment.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += LumaIntraPartitionMask.GetWordCount();
wordCount += IntraNeighbourAvailabilty.GetWordCount();
wordCount += LeftEdgeLumaPixels.GetWordCount();
wordCount += UpperLeftCornerLumaPixel.GetWordCount();
wordCount += UpperEdgeLumaPixels.GetWordCount();
wordCount += UpperRightEdgeLumaPixels.GetWordCount();
wordCount += SadAdjustment.GetWordCount();
wordCount += Payload.GetWordCount();
return(wordCount);
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResultType.GetWordCount();
wordCount += IdResult.GetWordCount();
wordCount += Queue.GetWordCount();
wordCount += Flags.GetWordCount();
wordCount += NDRange.GetWordCount();
wordCount += NumEvents.GetWordCount();
wordCount += WaitEvents.GetWordCount();
wordCount += RetEvent.GetWordCount();
wordCount += Invoke.GetWordCount();
wordCount += Param.GetWordCount();
wordCount += ParamSize.GetWordCount();
wordCount += ParamAlign.GetWordCount();
wordCount += LocalSize.GetWordCount();
return(wordCount);
}
public void SetUp()
{
_classFilter = MockRepository.GenerateMock <IClassFilter>();
_classFilter
.Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask())
.Return(false.AsCompletedTask());
_idMapper = MockRepository.GenerateMock <IIdMapper>();
_idMapper
.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask())
.Return(null);
_nameMapper = MockRepository.GenerateMock <INameMapper>();
_nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask());
_resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>();
_resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask());
_autoRouteCollection = new AutoRouteCollection(true)
.Assemblies(Assembly.GetExecutingAssembly())
.ClassFilters(_classFilter)
.NameMappers(_nameMapper)
.IdMappers(_idMapper)
.ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper);
}
public void SetUp()
{
_result = IdResult.IdNotMapped();
}
public void SetUp()
{
_classFilter = MockRepository.GenerateMock <IClassFilter>();
_classFilter
.Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask())
.Return(false.AsCompletedTask());
_idMapper = MockRepository.GenerateMock <IIdMapper>();
_idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask());
_nameMapper = MockRepository.GenerateMock <INameMapper>();
_nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask());
_resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>();
_resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask());
_authenticationProvider = MockRepository.GenerateMock <IAuthenticationProvider>();
_authenticationProvider
.Stub(arg => arg.AuthenticateAsync(Arg <HttpRequestBase> .Is.Anything, Arg <HttpResponseBase> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything))
.Return(AuthenticationResult.AuthenticationSucceeded.AsCompletedTask());
_authenticationStrategy = MockRepository.GenerateMock <IAuthenticationStrategy>();
_authenticationStrategy.Stub(arg => arg.MustAuthenticateAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(true.AsCompletedTask());
_autoRouteCollection = new AutoRouteCollection()
.Assemblies(Assembly.GetExecutingAssembly())
.ClassFilters(_classFilter)
.NameMappers(_nameMapper)
.IdMappers(_idMapper)
.ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper)
.Authenticate(_authenticationProvider, _authenticationStrategy);
_routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray();
}
public void SetUp()
{
_classFilter = MockRepository.GenerateMock <IClassFilter>();
_classFilter
.Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask())
.Return(false.AsCompletedTask());
_idMapper = MockRepository.GenerateMock <IIdMapper>();
_idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask());
_nameMapper = MockRepository.GenerateMock <INameMapper>();
_nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask());
_resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>();
_resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask());
_responseMapper = MockRepository.GenerateMock <IResponseMapper>();
_responseMapper
.Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything))
.Return(Task.Factory.Empty());
_restrictionMapper1 = MockRepository.GenerateMock <IRestrictionMapper>();
_restrictionMapper1
.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything, Arg <IContainer> .Is.Anything))
.WhenCalled(arg => ((Route.Routing.Route)arg.Arguments.Skip(2).First()).RestrictByMethods("GET").AsCompletedTask())
.Return(Task.Factory.Empty());
_restrictionMapper2 = MockRepository.GenerateMock <IRestrictionMapper>();
_restrictionMapper2
.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything, Arg <IContainer> .Is.Anything))
.WhenCalled(arg => ((Route.Routing.Route)arg.Arguments.Skip(2).First()).RestrictByMethods("POST").AsCompletedTask())
.Return(Task.Factory.Empty());
_autoRouteCollection = new AutoRouteCollection()
.Assemblies(Assembly.GetExecutingAssembly())
.ClassFilters(_classFilter)
.NameMappers(_nameMapper)
.IdMappers(_idMapper)
.ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper)
.ResponseMapper(_responseMapper)
.RestrictionMappers(_restrictionMapper1, _restrictionMapper2);
}
public void SetUp()
{
_classFilter = MockRepository.GenerateMock <IClassFilter>();
_classFilter
.Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask())
.Return(false.AsCompletedTask());
_idMapper1 = MockRepository.GenerateMock <IIdMapper>();
_idMapper1.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.Parse("1dffe3ee-1ade-4aa2-835a-9cb91b7e31c4")).AsCompletedTask());
_idMapper2 = MockRepository.GenerateMock <IIdMapper>();
_idMapper2.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.Parse("493e725c-cbc1-4ea4-b6d1-350018d4542d")).AsCompletedTask());
_nameMapper = MockRepository.GenerateMock <INameMapper>();
_nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask());
_resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>();
_resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask());
_responseMapper = MockRepository.GenerateMock <IResponseMapper>();
_responseMapper
.Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything))
.Return(Task.Factory.Empty());
_autoRouteCollection = new AutoRouteCollection()
.Assemblies(Assembly.GetExecutingAssembly())
.ClassFilters(_classFilter)
.NameMappers(_nameMapper)
.IdMappers(_idMapper1)
.ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper)
.ResponseMapper(_responseMapper);
_routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray();
}
public void SetUp()
{
_classFilter = MockRepository.GenerateMock <IClassFilter>();
_classFilter
.Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask())
.Return(false.AsCompletedTask());
_idMapper = MockRepository.GenerateMock <IIdMapper>();
_idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask());
_nameMapper = MockRepository.GenerateMock <INameMapper>();
_nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped(Guid.NewGuid().ToString()).AsCompletedTask());
_resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>();
_resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask());
_responseMapper = MockRepository.GenerateMock <IResponseMapper>();
_responseMapper
.Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything))
.Return(Task.Factory.Empty());
_httpRuntime = MockRepository.GenerateMock <IHttpRuntime>();
_autoRouteCollection = new AutoRouteCollection()
.Assemblies(typeof(Endpoint).Assembly)
.ClassFilters(_classFilter)
.NameMappers(_nameMapper)
.IdMappers(_idMapper)
.ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper)
.RestrictUsingAttributes <UrlRelativePathAttribute>()
.RestrictRelativePathsToRelativeClassNamespaceAndClassName("")
.ResponseMapper(_responseMapper)
.RestrictionContainer(new DefaultRestrictionContainer(_httpRuntime));
_routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray();
}
public void SetUp()
{
_classFilter = MockRepository.GenerateMock <IClassFilter>();
_classFilter
.Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask())
.Return(false.AsCompletedTask());
_methodFilter = MockRepository.GenerateMock <IMethodFilter>();
_methodFilter
.Stub(arg => arg.MatchesAsync(Arg <MethodInfo> .Is.Anything))
.WhenCalled(arg => arg.ReturnValue = (((MethodInfo)arg.Arguments.First()).Name == "IncludedMethod").AsCompletedTask())
.Return(false.AsCompletedTask());
_idMapper = MockRepository.GenerateMock <IIdMapper>();
_idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask());
_nameMapper = MockRepository.GenerateMock <INameMapper>();
_nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask());
_resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>();
_resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask());
_responseMapper = MockRepository.GenerateMock <IResponseMapper>();
_responseMapper
.Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything))
.Return(Task.Factory.Empty());
_autoRouteCollection = new AutoRouteCollection()
.Assemblies(Assembly.GetExecutingAssembly())
.ClassFilters(_classFilter)
.MethodFilters(_methodFilter)
.NameMappers(_nameMapper)
.IdMappers(_idMapper)
.ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper)
.ResponseMapper(_responseMapper);
_routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray();
}
/// <summary>
/// Calculate number of words to fit complete instruction bytecode.
/// </summary>
/// <returns>Number of words in instruction bytecode.</returns>
public override uint GetWordCount()
{
uint wordCount = 0;
wordCount += IdResult.GetWordCount();
return wordCount;
}
//
// Threading:
//
#region private void WorkerThread()
private void WorkerThread()
{
try
{
var allFiles = Directory.GetFiles(_rootPath, "*", SearchOption.AllDirectories);
var totalFiles = allFiles.Length;
var totalStopWatch = Stopwatch.StartNew();
var uniqueIdSet = new ConcurrentDictionary <Id, int>();
Dispatcher.Invoke((Action) delegate
{
try
{
_progressBar.Visibility = Visibility.Visible;
_progressBar.Value = 0;
}
catch { }
});
var lastUpdateStopWatch = Stopwatch.StartNew();
//
// Parallelize the file format identification:
//
Parallel.ForEach(allFiles, _parallelForOptions, file =>
{
try
{
IdResult docFormat = null;
// WARNING: This example does not support long path names (> 255 chars) - .NET solutions for getting valid FileStreams for 'long file paths'
// can be found on the internet - also .NET 4.6.2 supports long file paths (web search for how to enable)
// Note: Minimum recommended buffer size of 16kb for file identification
using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 16384))
{
docFormat = DocumentIdentifier.Identify(stream, file);
}
uniqueIdSet[docFormat.ID] = 1;
var docIdResult = new DocumentIdResult(file, docFormat);
if (docFormat.ID == Id.Unknown)
{
Interlocked.Increment(ref _totalUnknownFiles);
}
_resultsQueue.Enqueue(docIdResult);
var numFilesIdentified = _resultsQueue.Count;
var percentComplete = 100.0 * numFilesIdentified / totalFiles;
// Update progress bar and status every 250 [ms]:
if (lastUpdateStopWatch.ElapsedMilliseconds >= 250)
{
lastUpdateStopWatch.Restart();
Dispatcher.BeginInvoke((Action) delegate
{
try
{
_progressBar.Value = percentComplete;
_statusTextBox.Text = string.Format(" {0} files out of {1} identified...", numFilesIdentified, totalFiles);
}
catch { }
});
}
}
catch (Exception ex)
{
Interlocked.Increment(ref _totalExceptions);
var docIDResult = new DocumentIdResult(file, DocumentIdentifier.UnknownResult);
docIDResult.HasError = true;
docIDResult.ErrorMessage = ex.Message;
_resultsQueue.Enqueue(docIDResult);
}
});
totalStopWatch.Stop(); //This time is going to include file I/O along with ID time
_totalFileIdTimeMs = totalStopWatch.Elapsed.TotalMilliseconds;
_totalUniqueFileIds = uniqueIdSet.Count;
_avgTimeToIdFile = _totalFileIdTimeMs / Math.Max(1, _resultsQueue.Count);
}
catch (Exception ex)
{
Dispatcher.Invoke((Action) delegate
{
MessageBox.Show(this, ex.Message, "Exception caught on thread 'WorkerThread' method");
});
}
finally
{
WorkerCompleted();
}
}
/// <summary>
/// Override
/// </summary>
protected override void ProcessRecord()
{
IdResult idResult = null;
DocumentContent content = null;
var strBuilder = new StringBuilder();
var extractorType = ContentExtractorType.Document;
using (var stream = File.OpenRead(Path))
{
idResult = DocumentIdentifier.Identify(stream, Path);
//
// Content extraction settings:
//
var settings = new ContentExtractionSettings();
settings.ExtractionType = ExtractionType.TextAndMetadata;
settings.Hashing.HashingType = HashingType.BinaryAndContentHash;
settings.SensitiveItemCheck.Check = true;
//
// Get Content Extractor for identified file format type:
//
var contentExtractorResult = ContentExtractorFactory.GetContentExtractor(stream, idResult, Path, settings);
if (contentExtractorResult.HasError)
{
WriteObject("Error: " + contentExtractorResult.Error);
return;
}
else
{
extractorType = contentExtractorResult.ContentExtractor.ContentExtractorType;
switch (extractorType)
{
case ContentExtractorType.Archive:
#region Archive Extraction...
{
var archiveExtractor = (IArchiveExtractor)contentExtractorResult.ContentExtractor;
if (archiveExtractor.IsSplit)
{
// Detected that currently selected file is the main split segment for a split archive. Now we will use archive
// extractor helper method 'GetSplitSegmentStreamsInOrder' to get the other split archive segments (in proper order)
// in the same directory:
Stream[] splitSegmentStreamsInOrder = null;
string[] splitSegmentNameInOrder = null;
archiveExtractor.GetSplitSegmentStreamsInOrder(Path, out splitSegmentStreamsInOrder, out splitSegmentNameInOrder);
content = archiveExtractor.ExtractContent(splitSegmentStreamsInOrder, splitSegmentNameInOrder, Password);
//
// We have an archive level password (versus item level passwords):
//
if (content.Result == ContentResult.WrongPassword)
{
if (!string.IsNullOrWhiteSpace(Password))
{
strBuilder.AppendLine("ERROR: Wrong Password");
}
else
{
strBuilder.AppendLine("ERROR: Archive requires a password");
}
}
}
else
{
content = archiveExtractor.ExtractContent(Password);
//
// We have an archive level password (versus item level passwords):
//
if (content.Result == ContentResult.WrongPassword)
{
if (!string.IsNullOrWhiteSpace(Password))
{
strBuilder.AppendLine("ERROR: Wrong Password");
}
else
{
strBuilder.AppendLine("ERROR: Archive requires a password");
}
}
}
}
#endregion
break;
case ContentExtractorType.Document:
#region Document Extraction...
{
var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor);
content = docExtractor.ExtractContent();
// We have an encrypted document that is supported for decryption, keep prompting user for passwords until result is not
// ContentResult.WrongPassword or until user presses "Cancel" button:
if (content.Result == ContentResult.WrongPassword && content.IsEncrypted && docExtractor.SupportsDecryption)
{
if (!string.IsNullOrWhiteSpace(Password))
{
strBuilder.AppendLine("ERROR: Wrong Password");
}
else
{
strBuilder.AppendLine("ERROR: Document requires a password");
}
}
else if (content.Result == ContentResult.WrongPassword && content.IsEncrypted && !docExtractor.SupportsDecryption)
{
strBuilder.AppendLine("ERROR: Document is encrypted with a password but format is not supported for decryption.");
}
}
#endregion
break;
case ContentExtractorType.MailStore:
#region MailStore Extraction...
{
var mailStoreExtractor = ((IMailStoreExtractor)contentExtractorResult.ContentExtractor);
content = mailStoreExtractor.ExtractContent();
}
#endregion
break;
case ContentExtractorType.Database:
#region Database Extraction...
{
// We will only get table/column info (individual table extracted text can be quite large):
var databaseExtractor = ((IDatabaseExtractor)contentExtractorResult.ContentExtractor);
content = databaseExtractor.ExtractContent(Path);
}
#endregion
break;
case ContentExtractorType.DocumentStore:
#region DocumentStore Extraction...
{
var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor);
content = docExtractor.ExtractContent();
}
#endregion
break;
case ContentExtractorType.Unsupported:
#region Unsupported Type Extraction...
{
//
// Binary-to-text extraction: Note, if property ContentExtractionSettings.BinaryToTextOnUnsupportedTypes is false, then calling
// IUnsupportedExtractor.ExtractContent will only calculate binary hashes without performing binary-to-text.
// Binary-to-text is not useful for file formats that do not have any textual content (e.g., compressed archives or encrypted files)
// It is up to the user to filter these formats out using either file format Id or file format classification.
//
var docExtractor = ((IUnsupportedExtractor)contentExtractorResult.ContentExtractor);
content = docExtractor.ExtractContent();
}
#endregion
break;
case ContentExtractorType.LargeUnsupported:
// Ignore for this example
break;
case ContentExtractorType.LargeEncodedText:
// Ignore for this example
break;
}
}
}
strBuilder.AppendLine("File Format:");
strBuilder.AppendLine("------------");
strBuilder.AppendLine(string.Format(" ID: {0}", idResult.ID.ToString()));
strBuilder.AppendLine(string.Format(" Classification: {0}", idResult.Classification.ToString()));
strBuilder.AppendLine(string.Format(" MatchType: {0}", idResult.MatchType.ToString()));
strBuilder.AppendLine(string.Format(" Text Encoding ID: {0}", idResult.EncodingID.ToString()));
strBuilder.AppendLine(string.Format(" IsEncrypted: {0}", idResult.IsEncrypted.ToString()));
strBuilder.AppendLine(string.Format(" MediaType: {0}", idResult.MediaType.ToString()));
strBuilder.AppendLine(string.Format(" Description: {0}", idResult.Description.ToString()));
if (content != null)
{
strBuilder.AppendLine();
strBuilder.AppendLine("File Metadata:");
strBuilder.AppendLine("---------------");
foreach (var meta in content.Metadata)
{
string value = "";
switch (meta.Value.PropertyType)
{
case PropertyType.Boolean:
value = ((BooleanProperty)meta.Value).Value.ToString();
break;
case PropertyType.DateTime:
value = ((DateTimeProperty)meta.Value).Value.ToString();
break;
case PropertyType.Double:
value = ((DoubleProperty)meta.Value).Value.ToString();
break;
case PropertyType.Int32:
value = ((Int32Property)meta.Value).Value.ToString();
break;
case PropertyType.Int64:
value = ((Int64Property)meta.Value).Value.ToString();
break;
case PropertyType.String:
value = ((StringProperty)meta.Value).Value;
break;
case PropertyType.BooleanList:
value = string.Join("; ", ((BooleanListProperty)meta.Value).Value);
break;
case PropertyType.DateTimeList:
value = string.Join("; ", ((DateTimeListProperty)meta.Value).Value);
break;
case PropertyType.DoubleList:
value = string.Join("; ", ((DoubleListProperty)meta.Value).Value);
break;
case PropertyType.Int32List:
value = string.Join("; ", ((Int32ListProperty)meta.Value).Value);
break;
case PropertyType.Int64List:
value = string.Join("; ", ((Int64ListProperty)meta.Value).Value);
break;
case PropertyType.StringList:
value = string.Join("; ", ((StringListProperty)meta.Value).Value);
break;
}
strBuilder.AppendLine(string.Format(" {0,-35} {1}", meta.Key, value));
}
strBuilder.AppendLine();
strBuilder.AppendLine("Custom Metadata:");
strBuilder.AppendLine("-----------------");
foreach (var meta in content.CustomMetadata)
{
string value = "";
switch (meta.Value.PropertyType)
{
case PropertyType.Boolean:
value = ((BooleanProperty)meta.Value).Value.ToString();
break;
case PropertyType.DateTime:
value = ((DateTimeProperty)meta.Value).Value.ToString();
break;
case PropertyType.Double:
value = ((DoubleProperty)meta.Value).Value.ToString();
break;
case PropertyType.Int32:
value = ((Int32Property)meta.Value).Value.ToString();
break;
case PropertyType.Int64:
value = ((Int64Property)meta.Value).Value.ToString();
break;
case PropertyType.String:
value = ((StringProperty)meta.Value).Value;
break;
case PropertyType.BooleanList:
value = string.Join("; ", ((BooleanListProperty)meta.Value).Value);
break;
case PropertyType.DateTimeList:
value = string.Join("; ", ((DateTimeListProperty)meta.Value).Value);
break;
case PropertyType.DoubleList:
value = string.Join("; ", ((DoubleListProperty)meta.Value).Value);
break;
case PropertyType.Int32List:
value = string.Join("; ", ((Int32ListProperty)meta.Value).Value);
break;
case PropertyType.Int64List:
value = string.Join("; ", ((Int64ListProperty)meta.Value).Value);
break;
case PropertyType.StringList:
value = string.Join("; ", ((StringListProperty)meta.Value).Value);
break;
}
strBuilder.AppendLine(string.Format(" {0,-35} {1}", meta.Key, value));
}
strBuilder.AppendLine();
strBuilder.AppendLine("File Attributes:");
strBuilder.AppendLine("----------------");
if (content.Attributes.Count > 0)
{
foreach (var attr in content.Attributes)
{
strBuilder.AppendLine(string.Format(" {0}", attr.ToString()));
}
}
strBuilder.AppendLine();
if (content is DatabaseContent)
{
var dbContent = (DatabaseContent)content;
strBuilder.AppendLine("Database Tables:");
strBuilder.AppendLine("----------------");
if (dbContent.Tables != null && dbContent.Tables.Count > 0)
{
strBuilder.AppendLine(" [Name] [Row Count] [Num Columns] [Is User Table] ");
foreach (var table in dbContent.Tables)
{
strBuilder.AppendLine(string.Format(" {0,-30} {1,-20} {2,-15} {3}", table.Name, table.RowCount,
table.Columns != null ? table.Columns.Count.ToString() : "0", table.IsUserTable.ToString()));
}
}
strBuilder.AppendLine();
}
strBuilder.AppendLine("File Hyperlinks:");
strBuilder.AppendLine("----------------");
if (content.HyperLinks != null && content.HyperLinks.Count > 0)
{
foreach (var link in content.HyperLinks)
{
strBuilder.AppendLine(string.Format(" {0}", link.Url));
}
}
strBuilder.AppendLine();
strBuilder.AppendLine();
strBuilder.AppendLine("Detected Sensitive Items:");
strBuilder.AppendLine("-------------------------");
if (content.SensitiveItemResult != null && content.SensitiveItemResult.Items.Count > 0)
{
foreach (var item in content.SensitiveItemResult.Items)
{
strBuilder.AppendLine(string.Format(" {0,-30} {1,-20} {2,-15} {3}", item.ItemType.ToString(), item.MatchType.ToString(), item.LocationType.ToString(), item.Text));
}
}
strBuilder.AppendLine();
strBuilder.AppendLine();
strBuilder.AppendLine("Detected Languages:");
strBuilder.AppendLine("-------------------");
if (content.LanguageIdResults != null && content.LanguageIdResults.Count > 0)
{
foreach (var langIdResult in content.LanguageIdResults)
{
strBuilder.AppendLine(string.Format(" {0,-30} {1,-20} {2,-15}", langIdResult.Language, langIdResult.LangIso639, langIdResult.PercentOfFullText));
}
}
strBuilder.AppendLine();
if (ShowText)
{
strBuilder.AppendLine();
if (content.ExtractedText != null)
{
var charsToDisplay = Math.Min(1000, content.ExtractedText.Length);
strBuilder.AppendLine(string.Format("Extracted Text: Total Chars = {0}, Displayed Chars = {1}", content.ExtractedText.Length, charsToDisplay));
strBuilder.AppendLine("-------------------------------------------------------------------");
strBuilder.AppendLine(content.ExtractedText.Substring(0, charsToDisplay));
strBuilder.AppendLine();
}
else
{
strBuilder.AppendLine(string.Format("Extracted Text: Total Chars = {0}, Displayed Chars = {1}", 0, 0));
strBuilder.AppendLine("-------------------------------------------------------------------");
strBuilder.AppendLine();
}
}
}
WriteObject(strBuilder.ToString());
}
/// <summary>
/// Override
/// </summary>
protected override void ProcessRecord()
{
IdResult idResult = null;
DocumentContent content = null;
ContentExtractorType extractorType = ContentExtractorType.Document;
using (var stream = File.OpenRead(Path))
{
idResult = DocumentIdentifier.Identify(stream, Path);
//
// Extract metadata content from document:
//
var settings = new ContentExtractionSettings();
settings.ExtractionType = ExtractionType.TextAndMetadata;
settings.EmbeddedObjectExtraction = EmbeddedExtractionType.EmbeddedDocumentsAndMedia;
settings.SensitiveItemCheck.Check = true; // Enable sensitive item checks
settings.Hashing.HashingType = HashingType.BinaryAndContentHash;
settings.LargeDocumentCritera = 100 * 1024 * 1024; // Define a 'large' file as >= 100MB (this determines when
// ContentExtractorType.LargeUnsupported and ContentExtractorType.LargeEncodedText
// extractor interfaces are returned.
//
// Get Content Extractor for identified file format type:
//
var contentExtractorResult = ContentExtractorFactory.GetContentExtractor(stream, idResult, Path, settings);
if (contentExtractorResult.HasError)
{
WriteObject("Error: " + contentExtractorResult.Error);
return;
}
else
{
extractorType = contentExtractorResult.ContentExtractor.ContentExtractorType;
switch (extractorType)
{
case ContentExtractorType.Archive:
#region Archive Extraction...
{
var archiveExtractor = (IArchiveExtractor)contentExtractorResult.ContentExtractor;
if (archiveExtractor.IsSplit)
{
// Detected that currently selected file is the main split segment for a split archive. Now we will use archive
// extractor helper method 'GetSplitSegmentStreamsInOrder' to get the other split archive segments (in proper order)
// in the same directory:
Stream[] splitSegmentStreamsInOrder = null;
string[] splitSegmentNameInOrder = null;
archiveExtractor.GetSplitSegmentStreamsInOrder(Path, out splitSegmentStreamsInOrder, out splitSegmentNameInOrder);
content = archiveExtractor.ExtractContent(splitSegmentStreamsInOrder, splitSegmentNameInOrder, Password);
//
// We have an archive level password (versus item level passwords):
//
if (content.Result == ContentResult.WrongPassword)
{
// wrong password
}
}
else
{
content = archiveExtractor.ExtractContent(Password);
//
// We have an archive level password (versus item level passwords):
//
if (content.Result == ContentResult.WrongPassword)
{
// wrong password
}
}
}
#endregion
break;
case ContentExtractorType.Document:
#region Document Extraction...
{
var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor);
content = docExtractor.ExtractContent();
if (content.Result == ContentResult.WrongPassword)
{
// wrong password
}
}
#endregion
break;
case ContentExtractorType.MailStore:
#region MailStore Extraction...
{
var mailStoreExtractor = ((IMailStoreExtractor)contentExtractorResult.ContentExtractor);
content = mailStoreExtractor.ExtractContent();
}
#endregion
break;
case ContentExtractorType.Database:
#region Database Extraction...
{
// We will only get table/column info (individual table extracted text can be quite large):
var databaseExtractor = ((IDatabaseExtractor)contentExtractorResult.ContentExtractor);
content = databaseExtractor.ExtractContent(Path);
}
#endregion
break;
case ContentExtractorType.DocumentStore:
#region DocumentStore Extraction...
{
var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor);
content = docExtractor.ExtractContent();
}
#endregion
break;
case ContentExtractorType.Unsupported:
#region Unsupported Type Extraction...
{
//
// Binary-to-text extraction: Note, if property ContentExtractionSettings.BinaryToTextOnUnsupportedTypes is false, then calling
// IUnsupportedExtractor.ExtractContent will only calculate binary hashes without performing binary-to-text.
// Binary-to-text is not useful for file formats that do not have any textual content (e.g., compressed archives or encrypted files)
// It is up to the user to filter these formats out using either file format Id or file format classification.
//
var docExtractor = ((IUnsupportedExtractor)contentExtractorResult.ContentExtractor);
content = docExtractor.ExtractContent();
}
#endregion
break;
case ContentExtractorType.LargeUnsupported:
#region 'Large' Unsupported Type Extraction...
{
// Ignore for this example, very 'large' binary-to-text that needs a FileStream could be extracted
content = new DocumentContent(idResult);
content.Result = ContentResult.UnsupportedError;
content.ErrorMessage = "Not supported for this example. Users should write output to a file stream when implemented";
}
#endregion
break;
case ContentExtractorType.LargeEncodedText:
#region 'Large' Encoded Text File Extraction...
{
// Ignore for this example
content = new DocumentContent(idResult);
content.Result = ContentResult.UnsupportedError;
content.ErrorMessage = "Not supported for this example. Users should write output to a file stream when implemented";
}
#endregion
break;
}
}
}
WriteObject(content);
}
/// <summary>
/// Write instruction into bytecode stream.
/// </summary>
/// <param name="writer">Bytecode writer.</param>
public override void Write(WordWriter writer)
{
IdResult.Write(writer);
WriteOperands(writer);
WriteExtras(writer);
}
public void SetUp()
{
_id = Guid.NewGuid();
_result = IdResult.IdMapped(_id);
}
