/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += Execution.GetWordCount(); wordCount += Pipe.GetWordCount(); wordCount += NumPackets.GetWordCount(); wordCount += PacketSize.GetWordCount(); wordCount += PacketAlignment.GetWordCount(); return(wordCount); }
public void Generate_grows_success() { var g = new EasyGenerator(10); IdResult oldIdResult = new IdResult(); for (int i = 0; i < 5000000; i++) { IdResult newIdResult = g.GetIdResult(); Assert.True(newIdResult.GenerateId() > oldIdResult.GenerateId(), $"{newIdResult}>{oldIdResult}???"); oldIdResult = newIdResult; } ; }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += SrcImage.GetWordCount(); wordCount += FwdRefImage.GetWordCount(); wordCount += BwdRefImage.GetWordCount(); wordCount += Payload.GetWordCount(); wordCount += StreaminComponents.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += NDRange.GetWordCount(); wordCount += Invoke.GetWordCount(); wordCount += Param.GetWordCount(); wordCount += ParamSize.GetWordCount(); wordCount += ParamAlign.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += SampledImage.GetWordCount(); wordCount += Coordinate.GetWordCount(); wordCount += Granularity.GetWordCount(); wordCount += Coarse.GetWordCount(); wordCount += ImageOperands?.GetWordCount() ?? 0u; return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += Execution.GetWordCount(); wordCount += Destination.GetWordCount(); wordCount += Source.GetWordCount(); wordCount += NumElements.GetWordCount(); wordCount += Stride.GetWordCount(); wordCount += Event.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += Pipe.GetWordCount(); wordCount += ReserveId.GetWordCount(); wordCount += Index.GetWordCount(); wordCount += Pointer.GetWordCount(); wordCount += PacketSize.GetWordCount(); wordCount += PacketAlignment.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += SkipBlockPartitionType.GetWordCount(); wordCount += SkipMotionVectorMask.GetWordCount(); wordCount += MotionVectors.GetWordCount(); wordCount += BidirectionalWeight.GetWordCount(); wordCount += SadAdjustment.GetWordCount(); wordCount += Payload.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += Pointer.GetWordCount(); wordCount += Memory.GetWordCount(); wordCount += Equal.GetWordCount(); wordCount += Unequal.GetWordCount(); wordCount += Value.GetWordCount(); wordCount += Comparator.GetWordCount(); return(wordCount); }
private async Task <Guid?> GetIdAsync(Type matchingType, MethodInfo matchingMethod) { foreach (IIdMapper idMapper in _idMappers) { IdResult result = await idMapper.MapAsync(matchingType, matchingMethod); if (result.ResultType == IdResultType.IdMapped) { return(result.Id); } } return(null); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResult.GetWordCount(); wordCount += SampledType.GetWordCount(); wordCount += Dim.GetWordCount(); wordCount += Depth.GetWordCount(); wordCount += Arrayed.GetWordCount(); wordCount += MS.GetWordCount(); wordCount += Sampled.GetWordCount(); wordCount += ImageFormat.GetWordCount(); wordCount += AccessQualifier?.GetWordCount() ?? 0u; return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += SrcCoord.GetWordCount(); wordCount += MotionVectors.GetWordCount(); wordCount += MajorShapes.GetWordCount(); wordCount += MinorShapes.GetWordCount(); wordCount += Direction.GetWordCount(); wordCount += PixelResolution.GetWordCount(); wordCount += SadAdjustment.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += LumaIntraPartitionMask.GetWordCount(); wordCount += IntraNeighbourAvailabilty.GetWordCount(); wordCount += LeftEdgeLumaPixels.GetWordCount(); wordCount += UpperLeftCornerLumaPixel.GetWordCount(); wordCount += UpperEdgeLumaPixels.GetWordCount(); wordCount += UpperRightEdgeLumaPixels.GetWordCount(); wordCount += SadAdjustment.GetWordCount(); wordCount += Payload.GetWordCount(); return(wordCount); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResultType.GetWordCount(); wordCount += IdResult.GetWordCount(); wordCount += Queue.GetWordCount(); wordCount += Flags.GetWordCount(); wordCount += NDRange.GetWordCount(); wordCount += NumEvents.GetWordCount(); wordCount += WaitEvents.GetWordCount(); wordCount += RetEvent.GetWordCount(); wordCount += Invoke.GetWordCount(); wordCount += Param.GetWordCount(); wordCount += ParamSize.GetWordCount(); wordCount += ParamAlign.GetWordCount(); wordCount += LocalSize.GetWordCount(); return(wordCount); }
public void SetUp() { _classFilter = MockRepository.GenerateMock <IClassFilter>(); _classFilter .Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask()) .Return(false.AsCompletedTask()); _idMapper = MockRepository.GenerateMock <IIdMapper>(); _idMapper .Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask()) .Return(null); _nameMapper = MockRepository.GenerateMock <INameMapper>(); _nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask()); _resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>(); _resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask()); _autoRouteCollection = new AutoRouteCollection(true) .Assemblies(Assembly.GetExecutingAssembly()) .ClassFilters(_classFilter) .NameMappers(_nameMapper) .IdMappers(_idMapper) .ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper); }
public void SetUp() { _result = IdResult.IdNotMapped(); }
public void SetUp() { _classFilter = MockRepository.GenerateMock <IClassFilter>(); _classFilter .Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask()) .Return(false.AsCompletedTask()); _idMapper = MockRepository.GenerateMock <IIdMapper>(); _idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask()); _nameMapper = MockRepository.GenerateMock <INameMapper>(); _nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask()); _resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>(); _resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask()); _authenticationProvider = MockRepository.GenerateMock <IAuthenticationProvider>(); _authenticationProvider .Stub(arg => arg.AuthenticateAsync(Arg <HttpRequestBase> .Is.Anything, Arg <HttpResponseBase> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything)) .Return(AuthenticationResult.AuthenticationSucceeded.AsCompletedTask()); _authenticationStrategy = MockRepository.GenerateMock <IAuthenticationStrategy>(); _authenticationStrategy.Stub(arg => arg.MustAuthenticateAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(true.AsCompletedTask()); _autoRouteCollection = new AutoRouteCollection() .Assemblies(Assembly.GetExecutingAssembly()) .ClassFilters(_classFilter) .NameMappers(_nameMapper) .IdMappers(_idMapper) .ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper) .Authenticate(_authenticationProvider, _authenticationStrategy); _routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray(); }
public void SetUp() { _classFilter = MockRepository.GenerateMock <IClassFilter>(); _classFilter .Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask()) .Return(false.AsCompletedTask()); _idMapper = MockRepository.GenerateMock <IIdMapper>(); _idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask()); _nameMapper = MockRepository.GenerateMock <INameMapper>(); _nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask()); _resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>(); _resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask()); _responseMapper = MockRepository.GenerateMock <IResponseMapper>(); _responseMapper .Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything)) .Return(Task.Factory.Empty()); _restrictionMapper1 = MockRepository.GenerateMock <IRestrictionMapper>(); _restrictionMapper1 .Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything, Arg <IContainer> .Is.Anything)) .WhenCalled(arg => ((Route.Routing.Route)arg.Arguments.Skip(2).First()).RestrictByMethods("GET").AsCompletedTask()) .Return(Task.Factory.Empty()); _restrictionMapper2 = MockRepository.GenerateMock <IRestrictionMapper>(); _restrictionMapper2 .Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything, Arg <IContainer> .Is.Anything)) .WhenCalled(arg => ((Route.Routing.Route)arg.Arguments.Skip(2).First()).RestrictByMethods("POST").AsCompletedTask()) .Return(Task.Factory.Empty()); _autoRouteCollection = new AutoRouteCollection() .Assemblies(Assembly.GetExecutingAssembly()) .ClassFilters(_classFilter) .NameMappers(_nameMapper) .IdMappers(_idMapper) .ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper) .ResponseMapper(_responseMapper) .RestrictionMappers(_restrictionMapper1, _restrictionMapper2); }
public void SetUp() { _classFilter = MockRepository.GenerateMock <IClassFilter>(); _classFilter .Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask()) .Return(false.AsCompletedTask()); _idMapper1 = MockRepository.GenerateMock <IIdMapper>(); _idMapper1.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.Parse("1dffe3ee-1ade-4aa2-835a-9cb91b7e31c4")).AsCompletedTask()); _idMapper2 = MockRepository.GenerateMock <IIdMapper>(); _idMapper2.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.Parse("493e725c-cbc1-4ea4-b6d1-350018d4542d")).AsCompletedTask()); _nameMapper = MockRepository.GenerateMock <INameMapper>(); _nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask()); _resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>(); _resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask()); _responseMapper = MockRepository.GenerateMock <IResponseMapper>(); _responseMapper .Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything)) .Return(Task.Factory.Empty()); _autoRouteCollection = new AutoRouteCollection() .Assemblies(Assembly.GetExecutingAssembly()) .ClassFilters(_classFilter) .NameMappers(_nameMapper) .IdMappers(_idMapper1) .ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper) .ResponseMapper(_responseMapper); _routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray(); }
public void SetUp() { _classFilter = MockRepository.GenerateMock <IClassFilter>(); _classFilter .Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask()) .Return(false.AsCompletedTask()); _idMapper = MockRepository.GenerateMock <IIdMapper>(); _idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask()); _nameMapper = MockRepository.GenerateMock <INameMapper>(); _nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped(Guid.NewGuid().ToString()).AsCompletedTask()); _resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>(); _resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask()); _responseMapper = MockRepository.GenerateMock <IResponseMapper>(); _responseMapper .Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything)) .Return(Task.Factory.Empty()); _httpRuntime = MockRepository.GenerateMock <IHttpRuntime>(); _autoRouteCollection = new AutoRouteCollection() .Assemblies(typeof(Endpoint).Assembly) .ClassFilters(_classFilter) .NameMappers(_nameMapper) .IdMappers(_idMapper) .ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper) .RestrictUsingAttributes <UrlRelativePathAttribute>() .RestrictRelativePathsToRelativeClassNamespaceAndClassName("") .ResponseMapper(_responseMapper) .RestrictionContainer(new DefaultRestrictionContainer(_httpRuntime)); _routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray(); }
public void SetUp() { _classFilter = MockRepository.GenerateMock <IClassFilter>(); _classFilter .Stub(arg => arg.MatchesAsync(Arg <Type> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = ((Type)arg.Arguments.First() == typeof(Endpoint)).AsCompletedTask()) .Return(false.AsCompletedTask()); _methodFilter = MockRepository.GenerateMock <IMethodFilter>(); _methodFilter .Stub(arg => arg.MatchesAsync(Arg <MethodInfo> .Is.Anything)) .WhenCalled(arg => arg.ReturnValue = (((MethodInfo)arg.Arguments.First()).Name == "IncludedMethod").AsCompletedTask()) .Return(false.AsCompletedTask()); _idMapper = MockRepository.GenerateMock <IIdMapper>(); _idMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(IdResult.IdMapped(Guid.NewGuid()).AsCompletedTask()); _nameMapper = MockRepository.GenerateMock <INameMapper>(); _nameMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(NameResult.NameMapped("name").AsCompletedTask()); _resolvedRelativeUrlMapper = MockRepository.GenerateMock <IResolvedRelativeUrlMapper>(); _resolvedRelativeUrlMapper.Stub(arg => arg.MapAsync(Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything)).Return(ResolvedRelativeUrlResult.ResolvedRelativeUrlMapped("relative").AsCompletedTask()); _responseMapper = MockRepository.GenerateMock <IResponseMapper>(); _responseMapper .Stub(arg => arg.MapAsync(Arg <Func <IContainer> > .Is.Anything, Arg <Type> .Is.Anything, Arg <MethodInfo> .Is.Anything, Arg <Route.Routing.Route> .Is.Anything)) .Return(Task.Factory.Empty()); _autoRouteCollection = new AutoRouteCollection() .Assemblies(Assembly.GetExecutingAssembly()) .ClassFilters(_classFilter) .MethodFilters(_methodFilter) .NameMappers(_nameMapper) .IdMappers(_idMapper) .ResolvedRelativeUrlMappers(_resolvedRelativeUrlMapper) .ResponseMapper(_responseMapper); _routes = _autoRouteCollection.GenerateRouteCollectionAsync().Result.ToArray(); }
/// <summary> /// Calculate number of words to fit complete instruction bytecode. /// </summary> /// <returns>Number of words in instruction bytecode.</returns> public override uint GetWordCount() { uint wordCount = 0; wordCount += IdResult.GetWordCount(); return wordCount; }
// // Threading: // #region private void WorkerThread() private void WorkerThread() { try { var allFiles = Directory.GetFiles(_rootPath, "*", SearchOption.AllDirectories); var totalFiles = allFiles.Length; var totalStopWatch = Stopwatch.StartNew(); var uniqueIdSet = new ConcurrentDictionary <Id, int>(); Dispatcher.Invoke((Action) delegate { try { _progressBar.Visibility = Visibility.Visible; _progressBar.Value = 0; } catch { } }); var lastUpdateStopWatch = Stopwatch.StartNew(); // // Parallelize the file format identification: // Parallel.ForEach(allFiles, _parallelForOptions, file => { try { IdResult docFormat = null; // WARNING: This example does not support long path names (> 255 chars) - .NET solutions for getting valid FileStreams for 'long file paths' // can be found on the internet - also .NET 4.6.2 supports long file paths (web search for how to enable) // Note: Minimum recommended buffer size of 16kb for file identification using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 16384)) { docFormat = DocumentIdentifier.Identify(stream, file); } uniqueIdSet[docFormat.ID] = 1; var docIdResult = new DocumentIdResult(file, docFormat); if (docFormat.ID == Id.Unknown) { Interlocked.Increment(ref _totalUnknownFiles); } _resultsQueue.Enqueue(docIdResult); var numFilesIdentified = _resultsQueue.Count; var percentComplete = 100.0 * numFilesIdentified / totalFiles; // Update progress bar and status every 250 [ms]: if (lastUpdateStopWatch.ElapsedMilliseconds >= 250) { lastUpdateStopWatch.Restart(); Dispatcher.BeginInvoke((Action) delegate { try { _progressBar.Value = percentComplete; _statusTextBox.Text = string.Format(" {0} files out of {1} identified...", numFilesIdentified, totalFiles); } catch { } }); } } catch (Exception ex) { Interlocked.Increment(ref _totalExceptions); var docIDResult = new DocumentIdResult(file, DocumentIdentifier.UnknownResult); docIDResult.HasError = true; docIDResult.ErrorMessage = ex.Message; _resultsQueue.Enqueue(docIDResult); } }); totalStopWatch.Stop(); //This time is going to include file I/O along with ID time _totalFileIdTimeMs = totalStopWatch.Elapsed.TotalMilliseconds; _totalUniqueFileIds = uniqueIdSet.Count; _avgTimeToIdFile = _totalFileIdTimeMs / Math.Max(1, _resultsQueue.Count); } catch (Exception ex) { Dispatcher.Invoke((Action) delegate { MessageBox.Show(this, ex.Message, "Exception caught on thread 'WorkerThread' method"); }); } finally { WorkerCompleted(); } }
/// <summary> /// Override /// </summary> protected override void ProcessRecord() { IdResult idResult = null; DocumentContent content = null; var strBuilder = new StringBuilder(); var extractorType = ContentExtractorType.Document; using (var stream = File.OpenRead(Path)) { idResult = DocumentIdentifier.Identify(stream, Path); // // Content extraction settings: // var settings = new ContentExtractionSettings(); settings.ExtractionType = ExtractionType.TextAndMetadata; settings.Hashing.HashingType = HashingType.BinaryAndContentHash; settings.SensitiveItemCheck.Check = true; // // Get Content Extractor for identified file format type: // var contentExtractorResult = ContentExtractorFactory.GetContentExtractor(stream, idResult, Path, settings); if (contentExtractorResult.HasError) { WriteObject("Error: " + contentExtractorResult.Error); return; } else { extractorType = contentExtractorResult.ContentExtractor.ContentExtractorType; switch (extractorType) { case ContentExtractorType.Archive: #region Archive Extraction... { var archiveExtractor = (IArchiveExtractor)contentExtractorResult.ContentExtractor; if (archiveExtractor.IsSplit) { // Detected that currently selected file is the main split segment for a split archive. Now we will use archive // extractor helper method 'GetSplitSegmentStreamsInOrder' to get the other split archive segments (in proper order) // in the same directory: Stream[] splitSegmentStreamsInOrder = null; string[] splitSegmentNameInOrder = null; archiveExtractor.GetSplitSegmentStreamsInOrder(Path, out splitSegmentStreamsInOrder, out splitSegmentNameInOrder); content = archiveExtractor.ExtractContent(splitSegmentStreamsInOrder, splitSegmentNameInOrder, Password); // // We have an archive level password (versus item level passwords): // if (content.Result == ContentResult.WrongPassword) { if (!string.IsNullOrWhiteSpace(Password)) { strBuilder.AppendLine("ERROR: Wrong Password"); } else { strBuilder.AppendLine("ERROR: Archive requires a password"); } } } else { content = archiveExtractor.ExtractContent(Password); // // We have an archive level password (versus item level passwords): // if (content.Result == ContentResult.WrongPassword) { if (!string.IsNullOrWhiteSpace(Password)) { strBuilder.AppendLine("ERROR: Wrong Password"); } else { strBuilder.AppendLine("ERROR: Archive requires a password"); } } } } #endregion break; case ContentExtractorType.Document: #region Document Extraction... { var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor); content = docExtractor.ExtractContent(); // We have an encrypted document that is supported for decryption, keep prompting user for passwords until result is not // ContentResult.WrongPassword or until user presses "Cancel" button: if (content.Result == ContentResult.WrongPassword && content.IsEncrypted && docExtractor.SupportsDecryption) { if (!string.IsNullOrWhiteSpace(Password)) { strBuilder.AppendLine("ERROR: Wrong Password"); } else { strBuilder.AppendLine("ERROR: Document requires a password"); } } else if (content.Result == ContentResult.WrongPassword && content.IsEncrypted && !docExtractor.SupportsDecryption) { strBuilder.AppendLine("ERROR: Document is encrypted with a password but format is not supported for decryption."); } } #endregion break; case ContentExtractorType.MailStore: #region MailStore Extraction... { var mailStoreExtractor = ((IMailStoreExtractor)contentExtractorResult.ContentExtractor); content = mailStoreExtractor.ExtractContent(); } #endregion break; case ContentExtractorType.Database: #region Database Extraction... { // We will only get table/column info (individual table extracted text can be quite large): var databaseExtractor = ((IDatabaseExtractor)contentExtractorResult.ContentExtractor); content = databaseExtractor.ExtractContent(Path); } #endregion break; case ContentExtractorType.DocumentStore: #region DocumentStore Extraction... { var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor); content = docExtractor.ExtractContent(); } #endregion break; case ContentExtractorType.Unsupported: #region Unsupported Type Extraction... { // // Binary-to-text extraction: Note, if property ContentExtractionSettings.BinaryToTextOnUnsupportedTypes is false, then calling // IUnsupportedExtractor.ExtractContent will only calculate binary hashes without performing binary-to-text. // Binary-to-text is not useful for file formats that do not have any textual content (e.g., compressed archives or encrypted files) // It is up to the user to filter these formats out using either file format Id or file format classification. // var docExtractor = ((IUnsupportedExtractor)contentExtractorResult.ContentExtractor); content = docExtractor.ExtractContent(); } #endregion break; case ContentExtractorType.LargeUnsupported: // Ignore for this example break; case ContentExtractorType.LargeEncodedText: // Ignore for this example break; } } } strBuilder.AppendLine("File Format:"); strBuilder.AppendLine("------------"); strBuilder.AppendLine(string.Format(" ID: {0}", idResult.ID.ToString())); strBuilder.AppendLine(string.Format(" Classification: {0}", idResult.Classification.ToString())); strBuilder.AppendLine(string.Format(" MatchType: {0}", idResult.MatchType.ToString())); strBuilder.AppendLine(string.Format(" Text Encoding ID: {0}", idResult.EncodingID.ToString())); strBuilder.AppendLine(string.Format(" IsEncrypted: {0}", idResult.IsEncrypted.ToString())); strBuilder.AppendLine(string.Format(" MediaType: {0}", idResult.MediaType.ToString())); strBuilder.AppendLine(string.Format(" Description: {0}", idResult.Description.ToString())); if (content != null) { strBuilder.AppendLine(); strBuilder.AppendLine("File Metadata:"); strBuilder.AppendLine("---------------"); foreach (var meta in content.Metadata) { string value = ""; switch (meta.Value.PropertyType) { case PropertyType.Boolean: value = ((BooleanProperty)meta.Value).Value.ToString(); break; case PropertyType.DateTime: value = ((DateTimeProperty)meta.Value).Value.ToString(); break; case PropertyType.Double: value = ((DoubleProperty)meta.Value).Value.ToString(); break; case PropertyType.Int32: value = ((Int32Property)meta.Value).Value.ToString(); break; case PropertyType.Int64: value = ((Int64Property)meta.Value).Value.ToString(); break; case PropertyType.String: value = ((StringProperty)meta.Value).Value; break; case PropertyType.BooleanList: value = string.Join("; ", ((BooleanListProperty)meta.Value).Value); break; case PropertyType.DateTimeList: value = string.Join("; ", ((DateTimeListProperty)meta.Value).Value); break; case PropertyType.DoubleList: value = string.Join("; ", ((DoubleListProperty)meta.Value).Value); break; case PropertyType.Int32List: value = string.Join("; ", ((Int32ListProperty)meta.Value).Value); break; case PropertyType.Int64List: value = string.Join("; ", ((Int64ListProperty)meta.Value).Value); break; case PropertyType.StringList: value = string.Join("; ", ((StringListProperty)meta.Value).Value); break; } strBuilder.AppendLine(string.Format(" {0,-35} {1}", meta.Key, value)); } strBuilder.AppendLine(); strBuilder.AppendLine("Custom Metadata:"); strBuilder.AppendLine("-----------------"); foreach (var meta in content.CustomMetadata) { string value = ""; switch (meta.Value.PropertyType) { case PropertyType.Boolean: value = ((BooleanProperty)meta.Value).Value.ToString(); break; case PropertyType.DateTime: value = ((DateTimeProperty)meta.Value).Value.ToString(); break; case PropertyType.Double: value = ((DoubleProperty)meta.Value).Value.ToString(); break; case PropertyType.Int32: value = ((Int32Property)meta.Value).Value.ToString(); break; case PropertyType.Int64: value = ((Int64Property)meta.Value).Value.ToString(); break; case PropertyType.String: value = ((StringProperty)meta.Value).Value; break; case PropertyType.BooleanList: value = string.Join("; ", ((BooleanListProperty)meta.Value).Value); break; case PropertyType.DateTimeList: value = string.Join("; ", ((DateTimeListProperty)meta.Value).Value); break; case PropertyType.DoubleList: value = string.Join("; ", ((DoubleListProperty)meta.Value).Value); break; case PropertyType.Int32List: value = string.Join("; ", ((Int32ListProperty)meta.Value).Value); break; case PropertyType.Int64List: value = string.Join("; ", ((Int64ListProperty)meta.Value).Value); break; case PropertyType.StringList: value = string.Join("; ", ((StringListProperty)meta.Value).Value); break; } strBuilder.AppendLine(string.Format(" {0,-35} {1}", meta.Key, value)); } strBuilder.AppendLine(); strBuilder.AppendLine("File Attributes:"); strBuilder.AppendLine("----------------"); if (content.Attributes.Count > 0) { foreach (var attr in content.Attributes) { strBuilder.AppendLine(string.Format(" {0}", attr.ToString())); } } strBuilder.AppendLine(); if (content is DatabaseContent) { var dbContent = (DatabaseContent)content; strBuilder.AppendLine("Database Tables:"); strBuilder.AppendLine("----------------"); if (dbContent.Tables != null && dbContent.Tables.Count > 0) { strBuilder.AppendLine(" [Name] [Row Count] [Num Columns] [Is User Table] "); foreach (var table in dbContent.Tables) { strBuilder.AppendLine(string.Format(" {0,-30} {1,-20} {2,-15} {3}", table.Name, table.RowCount, table.Columns != null ? table.Columns.Count.ToString() : "0", table.IsUserTable.ToString())); } } strBuilder.AppendLine(); } strBuilder.AppendLine("File Hyperlinks:"); strBuilder.AppendLine("----------------"); if (content.HyperLinks != null && content.HyperLinks.Count > 0) { foreach (var link in content.HyperLinks) { strBuilder.AppendLine(string.Format(" {0}", link.Url)); } } strBuilder.AppendLine(); strBuilder.AppendLine(); strBuilder.AppendLine("Detected Sensitive Items:"); strBuilder.AppendLine("-------------------------"); if (content.SensitiveItemResult != null && content.SensitiveItemResult.Items.Count > 0) { foreach (var item in content.SensitiveItemResult.Items) { strBuilder.AppendLine(string.Format(" {0,-30} {1,-20} {2,-15} {3}", item.ItemType.ToString(), item.MatchType.ToString(), item.LocationType.ToString(), item.Text)); } } strBuilder.AppendLine(); strBuilder.AppendLine(); strBuilder.AppendLine("Detected Languages:"); strBuilder.AppendLine("-------------------"); if (content.LanguageIdResults != null && content.LanguageIdResults.Count > 0) { foreach (var langIdResult in content.LanguageIdResults) { strBuilder.AppendLine(string.Format(" {0,-30} {1,-20} {2,-15}", langIdResult.Language, langIdResult.LangIso639, langIdResult.PercentOfFullText)); } } strBuilder.AppendLine(); if (ShowText) { strBuilder.AppendLine(); if (content.ExtractedText != null) { var charsToDisplay = Math.Min(1000, content.ExtractedText.Length); strBuilder.AppendLine(string.Format("Extracted Text: Total Chars = {0}, Displayed Chars = {1}", content.ExtractedText.Length, charsToDisplay)); strBuilder.AppendLine("-------------------------------------------------------------------"); strBuilder.AppendLine(content.ExtractedText.Substring(0, charsToDisplay)); strBuilder.AppendLine(); } else { strBuilder.AppendLine(string.Format("Extracted Text: Total Chars = {0}, Displayed Chars = {1}", 0, 0)); strBuilder.AppendLine("-------------------------------------------------------------------"); strBuilder.AppendLine(); } } } WriteObject(strBuilder.ToString()); }
/// <summary> /// Override /// </summary> protected override void ProcessRecord() { IdResult idResult = null; DocumentContent content = null; ContentExtractorType extractorType = ContentExtractorType.Document; using (var stream = File.OpenRead(Path)) { idResult = DocumentIdentifier.Identify(stream, Path); // // Extract metadata content from document: // var settings = new ContentExtractionSettings(); settings.ExtractionType = ExtractionType.TextAndMetadata; settings.EmbeddedObjectExtraction = EmbeddedExtractionType.EmbeddedDocumentsAndMedia; settings.SensitiveItemCheck.Check = true; // Enable sensitive item checks settings.Hashing.HashingType = HashingType.BinaryAndContentHash; settings.LargeDocumentCritera = 100 * 1024 * 1024; // Define a 'large' file as >= 100MB (this determines when // ContentExtractorType.LargeUnsupported and ContentExtractorType.LargeEncodedText // extractor interfaces are returned. // // Get Content Extractor for identified file format type: // var contentExtractorResult = ContentExtractorFactory.GetContentExtractor(stream, idResult, Path, settings); if (contentExtractorResult.HasError) { WriteObject("Error: " + contentExtractorResult.Error); return; } else { extractorType = contentExtractorResult.ContentExtractor.ContentExtractorType; switch (extractorType) { case ContentExtractorType.Archive: #region Archive Extraction... { var archiveExtractor = (IArchiveExtractor)contentExtractorResult.ContentExtractor; if (archiveExtractor.IsSplit) { // Detected that currently selected file is the main split segment for a split archive. Now we will use archive // extractor helper method 'GetSplitSegmentStreamsInOrder' to get the other split archive segments (in proper order) // in the same directory: Stream[] splitSegmentStreamsInOrder = null; string[] splitSegmentNameInOrder = null; archiveExtractor.GetSplitSegmentStreamsInOrder(Path, out splitSegmentStreamsInOrder, out splitSegmentNameInOrder); content = archiveExtractor.ExtractContent(splitSegmentStreamsInOrder, splitSegmentNameInOrder, Password); // // We have an archive level password (versus item level passwords): // if (content.Result == ContentResult.WrongPassword) { // wrong password } } else { content = archiveExtractor.ExtractContent(Password); // // We have an archive level password (versus item level passwords): // if (content.Result == ContentResult.WrongPassword) { // wrong password } } } #endregion break; case ContentExtractorType.Document: #region Document Extraction... { var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor); content = docExtractor.ExtractContent(); if (content.Result == ContentResult.WrongPassword) { // wrong password } } #endregion break; case ContentExtractorType.MailStore: #region MailStore Extraction... { var mailStoreExtractor = ((IMailStoreExtractor)contentExtractorResult.ContentExtractor); content = mailStoreExtractor.ExtractContent(); } #endregion break; case ContentExtractorType.Database: #region Database Extraction... { // We will only get table/column info (individual table extracted text can be quite large): var databaseExtractor = ((IDatabaseExtractor)contentExtractorResult.ContentExtractor); content = databaseExtractor.ExtractContent(Path); } #endregion break; case ContentExtractorType.DocumentStore: #region DocumentStore Extraction... { var docExtractor = ((IDocumentContentExtractor)contentExtractorResult.ContentExtractor); content = docExtractor.ExtractContent(); } #endregion break; case ContentExtractorType.Unsupported: #region Unsupported Type Extraction... { // // Binary-to-text extraction: Note, if property ContentExtractionSettings.BinaryToTextOnUnsupportedTypes is false, then calling // IUnsupportedExtractor.ExtractContent will only calculate binary hashes without performing binary-to-text. // Binary-to-text is not useful for file formats that do not have any textual content (e.g., compressed archives or encrypted files) // It is up to the user to filter these formats out using either file format Id or file format classification. // var docExtractor = ((IUnsupportedExtractor)contentExtractorResult.ContentExtractor); content = docExtractor.ExtractContent(); } #endregion break; case ContentExtractorType.LargeUnsupported: #region 'Large' Unsupported Type Extraction... { // Ignore for this example, very 'large' binary-to-text that needs a FileStream could be extracted content = new DocumentContent(idResult); content.Result = ContentResult.UnsupportedError; content.ErrorMessage = "Not supported for this example. Users should write output to a file stream when implemented"; } #endregion break; case ContentExtractorType.LargeEncodedText: #region 'Large' Encoded Text File Extraction... { // Ignore for this example content = new DocumentContent(idResult); content.Result = ContentResult.UnsupportedError; content.ErrorMessage = "Not supported for this example. Users should write output to a file stream when implemented"; } #endregion break; } } } WriteObject(content); }
/// <summary> /// Write instruction into bytecode stream. /// </summary> /// <param name="writer">Bytecode writer.</param> public override void Write(WordWriter writer) { IdResult.Write(writer); WriteOperands(writer); WriteExtras(writer); }
public void SetUp() { _id = Guid.NewGuid(); _result = IdResult.IdMapped(_id); }