public static List <Encoding> GetFileEncoding(byte[] arrayByte) { List <Encoding> result = new List <Encoding>(); CharsetDetector detector = new CharsetDetector(); detector.Feed(arrayByte, 0, arrayByte.Length); detector.DataEnd(); if (!string.IsNullOrEmpty(detector.Charset) && detector.Confidence > 0.8f) { try { Encoding enc = Encoding.GetEncoding(detector.Charset); result.Add(enc); } catch (Exception ex) { DTEHelper.WriteExceptionToOutput(null, ex); #if DEBUG if (System.Diagnostics.Debugger.IsAttached) { System.Diagnostics.Debugger.Break(); } #endif } } return(result); }
public static bool TryGetEncoding(byte[] data, out Encoding encoding) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(data, 0, data.Length); cdet.DataEnd(); if (cdet.Charset != null) { if (cdet.Charset.ToLowerInvariant() == "big-5") { encoding = Encoding.GetEncoding("big5"); return(true); } else { try { encoding = Encoding.GetEncoding(cdet.Charset); return(true); } catch { encoding = Encoding.Default; return(false); } } } encoding = Encoding.Default; return(false); }
public static string LoadFile(FileInfo fileInfo) { string result; Encoding encoding = CurrentEncoding; using (Stream stream = fileInfo.OpenRead()) { CharsetDetector detector = new CharsetDetector(); detector.Feed(stream); detector.DataEnd(); stream.Position = 0; if (detector.Charset != null) { Debug.LogFormat("Detected charset of file: {0}", detector.Charset); try { encoding = Encoding.GetEncoding(detector.Charset); } catch { Debug.LogWarning("Failed to load encoding, will use default encoding."); encoding = CurrentEncoding; } } else { Debug.LogFormat("Failed to detect charset, will use default encoding."); } using (StreamReader reader = new StreamReader(stream, encoding)) result = reader.ReadToEnd(); } return(result); }
private string DetectCharset(byte[] bytes) { try { using (var ms = _memoryStreamProvider.CreateNew(bytes)) { var detector = new CharsetDetector(); detector.Feed(ms); detector.DataEnd(); var charset = detector.Charset; if (!string.IsNullOrWhiteSpace(charset)) { //_logger.Debug("UniversalDetector detected charset {0}", charset); } return(charset); } } catch (IOException ex) { _logger.ErrorException("Error attempting to determine web socket message charset", ex); } return(null); }
private string DetectCharset(string path) { try { using (var file = new FileStream(path, FileMode.Open)) { var detector = new CharsetDetector(); detector.Feed(file); detector.DataEnd(); var charset = detector.Charset; if (!string.IsNullOrWhiteSpace(charset)) { _logger.Info("UniversalDetector detected charset {0} for {1}", charset, path); } return(charset); } } catch (IOException ex) { _logger.ErrorException("Error attempting to determine subtitle charset from {0}", ex, path); } return(null); }
public static Encoding GetEncoding(byte[] data) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(data, 0, data.Length); cdet.DataEnd(); if (cdet.Charset != null && cdet.Confidence > 0.5) { if (cdet.Charset.ToLowerInvariant() == "big-5") { return(Encoding.GetEncoding("big5")); } else { try { return(Encoding.GetEncoding(cdet.Charset)); } catch { return(Encoding.Default); } } } return(null); }
/// <summary> /// Command line example: detects the encoding of the given file. /// </summary> /// <param name="args">a filename</param> public static void Main(string[] args) { if (args.Length == 0) { Console.WriteLine("Usage: udetect <filename>"); return; } var filename = args[0]; using (var fs = File.OpenRead(filename)) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); Console.WriteLine(); if (cdet.Charset != null) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Charset: {0}, confidence: {1}", cdet.Charset, cdet.Confidence); } else { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Detection failed."); } Console.ResetColor(); } Exit(); }
/// <summary> /// Command line example: detects the encoding of the given file. /// </summary> /// <param name="args">a filename</param> public static void Main(String[] args) { if (args.Length == 0) { Console.WriteLine("Usage: udetect <filename>"); return; } string filename = args[0]; using (FileStream fs = File.OpenRead(filename)) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); if (cdet.Charset != null) { Console.WriteLine("Charset: {0}, confidence: {1}", cdet.Charset, cdet.Confidence); } else { Console.WriteLine("Detection failed."); } } }
private async Task <string> DetectCharset(string path, MediaProtocol protocol, CancellationToken cancellationToken) { try { using (var file = await GetStream(path, protocol, cancellationToken).ConfigureAwait(false)) { var detector = new CharsetDetector(); detector.Feed(file); detector.DataEnd(); var charset = detector.Charset; if (!string.IsNullOrWhiteSpace(charset)) { _logger.Info("UniversalDetector detected charset {0} for {1}", charset, path); } return(charset); } } catch (IOException ex) { _logger.ErrorException("Error attempting to determine subtitle charset from {0}", ex, path); } return(null); }
public Encoding Detect(Stream stream) { var detector = new CharsetDetector(); detector.Feed(stream); detector.DataEnd(); return(detector.Charset.Return(Encoding.GetEncoding, null)); }
private static Encoding GetEncoding(string path) { using (var stream = File.OpenRead(path)) { var detector = new CharsetDetector(); detector.Feed(stream); detector.DataEnd(); return(detector.Charset == null ? null : Encoding.GetEncoding(detector.Charset)); } }
public static Encoding DetectEncoding(string filename) { using (var fs = File.OpenRead(filename)) { var cd = new CharsetDetector(); cd.Feed(fs); cd.DataEnd(); return(cd.Charset != null?Encoding.GetEncoding(cd.Charset) : Encoding.UTF8); } }
private static Encoding GetEncoding(byte[] data) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(data, 0, data.Length); cdet.DataEnd(); if (cdet.Charset != null && cdet.Confidence > 0.5) { return(Encoding.GetEncoding(cdet.Charset)); } return(null); }
public static Encoding GetFileEncoding(string fullPath) { using (var fs = File.OpenRead(fullPath)) { var charsetDetector = new CharsetDetector(); charsetDetector.Feed(fs); charsetDetector.DataEnd(); fs.Close(); return(EncodingHelper.GetEncodingFromCharSet(charsetDetector.Charset)); } }
private static void ActionWorkerDoWork(object sender, DoWorkEventArgs e) { BackgroundWorker worker = (BackgroundWorker)sender; WorkerArgs args = (WorkerArgs)e.Argument; string[] allFiles = Directory.GetFiles(args.BaseDirectory, "*.*", args.IncludeSubdirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); IEnumerable <Regex> maskPatterns = GenerateMaskPatterns(args.FileMasks); for (int i = 0; i < allFiles.Length; i++) { if (worker.CancellationPending) { e.Cancel = true; break; } string path = allFiles[i]; string fileName = Path.GetFileName(path); if (!SatisfiesMaskPatterns(fileName, maskPatterns)) { continue; } CharsetDetector detector = new CharsetDetector(); using (FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read)) { detector.Feed(fs); detector.DataEnd(); } if (args.Action == CurrentAction.Validate) { if (detector.Charset == null) { continue; } if (args.ValidCharsets.Contains(detector.Charset)) { continue; } } string directoryName = Path.GetDirectoryName(path); int percentageCompleted = (i * 100) / allFiles.Length; WorkerProgress progress = new WorkerProgress(); progress.Charset = detector.Charset ?? "(Unknown)"; progress.FileName = fileName; progress.DirectoryName = directoryName; worker.ReportProgress(percentageCompleted, progress); } }
private static string DetectFileCharset(string filePath) { string charset; using (var fs = File.OpenRead(filePath)) { var charsetDetector = new CharsetDetector(); charsetDetector.Feed(fs); charsetDetector.DataEnd(); charset = charsetDetector.Charset; } return(charset); }
private static ICharsetDetector GetCharsetDetector(string file) { ICharsetDetector cdet; using (var fs = File.OpenRead(file)) { cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); } return(cdet); }
/// <summary> /// 指定されたバイト配列の文字コードを推定します。 /// </summary> /// <param name="bytes">バイト配列</param> /// <param name="verifiedLength">検証される先頭からのバイト長</param> /// <returns>文字コード</returns> public static Encoding DetectEncoding(byte[] bytes, int verifiedLength) { const int MIN_LENGTH = 256; // 空の場合は null if (bytes.Length == 0) { return(null); } // BOM による判定 var bom = DetectEncodingByBom(bytes); if (bom != null) { return(bom); } // バイト配列の整形 if (bytes.Length < MIN_LENGTH) { // バイト長が短すぎる場合は複製して補う var count = MIN_LENGTH / bytes.Length + 1; var buffer = new byte[count * bytes.Length]; for (var i = 0; i < count; i++) { Array.Copy(bytes, 0, buffer, bytes.Length * i, bytes.Length); } bytes = buffer; } else if (MIN_LENGTH <= verifiedLength && verifiedLength < bytes.Length) { // バイト長が指定されている場合は切り出す var buffer = new byte[verifiedLength]; Array.Copy(bytes, 0, buffer, 0, verifiedLength); bytes = buffer; } // Mozilla Universal Charset Detector による判定 var ude = new CharsetDetector(); ude.Feed(bytes, 0, bytes.Length); ude.DataEnd(); if (string.IsNullOrEmpty(ude.Charset) == false) { return(Encoding.GetEncoding(ude.Charset)); } // いずれによっても判定できない場合は null return(null); }
/// <summary> /// 检测一个外部文件的编码 /// </summary> /// <param name="path"></param> /// <returns></returns> public static Encoding GetEncoding(string path) { using (var fs = File.OpenRead(path)) { CharsetDetector detector = new CharsetDetector(); detector.Feed(fs); detector.DataEnd(); if (!string.IsNullOrEmpty(detector.Charset)) { return(Encoding.GetEncoding(detector.Charset)); } } return(null); }
private static Encoding DetectWithUDE(FileInfo file, Encoding targetEncoding) { var detector = new CharsetDetector(); using (var stream = file.OpenRead()) detector.Feed(stream); detector.DataEnd(); if (targetEncoding.WebName.Equals(detector.Charset, StringComparison.InvariantCultureIgnoreCase)) { return(null); } if (string.IsNullOrEmpty(detector.Charset)) { Console.WriteLine($"[-] Charset undetected in file '{file.Name}'- skipped!"); return(null); } if (detector.Confidence < 0.9) { Console.WriteLine($"[-] Charset detected as '{detector.Charset}' with confidence {Math.Round(detector.Confidence * 100)}% in file '{file.Name}' - skipped - too low confidence!"); return(null); } Console.WriteLine($"[+] Charset detected as '{detector.Charset}' with confidence {Math.Round(detector.Confidence * 100)}% in file '{file.Name}'"); var charset = detector.Charset; if (charset == "windows-1252") // incorrect detection in our case - may not be correct in all cases { charset = "windows-1250"; } Encoding detectedEncoding; try { detectedEncoding = Encoding.GetEncoding(charset); } catch (ArgumentException) { Console.WriteLine($"[!] Encoding '{detector.Charset}' not recognized by .NET Framework"); return(null); } return(detectedEncoding); }
public static Encoding DetectEncoding(Stream stream) { if (stream == null) { return(null); } var detector = new CharsetDetector(); detector.Feed(stream); detector.DataEnd(); if (detector.Charset == null) { return(null); } return(Encoding.GetEncoding(detector.Charset)); }
private static Encoding findEncoding(string path) { using (FileStream fs = File.OpenRead(path)) { CharsetDetector cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); if (cdet.Charset != null) { return(Encoding.GetEncoding(cdet.Charset)); } } return(Encoding.ASCII); }
private static Encoding GetEncoding(string FullFileName) { var detector = new CharsetDetector(); byte[] bytes = File.ReadAllBytes(FullFileName); detector.Feed(bytes, 0, bytes.Length); detector.DataEnd(); string encoding = detector.Charset; if (encoding == "windows-1255") { encoding = "windows-1251"; } Encoding enc = Encoding.GetEncoding(encoding); return(enc); }
public static string DetectCharset(byte[] bytes) { try { var charsetDetector = new CharsetDetector(); charsetDetector.Feed(bytes, 0, bytes.Length); charsetDetector.DataEnd(); return(charsetDetector.Charset == null ? null : charsetDetector.Charset.ToLowerInvariant()); } catch (Exception) { return(null); } }
public static Encoding DetectCharset(Byte[] bytes) { Encoding detectedCharset = Encoding.ASCII; CharsetDetector detector = new CharsetDetector(); detector.Feed(bytes); detector.DataEnd(); if (detector.Confidence > 0.5) { detectedCharset = Encoding.GetEncoding(detector.Charset); } return(detectedCharset); }
public static string GetEncoding(string filename, out float confidence) { confidence = 0.0f; using (var fs = File.OpenRead(filename)) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); if (cdet.Charset != null) { Logger.Log($"Charset: {cdet.Charset}, confidence: {cdet.Confidence}"); confidence = cdet.Confidence; return(cdet.Charset); } Logger.Log($"{filename}: Detection failed."); return("UTF-8"); } }
private void CSVImport_Load(object sender, EventArgs e) { using (FileStream fs = File.OpenRead(CsvFile)) { CharsetDetector cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); if (cdet.Charset != null) { tfp = new TextFieldParser(CsvFile, Encoding.GetEncoding(cdet.Charset)); } else { tfp = new TextFieldParser(CsvFile, Encoding.ASCII); } } }
private static Encoding GetEncoding(string path) { try { using (var stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { var detector = new CharsetDetector(); detector.Feed(stream); detector.DataEnd(); return(detector.Charset == null ? null : Encoding.GetEncoding(detector.Charset)); } } catch { //File locks, etc. return(null); } }
/// <summary> /// Constructs a new <see cref="DynamicFileByteProvider" /> instance. /// </summary> /// <param name="fileName">The name of the file from which bytes should be provided.</param> /// <param name="readOnly">True, opens the file in read-only mode.</param> public DynamicFileByteProvider(string fileName, bool readOnly, bool isGoToGetDefaultCharset) { _fileName = fileName; if (!readOnly) { _stream = File.Open(fileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); } else { _stream = File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } _readOnly = readOnly; if (isGoToGetDefaultCharset) { try { CharsetDetector detector = new CharsetDetector(); detector.Feed(_stream); detector.DataEnd(); if (detector.Charset.IsValid()) { this._encoding = Encoding.GetEncoding(detector.Charset); } else { this._encoding = Encoding.ASCII; } } catch (Exception) { this._encoding = Encoding.ASCII; } } ReInitialize(); }
public static string DetectEncoding(byte[] decompressbuff, HttpWebResponse response) { var ms = new MemoryStream(decompressbuff); var cdet = new CharsetDetector(); cdet.Feed(ms); cdet.DataEnd(); ms.Close(); var encoding = cdet.Charset; if (encoding != null) { return(encoding); } encoding = response.CharacterSet; if (!String.IsNullOrEmpty(encoding) && encoding.IndexOf(',') != -1) { encoding = encoding.Split(',')[0]; } if (response.ResponseUri.ToString().EndsWith(".xml") || (response.ContentType == "text/xml")) { encoding = GetXmlEncoding(decompressbuff); } else if (string.IsNullOrEmpty(encoding) || (string.Compare(encoding, "ISO-8859-1") == 0)) { encoding = GetEncodingFromBuffer(decompressbuff); } if (!string.IsNullOrEmpty(encoding) && encoding.ToLower() == "gbk") { encoding = "GB18030"; } if (!CharsetExists(encoding)) { encoding = "utf-8"; } return(encoding); }
/// <summary> /// Command line example: detects the encoding of the given file. /// </summary> /// <param name="args">a filename</param> public static void Main(String[] args) { if (args.Length == 0) { Console.WriteLine("Usage: udetect <filename>"); return; } var filename = args[0]; using (var fs = File.OpenRead(filename)) { ICharsetDetector cdet = new CharsetDetector(); cdet.Feed(fs); cdet.DataEnd(); if (cdet.Charset != null) { Console.WriteLine("Charset: {0}, confidence: {1}", cdet.Charset, cdet.Confidence); } else Console.WriteLine("Detection failed."); } }
private async Task<string> DetectCharset(string path, string language, MediaProtocol protocol, CancellationToken cancellationToken) { try { using (var file = await GetStream(path, protocol, cancellationToken).ConfigureAwait(false)) { var detector = new CharsetDetector(); detector.Feed(file); detector.DataEnd(); var charset = detector.Charset; if (!string.IsNullOrWhiteSpace(charset)) { _logger.Info("UniversalDetector detected charset {0} for {1}", charset, path); } // This is often incorrectly indetected. If this happens, try to use other techniques instead if (string.Equals("x-mac-cyrillic", charset, StringComparison.OrdinalIgnoreCase)) { if (!string.IsNullOrWhiteSpace(language)) { return null; } } return charset; } } catch (IOException ex) { _logger.ErrorException("Error attempting to determine subtitle charset from {0}", ex, path); } return null; }
private async Task<string> DetectCharset(string path, MediaProtocol protocol, CancellationToken cancellationToken) { try { using (var file = await GetStream(path, protocol, cancellationToken).ConfigureAwait(false)) { var detector = new CharsetDetector(); detector.Feed(file); detector.DataEnd(); var charset = detector.Charset; if (!string.IsNullOrWhiteSpace(charset)) { _logger.Info("UniversalDetector detected charset {0} for {1}", charset, path); } return charset; } } catch (IOException ex) { _logger.ErrorException("Error attempting to determine subtitle charset from {0}", ex, path); } return null; }