public ActionResult DownloadTags(int id = invalidId, string formatString = "", bool setFormatString = false, bool includeHeader = false) { if (id == invalidId) { return(NoId()); } if (setFormatString) { userQueries.SetAlbumFormatString(formatString); } else if (string.IsNullOrEmpty(formatString) && LoginManager.IsLoggedIn) { formatString = LoginManager.LoggedUser.AlbumFormatString; } if (string.IsNullOrEmpty(formatString)) { formatString = TagFormatter.TagFormatStrings[0]; } var album = Service.GetAlbum(id); var tagString = Service.GetAlbumTagString(id, formatString, includeHeader); var enc = new UTF8Encoding(true); var data = enc.GetPreamble().Concat(enc.GetBytes(tagString)).ToArray(); return(File(data, "text/csv", album.Name + ".csv")); }
public void WhenLimitIsNotSpecifiedAndEncodingHasNoPreambleDataIsCorrectlyAppendedToFileSink() { var encoding = new UTF8Encoding(false); Assert.Equal(0, encoding.GetPreamble().Length); WriteTwoEventsAndCheckOutputFileLength(null, encoding); }
// http://www.unicode.org/faq/utf_bom.html public static bool IsByteArrayStartsWithUtf8Bom(byte[] first3Bytes) { if (first3Bytes == null || first3Bytes.Length <= 2) { return(false); } if (sUtf8Bom == null) { var utf8WithBom = new UTF8Encoding(true); sUtf8Bom = utf8WithBom.GetPreamble(); } // NO need to check which is shorter... // var shorter = sUtf8Bom.Length <= first3Bytes.Length ? sUtf8Bom : first3Bytes; for (int i = 0; i < sUtf8Bom.Length; ++i) { if (sUtf8Bom[i] != first3Bytes[i]) { return(false); } } // return preamble.Where( ( p, i ) => p != bytes[i] ).Any(); return(true); }
// For the input string, it simulates a XML string with or without Byte Order Mark (BOM) private void UTF8String(bool useBOM) { UTF8Encoding encoder = new UTF8Encoding(useBOM); byte[] bytesBOM = null; byte[] bytesData = null; byte[] bytesString = null; // Generating BOM if the flag is true if (useBOM) { bytesBOM = encoder.GetPreamble(); } // Generating data bytesData = encoder.GetBytes(_xmlSample); // Adding BOM if the flag is true if (useBOM) { bytesString = bytesBOM.Concat(bytesData).ToArray(); } else { bytesString = bytesData; } // Encoding to string _data.StringValue = Encoding.UTF8.GetString(bytesString); this.propertyGridResult.SelectedObject = _data; }
public static void DefaultEncodingBOMTest() { UTF8Encoding defaultEncoding = Encoding.Default as UTF8Encoding; Assert.True(defaultEncoding != null); Assert.Equal(0, defaultEncoding.GetPreamble().Length); }
public static void Print(this Encoding encoding) { if (encoding == null) { Console.WriteLine("NULL"); } //Console.Write("EncodingName: {0}, BodyName: {1}, WebName:{2}", encoding.EncodingName, encoding.BodyName, encoding.WebName); Console.Write("{0}", encoding.WebName); UTF8Encoding utf8 = encoding as UTF8Encoding; if (utf8 != null) { var arr = utf8.GetPreamble(); if (arr != null) { Console.Write(", WithBOM: {0}", string.Join("", arr.Select(i => i.ToString("X")))); } else { Console.Write(", WithBOM: {0}", "NONE"); } } Console.WriteLine(); }
static void UTF_8_BOM() { Encoding encodingUTF8 = new UTF8Encoding(true); byte[] preamble = encodingUTF8.GetPreamble(); foreach (var s in Selection.GetFiltered(typeof(TextAsset), SelectionMode.DeepAssets)) { TextAsset text = (TextAsset)s; string path = Path.Combine(Application.dataPath, AssetDatabase.GetAssetPath(text).Substring(7)); Debug.Log("reading " + path); byte[] bytes = File.ReadAllBytes(path); if (bytes[0] != preamble[0] || bytes[1] != preamble[1] || bytes[2] != preamble[2]) { byte[] newBytes = new byte[bytes.LongLength + 3]; Array.Copy(preamble, 0, newBytes, 0, preamble.Length); Array.Copy(bytes, 0, newBytes, 3, bytes.LongLength); File.WriteAllBytes(path, newBytes); Debug.Log("Convert done." + newBytes.Length); } else { Debug.Log("File already with BOM."); } } }
public void PosTest1() { UTF8Encoding UTF8NoPreamble = new UTF8Encoding(); Byte[] preamble; preamble = UTF8NoPreamble.GetPreamble(); }
public void PosTest1() { UTF8Encoding UTF8NoPreamble = new UTF8Encoding(); Byte[] preamble; preamble = UTF8NoPreamble.GetPreamble(); }
public static void Main() { UTF8Encoding utf8 = new UTF8Encoding(true, true); String s = "It was the best of times, it was the worst of times..."; // We need to dimension the array, since we'll populate it with 2 method calls. Byte[] bytes = new Byte[utf8.GetByteCount(s) + utf8.GetPreamble().Length]; // Encode the string. Array.Copy(utf8.GetPreamble(), bytes, utf8.GetPreamble().Length); utf8.GetBytes(s, 0, s.Length, bytes, utf8.GetPreamble().Length); // Decode the byte array. String s2 = utf8.GetString(bytes, 0, bytes.Length); Console.WriteLine(s2); }
public FileContentResult DownloadTags(int id) { var album = Service.GetAlbum(id); var tagString = Service.GetAlbumTagString(id, TagFormatter.TagFormatStrings[0]); var enc = new UTF8Encoding(true); var data = enc.GetPreamble().Concat(enc.GetBytes(tagString)).ToArray(); return(File(data, "text/csv", album.Name + ".csv")); }
static void Main(string[] args) { OutputEncoding = Encoding.UTF8; // Create a UTF-8 encoding that supports a BOM. Encoding utf8 = new UTF8Encoding(true); // A Unicode string with two characters outside an 8-bit code range. String unicodeString = "This Unicode string has 6 characters outside the " + "ASCII range:\n" + "Pi (\u03A0)), and Sigma (\u03A3), Ę(\u0119), LATIN SMALL LIGATURE FF(\uFB00), SMALL LETTER T WITH STROKE(\u0167), Alpha(\u03b1)"; Console.WriteLine("Original string:"); Console.WriteLine(unicodeString); Console.WriteLine(); // Encode the string. Byte[] encodedBytes = utf8.GetBytes(unicodeString); Console.WriteLine("The encoded string has {0} bytes.", encodedBytes.Length); Console.WriteLine(); // Write the bytes to a file with a BOM. var fs = new FileStream(@".\UTF8Encoding.txt", FileMode.Create); Byte[] bom = utf8.GetPreamble(); fs.Write(bom, 0, bom.Length); fs.Write(encodedBytes, 0, encodedBytes.Length); Console.WriteLine("Wrote {0} bytes to the file.", fs.Length); fs.Close(); Console.WriteLine(); // Open the file using StreamReader. var sr = new StreamReader(@".\UTF8Encoding.txt"); String newString = sr.ReadToEnd(); sr.Close(); Console.WriteLine("String read using StreamReader:"); Console.WriteLine(newString); Console.WriteLine(); // Open the file as a binary file and decode the bytes back to a string. fs = new FileStream(@".\UTF8Encoding.txt", FileMode.Open); Byte[] bytes = new Byte[fs.Length]; fs.Read(bytes, 0, (int)fs.Length); fs.Close(); String decodedString = utf8.GetString(bytes); Console.WriteLine("Decoded bytes:"); Console.WriteLine(decodedString); }
public ActionResult Export(int id) { var songList = queries.GetSongList(id); var formatString = "%notes%;%publishdate%;%title%;%url%;%pv.original.niconicodouga%;%pv.original.!niconicodouga%;%pv.reprint%"; var tagString = queries.GetTagString(id, formatString); var enc = new UTF8Encoding(true); var data = enc.GetPreamble().Concat(enc.GetBytes(tagString)).ToArray(); return(File(data, "text/csv", songList.Name + ".csv")); }
public static void Main() { UTF8Encoding utf8 = new UTF8Encoding(); UTF8Encoding utf8EmitBOM = new UTF8Encoding(true); Console.WriteLine("utf8 preamble:"); ShowArray(utf8.GetPreamble()); Console.WriteLine("utf8EmitBOM:"); ShowArray(utf8EmitBOM.GetPreamble()); }
public ActionResult Export(int id) { var songList = queries.GetSongList(id); var formatString = "%notes%;%publishdate%;%title%;%url%;%pv.original.niconicodouga%;%pv.original.!niconicodouga%;%pv.reprint%"; var tagString = queries.HandleQuery(ctx => new SongListFormatter(entryLinkFactory).ApplyFormat(ctx.Load(id), formatString, PermissionContext.LanguagePreference, true)); var enc = new UTF8Encoding(true); var data = enc.GetPreamble().Concat(enc.GetBytes(tagString)).ToArray(); return(File(data, "text/csv", songList.Name + ".csv")); }
private static void CoreSerializeWithBOM(object message, Stream stream) { var inputMessage = CoreJsonSerializer.Serialize(message); var encoding = new UTF8Encoding(true); stream.Write(encoding.GetPreamble()); var bytes = encoding.GetBytes(inputMessage); stream.Write(bytes); stream.Position = 0; }
public static void Run() { // Create a UTF-8 encoding that supports a BOM. Encoding utf8 = new UTF8Encoding(true); // A Unicode string with two characters outside an 8-bit code range. String unicodeString = "This Unicode string has 2 characters outside the " + "ASCII range:\n" + "Pi (\u03A0)), and Sigma (\u03A3)."; Console.WriteLine("Original string:"); Console.WriteLine(unicodeString); Console.WriteLine(); // Encode the string. Byte[] encodedBytes = utf8.GetBytes(unicodeString); Console.WriteLine("The encoded string has {0} bytes.", encodedBytes.Length); Console.WriteLine(); // Write the bytes to a file with a BOM. var fs = new FileStream(@".\UTF8Encoding.txt", FileMode.Create); Byte[] bom = utf8.GetPreamble(); fs.Write(bom, 0, bom.Length); fs.Write(encodedBytes, 0, encodedBytes.Length); Console.WriteLine("Wrote {0} bytes to the file.", fs.Length); fs.Dispose(); Console.WriteLine(); // Open the file using StreamReader. fs = new FileStream(@".\UTF8Encoding.txt", FileMode.Open); var sr = new StreamReader(fs); String newString = sr.ReadToEnd(); fs.Dispose(); sr.Dispose(); Console.WriteLine("String read using StreamReader:"); Console.WriteLine(newString); Console.WriteLine(); // Open the file as a binary file and decode the bytes back to a string. fs = new FileStream(@".\UTF8Encoding.txt", FileMode.Open); Byte[] bytes = new Byte[fs.Length]; fs.Read(bytes, 0, (int)fs.Length); fs.Dispose(); String decodedString = utf8.GetString(encodedBytes); Console.WriteLine("Decoded bytes:"); Console.WriteLine(decodedString); }
private static byte[] RemoveByteOrderMarkIfFound(byte[] bytes) { var enc = new UTF8Encoding(true); var preamble = enc.GetPreamble(); if (preamble.Where((p, i) => p != bytes[i]).Any()) { return(bytes); } return(bytes.Skip(preamble.Length).ToArray()); }
private static string ConvertFromUtf8(byte[] bytes) { var enc = new UTF8Encoding(true); var preamble = enc.GetPreamble(); if (preamble.Where((p, i) => p != bytes[i]).Any()) { throw new ArgumentException("Not utf8-BOM"); } return(enc.GetString(bytes.Skip(preamble.Length).ToArray())); }
public bool TestIsUTF8WithBOM() { var reader = new StreamReader(FileSystem.GetFile(XmlPath).OpenRead()); using (var memstream = new MemoryStream()) { reader.BaseStream.CopyTo(memstream); var bytes = memstream.ToArray(); var encoding = new UTF8Encoding(true); var preamble = encoding.GetPreamble(); return(preamble.Where((p, i) => p == bytes[i]).Any()); } }
private static void VerifyUtf8Encoding(UTF8Encoding encoding, bool encoderShouldEmitUTF8Identifier, bool throwOnInvalidBytes) { if (encoderShouldEmitUTF8Identifier) { Assert.Equal(new byte[] { 0xEF, 0xBB, 0xBF }, encoding.GetPreamble()); } else { Assert.Empty(encoding.GetPreamble()); } if (throwOnInvalidBytes) { Assert.Equal(EncoderFallback.ExceptionFallback, encoding.EncoderFallback); Assert.Equal(DecoderFallback.ExceptionFallback, encoding.DecoderFallback); } else { Assert.Equal(new EncoderReplacementFallback("\uFFFD"), encoding.EncoderFallback); Assert.Equal(new DecoderReplacementFallback("\uFFFD"), encoding.DecoderFallback); } }
public static void VerifyUtf8Encoding(UTF8Encoding encoding, bool encoderShouldEmitUTF8Identifier, bool throwOnInvalidBytes) { if (encoderShouldEmitUTF8Identifier) { Assert.Equal(new byte[] { 0xEF, 0xBB, 0xBF }, encoding.GetPreamble()); } else { Assert.Empty(encoding.GetPreamble()); } if (throwOnInvalidBytes) { Assert.Equal(EncoderFallback.ExceptionFallback, encoding.EncoderFallback); Assert.Equal(DecoderFallback.ExceptionFallback, encoding.DecoderFallback); } else { Assert.Equal(new EncoderReplacementFallback("\uFFFD"), encoding.EncoderFallback); Assert.Equal(new DecoderReplacementFallback("\uFFFD"), encoding.DecoderFallback); } }
public override void DrawConfigurables() { if (GUILayout.Button("Write Transaction Database to CSV")) { string baseName = string.Format( "{0}-{1}.csv", HighLogic.CurrentGame.Title, (int)Planetarium.GetUniversalTime() ); var file = KSP.IO.File.Create <VOID_CareerTracker>(baseName, null); UTF8Encoding enc = new UTF8Encoding(true); byte[] lineBytes = enc.GetPreamble(); file.Write(lineBytes, 0, lineBytes.Length); string transLine = string.Format( "{0}, {1}, \"{2}\", \"{3}\", \"{4}\"\n", "TimeStamp", "Reason", "Funds Delta", "Science Delta", "Reputation Delta" ); lineBytes = enc.GetBytes(transLine); file.Write(lineBytes, 0, lineBytes.Length); CurrencyTransaction trans; for (int idx = 0; idx < Tracker.TransactionList.Count; idx++) { trans = Tracker.TransactionList[idx]; transLine = string.Format( "{0}, \"{1}\", {2}, {3}, {4}\n", trans.TimeStamp.ToString("F2"), Enum.GetName(typeof(TransactionReasons), trans.Reason), trans.FundsDelta.ToString("F2"), trans.ScienceDelta.ToString("F2"), trans.ReputationDelta.ToString("F2") ); lineBytes = enc.GetBytes(transLine); file.Write(lineBytes, 0, lineBytes.Length); } } }
public void EncodingBOM() { var utf8BOM = new UTF8Encoding(encoderShouldEmitUTF8Identifier: true); var bytes = utf8BOM.GetPreamble().Concat(utf8BOM.GetBytes("abc")).ToArray(); Assert.Equal(utf8BOM, SourceText.From(bytes, bytes.Length, s_unicode).Encoding); Assert.Equal(utf8BOM, SourceText.From(bytes, bytes.Length, null).Encoding); var stream = new MemoryStream(bytes); Assert.Equal(utf8BOM, SourceText.From(stream, s_unicode).Encoding); Assert.Equal(utf8BOM, SourceText.From(stream, null).Encoding); }
public static bool FileHasBom(Stream stream) { using (var ms = new MemoryStream()) { stream.CopyTo(ms); var bytes = ms.ToArray(); // be nice and reset position stream.Position = 0; var enc = new UTF8Encoding(true); var preamble = enc.GetPreamble(); return(preamble.Where((p, i) => p == bytes[i]).Any()); } }
protected void lnkDownload_Click(object sender, EventArgs e) { gvDownload.DataSource = Endorsement.EndorsementsForUser(Student, Instructor); gvDownload.DataBind(); Response.ContentType = "text/csv"; // Give it a name that is the brand name, user's name, and date. Convert spaces to dashes, and then strip out ANYTHING that is not alphanumeric or a dash. string szFilename = String.Format(CultureInfo.CurrentCulture, "{0}-{1}-{2}", Branding.CurrentBrand.AppName, Resources.SignOff.DownloadEndorsementsFilename, String.IsNullOrEmpty(Student) ? Resources.SignOff.DownloadEndorsementsAllStudents : MyFlightbook.Profile.GetUser(Student).UserFullName); string szDisposition = String.Format(CultureInfo.InvariantCulture, "attachment;filename={0}.csv", Regex.Replace(szFilename, "[^0-9a-zA-Z-]", "")); Response.AddHeader("Content-Disposition", szDisposition); UTF8Encoding enc = new UTF8Encoding(true); // to include the BOM Response.Write(Encoding.UTF8.GetString(enc.GetPreamble())); // UTF-8 BOM Response.Write(gvDownload.CSVFromData()); Response.End(); }
static EncodingUtility() { Encoding utf32BE = new UTF32Encoding(true, true, true); Encoding utf32LE = new UTF32Encoding(false, true, true); Encoding utf16BE = new UnicodeEncoding(true, true, true); Encoding utf16LE = new UnicodeEncoding(false, true, true); Encoding utf8BOM = new UTF8Encoding(true, true); encodingLookup = new KeyValuePair <byte[], Encoding>[] { new KeyValuePair <byte[], Encoding>(utf32BE.GetPreamble(), utf32BE), new KeyValuePair <byte[], Encoding>(utf32LE.GetPreamble(), utf32LE), new KeyValuePair <byte[], Encoding>(utf16BE.GetPreamble(), utf16BE), new KeyValuePair <byte[], Encoding>(utf16LE.GetPreamble(), utf16LE), new KeyValuePair <byte[], Encoding>(utf8BOM.GetPreamble(), utf8BOM), }; }
protected override void Render(HtmlTextWriter writer) { if (OverrideRender) { StringBuilder sbOut = new StringBuilder(); using (StringWriter swOut = new StringWriter(sbOut, CultureInfo.InvariantCulture)) { UTF8Encoding enc = new UTF8Encoding(true); swOut.Write(Encoding.UTF8.GetString(enc.GetPreamble())); using (HtmlTextWriter htwOut = new HtmlTextWriter(swOut)) { base.Render(htwOut); } } PDFRenderer.RenderFile(sbOut.ToString(), PageOptions, () => // onError { util.NotifyAdminEvent("Error saving PDF", String.Format(CultureInfo.CurrentCulture, "User: {0}\r\nOptions:\r\n{1}\r\n\r\nQuery:{2}\r\n", CurrentUser.UserName, JsonConvert.SerializeObject(PageOptions), JsonConvert.SerializeObject(PrintOptions1.Options)), ProfileRoles.maskSiteAdminOnly); Response.Redirect(Request.Url.PathAndQuery + (Request.Url.PathAndQuery.Contains("?") ? "&" : "?") + "pdfErr=1"); }, (szOutputPDF) => // onSuccess { Page.Response.Clear(); Page.Response.ContentType = "application/pdf"; Response.AddHeader("content-disposition", String.Format(CultureInfo.CurrentCulture, @"attachment;filename=""{0}.pdf""", CurrentUser.UserFullName)); Response.WriteFile(szOutputPDF); Page.Response.Flush(); // See http://stackoverflow.com/questions/20988445/how-to-avoid-response-end-thread-was-being-aborted-exception-during-the-exce for the reason for the next two lines. Page.Response.SuppressContent = true; // Gets or sets a value indicating whether to send HTTP content to the client. HttpContext.Current.ApplicationInstance.CompleteRequest(); // Causes ASP.NET to bypass all events and filtering in the HTTP pipeline chain of execution and directly execute the EndRequest event. }); } else { base.Render(writer); } }
public void Check(string csFile) { //string csFile = file.FullName; byte[] bytes = File.ReadAllBytes(csFile); UTF8Encoding enc = new UTF8Encoding(true); byte[] preamble = enc.GetPreamble(); if (preamble.Where((p, i) => p != bytes[i]).Any()) { OnError($"EE\tNo utf8 BOM: {csFile}"); } else { OnInformation($"||\tUtf8 BOM OK: {csFile}"); } // return enc.GetString(bytes.Skip(preamble.Length).ToArray()); }
static EncodingUtility() { TextAsset.EncodingUtility.targetEncoding = Encoding.GetEncoding(Encoding.UTF8.CodePage, new EncoderReplacementFallback("�"), new DecoderReplacementFallback("�")); Encoding encoding = new UTF32Encoding(true, true, true); Encoding encoding2 = new UTF32Encoding(false, true, true); Encoding encoding3 = new UnicodeEncoding(true, true, true); Encoding encoding4 = new UnicodeEncoding(false, true, true); Encoding encoding5 = new UTF8Encoding(true, true); TextAsset.EncodingUtility.encodingLookup = new KeyValuePair <byte[], Encoding>[] { new KeyValuePair <byte[], Encoding>(encoding.GetPreamble(), encoding), new KeyValuePair <byte[], Encoding>(encoding2.GetPreamble(), encoding2), new KeyValuePair <byte[], Encoding>(encoding3.GetPreamble(), encoding3), new KeyValuePair <byte[], Encoding>(encoding4.GetPreamble(), encoding4), new KeyValuePair <byte[], Encoding>(encoding5.GetPreamble(), encoding5) }; }
public void Should_handle_message_without_UTF8_BOM() { var messageMapper = new MessageMapper(); var serializer = new JsonMessageSerializer(messageMapper, null, null, null, null); var utf8WithoutBomEncoding = new UTF8Encoding(false); var serialized = utf8WithoutBomEncoding .GetPreamble() .Concat(utf8WithoutBomEncoding.GetBytes($"{{\"{nameof(SimpleMessage.SomeProperty)}\":\"John\"}}")) .ToArray(); using (var stream = new MemoryStream(serialized)) { stream.Position = 0; var result = (SimpleMessage)serializer.Deserialize(stream, new[] { typeof(SimpleMessage) })[0]; Assert.AreEqual("John", result.SomeProperty); } }
public static void Main() { // The default constructor does not provide a preamble. UTF8Encoding UTF8NoPreamble = new UTF8Encoding(); UTF8Encoding UTF8WithPreamble = new UTF8Encoding(true); Byte[] preamble; preamble = UTF8NoPreamble.GetPreamble(); Console.WriteLine("UTF8NoPreamble"); Console.WriteLine(" preamble length: {0}", preamble.Length); Console.Write(" preamble: "); ShowArray(preamble); Console.WriteLine(); preamble = UTF8WithPreamble.GetPreamble(); Console.WriteLine("UTF8WithPreamble"); Console.WriteLine(" preamble length: {0}", preamble.Length); Console.Write(" preamble: "); ShowArray(preamble); }
public bool PosTest1() { bool retVal = true; // Add your scenario description here TestLibrary.TestFramework.BeginScenario("PosTest1: Verify method GetPreamble"); try { UTF8Encoding UTF8NoPreamble = new UTF8Encoding(); Byte[] preamble; preamble = UTF8NoPreamble.GetPreamble(); } catch (Exception e) { TestLibrary.TestFramework.LogError("001", "Unexpected exception: " + e); TestLibrary.TestFramework.LogInformation(e.StackTrace); retVal = false; } return retVal; }
static void UTF_8_BOM() { Encoding encodingUTF8 = new UTF8Encoding(true); byte[] preamble = encodingUTF8.GetPreamble(); foreach (var s in Selection.GetFiltered(typeof(TextAsset), SelectionMode.DeepAssets)) { TextAsset text = (TextAsset)s; string path = Path.Combine(Application.dataPath, AssetDatabase.GetAssetPath(text).Substring(7)); Debug.Log("reading " + path); byte[] bytes = File.ReadAllBytes(path); if (bytes[0] != preamble[0] || bytes[1] != preamble[1] || bytes[2] != preamble[2]) { byte[] newBytes = new byte[bytes.LongLength + 3]; Array.Copy(preamble, 0, newBytes, 0, preamble.Length); Array.Copy(bytes, 0, newBytes, 3, bytes.LongLength); File.WriteAllBytes(path, newBytes); Debug.Log("Convert done." + newBytes.Length); } else { Debug.Log("File already with BOM."); } } }