charset_test.go 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package charset
  5. import (
  6. "bytes"
  7. "encoding/xml"
  8. "io/ioutil"
  9. "runtime"
  10. "strings"
  11. "testing"
  12. "golang.org/x/text/transform"
  13. )
  14. func transformString(t transform.Transformer, s string) (string, error) {
  15. r := transform.NewReader(strings.NewReader(s), t)
  16. b, err := ioutil.ReadAll(r)
  17. return string(b), err
  18. }
  19. type testCase struct {
  20. utf8, other, otherEncoding string
  21. }
  22. // testCases for encoding and decoding.
  23. var testCases = []testCase{
  24. {"Résumé", "Résumé", "utf8"},
  25. {"Résumé", "R\xe9sum\xe9", "latin1"},
  26. {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
  27. {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
  28. {"Hello, world", "Hello, world", "ASCII"},
  29. {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
  30. {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
  31. {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
  32. {"latviešu", "latvie\xf0u", "ISO-8859-13"},
  33. {"Seònaid", "Se\xf2naid", "ISO-8859-14"},
  34. {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
  35. {"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
  36. {"nutraĵo", "nutra\xbco", "ISO-8859-3"},
  37. {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
  38. {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
  39. {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
  40. {"Kağan", "Ka\xf0an", "ISO-8859-9"},
  41. {"Résumé", "R\x8esum\x8e", "macintosh"},
  42. {"Gdańsk", "Gda\xf1sk", "windows-1250"},
  43. {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
  44. {"Résumé", "R\xe9sum\xe9", "windows-1252"},
  45. {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
  46. {"Kağan", "Ka\xf0an", "windows-1254"},
  47. {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
  48. {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
  49. {"latviešu", "latvie\xf0u", "windows-1257"},
  50. {"Việt", "Vi\xea\xf2t", "windows-1258"},
  51. {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
  52. {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
  53. {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
  54. {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
  55. {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
  56. {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
  57. {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
  58. {"㧯", "\x82\x31\x89\x38", "gb18030"},
  59. {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
  60. {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
  61. {"イウエオカ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
  62. {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
  63. {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
  64. {"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"},
  65. }
  66. func TestDecode(t *testing.T) {
  67. testCases := append(testCases, []testCase{
  68. // Replace multi-byte maximum subpart of ill-formed subsequence with
  69. // single replacement character (WhatWG requirement).
  70. {"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
  71. }...)
  72. for _, tc := range testCases {
  73. e, _ := Lookup(tc.otherEncoding)
  74. if e == nil {
  75. t.Errorf("%s: not found", tc.otherEncoding)
  76. continue
  77. }
  78. s, err := transformString(e.NewDecoder(), tc.other)
  79. if err != nil {
  80. t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
  81. continue
  82. }
  83. if s != tc.utf8 {
  84. t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
  85. }
  86. }
  87. }
  88. func TestEncode(t *testing.T) {
  89. testCases := append(testCases, []testCase{
  90. // Use Go-style replacement.
  91. {"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
  92. // U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
  93. {"Gdańsk", "Gdańsk", "ISO-8859-11"},
  94. {"\ufffd", "�", "ISO-8859-11"},
  95. {"a\xe1\x80b", "a��b", "ISO-8859-11"},
  96. }...)
  97. for _, tc := range testCases {
  98. e, _ := Lookup(tc.otherEncoding)
  99. if e == nil {
  100. t.Errorf("%s: not found", tc.otherEncoding)
  101. continue
  102. }
  103. s, err := transformString(e.NewEncoder(), tc.utf8)
  104. if err != nil {
  105. t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
  106. continue
  107. }
  108. if s != tc.other {
  109. t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
  110. }
  111. }
  112. }
  113. var sniffTestCases = []struct {
  114. filename, declared, want string
  115. }{
  116. {"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
  117. {"UTF-16LE-BOM.html", "", "utf-16le"},
  118. {"UTF-16BE-BOM.html", "", "utf-16be"},
  119. {"meta-content-attribute.html", "text/html", "iso-8859-15"},
  120. {"meta-charset-attribute.html", "text/html", "iso-8859-15"},
  121. {"No-encoding-declaration.html", "text/html", "utf-8"},
  122. {"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
  123. {"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
  124. {"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
  125. {"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
  126. {"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
  127. }
  128. func TestSniff(t *testing.T) {
  129. switch runtime.GOOS {
  130. case "nacl": // platforms that don't permit direct file system access
  131. t.Skipf("not supported on %q", runtime.GOOS)
  132. }
  133. for _, tc := range sniffTestCases {
  134. content, err := ioutil.ReadFile("testdata/" + tc.filename)
  135. if err != nil {
  136. t.Errorf("%s: error reading file: %v", tc.filename, err)
  137. continue
  138. }
  139. _, name, _ := DetermineEncoding(content, tc.declared)
  140. if name != tc.want {
  141. t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
  142. continue
  143. }
  144. }
  145. }
  146. func TestReader(t *testing.T) {
  147. switch runtime.GOOS {
  148. case "nacl": // platforms that don't permit direct file system access
  149. t.Skipf("not supported on %q", runtime.GOOS)
  150. }
  151. for _, tc := range sniffTestCases {
  152. content, err := ioutil.ReadFile("testdata/" + tc.filename)
  153. if err != nil {
  154. t.Errorf("%s: error reading file: %v", tc.filename, err)
  155. continue
  156. }
  157. r, err := NewReader(bytes.NewReader(content), tc.declared)
  158. if err != nil {
  159. t.Errorf("%s: error creating reader: %v", tc.filename, err)
  160. continue
  161. }
  162. got, err := ioutil.ReadAll(r)
  163. if err != nil {
  164. t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
  165. continue
  166. }
  167. e, _ := Lookup(tc.want)
  168. want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
  169. if err != nil {
  170. t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
  171. continue
  172. }
  173. if !bytes.Equal(got, want) {
  174. t.Errorf("%s: got %q, want %q", tc.filename, got, want)
  175. continue
  176. }
  177. }
  178. }
  179. var metaTestCases = []struct {
  180. meta, want string
  181. }{
  182. {"", ""},
  183. {"text/html", ""},
  184. {"text/html; charset utf-8", ""},
  185. {"text/html; charset=latin-2", "latin-2"},
  186. {"text/html; charset; charset = utf-8", "utf-8"},
  187. {`charset="big5"`, "big5"},
  188. {"charset='shift_jis'", "shift_jis"},
  189. }
  190. func TestFromMeta(t *testing.T) {
  191. for _, tc := range metaTestCases {
  192. got := fromMetaElement(tc.meta)
  193. if got != tc.want {
  194. t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
  195. }
  196. }
  197. }
  198. func TestXML(t *testing.T) {
  199. const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><a><Word>r\xe9sum\xe9</Word></a>"
  200. d := xml.NewDecoder(strings.NewReader(s))
  201. d.CharsetReader = NewReaderLabel
  202. var a struct {
  203. Word string
  204. }
  205. err := d.Decode(&a)
  206. if err != nil {
  207. t.Fatalf("Decode: %v", err)
  208. }
  209. want := "résumé"
  210. if a.Word != want {
  211. t.Errorf("got %q, want %q", a.Word, want)
  212. }
  213. }