本站源代码
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

258 lines
6.1KB

  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // Package charset provides common text encodings for HTML documents.
  5. //
  6. // The mapping from encoding labels to encodings is defined at
  7. // https://encoding.spec.whatwg.org/.
  8. package charset // import "golang.org/x/net/html/charset"
  9. import (
  10. "bytes"
  11. "fmt"
  12. "io"
  13. "mime"
  14. "strings"
  15. "unicode/utf8"
  16. "golang.org/x/net/html"
  17. "golang.org/x/text/encoding"
  18. "golang.org/x/text/encoding/charmap"
  19. "golang.org/x/text/encoding/htmlindex"
  20. "golang.org/x/text/transform"
  21. )
  22. // Lookup returns the encoding with the specified label, and its canonical
  23. // name. It returns nil and the empty string if label is not one of the
  24. // standard encodings for HTML. Matching is case-insensitive and ignores
  25. // leading and trailing whitespace. Encoders will use HTML escape sequences for
  26. // runes that are not supported by the character set.
  27. func Lookup(label string) (e encoding.Encoding, name string) {
  28. e, err := htmlindex.Get(label)
  29. if err != nil {
  30. return nil, ""
  31. }
  32. name, _ = htmlindex.Name(e)
  33. return &htmlEncoding{e}, name
  34. }
  35. type htmlEncoding struct{ encoding.Encoding }
  36. func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
  37. // HTML requires a non-terminating legacy encoder. We use HTML escapes to
  38. // substitute unsupported code points.
  39. return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
  40. }
  41. // DetermineEncoding determines the encoding of an HTML document by examining
  42. // up to the first 1024 bytes of content and the declared Content-Type.
  43. //
  44. // See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
  45. func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
  46. if len(content) > 1024 {
  47. content = content[:1024]
  48. }
  49. for _, b := range boms {
  50. if bytes.HasPrefix(content, b.bom) {
  51. e, name = Lookup(b.enc)
  52. return e, name, true
  53. }
  54. }
  55. if _, params, err := mime.ParseMediaType(contentType); err == nil {
  56. if cs, ok := params["charset"]; ok {
  57. if e, name = Lookup(cs); e != nil {
  58. return e, name, true
  59. }
  60. }
  61. }
  62. if len(content) > 0 {
  63. e, name = prescan(content)
  64. if e != nil {
  65. return e, name, false
  66. }
  67. }
  68. // Try to detect UTF-8.
  69. // First eliminate any partial rune at the end.
  70. for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
  71. b := content[i]
  72. if b < 0x80 {
  73. break
  74. }
  75. if utf8.RuneStart(b) {
  76. content = content[:i]
  77. break
  78. }
  79. }
  80. hasHighBit := false
  81. for _, c := range content {
  82. if c >= 0x80 {
  83. hasHighBit = true
  84. break
  85. }
  86. }
  87. if hasHighBit && utf8.Valid(content) {
  88. return encoding.Nop, "utf-8", false
  89. }
  90. // TODO: change default depending on user's locale?
  91. return charmap.Windows1252, "windows-1252", false
  92. }
  93. // NewReader returns an io.Reader that converts the content of r to UTF-8.
  94. // It calls DetermineEncoding to find out what r's encoding is.
  95. func NewReader(r io.Reader, contentType string) (io.Reader, error) {
  96. preview := make([]byte, 1024)
  97. n, err := io.ReadFull(r, preview)
  98. switch {
  99. case err == io.ErrUnexpectedEOF:
  100. preview = preview[:n]
  101. r = bytes.NewReader(preview)
  102. case err != nil:
  103. return nil, err
  104. default:
  105. r = io.MultiReader(bytes.NewReader(preview), r)
  106. }
  107. if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
  108. r = transform.NewReader(r, e.NewDecoder())
  109. }
  110. return r, nil
  111. }
  112. // NewReaderLabel returns a reader that converts from the specified charset to
  113. // UTF-8. It uses Lookup to find the encoding that corresponds to label, and
  114. // returns an error if Lookup returns nil. It is suitable for use as
  115. // encoding/xml.Decoder's CharsetReader function.
  116. func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
  117. e, _ := Lookup(label)
  118. if e == nil {
  119. return nil, fmt.Errorf("unsupported charset: %q", label)
  120. }
  121. return transform.NewReader(input, e.NewDecoder()), nil
  122. }
  123. func prescan(content []byte) (e encoding.Encoding, name string) {
  124. z := html.NewTokenizer(bytes.NewReader(content))
  125. for {
  126. switch z.Next() {
  127. case html.ErrorToken:
  128. return nil, ""
  129. case html.StartTagToken, html.SelfClosingTagToken:
  130. tagName, hasAttr := z.TagName()
  131. if !bytes.Equal(tagName, []byte("meta")) {
  132. continue
  133. }
  134. attrList := make(map[string]bool)
  135. gotPragma := false
  136. const (
  137. dontKnow = iota
  138. doNeedPragma
  139. doNotNeedPragma
  140. )
  141. needPragma := dontKnow
  142. name = ""
  143. e = nil
  144. for hasAttr {
  145. var key, val []byte
  146. key, val, hasAttr = z.TagAttr()
  147. ks := string(key)
  148. if attrList[ks] {
  149. continue
  150. }
  151. attrList[ks] = true
  152. for i, c := range val {
  153. if 'A' <= c && c <= 'Z' {
  154. val[i] = c + 0x20
  155. }
  156. }
  157. switch ks {
  158. case "http-equiv":
  159. if bytes.Equal(val, []byte("content-type")) {
  160. gotPragma = true
  161. }
  162. case "content":
  163. if e == nil {
  164. name = fromMetaElement(string(val))
  165. if name != "" {
  166. e, name = Lookup(name)
  167. if e != nil {
  168. needPragma = doNeedPragma
  169. }
  170. }
  171. }
  172. case "charset":
  173. e, name = Lookup(string(val))
  174. needPragma = doNotNeedPragma
  175. }
  176. }
  177. if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
  178. continue
  179. }
  180. if strings.HasPrefix(name, "utf-16") {
  181. name = "utf-8"
  182. e = encoding.Nop
  183. }
  184. if e != nil {
  185. return e, name
  186. }
  187. }
  188. }
  189. }
  190. func fromMetaElement(s string) string {
  191. for s != "" {
  192. csLoc := strings.Index(s, "charset")
  193. if csLoc == -1 {
  194. return ""
  195. }
  196. s = s[csLoc+len("charset"):]
  197. s = strings.TrimLeft(s, " \t\n\f\r")
  198. if !strings.HasPrefix(s, "=") {
  199. continue
  200. }
  201. s = s[1:]
  202. s = strings.TrimLeft(s, " \t\n\f\r")
  203. if s == "" {
  204. return ""
  205. }
  206. if q := s[0]; q == '"' || q == '\'' {
  207. s = s[1:]
  208. closeQuote := strings.IndexRune(s, rune(q))
  209. if closeQuote == -1 {
  210. return ""
  211. }
  212. return s[:closeQuote]
  213. }
  214. end := strings.IndexAny(s, "; \t\n\f\r")
  215. if end == -1 {
  216. end = len(s)
  217. }
  218. return s[:end]
  219. }
  220. return ""
  221. }
  222. var boms = []struct {
  223. bom []byte
  224. enc string
  225. }{
  226. {[]byte{0xfe, 0xff}, "utf-16be"},
  227. {[]byte{0xff, 0xfe}, "utf-16le"},
  228. {[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
  229. }
上海开阖软件有限公司 沪ICP备12045867号-1