charmap.go 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. //go:generate go run maketables.go
  5. // Package charmap provides simple character encodings such as IBM Code Page 437
  6. // and Windows 1252.
  7. package charmap // import "golang.org/x/text/encoding/charmap"
  8. import (
  9. "unicode/utf8"
  10. "golang.org/x/text/encoding"
  11. "golang.org/x/text/encoding/internal"
  12. "golang.org/x/text/encoding/internal/identifier"
  13. "golang.org/x/text/transform"
  14. )
  15. // These encodings vary only in the way clients should interpret them. Their
  16. // coded character set is identical and a single implementation can be shared.
  17. var (
  18. // ISO8859_6E is the ISO 8859-6E encoding.
  19. ISO8859_6E encoding.Encoding = &iso8859_6E
  20. // ISO8859_6I is the ISO 8859-6I encoding.
  21. ISO8859_6I encoding.Encoding = &iso8859_6I
  22. // ISO8859_8E is the ISO 8859-8E encoding.
  23. ISO8859_8E encoding.Encoding = &iso8859_8E
  24. // ISO8859_8I is the ISO 8859-8I encoding.
  25. ISO8859_8I encoding.Encoding = &iso8859_8I
  26. iso8859_6E = internal.Encoding{
  27. Encoding: ISO8859_6,
  28. Name: "ISO-8859-6E",
  29. MIB: identifier.ISO88596E,
  30. }
  31. iso8859_6I = internal.Encoding{
  32. Encoding: ISO8859_6,
  33. Name: "ISO-8859-6I",
  34. MIB: identifier.ISO88596I,
  35. }
  36. iso8859_8E = internal.Encoding{
  37. Encoding: ISO8859_8,
  38. Name: "ISO-8859-8E",
  39. MIB: identifier.ISO88598E,
  40. }
  41. iso8859_8I = internal.Encoding{
  42. Encoding: ISO8859_8,
  43. Name: "ISO-8859-8I",
  44. MIB: identifier.ISO88598I,
  45. }
  46. )
  47. // All is a list of all defined encodings in this package.
  48. var All []encoding.Encoding = listAll
  49. // TODO: implement these encodings, in order of importance.
  50. // ASCII, ISO8859_1: Rather common. Close to Windows 1252.
  51. // ISO8859_9: Close to Windows 1254.
  52. // utf8Enc holds a rune's UTF-8 encoding in data[:len].
  53. type utf8Enc struct {
  54. len uint8
  55. data [3]byte
  56. }
  57. // Charmap is an 8-bit character set encoding.
  58. type Charmap struct {
  59. // name is the encoding's name.
  60. name string
  61. // mib is the encoding type of this encoder.
  62. mib identifier.MIB
  63. // asciiSuperset states whether the encoding is a superset of ASCII.
  64. asciiSuperset bool
  65. // low is the lower bound of the encoded byte for a non-ASCII rune. If
  66. // Charmap.asciiSuperset is true then this will be 0x80, otherwise 0x00.
  67. low uint8
  68. // replacement is the encoded replacement character.
  69. replacement byte
  70. // decode is the map from encoded byte to UTF-8.
  71. decode [256]utf8Enc
  72. // encoding is the map from runes to encoded bytes. Each entry is a
  73. // uint32: the high 8 bits are the encoded byte and the low 24 bits are
  74. // the rune. The table entries are sorted by ascending rune.
  75. encode [256]uint32
  76. }
  77. // NewDecoder implements the encoding.Encoding interface.
  78. func (m *Charmap) NewDecoder() *encoding.Decoder {
  79. return &encoding.Decoder{Transformer: charmapDecoder{charmap: m}}
  80. }
  81. // NewEncoder implements the encoding.Encoding interface.
  82. func (m *Charmap) NewEncoder() *encoding.Encoder {
  83. return &encoding.Encoder{Transformer: charmapEncoder{charmap: m}}
  84. }
  85. // String returns the Charmap's name.
  86. func (m *Charmap) String() string {
  87. return m.name
  88. }
  89. // ID implements an internal interface.
  90. func (m *Charmap) ID() (mib identifier.MIB, other string) {
  91. return m.mib, ""
  92. }
  93. // charmapDecoder implements transform.Transformer by decoding to UTF-8.
  94. type charmapDecoder struct {
  95. transform.NopResetter
  96. charmap *Charmap
  97. }
  98. func (m charmapDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
  99. for i, c := range src {
  100. if m.charmap.asciiSuperset && c < utf8.RuneSelf {
  101. if nDst >= len(dst) {
  102. err = transform.ErrShortDst
  103. break
  104. }
  105. dst[nDst] = c
  106. nDst++
  107. nSrc = i + 1
  108. continue
  109. }
  110. decode := &m.charmap.decode[c]
  111. n := int(decode.len)
  112. if nDst+n > len(dst) {
  113. err = transform.ErrShortDst
  114. break
  115. }
  116. // It's 15% faster to avoid calling copy for these tiny slices.
  117. for j := 0; j < n; j++ {
  118. dst[nDst] = decode.data[j]
  119. nDst++
  120. }
  121. nSrc = i + 1
  122. }
  123. return nDst, nSrc, err
  124. }
  125. // DecodeByte returns the Charmap's rune decoding of the byte b.
  126. func (m *Charmap) DecodeByte(b byte) rune {
  127. switch x := &m.decode[b]; x.len {
  128. case 1:
  129. return rune(x.data[0])
  130. case 2:
  131. return rune(x.data[0]&0x1f)<<6 | rune(x.data[1]&0x3f)
  132. default:
  133. return rune(x.data[0]&0x0f)<<12 | rune(x.data[1]&0x3f)<<6 | rune(x.data[2]&0x3f)
  134. }
  135. }
  136. // charmapEncoder implements transform.Transformer by encoding from UTF-8.
  137. type charmapEncoder struct {
  138. transform.NopResetter
  139. charmap *Charmap
  140. }
  141. func (m charmapEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
  142. r, size := rune(0), 0
  143. loop:
  144. for nSrc < len(src) {
  145. if nDst >= len(dst) {
  146. err = transform.ErrShortDst
  147. break
  148. }
  149. r = rune(src[nSrc])
  150. // Decode a 1-byte rune.
  151. if r < utf8.RuneSelf {
  152. if m.charmap.asciiSuperset {
  153. nSrc++
  154. dst[nDst] = uint8(r)
  155. nDst++
  156. continue
  157. }
  158. size = 1
  159. } else {
  160. // Decode a multi-byte rune.
  161. r, size = utf8.DecodeRune(src[nSrc:])
  162. if size == 1 {
  163. // All valid runes of size 1 (those below utf8.RuneSelf) were
  164. // handled above. We have invalid UTF-8 or we haven't seen the
  165. // full character yet.
  166. if !atEOF && !utf8.FullRune(src[nSrc:]) {
  167. err = transform.ErrShortSrc
  168. } else {
  169. err = internal.RepertoireError(m.charmap.replacement)
  170. }
  171. break
  172. }
  173. }
  174. // Binary search in [low, high) for that rune in the m.charmap.encode table.
  175. for low, high := int(m.charmap.low), 0x100; ; {
  176. if low >= high {
  177. err = internal.RepertoireError(m.charmap.replacement)
  178. break loop
  179. }
  180. mid := (low + high) / 2
  181. got := m.charmap.encode[mid]
  182. gotRune := rune(got & (1<<24 - 1))
  183. if gotRune < r {
  184. low = mid + 1
  185. } else if gotRune > r {
  186. high = mid
  187. } else {
  188. dst[nDst] = byte(got >> 24)
  189. nDst++
  190. break
  191. }
  192. }
  193. nSrc += size
  194. }
  195. return nDst, nSrc, err
  196. }
  197. // EncodeRune returns the Charmap's byte encoding of the rune r. ok is whether
  198. // r is in the Charmap's repertoire. If not, b is set to the Charmap's
  199. // replacement byte. This is often the ASCII substitute character '\x1a'.
  200. func (m *Charmap) EncodeRune(r rune) (b byte, ok bool) {
  201. if r < utf8.RuneSelf && m.asciiSuperset {
  202. return byte(r), true
  203. }
  204. for low, high := int(m.low), 0x100; ; {
  205. if low >= high {
  206. return m.replacement, false
  207. }
  208. mid := (low + high) / 2
  209. got := m.encode[mid]
  210. gotRune := rune(got & (1<<24 - 1))
  211. if gotRune < r {
  212. low = mid + 1
  213. } else if gotRune > r {
  214. high = mid
  215. } else {
  216. return byte(got >> 24), true
  217. }
  218. }
  219. }