1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
7 // This program generates the trie for width operations. The generated table
8 // includes width category information as well as the normalization mappings.
19 "golang.org/x/text/internal/gen"
20 "golang.org/x/text/internal/triegen"
23 // See gen_common.go for flags.
29 gen.Repackage("gen_trieval.go", "trieval.go", "width")
30 gen.Repackage("gen_common.go", "common_test.go", "width")
34 t := triegen.NewTrie("width")
35 // fold and inverse mappings. See mapComment for a description of the format
36 // of each entry. Add dummy value to make an index of 0 mean no mapping.
37 inverse := [][4]byte{{}}
38 mapping := map[[4]byte]int{[4]byte{}: 0}
40 getWidthData(func(r rune, tag elem, alt rune) {
44 buf[0] = byte(utf8.EncodeRune(buf[1:], alt))
46 buf[buf[0]] ^= s[len(s)-1]
48 if idx, ok = mapping[buf]; !ok {
50 if idx > math.MaxUint8 {
51 log.Fatalf("Index %d does not fit in a byte.", idx)
54 inverse = append(inverse, buf)
57 t.Insert(r, uint64(tag|elem(idx)))
61 gen.WriteUnicodeVersion(w)
68 sz += writeMappings(w, inverse)
70 fmt.Fprintf(w, "// Total table size %d bytes (%dKiB)\n", sz, sz/1024)
72 gen.WriteVersionedGoFile(*outputFile, "width", w.Bytes())
75 const inverseDataComment = `
76 // inverseData contains 4-byte entries of the following format:
77 // <length> <modified UTF-8-encoded rune> <0 padding>
78 // The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the
79 // UTF-8 encoding of the original rune. Mappings often have the following
81 // A -> A (U+FF21 -> U+0041)
82 // B -> B (U+FF22 -> U+0042)
84 // By xor-ing the last byte the same entry can be shared by many mappings. This
85 // reduces the total number of distinct entries by about two thirds.
86 // The resulting entry for the aforementioned mappings is
87 // { 0x01, 0xE0, 0x00, 0x00 }
88 // Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get
90 // Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get
92 // Note that because of the xor-ing, the byte sequence stored in the entry is
95 func writeMappings(w io.Writer, data [][4]byte) int {
96 fmt.Fprintln(w, inverseDataComment)
97 fmt.Fprintf(w, "var inverseData = [%d][4]byte{\n", len(data))
98 for _, x := range data {
99 fmt.Fprintf(w, "{ 0x%02x, 0x%02x, 0x%02x, 0x%02x },\n", x[0], x[1], x[2], x[3])
107 fmt.Fprintf(w, "\nvar mapRunes = map[rune]struct{r rune; e elem}{\n")
108 getWidthData(func(r rune, tag elem, alt rune) {
110 fmt.Fprintf(w, "\t0x%X: {0x%X, 0x%X},\n", r, alt, tag)
114 gen.WriteGoFile("runes_test.go", "width", w.Bytes())