1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
7 // Normalization table generator.
8 // Data read from the web.
9 // See forminfo.go for a description of the trie values associated with each rune.
24 "golang.org/x/text/internal/gen"
25 "golang.org/x/text/internal/triegen"
26 "golang.org/x/text/internal/ucd"
33 loadCompositionExclusions()
34 completeCharFields(FCanonical)
35 completeCharFields(FCompatibility)
36 computeNonStarterCounts()
45 tablelist = flag.String("tables",
47 "comma-separated list of which tables to generate; "+
48 "can be 'decomp', 'recomp', 'info' and 'all'")
49 test = flag.Bool("test",
51 "test existing tables against DerivedNormalizationProps and generate test data for regression testing")
52 verbose = flag.Bool("verbose",
54 "write data to stdout as it is parsed")
57 const MaxChar = 0x10FFFF // anything above this shouldn't exist
59 // Quick Check properties of runes allow us to quickly
60 // determine whether a rune may occur in a normal form.
61 // For a given normal form, a rune may be guaranteed to occur
62 // verbatim (QC=Yes), may or may not combine with another
63 // rune (QC=Maybe), or may not occur (QC=No).
67 QCUnknown QCResult = iota
73 func (r QCResult) String() string {
82 return "***UNKNOWN***"
86 FCanonical = iota // NFC or NFD
87 FCompatibility // NFKC or NFKD
92 MComposed = iota // NFC or NFKC
93 MDecomposed // NFD or NFKD
97 // This contains only the properties we're interested in.
100 codePoint rune // if zero, this index is not a valid code point.
101 ccc uint8 // canonical combining class
103 excludeInComp bool // from CompositionExclusions.txt
104 compatDecomp bool // it has a compatibility expansion
106 nTrailingNonStarters uint8
107 nLeadingNonStarters uint8 // must be equal to trailing if non-zero
109 forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility
114 var chars = make([]Char, MaxChar+1)
115 var cccMap = make(map[uint8]uint8)
117 func (c Char) String() string {
118 buf := new(bytes.Buffer)
120 fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name)
121 fmt.Fprintf(buf, " ccc: %v\n", c.ccc)
122 fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp)
123 fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp)
124 fmt.Fprintf(buf, " state: %v\n", c.state)
125 fmt.Fprintf(buf, " NFC:\n")
126 fmt.Fprint(buf, c.forms[FCanonical])
127 fmt.Fprintf(buf, " NFKC:\n")
128 fmt.Fprint(buf, c.forms[FCompatibility])
133 // In UnicodeData.txt, some ranges are marked like this:
134 // 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
135 // 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
136 // parseCharacter keeps a state variable indicating the weirdness.
140 SNormal State = iota // known to be zero for the type
146 var lastChar = rune('\u0000')
148 func (c Char) isValid() bool {
149 return c.codePoint != 0 && c.state != SMissing
152 type FormInfo struct {
153 quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed
154 verified [MNumberOfModes]bool // index: MComposed or MDecomposed
156 combinesForward bool // May combine with rune on the right
157 combinesBackward bool // May combine with rune on the left
158 isOneWay bool // Never appears in result
159 inDecomp bool // Some decompositions result in this char.
161 expandedDecomp Decomposition
164 func (f FormInfo) String() string {
165 buf := bytes.NewBuffer(make([]byte, 0))
167 fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed])
168 fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed])
169 fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward)
170 fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward)
171 fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay)
172 fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp)
173 fmt.Fprintf(buf, " decomposition: %X\n", f.decomp)
174 fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp)
179 type Decomposition []rune
181 func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
182 decomp := strings.Split(s, " ")
183 if len(decomp) > 0 && skipfirst {
186 for _, d := range decomp {
187 point, err := strconv.ParseUint(d, 16, 64)
191 a = append(a, rune(point))
196 func loadUnicodeData() {
197 f := gen.OpenUCDFile("UnicodeData.txt")
201 r := p.Rune(ucd.CodePoint)
204 char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
205 decmap := p.String(ucd.DecompMapping)
207 exp, err := parseDecomposition(decmap, false)
211 exp, err = parseDecomposition(decmap, true)
213 log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
219 char.name = p.String(ucd.Name)
221 char.forms[FCompatibility].decomp = exp
223 char.forms[FCanonical].decomp = exp
225 char.compatDecomp = true
228 char.forms[FCompatibility].decomp = exp
231 if err := p.Err(); err != nil {
236 // compactCCC converts the sparse set of CCC values to a continguous one,
237 // reducing the number of bits needed from 8 to 6.
239 m := make(map[uint8]uint8)
240 for i := range chars {
245 for v, _ := range m {
246 cccs = append(cccs, int(v))
249 for i, c := range cccs {
250 cccMap[uint8(i)] = uint8(c)
251 m[uint8(c)] = uint8(i)
253 for i := range chars {
259 log.Fatalf("too many difference CCC values: %d >= 64", len(m))
263 // CompositionExclusions.txt has form:
265 // See https://unicode.org/reports/tr44/ for full explanation
266 func loadCompositionExclusions() {
267 f := gen.OpenUCDFile("CompositionExclusions.txt")
271 c := &chars[p.Rune(0)]
273 log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
275 c.excludeInComp = true
277 if e := p.Err(); e != nil {
282 // hasCompatDecomp returns true if any of the recursive
283 // decompositions contains a compatibility expansion.
284 // In this case, the character may not occur in NFK*.
285 func hasCompatDecomp(r rune) bool {
290 for _, d := range c.forms[FCompatibility].decomp {
291 if hasCompatDecomp(d) {
298 // Hangul related constants.
301 HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28)
310 JamoLVTCount = 19 * 21 * 28
314 func isHangul(r rune) bool {
315 return HangulBase <= r && r < HangulEnd
318 func isHangulWithoutJamoT(r rune) bool {
323 return r < JamoLVTCount && r%JamoTCount == 0
326 func ccc(r rune) uint8 {
330 // Insert a rune in a buffer, ordered by Canonical Combining Class.
331 func insertOrdered(b Decomposition, r rune) Decomposition {
338 if ccc(b[n-1]) <= cc {
348 // Recursively decompose.
349 func decomposeRecursive(form int, r rune, d Decomposition) Decomposition {
350 dcomp := chars[r].forms[form].decomp
352 return insertOrdered(d, r)
354 for _, c := range dcomp {
355 d = decomposeRecursive(form, c, d)
360 func completeCharFields(form int) {
361 // Phase 0: pre-expand decomposition.
362 for i := range chars {
363 f := &chars[i].forms[form]
364 if len(f.decomp) == 0 {
367 exp := make(Decomposition, 0)
368 for _, c := range f.decomp {
369 exp = decomposeRecursive(form, c, exp)
371 f.expandedDecomp = exp
374 // Phase 1: composition exclusion, mark decomposition.
375 for i := range chars {
379 // Marks script-specific exclusions and version restricted.
380 f.isOneWay = c.excludeInComp
383 f.isOneWay = f.isOneWay || len(f.decomp) == 1
385 // Non-starter decompositions
386 if len(f.decomp) > 1 {
387 chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0
388 f.isOneWay = f.isOneWay || chk
391 // Runes that decompose into more than two runes.
392 f.isOneWay = f.isOneWay || len(f.decomp) > 2
394 if form == FCompatibility {
395 f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint)
398 for _, r := range f.decomp {
399 chars[r].forms[form].inDecomp = true
403 // Phase 2: forward and backward combining.
404 for i := range chars {
408 if !f.isOneWay && len(f.decomp) == 2 {
409 f0 := &chars[f.decomp[0]].forms[form]
410 f1 := &chars[f.decomp[1]].forms[form]
412 f0.combinesForward = true
415 f1.combinesBackward = true
418 if isHangulWithoutJamoT(rune(i)) {
419 f.combinesForward = true
423 // Phase 3: quick check values.
424 for i := range chars {
429 case len(f.decomp) > 0:
430 f.quickCheck[MDecomposed] = QCNo
431 case isHangul(rune(i)):
432 f.quickCheck[MDecomposed] = QCNo
434 f.quickCheck[MDecomposed] = QCYes
438 f.quickCheck[MComposed] = QCNo
439 case (i & 0xffff00) == JamoLBase:
440 f.quickCheck[MComposed] = QCYes
441 if JamoLBase <= i && i < JamoLEnd {
442 f.combinesForward = true
444 if JamoVBase <= i && i < JamoVEnd {
445 f.quickCheck[MComposed] = QCMaybe
446 f.combinesBackward = true
447 f.combinesForward = true
449 if JamoTBase <= i && i < JamoTEnd {
450 f.quickCheck[MComposed] = QCMaybe
451 f.combinesBackward = true
453 case !f.combinesBackward:
454 f.quickCheck[MComposed] = QCYes
456 f.quickCheck[MComposed] = QCMaybe
461 func computeNonStarterCounts() {
462 // Phase 4: leading and trailing non-starter count
463 for i := range chars {
466 runes := []rune{rune(i)}
467 // We always use FCompatibility so that the CGJ insertion points do not
468 // change for repeated normalizations with different forms.
469 if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
472 // We consider runes that combine backwards to be non-starters for the
473 // purpose of Stream-Safe Text Processing.
474 for _, r := range runes {
475 if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
478 c.nLeadingNonStarters++
480 for i := len(runes) - 1; i >= 0; i-- {
481 if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
484 c.nTrailingNonStarters++
486 if c.nTrailingNonStarters > 3 {
487 log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
490 if isHangul(rune(i)) {
491 c.nTrailingNonStarters = 2
492 if isHangulWithoutJamoT(rune(i)) {
493 c.nTrailingNonStarters = 1
497 if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t {
498 log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t)
500 if t := c.nTrailingNonStarters; t > 3 {
501 log.Fatalf("%U: number of trailing non-starters is %d > 3", t)
506 func printBytes(w io.Writer, b []byte, name string) {
507 fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
508 fmt.Fprintf(w, "var %s = [...]byte {", name)
509 for i, c := range b {
512 fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
516 fmt.Fprintf(w, "0x%.2X, ", c)
518 fmt.Fprint(w, "\n}\n\n")
521 // See forminfo.go for format.
522 func makeEntry(f *FormInfo, c *Char) uint16 {
524 if r := c.codePoint; HangulBase <= r && r < HangulEnd {
527 if f.combinesForward {
530 if f.quickCheck[MDecomposed] == QCNo {
533 switch f.quickCheck[MComposed] {
540 log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed])
542 e |= uint16(c.nTrailingNonStarters)
546 // decompSet keeps track of unique decompositions, grouped by whether
547 // the decomposition is followed by a trailing and/or leading CCC.
548 type decompSet [7]map[string]bool
557 firstStarterWithNLead
561 var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"}
563 func makeDecompSet() decompSet {
566 m[i] = make(map[string]bool)
570 func (m *decompSet) insert(key int, s string) {
574 func printCharInfoTables(w io.Writer) int {
575 mkstr := func(r rune, f *FormInfo) (int, string) {
576 d := f.expandedDecomp
577 s := string([]rune(d))
578 if max := 1 << 6; len(s) >= max {
579 const msg = "%U: too many bytes in decomposition: %d >= %d"
580 log.Fatalf(msg, r, len(s), max)
582 head := uint8(len(s))
583 if f.quickCheck[MComposed] != QCYes {
586 if f.combinesForward {
589 s = string([]byte{head}) + s
592 tccc := ccc(d[len(d)-1])
594 if cc != 0 && lccc == 0 && tccc == 0 {
595 log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
597 if tccc < lccc && lccc != 0 {
598 const msg = "%U: lccc (%d) must be <= tcc (%d)"
599 log.Fatalf(msg, r, lccc, tccc)
601 index := normalDecomp
602 nTrail := chars[r].nTrailingNonStarters
603 nLead := chars[r].nLeadingNonStarters
604 if tccc > 0 || lccc > 0 || nTrail > 0 {
607 s += string([]byte{tccc})
609 for _, r := range d[1:] {
614 if lccc > 0 || nLead > 0 {
615 s += string([]byte{lccc})
616 if index == firstCCC {
617 log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
619 index = firstLeadingCCC
623 log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
625 index = firstCCCZeroExcept
627 } else if len(d) > 1 {
633 decompSet := makeDecompSet()
634 const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail.
635 decompSet.insert(firstStarterWithNLead, nLeadStr)
637 // Store the uniqued decompositions in a byte buffer,
638 // preceded by their byte length.
639 for _, c := range chars {
640 for _, f := range c.forms {
641 if len(f.expandedDecomp) == 0 {
644 if f.combinesBackward {
645 log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
647 index, s := mkstr(c.codePoint, &f)
648 decompSet.insert(index, s)
652 decompositions := bytes.NewBuffer(make([]byte, 0, 10000))
654 positionMap := make(map[string]uint16)
655 decompositions.WriteString("\000")
656 fmt.Fprintln(w, "const (")
657 for i, m := range decompSet {
663 for _, s := range sa {
664 p := decompositions.Len()
665 decompositions.WriteString(s)
666 positionMap[s] = uint16(p)
669 fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
672 fmt.Fprintln(w, "maxDecomp = 0x8000")
674 b := decompositions.Bytes()
675 printBytes(w, b, "decomps")
678 varnames := []string{"nfc", "nfkc"}
679 for i := 0; i < FNumberOfFormTypes; i++ {
680 trie := triegen.NewTrie(varnames[i])
682 for r, c := range chars {
684 d := f.expandedDecomp
686 _, key := mkstr(c.codePoint, &f)
687 trie.Insert(rune(r), uint64(positionMap[key]))
688 if c.ccc != ccc(d[0]) {
689 // We assume the lead ccc of a decomposition !=0 in this case.
691 log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
694 } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
695 // Handle cases where it can't be detected that the nLead should be equal
697 trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
698 } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
699 trie.Insert(c.codePoint, uint64(0x8000|v))
702 sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
711 func contains(sa []string, s string) bool {
712 for _, a := range sa {
724 if *tablelist == "" {
727 list := strings.Split(*tablelist, ",")
728 if *tablelist == "all" {
729 list = []string{"recomp", "info"}
732 // Compute maximum decomposition size.
734 for _, c := range chars {
735 if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max {
739 fmt.Fprintln(w, `import "sync"`)
742 fmt.Fprintln(w, "const (")
743 fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
744 fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
746 fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
747 fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
748 fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
749 fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
750 fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
751 fmt.Fprintln(w, ")\n")
753 // Print the CCC remap table.
755 fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
756 for i := 0; i < len(cccMap); i++ {
760 fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
762 fmt.Fprintln(w, "\n}\n")
764 if contains(list, "info") {
765 size += printCharInfoTables(w)
768 if contains(list, "recomp") {
769 // Note that we use 32 bit keys, instead of 64 bit.
770 // This clips the bits of three entries, but we know
771 // this won't cause a collision. The compiler will catch
772 // any changes made to UnicodeData.txt that introduces
774 // Note that the recomposition map for NFC and NFKC
779 for _, c := range chars {
780 f := c.forms[FCanonical]
781 if !f.isOneWay && len(f.decomp) > 0 {
787 fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
788 fmt.Fprintln(w, "var recompMap map[uint32]rune")
789 fmt.Fprintln(w, "var recompMapOnce sync.Once\n")
790 fmt.Fprintln(w, `const recompMapPacked = "" +`)
792 for i, c := range chars {
793 f := c.forms[FCanonical]
795 if !f.isOneWay && len(d) > 0 {
796 key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
797 binary.BigEndian.PutUint32(buf[:4], key)
798 binary.BigEndian.PutUint32(buf[4:], uint32(i))
799 fmt.Fprintf(w, "\t\t%q + // 0x%.8X: 0x%.8X\n", string(buf[:]), key, uint32(i))
802 // hack so we don't have to special case the trailing plus sign
803 fmt.Fprintf(w, ` ""`)
807 fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
808 gen.WriteVersionedGoFile("tables.go", "norm", w.Bytes())
813 for _, c := range chars {
814 if !c.isValid() || c.state == SMissing {
822 // verifyComputed does various consistency tests.
823 func verifyComputed() {
824 for i, c := range chars {
825 for _, f := range c.forms {
826 isNo := (f.quickCheck[MDecomposed] == QCNo)
827 if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) {
828 log.Fatalf("%U: NF*D QC must be No if rune decomposes", i)
831 isMaybe := f.quickCheck[MComposed] == QCMaybe
832 if f.combinesBackward != isMaybe {
833 log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i)
835 if len(f.decomp) > 0 && f.combinesForward && isMaybe {
836 log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i)
839 if len(f.expandedDecomp) != 0 {
842 if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
843 // We accept these runes to be treated differently (it only affects
844 // segment breaking in iteration, most likely on improper use), but
845 // reconsider if more characters are added.
846 // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
847 // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
848 // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
849 // U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
850 // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
851 // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
852 if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
853 log.Fatalf("%U: nLead was %v; want %v", i, a, b)
857 nfc := c.forms[FCanonical]
858 nfkc := c.forms[FCompatibility]
859 if nfc.combinesBackward != nfkc.combinesBackward {
860 log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
865 // Use values in DerivedNormalizationProps.txt to compare against the
866 // values we computed.
867 // DerivedNormalizationProps.txt has form:
868 // 00C0..00C5 ; NFD_QC; N # ...
869 // 0374 ; NFD_QC; N # ...
870 // See https://unicode.org/reports/tr44/ for full explanation
872 f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
883 ftype, mode = FCanonical, MComposed
885 ftype, mode = FCanonical, MDecomposed
887 ftype, mode = FCompatibility, MComposed
889 ftype, mode = FCompatibility, MDecomposed
902 log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
904 if got := c.forms[ftype].quickCheck[mode]; got != qr {
905 log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
907 c.forms[ftype].verified[mode] = true
909 if err := p.Err(); err != nil {
912 // Any unspecified value must be QCYes. Verify this.
913 for i, c := range chars {
914 for j, fd := range c.forms {
915 for k, qr := range fd.quickCheck {
916 if !fd.verified[k] && qr != QCYes {
917 m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
918 log.Printf(m, i, j, k, qr, c.name)
925 var testHeader = `const (
931 type formData struct {
937 type runeData struct {
942 f [2]formData // 0: canonical; 1: compatibility
945 func f(qc uint8, cf bool, dec string) [2]formData {
946 return [2]formData{{qc, cf, dec}, {qc, cf, dec}}
949 func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData {
950 return [2]formData{{qc, cf, d}, {qck, cfk, dk}}
953 var testData = []runeData{
956 func printTestdata() {
957 type lastInfo struct {
966 fmt.Fprintf(w, testHeader)
967 for r, c := range chars {
968 f := c.forms[FCanonical]
969 qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
970 f = c.forms[FCompatibility]
971 qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
973 if d == dk && qc == qck && cf == cfk {
974 s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d)
976 s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk)
978 current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
980 fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
985 gen.WriteVersionedGoFile("data_test.go", "norm", w.Bytes())