The Chinese Dictionary

Chinese is a complex language (aren’t they all :-( ). The written form is hieroglyphic, that is “pictograms” instead of using an alphabet. But this written form has evolved over time, and even recently split into two forms: “traditional” Chinese as used in Taiwan and Hong Kong, and “simplified” Chinese as used in mainland China. While most of the characters are the same, about 1,000 are different. Thus a Chinese dictionary will often have two written forms of the same character.

Most Westerners like me can’t understand these characters. So there is a “Latinised” form called Pinyin which writes the characters in a phonetic alphabet based on the Latin alphabet. It isn’t quite the Latin alphabet, because Chinese is a tonal language, and the Pinyin form has to show the tones (much like acccents in French and other European languages). So a typical dictionary has to show four things: the traditional form, the simplified form, the Pinyin and the English. For example,

Traditional Simplified Pinyin English
好 好 hǎo good

But again there is a little complication. There is a free Chinese/English dictionary and even better, you can download it as a UTF-8 file, which Go is well suited to handle. In this, the Chinese characters are written in Unicode but the Pinyin characters are not: although there are Unicode characters for letters such as ‘ǎ’, many dictionaries including this one use the Latin ‘a’ and place the tone at the end of the word. Here it is the third tone, so “hǎo” is written as “hao3”. This makes it easier for those who only have US keyboards and no Unicode editor to still communicate in Pinyin.

This data format mismatch is not a big deal: just that somewhere along the line, between the original text dictionary and the display in the browser, a data massage has to be performed. Go templates allow this to be done by defining a custom template, so I chose that route. Alternatives could have been to do this as the dictionary is read in, or in the Javascript to display the final characters.

The code for the Pinyin formatter is given below. Please don’t bother reading it unless you are really interested in knowing the rules for Pinyin formatting. 

  1. package pinyin
  3. import (
  4.     "io"
  5.     "strings"
  6. )
  8. func PinyinFormatter(w io.Writer, format string, value ...interface{}) {
  9.     line := value[0].(string)
  10.     words := strings.Fields(line)
  11.     for n, word := range words {
  12.         // convert "u:" to "ü" if present
  13.         uColon := strings.Index(word, "u:")
  14.         if uColon != -1 {
  15.             parts := strings.SplitN(word, "u:", 2)
  16.             word = parts[0] + "ü" + parts[1]
  17.         }
  18.         println(word)
  19.         // get last character, will be the tone if present
  20.         chars := []rune(word)
  21.         tone := chars[len(chars)-1]
  22.         if tone == '5' {
  23.             words[n] = string(chars[0 : len(chars)-1])
  24.             println("lost accent on", words[n])
  25.             continue
  26.         }
  27.         if tone < '1' || tone > '4' {
  28.             continue
  29.         }
  30.         words[n] = addAccent(word, int(tone))
  31.     }
  32.     line = strings.Join(words, ` `)
  33.     w.Write([]byte(line))
  34. }
  36. var (
  37.     // maps 'a1' to '\u0101' etc
  38.     aAccent = map[int]rune{
  39.         '1': '\u0101',
  40.         '2': '\u00e1',
  41.         '3': '\u01ce', // '\u0103',
  42.         '4': '\u00e0'}
  43.     eAccent = map[int]rune{
  44.         '1': '\u0113',
  45.         '2': '\u00e9',
  46.         '3': '\u011b', // '\u0115',
  47.         '4': '\u00e8'}
  48.     iAccent = map[int]rune{
  49.         '1': '\u012b',
  50.         '2': '\u00ed',
  51.         '3': '\u01d0', // '\u012d',
  52.         '4': '\u00ec'}
  53.     oAccent = map[int]rune{
  54.         '1': '\u014d',
  55.         '2': '\u00f3',
  56.         '3': '\u01d2', // '\u014f',
  57.         '4': '\u00f2'}
  58.     uAccent = map[int]rune{
  59.         '1': '\u016b',
  60.         '2': '\u00fa',
  61.         '3': '\u01d4', // '\u016d',
  62.         '4': '\u00f9'}
  63.     üAccent = map[int]rune{
  64.         '1': 'ǖ',
  65.         '2': 'ǘ',
  66.         '3': 'ǚ',
  67.         '4': 'ǜ'}
  68. )
  70. func addAccent(word string, tone int) string {
  71.     /*
  72.      * Based on "Where do the tone marks go?"
  73.      * at http://www.pinyin.info/rules/where.html
  74.      */
  76.     n := strings.Index(word, "a")
  77.     if n != -1 {
  78.         aAcc := aAccent[tone]
  79.         // replace 'a' with its tone version
  80.         word = word[0:n] + string(aAcc) + word[(n+1):len(word)-1]
  81.     } else {
  82.         n := strings.Index(word, "e")
  83.         if n != -1 {
  84.             eAcc := eAccent[tone]
  85.             word = word[0:n] + string(eAcc) +
  86.                 word[(n+1):len(word)-1]
  87.         } else {
  88.             n = strings.Index(word, "ou")
  89.             if n != -1 {
  90.                 oAcc := oAccent[tone]
  91.                 word = word[0:n] + string(oAcc) + "u" +
  92.                     word[(n+2):len(word)-1]
  93.             } else {
  94.                 chars := []rune(word)
  95.                 length := len(chars)
  96.                 // put tone onthe last vowel
  97.             L:
  98.                 for n, _ := range chars {
  99.                     m := length - n - 1
  100.                     switch chars[m] {
  101.                     case 'i':
  102.                         chars[m] = iAccent[tone]
  103.                         break L
  104.                     case 'o':
  105.                         chars[m] = oAccent[tone]
  106.                         break L
  107.                     case 'u':
  108.                         chars[m] = uAccent[tone]
  109.                         break L
  110.                     case 'ü':
  111.                         chars[m] = üAccent[tone]
  112.                         break L
  113.                     default:
  114.                     }
  115.                 }
  116.                 word = string(chars[0 : len(chars)-1])
  117.             }
  118.         }
  119.     }
  121.     return word
  122. }

How this is used is illustrated by the function lookupWord. This is called in response to an HTML Form request to find the English words in a dictionary.

  1. func lookupWord(rw http.ResponseWriter, req *http.Request) {
  2.         word := req.FormValue("word")
  3.         words := d.LookupEnglish(word)
  5.         pinyinMap := template.FormatterMap {"pinyin": pinyin.PinyinFormatter}
  6.         t, err := template.ParseFile("html/DictionaryEntry.html", pinyinMap)
  7.         if err != nil {
  8.                 http.Error(rw, err.String(), http.StatusInternalServerError)
  9.                 return
  10.         }
  11.         t.Execute(rw, words)
  12. }

The HTML code is

  1. <html>
  2.   <body>
  3.     <table border="1">
  4.       <tr>
  5.     <th>Word</th>
  6.     <th>Traditional</th>
  7.     <th>Simplified</th>
  8.     <th>Pinyin</th>
  9.     <th>English</th>
  10.       </tr>
  11.       {{with .Entries}}
  12.       {{range .}}
  13.       {.repeated section Entries}
  14.       <tr>
  15.     <td>{{.Word}}</td>
  16.     <td>{{.Traditional}}</td> 
  17.     <td>{{.Simplified}}</td>
  18.     <td>{{.Pinyin|pinyin}}</td>
  19.     <td>
  20.       <pre>
  21.         {.repeated section Translations} 
  22.         {@|html} 
  23.         {.end}
  24.       </pre>
  25.     </td>
  26.       </tr>
  27.       {.end} 
  28.       {{end}}
  29.       {{end}}
  30.     </table>
  31.   </body>
  32. </html>

The Dictionary type

The text file containing the dictionary has lines of the form

traditional simplified [pinyin] /translation/translation/…/

For example,

好 好 [hao3] /good/well/proper/good to/easy to/very/so/(suffix indicating completion or readiness)/

We store each line as an Entry within the Dictionary package:

  1. type Entry struct {
  2.     Traditional string
  3.     Simplified string
  4.     Pinyin     string
  5.     Translations []string
  6. }

The dictionary itself is just an array of these entries:

  1. type Dictionary struct {
  2.     Entries []*Entry
  3. }

Building the dictionary is easy enough. Just read each line and break the line into its various bits using simple string methods. Then add the line to the dictionary slice.

Looking up entries in this dictionary is straightforward: just search through until we find the appropriate key. There are about 100,000 entries in this dictionary: brute force by a linear search is fast enough. If it were necessary, faster storage and search mechanisms could easily be used.

The original dictionary grows by people on the Web adding in entries as they see fit. Consequently it isn’t that well organised and contains repetitions and multiple entries. So looking up any word - either by Pinyin or by English - may return multiple matches. To cater for this, each lookup returns a “mini dictionary”, just those lines in the full dictionary that match.

The Dictionary code is

  1. package dictionary
  3. import (
  4.     "bufio"
  5.     //"fmt"
  6.     "os"
  7.     "strings"
  8. )
  10. type Entry struct {
  11.     Traditional  string
  12.     Simplified   string
  13.     Pinyin       string
  14.     Translations []string
  15. }
  17. func (de Entry) String() string {
  18.     str := de.Traditional + ` ` + de.Simplified + ` ` + de.Pinyin
  19.     for _, t := range de.Translations {
  20.         str = str + "\n    " + t
  21.     }
  22.     return str
  23. }
  25. type Dictionary struct {
  26.     Entries []*Entry
  27. }
  29. func (d *Dictionary) String() string {
  30.     str := ""
  31.     for n := 0; n < len(d.Entries); n++ {
  32.         de := d.Entries[n]
  33.         str += de.String() + "\n"
  34.     }
  35.     return str
  36. }
  38. func (d *Dictionary) LookupPinyin(py string) *Dictionary {
  39.     newD := new(Dictionary)
  40.     v := make([]*Entry, 0, 100)
  41.     for n := 0; n < len(d.Entries); n++ {
  42.         de := d.Entries[n]
  43.         if de.Pinyin == py {
  44.             v = append(v, de)
  45.         }
  46.     }
  47.     newD.Entries = v
  48.     return newD
  49. }
  51. func (d *Dictionary) LookupEnglish(eng string) *Dictionary {
  52.     newD := new(Dictionary)
  53.     v := make([]*Entry, 0, 100)
  54.     for n := 0; n < len(d.Entries); n++ {
  55.         de := d.Entries[n]
  56.         for _, e := range de.Translations {
  57.             if e == eng {
  58.                 v = append(v, de)
  59.             }
  60.         }
  61.     }
  62.     newD.Entries = v
  63.     return newD
  64. }
  66. func (d *Dictionary) LookupSimplified(simp string) *Dictionary {
  67.     newD := new(Dictionary)
  68.     v := make([]*Entry, 0, 100)
  70.     for n := 0; n < len(d.Entries); n++ {
  71.         de := d.Entries[n]
  72.         if de.Simplified == simp {
  73.             v = append(v, de)
  74.         }
  75.     }
  76.     newD.Entries = v
  77.     return newD
  78. }
  80. func (d *Dictionary) Load(path string) {
  82.     f, err := os.Open(path)
  83.     r := bufio.NewReader(f)
  84.     if err != nil {
  85.         println(err.Error())
  86.         os.Exit(1)
  87.     }
  89.     v := make([]*Entry, 0, 100000)
  90.     numEntries := 0
  91.     for {
  92.         line, err := r.ReadString('\n')
  93.         if err != nil {
  94.             break
  95.         }
  96.         if line[0] == '#' {
  97.             continue
  98.         }
  99.         // fmt.Println(line)
  100.         trad, simp, pinyin, translations := parseDictEntry(line)
  102.         de := Entry{
  103.             Traditional:  trad,
  104.             Simplified:   simp,
  105.             Pinyin:       pinyin,
  106.             Translations: translations}
  108.         v = append(v, &de)
  109.         numEntries++
  110.     }
  111.     // fmt.Printf("Num entries %d\n", numEntries)
  112.     d.Entries = v
  113. }
  115. func parseDictEntry(line string) (string, string, string, []string) {
  116.     // format is
  117.     //    trad simp [pinyin] /trans/trans/.../
  118.     tradEnd := strings.Index(line, " ")
  119.     trad := line[0:tradEnd]
  120.     line = strings.TrimSpace(line[tradEnd:])
  122.     simpEnd := strings.Index(line, " ")
  123.     simp := line[0:simpEnd]
  124.     line = strings.TrimSpace(line[simpEnd:])
  126.     pinyinEnd := strings.Index(line, "]")
  127.     pinyin := line[1:pinyinEnd]
  128.     line = strings.TrimSpace(line[pinyinEnd+1:])
  130.     translations := strings.Split(line, "/")
  131.     // includes empty at start and end, so
  132.     translations = translations[1 : len(translations)-1]
  134.     return trad, simp, pinyin, translations
  135. }