| 35 | |
| 36 | |
| 37 | def parse_gb18030map(fo): |
| 38 | m, gbuni = {}, {} |
| 39 | for i in range(65536): |
| 40 | if i < 0xd800 or i > 0xdfff: # exclude unicode surrogate area |
| 41 | gbuni[i] = None |
| 42 | for uni, native in re_gb18030ass.findall(fo.read()): |
| 43 | uni = eval('0x'+uni) |
| 44 | native = [eval('0x'+u) for u in native.split()] |
| 45 | if len(native) <= 2: |
| 46 | del gbuni[uni] |
| 47 | if len(native) == 2: # we can decode algorithmically for 1 or 4 bytes |
| 48 | m.setdefault(native[0], {}) |
| 49 | m[native[0]][native[1]] = uni |
| 50 | gbuni = [k for k in gbuni.keys()] |
| 51 | gbuni.sort() |
| 52 | return m, gbuni |
| 53 | |
| 54 | def main(): |
| 55 | print("Loading Mapping File...") |