| 8 | class TextClean(object): |
| 9 | |
| 10 | def __init__(self): |
| 11 | spu = [ |
| 12 | 0xA0, 0x1680, 0x202f, 0x205F, 0x3000, 0xFEFF, 8203, 8206, 8207, |
| 13 | 8298, 8300, 65279 |
| 14 | ] |
| 15 | spu.extend(range(0xE000, 0xF8FF + 1)) |
| 16 | spu.extend(range(0x2000, 0x200A + 1)) |
| 17 | spu.extend(range(0x7F, 0xA0 + 1)) |
| 18 | |
| 19 | self.spaces = set([chr(i) for i in spu]) |
| 20 | |
| 21 | self.space_pat = re.compile(r'\s+', re.UNICODE) |
| 22 | |
| 23 | self.replace_char = { |
| 24 | u'`': u"'", |
| 25 | u'’': u"'", |
| 26 | u'´': u"'", |
| 27 | u'‘': u"'", |
| 28 | u'º': u'°', |
| 29 | u'–': u'-', |
| 30 | u'—': u'-' |
| 31 | } |
| 32 | |
| 33 | def sbc2dbc(self, ch): |
| 34 | n = ord(ch) |