Method test_decode_non_utf8

tests/test_robotstxt_interface.py:129–132

(self)

        assert decoded_content == "User-agent: *\nDisallow: /\n"

    def test_decode_non_utf8(self):
        robotstxt_body = b"User-agent: *\n\xffDisallow: /\n"
        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
        assert decoded_content == "User-agent: *\nDisallow: /\n"

    # UTF-8 BOM at the beginning of the file ignored
    def test_decode_utf8_bom(self):