Skip to content

INTERNALERROR (UnicodeDecodeError): ReprFuncArgs don't handle mixed unicode and utf-8 strings on the same call #2731

New issue
New issue
Closed
Closed
INTERNALERROR (UnicodeDecodeError): ReprFuncArgs don't handle mixed unicode and utf-8 strings on the same call#2731
Labels
topic: reportingrelated to terminal output and user-facing messages and errorstype: bugproblem that needs to be addressed
@fgmacedo

Description

@fgmacedo
fgmacedo
opened on Aug 29, 2017

Versions:

Python 2.7.13
pytest==3.2.1
Linux L251 4.4.0-92-generic #115-Ubuntu x86_64 x86_64 x86_64 GNU/Linux

The class _pytest._code.code.ReprFuncArgs crashes with UnicodeDecodeError when trying to write args with mixed unicode and utf-8 strings.

There's a test that reproduces the bug:

# coding: utf-8


class MockTerminalWritter(object):
    fullwidth = 80

    def __init__(self):
        self.lines = []

    def line(self, line):
        self.lines.append(line)


def test_repr():
    from _pytest._code.code import ReprFuncArgs

    tw = MockTerminalWritter()

    args = [
        ('unicode_string', u"'São Paulo'"),
        ('utf8_string', 'S\xc3\xa3o Paulo'),
    ]

    r = ReprFuncArgs(args)
    r.toterminal(tw)

Resulting in:

_______________________________ test_repr _____________________________________

    def test_repr():
        from _pytest._code.code import ReprFuncArgs
    
        tw = MockTerminalWritter()
    
        args = [
            ('unicode_string', u"'São Paulo'"),
            ('utf8_string', 'S\xc3\xa3o Paulo'),
        ]
    
        r = ReprFuncArgs(args)
>       r.toterminal(tw)

test_pytestunicode.py:31: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <<class '_pytest._code.code.ReprFuncArgs'> instance at 7fb056317d50>, 
tw = <test_pytestunicode.MockTerminalWritter object at 0x7fb056317f10>

    def toterminal(self, tw):
        if self.args:
            linesofar = ""
            for name, value in self.args:
                ns = "%s = %s" % (name, value)
                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
                    if linesofar:
                        tw.line(linesofar)
                    linesofar = ns
                else:
                    if linesofar:
>                       linesofar += ", " + ns
E                       UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 17: ordinal not in range(128)

../../.virtualenvs/pytest-error/lib/python2.7/site-packages/_pytest/_code/code.py:873: UnicodeDecodeError

Here the actual traceback of the error when the used by running our tests:

INTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/main.py", line 98, in wrap_session
INTERNALERROR>     session.exitstatus = doit(config, session) or 0
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/main.py", line 133, in _main
INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "<remote exec>", line 61, in pytest_runtestloop
INTERNALERROR>   File "<remote exec>", line 77, in run_tests
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 613, in execute
INTERNALERROR>     return _wrapped_call(hook_impl.function(*args), self.execute)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 254, in _wrapped_call
INTERNALERROR>     return call_outcome.get_result()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 280, in get_result
INTERNALERROR>     _reraise(*ex)  # noqa
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 265, in __init__
INTERNALERROR>     self.result = func()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/runner.py", line 66, in pytest_runtest_protocol
INTERNALERROR>     runtestprotocol(item, nextitem=nextitem)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/runner.py", line 79, in runtestprotocol
INTERNALERROR>     reports.append(call_and_report(item, "call", log))
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/runner.py", line 137, in call_and_report
INTERNALERROR>     hook.pytest_runtest_logreport(report=report)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "<remote exec>", line 89, in pytest_runtest_logreport
INTERNALERROR>   File "<remote exec>", line 103, in serialize_report
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 654, in __str__
INTERNALERROR>     s = self.__unicode__()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 664, in __unicode__
INTERNALERROR>     self.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 709, in toterminal
INTERNALERROR>     self.reprtraceback.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 725, in toterminal
INTERNALERROR>     entry.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 769, in toterminal
INTERNALERROR>     self.reprfuncargs.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 826, in toterminal
INTERNALERROR>     linesofar += ", " + ns
INTERNALERROR> UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 17: ordinal not in range(128)
INTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/main.py", line 98, in wrap_session
INTERNALERROR>     session.exitstatus = doit(config, session) or 0
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/main.py", line 133, in _main
INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "<remote exec>", line 61, in pytest_runtestloop
INTERNALERROR>   File "<remote exec>", line 77, in run_tests
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 613, in execute
INTERNALERROR>     return _wrapped_call(hook_impl.function(*args), self.execute)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 254, in _wrapped_call
INTERNALERROR>     return call_outcome.get_result()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 280, in get_result
INTERNALERROR>     _reraise(*ex)  # noqa
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 265, in __init__
INTERNALERROR>     self.result = func()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/runner.py", line 66, in pytest_runtest_protocol
INTERNALERROR>     runtestprotocol(item, nextitem=nextitem)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/runner.py", line 79, in runtestprotocol
INTERNALERROR>     reports.append(call_and_report(item, "call", log))
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/runner.py", line 137, in call_and_report
INTERNALERROR>     hook.pytest_runtest_logreport(report=report)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "<remote exec>", line 89, in pytest_runtest_logreport
INTERNALERROR>   File "<remote exec>", line 103, in serialize_report
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 654, in __str__
INTERNALERROR>     s = self.__unicode__()
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 664, in __unicode__
INTERNALERROR>     self.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 709, in toterminal
INTERNALERROR>     self.reprtraceback.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 725, in toterminal
INTERNALERROR>     entry.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 769, in toterminal
INTERNALERROR>     self.reprfuncargs.toterminal(tw)
INTERNALERROR>   File "/usr/local/lib/python2.7/dist-packages/_pytest/_code/code.py", line 826, in toterminal
INTERNALERROR>     linesofar += ", " + ns
INTERNALERROR> UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 17: ordinal not in range(128)

Here's a quick fix (diff):

diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
index 0230c56..d5c1dca 100644
--- a/_pytest/_code/code.py
+++ b/_pytest/_code/code.py
@@ -860,22 +860,23 @@ class ReprFuncArgs(TerminalRepr):
         self.args = args
 
     def toterminal(self, tw):
+        from _pytest.compat import safe_decode
         if self.args:
-            linesofar = ""
+            linesofar = u""
             for name, value in self.args:
-                ns = "%s = %s" % (name, value)
+                ns = u"%s = %s" % (safe_decode(name), safe_decode(value))
                 if len(ns) + len(linesofar) + 2 > tw.fullwidth:
                     if linesofar:
                         tw.line(linesofar)
                     linesofar = ns
                 else:
                     if linesofar:
-                        linesofar += ", " + ns
+                        linesofar += u", " + ns
                     else:
                         linesofar = ns
             if linesofar:
                 tw.line(linesofar)
-            tw.line("")
+            tw.line(u"")
 
 
 def getrawcode(obj, trycall=True):
diff --git a/_pytest/compat.py b/_pytest/compat.py
index 45f9f86..a8b001b 100644
--- a/_pytest/compat.py
+++ b/_pytest/compat.py
@@ -258,6 +258,19 @@ else:
             return v.encode('utf-8', errors)
 
 
+def safe_decode(v):
+    import locale
+    if hasattr(v, 'decode'):
+        encoding = locale.getpreferredencoding()
+        try:
+            v = v.decode(encoding)
+        except UnicodeDecodeError:
+            encoding = 'utf-8'
+            v = v.decode(encoding, errors='replace')
+
+    return v
+
+
 COLLECT_FAKEMODULE_ATTRIBUTES = (
     'Collector',
     'Module',

Please let me know if I should open a PR.

Metadata

Metadata

Assignees

No one assigned

    Labels

    topic: reportingrelated to terminal output and user-facing messages and errorstype: bugproblem that needs to be addressed

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions