Un-revert Work around Python/C JSON unicode differences

This fix was reverted because it depended on a small bit of code
in a patch that was reverted that changed some python/ovs testing
and build. The fix is still necessary.

The OVS C-based JSON parser operates on bytes, so the parser_feed
function returns the number of bytes that are processed. The pure
Python JSON parser currently operates on unicode, so it expects
that Parser.feed() returns a number of characters. This difference
leads to parsing errors when unicode characters are passed to the
C JSON parser from Python.

Acked-by: Lucas Alvares Gomes <lucasagomes@gmail.com>
Signed-off-by: Terry Wilson <twilson@redhat.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

diff --git a/python/ovs/json.py b/python/ovs/json.py
index 94c8e30f365d09ebc5ac889679b0640f1be3a61e..96a07513d747e96b0b51de0f1d55693c8b314f99 100644 (file)
--- a/python/ovs/json.py
+++ b/python/ovs/json.py
@@ -21,10 +21,13 @@ import sys
 
 import six
 
+PARSER_C = 'C'
+PARSER_PY = 'PYTHON'
 try:
     import ovs._json
+    PARSER = PARSER_C
 except ImportError:
-    pass
+    PARSER = PARSER_PY
 
 __pychecker__ = 'no-stringiter'
 
@@ -91,10 +94,9 @@ class Parser(object):
     MAX_HEIGHT = 1000
 
     def __new__(cls, *args, **kwargs):
-        try:
+        if PARSER == PARSER_C:
             return ovs._json.Parser(*args, **kwargs)
-        except NameError:
-            return super(Parser, cls).__new__(cls)
+        return super(Parser, cls).__new__(cls)
 
     def __init__(self, check_trailer=False):
         self.check_trailer = check_trailer

diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py
index cc7b2cd52aa02d28c368f02ef8ecb154ca503180..4a3027e9e8718fa2a4e2b51764d2d51c2e6dfbce 100644 (file)
--- a/python/ovs/jsonrpc.py
+++ b/python/ovs/jsonrpc.py
@@ -272,7 +272,8 @@ class Connection(object):
                 # data, so we convert it here as soon as possible.
                 if data and not error:
                     try:
-                        data = decoder.decode(data)
+                        if six.PY3 or ovs.json.PARSER == ovs.json.PARSER_PY:
+                            data = decoder.decode(data)
                     except UnicodeError:
                         error = errno.EILSEQ
                 if error:
@@ -298,7 +299,11 @@ class Connection(object):
             else:
                 if self.parser is None:
                     self.parser = ovs.json.Parser()
-                self.input = self.input[self.parser.feed(self.input):]
+                if six.PY3 and ovs.json.PARSER == ovs.json.PARSER_C:
+                    self.input = self.input.encode('utf-8')[
+                        self.parser.feed(self.input):].decode()
+                else:
+                    self.input = self.input[self.parser.feed(self.input):]
                 if self.parser.is_done():
                     msg = self.__process_msg()
                     if msg: