[PATCH] D68796: [LNT] Python 3 support: fix storage of json data as BLOB

[Thomas Preud'homme via Phabricator via llvm-commits]

thopre created this revision.
thopre added reviewers: cmatthews, hubert.reinterpretcast, kristof.beyls.

Machine and Run tables store JSON as binary data in their Parameters
column. However, JSON is inserted as text data in several places,
causing problem when SQLAlchemy reads it back since it expects binary
data but receives text, which Python3 throws an exception for.

2 constructs are responsible for inserting JSON as text:

1. json.dumps() method is used in several places to convert a dictionary to JSON which returns it as text. This commit adds an encoding step to UTF-8 since it is the recommended encoding for JSON.

2. One of the insertion with a JSON Parameters column in lnt_db_create.sql is missing a cast to BLOB. This commit adds the missing cast.


https://reviews.llvm.org/D68796

Files:
  lnt/server/db/migrations/upgrade_11_to_12.py
  lnt/server/db/migrations/upgrade_1_to_2.py
  lnt/server/db/testsuite.py
  lnt/server/db/testsuitedb.py
  tests/SharedInputs/SmallInstance/data/lnt_db_create.sql


Index: tests/SharedInputs/SmallInstance/data/lnt_db_create.sql
===================================================================
--- tests/SharedInputs/SmallInstance/data/lnt_db_create.sql
+++ tests/SharedInputs/SmallInstance/data/lnt_db_create.sql
@@ -153,7 +153,7 @@
 );
 INSERT INTO "NT_Machine" ("Name", "Parameters", "hardware", "os")
  VALUES('localhost__clang_DEV__x86_64',
-        '[["name", "localhost"], ["uname", "Darwin localhost 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12 18:47:41 PST 2012; root:xnu-1699.24.23~1/RELEASE_X86_64 x86_64"]]',
+        CAST('[["name", "localhost"], ["uname", "Darwin localhost 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12 18:47:41 PST 2012; root:xnu-1699.24.23~1/RELEASE_X86_64 x86_64"]]' AS BLOB),
         'x86_64','Darwin 11.3.0'); -- ID 1
 CREATE TABLE "NT_Test" (
 	"ID" INTEGER PRIMARY KEY NOT NULL, 
Index: lnt/server/db/testsuitedb.py
===================================================================
--- lnt/server/db/testsuitedb.py
+++ lnt/server/db/testsuitedb.py
@@ -143,7 +143,7 @@
 
             @parameters.setter
             def parameters(self, data):
-                self.parameters_data = json.dumps(sorted(data.items()))
+                self.parameters_data = json.dumps(sorted(data.items())).encode("utf-8")
 
             def get_baseline_run(self, session):
                 ts = Machine.testsuite
@@ -393,7 +393,7 @@
 
             @parameters.setter
             def parameters(self, data):
-                self.parameters_data = json.dumps(sorted(data.items()))
+                self.parameters_data = json.dumps(sorted(data.items())).encode("utf-8")
 
             def __json__(self, flatten_order=True):
                 result = {
Index: lnt/server/db/testsuite.py
===================================================================
--- lnt/server/db/testsuite.py
+++ lnt/server/db/testsuite.py
@@ -106,7 +106,7 @@
 
     def __init__(self, testsuite_name, data):
         self.testsuite_name = testsuite_name
-        self.jsonschema = json.dumps(data, encoding='utf-8', sort_keys=True)
+        self.jsonschema = json.dumps(data, sort_keys=True).encode('utf-8')
 
 
 class TestSuite(Base):
Index: lnt/server/db/migrations/upgrade_1_to_2.py
===================================================================
--- lnt/server/db/migrations/upgrade_1_to_2.py
+++ lnt/server/db/migrations/upgrade_1_to_2.py
@@ -84,7 +84,7 @@
         # Update the run info.
         run_info['inferred_run_order'] = run_order
         run_info['__report_version__'] = '1'
-        run.Parameters = json.dumps(sorted(run_info.items()))
+        run.Parameters = json.dumps(sorted(run_info.items())).encode('utf-8')
 
         if run_order != orig_order:
             # Lookup the new run order.
Index: lnt/server/db/migrations/upgrade_11_to_12.py
===================================================================
--- lnt/server/db/migrations/upgrade_11_to_12.py
+++ lnt/server/db/migrations/upgrade_11_to_12.py
@@ -21,7 +21,7 @@
         name = info.pop('name', None)
         if name is not None:
             info['hostname'] = name
-        machine.Parameters = json.dumps(sorted(info.items()))
+        machine.Parameters = json.dumps(sorted(info.items())).encode("utf-8")
 
     session.commit()
     session.close()


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D68796.224361.patch
Type: text/x-patch
Size: 3292 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20191010/1415da90/attachment.bin>