Ticket #30190: patch-2-wip.diff

File patch-2-wip.diff, 13.8 KB (added by aliva, 6 years ago)

patch 2 WIP

  • django/core/serializers/__init__.py

    diff --git a/django/core/serializers/__init__.py b/django/core/serializers/__init__.py
    index 4b0ae078e2..bcf6928c5f 100644
    a b BUILTIN_SERIALIZERS = {  
    2828    "python": "django.core.serializers.python",
    2929    "json": "django.core.serializers.json",
    3030    "yaml": "django.core.serializers.pyyaml",
     31    "jsonl": "django.core.serializers.jsonl",
    3132}
    3233
    3334_serializers = {}
  • new file django/core/serializers/jsonl.py

    diff --git a/django/core/serializers/jsonl.py b/django/core/serializers/jsonl.py
    new file mode 100644
    index 0000000000..f6852e5a4b
    - +  
     1"""
     2Serialize data to/from JSON lines
     3"""
     4
     5import json
     6
     7from django.core.serializers.base import DeserializationError
     8from django.core.serializers.json import DjangoJSONEncoder
     9from django.core.serializers.python import (
     10    Deserializer as PythonDeserializer, Serializer as PythonSerializer,
     11)
     12
     13
     14class Serializer(PythonSerializer):
     15    internal_use_only = False
     16
     17    def start_serialization(self):
     18        self._current = None
     19        self.json_kwargs = self.options.copy()
     20        self.json_kwargs.pop('stream', None)
     21        self.json_kwargs.pop('fields', None)
     22        self.json_kwargs.pop('indent', None)
     23        self.json_kwargs.setdefault('cls', DjangoJSONEncoder)
     24
     25    def end_object(self, obj):
     26        json.dump(self.get_dump_object(obj), self.stream, **self.json_kwargs)
     27        self.stream.write("\n")
     28        self._current = None
     29
     30    def getvalue(self):
     31        # Grandparent super
     32        return super(PythonSerializer, self).getvalue()
     33
     34
     35def Deserializer(stream_or_string, **options):
     36    if isinstance(stream_or_string, (bytes, str)):
     37        stream_or_string = stream_or_string.split('\n')
     38
     39    for line in stream_or_string:
     40        line = line.strip()
     41        if line == '':
     42            continue
     43        try:
     44            yield list(PythonDeserializer([json.loads(line), ], **options))[0]
     45        except (GeneratorExit, DeserializationError):
     46            raise
     47        except Exception as exc:
     48            raise DeserializationError() from exc
  • new file patch.diff

    diff --git a/patch.diff b/patch.diff
    new file mode 100644
    index 0000000000..db4f9def83
    - +  
     1diff --git a/django/core/serializers/__init__.py b/django/core/serializers/__init__.py
     2index 4b0ae078e2..bcf6928c5f 100644
     3--- a/django/core/serializers/__init__.py
     4+++ b/django/core/serializers/__init__.py
     5@@ -28,6 +28,7 @@ BUILTIN_SERIALIZERS = {
     6     "python": "django.core.serializers.python",
     7     "json": "django.core.serializers.json",
     8     "yaml": "django.core.serializers.pyyaml",
     9+    "jsonl": "django.core.serializers.jsonl",
     10 }
     11 
     12 _serializers = {}
     13diff --git a/django/core/serializers/jsonl.py b/django/core/serializers/jsonl.py
     14new file mode 100644
     15index 0000000000..15c981a8d8
     16--- /dev/null
     17+++ b/django/core/serializers/jsonl.py
     18@@ -0,0 +1,48 @@
     19+"""
     20+Serialize data to/from JSON lines
     21+"""
     22+
     23+import json
     24+
     25+from django.core.serializers.base import DeserializationError
     26+from django.core.serializers.json import DjangoJSONEncoder
     27+from django.core.serializers.python import (
     28+    Deserializer as PythonDeserializer, Serializer as PythonSerializer,
     29+)
     30+
     31+
     32+class Serializer(PythonSerializer):
     33+    internal_use_only = False
     34+
     35+    def start_serialization(self):
     36+        self._current = None
     37+        self.json_kwargs = self.options.copy()
     38+        self.json_kwargs.pop('stream', None)
     39+        self.json_kwargs.pop('fields', None)
     40+        self.options.pop('indent', None)
     41+        self.json_kwargs.setdefault('cls', DjangoJSONEncoder)
     42+
     43+    def end_object(self, obj):
     44+        json.dump(self.get_dump_object(obj), self.stream, **self.json_kwargs)
     45+        self.stream.write("\n")
     46+        self._current = None
     47+
     48+    def getvalue(self):
     49+        # Grandparent super
     50+        return super(PythonSerializer, self).getvalue()
     51+
     52+
     53+def Deserializer(stream_or_string, **options):
     54+    if isinstance(stream_or_string, (bytes, str)):
     55+        stream_or_string = stream_or_string.split('\n')
     56+
     57+    for line in stream_or_string:
     58+        line = line.strip()
     59+        if line == '':
     60+            continue
     61+        try:
     62+            yield list(PythonDeserializer([json.loads(line), ], **options))[0]
     63+        except (GeneratorExit, DeserializationError):
     64+            raise
     65+        except Exception as exc:
     66+            raise DeserializationError() from exc
  • new file tests/serializers/test_jsonl.py

    diff --git a/tests/serializers/test_jsonl.py b/tests/serializers/test_jsonl.py
    new file mode 100644
    index 0000000000..9096ce09dd
    - +  
     1import decimal
     2import json
     3import re
     4
     5from django.core import serializers
     6from django.core.serializers.base import DeserializationError
     7from django.db import models
     8from django.test import TestCase, TransactionTestCase
     9from django.test.utils import isolate_apps
     10
     11from .models import Score
     12from .tests import SerializersTestBase, SerializersTransactionTestBase
     13
     14
     15class JsonlSerializerTestCase(SerializersTestBase, TestCase):
     16    serializer_name = "jsonl"
     17    pkless_str = """
     18    {"pk": null, "model": "serializers.category", "fields": {"name": "Reference"}}
     19    {"model": "serializers.category","fields": {"name": "Non-fiction"}}
     20    """
     21    mapping_ordering_str = """{"model": "serializers.article", "pk": %(article_pk)s, """ \
     22        """"fields": {"author": %(author_pk)s, "headline": "Poker has no place on ESPN", """ \
     23        """"pub_date": "2006-06-16T11:00:00", """ \
     24        """"categories": [%(first_category_pk)s, %(second_category_pk)s], "meta_data": []}}\n"""
     25
     26    def test_deterministic_mapping_ordering(self):
     27        super(JsonlSerializerTestCase, self).test_deterministic_mapping_ordering()
     28
     29    @staticmethod
     30    def _validate_output(serial_str):
     31        try:
     32            [json.loads(line) for line in serial_str.splitlines()]
     33        except Exception:
     34            return False
     35        else:
     36            return True
     37
     38    @staticmethod
     39    def _get_pk_values(serial_str):
     40        serial_list = [json.loads(line) for line in serial_str.splitlines()]
     41        return [obj_dict['pk'] for obj_dict in serial_list]
     42
     43    @staticmethod
     44    def _get_field_values(serial_str, field_name):
     45        serial_list = [json.loads(line) for line in serial_str.splitlines()]
     46        return [obj_dict['fields'][field_name] for obj_dict in serial_list if field_name in obj_dict['fields']]
     47
     48    def test_indentation_whitespace(self):
     49        s = serializers.json.Serializer()
     50        json_data = s.serialize([Score(score=5.0), Score(score=6.0)], indent=2)
     51        for line in json_data.splitlines():
     52            if re.search(r'.+,\s*$', line):
     53                self.assertEqual(line, line.rstrip())
     54
     55    @isolate_apps('serializers')
     56    def test_custom_encoder(self):
     57        class ScoreDecimal(models.Model):
     58            score = models.DecimalField()
     59
     60        class CustomJSONEncoder(json.JSONEncoder):
     61            def default(self, o):
     62                if isinstance(o, decimal.Decimal):
     63                    return str(o)
     64                return super().default(o)
     65
     66        s = serializers.json.Serializer()
     67        json_data = s.serialize(
     68            [ScoreDecimal(score=decimal.Decimal(1.0))], cls=CustomJSONEncoder
     69        )
     70        self.assertIn('"fields": {"score": "1"}', json_data)
     71
     72    def test_json_deserializer_exception(self):
     73        with self.assertRaises(DeserializationError):
     74            for obj in serializers.deserialize("json", """[{"pk":1}"""):
     75                pass
     76
     77    def test_helpful_error_message_invalid_pk(self):
     78        """
     79        If there is an invalid primary key, the error message should contain
     80        the model associated with it.
     81        """
     82        test_string = """[{
     83            "pk": "badpk",
     84            "model": "serializers.player",
     85            "fields": {
     86                "name": "Bob",
     87                "rank": 1,
     88                "team": "Team"
     89            }
     90        }]"""
     91        with self.assertRaisesMessage(DeserializationError, "(serializers.player:pk=badpk)"):
     92            list(serializers.deserialize('json', test_string))
     93
     94    def test_helpful_error_message_invalid_field(self):
     95        """
     96        If there is an invalid field value, the error message should contain
     97        the model associated with it.
     98        """
     99        test_string = """[{
     100            "pk": "1",
     101            "model": "serializers.player",
     102            "fields": {
     103                "name": "Bob",
     104                "rank": "invalidint",
     105                "team": "Team"
     106            }
     107        }]"""
     108        expected = "(serializers.player:pk=1) field_value was 'invalidint'"
     109        with self.assertRaisesMessage(DeserializationError, expected):
     110            list(serializers.deserialize('json', test_string))
     111
     112    def test_helpful_error_message_for_foreign_keys(self):
     113        """
     114        Invalid foreign keys with a natural key should throw a helpful error
     115        message, such as what the failing key is.
     116        """
     117        test_string = """[{
     118            "pk": 1,
     119            "model": "serializers.category",
     120            "fields": {
     121                "name": "Unknown foreign key",
     122                "meta_data": [
     123                    "doesnotexist",
     124                    "metadata"
     125                ]
     126            }
     127        }]"""
     128        key = ["doesnotexist", "metadata"]
     129        expected = "(serializers.category:pk=1) field_value was '%r'" % key
     130        with self.assertRaisesMessage(DeserializationError, expected):
     131            list(serializers.deserialize('json', test_string))
     132
     133    def test_helpful_error_message_for_many2many_non_natural(self):
     134        """
     135        Invalid many-to-many keys should throw a helpful error message.
     136        """
     137        test_string = """[{
     138            "pk": 1,
     139            "model": "serializers.article",
     140            "fields": {
     141                "author": 1,
     142                "headline": "Unknown many to many",
     143                "pub_date": "2014-09-15T10:35:00",
     144                "categories": [1, "doesnotexist"]
     145            }
     146        }, {
     147            "pk": 1,
     148            "model": "serializers.author",
     149            "fields": {
     150                "name": "Agnes"
     151            }
     152        }, {
     153            "pk": 1,
     154            "model": "serializers.category",
     155            "fields": {
     156                "name": "Reference"
     157            }
     158        }]"""
     159        expected = "(serializers.article:pk=1) field_value was 'doesnotexist'"
     160        with self.assertRaisesMessage(DeserializationError, expected):
     161            list(serializers.deserialize('json', test_string))
     162
     163    def test_helpful_error_message_for_many2many_natural1(self):
     164        """
     165        Invalid many-to-many keys should throw a helpful error message.
     166        This tests the code path where one of a list of natural keys is invalid.
     167        """
     168        test_string = """[{
     169            "pk": 1,
     170            "model": "serializers.categorymetadata",
     171            "fields": {
     172                "kind": "author",
     173                "name": "meta1",
     174                "value": "Agnes"
     175            }
     176        }, {
     177            "pk": 1,
     178            "model": "serializers.article",
     179            "fields": {
     180                "author": 1,
     181                "headline": "Unknown many to many",
     182                "pub_date": "2014-09-15T10:35:00",
     183                "meta_data": [
     184                    ["author", "meta1"],
     185                    ["doesnotexist", "meta1"],
     186                    ["author", "meta1"]
     187                ]
     188            }
     189        }, {
     190            "pk": 1,
     191            "model": "serializers.author",
     192            "fields": {
     193                "name": "Agnes"
     194            }
     195        }]"""
     196        key = ["doesnotexist", "meta1"]
     197        expected = "(serializers.article:pk=1) field_value was '%r'" % key
     198        with self.assertRaisesMessage(DeserializationError, expected):
     199            for obj in serializers.deserialize('json', test_string):
     200                obj.save()
     201
     202    def test_helpful_error_message_for_many2many_natural2(self):
     203        """
     204        Invalid many-to-many keys should throw a helpful error message. This
     205        tests the code path where a natural many-to-many key has only a single
     206        value.
     207        """
     208        test_string = """[{
     209            "pk": 1,
     210            "model": "serializers.article",
     211            "fields": {
     212                "author": 1,
     213                "headline": "Unknown many to many",
     214                "pub_date": "2014-09-15T10:35:00",
     215                "meta_data": [1, "doesnotexist"]
     216            }
     217        }, {
     218            "pk": 1,
     219            "model": "serializers.categorymetadata",
     220            "fields": {
     221                "kind": "author",
     222                "name": "meta1",
     223                "value": "Agnes"
     224            }
     225        }, {
     226            "pk": 1,
     227            "model": "serializers.author",
     228            "fields": {
     229                "name": "Agnes"
     230            }
     231        }]"""
     232        expected = "(serializers.article:pk=1) field_value was 'doesnotexist'"
     233        with self.assertRaisesMessage(DeserializationError, expected):
     234            for obj in serializers.deserialize('json', test_string, ignore=False):
     235                obj.save()
     236
     237
     238class JsonSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase):
     239    serializer_name = "json"
     240    fwd_ref_str = """[
     241    {
     242        "pk": 1,
     243        "model": "serializers.article",
     244        "fields": {
     245            "headline": "Forward references pose no problem",
     246            "pub_date": "2006-06-16T15:00:00",
     247            "categories": [1],
     248            "author": 1
     249        }
     250    },
     251    {
     252        "pk": 1,
     253        "model": "serializers.category",
     254        "fields": {
     255            "name": "Reference"
     256        }
     257    },
     258    {
     259        "pk": 1,
     260        "model": "serializers.author",
     261        "fields": {
     262            "name": "Agnes"
     263        }
     264    }]"""
Back to Top