Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions Doc/library/json.rst
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,10 @@ Basic Usage
.. versionchanged:: 3.6
All optional parameters are now :ref:`keyword-only <keyword-only_parameter>`.

.. versionchanged:: next
*sort_keys* no longer fails for keys of different basic types
or for unsupported keys skipped due to *skipkeys*.


.. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \
check_circular=True, allow_nan=True, cls=None, \
Expand Down Expand Up @@ -536,6 +540,11 @@ Encoders and Decoders
If *sort_keys* is true (default: ``False``), then the output of dictionaries
will be sorted by key; this is useful for regression tests to ensure that
JSON serializations can be compared on a day-to-day basis.
Keys of mixed types are sorted by groups: strings, numbers and ``None``.

.. versionchanged:: next
*sort_keys* no longer fails for keys of different basic types
or for unsupported keys skipped due to *skipkeys*.

If *indent* is a non-negative integer or string, then JSON array elements and
object members will be pretty-printed with that indent level. An indent level
Expand Down
33 changes: 32 additions & 1 deletion Lib/json/encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,32 @@ def floatstr(o, allow_nan=self.allow_nan,
self.skipkeys, _one_shot)
return _iterencode(o, 0)

def _sort_items(items, skipkeys):
"""Sort (key, value) pairs in separate groups, because keys of
different types are not comparable: strings, numbers and ``None``.

Unsupported keys are skipped if *skipkeys* is true and reported
otherwise.
"""
strings = []
nones = []
numbers = []
for item in items:
key, value = item
if isinstance(key, str):
strings.append(item)
elif key is None:
nones.append(item)
elif isinstance(key, (int, float)): # includes bool
numbers.append(item)
elif not skipkeys:
raise TypeError(f'keys must be str, int, float, bool or None, '
f'not {key.__class__.__name__}')
strings.sort()
numbers.sort()
return strings + numbers + nones


def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
_key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
):
Expand Down Expand Up @@ -343,7 +369,12 @@ def _iterencode_dict(dct, _current_indent_level):
item_separator = _item_separator
first = True
if _sort_keys:
items = sorted(dct.items())
items = list(dct.items())
try:
items.sort()
except TypeError:
# Keys of different types are not comparable.
items = _sort_items(items, _skipkeys)
else:
items = dct.items()
for key, value in items:
Expand Down
37 changes: 37 additions & 0 deletions Lib/test/test_json/test_dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,43 @@ def test_skipkeys_indent(self):
v = {b'invalid_key': False, 'valid_key': True}
self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}')

def test_dump_sort_keys_mixed_types(self):
# Keys of different types are sorted in separate groups.
self.assertEqual(
self.dumps({1: 'a', 'z': 'b', 'a': 'c'}, sort_keys=True),
'{"a": "c", "z": "b", "1": "a"}')
self.assertEqual(
self.dumps({None: 0, True: 1, False: 4, 2: 2, 'a': 3},
sort_keys=True),
'{"a": 3, "false": 4, "true": 1, "2": 2, "null": 0}')
# Numbers are still sorted as numbers, and adding a string key
# does not change their order.
self.assertEqual(
self.dumps({10: 1, 2: 2}, sort_keys=True),
'{"2": 2, "10": 1}')
self.assertEqual(
self.dumps({10: 1, 2: 2, 'a': 3}, sort_keys=True),
'{"a": 3, "2": 2, "10": 1}')
# Unsupported keys are still reported, or skipped.
with self.assertRaises(TypeError):
self.dumps({(1, 2): 'x', 'z': 'b'}, sort_keys=True)
self.assertEqual(
self.dumps({(1, 2): 'x', 'z': 'b'}, skipkeys=True, sort_keys=True),
'{"z": "b"}')

def test_dump_sort_keys_unsupported(self):
# Unsupported keys are reported or skipped, whether or not they are
# comparable with each other.
for d in ({(2,): 1, (1,): 2}, # comparable
{(2,): 1, (1,): 2, 'z': 3},
{(1,): 1, ('a',): 2, 'z': 3}): # not comparable
with self.subTest(d=d):
with self.assertRaises(TypeError):
self.dumps(d, sort_keys=True)
self.assertEqual(
self.dumps(d, skipkeys=True, sort_keys=True),
'{"z": 3}' if 'z' in d else '{}')

def test_encode_truefalse(self):
self.assertEqual(self.dumps(
{True: False, False: True}, sort_keys=True),
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_json/test_speedups.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ def test(name):
self.assertRaises(ZeroDivisionError, test, 'allow_nan')
self.assertRaises(ZeroDivisionError, test, 'sort_keys')

def test_unsortable_keys(self):
with self.assertRaises(TypeError):
self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'})

def test_current_indent_level(self):
enc = self.json.encoder.c_make_encoder(
markers=None,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
:func:`json.dump` and :func:`json.dumps` with ``sort_keys=True`` no longer
fail for keys of different basic types or for unsupported keys skipped due
to *skipkeys*. Keys of mixed types are sorted by groups: strings, numbers
and ``None``.
85 changes: 83 additions & 2 deletions Modules/_json.c
Original file line number Diff line number Diff line change
Expand Up @@ -1793,6 +1793,76 @@ _encoder_iterate_dict_lock_held(PyEncoderObject *s, PyUnicodeWriter *writer,
return 0;
}

/* Sort the (key, value) pairs in separate groups, because keys of
different types are not comparable: strings, numbers and None.
Unsupported keys are skipped if skipkeys is true and reported otherwise.
Return a new list, or NULL on error. */
static PyObject *
encoder_sort_items(PyObject *items, int skipkeys)
{
enum {STRINGS, NUMBERS, NONES, NGROUPS};
PyObject *groups[NGROUPS] = {NULL};
PyObject *result = NULL;

for (int i = 0; i < NGROUPS; i++) {
groups[i] = PyList_New(0);
if (groups[i] == NULL) {
goto done;
}
}
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) {
PyObject *item = PyList_GET_ITEM(items, i);
if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
PyErr_SetString(PyExc_ValueError, "items must return 2-tuples");
goto done;
}
PyObject *key = PyTuple_GET_ITEM(item, 0);
int group;
if (PyUnicode_Check(key)) {
group = STRINGS;
}
else if (key == Py_None) {
group = NONES;
}
else if (PyLong_Check(key) || PyFloat_Check(key)) { // includes bool
group = NUMBERS;
}
else if (skipkeys) {
continue;
}
else {
PyErr_Format(PyExc_TypeError,
"keys must be str, int, float, bool or None, "
"not %.100s", Py_TYPE(key)->tp_name);
goto done;
}
if (PyList_Append(groups[group], item) < 0) {
goto done;
}
}
/* There is at most one None key. */
if (PyList_Sort(groups[STRINGS]) < 0 ||
PyList_Sort(groups[NUMBERS]) < 0)
{
goto done;
}
result = groups[STRINGS];
groups[STRINGS] = NULL;
for (int i = STRINGS + 1; i < NGROUPS; i++) {
Py_ssize_t size = PyList_GET_SIZE(result);
if (PyList_SetSlice(result, size, size, groups[i]) < 0) {
Py_CLEAR(result);
goto done;
}
}

done:
for (int i = 0; i < NGROUPS; i++) {
Py_XDECREF(groups[i]);
}
return result;
}

static int
encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
PyObject *dct,
Expand Down Expand Up @@ -1837,10 +1907,21 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,

if (s->sort_keys || !PyAnyDict_CheckExact(dct)) {
PyObject *items = PyMapping_Items(dct);
if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) {
Py_XDECREF(items);
if (items == NULL) {
goto bail;
}
if (s->sort_keys && PyList_Sort(items) < 0) {
if (!PyErr_ExceptionMatches(PyExc_TypeError)) {
Py_DECREF(items);
goto bail;
}
/* Keys of different types are not comparable. */
PyErr_Clear();
Py_SETREF(items, encoder_sort_items(items, s->skipkeys));
if (items == NULL) {
goto bail;
}
}

int result;
Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(items);
Expand Down
Loading