PanicError in read_ndjson and polars.str.json_decode with empty struct about polars HOT 3 OPEN

OrderedDict([('offer', Struct({'my_value': Int64, 'condition': List(Struct({'applicationId': Int64, 'conditionRequestReason': List(Struct({'a': Int64}))}))}))])

# pylint: disable=missing-class-docstring, missing-function-docstring
""" Test for reading NDJSON files with nested fields. """
import io
import json
import unittest
from typing import Sequence, Any, cast

import polars as pl
import polars.testing


class TestNdJson(unittest.TestCase):
    def test_json_format(self):
        well_defined_data = io.StringIO(
            """{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": []}]}}
            {"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{"a":0}]}]}}""")

        # Initialize the frame from a sequence of dictionaries.
        dicts = cast(Sequence[dict[str, Any]],
                     (json.loads(line) for line in well_defined_data))
        frame_from_dicts = pl.from_dicts(dicts)
        print(frame_from_dicts.schema)

        # Initialize the frame from the NDJSON file.
        frame_from_ndjson = pl.read_ndjson(well_defined_data)
        print(frame_from_ndjson.schema)

        # Compare the content of the frames.
        polars.testing.assert_frame_equal(frame_from_dicts, frame_from_ndjson)

        # Initialize the frame from a malformed NDJSON
        incomplete_schema_data = io.StringIO(
            """{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{}]}]}}""")
        frame_with_schema = pl.read_ndjson(incomplete_schema_data, schema=frame_from_ndjson.schema)
        frame_with_schema_export = frame_with_schema.write_ndjson()

        self.assertEqual(
            '{"offer":{"my_value":0,"condition":[{"applicationId":0,"conditionRequestReason":[{"a":null}]}]}}\n',
            frame_with_schema_export)


if __name__ == '__main__':
    unittest.main()

    def test_read_ndjson_list_of_awkward_struct(self):
        input_data = io.StringIO(
            """{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{}]}]}}
            {"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{"a":0}]}]}}""")

        # Initialize the frame from a sequence of dictionaries.
        dicts = cast(Sequence[dict[str, Any]],
                     (json.loads(line) for line in input_data))
        frame_from_dicts = pl.from_dicts(dicts)
        print(frame_from_dicts.schema)

        # Initialize the frame from the NDJSON file.
        frame_from_ndjson = pl.read_ndjson(input_data)
        print(frame_from_ndjson.schema)

        # Compare the content of the frames.
        polars.testing.assert_frame_equal(frame_from_dicts, frame_from_ndjson)

>>> frame_from_dicts
shape: (2, 1)
┌─────────────────────────┐
│ offer                   │
│ ---                     │
│ struct[2]               │
╞═════════════════════════╡
│ {0,[{0,[{null,null}]}]} │
│ {0,[{0,[{null,0}]}]}    │
└─────────────────────────┘
>>> frame_from_dicts.schema
OrderedDict([('offer', Struct({'my_value': Int64, 'condition': List(Struct({'applicationId': Int64, 'conditionRequestReason': List(Struct({'': Null, 'a': Int64}))}))}))])

# pylint: disable=missing-class-docstring, missing-function-docstring, too-few-public-methods
""" Unit tests for the polars JSON decode functionality. """
import io
import pytest
import polars as pl
import polars.testing


class TestPolarsJsonDecode:
    @pytest.mark.parametrize('test_name, input_ndjson', [
        ("str", """{"list_field": ["a", "b"]}
                   {"list_field": []}
                   {"list_field": ["c", "d", "e"]}"""),
        ("int", """{"list_field": [1, 2]}
                   {"list_field": []}
                   {"list_field": [4, 5, 6]}""")
    ])
    def test_json_decode_list_of_basic_type(self, test_name, input_ndjson):
        print(f"Reading list of {test_name}...")
        input_buf = io.StringIO(input_ndjson)
        frame_from_ndjson = pl.read_ndjson(input_buf)

        # pylint: disable-next=assignment-from-no-return
        series_with_json = pl.Series(values=input_buf).str.strip_chars_start()
        series_decoded_unnested = series_with_json.str.json_decode().struct.unnest()

        assert frame_from_ndjson.schema == series_decoded_unnested.schema
        polars.testing.assert_frame_equal(frame_from_ndjson, series_decoded_unnested)


if __name__ == '__main__':
    pytest.main()