Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion python/datafusion/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,8 @@ def __init__(
- ``lz4``: LZ4 compression.
- ``lz4_raw``: LZ4_RAW compression.
- ``zstd(n)``: Zstandard compression with level n.
compression_level: Compression level to set.
compression_level: Compression level to set, overriding any level in
``compression``.
dictionary_enabled: Sets if dictionary encoding is enabled. If ``None``,
uses the default parquet writer setting.
dictionary_page_size_limit: Sets best effort maximum dictionary page size,
Expand Down Expand Up @@ -292,12 +293,23 @@ def __init__(
column_specific_options: Overrides options for specific columns. If a column
is not a part of this dictionary, it will use the parameters provided
here.

Examples:
>>> ParquetWriterOptions().compression
'zstd(3)'
>>> ParquetWriterOptions(compression_level=6).compression
'zstd(6)'
>>> opts = ParquetWriterOptions(compression="gzip(9)", compression_level=6)
>>> opts.compression
'gzip(6)'
"""
self.data_pagesize_limit = data_pagesize_limit
self.write_batch_size = write_batch_size
self.writer_version = writer_version
self.skip_arrow_metadata = skip_arrow_metadata
if compression_level is not None:
if compression is not None:
compression = compression.split("(", 1)[0]
self.compression = f"{compression}({compression_level})"
else:
self.compression = compression
Expand Down
52 changes: 52 additions & 0 deletions python/tests/test_dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -3069,6 +3069,58 @@ def test_write_parquet_options(df, tmp_path):
assert result == expected


@pytest.mark.parametrize("use_options_method", [False, True])
@pytest.mark.parametrize(
("compression", "level", "codec"),
[
(None, 6, "zstd"),
("zstd(3)", 6, "zstd"),
("gzip(9)", 6, "gzip"),
("brotli(7)", 4, "brotli"),
("gzip", 6, "gzip"),
],
)
def test_write_parquet_options_override_compression_level(
df, tmp_path, compression, level, codec, use_options_method
):
kwargs = {} if compression is None else {"compression": compression}
options = ParquetWriterOptions(**kwargs, compression_level=level)
if use_options_method:
df.write_parquet_with_options(tmp_path, options)
else:
df.write_parquet(tmp_path, options)

files = list(tmp_path.rglob("*.parquet"))
assert files
for file in files:
metadata = pq.ParquetFile(file).metadata
for row_group in range(metadata.num_row_groups):
for column_index in range(metadata.num_columns):
assert (
metadata.row_group(row_group)
.column(column_index)
.compression.lower()
== codec
)
assert pq.read_table(tmp_path).to_pydict() == df.to_pydict()


@pytest.mark.parametrize(
("compression", "level"),
[("gzip(6)", 12), ("brotli(7)", 15), ("zstd(3)", 23)],
)
def test_write_parquet_options_override_invalid_level(df, tmp_path, compression, level):
options = ParquetWriterOptions(compression=compression, compression_level=level)
with pytest.raises(Exception, match=r"valid compression range .*? exceeded."):
df.write_parquet_with_options(tmp_path, options)


def test_write_parquet_options_level_without_codec(df, tmp_path):
options = ParquetWriterOptions(compression=None, compression_level=6)
with pytest.raises(Exception, match="Unknown or unsupported parquet compression"):
df.write_parquet_with_options(tmp_path, options)


def test_write_parquet_options_error(df, tmp_path):
options = ParquetWriterOptions(compression="gzip", compression_level=6)
with pytest.raises(ValueError):
Expand Down