From d6cfbd0aac21d7ff5963f346726dfeb2736dad4e Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Fri, 20 Jun 2025 16:11:15 +0800 Subject: [PATCH 1/2] doc: upgrade guide for new compression option for spill files --- docs/source/library-user-guide/upgrading.md | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md index 700b05a7ff869..378ca1b01b32a 100644 --- a/docs/source/library-user-guide/upgrading.md +++ b/docs/source/library-user-guide/upgrading.md @@ -65,6 +65,35 @@ See [#16317] for details. [#16317]: https://github.com/apache/datafusion/pull/16317 +### New `datafusion.execution.spill_compression` configuration option + +DataFusion 49.0.0 adds support for compressing spill files when data is written to disk during spilling query execution. A new configuration option `datafusion.execution.spill_compression` controls the compression codec used. + +**Configuration:** +- **Key**: `datafusion.execution.spill_compression` +- **Default**: `uncompressed` +- **Valid values**: `uncompressed`, `lz4_frame`, `zstd` + +**Usage:** + +```rust +# /* comment to avoid running +use datafusion::prelude::*; +use datafusion_common::config::SpillCompression; + +let config = SessionConfig::default() + .with_spill_compression(SpillCompression::Zstd); +let ctx = SessionContext::new_with_config(config); +# */ +``` + +Or via SQL: +```sql +SET datafusion.execution.spill_compression = 'zstd'; +``` + +For more details about this configuration option, including performance trade-offs between different compression codecs, see the [Configuration Settings](../user-guide/configs.md#datafusionexecutionspill_compression) documentation. + ## DataFusion `48.0.0` ### `Expr::Literal` has optional metadata From 2ed41745ab52311291e2ecb6a4eebf8bda3a14aa Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Fri, 20 Jun 2025 16:26:08 +0800 Subject: [PATCH 2/2] fix CI --- docs/source/library-user-guide/upgrading.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md index 378ca1b01b32a..e74d415d823df 100644 --- a/docs/source/library-user-guide/upgrading.md +++ b/docs/source/library-user-guide/upgrading.md @@ -70,6 +70,7 @@ See [#16317] for details. DataFusion 49.0.0 adds support for compressing spill files when data is written to disk during spilling query execution. A new configuration option `datafusion.execution.spill_compression` controls the compression codec used. **Configuration:** + - **Key**: `datafusion.execution.spill_compression` - **Default**: `uncompressed` - **Valid values**: `uncompressed`, `lz4_frame`, `zstd` @@ -88,11 +89,12 @@ let ctx = SessionContext::new_with_config(config); ``` Or via SQL: + ```sql SET datafusion.execution.spill_compression = 'zstd'; ``` -For more details about this configuration option, including performance trade-offs between different compression codecs, see the [Configuration Settings](../user-guide/configs.md#datafusionexecutionspill_compression) documentation. +For more details about this configuration option, including performance trade-offs between different compression codecs, see the [Configuration Settings](../user-guide/configs.md) documentation. ## DataFusion `48.0.0`