Skip to content

feat: Support RunEndEncoded type in approx_distinct #25951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jayzhan211 merged 2 commits into from
Oct 3, 2026
Merged

feat: Support RunEndEncoded type in approx_distinct #25951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8be94e0
Support RunEndEncoded for approx_distinct
mkleen Oct 1, 2026
9bfac5f
Merge branch 'main' into approx_distinct_runee
mkleen Oct 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions datafusion/functions-aggregate/src/approx_distinct.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -835,7 +835,9 @@ impl AggregateUDFImpl for ApproxDistinct {
| DataType::Struct(_)
| DataType::Union(_, _)
| DataType::LargeBinary => Box::new(HLLAccumulator::new()),
DataType::Dictionary(_, _) if is_supported_type(data_type) => {
DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
if is_supported_type(data_type) =>
{
Box::new(HLLAccumulator::new())
}
DataType::Null => {
Expand Down Expand Up @@ -894,25 +896,31 @@ fn is_fixed_domain_type(data_type: &DataType) -> bool {
}

fn is_supported_type(data_type: &DataType) -> bool {
let value_type = dictionary_value_type(data_type);
let value_type = encoded_value_type(data_type);
matches!(value_type, DataType::Null)
|| is_fixed_domain_type(value_type)
|| is_hll_groups_type(value_type)
}

fn dictionary_value_type(data_type: &DataType) -> &DataType {
fn encoded_value_type(data_type: &DataType) -> &DataType {
let mut value_type = data_type;
while let DataType::Dictionary(_, inner) = value_type {
value_type = inner;
loop {
match value_type {
DataType::Dictionary(_, inner) => value_type = inner,
DataType::RunEndEncoded(_, values) => value_type = values.data_type(),
_ => return value_type,
}
}
value_type
}

/// Returns true for the data types backed by the HyperLogLog
/// [`HllGroupsAccumulator`]. The fixed-domain types (booleans / small ints) and
/// `Null` fall back to the per-group [`Accumulator`] path.
fn is_hll_groups_type(data_type: &DataType) -> bool {
if matches!(data_type, DataType::Dictionary(_, _)) {
if matches!(
data_type,
DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
) {
return is_supported_type(data_type);
}

Expand Down
49 changes: 49 additions & 0 deletions datafusion/sqllogictest/test_files/aggregate.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1862,6 +1862,55 @@ SELECT approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type Dictionary\(Int32, Float64\) is not implemented
SELECT g, approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_distinct_group_test GROUP BY g;

# RunEndEncoded: run-end encoded values must hash identically to the plain
# values, so the counts below match the Utf8 / Int32 cases above.
query I
SELECT approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test WHERE g = 2;
----
2

query II
SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
----
1 2
2 2
3 0
4 1

query II
SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int64, "values": Int32)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
----
1 2
2 2
3 0
4 1

# RunEndEncoded over a fixed-domain value type (Int8) goes through the
# HyperLogLog accumulators and must match the Int32 counts above.
query II
SELECT g, approx_distinct(arrow_cast(arrow_cast(i, 'Int8'), 'RunEndEncoded("run_ends": non-null Int32, "values": Int8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
----
1 2
2 2
3 0
4 1

# RunEndEncoded over a dictionary
query II
SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(Int32, Utf8))')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
----
1 2
2 2
3 0
4 1

# Unsupported value types are rejected, as for a bare Float64
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
SELECT approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test;

statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test GROUP BY g;


# List
statement ok
Expand Down
Loading