From 037091ca4b922c7f9e09c9a72aac01432a3c69a6 Mon Sep 17 00:00:00 2001 From: ahmedabu98 Date: Fri, 13 Nov 2026 18:20:34 +0000 Subject: [PATCH 1/2] Return rows of unbounded Beam SQL queries while the pipeline runs --- .../iceberg/BeamSqlCliIcebergCdcTest.java | 28 ++ .../sql/impl/rel/BeamEnumerableConverter.java | 152 ++++++++- .../extensions/sql/impl/rel/LiveResult.java | 228 +++++++++++++ .../extensions/sql/impl/JdbcDriverTest.java | 11 +- .../sql/impl/rel/LiveResultTest.java | 322 ++++++++++++++++++ 5 files changed, 723 insertions(+), 18 deletions(-) create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java diff --git a/sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergCdcTest.java b/sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergCdcTest.java index 09db80804b24..62a18d5747cf 100644 --- a/sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergCdcTest.java +++ b/sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergCdcTest.java @@ -34,6 +34,7 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -368,6 +369,33 @@ public void streamingChangesAreUnbounded() throws SQLException { + " 'triggering_frequency_seconds'")); } + /** The read never ends, so the rows have to arrive while it runs. */ + @Test(timeout = 120_000) + public void streamingChangesArriveWhileThePipelineRuns() throws Exception { + seedOrders(); + List rows = new ArrayList<>(); + try (Connection connection = + JdbcDriver.connect(catalogManager, PipelineOptionsFactory.create()); + Statement statement = connection.createStatement(); + ResultSet result = + statement.executeQuery( + "SELECT * FROM CHANGES(orders) WITH ('streaming' = true, 'starting_strategy' =" + + " 'earliest', 'poll_interval_seconds' = 1, " + + CHANGE_TYPE + + ")")) { + while (rows.size() < ALL_CHANGES.size() && result.next()) { + rows.add( + format( + "%s:%s:%s:%s", + result.getObject(1), + result.getObject(2), + result.getObject(3), + result.getObject(4))); + } + } + assertThat(rows, containsInAnyOrder(ALL_CHANGES.toArray())); + } + @Test public void eachReferenceReadsItsOwnChanges() { seedOrders(); diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java index 12a939754602..6de4250f3515 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java @@ -27,10 +27,12 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Queue; import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -61,12 +63,15 @@ import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.DataContext; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.EnumerableRel; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.PhysType; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.AbstractEnumerable; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Enumerable; +import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Enumerator; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Linq4j; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.tree.Expression; @@ -180,10 +185,132 @@ static Enumerable toEnumerable( return count(options, node, cancelFlag); } else if (isLimitQuery(node)) { return limitCollect(options, node, cancelFlag); + } else if (node.isBounded() == IsBounded.UNBOUNDED) { + return liveCollect(options, node, cancelFlag); } return Linq4j.asEnumerable(rowToAvaticaAndUnboxValues(collectRows(options, node, cancelFlag))); } + /** + * Runs the pipeline of an unbounded query and hands over its rows while it runs. + * + *

If the shell set a {@link LiveResult.Viewer}, it shows the rows and the query returns none. + * Otherwise the query returns the rows as they arrive, until the pipeline ends, the statement is + * cancelled, or the result set is closed. + */ + private static Enumerable liveCollect( + PipelineOptions options, BeamRelNode node, AtomicBoolean cancelFlag) { + checkArgument( + options + .getRunner() + .getCanonicalName() + .equals("org.apache.beam.runners.direct.DirectRunner"), + "SELECT without INSERT is only supported in DirectRunner in SQL Shell."); + Pipeline pipeline = Pipeline.create(options); + PCollection rows = BeamSqlRelUtils.toPCollection(pipeline, node); + rows.apply(ParDo.of(new LiveCollector())); + LiveResult.Viewer viewer = LiveResult.getViewer(); + LiveResult result = LiveResult.start(pipeline, rows.getSchema(), cancelFlag); + if (viewer == null) { + return new LiveRows(result); + } + + RuntimeException failure; + try { + viewer.show(result); + } catch (IOException e) { + throw new RuntimeException("Failed to show the rows of the query", e); + } finally { + // Read the failure first: errors after closing come from cancelling the pipeline. + failure = result.getFailure(); + result.close(); + } + if (failure != null) { + throw failure; + } + return Linq4j.emptyEnumerable(); + } + + /** The rows of a {@link LiveResult} as they arrive. Closing them cancels the pipeline. */ + private static class LiveRows extends AbstractEnumerable { + private final LiveResult result; + + LiveRows(LiveResult result) { + this.result = result; + } + + @Override + public Enumerator enumerator() { + return Linq4j.iterableEnumerator(this); + } + + /** + * JDBC reads the rows through this iterator. Unlike the default one, it doesn't wait for the + * next row before it returns the current one, which would hold back the newest row. + */ + @Override + public Iterator iterator() { + return new LiveIterator(); + } + + private class LiveIterator implements Iterator, AutoCloseable { + private LiveResult.@Nullable Change next; + + @Override + public boolean hasNext() { + // A cancelled statement ends the rows. + while (next == null && !result.isCancelRequested()) { + // No rows come after the pipeline ends, so then an empty poll means there are no more. + boolean finished = result.isFinished(); + try { + next = result.poll(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + if (next == null && finished) { + RuntimeException failure = result.getFailure(); + close(); + if (failure != null) { + throw failure; + } + return false; + } + } + if (next == null) { + close(); + } + return next != null; + } + + @Override + public Object next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Row row = next.getRow(); + next = null; + return rowToAvaticaAndUnbox(row); + } + + @Override + public void close() { + result.close(); + } + } + } + + /** Adds each row and its change kind to the {@link LiveResult} of its pipeline. */ + private static class LiveCollector extends DoFn { + @ProcessElement + public void processElement(@Element Row row, ValueKind kind, PipelineOptions options) { + LiveResult result = LiveResult.forOptionsId(options.getOptionsId()); + if (result != null) { + result.add(kind, row); + } + } + } + private static PipelineResult limitRun( PipelineOptions options, BeamRelNode node, @@ -360,21 +487,22 @@ public void processElement(ProcessContext context) { private static List rowToAvaticaAndUnboxValues(Queue values) { return values.stream() - .map( - row -> { - Object[] objects = rowToAvatica(row); - if (objects.length == 1) { - // if objects.length == 1, that means input Row contains only 1 column/element, - // then an Object instead of Object[] should be returned because of - // CalciteResultSet's behaviour that tries to convert one column row to an Object. - return objects[0]; - } else { - return objects; - } - }) + .map(BeamEnumerableConverter::rowToAvaticaAndUnbox) .collect(Collectors.toList()); } + private static Object rowToAvaticaAndUnbox(Row row) { + Object[] objects = rowToAvatica(row); + if (objects.length == 1) { + // if objects.length == 1, that means input Row contains only 1 column/element, + // then an Object instead of Object[] should be returned because of + // CalciteResultSet's behaviour that tries to convert one column row to an Object. + return objects[0]; + } else { + return objects; + } + } + private static Object[] rowToAvatica(Row row) { Schema schema = row.getSchema(); Object[] convertedColumns = new Object[schema.getFields().size()]; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java new file mode 100644 index 000000000000..7735d83150e7 --- /dev/null +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.sql.impl.rel; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.beam.runners.direct.DirectOptions; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.PipelineResult.State; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The rows of an unbounded query, handed over while its pipeline runs. + * + *

{@link BeamEnumerableConverter} runs the pipeline on the DirectRunner. Its last step adds each + * row and its {@link ValueKind} here, and a JDBC result set or the {@link Viewer} takes them out. + * This only works on the DirectRunner, because it runs the pipeline in this JVM. + */ +@Internal +public final class LiveResult { + /** Shows the rows of unbounded queries while their pipelines run, in place of JDBC results. */ + public interface Viewer { + /** + * Shows the rows until the user is done or {@link LiveResult#isCancelRequested()} turns true. + * It runs on the statement's thread. Once it returns, the pipeline is cancelled and the + * statement returns no rows, or fails if the pipeline failed. + */ + void show(LiveResult result) throws IOException; + } + + /** A row and the kind of change it is. */ + public static final class Change { + private final ValueKind kind; + private final Row row; + + Change(ValueKind kind, Row row) { + this.kind = kind; + this.row = row; + } + + public ValueKind getKind() { + return kind; + } + + public Row getRow() { + return row; + } + } + + private static final Logger LOG = LoggerFactory.getLogger(LiveResult.class); + + /** How many rows wait to be taken out before the pipeline waits for room. */ + private static final int CAPACITY = 10_000; + + /** The live results of running pipelines, by pipeline options id. */ + private static final Map RESULTS = new ConcurrentHashMap<>(); + + private static volatile @Nullable Viewer viewer; + + private final Schema schema; + private final AtomicBoolean cancelFlag; + private final long id; + private final LinkedBlockingDeque changes = new LinkedBlockingDeque<>(CAPACITY); + private final CountDownLatch finished = new CountDownLatch(1); + private volatile @Nullable PipelineResult pipelineResult; + private volatile State state = State.RUNNING; + private volatile @Nullable RuntimeException failure; + private volatile boolean closed; + + private LiveResult(Schema schema, AtomicBoolean cancelFlag, long id) { + this.schema = schema; + this.cancelFlag = cancelFlag; + this.id = id; + } + + /** + * Sets the viewer for the rows of unbounded queries. The Beam SQL shell sets one when it runs in + * a terminal. Without one, the rows are returned through JDBC as they arrive. + */ + public static void setViewer(@Nullable Viewer newViewer) { + viewer = newViewer; + } + + static @Nullable Viewer getViewer() { + return viewer; + } + + static @Nullable LiveResult forOptionsId(long optionsId) { + return RESULTS.get(optionsId); + } + + /** + * Starts the pipeline without waiting for it, and watches it until it ends. + * + * @param cancelFlag set when the statement is cancelled, for example with Ctrl+C in the shell + */ + static LiveResult start(Pipeline pipeline, Schema schema, AtomicBoolean cancelFlag) { + LiveResult result = new LiveResult(schema, cancelFlag, pipeline.getOptions().getOptionsId()); + RESULTS.put(result.id, result); + PipelineResult pipelineResult; + try { + pipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); + pipelineResult = pipeline.run(); + } catch (RuntimeException e) { + RESULTS.remove(result.id); + throw e; + } + result.pipelineResult = pipelineResult; + Thread watcher = new Thread(() -> result.watch(pipelineResult), "BeamSqlLiveResultWatcher"); + watcher.setDaemon(true); + watcher.start(); + return result; + } + + private void watch(PipelineResult pipelineResult) { + @Nullable State end = null; + try { + end = pipelineResult.waitUntilFinish(); + } catch (RuntimeException e) { + failure = e; + end = State.FAILED; + } finally { + state = end == null ? State.UNKNOWN : end; + finished.countDown(); + } + } + + /** + * Adds a row, waiting for room if the buffer is full. The pipeline calls this from many threads. + */ + void add(ValueKind kind, Row row) { + Change change = new Change(kind, row); + try { + while (!closed) { + if (changes.offerLast(change, 100, TimeUnit.MILLISECONDS)) { + return; + } + } + } catch (InterruptedException e) { + // The pipeline is being cancelled. + Thread.currentThread().interrupt(); + } + } + + /** The schema of the query's rows. */ + public Schema getSchema() { + return schema; + } + + /** Waits up to {@code timeout} for the next row, and returns null if none came. */ + public @Nullable Change poll(long timeout, TimeUnit unit) throws InterruptedException { + return changes.pollFirst(timeout, unit); + } + + /** Moves the rows that are waiting into {@code sink}, without waiting for more. */ + public void drainTo(Collection sink) { + changes.drainTo(sink); + } + + /** Whether the statement was cancelled, for example with Ctrl+C in the shell. */ + public boolean isCancelRequested() { + return cancelFlag.get(); + } + + /** Whether the pipeline has ended. Rows may still be waiting to be taken out. */ + public boolean isFinished() { + return finished.getCount() == 0; + } + + /** {@link State#RUNNING} until the pipeline ends, and then the state it ended in. */ + public State getState() { + return isFinished() ? state : State.RUNNING; + } + + /** The error that the pipeline failed with, or null if it hasn't failed. */ + public @Nullable RuntimeException getFailure() { + return isFinished() ? failure : null; + } + + /** Stops taking rows, cancels the pipeline and waits up to 10 seconds for it to stop. */ + void close() { + if (closed) { + return; + } + closed = true; + RESULTS.remove(id); + PipelineResult result = pipelineResult; + if (result != null && !isFinished()) { + try { + result.cancel(); + finished.await(10, TimeUnit.SECONDS); + } catch (IOException | RuntimeException e) { + LOG.warn("Failed to cancel the pipeline", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + changes.clear(); + } +} diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java index b022d94777ba..ec0779128142 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java @@ -38,7 +38,6 @@ import java.sql.Statement; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.Locale; @@ -482,11 +481,11 @@ public void testCancelUnboundedQuery() throws Exception { } assertCancels(insert, insertRun); - for (String select : - Arrays.asList("SELECT sequence FROM seq", "SELECT sequence FROM seq LIMIT 1000000")) { - Statement statement = connection.createStatement(); - assertCancels(statement, executor.submit(() -> statement.executeQuery(select))); - } + // A SELECT without LIMIT returns rows while its pipeline runs. LiveResultTest cancels it. + Statement select = connection.createStatement(); + assertCancels( + select, + executor.submit(() -> select.executeQuery("SELECT sequence FROM seq LIMIT 1000000"))); } finally { executor.shutdownNow(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java new file mode 100644 index 000000000000..845b4f075027 --- /dev/null +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.sql.impl.rel; + +import static java.util.stream.Collectors.joining; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.apache.beam.sdk.coders.VarLongCoder; +import org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics; +import org.apache.beam.sdk.extensions.sql.impl.JdbcDriver; +import org.apache.beam.sdk.extensions.sql.meta.SchemaBaseBeamTable; +import org.apache.beam.sdk.extensions.sql.meta.provider.ReadOnlyTableProvider; +import org.apache.beam.sdk.io.GenerateSequence; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.joda.time.Duration; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unbounded queries hand over their rows while their pipelines run. */ +@RunWith(JUnit4.class) +public class LiveResultTest { + private static final Schema SCHEMA = + Schema.builder().addInt64Field("id").addStringField("name").build(); + + /** The changes that {@link ChangesTable} emits, in order. */ + private static final List KINDS = + ImmutableList.of( + ValueKind.INSERT, + ValueKind.INSERT, + ValueKind.UPDATE_BEFORE, + ValueKind.UPDATE_AFTER, + ValueKind.DELETE); + + private static final List ROWS = + ImmutableList.of(row(1, "a"), row(2, "b"), row(1, "a"), row(1, "aa"), row(2, "b")); + + @After + public void resetViewer() { + LiveResult.setViewer(null); + } + + @Test + public void rowsArriveWhileThePipelineRunsUntilCancelled() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (Connection connection = connect(); + Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT id FROM sparse")) { + // The second row comes 30 seconds after the first, and next() must not wait for it. + long start = System.nanoTime(); + assertTrue(result.next()); + assertTrue(System.nanoTime() - start < TimeUnit.SECONDS.toNanos(20)); + + // Cancel while next() waits for the second row. + Future cancel = + executor.submit( + () -> { + Thread.sleep(300); + statement.cancel(); + return null; + }); + assertFalse(result.next()); + cancel.get(); + } finally { + executor.shutdownNow(); + } + assertPipelinesStopped(); + } + + @Test + public void rowsEndWithThePipeline() throws Exception { + List rows = new ArrayList<>(); + try (Connection connection = connect(); + Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT * FROM changes")) { + while (result.next()) { + rows.add(result.getLong(1) + ":" + result.getString(2)); + } + } + assertThat(rows, containsInAnyOrder("1:a", "2:b", "1:a", "1:aa", "2:b")); + } + + @Test + public void viewerSeesEachRowWithItsKind() throws Exception { + List changes = new CopyOnWriteArrayList<>(); + LiveResult.setViewer(result -> collect(result, changes, ROWS.size())); + + try (Connection connection = connect(); + Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT id, UPPER(name) FROM changes")) { + // The viewer showed the rows, so the query returns none. + assertFalse(result.next()); + } + // The kinds survive the projection. + assertThat( + changes, + containsInAnyOrder( + "INSERT:1:A", "INSERT:2:B", "UPDATE_BEFORE:1:A", "UPDATE_AFTER:1:AA", "DELETE:2:B")); + assertPipelinesStopped(); + } + + @Test + public void viewerReturnsWhenTheStatementIsCancelled() throws Exception { + CountDownLatch showing = new CountDownLatch(1); + AtomicBoolean sawCancel = new AtomicBoolean(); + LiveResult.setViewer( + result -> { + showing.countDown(); + sawCancel.set(waitFor(result::isCancelRequested)); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (Connection connection = connect(); + Statement statement = connection.createStatement()) { + Future query = + executor.submit(() -> statement.executeQuery("SELECT * FROM sparse")); + assertTrue(showing.await(30, TimeUnit.SECONDS)); + statement.cancel(); + // The statement ends quietly: cancelling is how the user leaves the viewer. + query.get(30, TimeUnit.SECONDS).close(); + } finally { + executor.shutdownNow(); + } + assertTrue(sawCancel.get()); + assertPipelinesStopped(); + } + + @Test + public void failedPipelineFailsTheQuery() throws Exception { + try (Connection connection = connect(); + Statement statement = connection.createStatement()) { + Exception rows = + assertThrows( + Exception.class, + () -> { + try (ResultSet result = statement.executeQuery("SELECT * FROM failing")) { + result.next(); + } + }); + assertThat(getStackTraceAsString(rows), containsString("bad change")); + + LiveResult.setViewer(result -> waitFor(result::isFinished)); + Exception viewed = + assertThrows(Exception.class, () -> statement.executeQuery("SELECT * FROM failing")); + assertThat(getStackTraceAsString(viewed), containsString("bad change")); + } + } + + private static Row row(long id, String name) { + return Row.withSchema(SCHEMA).addValues(id, name).build(); + } + + private static Connection connect() { + return JdbcDriver.connect( + new ReadOnlyTableProvider( + "test", + ImmutableMap.of( + "changes", new ChangesTable(Source.ONCE), + "sparse", new ChangesTable(Source.SPARSE), + "failing", new ChangesTable(Source.FAILING))), + PipelineOptionsFactory.create()); + } + + /** Takes up to {@code count} changes as "KIND:value:value", or fewer if they stop coming. */ + private static void collect(LiveResult result, List changes, int count) + throws IOException { + try { + while (changes.size() < count) { + LiveResult.Change change = result.poll(1, TimeUnit.MINUTES); + if (change == null) { + return; + } + changes.add( + change.getKind() + + ":" + + change.getRow().getValues().stream().map(String::valueOf).collect(joining(":"))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + + /** Waits for every pipeline that a live result started to stop. */ + private static void assertPipelinesStopped() { + assertTrue( + "A pipeline kept running", + waitFor( + () -> + Thread.getAllStackTraces().keySet().stream() + .noneMatch(thread -> thread.getName().equals("BeamSqlLiveResultWatcher")))); + } + + /** Waits up to a minute for {@code condition}, and returns whether it held. */ + private static boolean waitFor(BooleanSupplier condition) { + long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(1); + while (!condition.getAsBoolean()) { + if (System.nanoTime() > deadline) { + return false; + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return true; + } + + private enum Source { + /** Emits each change once, and then the pipeline ends. */ + ONCE, + /** Repeats the changes forever, one every 30 seconds. */ + SPARSE, + /** Fails on its first change. */ + FAILING + } + + /** An unbounded table whose rows carry change kinds, like a CDC source. */ + private static class ChangesTable extends SchemaBaseBeamTable { + private final Source source; + + ChangesTable(Source source) { + super(SCHEMA); + this.source = source; + } + + @Override + public PCollection buildIOReader(PBegin begin) { + PCollection indexes = + source == Source.SPARSE + ? begin.apply(GenerateSequence.from(0).withRate(1, Duration.standardSeconds(30))) + : begin.apply( + TestStream.create(VarLongCoder.of()) + .addElements(0L, 1L, 2L, 3L, 4L) + .advanceWatermarkToInfinity()); + return indexes.apply(ParDo.of(new ToChange(source == Source.FAILING))).setRowSchema(SCHEMA); + } + + @Override + public PCollection.IsBounded isBounded() { + return PCollection.IsBounded.UNBOUNDED; + } + + @Override + public BeamTableStatistics getTableStatistics(PipelineOptions options) { + return BeamTableStatistics.UNBOUNDED_UNKNOWN; + } + + @Override + public POutput buildIOWriter(PCollection input) { + throw new UnsupportedOperationException("read-only"); + } + } + + /** Turns an index into the change at that position, with its kind. */ + private static class ToChange extends DoFn { + private final boolean fail; + + ToChange(boolean fail) { + this.fail = fail; + } + + @ProcessElement + public void processElement(@Element Long index, OutputReceiver out) { + if (fail) { + throw new IllegalStateException("bad change " + index); + } + int i = (int) (index % ROWS.size()); + out.builder(ROWS.get(i)).setValueKind(KINDS.get(i)).output(); + } + } +} From 075e2a9d75c7441aaed92d7396751b7e224b20a7 Mon Sep 17 00:00:00 2001 From: ahmedabu98 Date: Mon, 16 Nov 2026 20:15:50 +0000 Subject: [PATCH 2/2] address bot comments --- .../sql/impl/rel/BeamEnumerableConverter.java | 17 ++++++----------- .../extensions/sql/impl/rel/LiveResult.java | 10 +++++++--- .../sql/impl/rel/LiveResultTest.java | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java index 6de4250f3515..40c404d00bc3 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java @@ -69,9 +69,8 @@ import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.PhysType; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.PhysTypeImpl; -import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.AbstractEnumerable2; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Enumerable; -import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Enumerator; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Linq4j; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.tree.Expression; @@ -232,21 +231,17 @@ private static Enumerable liveCollect( } /** The rows of a {@link LiveResult} as they arrive. Closing them cancels the pipeline. */ - private static class LiveRows extends AbstractEnumerable { + private static class LiveRows extends AbstractEnumerable2 { private final LiveResult result; LiveRows(LiveResult result) { this.result = result; } - @Override - public Enumerator enumerator() { - return Linq4j.iterableEnumerator(this); - } - /** - * JDBC reads the rows through this iterator. Unlike the default one, it doesn't wait for the - * next row before it returns the current one, which would hold back the newest row. + * JDBC reads the rows through this iterator, and {@link #enumerator()} wraps it. It doesn't + * wait for the next row before it returns the current one, which would hold back the newest + * row. */ @Override public Iterator iterator() { @@ -263,7 +258,7 @@ public boolean hasNext() { // No rows come after the pipeline ends, so then an empty poll means there are no more. boolean finished = result.isFinished(); try { - next = result.poll(100, TimeUnit.MILLISECONDS); + next = result.poll(finished ? 0 : 100, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java index 7735d83150e7..ed0284ae942e 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResult.java @@ -144,8 +144,12 @@ private void watch(PipelineResult pipelineResult) { @Nullable State end = null; try { end = pipelineResult.waitUntilFinish(); - } catch (RuntimeException e) { - failure = e; + if (end == State.FAILED) { + failure = new IllegalStateException("The pipeline failed"); + } + } catch (Throwable e) { + // The DirectRunner rethrows an Error, like NoClassDefFoundError, without wrapping it. + failure = e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); end = State.FAILED; } finally { state = end == null ? State.UNKNOWN : end; @@ -159,7 +163,7 @@ private void watch(PipelineResult pipelineResult) { void add(ValueKind kind, Row row) { Change change = new Change(kind, row); try { - while (!closed) { + while (!closed && !cancelFlag.get()) { if (changes.offerLast(change, 100, TimeUnit.MILLISECONDS)) { return; } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java index 845b4f075027..596f1cdaf1a8 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/LiveResultTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; import org.apache.beam.sdk.coders.VarLongCoder; +import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics; import org.apache.beam.sdk.extensions.sql.impl.JdbcDriver; import org.apache.beam.sdk.extensions.sql.meta.SchemaBaseBeamTable; @@ -57,6 +58,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Enumerator; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; @@ -128,6 +130,23 @@ public void rowsEndWithThePipeline() throws Exception { assertThat(rows, containsInAnyOrder("1:a", "2:b", "1:a", "1:aa", "2:b")); } + @Test + public void enumeratorReadsTheRows() throws Exception { + BeamRelNode node = + BeamSqlEnv.readOnly("test", ImmutableMap.of("changes", new ChangesTable(Source.ONCE))) + .parseQuery("SELECT id FROM changes"); + Enumerator rows = + BeamEnumerableConverter.toEnumerable( + PipelineOptionsFactory.create(), node, new AtomicBoolean()) + .enumerator(); + List ids = new ArrayList<>(); + while (rows.moveNext()) { + ids.add((Long) rows.current()); + } + rows.close(); + assertThat(ids, containsInAnyOrder(1L, 2L, 1L, 1L, 2L)); + } + @Test public void viewerSeesEachRowWithItsKind() throws Exception { List changes = new CopyOnWriteArrayList<>();