Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
Expand Down Expand Up @@ -368,6 +369,33 @@ public void streamingChangesAreUnbounded() throws SQLException {
+ " 'triggering_frequency_seconds'"));
}

/** The read never ends, so the rows have to arrive while it runs. */
@Test(timeout = 120_000)
public void streamingChangesArriveWhileThePipelineRuns() throws Exception {
seedOrders();
List<String> rows = new ArrayList<>();
try (Connection connection =
JdbcDriver.connect(catalogManager, PipelineOptionsFactory.create());
Statement statement = connection.createStatement();
ResultSet result =
statement.executeQuery(
"SELECT * FROM CHANGES(orders) WITH ('streaming' = true, 'starting_strategy' ="
+ " 'earliest', 'poll_interval_seconds' = 1, "
+ CHANGE_TYPE
+ ")")) {
while (rows.size() < ALL_CHANGES.size() && result.next()) {
rows.add(
format(
"%s:%s:%s:%s",
result.getObject(1),
result.getObject(2),
result.getObject(3),
result.getObject(4)));
}
}
assertThat(rows, containsInAnyOrder(ALL_CHANGES.toArray()));
}

@Test
public void eachReferenceReadsItsOwnChanges() {
seedOrders();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Queue;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
Expand Down Expand Up @@ -61,11 +63,13 @@
import org.apache.beam.sdk.values.PCollection.IsBounded;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.ValueKind;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.DataContext;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.EnumerableRel;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.PhysType;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.adapter.enumerable.PhysTypeImpl;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.AbstractEnumerable2;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Enumerable;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.Linq4j;
import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.linq4j.tree.BlockBuilder;
Expand Down Expand Up @@ -180,10 +184,128 @@ static Enumerable<Object> toEnumerable(
return count(options, node, cancelFlag);
} else if (isLimitQuery(node)) {
return limitCollect(options, node, cancelFlag);
} else if (node.isBounded() == IsBounded.UNBOUNDED) {
return liveCollect(options, node, cancelFlag);
}
return Linq4j.asEnumerable(rowToAvaticaAndUnboxValues(collectRows(options, node, cancelFlag)));
}

/**
* Runs the pipeline of an unbounded query and hands over its rows while it runs.
*
* <p>If the shell set a {@link LiveResult.Viewer}, it shows the rows and the query returns none.
* Otherwise the query returns the rows as they arrive, until the pipeline ends, the statement is
* cancelled, or the result set is closed.
*/
private static Enumerable<Object> liveCollect(
PipelineOptions options, BeamRelNode node, AtomicBoolean cancelFlag) {
checkArgument(
options
.getRunner()
.getCanonicalName()
.equals("org.apache.beam.runners.direct.DirectRunner"),
"SELECT without INSERT is only supported in DirectRunner in SQL Shell.");
Pipeline pipeline = Pipeline.create(options);
PCollection<Row> rows = BeamSqlRelUtils.toPCollection(pipeline, node);
rows.apply(ParDo.of(new LiveCollector()));
LiveResult.Viewer viewer = LiveResult.getViewer();
LiveResult result = LiveResult.start(pipeline, rows.getSchema(), cancelFlag);
if (viewer == null) {
return new LiveRows(result);
}

RuntimeException failure;
try {
viewer.show(result);
} catch (IOException e) {
throw new RuntimeException("Failed to show the rows of the query", e);
} finally {
// Read the failure first: errors after closing come from cancelling the pipeline.
failure = result.getFailure();
result.close();
}
if (failure != null) {
throw failure;
}
return Linq4j.emptyEnumerable();
}

/** The rows of a {@link LiveResult} as they arrive. Closing them cancels the pipeline. */
private static class LiveRows extends AbstractEnumerable2<Object> {
private final LiveResult result;

LiveRows(LiveResult result) {
this.result = result;
}

/**
* JDBC reads the rows through this iterator, and {@link #enumerator()} wraps it. It doesn't
* wait for the next row before it returns the current one, which would hold back the newest
* row.
*/
@Override
public Iterator<Object> iterator() {
return new LiveIterator();
}

private class LiveIterator implements Iterator<Object>, AutoCloseable {
private LiveResult.@Nullable Change next;

@Override
public boolean hasNext() {
// A cancelled statement ends the rows.
while (next == null && !result.isCancelRequested()) {
// No rows come after the pipeline ends, so then an empty poll means there are no more.
boolean finished = result.isFinished();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟢 Low: When finished is already true and the buffer is empty, result.poll(100, TimeUnit.MILLISECONDS) still blocks for 100ms even though no further rows can be produced. Using a timeout of 0 when finished is true avoids adding a 100ms delay at the end of every bounded-completion or cancelled query.

Suggested change
boolean finished = result.isFinished();
next = result.poll(finished ? 0 : 100, TimeUnit.MILLISECONDS);

try {
next = result.poll(finished ? 0 : 100, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
if (next == null && finished) {
RuntimeException failure = result.getFailure();
close();
if (failure != null) {
throw failure;
}
return false;
}
}
if (next == null) {
close();
}
return next != null;
}

@Override
public Object next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
Row row = next.getRow();
next = null;
return rowToAvaticaAndUnbox(row);
}

@Override
public void close() {
result.close();
}
}
}

/** Adds each row and its change kind to the {@link LiveResult} of its pipeline. */
private static class LiveCollector extends DoFn<Row, Void> {
@ProcessElement
public void processElement(@Element Row row, ValueKind kind, PipelineOptions options) {
LiveResult result = LiveResult.forOptionsId(options.getOptionsId());
if (result != null) {
result.add(kind, row);
}
}
}

private static PipelineResult limitRun(
PipelineOptions options,
BeamRelNode node,
Expand Down Expand Up @@ -360,21 +482,22 @@ public void processElement(ProcessContext context) {

private static List<Object> rowToAvaticaAndUnboxValues(Queue<Row> values) {
return values.stream()
.map(
row -> {
Object[] objects = rowToAvatica(row);
if (objects.length == 1) {
// if objects.length == 1, that means input Row contains only 1 column/element,
// then an Object instead of Object[] should be returned because of
// CalciteResultSet's behaviour that tries to convert one column row to an Object.
return objects[0];
} else {
return objects;
}
})
.map(BeamEnumerableConverter::rowToAvaticaAndUnbox)
.collect(Collectors.toList());
}

private static Object rowToAvaticaAndUnbox(Row row) {
Object[] objects = rowToAvatica(row);
if (objects.length == 1) {
// if objects.length == 1, that means input Row contains only 1 column/element,
// then an Object instead of Object[] should be returned because of
// CalciteResultSet's behaviour that tries to convert one column row to an Object.
return objects[0];
} else {
return objects;
}
}

private static Object[] rowToAvatica(Row row) {
Schema schema = row.getSchema();
Object[] convertedColumns = new Object[schema.getFields().size()];
Expand Down
Loading
Loading