Skip to content

Commit ac9b1e0

Browse files
itinychengitinycheng
committed
[fix]: CI build failures #173 
1 parent 85523c9 commit ac9b1e0

6 files changed

Lines changed: 49 additions & 35 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,13 @@ jobs:
3939
uses: docker/setup-buildx-action@v3.10.0
4040
- name: Build with Maven
4141
run: ./mvnw verify
42+
- name: Upload test logs on failure
43+
if: failure()
44+
uses: actions/upload-artifact@v4
45+
with:
46+
name: test-logs-java-${{ matrix.java-version }}
47+
path: |
48+
**/target/surefire-reports/
49+
**/target/failsafe-reports/
50+
**/hs_err_*.log
51+
retention-days: 7

‎flink-connector-clickhouse-e2e-test/pom.xml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,21 @@ limitations under the License.
3636
<dependencies>
3737
<dependency>
3838
<groupId>org.testcontainers</groupId>
39-
<artifactId>clickhouse</artifactId>
39+
<artifactId>testcontainers-clickhouse</artifactId>
4040
<version>${testcontainer.version}</version>
41+
<scope>test</scope>
4142
</dependency>
4243
<dependency>
4344
<groupId>org.apache.flink</groupId>
4445
<artifactId>flink-connector-test-utils</artifactId>
4546
<version>${flink.version}</version>
4647
<scope>test</scope>
48+
<exclusions>
49+
<exclusion>
50+
<artifactId>commons-io</artifactId>
51+
<groupId>commons-io</groupId>
52+
</exclusion>
53+
</exclusions>
4754
</dependency>
4855
<dependency>
4956
<groupId>org.apache.flink</groupId>

flink-connector-clickhouse-e2e-test/src/test/java/org/apache/flink/connector/clickhouse/ClickhouseE2ECase.java renamed to flink-connector-clickhouse-e2e-test/src/test/java/org/apache/flink/connector/clickhouse/ClickhouseE2ETestCase.java

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
import org.junit.After;
44
import org.junit.Test;
5-
import org.slf4j.Logger;
6-
import org.slf4j.LoggerFactory;
75

86
import java.time.Duration;
97
import java.time.temporal.ChronoUnit;
@@ -12,9 +10,7 @@
1210
import java.util.List;
1311

1412
/** End-to-end test for Clickhouse. */
15-
public class ClickhouseE2ECase extends FlinkContainerTestEnvironment {
16-
17-
private static final Logger logger = LoggerFactory.getLogger(ClickhouseE2ECase.class);
13+
public class ClickhouseE2ETestCase extends FlinkContainerEnvironment {
1814

1915
ClickhouseProxy proxy;
2016

@@ -27,6 +23,8 @@ public void testSink() throws Exception {
2723
CLICKHOUSE_CONTAINER.getJdbcUrl(),
2824
CLICKHOUSE_CONTAINER.getUsername(),
2925
CLICKHOUSE_CONTAINER.getPassword());
26+
Thread.sleep(5000);
27+
3028
proxy.execute(
3129
"create table test (id Int32, name String, float32_column Float32, date_column Date,datetime_column DateTime, array_column Array(Int32)) engine = Memory");
3230
proxy.execute(
@@ -52,7 +50,9 @@ public void testSink() throws Exception {
5250
+ "',\n"
5351
+ " 'table-name' = 'test',\n"
5452
+ " 'username'='test_username',\n"
55-
+ " 'password'='test_password'\n"
53+
+ " 'password'='test_password',\n"
54+
+ " 'properties.compress' = 'false',\n"
55+
+ " 'properties.decompress' = 'false'\n"
5656
+ ");");
5757
sqlLines.add(
5858
"create table test (id int, name varchar,float32_column FLOAT,\n"
@@ -63,15 +63,17 @@ public void testSink() throws Exception {
6363
+ "',\n"
6464
+ " 'table-name' = 'test_insert',\n"
6565
+ " 'username'='test_username',\n"
66-
+ " 'password'='test_password'\n"
66+
+ " 'password'='test_password',\n"
67+
+ " 'properties.compress' = 'false',\n"
68+
+ " 'properties.decompress' = 'false'\n"
6769
+ ");");
6870
sqlLines.add("insert into test select * from clickhouse_test;");
6971

7072
submitSQLJob(
7173
sqlLines,
7274
SQL_CONNECTOR_CLICKHOUSE_JAR,
7375
CLICKHOUSE_JDBC_JAR,
74-
HTTPCORE_JAR,
76+
HTTP_CORE_JAR,
7577
HTTPCLIENT_JAR,
7678
HTTPCLIENT_H2_JAR);
7779
waitUntilJobRunning(Duration.of(1, ChronoUnit.MINUTES));
@@ -86,7 +88,7 @@ public void testSink() throws Exception {
8688
expectedResult,
8789
"test_insert",
8890
Arrays.asList("id", "name", "float32_column", "datetime_column", "array_column"),
89-
60000);
91+
600_000);
9092
}
9193

9294
@After

‎flink-connector-clickhouse-e2e-test/src/test/java/org/apache/flink/connector/clickhouse/ClickhouseProxy.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
import com.clickhouse.jdbc.ClickHouseDriver;
66
import com.clickhouse.jdbc.ClickHouseStatement;
77
import org.junit.Assert;
8-
import org.slf4j.Logger;
9-
import org.slf4j.LoggerFactory;
108

119
import java.sql.Array;
1210
import java.sql.ResultSet;
@@ -20,10 +18,10 @@
2018

2119
/** A proxy for Clickhouse to execute SQLs and check results. */
2220
public class ClickhouseProxy {
21+
2322
private final String jdbcUrl;
2423
private final String username;
2524
private final String password;
26-
private static final Logger logger = LoggerFactory.getLogger(ClickhouseProxy.class);
2725
ClickHouseDriver driver;
2826
ClickHouseStatement statement;
2927
ClickHouseConnection connection;
@@ -41,13 +39,15 @@ public void connect() {
4139
Properties properties = new Properties();
4240
properties.put("username", username);
4341
properties.put("password", password);
42+
properties.put("compress", "false");
43+
properties.put("decompress", "false");
4444
ClickHouseDataSource clickHouseDataSource =
4545
new ClickHouseDataSource(jdbcUrl, properties);
4646
connection = clickHouseDataSource.getConnection(username, password);
4747
statement = connection.createStatement();
4848
}
4949
} catch (Exception e) {
50-
logger.error("Failed to connect to clickhouse", e);
50+
throw new RuntimeException("Cannot establish ClickHouse connection", e);
5151
}
5252
}
5353

flink-connector-clickhouse-e2e-test/src/test/java/org/apache/flink/connector/clickhouse/FlinkContainerTestEnvironment.java renamed to flink-connector-clickhouse-e2e-test/src/test/java/org/apache/flink/connector/clickhouse/FlinkContainerEnvironment.java

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import org.apache.flink.client.program.rest.RestClusterClient;
77
import org.apache.flink.configuration.Configuration;
88
import org.apache.flink.configuration.RestOptions;
9-
import org.apache.flink.connector.testframe.container.TestcontainersSettings;
109
import org.apache.flink.runtime.client.JobStatusMessage;
1110
import org.apache.flink.table.api.ValidationException;
1211
import org.apache.flink.test.resources.ResourceTestUtils;
@@ -22,6 +21,7 @@
2221
import org.testcontainers.containers.GenericContainer;
2322
import org.testcontainers.containers.Network;
2423
import org.testcontainers.containers.output.Slf4jLogConsumer;
24+
import org.testcontainers.containers.wait.strategy.Wait;
2525
import org.testcontainers.lifecycle.Startables;
2626
import org.testcontainers.utility.DockerImageName;
2727
import org.testcontainers.utility.MountableFile;
@@ -41,10 +41,9 @@
4141
import static org.assertj.core.util.Preconditions.checkState;
4242

4343
/** Test environment running job on Flink containers. */
44-
public class FlinkContainerTestEnvironment {
44+
public class FlinkContainerEnvironment {
4545

46-
private static final Logger logger =
47-
LoggerFactory.getLogger(FlinkContainerTestEnvironment.class);
46+
private static final Logger logger = LoggerFactory.getLogger(FlinkContainerEnvironment.class);
4847
public static final Network NETWORK = Network.newNetwork();
4948

5049
static final ClickHouseContainer CLICKHOUSE_CONTAINER =
@@ -54,20 +53,20 @@ public class FlinkContainerTestEnvironment {
5453
.withExposedPorts(8123, 9000)
5554
.withUsername("test_username")
5655
.withPassword("test_password")
56+
.withStartupTimeout(Duration.ofMinutes(10))
57+
.withStartupAttempts(3)
58+
.waitingFor(
59+
Wait.forHttp("/ping")
60+
.forPort(8123)
61+
.forStatusCode(200)
62+
.withStartupTimeout(Duration.ofMinutes(10)))
5763
.withLogConsumer(new Slf4jLogConsumer(logger));
5864

59-
private static final TestcontainersSettings TESTCONTAINERS_SETTINGS =
60-
TestcontainersSettings.builder()
61-
.logger(logger)
62-
.network(NETWORK)
63-
.dependsOn(CLICKHOUSE_CONTAINER)
64-
.build();
65-
6665
public static final Path SQL_CONNECTOR_CLICKHOUSE_JAR =
6766
ResourceTestUtils.getResource("flink-connector-clickhouse-1.0.0-SNAPSHOT.jar");
6867
public static final Path CLICKHOUSE_JDBC_JAR =
6968
ResourceTestUtils.getResource("clickhouse-jdbc-0.6.4.jar");
70-
public static final Path HTTPCORE_JAR = ResourceTestUtils.getResource("httpcore5-5.2.jar");
69+
public static final Path HTTP_CORE_JAR = ResourceTestUtils.getResource("httpcore5-5.2.jar");
7170
public static final Path HTTPCLIENT_JAR =
7271
ResourceTestUtils.getResource("httpclient5-5.2.1.jar");
7372
public static final Path HTTPCLIENT_H2_JAR =
@@ -81,6 +80,7 @@ public class FlinkContainerTestEnvironment {
8180
@Before
8281
public void setUp() throws Exception {
8382
CLICKHOUSE_CONTAINER.start();
83+
Thread.sleep(30000);
8484

8585
String properties =
8686
String.join(
@@ -90,7 +90,7 @@ public void setUp() throws Exception {
9090
"heartbeat.timeout: 60000",
9191
"parallelism.default: 1"));
9292
jobManager =
93-
new GenericContainer<>(new DockerImageName("flink:2.1-scala_2.12"))
93+
new GenericContainer<>(DockerImageName.parse("flink:2.1-scala_2.12"))
9494
.withCommand("jobmanager")
9595
.withNetwork(NETWORK)
9696
.withExtraHost("host.docker.internal", "host-gateway")
@@ -101,7 +101,7 @@ public void setUp() throws Exception {
101101
.withEnv("FLINK_PROPERTIES", properties)
102102
.withLogConsumer(new Slf4jLogConsumer(logger));
103103
taskManager =
104-
new GenericContainer<>(new DockerImageName("flink:2.1-scala_2.12"))
104+
new GenericContainer<>(DockerImageName.parse("flink:2.1-scala_2.12"))
105105
.withCommand("taskmanager")
106106
.withExtraHost("host.docker.internal", "host-gateway")
107107
.withNetwork(NETWORK)
@@ -184,11 +184,6 @@ private String copyAndGetContainerPath(GenericContainer<?> container, String fil
184184
return containerPath;
185185
}
186186

187-
private static List<String> readSqlFile(final String resourceName) throws Exception {
188-
return Files.readAllLines(
189-
Paths.get(ClickhouseE2ECase.class.getResource("/" + resourceName).toURI()));
190-
}
191-
192187
public void waitUntilJobRunning(Duration timeout) {
193188
RestClusterClient<?> clusterClient = getRestClusterClient();
194189
Deadline deadline = Deadline.fromNow(timeout);

‎pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ limitations under the License.
6161
<clickhouse-jdbc.version>0.6.4</clickhouse-jdbc.version>
6262
<scala.binary.version>2.12</scala.binary.version>
6363
<flink.version>2.1.0</flink.version>
64-
<commons-lang3.version>3.13.0</commons-lang3.version>
65-
<testcontainer.version>1.19.8</testcontainer.version>
64+
<commons-lang3.version>3.18.0</commons-lang3.version>
65+
<testcontainer.version>2.0.2</testcontainer.version>
6666
<httpclient5.version>5.2.1</httpclient5.version>
6767
<httpcore5.version>5.2</httpcore5.version>
6868
<shade.base>org.apache.flink.shaded.clickhouse</shade.base>

0 commit comments

Comments
 (0)