Skip to content

Commit 592cf7f

Browse files
committed
Fix broken integration test.
Add nci-develop to list of pull-request build triggers.
1 parent ec96fb8 commit 592cf7f

7 files changed

Lines changed: 193 additions & 3 deletions

File tree

.github/workflows/master-build.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919
branches: [master]
2020
pull_request:
2121
types: [opened, synchronize, reopened]
22-
branches: [master]
22+
branches: [master, nci-develop]
2323
jobs:
2424
javadoc:
2525
strategy:
@@ -104,6 +104,8 @@ jobs:
104104
- 'src/testresources/**'
105105
plugins:
106106
- 'src/plugin/**'
107+
indexer_plugins:
108+
- 'src/plugin/indexer-*/**'
107109
buildconf:
108110
- 'build.xml'
109111
- 'ivy/ivy.xml'
@@ -120,6 +122,10 @@ jobs:
120122
- name: test plugins
121123
if: ${{ steps.filter.outputs.plugins == 'true' && steps.filter.outputs.core == 'false' && steps.filter.outputs.buildconf == 'false' }}
122124
run: ant clean test-plugins -buildfile build.xml
125+
# run indexer integration tests when indexer plugin files change (Docker required, ubuntu-latest only)
126+
- name: test indexer integration
127+
if: ${{ steps.filter.outputs.indexer_plugins == 'true' && matrix.os == 'ubuntu-latest' }}
128+
run: ant clean test-indexer-integration -buildfile build.xml
123129
- name: Check for test results
124130
id: check_tests
125131
if: always() && matrix.os == 'ubuntu-latest'
@@ -139,4 +145,4 @@ jobs:
139145
path: |
140146
./build/test/TEST-*.xml
141147
./build/**/test/TEST-*.xml
142-
retention-days: 1
148+
retention-days: 1

build.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,10 @@
534534
<ant dir="src/plugin" target="test-single" inheritAll="false"/>
535535
</target>
536536

537+
<target name="test-indexer-integration" depends="resolve-test, compile, compile-core-test, job" description="--> run indexer plugin integration tests (Testcontainers)">
538+
<ant dir="src/plugin" target="test-indexer-integration" inheritAll="false"/>
539+
</target>
540+
537541
<target name="nightly" depends="test, tar-src, zip-src" description="--> run the nightly target build">
538542
</target>
539543

ivy/ivy.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,10 @@
134134
<dependency org="org.mockito" name="mockito-core" rev="5.18.0" conf="test->default"/>
135135
<dependency org="org.mockito" name="mockito-junit-jupiter" rev="5.18.0" conf="test->default"/>
136136

137+
<!-- Testcontainers for indexer plugin integration tests -->
138+
<dependency org="org.testcontainers" name="testcontainers" rev="2.0.3" conf="test->default"/>
139+
<dependency org="org.testcontainers" name="junit-jupiter" rev="1.21.4" conf="test->default"/>
140+
137141
<!-- Jetty used to serve test pages for unit tests, but is also provided as dependency of Hadoop -->
138142
<dependency org="org.eclipse.jetty" name="jetty-server" rev="12.1.5" conf="test->default">
139143
<exclude org="ch.qos.reload4j" module="*" />

src/plugin/build-plugin.xml

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
<include name="hamcrest*.jar" />
8585
<include name="junit*.jar" />
8686
<include name="opentest4j*.jar" />
87+
<include name="testcontainers*.jar" />
8788
</fileset>
8889
<path refid="classpath"/>
8990
</path>
@@ -189,7 +190,7 @@
189190
<!-- ================================================================== -->
190191
<!-- Compile test code -->
191192
<!-- ================================================================== -->
192-
<target name="compile-test" depends="compile, deps-test-compile" if="test.available">
193+
<target name="compile-test" depends="resolve-test, compile, deps-test-compile" if="test.available">
193194
<javac
194195
encoding="${build.encoding}"
195196
srcdir="${src.test}"
@@ -242,6 +243,33 @@
242243
<fail if="tests.failed">Tests failed!</fail>
243244
</target>
244245

246+
<!-- ================================================================== -->
247+
<!-- Run indexer plugin integration tests (Testcontainers) -->
248+
<!-- ================================================================== -->
249+
<target name="test-indexer-integration" depends="compile-test, deploy" if="test.available">
250+
<echo message="Running indexer integration tests for plugin: ${name}"/>
251+
<junitlauncher printSummary="true" haltOnFailure="false" failureProperty="integration.tests.failed">
252+
<classpath refid="test.classpath"/>
253+
<testclasses outputDir="${build.test}">
254+
<listener type="legacy-plain" sendSysOut="true" sendSysErr="true"/>
255+
<listener type="legacy-xml" sendSysOut="true" sendSysErr="true"/>
256+
<fork forkMode="perTestClass">
257+
<jvmarg value="-Xmx2000m"/>
258+
<sysproperty key="test.data" value="${build.test}/data"/>
259+
<sysproperty key="test.input" value="${root}/data"/>
260+
<sysproperty key="testcontainers.reuse.enable" value="true"/>
261+
<sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
262+
</fork>
263+
<fileset dir="${build.test}">
264+
<include name="**/IT*.class"/>
265+
<include name="**/*IT.class"/>
266+
<include name="**/*IntegrationTest.class"/>
267+
</fileset>
268+
</testclasses>
269+
</junitlauncher>
270+
<fail if="integration.tests.failed">Indexer integration tests failed!</fail>
271+
</target>
272+
245273
<!-- target: resolve ================================================= -->
246274
<target name="resolve-default" depends="clean-lib" description="resolve and retrieve dependencies with ivy">
247275
<ivy:resolve file="ivy.xml" conf="default" log="download-only"/>

src/plugin/build.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,13 @@
172172
<ant dir="${plugin}" target="test"/>
173173
</target>
174174

175+
<!-- ====================================================== -->
176+
<!-- Indexer plugin integration tests (Testcontainers) -->
177+
<!-- ====================================================== -->
178+
<target name="test-indexer-integration">
179+
<ant dir="indexer-elastic" target="test-indexer-integration"/>
180+
</target>
181+
175182
<!-- ====================================================== -->
176183
<!-- Clean all of the plugins. -->
177184
<!-- ====================================================== -->
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.nutch.indexer;
18+
19+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
20+
21+
import org.junit.jupiter.api.AfterEach;
22+
import org.junit.jupiter.api.BeforeEach;
23+
import org.junit.jupiter.api.Test;
24+
import org.testcontainers.junit.jupiter.Testcontainers;
25+
26+
/**
27+
* Abstract base for IndexWriter integration tests. Provides common test logic
28+
* for write/commit and delete operations.
29+
*/
30+
@Testcontainers(disabledWithoutDocker = true)
31+
public abstract class AbstractIndexWriterIT
32+
implements IndexWriterIntegrationTest {
33+
34+
@BeforeEach
35+
void setUp() throws Exception {
36+
setUpIndexWriter();
37+
}
38+
39+
@AfterEach
40+
void tearDown() throws Exception {
41+
tearDownIndexWriter();
42+
}
43+
44+
@Test
45+
void testWriteAndCommitDocument() throws Exception {
46+
NutchDocument doc = createTestDocument("test-doc-1", "Test Document",
47+
"This is a test document for integration testing.");
48+
assertDoesNotThrow(() -> getIndexWriter().write(doc));
49+
assertDoesNotThrow(() -> getIndexWriter().commit());
50+
tearDownIndexWriter();
51+
verifyDocumentWritten("test-doc-1", "Test Document");
52+
}
53+
54+
@Test
55+
void testDeleteDocument() throws Exception {
56+
if (!supportsDelete()) {
57+
return;
58+
}
59+
String docId = "test-doc-to-delete";
60+
NutchDocument doc = createTestDocument(docId, "Document to Delete", "");
61+
62+
IndexWriter writer = getIndexWriter();
63+
writer.write(doc);
64+
writer.commit();
65+
66+
IndexWriter deleteWriter = prepareWriterForDeleteTest();
67+
if (deleteWriter == null) {
68+
deleteWriter = writer;
69+
}
70+
final IndexWriter writerForDelete = deleteWriter;
71+
assertDoesNotThrow(() -> writerForDelete.delete(docId));
72+
assertDoesNotThrow(() -> writerForDelete.commit());
73+
if (deleteWriter != writer) {
74+
try {
75+
deleteWriter.close();
76+
} catch (Exception e) {
77+
// Ignore
78+
}
79+
}
80+
}
81+
82+
/** Create a NutchDocument with id, title, and content. */
83+
protected NutchDocument createTestDocument(String id, String title,
84+
String content) {
85+
NutchDocument doc = new NutchDocument();
86+
doc.add("id", id);
87+
doc.add("title", title);
88+
doc.add("content", content);
89+
return doc;
90+
}
91+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.nutch.indexer;
18+
19+
/**
20+
* Contract for IndexWriter integration tests. Implementations run against
21+
* real backends via Testcontainers.
22+
*/
23+
public interface IndexWriterIntegrationTest {
24+
25+
/** Open the index writer before tests. */
26+
void setUpIndexWriter() throws Exception;
27+
28+
/** Close the index writer after tests. */
29+
void tearDownIndexWriter() throws Exception;
30+
31+
/** The IndexWriter under test. */
32+
IndexWriter getIndexWriter();
33+
34+
/** Whether this writer supports document deletion. */
35+
boolean supportsDelete();
36+
37+
/** Optional verification that a document was indexed. */
38+
default void verifyDocumentWritten(String docId, String expectedTitle)
39+
throws Exception {
40+
// no-op
41+
}
42+
43+
/**
44+
* Optional writer configured for delete operations. Default returns null to
45+
* use {@link #getIndexWriter()}.
46+
*/
47+
default IndexWriter prepareWriterForDeleteTest() throws Exception {
48+
return null;
49+
}
50+
}

0 commit comments

Comments
 (0)