Skip to content

Commit 455a05a

Browse files
refactor: move header validation into streamLines to avoid opening file twice
1 parent cb454b7 commit 455a05a

1 file changed

Lines changed: 31 additions & 39 deletions

File tree

wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/sources/fs/JavaCSVTableSource.java

Lines changed: 31 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@ public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> eval
116116
}
117117

118118
private Stream<Record> createStream(final String actualInputPath) {
119-
validateHeaderLine(actualInputPath);
120119
return streamLines(actualInputPath).map(this::parseLine);
121120
}
122121

@@ -128,11 +127,9 @@ private Record parseLine(final String s) {
128127
if (tokens.length != fieldTypes.size())
129128
throw new IllegalStateException(
130129
String.format(
131-
"Column count mismatch in CSV file '%s': expected %d columns but found %d "
132-
+ "(separator '%s'). Line: '%s'. "
133-
+ "Ensure the header uses 'name:type' format with commas "
134-
+ "and data rows use '%s' as delimiter.",
135-
sourcePath, fieldTypes.size(), tokens.length, separator, s, separator));
130+
"CSV file '%s': data row has %d columns but expected %d "
131+
+ "(separator '%s'). Line: '%s'.",
132+
sourcePath, tokens.length, fieldTypes.size(), separator, s));
136133
// now tokens.length == fieldtypes.size
137134

138135
final Object[] objects = new Object[tokens.length];
@@ -174,55 +171,50 @@ private static Stream<String> streamLines(final String path) {
174171
() -> new IllegalStateException(String.format("No file system found for %s", path)));
175172
try {
176173
final Iterator<String> lineIterator = createLineIterator(fileSystem, path);
177-
lineIterator.next(); // skip header row
174+
if (!lineIterator.hasNext()) {
175+
throw new IllegalStateException(String.format("CSV file '%s' is empty. Expected a header row (e.g., 'id:int,name:string').",path));
176+
}
177+
String headerLine = lineIterator.next(); // read and skip header line
178+
validateHeaderLine(path, headerLine);
178179
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(lineIterator, 0), false);
179180
} catch (final IOException e) {
180181
throw new WayangException(String.format("%s failed to read %s.", FileUtils.class, path), e);
181182
}
182-
183183
}
184184

185185
/**
186186
* Validates the CSV header for Calcite compatibility.
187-
* Checks that the header is present, uses comma separators (not the data
188-
* delimiter), and each column follows the 'name:type' format
189-
* (e.g., 'id:int,name:string,email:string'). Note that Calcite hardcodes
190-
* commas for header parsing, while data rows use Wayang's configurable
191-
* separator (default ';').
187+
* Checks that each column follows the 'name:type' format
188+
* (e.g., 'id:int,name:string,email:string') and that commas
189+
* are used as the header separator.
192190
*
193-
* @param path the filesystem path to the CSV file
191+
* @param path the filesystem path to the CSV file
192+
* @param headerLine the first line of the CSV file
194193
*/
195-
private void validateHeaderLine(final String path) {
196-
final FileSystem fileSystem = FileSystems.getFileSystem(path).orElseThrow(
197-
() -> new IllegalStateException(String.format("No file system found for %s", path)));
198-
try {
199-
final Iterator<String> lineIterator = createLineIterator(fileSystem, path);
194+
private static void validateHeaderLine(final String path, final String headerLine) {
195+
final String[] headerColumns = headerLine.split(","); // split header row into columns
200196

201-
if (!lineIterator.hasNext()) {
202-
throw new IllegalStateException(String.format("CSV file '%s' is empty. Expected a header row (e.g., 'id:int,name:string').",path));
197+
int colonCount = 0;
198+
for (int i = 0; i < headerLine.length(); i++) {
199+
if (headerLine.charAt(i) == ':') {
200+
colonCount++;
203201
}
204-
205-
final String headerLine = lineIterator.next(); // read header row
206-
final String[] headerColumns = headerLine.split(","); // split header row into columns
202+
}
207203

208-
if (headerColumns.length == 1 && headerLine.contains(String.valueOf(separator))) {
204+
for (final String column : headerColumns) {
205+
if (!column.trim().contains(":")) {
209206
throw new IllegalStateException(String.format(
210-
"CSV file '%s': header uses '%s' as separator, but Calcite requires commas. "
211-
+ "Header: '%s'. "
212-
+ "Expected format: %s.",
213-
sourcePath, separator, headerLine, headerLine.replace(String.valueOf(separator), ",")));
207+
"CSV file '%s': header column '%s' missing required type. "
208+
+ "Expected 'name:type' format (e.g., 'id:int'). Header: '%s'.",
209+
path, column.trim(), headerLine));
214210
}
211+
}
215212

216-
for (final String column : headerColumns) {
217-
if (!column.trim().contains(":")) {
218-
throw new IllegalStateException(String.format(
219-
"CSV file '%s': header column '%s' missing required type. "
220-
+ "Expected 'name:type' format (e.g., 'id:int'). Full header: '%s'.",
221-
sourcePath, column.trim(), headerLine));
222-
}
223-
}
224-
} catch (final IOException e) {
225-
throw new WayangException(String.format("%s failed to read %s.", FileUtils.class, path), e);
213+
if (headerColumns.length != colonCount) {
214+
throw new IllegalStateException(String.format(
215+
"CSV file '%s': column count mismatch. Expected %d comma-separated 'name:type' columns "
216+
+ "but found %d. Header: '%s'.",
217+
path, colonCount, headerColumns.length, headerLine));
226218
}
227219
}
228220

0 commit comments

Comments
 (0)