Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,19 @@

### New Features

- **[client-v2, jdbc-v2]** Added support for the `MultiPoint` geo data type (ClickHouse `26.8+`). Previously the type was
unknown to the client, so reading or writing a `MultiPoint` column failed with `Unknown data type: MultiPoint`, and a
`MultiPoint` value inside a `Geometry` column failed with an out-of-range variant discriminator. `MultiPoint` is
`Array(Point)` on the wire, exactly like `Ring` and `LineString`, so it is read and written as `double[][]` through
generic records, binary readers, POJO binding, and SQL parameter formatting, and is read from `Dynamic` columns. In the
JDBC driver (`jdbc-v2`) `MultiPoint` maps
to `java.sql.Types.ARRAY`, is returned as `double[][]` from `getObject` and as a `java.sql.Array` from `getArray`, and is
reported by `ResultSetMetaData` and `DatabaseMetaData`. ClickHouse `26.8` also adds `MultiPoint` to the `Geometry`
variant; the server appends it after the existing six variants instead of ordering it by type name, so the client now
keeps that order and decodes a `MultiPoint` held in a `Geometry` column. Because `MultiPoint` shares its Java
representation (`double[][]`) with `Ring` and `LineString`, it is not selectable through the shape-based `Geometry`
write path — a 2D value keeps resolving to `Ring` as before, and writing `MultiPoint` requires a concrete `MultiPoint`
column. (https://github.com/ClickHouse/clickhouse-java/issues/3048)
- **[client-v2]** Added an OpenTelemetry implementation of the observability SPI.
`Client.Builder.setSpanRecorder(new OpenTelemetrySpanRecorder(openTelemetry))`
reports every client operation and every transport request as an OpenTelemetry `CLIENT` span: an operation span is
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,9 @@ private static ClickHouseColumn update(ClickHouseColumn column) {
case LineString:
column.template = ClickHouseGeoRingValue.ofEmpty();
break;
case MultiPoint:
column.template = ClickHouseGeoRingValue.ofEmpty();
break;
case Polygon:
column.template = ClickHouseGeoPolygonValue.ofEmpty();
break;
Expand Down Expand Up @@ -363,8 +366,27 @@ private static ClickHouseColumn update(ClickHouseColumn column) {
}

private static ClickHouseColumn createGeometryVariantColumn() {
ClickHouseColumn column = ClickHouseColumn.of("v",
// The six geometry variants that exist since CH 25.11. Variant nested columns are ordered by
// type name, which reproduces the discriminators the server assigns to them.
ClickHouseColumn base = ClickHouseColumn.of("v",
"Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)");

// CH 26.8 added MultiPoint to Geometry without renumbering the existing variants: the server
// appends it after MultiPolygon instead of inserting it in type-name order, so it is appended
// here as well rather than relying on the generic Variant ordering.
List<ClickHouseColumn> nestedColumns = new ArrayList<>(base.nested);
nestedColumns.add(ClickHouseColumn.of("v." + ClickHouseDataType.MultiPoint.name(),
ClickHouseDataType.MultiPoint.name()));

ClickHouseColumn column = new ClickHouseColumn(ClickHouseDataType.Variant, "v",
"Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon, MultiPoint)",
false, false, null, nestedColumns);

// MultiPoint shares its Java representation (double[][]) with Ring and LineString, so it is
// deliberately left out of both write-side mappings: a Java value written to a Geometry column
// keeps resolving to the same variant it resolved to before. MultiPoint is read-only through
// Geometry and has to be written through a concrete MultiPoint column.
column.classToVariantOrdNumMap = base.classToVariantOrdNumMap;
Map<Integer, Integer> map = new HashMap<>();
map.put(1, getVariantOrdNum(column.nested, ClickHouseDataType.Point));
map.put(2, getVariantOrdNum(column.nested, ClickHouseDataType.Ring));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ public enum ClickHouseDataType implements SQLType {
Ring(Object.class, false, true, true, 0, 0, 0, 0, 0, true), // same as Array(Point)
LineString( Object.class, false, true, true, 0, 0, 0, 0, 0, true), // same as Array(Point)
MultiLineString(Object.class, false, true, true, 0, 0, 0, 0, 0, true), // same as Array(Ring)
MultiPoint(Object.class, false, true, true, 0, 0, 0, 0, 0, true), // same as Array(Point)
Geometry(Object.class, false, true, true, 0, 0, 0, 0, 0, true), // same as Variant(Point, ...)
JSON(Object.class, false, false, false, 0, 0, 0, 0, 0, true, 0x30),
@Deprecated // (since = "CH 25.11")
Expand Down Expand Up @@ -216,6 +217,7 @@ static Map<ClickHouseDataType, Set<Class<?>>> dataTypeClassMap() {
map.put(Point, setOf(double[].class, ClickHouseGeoPointValue.class));
map.put(Ring, setOf(double[][].class, ClickHouseGeoRingValue.class));
map.put(LineString, setOf(double[][].class, ClickHouseGeoRingValue.class));
map.put(MultiPoint, setOf(double[][].class, ClickHouseGeoRingValue.class));
map.put(Polygon, setOf(double[][][].class, ClickHouseGeoPolygonValue.class));
map.put(MultiLineString, setOf(double[][][].class, ClickHouseGeoPolygonValue.class));
map.put(MultiPolygon, setOf(double[][][][].class, ClickHouseGeoMultiPolygonValue.class));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,45 @@ public void testGeometryVariantOrdNumUsesArrayDimensions() {
Assert.assertEquals(geometry.getGeometryVariantOrdNum(new Object()), -1);
}

@Test(groups = { "unit" })
public void testGeometryVariantOrder() {
ClickHouseColumn geometry = ClickHouseColumn.of("v", "Geometry");

// The discriminators the server assigns to the Geometry variants. MultiPoint was added in
// 26.8 after the other six and keeps the last position instead of being ordered by name.
List<ClickHouseDataType> expected = Arrays.asList(
ClickHouseDataType.LineString,
ClickHouseDataType.MultiLineString,
ClickHouseDataType.MultiPolygon,
ClickHouseDataType.Point,
ClickHouseDataType.Polygon,
ClickHouseDataType.Ring,
ClickHouseDataType.MultiPoint);

List<ClickHouseDataType> actual = new LinkedList<>();
geometry.getNestedColumns().forEach(c -> actual.add(c.getDataType()));
Assert.assertEquals(actual, expected);

// MultiPoint shares double[][] with Ring and LineString, so a Java value written to a
// Geometry column must keep resolving to the variant it resolved to before.
Assert.assertEquals(geometry.getGeometryVariantOrdNum(2),
getVariantOrdNum(geometry, ClickHouseDataType.Ring));
Assert.assertEquals(geometry.getGeometryVariantOrdNum(
ClickHouseGeoRingValue.of(new double[][] { { 1D, 2D }, { 3D, 4D } })),
getVariantOrdNum(geometry, ClickHouseDataType.Ring));
}

@Test(groups = { "unit" })
public void testMultiPointColumn() {
ClickHouseColumn column = ClickHouseColumn.of("m", "MultiPoint");

Assert.assertEquals(column.getDataType(), ClickHouseDataType.MultiPoint);
Assert.assertFalse(column.isNullable());
Assert.assertTrue(column.newValue(null) instanceof ClickHouseGeoRingValue);
Assert.assertEquals(ClickHouseColumn.of("m", "Array(MultiPoint)").getArrayBaseColumn().getDataType(),
ClickHouseDataType.MultiPoint);
}

private static int getVariantOrdNum(ClickHouseColumn column, ClickHouseDataType dataType) {
for (int i = 0; i < column.getNestedColumns().size(); i++) {
if (column.getNestedColumns().get(i).getDataType() == dataType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,8 @@ private <T> T readValue(ClickHouseColumn column, Class<?> typeHint, boolean stri
return (T) readGeoRing();
case LineString:
return (T) readGeoRing();
case MultiPoint:
return (T) readGeoRing();
Comment on lines +247 to +248

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you — the technical description of the Native layout is correct, and I verified it. But the behavior is pre-existing for every geo type and is not introduced by this PR, so I am not changing it here.

What this PR changes in this file: one line, case MultiPoint: return (T) readGeoRing();. NativeFormatReader is untouched by the diff.

Why MultiPoint is not special: NativeFormatReader.readBlock() enters its columnar Array branch only when column.isArray() is true. Point, Ring, LineString, MultiLineString, Polygon and MultiPolygon are all distinct ClickHouseDataType values, so isArray() is already false for each of them and each already falls through to binaryStreamReader.readValue(column). MultiPoint inherits exactly the same routing as its siblings.

Measured on main's code path (ClickHouse 26.7.3.19), using RingSELECT r FROM ... FORMAT Native for the rows [(1,2),(3,4)] and [(5,6)]:

010201720452696e67                                     1 col, 2 rows, name 'r', type Ring
02000000 00000000  03000000 00000000                   cumulative offsets 2, 3
000000000000f03f 0000000000000840 0000000000001440     x column: 1, 3, 5
0000000000000040 0000000000001040 0000000000001840     y column: 2, 4, 6

Fed to NativeFormatReader:

row 0 = [[3.13151306251402E-294, 0.0], [1.7765824089018436E-307, 1.7835357647096786E-307]]
row 1 = desynchronized (NoSuchColumnException)

Expected [[1.0,2.0],[3.0,4.0]] then [[5.0,6.0]]. So Ring — with no code from this PR involved — already shows the failure mode you describe.

Why not fix it in this PR: either remedy (a Native columnar decode path for geo columns, or an explicit ClientException for geo in Native as readBlock already does for undecodable QBit shapes) changes the observable behavior of Point/Ring/LineString/MultiLineString/Polygon/MultiPolygon. That is a behavior change on existing types and needs its own PR and maintainer buy-in; bundling it into an additive MultiPoint feature would mix two concerns. Rejecting MultiPoint alone in Native would also make it inconsistent with the siblings that share its wire format.

The MultiPoint read/write tests in this PR run through RowBinaryWithNamesAndTypes, which is correct for all geo types today.

I have logged the pre-existing Native geo defect for a separate fix, including the reproduction above. It is distinct from #2955 / #2956 and from the Array element sub-structure work, because geo columns never enter the Array branch at all.

Leaving this thread open for maintainer visibility.

case JSON: // experimental https://clickhouse.com/docs/en/sql-reference/data-types/newjson
if (jsonAsString) {
return (T) readString(input);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo
break;
case Ring:
case LineString:
case MultiPoint:
value = value instanceof ClickHouseGeoRingValue ? ((ClickHouseGeoRingValue)value).getValue() : value;
serializeArrayData(stream, value, GEO_RING_ARRAY);
break;
Expand Down Expand Up @@ -388,6 +389,7 @@ public static void writeDynamicTypeTag(OutputStream stream, ClickHouseColumn typ
case Point:
case LineString:
case MultiLineString:
case MultiPoint:
case Polygon:
case Ring:
case MultiPolygon:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ public String convertToString(Object value, ClickHouseColumn column) {
case Point:
case Ring:
case LineString:
case MultiPoint:
case Polygon:
case MultiLineString:
case MultiPolygon:
Expand Down Expand Up @@ -487,6 +488,7 @@ private boolean isGeoType(ClickHouseDataType dataType) {
case Point:
case Ring:
case LineString:
case MultiPoint:
case Polygon:
case MultiLineString:
case MultiPolygon:
Expand All @@ -502,7 +504,8 @@ private boolean isGeoTypeForDimensions(ClickHouseDataType dataType, int dimensio
case 1:
return dataType == ClickHouseDataType.Point;
case 2:
return dataType == ClickHouseDataType.Ring || dataType == ClickHouseDataType.LineString;
return dataType == ClickHouseDataType.Ring || dataType == ClickHouseDataType.LineString
|| dataType == ClickHouseDataType.MultiPoint;
case 3:
return dataType == ClickHouseDataType.Polygon || dataType == ClickHouseDataType.MultiLineString;
case 4:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.clickhouse.data.ClickHouseColumn;
import com.clickhouse.data.ClickHouseDataType;
import com.clickhouse.data.value.ClickHouseGeoPolygonValue;
import com.clickhouse.data.value.ClickHouseGeoRingValue;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
Expand Down Expand Up @@ -127,9 +128,27 @@ public void testDynamicWithGeoCustomTypeRoundTrip() throws Exception {
public void testDynamicTypeTagUsesCustomEncodingForGeoTypes() throws Exception {
assertCustomGeoTypeTag("LineString");
assertCustomGeoTypeTag("MultiLineString");
assertCustomGeoTypeTag("MultiPoint");
assertCustomGeoTypeTag("Geometry");
}

@Test
public void testMultiPointRoundTrip() throws Exception {
ClickHouseColumn multiPoint = ClickHouseColumn.of("v", "MultiPoint");
double[][] points = new double[][] {{1D, 2D}, {3D, 4D}, {5D, 6D}};

ByteArrayOutputStream out = new ByteArrayOutputStream();
SerializerUtils.serializeData(out, ClickHouseGeoRingValue.of(points), multiPoint);

// Identical wire representation to Ring: a var-uint point count followed by two Float64 per point.
ByteArrayOutputStream ring = new ByteArrayOutputStream();
SerializerUtils.serializeData(ring, ClickHouseGeoRingValue.of(points), ClickHouseColumn.of("v", "Ring"));
Assert.assertEquals(out.toByteArray(), ring.toByteArray());

Object value = newReader(out.toByteArray()).readValue(multiPoint);
Assert.assertTrue(Arrays.deepEquals((double[][]) value, points));
}

@Test
public void testGeometrySerializationRejectsUnsupportedValue() {
Assert.assertThrows(IllegalArgumentException.class,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,24 @@ public void testVariantOrDynamicGeoToString() {
"[[1.0, 2.0, 3.0]]");
}

@Test
public void testVariantTwoDimensionalGeoToString() {
DataTypeConverter converter = new DataTypeConverter();
double[][] value = new double[][] {{1D, 2D}, {3D, 4D}};

// every two dimensional geo type is written as a point sequence
assertEquals(converter.convertToString(value, ClickHouseColumn.of("field", "Variant(String, Ring)")),
"[(1.0,2.0),(3.0,4.0)]");
assertEquals(converter.convertToString(value, ClickHouseColumn.of("field", "Variant(String, LineString)")),
"[(1.0,2.0),(3.0,4.0)]");
assertEquals(converter.convertToString(value, ClickHouseColumn.of("field", "Variant(String, MultiPoint)")),
"[(1.0,2.0),(3.0,4.0)]");

// no variant matches the two dimensional shape, thus the value keeps the plain array form
assertEquals(converter.convertToString(value, ClickHouseColumn.of("field", "Variant(String, Polygon)")),
"[[1.0, 2.0], [3.0, 4.0]]");
}

@DataProvider(name = "queryParameters")
public static Object[][] queryParameters() {
return new Object[][] {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@
case Nullable: // virtual type
case LowCardinality: // virtual type
case LineString: // same as Ring
case MultiPoint: // same as Ring
case MultiLineString: // same as MultiPolygon
case Time:
case Time64:
Expand Down Expand Up @@ -1173,6 +1174,7 @@
case LowCardinality: // virtual type
case Enum: // virtual type
case LineString: // same as Ring
case MultiPoint: // same as Ring
case MultiLineString: // same as MultiPolygon
case Time:
case Time64:
Expand Down Expand Up @@ -2008,6 +2010,82 @@
}
}

private static final String MULTI_POINT_UNSUPPORTED_VERSIONS = "(,26.7]";

@Data
@AllArgsConstructor
public static class DTOForMultiPointTests {
private int rowId;
private double[][] geom;
private double marker;
}

@Test(groups = {"integration"})
public void testMultiPoint() throws Exception {
if (isVersionMatch(MULTI_POINT_UNSUPPORTED_VERSIONS)) {
return;
}

final String table = "test_multi_point";
final double[][] expected = new double[][] {{1D, 2D}, {3D, 4D}, {5D, 6D}};

client.execute("DROP TABLE IF EXISTS " + table).get().close();
client.execute(tableDefinition(table, "rowId Int32", "geom MultiPoint", "marker Float64")).get().close();
client.register(DTOForMultiPointTests.class, client.getTableSchema(table));

client.insert(table, Collections.singletonList(new DTOForMultiPointTests(0, expected, 42D))).get().close();
client.execute("INSERT INTO " + table + " VALUES (1, readWKTMultiPoint('MULTIPOINT(1 2, 3 4, 5 6)'), 42)")
.get().close();

List<GenericRecord> records = client.queryAll("SELECT * FROM " + table + " ORDER BY rowId");
Assert.assertEquals(records.size(), 2);
for (GenericRecord record : records) {
Assert.assertTrue(Arrays.deepEquals((double[][]) record.getObject("geom"), expected));
Assert.assertTrue(Arrays.deepEquals(record.getGeoRing("geom").getValue(), expected));
Assert.assertEquals(record.getDouble("marker"), 42D);
}

try (QueryResponse response = client.query("SELECT * FROM " + table + " ORDER BY rowId").get()) {
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response);
int rows = 0;
while (reader.next() != null) {
Assert.assertTrue(Arrays.deepEquals((double[][]) reader.readValue("geom"), expected));
Assert.assertEquals(reader.getString("geom"), "[(1.0,2.0),(3.0,4.0),(5.0,6.0)]");
Assert.assertEquals(reader.getDouble("marker"), 42D);
rows++;
}
Assert.assertEquals(rows, 2);
}
}

@Test(groups = {"integration"})
public void testGeometryWithMultiPoint() throws Exception {
if (isVersionMatch(MULTI_POINT_UNSUPPORTED_VERSIONS)) {
return;
}

final String table = "test_geometry_multi_point";
final double[][] points = new double[][] {{1D, 2D}, {3D, 4D}, {5D, 6D}};
final double[][] ring = new double[][] {{1D, 2D}, {3D, 4D}, {1D, 2D}};

client.execute("DROP TABLE IF EXISTS " + table).get().close();
client.execute(tableDefinition(table, "rowId Int32", "geom Geometry", "marker Float64"),
(CommandSettings) new CommandSettings().serverSetting("allow_suspicious_variant_types", "1"))
.get().close();
client.execute("INSERT INTO " + table + " VALUES "
+ "(0, readWKTMultiPoint('MULTIPOINT(1 2, 3 4, 5 6)'), 42), "
+ "(1, CAST([(1, 2), (3, 4), (1, 2)] AS Ring), 42)").get().close();

List<GenericRecord> records = client.queryAll("SELECT * FROM " + table + " ORDER BY rowId");
Assert.assertEquals(records.size(), 2);
// A MultiPoint value stored in a Geometry column decodes to the same double[][] shape as a
// Ring value, which keeps decoding unchanged.
Assert.assertTrue(Arrays.deepEquals((double[][]) records.get(0).getObject("geom"), points));
Assert.assertTrue(Arrays.deepEquals((double[][]) records.get(1).getObject("geom"), ring));
Assert.assertEquals(records.get(0).getDouble("marker"), 42D);
Assert.assertEquals(records.get(1).getDouble("marker"), 42D);
}

@Test(groups = {"integration"})
public void testDates() throws Exception {
LocalDate date = LocalDate.of(2024, 1, 15);
Expand Down Expand Up @@ -2494,7 +2572,7 @@
List<GenericRecord> records = client.queryAll("SELECT * FROM " + table + " ORDER BY id");
Assert.assertEquals(records.size(), 2);

for (GenericRecord record : records) {

Check warning on line 2575 in client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Rename this variable to not match a restricted identifier.

See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ_6XJ5NdbvXhE_aw97x&open=AZ_6XJ5NdbvXhE_aw97x&pullRequest=3050
@SuppressWarnings("unchecked")
Map<String, Object> data = (Map<String, Object>) record.getObject("data");
Assert.assertNotNull(data, "JSON column should not be null");
Expand Down
Loading
Loading