[Appender] Cast float64 to numeric destination type (#198)

* initial commit

* remove appender from vector functions

* more appender removal

* new vector.go file

* support missing timestamp types

* implicit types init commit

* fix typo

* formatting

* better type casting

* small fixes

* enabling JSON parsing

* renamed vector.go to appender_vector.go

---------

Co-authored-by: Marc Boeker <37560+marcboeker@users.noreply.github.com>

appender_test.go

@@ -763,39 +763,67 @@ func TestAppenderUint8SliceTinyInt(t *testing.T) {
 	cleanupAppender(t, c, con, a)
 }
 
+var jsonInputs = [][]byte{
+	[]byte(`{"c1": 42, "l1": [1, 2, 3], "s1": {"a": 101, "b": ["hello", "world"]}, "l2": [{"a": [{"a": [4.2, 7.9]}]}]}`),
+	[]byte(`{"c1": null, "l1": [null, 2, null], "s1": {"a": null, "b": ["hello", null]}, "l2": [{"a": [{"a": [null, 7.9]}]}]}`),
+	[]byte(`{"c1": null, "l1": null, "s1": {"a": null, "b": null}, "l2": [{"a": [{"a": null}]}]}`),
+	[]byte(`{"c1": null, "l1": null, "s1": null, "l2": [{"a": [null, {"a": null}]}]}`),
+	[]byte(`{"c1": null, "l1": null, "s1": null, "l2": [{"a": null}]}`),
+	[]byte(`{"c1": null, "l1": null, "s1": null, "l2": [null, null]}`),
+	[]byte(`{"c1": null, "l1": null, "s1": null, "l2": null}`),
+}
+
+var jsonResults = [][]string{
+	{"42", "[1 2 3]", "map[a:101 b:[hello world]]", "[map[a:[map[a:[4.2 7.9]]]]]"},
+	{"<nil>", "[<nil> 2 <nil>]", "map[a:<nil> b:[hello <nil>]]", "[map[a:[map[a:[<nil> 7.9]]]]]"},
+	{"<nil>", "<nil>", "map[a:<nil> b:<nil>]", "[map[a:[map[a:<nil>]]]]"},
+	{"<nil>", "<nil>", "<nil>", "[map[a:[<nil> map[a:<nil>]]]]"},
+	{"<nil>", "<nil>", "<nil>", "[map[a:<nil>]]"},
+	{"<nil>", "<nil>", "<nil>", "[<nil> <nil>]"},
+	{"<nil>", "<nil>", "<nil>", "<nil>"},
+}
+
 func TestAppenderWithJSON(t *testing.T) {
 	c, con, a := prepareAppender(t, `
 		CREATE TABLE test (
-		    id DOUBLE,
-			l DOUBLE[],
-			s STRUCT(a DOUBLE, b VARCHAR)
+		    c1 UBIGINT,
+			l1 TINYINT[],
+			s1 STRUCT(a INTEGER, b VARCHAR[]),
+		    l2 STRUCT(a STRUCT(a FLOAT[])[])[]              
 	  	)`)
 
-	jsonBytes := []byte(`{"id": 42, "l":[1, 2, 3], "s":{"a":101, "b":"hello"}}`)
-	var jsonData map[string]interface{}
-	err := json.Unmarshal(jsonBytes, &jsonData)
-	require.NoError(t, err)
+	for _, jsonInput := range jsonInputs {
+		var jsonData map[string]interface{}
+		err := json.Unmarshal(jsonInput, &jsonData)
+		require.NoError(t, err)
+		require.NoError(t, a.AppendRow(jsonData["c1"], jsonData["l1"], jsonData["s1"], jsonData["l2"]))
+	}
 
-	require.NoError(t, a.AppendRow(jsonData["id"], jsonData["l"], jsonData["s"]))
 	require.NoError(t, a.Flush())
 
 	// Verify results.
 	res, err := sql.OpenDB(c).QueryContext(context.Background(), `SELECT * FROM test`)
 	require.NoError(t, err)
 
+	i := 0
 	for res.Next() {
 		var (
-			id uint64
-			l  interface{}
-			s  interface{}
+			c1 interface{}
+			l1 interface{}
+			s1 interface{}
+			l2 interface{}
 		)
-		err := res.Scan(&id, &l, &s)
+		err = res.Scan(&c1, &l1, &s1, &l2)
 		require.NoError(t, err)
-		require.Equal(t, uint64(42), id)
-		require.Equal(t, "[1 2 3]", fmt.Sprint(l))
-		require.Equal(t, "map[a:101 b:hello]", fmt.Sprint(s))
+		require.Equal(t, jsonResults[i][0], fmt.Sprint(c1))
+		require.Equal(t, jsonResults[i][1], fmt.Sprint(l1))
+		require.Equal(t, jsonResults[i][2], fmt.Sprint(s1))
+		require.Equal(t, jsonResults[i][3], fmt.Sprint(l2))
+		i++
 	}
 
-	require.NoError(t, res.Close())
+	require.Equal(t, len(jsonInputs), i)
+
+  require.NoError(t, res.Close())
 	cleanupAppender(t, c, con, a)
 }

appender_vector.go

@@ -37,25 +37,25 @@ func (vec *vector) tryCast(val any) (any, error) {
 
 	switch vec.duckdbType {
 	case C.DUCKDB_TYPE_UTINYINT:
-		return tryPrimitiveCast[uint8](val, reflect.Uint8.String())
+		return tryNumericCast[uint8](val, reflect.Uint8.String())
 	case C.DUCKDB_TYPE_TINYINT:
-		return tryPrimitiveCast[int8](val, reflect.Int8.String())
+		return tryNumericCast[int8](val, reflect.Int8.String())
 	case C.DUCKDB_TYPE_USMALLINT:
-		return tryPrimitiveCast[uint16](val, reflect.Uint16.String())
+		return tryNumericCast[uint16](val, reflect.Uint16.String())
 	case C.DUCKDB_TYPE_SMALLINT:
-		return tryPrimitiveCast[int16](val, reflect.Int16.String())
+		return tryNumericCast[int16](val, reflect.Int16.String())
 	case C.DUCKDB_TYPE_UINTEGER:
-		return tryPrimitiveCast[uint32](val, reflect.Uint32.String())
+		return tryNumericCast[uint32](val, reflect.Uint32.String())
 	case C.DUCKDB_TYPE_INTEGER:
-		return tryPrimitiveCast[int32](val, reflect.Int32.String())
+		return tryNumericCast[int32](val, reflect.Int32.String())
 	case C.DUCKDB_TYPE_UBIGINT:
-		return tryPrimitiveCast[uint64](val, reflect.Uint64.String())
+		return tryNumericCast[uint64](val, reflect.Uint64.String())
 	case C.DUCKDB_TYPE_BIGINT:
-		return tryPrimitiveCast[int64](val, reflect.Int64.String())
+		return tryNumericCast[int64](val, reflect.Int64.String())
 	case C.DUCKDB_TYPE_FLOAT:
-		return tryPrimitiveCast[float32](val, reflect.Float32.String())
+		return tryNumericCast[float32](val, reflect.Float32.String())
 	case C.DUCKDB_TYPE_DOUBLE:
-		return tryPrimitiveCast[float64](val, reflect.Float64.String())
+		return tryNumericCast[float64](val, reflect.Float64.String())
 	case C.DUCKDB_TYPE_BOOLEAN:
 		return tryPrimitiveCast[bool](val, reflect.Bool.String())
 	case C.DUCKDB_TYPE_VARCHAR:
@@ -94,6 +94,22 @@ func tryPrimitiveCast[T any](val any, expected string) (any, error) {
 	return nil, castError(goType.String(), expected)
 }
 
+func tryNumericCast[T numericType](val any, expected string) (any, error) {
+	if v, ok := val.(T); ok {
+		return v, nil
+	}
+
+	// JSON unmarshalling uses float64 for numbers.
+	// We might want to add more implicit casts here.
+	switch v := val.(type) {
+	case float64:
+		return convertNumericType[float64, T](v), nil
+	}
+
+	goType := reflect.TypeOf(val)
+	return nil, castError(goType.String(), expected)
+}
+
 func (vec *vector) tryCastList(val any) ([]any, error) {
 	goType := reflect.TypeOf(val)
 	if goType.Kind() != reflect.Slice {

types.go

@@ -28,6 +28,14 @@ var unsupportedAppenderTypeMap = map[C.duckdb_type]string{
 	C.DUCKDB_TYPE_TIME_TZ:  "TIME_TZ",
 }
 
+type numericType interface {
+	int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64
+}
+
+func convertNumericType[srcT numericType, destT numericType](val srcT) destT {
+	return destT(val)
+}
+
 type UUID [16]byte
 
 func (u *UUID) Scan(v any) error {