Skip to content
Snippets Groups Projects
Commit 4fbdc650 authored by Pavel Antonov's avatar Pavel Antonov :asterisk:
Browse files

fix(schema): Исправлена ошибка декодирования типа Number и Timestamp из JSON (json.Number) 

Issue: #3355
parents ab0aede8 bb6ba200
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
go.mod
+ 36
− 4
View file @ 4fbdc650
......@@ -14,13 +14,14 @@ require (
github.com/mitchellh/mapstructure v1.5.0
github.com/nats-io/nats.go v1.42.0
github.com/pkg/errors v0.9.1
github.com/redpanda-data/benthos/v4 v4.53.0
github.com/rs/xid v1.6.0
github.com/stretchr/testify v1.10.0
go.mongodb.org/mongo-driver v1.17.3
go.opentelemetry.io/otel v1.36.0
go.opentelemetry.io/otel/trace v1.36.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.38.0
golang.org/x/crypto v0.39.0
golang.org/x/image v0.27.0
golang.org/x/net v0.40.0
golang.org/x/oauth2 v0.30.0
......@@ -29,7 +30,38 @@ require (
gopkg.in/yaml.v3 v3.0.1
)
require go.opentelemetry.io/auto/sdk v1.1.0 // indirect
require (
cuelang.org/go v0.13.0 // indirect
github.com/Jeffail/gabs/v2 v2.7.0 // indirect
github.com/Jeffail/shutdown v1.0.0 // indirect
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gofrs/uuid/v5 v5.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
github.com/gorilla/handlers v1.5.2 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/matoous/go-nanoid/v2 v2.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/segmentio/ksuid v1.0.4 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/tilinna/z85 v1.0.0 // indirect
github.com/urfave/cli/v2 v2.27.6 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
)
require (
cloud.google.com/go/compute/metadata v0.7.0 // indirect
......@@ -56,8 +88,8 @@ require (
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.opentelemetry.io/otel/metric v1.36.0
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sync v0.15.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.25.0
golang.org/x/text v0.26.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
)
go.sum
+ 130
− 8
View file @ 4fbdc650
This diff is collapsed.
pkg/items/item_test.go
+ 124
− 1
View file @ 4fbdc650
......@@ -8,6 +8,7 @@ import (
"git.perx.ru/perxis/perxis-go/pkg/schema"
"git.perx.ru/perxis/perxis-go/pkg/schema/field"
"github.com/redpanda-data/benthos/v4/public/service"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
......@@ -82,7 +83,12 @@ func TestGetField(t *testing.T) {
}{
{"Simple", "a", field.String(), assert.NoError},
{"Incorrect field", "b", nil, assert.Error},
{"Object", "obj", field.Object("a", field.Number(field.NumberFormatFloat), "b", field.String()), assert.NoError},
{
"Object",
"obj",
field.Object("a", field.Number(field.NumberFormatFloat), "b", field.String()),
assert.NoError,
},
{"Object path", "obj.a", field.Number(field.NumberFormatFloat), assert.NoError},
{"Array", "arr", field.Array(field.Object("a", field.Time())), assert.NoError},
{"Array path", "arr.a", field.Time(), assert.NoError},
......@@ -746,3 +752,120 @@ func TestFromMap(t *testing.T) {
})
}
}
func TestFromMap_WithAsStructuredMut(t *testing.T) {
// Схема для данных
sch := schema.New(
"number", field.Number(field.NumberFormatInt),
"obj", field.Object(
"arr", field.Array(field.String()),
"bool", field.Bool(),
"date", field.Time(),
"stamp", field.Timestamp(),
),
"text", field.String(),
)
// Ожидаемые результаты
expectedDate, _ := time.Parse(time.RFC3339, "2025-05-31T21:00:00Z")
tests := []struct {
name string
data interface{}
}{
{
name: "JSON",
data: []byte(`{
"collectionId":"test_collection_1",
"createdAt":"2025-06-18T09:52:11Z",
"createdBy":"c361pni1l3r7ve137ieg",
"createdRevAt":"2025-06-18T09:52:11Z",
"data":{
"number":1,
"obj":{
"arr":["1","2","3"],
"bool":true,
"date":"2025-05-31T21:00:00Z",
"stamp":1000
},
"text":"massive"
},
"envId":"d0s98dajvknu95q7ia40",
"id":"d198oiqjvknp270h3tf0",
"revId":"d198oiqjvknp270h3tfg",
"spaceId":"d0s98dajvknu95q7ia3g",
"state":1,
"updatedAt":"2025-06-18T09:52:11Z",
"updatedBy":"c361pni1l3r7ve137ieg"
}`),
},
{
name: "Map",
data: map[string]any{
"collectionId": "test_collection_1",
"createdAt": "2025-06-18T09:52:11Z",
"createdBy": "c361pni1l3r7ve137ieg",
"createdRevAt": "2025-06-18T09:52:11Z",
"data": map[string]any{
"number": 1,
"obj": map[string]any{
"arr": []any{"1", "2", "3"},
"bool": true,
"date": "2025-05-31T21:00:00Z",
"stamp": 1000,
},
"text": "massive",
},
"envId": "d0s98dajvknu95q7ia40",
"id": "d198oiqjvknp270h3tf0",
"revId": "d198oiqjvknp270h3tfg",
"spaceId": "d0s98dajvknu95q7ia3g",
"state": 1,
"updatedAt": "2025-06-18T09:52:11Z",
"updatedBy": "c361pni1l3r7ve137ieg",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var v any
var err error
// Обрабатываем разные типы входных данных
if bytes, ok := tc.data.([]byte); ok {
msg := service.NewMessage(bytes)
v, err = msg.AsStructuredMut()
require.NoError(t, err, "AsStructuredMut не должен возвращать ошибку для %s", tc.name)
} else {
msg := service.NewMessage(nil)
msg.SetStructuredMut(tc.data)
v, err = msg.AsStructuredMut()
require.NoError(t, err, "AsStructuredMut не должен возвращать ошибку для %s", tc.name)
}
data, _ := v.(map[string]any)
item, err := FromMap(data, sch)
require.NoError(t, err, "FromMap должен корректно обработать данные из %s", tc.name)
// Проверяем, что число корректно преобразовано (может быть int64 после AsStructuredMut)
number := item.Data["number"]
if num, ok := number.(int64); ok {
assert.Equal(t, int64(1), num)
} else {
assert.Equal(t, 1, number)
}
assert.Equal(t, "massive", item.Data["text"])
// Проверяем объект
obj, ok := item.Data["obj"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, true, obj["bool"])
assert.Equal(t, []interface{}{"1", "2", "3"}, obj["arr"])
// Проверяем дату
assert.Equal(t, expectedDate, obj["date"])
})
}
}
pkg/schema/field/number.go
+ 13
− 0
View file @ 4fbdc650
......@@ -2,6 +2,7 @@ package field
import (
"context"
"encoding/json"
"math"
"reflect"
......@@ -58,6 +59,18 @@ func toNumberReflect(i interface{}) (interface{}, error) {
func ToNumber(i interface{}) (interface{}, error) {
switch v := i.(type) {
case json.Number:
// Сначала пробуем парсить как целое число
intVal, err := v.Int64()
if err == nil {
return intVal, nil
}
// Если не получилось, пробуем как дробное число
floatVal, err := v.Float64()
if err == nil {
return floatVal, nil
}
return nil, err
case int64:
return v, nil
case int:
......
pkg/schema/field/number_test.go
+ 8
− 4
View file @ 4fbdc650
......@@ -2,6 +2,7 @@ package field
import (
"context"
"encoding/json"
"math"
"reflect"
"testing"
......@@ -39,6 +40,9 @@ func TestNumberField_Decode(t *testing.T) {
{"Correct", Number("float"), 2.2, 2.2, false}, // #16
{"Correct", Number("float"), 2, 2.0, false}, // #17
{"Correct", Number("float"), int64(2), 2.0, false}, // #18
{"Correct", Number("float"), json.Number("2"), 2.0, false}, // #19
{"Correct", Number("int"), json.Number("2"), int64(2), false}, // #20
{"Correct", Number("float"), json.Number("2.2"), 2.2, false}, // #21
{"Wrong data", Number("int"), "", nil, true}, // #0
{"Wrong data", Number("int"), []byte(""), nil, true}, // #1
......
pkg/schema/field/timestamp.go
+ 13
− 1
View file @ 4fbdc650
......@@ -2,6 +2,7 @@ package field
import (
"context"
"encoding/json"
"fmt"
"time"
)
......@@ -69,11 +70,22 @@ func (TimestampType) Decode(_ context.Context, _ *Field, v interface{}) (interfa
return duration.Nanoseconds(), nil
}
t, err := time.Parse(time.TimeOnly, i)
if err == nil {
return t.AddDate(1, 0, 0).Sub(zeroTime).Nanoseconds(), nil
}
return nil, err
case json.Number:
// Пробуем парсить json.Number как int64
intVal, err := i.Int64()
if err == nil {
return intVal, nil
}
// Если не получилось, пробуем как float64 и приводим к int64
floatVal, err := i.Float64()
if err == nil {
return int64(floatVal), nil
}
return nil, err
default:
return toTimestamp(i)
}
......
pkg/schema/field/timestamp_test.go
+ 35
− 4
View file @ 4fbdc650
......@@ -2,6 +2,7 @@ package field
import (
"context"
"encoding/json"
"reflect"
"testing"
......@@ -31,11 +32,41 @@ func TestTimestamp_Decode(t *testing.T) {
{"Correct", Timestamp(), uint64(2), int64(2), false, ""}, // #11
{"Correct", Timestamp(), nil, nil, false, ""}, // #12
{"Correct", Timestamp(), 2.0, int64(2), false, ""}, // #13
{"Correct", Timestamp(), json.Number("24"), int64(24), false, ""}, // #14
{"Correct", Timestamp(), json.Number("2.4"), int64(2), false, ""}, // #15
{"Wrong data", Timestamp(), "", nil, true, "decode error: parsing time \"\" as \"15:04:05\": cannot parse \"\" as \"15\""}, // #0
{"Wrong data", Timestamp(), []byte(""), nil, true, "decode error: unsupported value type: \"[]uint8\""}, // #1
{"Wrong data", Timestamp(), "13:10", nil, true, "decode error: parsing time \"13:10\" as \"15:04:05\": cannot parse \"\" as \":\""}, // #2
{"Wrong data", Timestamp(), "24:00:00", nil, true, "decode error: parsing time \"24:00:00\": hour out of range"}, // #3
{
"Wrong data",
Timestamp(),
"",
nil,
true,
"decode error: parsing time \"\" as \"15:04:05\": cannot parse \"\" as \"15\"",
}, // #0
{
"Wrong data",
Timestamp(),
[]byte(""),
nil,
true,
"decode error: unsupported value type: \"[]uint8\"",
}, // #1
{
"Wrong data",
Timestamp(),
"13:10",
nil,
true,
"decode error: parsing time \"13:10\" as \"15:04:05\": cannot parse \"\" as \":\"",
}, // #2
{
"Wrong data",
Timestamp(),
"24:00:00",
nil,
true,
"decode error: parsing time \"24:00:00\": hour out of range",
}, // #3
}
for _, tt := range tests {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment