ZMQ server: Encode NaN, Inf, -Inf as null in JSON (#1020)

Author: visr

Manifest.toml

@@ -7,16 +7,14 @@ authors = ["Deltares and contributors"]
 projects = ["test"]
 
 [deps]
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
-StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 Wflow = "d48b7d99-76e7-47ae-b1d5-ff0c1cf9a818"
 ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1"
 
 [compat]
-JSON3 = "1.14"
+JSON = "1"
 Logging = "1"
-StructTypes = "1.10"
 Wflow = "1"
 ZMQ = "1.2"
 julia = "1.10"

server/Project.toml

@@ -1,7 +1,6 @@
 module WflowServer
 using ZMQ: ZMQ
-using JSON3: JSON3
-using StructTypes: StructTypes
+using JSON: JSON
 using Wflow: Wflow
 
 include("bmi_service.jl")

server/src/WflowServer.jl

@@ -49,18 +49,29 @@ function shutdown(s::ZMQ.Socket, ctx::ZMQ.Context)
     return ZMQ.close(ctx)
 end
 
+"""
+    serialize_response(ret)
+
+Serialize the return value `ret` of a Wflow BMI function to a JSON string. JSON has no
+representation for the non-finite floating point values `NaN`, `Inf` and `-Inf`, so these
+are encoded as `null`.
+"""
+function serialize_response(ret)
+    return JSON.json(ret; allownan = true, nan = "null", inf = "null", ninf = "null")
+end
+
 "Error response ZMQ server"
 function response(err::AbstractString, s::ZMQ.Socket)
     @info "Send error response"
     resp = Dict{String, String}("status" => "ERROR", "error" => err)
-    return ZMQ.send(s, JSON3.write(resp))
+    return ZMQ.send(s, JSON.json(resp))
 end
 
 "Status response ZMQ server"
 function response(s::ZMQ.Socket)
     @info "Send status response"
     resp = Dict{String, String}("status" => "OK")
-    return ZMQ.send(s, JSON3.write(resp))
+    return ZMQ.send(s, JSON.json(resp))
 end
 
 "Validate JSON request against mapped Struct"
@@ -89,7 +100,7 @@ function wflow_bmi(s::ZMQ.Socket, handler::ModelHandler, f)
             response(s)
         else
             @info "Send response including output from Wflow function `$(f.fn)`"
-            ZMQ.send(s, JSON3.write(ret; allow_inf = true))
+            ZMQ.send(s, serialize_response(ret))
         end
     catch e
         @error "Wflow function `$(f.fn)` failed" exception = (e, catch_backtrace())
@@ -151,13 +162,13 @@ function start(port::Int)
         while true
             # Wait for next request from client
             req = ZMQ.recv(socket)
-            json = JSON3.read(req; allow_inf = true)
+            json = JSON.parse(req; allownan = true)
             @info "Received request to run function `$(json.fn)`..."
 
             if haskey(MAP_STRUCTS, json.fn)
                 v = valid_request(json)
                 if isnothing(v)
-                    f = StructTypes.constructfrom(MAP_STRUCTS[json.fn], json)
+                    f = JSON.parse(req, MAP_STRUCTS[json.fn]; allownan = true)
                     wflow_bmi(socket, handler, f)
                 else
                     err = (

server/src/server.jl

@@ -1,10 +1,9 @@
 [deps]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
-StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 Wflow = "d48b7d99-76e7-47ae-b1d5-ff0c1cf9a818"

server/test/Project.toml

@@ -1,5 +1,5 @@
 @testitem "Client server Wflow ZMQ Server" begin
-    import ZMQ, JSON3, StructTypes, Wflow
+    import ZMQ, JSON, Wflow
     using Statistics: mean
     using Logging: with_logger, NullLogger
     using Wflow: to_SI, MM
@@ -15,8 +15,8 @@
     ZMQ.connect(socket, "tcp://localhost:5555")
 
     function request(message)
-        ZMQ.send(socket, JSON3.write(message))
-        ret_value = JSON3.read(ZMQ.recv(socket), Dict; allow_inf = true)
+        ZMQ.send(socket, JSON.json(message))
+        ret_value = JSON.parse(ZMQ.recv(socket); dicttype = Dict{String, Any})
         return ret_value
     end
 
@@ -33,13 +33,21 @@
         @test request((fn = "get_time_units",)) == Dict("time_units" => "s")
     end
 
-    @testset "Reading and writing NaN values allowed" begin
+    @testset "non-finite responses encoded as null" begin
+        # the server encoder maps NaN, Inf and -Inf to null, since JSON cannot represent them.
+        encoded = WflowServer.serialize_response(Dict("value" => [NaN, Inf, -Inf, 1.0]))
+        @test !occursin("NaN", encoded)
+        @test !occursin("Infinity", encoded)
+        @test occursin("[null,null,null,1.0]", encoded)
+        @test JSON.parse(encoded)["value"] == [nothing, nothing, nothing, 1.0]
+
+        # end-to-end check that a model response containing NaN arrives as null
         msg = (
             fn = "get_value",
             name = "soil_layer_1_water__volume_fraction",
             dest = fill(0.0, 50063),
         )
-        @test isnan(mean(request(msg)["value"]))
+        @test any(isnothing, request(msg)["value"])
     end
 
     @testset "update functions" begin