Training a Neural ODE to Model Gravitational Waveforms
This code is adapted from Astroinformatics/ScientificMachineLearning
The code has been minimally adapted from Keith et. al. 2021 which originally used Flux.jl
Package Imports
using Lux,
ComponentArrays,
LineSearches,
OrdinaryDiffEqLowOrderRK,
Optimization,
OptimizationOptimJL,
Printf,
Random,
SciMLSensitivity
using CairoMakieDefine some Utility Functions
Tip
This section can be skipped. It defines functions to simulate the model, however, from a scientific machine learning perspective, isn't super relevant.
We need a very crude 2-body path. Assume the 1-body motion is a Newtonian 2-body position vector
function one2two(path, m₁, m₂)
M = m₁ + m₂
r₁ = m₂ / M .* path
r₂ = -m₁ / M .* path
return r₁, r₂
endNext we define a function to perform the change of variables:
@views function soln2orbit(soln, model_params=nothing)
@assert size(soln, 1) ∈ [2, 4] "size(soln,1) must be either 2 or 4"
if size(soln, 1) == 2
χ = soln[1, :]
ϕ = soln[2, :]
@assert length(model_params) == 3 "model_params must have length 3 when size(soln,2) = 2"
p, M, e = model_params
else
χ = soln[1, :]
ϕ = soln[2, :]
p = soln[3, :]
e = soln[4, :]
end
r = p ./ (1 .+ e .* cos.(χ))
x = r .* cos.(ϕ)
y = r .* sin.(ϕ)
orbit = vcat(x', y')
return orbit
endThis function uses second-order one-sided difference stencils at the endpoints; see https://doi.org/10.1090/S0025-5718-1988-0935077-0
function d_dt(v::AbstractVector, dt)
a = -3 / 2 * v[1] + 2 * v[2] - 1 / 2 * v[3]
b = (v[3:end] .- v[1:(end - 2)]) / 2
c = 3 / 2 * v[end] - 2 * v[end - 1] + 1 / 2 * v[end - 2]
return [a; b; c] / dt
endThis function uses second-order one-sided difference stencils at the endpoints; see https://doi.org/10.1090/S0025-5718-1988-0935077-0
function d2_dt2(v::AbstractVector, dt)
a = 2 * v[1] - 5 * v[2] + 4 * v[3] - v[4]
b = v[1:(end - 2)] .- 2 * v[2:(end - 1)] .+ v[3:end]
c = 2 * v[end] - 5 * v[end - 1] + 4 * v[end - 2] - v[end - 3]
return [a; b; c] / (dt^2)
endNow we define a function to compute the trace-free moment tensor from the orbit
function orbit2tensor(orbit, component, mass=1)
x = orbit[1, :]
y = orbit[2, :]
Ixx = x .^ 2
Iyy = y .^ 2
Ixy = x .* y
trace = Ixx .+ Iyy
if component[1] == 1 && component[2] == 1
tmp = Ixx .- trace ./ 3
elseif component[1] == 2 && component[2] == 2
tmp = Iyy .- trace ./ 3
else
tmp = Ixy
end
return mass .* tmp
end
function h_22_quadrupole_components(dt, orbit, component, mass=1)
mtensor = orbit2tensor(orbit, component, mass)
mtensor_ddot = d2_dt2(mtensor, dt)
return 2 * mtensor_ddot
end
function h_22_quadrupole(dt, orbit, mass=1)
h11 = h_22_quadrupole_components(dt, orbit, (1, 1), mass)
h22 = h_22_quadrupole_components(dt, orbit, (2, 2), mass)
h12 = h_22_quadrupole_components(dt, orbit, (1, 2), mass)
return h11, h12, h22
end
function h_22_strain_one_body(dt::T, orbit) where {T}
h11, h12, h22 = h_22_quadrupole(dt, orbit)
h₊ = h11 - h22
hₓ = T(2) * h12
scaling_const = √(T(π) / 5)
return scaling_const * h₊, -scaling_const * hₓ
end
function h_22_quadrupole_two_body(dt, orbit1, mass1, orbit2, mass2)
h11_1, h12_1, h22_1 = h_22_quadrupole(dt, orbit1, mass1)
h11_2, h12_2, h22_2 = h_22_quadrupole(dt, orbit2, mass2)
h11 = h11_1 + h11_2
h12 = h12_1 + h12_2
h22 = h22_1 + h22_2
return h11, h12, h22
end
function h_22_strain_two_body(dt::T, orbit1, mass1, orbit2, mass2) where {T}
# compute (2,2) mode strain from orbits of BH1 of mass1 and BH2 of mass 2
@assert abs(mass1 + mass2 - 1.0) < 1.0e-12 "Masses do not sum to unity"
h11, h12, h22 = h_22_quadrupole_two_body(dt, orbit1, mass1, orbit2, mass2)
h₊ = h11 - h22
hₓ = T(2) * h12
scaling_const = √(T(π) / 5)
return scaling_const * h₊, -scaling_const * hₓ
end
function compute_waveform(dt::T, soln, mass_ratio, model_params=nothing) where {T}
@assert mass_ratio ≤ 1 "mass_ratio must be <= 1"
@assert mass_ratio ≥ 0 "mass_ratio must be non-negative"
orbit = soln2orbit(soln, model_params)
if mass_ratio > 0
m₂ = inv(T(1) + mass_ratio)
m₁ = mass_ratio * m₂
orbit₁, orbit₂ = one2two(orbit, m₁, m₂)
waveform = h_22_strain_two_body(dt, orbit₁, m₁, orbit₂, m₂)
else
waveform = h_22_strain_one_body(dt, orbit)
end
return waveform
endSimulating the True Model
RelativisticOrbitModel defines system of odes which describes motion of point like particle in Schwarzschild background, uses
where,
function RelativisticOrbitModel(u, (p, M, e), t)
χ, ϕ = u
numer = (p - 2 - 2 * e * cos(χ)) * (1 + e * cos(χ))^2
denom = sqrt((p - 2)^2 - 4 * e^2)
χ̇ = numer * sqrt(p - 6 - 2 * e * cos(χ)) / (M * (p^2) * denom)
ϕ̇ = numer / (M * (p^(3 / 2)) * denom)
return [χ̇, ϕ̇]
end
mass_ratio = 0.0 # test particle
u0 = Float64[π, 0.0] # initial conditions
datasize = 250
tspan = (0.0f0, 6.0f4) # timespace for GW waveform
tsteps = range(tspan[1], tspan[2]; length=datasize) # time at each timestep
dt_data = tsteps[2] - tsteps[1]
dt = 100.0
const ode_model_params = [100.0, 1.0, 0.5]; # p, M, eLet's simulate the true model and plot the results using OrdinaryDiffEq.jl
prob = ODEProblem(RelativisticOrbitModel, u0, tspan, ode_model_params)
soln = Array(solve(prob, RK4(); saveat=tsteps, dt, adaptive=false))
waveform = first(compute_waveform(dt_data, soln, mass_ratio, ode_model_params))
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")
l = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
s = scatter!(ax, tsteps, waveform; marker=:circle, markersize=12, alpha=0.5)
axislegend(ax, [[l, s]], ["Waveform Data"])
fig
endDefining a Neural Network Model
Next, we define the neural network model that takes 1 input (time) and has two outputs. We'll make a function ODE_model that takes the initial conditions, neural network parameters and a time as inputs and returns the derivatives.
It is typically never recommended to use globals but in case you do use them, make sure to mark them as const.
We will deviate from the standard Neural Network initialization and use WeightInitializers.jl,
const nn = Chain(
Base.Fix1(fast_activation, cos),
Dense(1 => 32, cos; init_weight=truncated_normal(; std=1.0e-4), init_bias=zeros32),
Dense(32 => 32, cos; init_weight=truncated_normal(; std=1.0e-4), init_bias=zeros32),
Dense(32 => 2; init_weight=truncated_normal(; std=1.0e-4), init_bias=zeros32),
)
ps, st = Lux.setup(Random.default_rng(), nn)((layer_1 = NamedTuple(), layer_2 = (weight = Float32[-5.8348374f-5; -9.876448f-5; 0.00012119508; -9.61561f-6; -3.6043346f-6; 5.899972f-5; -0.00018122746; 9.657172f-5; -7.5578083f-7; -0.00011121817; -0.00015208033; 0.00010028688; 7.760727f-5; -0.00010268387; -5.642814f-5; 0.00010963677; 4.8780985f-5; 9.687141f-5; -0.00011957743; -3.3345277f-5; 2.436823f-5; 0.00012688052; -5.2860596f-5; 8.907581f-5; 9.935847f-5; -6.452864f-5; -1.1268748f-5; -0.00021390352; -0.000104412014; -0.00014843087; 0.000107153974; -9.729836f-5;;], bias = Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), layer_3 = (weight = Float32[-1.8668805f-5 3.582669f-5 3.3163768f-5 -0.00019120348 0.0001372589 -7.1804454f-5 -8.712347f-5 -6.905994f-5 2.0078533f-5 0.00014296938 9.471705f-5 -0.00012800335 -3.7012604f-5 1.907192f-5 -0.00020639854 0.000119270066 7.035698f-6 1.5725363f-5 5.3540563f-5 -2.6147347f-5 0.00021266211 1.9117779f-5 9.9451885f-5 -1.8201885f-5 -5.19872f-5 0.00013282588 -1.7630024f-5 3.493997f-5 -4.227899f-6 9.470231f-5 -0.000101453 0.0001136428; 0.00018772091 0.00017170818 2.6265914f-5 3.8836795f-5 -0.00013627528 -9.247766f-5 -0.00013811015 -0.00013960659 0.00028160957 7.4565796f-5 -0.00018021406 -0.00012523573 2.4525003f-5 -5.278551f-5 9.0219f-5 -9.067837f-5 3.2154523f-5 8.102905f-5 2.5942692f-5 -7.058045f-5 -0.00011492865 8.9720845f-5 2.2314207f-5 8.84276f-6 -4.1187162f-5 -6.847978f-5 0.00014521962 -3.5768804f-5 7.5798f-5 -4.8830978f-5 9.227155f-5 -5.5563723f-5; 0.000178805 -8.595718f-5 4.3519114f-5 -5.100334f-5 1.45954255f-5 6.1711806f-5 -2.487818f-6 1.8394941f-5 -0.0001666309 -7.0548405f-5 -9.6052725f-5 4.3554413f-5 4.8849288f-5 -8.735577f-5 -0.00013988133 0.0002955819 3.1695358f-6 4.3652713f-6 0.00010313911 9.819785f-5 -6.160957f-5 -6.0200775f-5 6.172343f-5 1.5346572f-5 7.297788f-5 -9.3491944f-5 -0.00021940359 0.00010485367 0.00013447957 2.9692568f-5 -0.00015388658 -0.0001074127; 9.259429f-5 -6.510644f-5 -0.00011919127 -0.0001798583 -4.5997513f-5 5.7395977f-5 7.70142f-5 9.059317f-5 3.8464248f-5 -2.41091f-6 0.00010552939 -5.042836f-5 6.8878704f-5 -4.6673354f-6 0.00013026137 1.5497677f-6 -9.645844f-5 -5.5899454f-5 7.648796f-6 4.657987f-6 -6.68062f-5 -5.7248246f-5 8.7148124f-5 2.3410526f-5 5.8693473f-5 -4.7692538f-5 -2.2363054f-5 0.000115981034 8.1278464f-5 -6.75694f-5 0.00015751434 -0.00012262547; 7.2260327f-6 -7.8243494f-5 1.6625983f-5 5.144557f-5 -1.9628384f-5 6.767262f-6 5.6363806f-5 0.00019119385 -0.00013999299 1.6062126f-5 -1.4986463f-5 0.00019346704 7.3822805f-5 -0.000103250764 1.406671f-5 0.00012881249 2.91033f-6 -0.0001195426 -3.5115274f-5 0.00010512641 4.0520692f-7 8.562195f-5 0.00023492887 -7.472891f-6 4.6771544f-5 5.709192f-5 -8.135603f-5 1.4331183f-5 -2.637332f-5 0.00025164522 0.00022846553 -5.6744916f-6; -0.00010322685 5.743086f-5 3.4199722f-5 -6.0650204f-5 -0.000109708504 1.8495713f-5 -4.2140786f-5 0.00014654997 -0.00020358755 7.310547f-5 8.177945f-6 -1.1077046f-5 -5.263695f-6 -9.5815834f-5 0.00012992868 -1.46531875f-5 5.698459f-5 8.070533f-5 0.00014072974 8.261127f-5 0.00018273978 4.32997f-5 -6.747821f-5 5.371308f-5 6.135542f-6 -5.9303227f-5 7.817134f-5 -8.325223f-5 -8.402641f-5 -9.679683f-5 -2.2898825f-5 3.413551f-5; -4.4073317f-5 2.7275977f-5 -0.0001298853 -7.630096f-6 1.3358331f-5 8.0846585f-5 0.00011925573 -0.00012388141 0.00010124177 4.686407f-5 -7.991484f-5 -0.00010383752 9.1115425f-7 4.4782897f-5 1.2265989f-5 -6.156948f-5 0.00010292579 6.448842f-5 3.4611374f-5 -0.0001135545 -0.00017832097 4.8059017f-5 8.5970794f-5 1.9724275f-5 1.4930607f-5 4.8618444f-6 1.2598952f-5 -4.363955f-6 0.00019778329 0.0001305517 -0.0001677117 0.000119327975; -5.7099915f-5 6.9759386f-5 -4.055336f-6 8.983842f-5 6.255421f-5 0.00018410856 -0.000109972374 -0.0001632548 -3.0787665f-5 -1.906711f-5 5.8081547f-5 7.3935094f-5 0.00010043877 -4.0741306f-5 5.7227033f-5 4.4911794f-5 0.00017947025 2.949328f-5 -0.0002142254 7.1590344f-5 -1.7454404f-5 -0.000111240995 6.6780894f-6 3.5890025f-5 3.3431053f-5 -8.3752544f-5 -3.7923408f-5 -0.00012639852 -0.00013514979 0.00017401269 -3.3037242f-5 8.641969f-5; -7.412197f-5 -6.344727f-5 6.7137444f-5 -0.00011289815 -4.790401f-5 8.655701f-5 6.383271f-5 -9.812945f-5 -0.000102941296 -6.364722f-5 0.00017590856 -0.00015643539 -0.00012978159 0.00013316414 -8.65311f-5 1.3504223f-5 -0.00025869565 -9.855057f-5 0.00023505077 0.00012223877 4.836017f-5 -9.042775f-5 -6.925345f-5 8.36833f-5 -2.1982663f-5 8.7313245f-5 -9.31693f-5 0.00011150176 -8.2252234f-5 -3.9185135f-5 -2.7172156f-5 6.818709f-5; 0.00010678034 3.617757f-5 -1.3143006f-5 0.00011182833 9.8506884f-5 4.0602183f-5 -0.00012342166 -2.9673085f-6 -0.000118087126 6.189607f-5 0.00012855513 1.905334f-6 7.544396f-5 -8.458957f-5 -1.6054362f-5 -3.0408366f-5 3.150619f-5 -6.760735f-5 6.6923814f-5 -6.797528f-6 -6.1812716f-5 5.251468f-5 2.4940542f-5 1.1779289f-5 0.00016093098 -9.990018f-5 -2.7306f-5 0.00014413612 -0.00013049912 -7.51237f-5 -0.00019567831 4.4555116f-5; 7.866241f-5 6.324201f-5 -7.766566f-5 -5.4740878f-5 -3.220369f-5 4.2940894f-5 0.00013914131 -3.0543524f-5 0.00020955304 2.5670568f-5 0.00016678897 -6.3236424f-5 0.0001627412 -4.7291533f-6 8.217039f-5 1.3586791f-5 4.1643478f-5 -0.00010451503 -9.4598705f-5 -5.503092f-5 -8.808115f-5 3.4125536f-5 -0.00014314728 1.2982209f-5 6.0949074f-5 -0.00018566583 0.00017133533 -0.00016779268 0.00011038795 1.5813997f-5 8.593359f-5 0.00011040944; -6.1550425f-5 -2.3709503f-5 3.1105566f-5 -8.874214f-5 -4.4673317f-5 -0.0001846917 4.5776327f-5 7.664195f-5 6.294843f-7 5.2682208f-5 2.7718117f-5 6.716495f-5 0.00011583606 8.690497f-6 8.7270375f-5 0.00010652735 -4.447465f-6 -4.3212214f-5 7.0800364f-5 -4.6065896f-5 -7.215428f-5 7.678661f-6 -0.00011783728 -1.1832458f-5 -3.2922928f-6 -3.7849422f-5 0.00010475854 4.7761754f-5 -0.00010881852 -0.00017345937 9.182303f-6 1.0893005f-5; -4.125227f-5 9.5546544f-5 -5.7037927f-5 0.00012168478 0.00013969779 4.9814884f-5 -0.00012058344 0.00018792666 -0.00012954717 4.0372288f-5 -4.073608f-5 -0.00020077577 -6.2347965f-5 0.00013898774 0.00015038004 -0.0001422307 5.741513f-5 -3.3815304f-5 -1.8594452f-5 7.7310884f-5 -6.7778536f-5 -0.00011506184 -0.00012988724 8.71114f-5 0.00011507885 0.00012965493 0.00016975487 -0.00014526135 -6.059203f-6 8.084686f-5 -4.118684f-5 -4.232304f-5; -7.872805f-5 -4.073728f-6 3.0086223f-5 0.00016417092 8.085073f-5 -0.0001035475 4.026178f-5 0.00017359834 -0.00010359966 -0.00021277538 -3.508859f-5 -0.00016853794 -0.00010978249 0.00011407171 -0.00025826404 7.6030185f-5 -0.00010497797 -0.00025259051 -1.3771323f-5 0.000117471056 4.8935748f-5 -0.00011597259 -4.0221832f-5 0.00013946078 8.000082f-5 2.3506987f-5 3.9880804f-5 1.2160578f-5 1.7364467f-6 1.4217318f-5 -4.0581468f-5 -3.6872127f-5; 0.00012325807 -8.8001216f-5 5.188272f-5 -8.49933f-6 -3.9027796f-5 0.00010778666 -3.514237f-5 6.967995f-5 -0.00014925581 -6.4074284f-6 -7.900031f-5 -6.073437f-6 -0.00018893734 -8.5106505f-5 -0.00029727194 0.00012964441 0.0001314124 -0.00013782788 -2.3002565f-5 -2.1128002f-5 6.531919f-5 1.6641041f-5 -0.00013628005 6.938581f-5 -9.003848f-6 9.694454f-5 3.2118544f-6 0.00012561018 -2.319881f-6 -8.3875006f-5 -2.4464682f-5 -1.0085286f-5; -0.00010202448 4.95222f-5 2.6988857f-5 5.8350568f-5 2.7979366f-5 -4.3161093f-5 3.6365588f-8 4.585672f-6 0.00016051176 3.719396f-5 1.6391028f-5 0.00010761394 8.567692f-5 -2.0019566f-5 -4.896406f-5 -9.366821f-5 9.404927f-5 -3.9895367f-5 -3.2931785f-5 2.8960156f-5 0.000121102516 -4.75475f-5 1.3998798f-5 -0.00016584857 2.3259085f-5 -1.9214853f-5 -1.7622813f-5 7.681458f-5 2.5161394f-6 -6.466256f-5 -1.0205686f-5 0.00025454335; -6.90344f-5 -0.00013407167 -1.1767117f-5 2.9079521f-5 0.0001304263 -3.792497f-5 -0.00015737175 -5.3776017f-5 4.8925893f-5 -4.6064983f-5 -9.167231f-5 0.00027149473 -3.603722f-5 -4.1691226f-5 3.3781748f-6 -2.9136012f-5 3.5376826f-5 -2.1586222f-5 9.23938f-6 0.00015016452 -2.2445216f-5 -9.795376f-5 2.868583f-5 -0.00012655379 6.0694914f-5 -8.782607f-5 4.1720858f-5 -1.6417287f-5 -6.8041923f-6 -0.00011119281 2.9032117f-5 1.8462191f-7; -4.0528626f-5 3.6504043f-5 -3.6995178f-5 -0.00017494564 -6.709162f-5 2.5705885f-5 -8.510146f-5 3.1299085f-5 6.9587f-6 0.00018017621 8.914129f-5 3.9096714f-5 -0.00014975824 -1.8155542f-5 -0.000106488886 -0.00014602067 -0.00013553581 1.4857246f-6 4.0629788f-5 -1.6190754f-5 0.000112203576 3.895496f-5 0.00013391991 -3.6400513f-6 -7.430958f-7 0.00018591434 -9.0221976f-5 0.00018314252 -0.00023236197 -6.269749f-5 9.384995f-5 -4.198353f-5; -1.5182989f-5 -5.798204f-5 -3.951506f-5 8.770751f-6 4.2634252f-5 0.00017770914 4.135522f-5 -4.0808278f-5 -1.6481556f-5 -0.00024844686 0.00011901454 0.00010770738 -0.0001750306 -0.00017351042 1.2258177f-5 -5.714537f-6 -0.00017381136 1.1585549f-5 5.149784f-5 2.0796619f-5 1.2299424f-5 2.7931776f-6 -3.4180302f-5 7.975648f-6 -0.0001208594 1.2211348f-5 1.0895036f-5 -0.00017020131 0.00013477799 1.1158511f-5 -1.6464384f-5 7.838931f-5; -4.7005986f-5 0.00010189473 0.00023792495 -5.0936615f-5 1.4457192f-5 3.609661f-5 8.511194f-5 0.00011359169 -7.637319f-5 9.2589784f-5 6.3246225f-5 -0.00018655814 7.145045f-5 -5.4244072f-5 -2.171643f-5 -7.0437905f-5 1.5600644f-5 8.4449974f-5 0.00018714445 4.357966f-5 1.5771004f-5 -0.00017935953 7.0828937f-6 -2.8323606f-5 -0.00017892648 1.0163123f-5 -6.2054685f-5 -0.00015607451 -7.3581985f-5 -7.5289854f-6 -2.733882f-5 1.6308582f-5; -6.632195f-6 -9.728169f-5 -0.00010843369 8.4823725f-5 -0.00010015622 -0.000120785786 -5.774221f-5 9.17833f-5 0.00014111184 -4.5533223f-5 -4.7735277f-5 -0.00020020184 -5.97778f-5 0.0001171227 -1.8025248f-5 7.070286f-5 4.9758975f-5 0.000113679795 0.00013957817 0.00024601037 -6.408259f-5 8.4029976f-5 2.1468562f-5 -1.118792f-5 5.2142626f-5 2.8656608f-5 3.6030542f-5 -0.0001651316 -1.9895791f-5 4.568944f-5 2.2872337f-5 8.644947f-5; 0.00015099003 -2.4772768f-5 1.8022003f-5 -5.8446334f-5 0.00012379372 -7.447553f-5 1.9523908f-5 1.8614455f-5 3.3289473f-6 9.5214986f-5 -1.6094364f-5 0.00015316215 -0.00019020907 6.0708273f-5 -0.0001361975 -0.00027116327 -1.373464f-6 6.176239f-5 6.2120285f-5 4.6654684f-5 0.0001764337 -0.00019225273 -1.8877085f-5 3.200361f-5 7.265681f-5 0.00029059398 0.00013931139 -4.6611665f-5 0.00011881269 -0.00010924844 -2.5692501f-5 0.00014669477; -8.306499f-5 -0.00010236705 -0.000108781314 1.02758395f-5 5.9223396f-5 -6.188559f-5 -6.211359f-5 8.42997f-5 -3.999993f-5 3.706418f-5 2.1479811f-5 6.244985f-5 -4.6485085f-5 0.00011913888 -5.573347f-6 6.3916186f-6 -6.200957f-6 0.00011328789 -9.348446f-5 -9.881842f-5 7.5154334f-5 6.68273f-5 -7.2349903f-6 9.610861f-5 2.8986715f-6 1.8459194f-5 -0.00019489758 -5.6652025f-5 -2.5276922f-6 -0.00013769699 9.355618f-5 -0.00013061326; 4.0602885f-5 -5.2530268f-5 -0.00019219781 8.803308f-5 5.7873014f-5 8.246208f-5 6.670098f-5 -5.5866065f-5 -0.00017423845 -0.00010859543 -1.9457153f-5 -1.7794315f-5 -0.00010353774 -0.00024356009 -2.3843933f-5 -4.339431f-5 -4.308275f-5 -8.711181f-5 7.499794f-5 6.0530314f-5 6.792406f-5 0.00014560232 7.7612814f-5 6.289713f-6 0.000102513135 -3.066662f-5 8.4256855f-5 -4.1275776f-5 -9.165219f-5 5.3934284f-5 -8.955513f-5 2.8145283f-5; 5.47765f-5 -8.857444f-6 -0.00011222946 -0.00010341099 5.123414f-5 -0.00011648262 -5.0840223f-5 -0.00014317682 -6.151591f-5 -1.6116448f-6 -0.00013006902 -5.1536485f-5 -0.00013057752 5.668812f-5 1.4020236f-5 0.000110635745 6.6903235f-6 -3.85035f-5 6.203425f-5 -0.00014279172 -0.0001781243 -0.00011709085 5.7553356f-5 -4.0651063f-5 0.00013685464 4.4272597f-6 -7.2951334f-5 5.4782864f-5 -3.267499f-5 -0.0001790802 -0.00014754127 9.328315f-6; 9.269358f-5 -2.6185013f-5 -7.3304f-5 -9.595472f-5 0.000115709394 3.4146806f-6 0.00022821697 2.8725048f-5 -7.5017655f-5 5.109157f-5 -1.2287818f-5 3.1886448f-5 -4.208462f-5 7.743377f-5 3.329134f-5 -0.000118974924 1.9989704f-6 0.000102279206 0.0001055711 -1.6677022f-5 2.4690427f-5 -7.526083f-5 6.221869f-5 2.6277543f-5 8.114365f-5 -0.00019512963 -1.2115424f-5 -9.1851885f-5 8.284925f-5 6.154138f-5 0.00013860075 -0.00014217527; 0.00016121926 0.00011996626 -5.165581f-6 3.0095589f-5 6.43308f-5 -0.00010438062 -0.000115888426 3.2026648f-5 -0.00017575565 -4.8505055f-5 -0.0001729564 7.12783f-5 0.00012073535 6.3341383f-7 -1.5338179f-5 -8.6135085f-5 -4.5970708f-5 0.00013874446 -0.000107713866 -0.00018244922 0.00020025985 5.723756f-6 -5.3542364f-5 4.342466f-5 -6.6845176f-5 -6.5517364f-5 -3.6826445f-5 -0.00021503215 -6.181883f-5 -0.000114291135 0.00021477226 3.7397756f-5; 0.00010745972 -2.740469f-5 0.0001550886 0.00010056456 0.00014794125 -8.208977f-6 3.4907065f-5 1.1085824f-5 -0.00012296403 -7.034539f-5 8.36932f-5 0.00021919633 3.7078196f-5 -6.3416286f-5 4.9852148f-5 -2.5337291f-5 2.7409564f-5 -8.090568f-6 -8.499931f-5 -6.674316f-5 0.000105325045 -0.00029841813 -7.783665f-5 -8.351863f-5 -5.0079934f-6 9.4368725f-6 -1.3138666f-5 8.626741f-5 -6.0838618f-5 -0.0003967997 3.7571117f-5 3.7966925f-5; 7.497475f-5 -8.13362f-5 -0.00012389413 -7.907844f-6 -1.828185f-5 3.742274f-5 -0.00012604592 -1.9765803f-5 4.9770584f-5 0.00018731263 -8.914034f-5 0.0002680948 0.00014337164 0.00013143705 -0.00014196469 -0.00014767448 0.00011116728 1.485507f-5 4.4098408f-5 -0.00021469478 5.8803835f-5 -2.1461878f-5 2.0072739f-5 4.2081847f-5 7.6898024f-5 0.00014227697 -8.567473f-5 4.9282193f-5 -0.00020578832 -0.000152279 9.8155215f-5 7.899175f-5; 4.9119128f-5 -0.00013359996 6.4877975f-5 -4.5627123f-5 4.8682614f-5 4.1882307f-5 1.711162f-5 -0.00012438916 2.644266f-5 -2.82547f-5 5.241305f-5 4.6142977f-5 -5.672127f-5 0.0001746392 -8.3533814f-5 3.5351382f-5 1.7182001f-5 -5.3050127f-5 0.00022973749 6.2565676f-5 2.6296216f-6 8.87548f-5 -9.779842f-5 -0.00013249954 5.3909404f-5 9.3854585f-5 -5.2240484f-6 -4.0277984f-5 2.4504248f-5 3.0046596f-5 -4.8445403f-5 3.8707485f-6; 1.704329f-5 -7.769637f-6 -0.00010364993 -4.019549f-5 5.6116696f-5 -1.1043193f-5 3.9614308f-5 3.381184f-5 0.00010750851 -3.7334008f-5 -6.209504f-6 2.4276742f-5 -6.197409f-6 6.289988f-5 -5.6212735f-5 -1.0209883f-6 -7.3071446f-5 4.335046f-5 5.292299f-5 -0.00018309934 -0.00012126875 0.00013110266 7.9121244f-5 0.00012760781 -7.442387f-5 -0.00023450858 -1.6719896f-5 7.446546f-5 0.000114984265 2.6593067f-5 -5.7959587f-5 -8.2231145f-6; 1.3634352f-5 -8.507019f-5 -0.00018329339 5.7341145f-5 -3.967044f-5 9.150825f-5 -4.5447337f-5 -7.508787f-5 -6.501526f-6 1.0647892f-5 -3.81255f-5 -0.00016447679 7.79185f-5 2.2667426f-5 0.00014416428 -1.0217069f-5 0.00012669688 2.2144768f-5 7.5454984f-5 1.457007f-5 4.4653472f-5 -1.0623837f-5 5.1792293f-5 -5.5606088f-5 8.21392f-5 -7.5122334f-5 -9.413978f-5 2.2343464f-5 -0.00023895802 -2.5393718f-5 3.913376f-5 1.5355401f-5], bias = Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), layer_4 = (weight = Float32[1.29448135f-5 7.357276f-6 1.5366813f-5 1.9058441f-5 -7.488697f-6 7.1884104f-5 -9.038628f-7 0.00012023569 -0.00016395425 3.3367563f-5 -5.281665f-6 0.00015298683 -0.00011824935 -0.00013802972 -8.2075865f-5 -0.00022923962 -8.1164595f-5 1.5034316f-5 0.0001772002 3.0051022f-5 7.385093f-5 6.368679f-5 -4.561643f-5 -0.00020627433 2.7028058f-5 -0.00019484955 -0.00012353783 -4.542493f-6 6.477746f-5 -0.00020185347 -0.00018942049 7.917334f-5; -3.5029694f-5 0.00010797187 -1.1301062f-6 -9.895781f-5 -6.2111358f-6 -0.0001234214 5.128497f-5 -3.309298f-5 0.0002025835 -3.3101758f-5 6.147855f-5 0.000143699 -2.873445f-5 0.000114899776 -0.00017635307 -2.2271502f-6 2.3233264f-5 -1.6880065f-5 4.201704f-5 -0.00011256594 3.7162423f-5 0.00031406694 -0.00014558544 -0.00013784724 9.384956f-5 -6.3827565f-6 0.00015267596 -0.00014746338 -4.6161345f-5 -7.609254f-5 -6.9909824f-5 -2.1163583f-5], bias = Float32[0.0, 0.0])), (layer_1 = NamedTuple(), layer_2 = NamedTuple(), layer_3 = NamedTuple(), layer_4 = NamedTuple()))Similar to most deep learning frameworks, Lux defaults to using Float32. However, in this case we need Float64
const params = ComponentArray(f64(ps))
const nn_model = StatefulLuxLayer(nn, nothing, st)StatefulLuxLayer{Val{true}()}(
Chain(
layer_1 = WrappedFunction(Base.Fix1{typeof(LuxLib.API.fast_activation), typeof(cos)}(LuxLib.API.fast_activation, cos)),
layer_2 = Dense(1 => 32, cos), # 64 parameters
layer_3 = Dense(32 => 32, cos), # 1_056 parameters
layer_4 = Dense(32 => 2), # 66 parameters
),
) # Total: 1_186 parameters,
# plus 0 states.Now we define a system of odes which describes motion of point like particle with Newtonian physics, uses
where,
function ODE_model(u, nn_params, t)
χ, ϕ = u
p, M, e = ode_model_params
# In this example we know that `st` is am empty NamedTuple hence we can safely ignore
# it, however, in general, we should use `st` to store the state of the neural network.
y = 1 .+ nn_model([first(u)], nn_params)
numer = (1 + e * cos(χ))^2
denom = M * (p^(3 / 2))
χ̇ = (numer / denom) * y[1]
ϕ̇ = (numer / denom) * y[2]
return [χ̇, ϕ̇]
endLet us now simulate the neural network model and plot the results. We'll use the untrained neural network parameters to simulate the model.
prob_nn = ODEProblem(ODE_model, u0, tspan, params)
soln_nn = Array(solve(prob_nn, RK4(); u0, p=params, saveat=tsteps, dt, adaptive=false))
waveform_nn = first(compute_waveform(dt_data, soln_nn, mass_ratio, ode_model_params))
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")
l1 = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
s1 = scatter!(
ax, tsteps, waveform; marker=:circle, markersize=12, alpha=0.5, strokewidth=2
)
l2 = lines!(ax, tsteps, waveform_nn; linewidth=2, alpha=0.75)
s2 = scatter!(
ax, tsteps, waveform_nn; marker=:circle, markersize=12, alpha=0.5, strokewidth=2
)
axislegend(
ax,
[[l1, s1], [l2, s2]],
["Waveform Data", "Waveform Neural Net (Untrained)"];
position=:lb,
)
fig
endSetting Up for Training the Neural Network
Next, we define the objective (loss) function to be minimized when training the neural differential equations.
const mseloss = MSELoss()
function loss(θ)
pred = Array(solve(prob_nn, RK4(); u0, p=θ, saveat=tsteps, dt, adaptive=false))
pred_waveform = first(compute_waveform(dt_data, pred, mass_ratio, ode_model_params))
return mseloss(pred_waveform, waveform)
endWarmup the loss function
loss(params)0.0006828148459814618Now let us define a callback function to store the loss over time
const losses = Float64[]
function callback(θ, l)
push!(losses, l)
@printf "Training \t Iteration: %5d \t Loss: %.10f\n" θ.iter l
return false
endTraining the Neural Network
Training uses the BFGS optimizers. This seems to give good results because the Newtonian model seems to give a very good initial guess
adtype = Optimization.AutoZygote()
optf = Optimization.OptimizationFunction((x, p) -> loss(x), adtype)
optprob = Optimization.OptimizationProblem(optf, params)
res = Optimization.solve(
optprob,
BFGS(; initial_stepnorm=0.01, linesearch=LineSearches.BackTracking());
callback,
maxiters=1000,
)retcode: Success
u: ComponentVector{Float64}(layer_1 = Float64[], layer_2 = (weight = [-5.834837429573833e-5; -9.876448166321365e-5; 0.00012119508028260125; -9.61560999712665e-6; -3.604334551710514e-6; 5.899972165933541e-5; -0.00018122745677817958; 9.657171904102456e-5; -7.557808316952468e-7; -0.00011121817078660818; -0.0001520803343736575; 0.00010028688120650281; 7.760727021368201e-5; -0.00010268387268295828; -5.642813994197312e-5; 0.00010963677050320207; 4.878098479817058e-5; 9.687140845923874e-5; -0.00011957743117819968; -3.334527718832182e-5; 2.4368229787790325e-5; 0.00012688052083825082; -5.286059604256085e-5; 8.907580922825896e-5; 9.935846901496912e-5; -6.452864181481293e-5; -1.1268747584828231e-5; -0.00021390352048921663; -0.00010441201447967381; -0.00014843087410527271; 0.00010715397365847818; -9.729836165199876e-5;;], bias = [-8.891303677602294e-18, -2.838277157730841e-17, 3.5387587409600525e-17, -2.549220392461631e-20, 5.339460588768349e-19, 1.3487181460004428e-17, -1.229785258333336e-16, 7.804135303568906e-17, -8.894762252530783e-19, -1.5099386087826267e-16, -1.050031947957065e-16, 6.074313436240169e-17, 1.0683385728512984e-16, -1.8480289008569346e-17, -3.122403531863356e-17, -8.001637306743766e-17, 4.8224951868446606e-17, 2.925387967360732e-17, 1.4311932852162363e-17, -2.6912775625412213e-17, 6.807668300419693e-18, 7.027413585281725e-17, -3.552638259238728e-17, -2.949119152813496e-17, 2.0462828978047663e-17, -1.9879271431373452e-17, -8.752066325280991e-18, 1.1871272086353752e-16, -8.456419006375275e-17, -3.066647851725688e-16, 1.0305051792104961e-16, -7.476779165162449e-17]), layer_3 = (weight = [-1.8667517394571942e-5 3.5827979047966885e-5 3.3165054958728634e-5 -0.00019120219116033793 0.00013726019057044958 -7.180316646285304e-5 -8.71221859024955e-5 -6.905865342291264e-5 2.0079819990277405e-5 0.00014297066769627533 9.47183357884099e-5 -0.00012800205917032657 -3.701131612346562e-5 1.9073207625255352e-5 -0.0002063972573860542 0.00011927135307173869 7.036985664815693e-6 1.5726650929495606e-5 5.3541850832473216e-5 -2.614605949660612e-5 0.00021266339859990575 1.9119066167105823e-5 9.945317248180591e-5 -1.8200597366598846e-5 -5.198591355751484e-5 0.0001328271678748543 -1.762873620195122e-5 3.4941258093511294e-5 -4.22661145152782e-6 9.470359469828403e-5 -0.0001014517114863759 0.00011364409107314226; 0.0001877211926808226 0.0001717084543384251 2.6266192555833436e-5 3.8837073709640884e-5 -0.00013627500253956558 -9.247738295694635e-5 -0.00013810986808262854 -0.0001396063142851355 0.0002816098441440261 7.456607466569266e-5 -0.00018021378297586 -0.00012523545398604008 2.4525281453446175e-5 -5.2785233372066894e-5 9.021927883854317e-5 -9.067808957066287e-5 3.215480149496536e-5 8.102932964224494e-5 2.59429708717492e-5 -7.05801702383327e-5 -0.00011492836880957505 8.972112312452931e-5 2.231448537035564e-5 8.843038024263435e-6 -4.118688399781353e-5 -6.847949941171162e-5 0.00014521989408653436 -3.5768525656258884e-5 7.57982799155276e-5 -4.883069947705595e-5 9.227182648150354e-5 -5.556344500969972e-5; 0.0001788047728222337 -8.595740762767255e-5 4.3518887487055504e-5 -5.10035653679626e-5 1.4595198826911481e-5 6.17115791282159e-5 -2.4880445654212336e-6 1.839471480697517e-5 -0.0001666311229019886 -7.054863197491358e-5 -9.605295161246097e-5 4.3554186795420376e-5 4.884906122232888e-5 -8.735599947622844e-5 -0.00013988155949856077 0.00029558168063876367 3.1693092022505896e-6 4.365044624039519e-6 0.00010313888089503321 9.81976252841304e-5 -6.160979790375508e-5 -6.0201001524644094e-5 6.172320610848331e-5 1.5346345044138117e-5 7.297765007014515e-5 -9.34921710541929e-5 -0.00021940381810064372 0.00010485344575490261 0.00013447934799832426 2.9692341577077597e-5 -0.00015388680292754295 -0.0001074129260633594; 9.259499190011554e-5 -6.510573596824545e-5 -0.00011919056281112831 -0.0001798575919856354 -4.5996808450579544e-5 5.739668110254711e-5 7.701490382532409e-5 9.059387072557009e-5 3.8464952256620036e-5 -2.410205735942046e-6 0.00010553009475915734 -5.0427655634848736e-5 6.887940823435283e-5 -4.6666311682204055e-6 0.00013026207149596447 1.5504719094673384e-6 -9.645773546435356e-5 -5.589875011719408e-5 7.649500064870904e-6 4.658691312217464e-6 -6.680549425436559e-5 -5.72475416738613e-5 8.714882848653517e-5 2.3411230602505766e-5 5.8694177158031325e-5 -4.7691833726274706e-5 -2.236234989322622e-5 0.00011598173866149494 8.127916796001926e-5 -6.756869855235267e-5 0.0001575150438548003 -0.0001226247638733184; 7.230161065865419e-6 -7.823936615513687e-5 1.663011167929261e-5 5.144969978811536e-5 -1.962425525324092e-5 6.77139056516556e-6 5.636793426908196e-5 0.00019119797358380634 -0.00013998886137176904 1.606625406802696e-5 -1.4982334357607014e-5 0.0001934711719173743 7.382693289638019e-5 -0.0001032466360242308 1.4070838153759424e-5 0.00012881661965602654 2.914458411954784e-6 -0.00011953847282474861 -3.511114558696476e-5 0.0001051305399984046 4.0933526303569726e-7 8.562607687289857e-5 0.00023493299810307987 -7.468762612365338e-6 4.677567191047529e-5 5.709604662352943e-5 -8.13519044071498e-5 1.4335311027525506e-5 -2.636919165288302e-5 0.0002516493506505729 0.00022846966309098243 -5.670363259374977e-6; -0.00010322623530721599 5.74314744773557e-5 3.420033498348136e-5 -6.064959053385313e-5 -0.00010970789118057358 1.8496326436592903e-5 -4.214017281977419e-5 0.00014655057929137297 -0.00020358693837972599 7.310608087952697e-5 8.178558335357818e-6 -1.107643312191331e-5 -5.263082140869325e-6 -9.581522136779949e-5 0.00012992929440604217 -1.465257447473717e-5 5.698520361709121e-5 8.070594229882158e-5 0.0001407303516208887 8.261187939585857e-5 0.0001827403921089855 4.330031332590058e-5 -6.74776001795411e-5 5.371369296715199e-5 6.136155196302875e-6 -5.930261432861301e-5 7.817195181648935e-5 -8.325161435262513e-5 -8.402579807064018e-5 -9.679621690504929e-5 -2.2898211693088538e-5 3.413612465753724e-5; -4.4072358488126677e-5 2.727693516025306e-5 -0.00012988434411564546 -7.629137709233358e-6 1.3359288803979863e-5 8.084754279370136e-5 0.00011925669003065406 -0.00012388045672306327 0.0001012427286503167 4.686502632356618e-5 -7.991388212868908e-5 -0.00010383656137601963 9.121122878164264e-7 4.4783855063351424e-5 1.2266947468372116e-5 -6.156852270688479e-5 0.00010292674904011872 6.448937836356925e-5 3.461233189117473e-5 -0.00011355354456209788 -0.00017832001560346415 4.805997503243641e-5 8.597175235881207e-5 1.9725233304407882e-5 1.4931565036740912e-5 4.862802487689049e-6 1.259990984071333e-5 -4.362997170776093e-6 0.00019778424519135103 0.0001305526651583805 -0.00016771073807941354 0.00011932893301381259; -5.709927596895931e-5 6.976002453897042e-5 -4.054697333653212e-6 8.983905548533516e-5 6.255484744045976e-5 0.00018410919995816448 -0.00010997173546044738 -0.0001632541551457169 -3.078702672197388e-5 -1.9066472182624716e-5 5.8082185327779406e-5 7.39357329935383e-5 0.00010043940540939539 -4.074066770541605e-5 5.722767139972216e-5 4.491243283567798e-5 0.00017947089339444465 2.949391831365315e-5 -0.00021422475571440734 7.159098289279338e-5 -1.7453765272385545e-5 -0.00011124035687813772 6.678728036184703e-6 3.589066376165417e-5 3.343169204030076e-5 -8.375190542707212e-5 -3.79227693569744e-5 -0.00012639787916256964 -0.00013514915359834337 0.00017401332855527857 -3.303660365915068e-5 8.642033038498732e-5; -7.412341185337094e-5 -6.344870891844712e-5 6.713600194124879e-5 -0.00011289959477218185 -4.790545174111004e-5 8.655556919328711e-5 6.383126926885293e-5 -9.813089238669603e-5 -0.00010294273781547644 -6.364865950964125e-5 0.00017590711538723516 -0.00015643682965965787 -0.0001297830324103501 0.0001331626979452203 -8.653254301244565e-5 1.3502781054552336e-5 -0.00025869709609033746 -9.855201026148971e-5 0.0002350493294004946 0.00012223732371857118 4.835872712354812e-5 -9.042918938867306e-5 -6.925489399073837e-5 8.368185697399135e-5 -2.1984105135932033e-5 8.731180312387522e-5 -9.31707411138745e-5 0.0001115003167586214 -8.225367603082014e-5 -3.918657714457458e-5 -2.717359823073503e-5 6.818564613858365e-5; 0.0001067808655180636 3.617809785382184e-5 -1.3142479237017941e-5 0.00011182885942716608 9.850741049476426e-5 4.0602709542414436e-5 -0.0001234211347208741 -2.966781711534049e-6 -0.0001180865990490048 6.189659621767947e-5 0.00012855565297385742 1.9058608232580743e-6 7.544448750295102e-5 -8.45890433437536e-5 -1.605383544202024e-5 -3.0407839166314605e-5 3.1506718424767714e-5 -6.760682002905456e-5 6.692434113674079e-5 -6.797001095859129e-6 -6.181218917745648e-5 5.2515205662615574e-5 2.4941068896711257e-5 1.1779815632594722e-5 0.00016093151067352211 -9.989965177638067e-5 -2.7305472703031354e-5 0.0001441366505433401 -0.00013049858965942363 -7.51231752828128e-5 -0.00019567778212049711 4.45556427267503e-5; 7.866428651364302e-5 6.32438911565705e-5 -7.766378101910443e-5 -5.473899935854614e-5 -3.220181159808311e-5 4.294277251797505e-5 0.00013914318852806308 -3.054164543532417e-5 0.00020955491501122703 2.5672446747600804e-5 0.0001667908524836457 -6.323454571206748e-5 0.00016274307719181842 -4.727274506468832e-6 8.217226578604224e-5 1.3588670037686345e-5 4.16453564980238e-5 -0.00010451314873312255 -9.45968261362887e-5 -5.5029040856920967e-5 -8.80792706880537e-5 3.4127415085499454e-5 -0.00014314539865840407 1.29840879821588e-5 6.095095314999427e-5 -0.00018566395582415434 0.00017133720549505186 -0.00016779079877558183 0.00011038982639770277 1.581587613367513e-5 8.593547109962656e-5 0.00011041131957649595; -6.15509781289179e-5 -2.371005589678155e-5 3.110501333710158e-5 -8.87426942006024e-5 -4.467387057387181e-5 -0.00018469225338206476 4.5775774057141756e-5 7.664139397663322e-5 6.289311920844293e-7 5.2681654639191304e-5 2.7717563940561752e-5 6.71643937005409e-5 0.00011583550790080067 8.689944339739608e-6 8.726982182112667e-5 0.00010652680052432538 -4.448018129307165e-6 -4.321276733547193e-5 7.079981137055318e-5 -4.606644884345945e-5 -7.215483506454989e-5 7.678107837155973e-6 -0.00011783782988798666 -1.1833010970558992e-5 -3.2928459438428037e-6 -3.784997548221448e-5 0.00010475798612448476 4.776120099111439e-5 -0.00010881907669839166 -0.00017345992105558962 9.181749961744681e-6 1.0892452197612986e-5; -4.125140244177275e-5 9.554741277788853e-5 -5.703705814703419e-5 0.00012168565022257519 0.00013969865845247535 4.9815752711950654e-5 -0.00012058257256904253 0.00018792753267298548 -0.00012954630496714832 4.037315680143437e-5 -4.073520962884011e-5 -0.0002007749032116394 -6.23470956696085e-5 0.00013898861230081944 0.0001503809138313988 -0.00014222982703977242 5.741599975699856e-5 -3.381443516154676e-5 -1.8593582942048497e-5 7.731175296903922e-5 -6.777766670602025e-5 -0.00011506097202691341 -0.00012988636867412084 8.711227142448674e-5 0.00011507972099316391 0.00012965579967676906 0.00016975574122007363 -0.00014526048166481781 -6.058333971667482e-6 8.084773012633174e-5 -4.1185969754950685e-5 -4.2322172382044585e-5; -7.872986591987022e-5 -4.075542649060075e-6 3.0084408506686193e-5 0.0001641691096373098 8.084891757594511e-5 -0.0001035493142191105 4.02599661973696e-5 0.0001735965242622671 -0.00010360147555924864 -0.00021277719773219071 -3.509040382789843e-5 -0.0001685397537877528 -0.0001097843078533888 0.00011406989230441648 -0.0002582658590037161 7.602837010186972e-5 -0.00010497978203797484 -0.00025259232919064636 -1.377313768183337e-5 0.00011746924152963018 4.893393358348061e-5 -0.00011597440155322547 -4.022364649439907e-5 0.00013945896804931204 7.999900569108061e-5 2.3505172946768458e-5 3.987898978071384e-5 1.2158763609372763e-5 1.7346322627983663e-6 1.421550311340024e-5 -4.058328252734774e-5 -3.687394112797733e-5; 0.00012325649627986752 -8.800278841713666e-5 5.1881145608285744e-5 -8.500902496378557e-6 -3.902936909194823e-5 0.00010778508918396588 -3.514394224233899e-5 6.967837440119812e-5 -0.00014925738027282776 -6.409001017665436e-6 -7.900188315156385e-5 -6.075009552071298e-6 -0.00018893891059756687 -8.510807778416382e-5 -0.0002972735102424454 0.0001296428371084843 0.00013141082204914985 -0.0001378294556537667 -2.300413793032047e-5 -2.1129574762524213e-5 6.531761651662424e-5 1.6639468284348125e-5 -0.0001362816266385896 6.938423653868645e-5 -9.005421035319855e-6 9.694296646434309e-5 3.210281746434491e-6 0.0001256086096497414 -2.3214537498935782e-6 -8.387657830221172e-5 -2.4466254345817888e-5 -1.0086858438947268e-5; -0.0001020228176547273 4.9523865245785556e-5 2.699052100992957e-5 5.835223209047286e-5 2.798102986176564e-5 -4.315942925117567e-5 3.8029681361716246e-8 4.587335925091984e-6 0.00016051342180631656 3.7195624545485206e-5 1.639269220517682e-5 0.00010761560196456483 8.567858248184638e-5 -2.001790227027188e-5 -4.896239471221916e-5 -9.366654768186745e-5 9.405093232104024e-5 -3.989370298754404e-5 -3.293012042363423e-5 2.896181985320995e-5 0.00012110417964336961 -4.754583488634203e-5 1.4000461933535134e-5 -0.00016584690666590716 2.326074872157995e-5 -1.9213188634097323e-5 -1.7621149081461463e-5 7.68162405530632e-5 2.517803468574368e-6 -6.466089899718503e-5 -1.0204021518664455e-5 0.0002545450167328027; -6.903567596723877e-5 -0.0001340729435990873 -1.176839073864362e-5 2.9078247773798713e-5 0.00013042502500197153 -3.7926242346070636e-5 -0.0001573730191533666 -5.377729072038392e-5 4.89246190615115e-5 -4.606625630672271e-5 -9.16735811709679e-5 0.000271493459461133 -3.603849514306968e-5 -4.169249992387626e-5 3.376901115775297e-6 -2.9137285725802273e-5 3.537555270908735e-5 -2.158749562782075e-5 9.238106524798944e-6 0.00015016324281772885 -2.2446489726786692e-5 -9.795503179461723e-5 2.8684556621232794e-5 -0.000126555060394233 6.0693640573524816e-5 -8.78273426290933e-5 4.171958384417238e-5 -1.6418560751240724e-5 -6.805465961726242e-6 -0.00011119408050703597 2.9030843090957645e-5 1.8334820210617333e-7; -4.052938109692314e-5 3.650328855416765e-5 -3.699593233489282e-5 -0.00017494639703966884 -6.70923743952087e-5 2.570513080842667e-5 -8.510221758353862e-5 3.1298330411454826e-5 6.957945258697942e-6 0.00018017545718737037 8.914053816385922e-5 3.909595963466016e-5 -0.00014975899628792022 -1.8156297103478098e-5 -0.00010648964033192159 -0.00014602142417300407 -0.00013553656552415494 1.484969941311469e-6 4.0629033007800107e-5 -1.619150842328274e-5 0.00011220282087139586 3.8954205790443314e-5 0.00013391915328372128 -3.640806001049519e-6 -7.438505077789741e-7 0.00018591358295213324 -9.022273096218915e-5 0.00018314176324327452 -0.00023236272480301503 -6.269824169305755e-5 9.384919596174805e-5 -4.198428521943258e-5; -1.518455810932338e-5 -5.798360980755921e-5 -3.951662911625602e-5 8.769182201064486e-6 4.263268310401754e-5 0.00017770756930636584 4.13536497909945e-5 -4.080984690866726e-5 -1.648312464007478e-5 -0.00024844842592195744 0.00011901297220553678 0.00010770581393095409 -0.00017503216550161593 -0.00017351198512536705 1.2256607802699788e-5 -5.716106173928534e-6 -0.00017381293328420902 1.1583980082070342e-5 5.149626922153826e-5 2.079504956432839e-5 1.2297855206413276e-5 2.791608536577468e-6 -3.4181871527691104e-5 7.974079195411027e-6 -0.00012086096576589426 1.2209778830000695e-5 1.0893467148122689e-5 -0.00017020287960241854 0.00013477642168815035 1.1156942306269209e-5 -1.6465953380099475e-5 7.838774290227046e-5; -4.700607306205846e-5 0.00010189464236844171 0.00023792486180553065 -5.093670179832914e-5 1.4457105037016319e-5 3.6096524210896426e-5 8.511185305194835e-5 0.00011359160279624002 -7.637327499453164e-5 9.258969666205196e-5 6.324613795384642e-5 -0.000186558230696229 7.14503622404469e-5 -5.424415931685986e-5 -2.171651786540813e-5 -7.043799260629495e-5 1.5600557246378682e-5 8.444988642820928e-5 0.0001871443674573846 4.357957376867349e-5 1.5770916517957122e-5 -0.00017935961651148417 7.08280648921437e-6 -2.8323693102557925e-5 -0.0001789265660662034 1.0163035851454471e-5 -6.20547724864278e-5 -0.00015607460218945574 -7.35820721345783e-5 -7.52907263433453e-6 -2.7338906791393328e-5 1.6308495189313105e-5; -6.631072351210731e-6 -9.728056980273565e-5 -0.00010843256828119107 8.48248476770288e-5 -0.0001001550969292828 -0.000120784663208524 -5.774108799850359e-5 9.178442482627133e-5 0.0001411129654527224 -4.5532100243493665e-5 -4.7734154263514974e-5 -0.00020020072193327254 -5.97766790222351e-5 0.00011712382033257837 -1.802412557915891e-5 7.070398200559466e-5 4.976009799540431e-5 0.00011368091722508094 0.00013957929545105387 0.00024601149411430093 -6.408146660285547e-5 8.403109890897648e-5 2.1469685039137497e-5 -1.1186797250758857e-5 5.214374900353257e-5 2.8657730895447057e-5 3.6031664735875144e-5 -0.00016513047526568432 -1.9894668780382876e-5 4.569056395231418e-5 2.287345918850099e-5 8.645059492229616e-5; 0.00015099225610179688 -2.4770543597649267e-5 1.802422710902495e-5 -5.844411017455058e-5 0.0001237959489008609 -7.447330673307326e-5 1.9526132098720863e-5 1.8616679233721354e-5 3.3311712632531426e-6 9.521720975321858e-5 -1.6092139696373526e-5 0.0001531643768321847 -0.00019020684950368807 6.071049691677367e-5 -0.00013619527700206387 -0.000271161045668077 -1.3712400113416241e-6 6.176461219006574e-5 6.212250926914624e-5 4.6656908387430314e-5 0.0001764359306326329 -0.00019225050592645203 -1.887486060872301e-5 3.200583277071358e-5 7.26590344930193e-5 0.00029059620036428604 0.00013931361653527348 -4.660944133500068e-5 0.00011881491562967438 -0.00010924621411682669 -2.569027739077601e-5 0.00014669699638766593; -8.306632328101407e-5 -0.0001023683838175859 -0.00010878264977151944 1.0274504053016411e-5 5.9222060077375444e-5 -6.188692743678549e-5 -6.211492684457081e-5 8.429836787029453e-5 -4.000126530088606e-5 3.70628437630594e-5 2.1478475606212813e-5 6.244851435979211e-5 -4.648642090985413e-5 0.00011913754467598496 -5.574682585643447e-6 6.390283193403649e-6 -6.202292592275379e-6 0.00011328655425670843 -9.348579289878223e-5 -9.881975377000012e-5 7.515299846629515e-5 6.68259632911311e-5 -7.236325767803948e-6 9.610727618194407e-5 2.897336049434524e-6 1.8457858164697534e-5 -0.00019489891958444493 -5.66533602287155e-5 -2.5290276644879403e-6 -0.0001376983239643472 9.35548411470596e-5 -0.00013061459715033594; 4.060155833816607e-5 -5.253159406616558e-5 -0.00019219913747415868 8.803175069136852e-5 5.787168765767782e-5 8.246075367300295e-5 6.66996543247053e-5 -5.586739143960401e-5 -0.00017423977487975427 -0.00010859675467268655 -1.9458479529340727e-5 -1.7795641271147933e-5 -0.00010353906918804528 -0.0002435614173931366 -2.3845259876325964e-5 -4.339563611334296e-5 -4.3084075970329e-5 -8.711313648869504e-5 7.499661069109096e-5 6.0528987431678714e-5 6.792273552367794e-5 0.00014560098997994396 7.761148799421692e-5 6.288386429006423e-6 0.00010251180856234193 -3.0667945340028774e-5 8.425552868960734e-5 -4.127710281076417e-5 -9.165351593888521e-5 5.393295714261337e-5 -8.955645398724022e-5 2.8143956568457804e-5; 5.477205330950497e-5 -8.861892191837091e-6 -0.00011223391130353264 -0.00010341543612324826 5.1229691499395126e-5 -0.00011648706505096219 -5.084467057621207e-5 -0.000143181265692895 -6.152035576545839e-5 -1.616092739839639e-6 -0.000130073467979856 -5.1540932426197964e-5 -0.0001305819701056209 5.6683672929447346e-5 1.4015788557087311e-5 0.00011063129703960055 6.68587562446739e-6 -3.850794610936901e-5 6.202979920179589e-5 -0.00014279616380830334 -0.00017812875240364232 -0.00011709529872770154 5.754890798104196e-5 -4.065551048281588e-5 0.0001368501884744346 4.422811790362961e-6 -7.295578189681472e-5 5.477841613449996e-5 -3.267943839911859e-5 -0.00017908465316304742 -0.00014754571976392932 9.323867280706797e-6; 9.269487189346004e-5 -2.618372356476986e-5 -7.33027104778914e-5 -9.59534286429811e-5 0.00011571068305972188 3.4159697698360083e-6 0.0002282182626750476 2.872633693374453e-5 -7.501636584305048e-5 5.1092859832407956e-5 -1.2286529015858045e-5 3.188773687912811e-5 -4.208332986121166e-5 7.743505692093341e-5 3.329262970698086e-5 -0.0001189736344876455 2.0002596393774272e-6 0.0001022804949407721 0.0001055723928239431 -1.6675733166205554e-5 2.4691716573276916e-5 -7.525954289843857e-5 6.221998181178271e-5 2.627883212159206e-5 8.114493674116877e-5 -0.00019512833979927957 -1.2114135204621968e-5 -9.185059617989615e-5 8.285053762125476e-5 6.154267019342783e-5 0.0001386020389709868 -0.0001421739784909157; 0.0001612177526708934 0.00011996475374468534 -5.1670857232523515e-6 3.0094084345874527e-5 6.432929664293686e-5 -0.00010438212333104949 -0.0001158899306887374 3.202514350556682e-5 -0.00017575715022252487 -4.8506559628033373e-5 -0.00017295790015343912 7.127679570254816e-5 0.00012073384429237554 6.319092249818121e-7 -1.5339683463423697e-5 -8.613659004771104e-5 -4.5972212623781184e-5 0.0001387429598849538 -0.00010771537048133861 -0.00018245072563735328 0.0002002583438894008 5.722251594824632e-6 -5.354386879338363e-5 4.342315666287051e-5 -6.684668036035325e-5 -6.551886902747087e-5 -3.682794916534408e-5 -0.0002150336570906933 -6.1820332425153e-5 -0.00011429263971370105 0.00021477075066792797 3.739625177837828e-5; 0.00010745912943908956 -2.7405277861392663e-5 0.00015508800636686613 0.00010056397361855904 0.0001479406585794682 -8.209564540297592e-6 3.4906477893829484e-5 1.1085236409568538e-5 -0.00012296462016005123 -7.034597881195884e-5 8.369261407003192e-5 0.000219195741309674 3.7077609093981474e-5 -6.341687337320151e-5 4.985156040580951e-5 -2.5337878169951133e-5 2.7408976465235386e-5 -8.091155606083248e-6 -8.499989569011855e-5 -6.674374678462774e-5 0.00010532445806253661 -0.0002984187209868034 -7.783723928790537e-5 -8.35192164877606e-5 -5.008580657007222e-6 9.436285225970468e-6 -1.3139253197582943e-5 8.62668260460615e-5 -6.0839205317309444e-5 -0.0003968002908286788 3.7570529756491795e-5 3.7966338212724186e-5; 7.497545233891174e-5 -8.133549486556406e-5 -0.0001238934295139225 -7.907139984672902e-6 -1.828114578129351e-5 3.742344201273113e-5 -0.00012604522121874528 -1.9765099164581187e-5 4.9771287780822266e-5 0.00018731333479481377 -8.913963607083694e-5 0.000268095498217369 0.0001433723424674082 0.0001314377492639296 -0.00014196398329259063 -0.00014767377647840782 0.00011116798636467048 1.4855773446831583e-5 4.409911129586912e-5 -0.00021469407725417173 5.880453831596194e-5 -2.1461173997162774e-5 2.0073442687497773e-5 4.2082550539269176e-5 7.689872804294022e-5 0.0001422776746443547 -8.567402469962001e-5 4.928289640191739e-5 -0.0002057876211943977 -0.00015227829349497 9.815591842597925e-5 7.899245033004012e-5; 4.9120218682828204e-5 -0.0001335988639760517 6.487906612225296e-5 -4.5626031714463514e-5 4.868370488172409e-5 4.1883398444651444e-5 1.7112711794341497e-5 -0.00012438806916118922 2.6443750903736937e-5 -2.8253609706856873e-5 5.241413928221098e-5 4.614406825627786e-5 -5.6720179053928855e-5 0.00017464028460441726 -8.353272314646382e-5 3.53524734243057e-5 1.7183092132354373e-5 -5.30490364320329e-5 0.00022973857958337728 6.25667667924703e-5 2.630712606942902e-6 8.875589228470248e-5 -9.779732536056167e-5 -0.00013249844814648188 5.391049453525465e-5 9.385567646010644e-5 -5.222957406048574e-6 -4.027689319568443e-5 2.4505339388876195e-5 3.0047687324133422e-5 -4.844431205068181e-5 3.87183954505158e-6; 1.7043125252157068e-5 -7.769801970071194e-6 -0.0001036500958540363 -4.019565459349063e-5 5.6116531039505625e-5 -1.1043358073593958e-5 3.961414257754879e-5 3.381167551960045e-5 0.00010750834407043774 -3.7334173258941515e-5 -6.2096693105644194e-6 2.4276576721958793e-5 -6.197574395273816e-6 6.289971714831143e-5 -5.6212900391629354e-5 -1.021153467818309e-6 -7.307161068784862e-5 4.335029588072741e-5 5.292282584231554e-5 -0.00018309950756848655 -0.00012126891452827356 0.00013110249200362002 7.912107902337942e-5 0.000127607645834714 -7.442403658534218e-5 -0.0002345087465179044 -1.6720061113674748e-5 7.446529560947235e-5 0.00011498409948070722 2.6592901475296984e-5 -5.795975231340894e-5 -8.223279666353998e-6; 1.3633493381736264e-5 -8.507104985216928e-5 -0.0001832942512006142 5.73402862759623e-5 -3.967130078269736e-5 9.150739015278867e-5 -4.544819648760025e-5 -7.508872704381431e-5 -6.502385076214413e-6 1.0647032923251485e-5 -3.812636032268329e-5 -0.00016447764983201687 7.791764028336453e-5 2.266656656541122e-5 0.00014416342264705792 -1.0217927865206733e-5 0.00012669601665077232 2.2143909064132447e-5 7.545412472648375e-5 1.456921063475007e-5 4.465261326952285e-5 -1.0624695732594732e-5 5.179143363459477e-5 -5.5606946716190986e-5 8.213834325981966e-5 -7.512319325503413e-5 -9.414064015761582e-5 2.2342604552649502e-5 -0.00023895887761643422 -2.5394576823208028e-5 3.913290083840241e-5 1.5354542029333983e-5], bias = [1.2874470112129847e-9, 2.784259265749326e-10, -2.2662798818678674e-10, 7.041926459452476e-10, 4.128344350986865e-9, 6.129886744787956e-10, 9.580424317369544e-10, 6.385876434109405e-10, -1.4417521129076619e-9, 5.268348985585266e-10, 1.878754533724061e-9, -5.5311259725312e-10, 8.688886779493627e-10, -1.8144711656956599e-9, -1.5726449039935901e-9, 1.6640935053288343e-9, -1.2737085460404368e-9, -7.546829080867976e-10, -1.5690637549696826e-9, -8.722765558515411e-11, 1.122630438864066e-9, 2.2239765349672395e-9, -1.335432968392058e-9, -1.3264992616501163e-9, -4.447892058826456e-9, 1.2891924303546494e-9, -1.504608407513894e-9, -5.872343858403378e-10, 7.036254825141248e-10, 1.0910304543974322e-9, -1.6517479773984885e-10, -8.590173591797908e-10]), layer_4 = (weight = [-0.0006556234984295367 -0.0006612110765923384 -0.0006532015399064846 -0.0006495099005237331 -0.0006760566080867875 -0.0005966842414450621 -0.0006694721934364771 -0.0005483326574645804 -0.000832522541669649 -0.0006352007840818512 -0.0006738499269407783 -0.0005155815155884456 -0.0007868176848258587 -0.0008065979812013108 -0.0007506441549356489 -0.0008978079000163388 -0.0007497329060695383 -0.0006535340241734518 -0.0004913680946911727 -0.0006385173325314593 -0.0005947173892095882 -0.000604881432256835 -0.0007141847391026195 -0.0008748426357684999 -0.0006415397817042898 -0.0008634178564472921 -0.0007921061212374032 -0.0006731108389308468 -0.0006037908844002258 -0.0008704217939604213 -0.0008579888445066166 -0.0005893949960003867; 0.0001973562531420948 0.00034035783265589536 0.0002312558554014627 0.0001334281465250506 0.0002261746726499246 0.00010896455916981808 0.00028367092294843553 0.00019929297881634387 0.00043496943991010493 0.00019928420141274224 0.0002938644830701823 0.0003760849581086847 0.00020365150486042517 0.0003472857065879899 5.603286511759712e-5 0.0002301587846980716 0.0002556192110143775 0.0002155058917158115 0.00027440298112615095 0.00011982002189311677 0.00026954837376099155 0.0005464528600230506 8.680050976170833e-5 9.45387081713872e-5 0.0003262353414424768 0.00022600318943857213 0.000385061899075553 8.492258351285086e-5 0.00018622461232794857 0.00015629341105919688 0.00016247613811899638 0.00021122237223289184], bias = [-0.0006685683546190095, 0.0002323859620311792]))Visualizing the Results
Let us now plot the loss over time
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Iteration", ylabel="Loss")
lines!(ax, losses; linewidth=4, alpha=0.75)
scatter!(ax, 1:length(losses), losses; marker=:circle, markersize=12, strokewidth=2)
fig
endFinally let us visualize the results
prob_nn = ODEProblem(ODE_model, u0, tspan, res.u)
soln_nn = Array(solve(prob_nn, RK4(); u0, p=res.u, saveat=tsteps, dt, adaptive=false))
waveform_nn_trained = first(
compute_waveform(dt_data, soln_nn, mass_ratio, ode_model_params)
)
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")
l1 = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
s1 = scatter!(
ax, tsteps, waveform; marker=:circle, alpha=0.5, strokewidth=2, markersize=12
)
l2 = lines!(ax, tsteps, waveform_nn; linewidth=2, alpha=0.75)
s2 = scatter!(
ax, tsteps, waveform_nn; marker=:circle, alpha=0.5, strokewidth=2, markersize=12
)
l3 = lines!(ax, tsteps, waveform_nn_trained; linewidth=2, alpha=0.75)
s3 = scatter!(
ax,
tsteps,
waveform_nn_trained;
marker=:circle,
alpha=0.5,
strokewidth=2,
markersize=12,
)
axislegend(
ax,
[[l1, s1], [l2, s2], [l3, s3]],
["Waveform Data", "Waveform Neural Net (Untrained)", "Waveform Neural Net"];
position=:lb,
)
fig
endAppendix
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
endJulia Version 1.12.7
Commit 6d172b025e4 (2026-08-15 08:05 UTC)
Build Info:
Official https://julialang.org release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 4 × AMD EPYC 9V45 96-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-18.1.7 (ORCJIT, znver5)
GC: Built with stock GC
Threads: 4 default, 1 interactive, 4 GC (on 4 virtual cores)
Environment:
JULIA_DEBUG = Literate
LD_LIBRARY_PATH =
JULIA_NUM_THREADS = 4
JULIA_CPU_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0This page was generated using Literate.jl.