Training a Neural ODE to Model Gravitational Waveforms
This code is adapted from Astroinformatics/ScientificMachineLearning
The code has been minimally adapted from Keith et. al. 2021 which originally used Flux.jl
Package Imports
using Lux,
ComponentArrays,
LineSearches,
OrdinaryDiffEqLowOrderRK,
Optimization,
OptimizationOptimJL,
Printf,
Random,
SciMLSensitivity
using CairoMakieDefine some Utility Functions
Tip
This section can be skipped. It defines functions to simulate the model, however, from a scientific machine learning perspective, isn't super relevant.
We need a very crude 2-body path. Assume the 1-body motion is a Newtonian 2-body position vector
function one2two(path, m₁, m₂)
M = m₁ + m₂
r₁ = m₂ / M .* path
r₂ = -m₁ / M .* path
return r₁, r₂
endNext we define a function to perform the change of variables:
@views function soln2orbit(soln, model_params=nothing)
@assert size(soln, 1) ∈ [2, 4] "size(soln,1) must be either 2 or 4"
if size(soln, 1) == 2
χ = soln[1, :]
ϕ = soln[2, :]
@assert length(model_params) == 3 "model_params must have length 3 when size(soln,2) = 2"
p, M, e = model_params
else
χ = soln[1, :]
ϕ = soln[2, :]
p = soln[3, :]
e = soln[4, :]
end
r = p ./ (1 .+ e .* cos.(χ))
x = r .* cos.(ϕ)
y = r .* sin.(ϕ)
orbit = vcat(x', y')
return orbit
endThis function uses second-order one-sided difference stencils at the endpoints; see https://doi.org/10.1090/S0025-5718-1988-0935077-0
function d_dt(v::AbstractVector, dt)
a = -3 / 2 * v[1] + 2 * v[2] - 1 / 2 * v[3]
b = (v[3:end] .- v[1:(end - 2)]) / 2
c = 3 / 2 * v[end] - 2 * v[end - 1] + 1 / 2 * v[end - 2]
return [a; b; c] / dt
endThis function uses second-order one-sided difference stencils at the endpoints; see https://doi.org/10.1090/S0025-5718-1988-0935077-0
function d2_dt2(v::AbstractVector, dt)
a = 2 * v[1] - 5 * v[2] + 4 * v[3] - v[4]
b = v[1:(end - 2)] .- 2 * v[2:(end - 1)] .+ v[3:end]
c = 2 * v[end] - 5 * v[end - 1] + 4 * v[end - 2] - v[end - 3]
return [a; b; c] / (dt^2)
endNow we define a function to compute the trace-free moment tensor from the orbit
function orbit2tensor(orbit, component, mass=1)
x = orbit[1, :]
y = orbit[2, :]
Ixx = x .^ 2
Iyy = y .^ 2
Ixy = x .* y
trace = Ixx .+ Iyy
if component[1] == 1 && component[2] == 1
tmp = Ixx .- trace ./ 3
elseif component[1] == 2 && component[2] == 2
tmp = Iyy .- trace ./ 3
else
tmp = Ixy
end
return mass .* tmp
end
function h_22_quadrupole_components(dt, orbit, component, mass=1)
mtensor = orbit2tensor(orbit, component, mass)
mtensor_ddot = d2_dt2(mtensor, dt)
return 2 * mtensor_ddot
end
function h_22_quadrupole(dt, orbit, mass=1)
h11 = h_22_quadrupole_components(dt, orbit, (1, 1), mass)
h22 = h_22_quadrupole_components(dt, orbit, (2, 2), mass)
h12 = h_22_quadrupole_components(dt, orbit, (1, 2), mass)
return h11, h12, h22
end
function h_22_strain_one_body(dt::T, orbit) where {T}
h11, h12, h22 = h_22_quadrupole(dt, orbit)
h₊ = h11 - h22
hₓ = T(2) * h12
scaling_const = √(T(π) / 5)
return scaling_const * h₊, -scaling_const * hₓ
end
function h_22_quadrupole_two_body(dt, orbit1, mass1, orbit2, mass2)
h11_1, h12_1, h22_1 = h_22_quadrupole(dt, orbit1, mass1)
h11_2, h12_2, h22_2 = h_22_quadrupole(dt, orbit2, mass2)
h11 = h11_1 + h11_2
h12 = h12_1 + h12_2
h22 = h22_1 + h22_2
return h11, h12, h22
end
function h_22_strain_two_body(dt::T, orbit1, mass1, orbit2, mass2) where {T}
# compute (2,2) mode strain from orbits of BH1 of mass1 and BH2 of mass 2
@assert abs(mass1 + mass2 - 1.0) < 1.0e-12 "Masses do not sum to unity"
h11, h12, h22 = h_22_quadrupole_two_body(dt, orbit1, mass1, orbit2, mass2)
h₊ = h11 - h22
hₓ = T(2) * h12
scaling_const = √(T(π) / 5)
return scaling_const * h₊, -scaling_const * hₓ
end
function compute_waveform(dt::T, soln, mass_ratio, model_params=nothing) where {T}
@assert mass_ratio ≤ 1 "mass_ratio must be <= 1"
@assert mass_ratio ≥ 0 "mass_ratio must be non-negative"
orbit = soln2orbit(soln, model_params)
if mass_ratio > 0
m₂ = inv(T(1) + mass_ratio)
m₁ = mass_ratio * m₂
orbit₁, orbit₂ = one2two(orbit, m₁, m₂)
waveform = h_22_strain_two_body(dt, orbit₁, m₁, orbit₂, m₂)
else
waveform = h_22_strain_one_body(dt, orbit)
end
return waveform
endSimulating the True Model
RelativisticOrbitModel defines system of odes which describes motion of point like particle in Schwarzschild background, uses
where,
function RelativisticOrbitModel(u, (p, M, e), t)
χ, ϕ = u
numer = (p - 2 - 2 * e * cos(χ)) * (1 + e * cos(χ))^2
denom = sqrt((p - 2)^2 - 4 * e^2)
χ̇ = numer * sqrt(p - 6 - 2 * e * cos(χ)) / (M * (p^2) * denom)
ϕ̇ = numer / (M * (p^(3 / 2)) * denom)
return [χ̇, ϕ̇]
end
mass_ratio = 0.0 # test particle
u0 = Float64[π, 0.0] # initial conditions
datasize = 250
tspan = (0.0f0, 6.0f4) # timespace for GW waveform
tsteps = range(tspan[1], tspan[2]; length=datasize) # time at each timestep
dt_data = tsteps[2] - tsteps[1]
dt = 100.0
const ode_model_params = [100.0, 1.0, 0.5]; # p, M, eLet's simulate the true model and plot the results using OrdinaryDiffEq.jl
prob = ODEProblem(RelativisticOrbitModel, u0, tspan, ode_model_params)
soln = Array(solve(prob, RK4(); saveat=tsteps, dt, adaptive=false))
waveform = first(compute_waveform(dt_data, soln, mass_ratio, ode_model_params))
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")
l = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
s = scatter!(ax, tsteps, waveform; marker=:circle, markersize=12, alpha=0.5)
axislegend(ax, [[l, s]], ["Waveform Data"])
fig
endDefining a Neural Network Model
Next, we define the neural network model that takes 1 input (time) and has two outputs. We'll make a function ODE_model that takes the initial conditions, neural network parameters and a time as inputs and returns the derivatives.
It is typically never recommended to use globals but in case you do use them, make sure to mark them as const.
We will deviate from the standard Neural Network initialization and use WeightInitializers.jl,
const nn = Chain(
Base.Fix1(fast_activation, cos),
Dense(1 => 32, cos; init_weight=truncated_normal(; std=1.0e-4), init_bias=zeros32),
Dense(32 => 32, cos; init_weight=truncated_normal(; std=1.0e-4), init_bias=zeros32),
Dense(32 => 2; init_weight=truncated_normal(; std=1.0e-4), init_bias=zeros32),
)
ps, st = Lux.setup(Random.default_rng(), nn)((layer_1 = NamedTuple(), layer_2 = (weight = Float32[0.0003408975; 9.588517f-6; -7.5587886f-6; 3.6867044f-5; 5.658439f-5; 0.00013100529; 1.558933f-5; 0.00019127454; 4.8222806f-5; 1.282878f-5; 3.870084f-5; -4.389879f-5; -0.00013722468; -0.00014278658; -0.00015196919; -0.0001068889; 6.517027f-5; -0.00025143105; 0.00015317238; 4.2008182f-6; -0.00010311469; -2.6292255f-5; 0.00016097332; -0.00013705362; -0.00014839703; -3.4639914f-5; -6.309642f-5; -0.00015577032; 7.645379f-5; -1.3043067f-5; 1.3373255f-5; -4.203037f-5;;], bias = Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), layer_3 = (weight = Float32[1.4312949f-5 -0.00016423331 2.7365091f-5 8.5241714f-5 0.00016539256 0.00025495276 1.3353689f-5 -2.3408837f-5 -8.2642254f-7 0.00014190844 1.7294396f-5 2.3236658f-5 -0.0002350252 0.00023850436 -1.7712791f-5 8.028815f-5 3.7223408f-6 3.204131f-5 -7.873096f-5 -9.328036f-5 -6.590113f-5 -2.8658786f-5 -0.00014137142 -9.568968f-5 -3.8996743f-5 6.190843f-5 -7.978354f-5 -5.231559f-6 -0.00020617324 -2.9564699f-5 -4.321614f-5 -9.951595f-5; -2.0302223f-5 4.143261f-6 -6.219476f-5 5.0101906f-5 0.0002109006 2.6636175f-5 -9.033563f-5 5.30095f-6 -0.00013805051 0.00012935106 -3.8971066f-5 6.630307f-5 -0.000107967746 -0.00012385348 5.8465746f-5 -3.519388f-5 0.000121043944 5.388883f-5 0.00023917221 -6.873091f-5 6.6200264f-5 -3.1021435f-5 -3.055443f-5 6.8014656f-6 2.5323565f-5 -0.00013522254 -4.2865802f-5 6.205318f-5 0.00011349655 -5.2664524f-5 -0.00010611314 -8.7807595f-5; 0.00022273826 7.4884454f-5 -0.0002759887 2.9063825f-5 0.00011327373 6.7716195f-5 0.00010199822 -1.990387f-5 -9.253616f-5 5.01193f-5 7.106522f-5 -6.5570464f-5 -6.235287f-5 0.00013358868 -2.8831416f-6 -3.0244584f-5 -8.34908f-5 -8.88178f-5 -8.58116f-5 5.765341f-5 -2.339993f-5 -1.1753245f-5 -0.00019570545 3.7351503f-5 -0.00011546451 -0.0003139264 1.9874915f-5 -0.00013060906 9.1970665f-5 -4.8956917f-5 -7.8362216f-5 -0.00016720123; -0.00011085721 8.2279286f-5 0.00011757132 1.6976624f-5 3.5129546f-5 -3.3748616f-5 -8.644605f-5 3.691042f-5 5.5642904f-5 -0.00013741864 -0.00013700667 3.834346f-5 -4.8326114f-5 9.2168506f-5 -2.243117f-5 -7.0639246f-5 -8.02217f-5 -9.003218f-6 0.000107687556 -0.00013319262 -2.473832f-5 -1.39530675f-5 -1.2474356f-5 -5.0552982f-5 -0.0001615242 0.00018679255 9.733559f-5 3.8347498f-5 -6.747756f-5 -3.6616635f-5 0.000154568 0.00013818048; -0.00016759196 -7.276709f-5 3.166115f-5 -9.400629f-5 -1.4379692f-6 0.00017753437 3.8851435f-5 -2.4023757f-5 2.0732163f-5 9.4633804f-5 3.7004254f-5 6.7476913f-6 -0.00016703321 7.3526684f-7 -0.00019296486 -5.2198753f-5 -1.4375004f-5 8.1301274f-5 -3.4305365f-5 -0.00010891633 8.6330256f-5 2.6766633f-5 -0.00012990968 -0.00016843442 0.00015807556 -2.6697035f-5 0.00026596567 -8.95835f-6 -6.859496f-6 -0.00017495593 5.556368f-5 0.0001240053; 1.0590917f-5 -0.00020593955 8.4479f-6 0.00011018577 3.9458948f-5 -6.8411755f-5 6.313785f-6 3.9817787f-5 -0.00025018398 3.778762f-5 5.1805546f-5 -0.00013368447 -4.7728397f-5 0.00015863667 -0.00012815953 -0.00010691685 -0.00010899474 -0.00016654478 8.745263f-5 -8.059468f-5 7.3681226f-6 -2.7314098f-5 6.934999f-5 5.6424746f-5 0.00010620131 -6.5416396f-5 -2.0219994f-5 -5.765975f-5 -0.00024750913 -6.849772f-5 -8.1214974f-5 8.3871186f-5; 2.9330577f-5 1.9917767f-5 -5.503175f-5 -5.8357302f-5 0.00020061927 5.1326864f-5 1.1243968f-5 -7.4285155f-5 6.420257f-5 -3.773127f-5 -0.00016378098 -0.00010297794 5.8763115f-5 0.00011095479 0.00026250348 -0.00016945497 -0.00023827673 0.00020761965 5.4364675f-5 4.0375904f-5 8.717823f-6 5.037518f-5 1.14503855f-5 -3.835111f-5 0.00027895806 0.000107742584 0.00016066564 -0.00017468754 -8.859432f-5 -4.2226475f-5 1.5609417f-5 -6.9924026f-5; -5.7253965f-5 0.00017582225 -0.000103853854 2.2764869f-5 9.269073f-6 -7.8505036f-5 -0.00012685372 -0.00013936822 8.188173f-5 9.266611f-5 0.00017562223 9.244195f-5 9.736536f-6 -1.6115358f-5 -1.6053831f-5 9.160241f-5 4.0178336f-5 -7.990637f-6 -0.000104188395 -6.3390165f-5 1.0732429f-5 5.0319075f-5 -3.705854f-5 -5.2444226f-5 -8.831436f-5 3.9478182f-5 -2.6675494f-5 -0.00015476078 -0.00018616152 -0.00017407445 0.000108545544 6.4483465f-5; 4.9464892f-5 0.00014221057 3.540581f-5 6.570273f-5 -0.00013204102 -7.310944f-5 1.8873676f-5 5.0479874f-5 -8.121562f-5 -0.0001173974 5.1002728f-5 3.5685996f-5 -0.000101424506 -0.00015413125 0.000118730284 -4.7622827f-5 0.00015951136 -0.00017665816 0.00011259204 -0.00023239305 5.4915267f-6 0.00010864195 3.0111582f-5 -2.2019398f-5 5.3601587f-5 0.000389346 -1.728469f-5 -0.0001056573 -0.00016274351 -9.019271f-5 3.6165166f-5 -0.00010899792; -5.46585f-6 0.00017778057 -2.5869792f-7 -0.00014455408 -0.00016085124 1.8685816f-5 -2.2597067f-5 0.0001682637 -9.742669f-5 -9.709847f-5 -3.7047f-5 2.3323517f-5 2.3298535f-5 -5.4008164f-5 9.055093f-5 7.955123f-5 -0.00011191513 0.000106101055 0.00015586682 9.293283f-5 5.56297f-5 -0.00010621127 -2.0658374f-5 4.286999f-6 -4.4223248f-6 -9.850715f-5 -0.00015115735 0.00011411669 0.00015363246 6.877254f-5 2.036752f-5 6.3822845f-5; -0.00010512996 -0.00018773509 -9.437513f-5 -9.148275f-5 1.3678477f-5 4.5199245f-6 1.8573866f-5 8.0506225f-6 1.877968f-5 2.3251441f-5 -5.162989f-5 4.1832078f-5 5.4963602f-6 -0.00012860174 1.5439839f-5 -0.0001314776 8.054647f-5 0.00013066792 -1.894369f-5 1.3571411f-5 -4.462639f-5 -0.00015510328 7.66786f-5 3.8039525f-5 -6.149753f-5 -0.00021104014 6.304548f-6 -7.1596376f-5 -4.9446604f-5 8.345504f-7 0.00022314629 0.00025531516; -7.229343f-6 2.7222824f-5 5.09293f-5 -1.0695267f-5 6.2808875f-5 -4.605681f-5 4.2323063f-5 7.693944f-6 0.000114866634 8.25511f-5 -0.00017159194 6.880147f-5 2.0048208f-5 -0.00014076893 -0.00013398158 4.1120267f-5 -0.00013743585 4.9631813f-5 9.3939954f-5 7.6180404f-5 0.00015965487 -1.1586632f-5 -2.0509704f-5 -6.251024f-5 -4.414834f-5 -4.9841507f-5 3.9479324f-5 8.8770124f-5 -8.174318f-6 6.71258f-5 -8.899595f-5 7.713799f-5; 7.7500066f-5 0.00010934406 8.947208f-5 -5.501046f-5 8.042043f-5 -1.3496593f-5 0.00017223413 2.9001569f-5 -2.9438843f-5 2.9055833f-5 9.786438f-5 -0.00014856036 -8.872925f-5 2.0984196f-5 -0.00011779838 -0.00016966142 2.8239183f-5 -9.657625f-5 0.00012578594 9.4760435f-5 -5.90738f-5 0.00014593115 -0.00013165927 0.00022205686 0.000104846746 1.578943f-5 -0.00010876161 -4.817776f-5 3.9655428f-5 -3.3786215f-5 -0.00012236135 0.00013680998; -8.203431f-6 4.7444762f-5 -6.1390147f-6 1.772023f-5 0.00017802672 3.8814043f-5 -7.574072f-5 -0.00012658257 8.485407f-5 2.3205835f-6 0.0001373197 -8.002098f-5 -0.00013223007 -0.00012182618 -7.94925f-5 -3.58164f-5 6.9681206f-5 9.04981f-5 -1.7590342f-5 -8.045428f-5 -3.686074f-5 0.00016558093 -0.0001664669 0.00016739772 0.00017477493 0.00018823537 0.000105530235 -4.0881558f-5 -4.635198f-5 -1.238477f-5 0.00010703721 -0.00011996226; 4.4948898f-5 -2.44629f-5 9.372463f-6 0.00012850564 7.188311f-5 7.283078f-5 -4.0446554f-5 -2.4361845f-5 -8.592871f-6 2.0358902f-5 4.978393f-5 -6.095383f-5 6.9283425f-7 -3.1672666f-5 -7.0118156f-5 -2.0417647f-5 -4.1076844f-5 -4.785753f-5 -9.880143f-5 2.3627035f-5 -4.2055173f-5 1.632221f-5 0.00014261226 0.00013266566 6.929001f-6 -4.5729546f-5 3.2110154f-5 7.927975f-5 0.00013082099 -0.0002473624 -0.00012790166 7.9655714f-5; 4.3818356f-5 -0.00014698318 -0.00015100688 -7.12934f-5 -3.6446f-5 5.1972595f-5 0.00018467389 0.00014821341 2.8167215f-5 -0.00018746122 -0.00017924227 0.00015473753 -9.861068f-5 0.00019874179 0.00016385029 8.873239f-5 -4.107296f-5 -3.649333f-5 4.7770576f-5 -9.329239f-5 4.292059f-5 -0.00018092537 3.5471243f-5 0.0002271382 0.00010421629 -4.3154723f-5 3.9953808f-5 0.00010449556 9.133511f-6 -7.3122435f-5 9.221615f-5 7.184519f-5; -7.631425f-5 -7.185064f-6 -4.1593037f-5 9.817358f-5 0.00012651415 -0.00014746627 3.198354f-7 -0.00012240674 4.9401544f-5 -4.5115994f-5 -1.4600527f-5 -7.178937f-6 8.061043f-5 0.00017957615 7.735617f-5 -0.0001271342 0.00020494178 -1.0197696f-5 1.6909587f-5 -3.6866357f-5 0.00015858213 0.00013527331 7.5300355f-5 0.00010308625 -0.00017374929 0.00014144927 -6.9794434f-5 -3.964003f-5 1.9634414f-5 -7.068755f-5 -2.8720875f-5 -6.252947f-6; -7.2228264f-5 1.1162751f-5 -3.2868225f-5 0.00022954174 7.6270495f-5 -5.110049f-5 -0.000109708715 8.200776f-5 -7.741359f-5 -7.9803656f-5 -1.1042274f-6 5.575357f-5 6.588144f-5 7.525665f-5 -1.2007606f-5 -4.8776543f-5 2.3052795f-5 -4.602598f-5 -4.9711034f-6 0.000118631855 0.00017901348 0.00016072199 9.2473754f-5 -9.150679f-5 4.1139498f-5 0.00015636825 -0.00010168189 6.925298f-5 -5.545547f-5 4.0401595f-5 0.00011199934 -0.000112203525; -0.000102826714 -0.0001547092 0.00014495847 -9.071976f-5 -7.632195f-5 -0.00013334413 2.4445393f-5 -0.0001072046 -6.320716f-5 -7.729123f-5 -7.615674f-5 -0.0001988333 -0.00016910225 -3.5977584f-5 3.6255788f-5 -7.824544f-5 8.534133f-6 -0.000108840475 7.843443f-6 -0.00010149536 -8.17239f-5 -1.0414902f-5 4.5968416f-5 -8.9763336f-5 -4.5073916f-6 -9.963447f-5 -0.00011344184 0.0001458412 -5.208258f-5 5.429468f-5 0.00019605084 3.981282f-5; 5.9826376f-5 5.0349612f-5 5.4788684f-6 -2.0530367f-6 -6.323617f-5 -4.862108f-5 -0.00011941415 -4.0640338f-5 -8.583022f-5 0.00013929504 -0.0001143893 -0.00013914974 -0.00014296547 0.00010366485 -7.928279f-5 -0.00020206282 -7.535412f-5 0.00015240347 7.674123f-5 0.00014761461 0.00014422032 3.3488715f-5 1.682675f-5 1.0269651f-5 0.0001310744 3.2633966f-6 -0.00013815472 -4.9644084f-5 -0.000111167974 -7.343229f-5 3.7496746f-5 8.601112f-5; 5.5232693f-5 2.6073021f-5 1.6038683f-5 2.7026574f-5 9.314611f-5 2.6181424f-5 5.6184203f-5 -5.231742f-5 -7.1954666f-5 -2.7529526f-5 0.00021916936 -0.00010245069 -5.539335f-6 -8.825235f-5 -2.4794319f-5 -9.64513f-5 -0.00010328492 3.1663414f-5 -9.347542f-5 0.00014924009 -0.000119355085 0.0001693542 -9.1529204f-5 -5.3843332f-6 7.109199f-5 4.3989512f-5 -0.000102531 -0.0001529021 -9.628087f-5 -0.00010028387 2.0546817f-5 -7.995879f-5; -0.000119721015 -4.299299f-5 0.00017804216 0.00014053317 6.479252f-5 -9.206217f-5 -5.7847476f-5 1.784703f-5 -5.0416816f-6 -1.3141258f-5 -6.380795f-6 8.581876f-7 7.878076f-5 7.521673f-5 -0.00012573435 1.5355612f-5 -1.0463815f-5 -1.8210132f-5 9.384263f-5 1.8299947f-5 0.00014509143 9.248436f-5 -9.002327f-5 5.876017f-5 -0.00013419238 0.0001150451 -8.745645f-6 6.1989062f-6 1.715965f-5 0.0001790666 7.0106886f-5 -5.034078f-5; -0.00018500265 9.7235525f-6 5.972692f-5 -1.0527123f-5 -0.00021889436 -4.145049f-5 -0.00021679487 -0.00017133425 5.6204077f-5 -0.000109645065 5.041278f-5 -3.110386f-5 4.390971f-5 -9.558353f-5 -3.2894655f-5 0.00015825822 -0.00016250087 -3.3592092f-5 -0.00016454923 -2.7657725f-5 9.295586f-5 2.6141892f-5 -0.0001321624 -0.00013439641 -7.236614f-5 0.00014310682 9.876695f-5 -7.102281f-5 5.6551165f-5 1.8286639f-5 -0.0001684476 -0.00012376937; 0.00010999622 0.00019965263 0.00012536332 4.6626283f-5 5.687285f-5 0.00019896086 -6.874966f-6 0.0001863757 6.7552566f-5 -5.047448f-5 -0.00017553549 -2.5968544f-5 7.627189f-5 -1.44438445f-5 0.00010213394 7.34237f-6 -0.000100847414 -0.00016858742 5.367672f-5 5.614477f-5 -3.2723062f-6 0.0001304001 0.00012555752 0.0001677333 -2.1791531f-5 -5.3986427f-5 0.00013869652 3.040996f-5 3.3392023f-6 4.6557554f-5 -0.00014848483 3.8228623f-5; 0.00022994775 4.4293127f-5 -0.00012591756 -4.1733507f-5 -5.249876f-7 8.3213214f-5 3.9648527f-5 -0.00015865029 0.00016916374 -1.2199545f-5 -1.3449696f-6 4.249689f-5 -0.00020813392 -0.00015231388 8.288054f-5 -5.8885023f-5 0.00013201086 -2.0647889f-5 -0.0003267711 -0.000110894136 0.00010208311 2.7059874f-5 0.00013497294 5.721646f-5 -9.2028364f-5 4.6772213f-5 -0.00010365191 0.00018327487 -9.842112f-5 -6.0175935f-5 5.8924634f-5 -4.1692616f-5; 2.365813f-5 -9.0289177f-7 -3.3376473f-6 9.755106f-5 3.855743f-5 2.5284178f-5 -3.1006105f-6 9.885389f-6 -9.1379225f-5 4.074834f-5 2.9916455f-5 0.00012762734 -0.00012639577 -2.8954437f-5 4.6505018f-5 -4.436981f-6 -5.332023f-5 4.0134706f-5 0.0001163185 1.595353f-5 -0.00015209241 -7.1901835f-5 -0.00010128696 -0.00010908914 -6.129624f-5 -0.00010390137 -7.960629f-5 6.825294f-5 -0.0001603811 7.074171f-5 -3.553227f-5 6.484228f-5; 3.1392097f-5 6.5899108f-6 -7.40465f-5 7.40787f-5 -6.753723f-5 2.2722646f-5 0.00015718702 8.9052366f-5 3.081749f-5 4.710547f-6 -2.7446913f-6 5.248134f-5 -6.1003415f-5 -0.00010542652 0.0001081651 0.00015688433 0.00015760375 -0.00012297957 -0.00014618847 -9.315969f-5 -6.534499f-5 4.231428f-6 -5.538033f-5 0.000103207894 -0.00019403773 0.00020444732 5.286338f-5 0.0001816561 5.3409876f-5 0.000120428616 3.1126438f-6 -0.00014910642; -0.00010299324 2.9094188f-5 -0.00011410129 -0.00013854037 3.1414484f-6 -0.00026100106 -0.000105816565 5.214107f-5 -0.00017786559 -0.00013400812 9.577253f-5 0.00016020282 -0.00020699957 8.2247134f-5 8.5247586f-5 2.0168056f-5 1.0130955f-5 -0.00022800219 0.0002578489 -5.7309106f-5 -9.255319f-5 -6.8842214f-6 -0.00017445988 -0.00014112065 4.780876f-6 2.3355635f-5 1.2308216f-5 -3.441737f-5 -0.00013607889 0.0001079759 6.1557395f-5 -0.00010309489; -5.550049f-6 3.990381f-5 7.027656f-5 0.0001526735 9.724347f-5 2.015505f-5 -9.211774f-5 0.00020921604 -0.00017133691 -6.5810396f-5 0.00021196202 0.00014374194 -4.8756738f-5 0.000118726806 -6.196058f-5 6.3991785f-5 0.00010933077 4.198802f-5 8.0595964f-5 -0.00014813675 -2.8137662f-5 9.495032f-5 0.00010679555 0.0001440846 0.000111458576 -4.8946666f-5 9.074365f-5 1.7690863f-5 0.00013222263 8.988761f-5 -0.00012865532 1.6294684f-5; -9.412849f-5 -5.3499363f-5 -0.00012078635 4.0636933f-5 0.00014675684 1.0763701f-5 -3.5245717f-5 -7.055218f-5 0.00012693714 -0.00015543446 0.0001326975 -1.575651f-5 -7.2611074f-5 4.6198147f-5 1.7930213f-5 -1.1584197f-5 -2.4098932f-5 0.00010056959 -0.00016141421 -2.3000353f-5 -2.3644332f-5 -2.3099778f-5 -0.00012094341 -0.00015053609 0.00013671812 0.00011112342 6.0355804f-5 0.00010633275 -5.622791f-5 -0.00010534279 1.113509f-5 6.3310144f-5; -0.00020688299 -0.000114638315 0.00019582285 -0.00012626464 -0.00011377841 9.7522745f-5 4.673038f-5 6.561178f-6 0.0003003753 0.00012956678 -8.0218604f-5 9.545677f-5 0.00018533609 1.7533922f-6 2.8798811f-5 0.00017805505 5.3260743f-7 -1.2940112f-5 7.1418044f-6 3.803458f-5 -0.00029269882 -6.78161f-5 0.00010332184 7.956728f-5 -0.00014949184 1.32291225f-5 0.00013449778 -4.7996575f-5 3.334115f-5 -1.874586f-5 0.00015622331 -5.6772602f-5; -5.5945064f-5 -2.3521872f-5 0.00026331315 -0.00013016301 4.931833f-5 6.065834f-5 5.009691f-5 0.0001488789 -0.000109708155 -3.68177f-5 2.3555293f-5 -0.000115417344 0.00018116242 -9.4495714f-5 -0.00022555131 2.2833827f-5 -2.9076991f-6 7.945719f-5 -6.6351466f-5 -2.0130548f-5 -0.00011717599 9.337776f-5 -5.5996945f-5 3.117818f-5 9.37729f-5 -0.00013128007 -2.8956927f-5 -5.4588403f-5 0.00021197955 -2.0327161f-5 0.00016455082 -6.913506f-5], bias = Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), layer_4 = (weight = Float32[6.4882144f-5 2.4759454f-5 2.5532321f-5 5.1265866f-5 -0.0001849099 2.9254474f-5 0.0001310776 0.000103382 2.976457f-5 9.9066136f-5 4.8375852f-5 4.874356f-5 -3.483619f-5 -3.4981345f-5 -7.3966876f-5 -0.00019371658 6.304425f-5 3.624392f-5 5.864231f-5 -1.3367377f-5 -4.161002f-5 -8.484441f-6 0.00015827575 -9.059459f-6 6.2700434f-5 -7.580367f-5 0.00010702232 4.735187f-5 -0.00018905872 -1.6213156f-5 6.9961425f-6 5.6776258f-5; 0.00010391655 -9.829624f-7 9.739935f-5 2.6735834f-5 5.8385398f-5 -4.0175586f-5 6.794792f-5 -0.00012019132 -0.00014325933 -0.00018738309 0.00016203373 5.685149f-5 -2.9941923f-5 -2.0218926f-5 0.00021557993 -0.00020218192 0.000102134414 -3.5582925f-5 -2.9775723f-5 0.00010097515 3.764004f-5 -3.805045f-5 3.440298f-5 4.373058f-5 -0.00012929144 -0.0001813774 0.00011190267 -3.5943074f-5 -5.5829772f-5 -0.0001237597 -2.5535284f-5 -3.2734777f-5], bias = Float32[0.0, 0.0])), (layer_1 = NamedTuple(), layer_2 = NamedTuple(), layer_3 = NamedTuple(), layer_4 = NamedTuple()))Similar to most deep learning frameworks, Lux defaults to using Float32. However, in this case we need Float64
const params = ComponentArray(f64(ps))
const nn_model = StatefulLuxLayer(nn, nothing, st)StatefulLuxLayer{Val{true}()}(
Chain(
layer_1 = WrappedFunction(Base.Fix1{typeof(LuxLib.API.fast_activation), typeof(cos)}(LuxLib.API.fast_activation, cos)),
layer_2 = Dense(1 => 32, cos), # 64 parameters
layer_3 = Dense(32 => 32, cos), # 1_056 parameters
layer_4 = Dense(32 => 2), # 66 parameters
),
) # Total: 1_186 parameters,
# plus 0 states.Now we define a system of odes which describes motion of point like particle with Newtonian physics, uses
where,
function ODE_model(u, nn_params, t)
χ, ϕ = u
p, M, e = ode_model_params
# In this example we know that `st` is am empty NamedTuple hence we can safely ignore
# it, however, in general, we should use `st` to store the state of the neural network.
y = 1 .+ nn_model([first(u)], nn_params)
numer = (1 + e * cos(χ))^2
denom = M * (p^(3 / 2))
χ̇ = (numer / denom) * y[1]
ϕ̇ = (numer / denom) * y[2]
return [χ̇, ϕ̇]
endLet us now simulate the neural network model and plot the results. We'll use the untrained neural network parameters to simulate the model.
prob_nn = ODEProblem(ODE_model, u0, tspan, params)
soln_nn = Array(solve(prob_nn, RK4(); u0, p=params, saveat=tsteps, dt, adaptive=false))
waveform_nn = first(compute_waveform(dt_data, soln_nn, mass_ratio, ode_model_params))
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")
l1 = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
s1 = scatter!(
ax, tsteps, waveform; marker=:circle, markersize=12, alpha=0.5, strokewidth=2
)
l2 = lines!(ax, tsteps, waveform_nn; linewidth=2, alpha=0.75)
s2 = scatter!(
ax, tsteps, waveform_nn; marker=:circle, markersize=12, alpha=0.5, strokewidth=2
)
axislegend(
ax,
[[l1, s1], [l2, s2]],
["Waveform Data", "Waveform Neural Net (Untrained)"];
position=:lb,
)
fig
endSetting Up for Training the Neural Network
Next, we define the objective (loss) function to be minimized when training the neural differential equations.
const mseloss = MSELoss()
function loss(θ)
pred = Array(solve(prob_nn, RK4(); u0, p=θ, saveat=tsteps, dt, adaptive=false))
pred_waveform = first(compute_waveform(dt_data, pred, mass_ratio, ode_model_params))
return mseloss(pred_waveform, waveform)
endWarmup the loss function
loss(params)0.0007302776007983661Now let us define a callback function to store the loss over time
const losses = Float64[]
function callback(θ, l)
push!(losses, l)
@printf "Training \t Iteration: %5d \t Loss: %.10f\n" θ.iter l
return false
endTraining the Neural Network
Training uses the BFGS optimizers. This seems to give good results because the Newtonian model seems to give a very good initial guess
adtype = Optimization.AutoZygote()
optf = Optimization.OptimizationFunction((x, p) -> loss(x), adtype)
optprob = Optimization.OptimizationProblem(optf, params)
res = Optimization.solve(
optprob,
BFGS(; initial_stepnorm=0.01, linesearch=LineSearches.BackTracking());
callback,
maxiters=1000,
)retcode: Success
u: ComponentVector{Float64}(layer_1 = Float64[], layer_2 = (weight = [0.0003408975026101928; 9.588517059439818e-6; -7.558788638550984e-6; 3.686704440036965e-5; 5.658439113167257e-5; 0.00013100529031336749; 1.5589330359929346e-5; 0.0001912745356088529; 4.822280607178587e-5; 1.2828780199925254e-5; 3.870084037776231e-5; -4.3898791773185975e-5; -0.00013722467701847386; -0.00014278657909010548; -0.00015196918684515372; -0.00010688890324666012; 6.517027213696115e-5; -0.00025143104721762183; 0.00015317238285192063; 4.2008182390389545e-6; -0.00010311468940919812; -2.629225491542421e-5; 0.00016097331535966264; -0.00013705361925482653; -0.00014839702635065195; -3.463991379244555e-5; -6.309642049011164e-5; -0.0001557703217257067; 7.645379082513084e-5; -1.3043067156092321e-5; 1.3373254660118636e-5; -4.203036951366147e-5;;], bias = [3.008821426460242e-16, 1.7911226528370474e-17, -1.2513273591963364e-17, 6.169133323837949e-17, 1.0380870701357692e-16, 2.930466995342326e-16, 1.6631990088489206e-17, 6.379895080255005e-16, 8.659969045221372e-17, -6.160995874736102e-19, -5.382433738562075e-17, -6.768205188320569e-17, -3.94748821419937e-16, -7.729687080997708e-17, -9.119975738990878e-17, -7.844935927900683e-17, 8.767353503830537e-17, -3.4156912241678667e-16, 1.6177177195343332e-16, 3.228111115786596e-18, -7.938993292539713e-17, -4.1653384747117544e-17, 4.1194716825357904e-16, -5.56373626386643e-16, -8.117816398820033e-17, -4.881462207249965e-17, -8.190070036080159e-17, -1.823707220735968e-16, 2.2469128868091173e-16, -1.3837727062453166e-17, -1.0518147691545537e-17, -7.593827630912322e-18]), layer_3 = (weight = [1.431172866448196e-5 -0.0001642345286380786 2.7363870758674328e-5 8.524049379059094e-5 0.00016539133692287895 0.0002549515357533252 1.33524682315712e-5 -2.3410057037710334e-5 -8.276430047561992e-7 0.00014190722329669565 1.7293175988127343e-5 2.3235437838705288e-5 -0.00023502642390655607 0.00023850314217282266 -1.7714011773812496e-5 8.028692725940989e-5 3.721120338126579e-6 3.204008869956831e-5 -7.873218230076203e-5 -9.328158004780371e-5 -6.590234799648285e-5 -2.866000606434466e-5 -0.00014137264035513598 -9.569089699641128e-5 -3.8997963128957304e-5 6.190721076575082e-5 -7.978475870902564e-5 -5.232779512759676e-6 -0.00020617445799860014 -2.956591917598221e-5 -4.321736007102831e-5 -9.95171703106072e-5; -2.0301937440674072e-5 4.143546191344111e-6 -6.219447667025651e-5 5.010219112374187e-5 0.00021090088713900465 2.6636460204470554e-5 -9.033534141271067e-5 5.301235176017215e-6 -0.00013805022743544252 0.00012935134301879752 -3.8970780481022606e-5 6.630335857845204e-5 -0.00010796746053746768 -0.00012385318976375156 5.8466031730501946e-5 -3.519359441487845e-5 0.00012104422941557402 5.388911605914778e-5 0.00023917249356382618 -6.873062219018738e-5 6.620054929736281e-5 -3.1021149227299476e-5 -3.0554145525814265e-5 6.801750912571057e-6 2.5323850166030363e-5 -0.0001352222518857073 -4.286551691838266e-5 6.20534644600478e-5 0.00011349683492663571 -5.26642382123011e-5 -0.00010611285532139863 -8.780730993966278e-5; 0.00022273480172816678 7.488099109683088e-5 -0.00027599217682076725 2.906036244017583e-5 0.00011327026769355194 6.771273220727192e-5 0.00010199475983459422 -1.9907332525324707e-5 -9.253962306469525e-5 5.011583599330767e-5 7.10617536336562e-5 -6.557392734033877e-5 -6.235633153630257e-5 0.00013358521428961644 -2.8866046155732638e-6 -3.0248047177910127e-5 -8.349426169810444e-5 -8.882125947781307e-5 -8.581505934636353e-5 5.764994822162097e-5 -2.3403393779181164e-5 -1.1756708399518693e-5 -0.00019570891125987247 3.734804014164179e-5 -0.00011546796994445322 -0.0003139298638322016 1.9871451886830557e-5 -0.0001306125191964504 9.196720239775808e-5 -4.896038043593145e-5 -7.836567928247242e-5 -0.00016720469247015097; -0.00011085737496529288 8.227912185332531e-5 0.00011757115426092202 1.6976460029518105e-5 3.51293812863603e-5 -3.3748780696504964e-5 -8.64462170275787e-5 3.691025558594605e-5 5.564273960856528e-5 -0.00013741880393269677 -0.00013700683921258162 3.8343295455824495e-5 -4.8326278191843196e-5 9.216834150805041e-5 -2.2431334217862122e-5 -7.063941011353542e-5 -8.022186084333061e-5 -9.003382546385818e-6 0.00010768739157440993 -0.00013319278223122675 -2.4738483988648837e-5 -1.3953231972728088e-5 -1.2474520007407498e-5 -5.0553146865303754e-5 -0.00016152437165062035 0.00018679238128327565 9.73354282913804e-5 3.8347333612301566e-5 -6.747772276782608e-5 -3.6616799478970645e-5 0.0001545678343981577 0.0001381803114790543; -0.00016759293912564912 -7.276806836839348e-5 3.166017513481383e-5 -9.400726530825263e-5 -1.4389453798198587e-6 0.00017753339370588805 3.885045834809031e-5 -2.4024733012842884e-5 2.073118657622508e-5 9.463282794800075e-5 3.700327824676871e-5 6.746715096735953e-6 -0.00016703418923665682 7.342906752997716e-7 -0.00019296583314084222 -5.219972927247325e-5 -1.4375980119299237e-5 8.130029773217279e-5 -3.43063414237252e-5 -0.00010891730317039 8.632927956401193e-5 2.6765656637646884e-5 -0.00012991065277841564 -0.0001684353931539963 0.0001580745872144327 -2.6698010789775432e-5 0.00026596469860865345 -8.959326170806015e-6 -6.860472059857376e-6 -0.000174956906723171 5.556270361753599e-5 0.00012400431905748794; 1.0586703662903226e-5 -0.00020594376130520162 8.443686762808021e-6 0.00011018155762668145 3.9454734885121484e-5 -6.84159678921501e-5 6.309571997013327e-6 3.981357420076237e-5 -0.00025018819016894747 3.77834074033725e-5 5.1801332826478645e-5 -0.00013368868551241707 -4.773261045826618e-5 0.00015863245769381714 -0.00012816374530571877 -0.00010692106318217806 -0.00010899895298551926 -0.00016654899102186308 8.744841741376242e-5 -8.059888952902321e-5 7.363909641773406e-6 -2.7318310661024346e-5 6.934577666201649e-5 5.642053272552762e-5 0.00010619709771803067 -6.542060893752255e-5 -2.02242065124559e-5 -5.766396154554767e-5 -0.0002475133444231555 -6.850193333138494e-5 -8.121918674350497e-5 8.386697279730315e-5; 2.933242843255989e-5 1.9919618473599134e-5 -5.5029897218824875e-5 -5.8355450043943145e-5 0.00020062111810382864 5.1328716047075935e-5 1.1245820253483328e-5 -7.42833034268554e-5 6.420442162614778e-5 -3.772941988010296e-5 -0.0001637791245843046 -0.00010297608593411255 5.8764966645477626e-5 0.0001109566402488512 0.00026250533495164623 -0.00016945312005866404 -0.00023827487518012175 0.00020762150609929632 5.436652657642151e-5 4.037775591549135e-5 8.719675072440515e-6 5.03770317050871e-5 1.1452237351992844e-5 -3.8349258709266475e-5 0.00027895991309612276 0.00010774443592943782 0.00016066749606814598 -0.0001746856831845727 -8.859247088132174e-5 -4.2224622909467315e-5 1.5611269311724573e-5 -6.992217446842664e-5; -5.725555299782984e-5 0.00017582066150141637 -0.00010385544217199332 2.2763280616414825e-5 9.267485187662876e-6 -7.850662430056666e-5 -0.00012685530443916332 -0.00013936980601638783 8.188013973638287e-5 9.266452045637875e-5 0.0001756206454266032 9.24403627542019e-5 9.734948183982866e-6 -1.6116946677278167e-5 -1.6055419360701836e-5 9.16008191089366e-5 4.017674811991834e-5 -7.992224960515967e-6 -0.0001041899834271238 -6.339175367968403e-5 1.0730840334946499e-5 5.0317486715426425e-5 -3.706012855755943e-5 -5.2445813958833254e-5 -8.831594666472682e-5 3.947659363264034e-5 -2.6677082383614452e-5 -0.00015476237083558855 -0.00018616310469580762 -0.00017407603652457016 0.00010854395597932623 6.44818771653048e-5; 4.946420861119616e-5 0.00014220988716871907 3.540512673382144e-5 6.570204325427758e-5 -0.0001320417063409692 -7.311012402467552e-5 1.8872992338146698e-5 5.0479190146441855e-5 -8.121630478242865e-5 -0.00011739808494283387 5.1002044098568104e-5 3.5685312947607655e-5 -0.00010142518974432809 -0.0001541319356631679 0.00011872960069624717 -4.7623510429982726e-5 0.00015951067373564057 -0.00017665883885753373 0.00011259135547036323 -0.00023239373647200446 5.4908432746034865e-6 0.00010864126718544844 3.0110898048156657e-5 -2.2020081335791322e-5 5.360090338101596e-5 0.0003893453254265477 -1.7285373790654778e-5 -0.00010565798536233569 -0.00016274419565277546 -9.019339583872387e-5 3.6164482050225726e-5 -0.00010899860305769834; -5.464528696541625e-6 0.00017778189490663077 -2.573767388718037e-7 -0.00014455276263298334 -0.00016084991815851947 1.8687137215132095e-5 -2.2595745887982685e-5 0.00016826501510683943 -9.742536653185169e-5 -9.709714808387494e-5 -3.704567947544028e-5 2.332483805538732e-5 2.3299856054911808e-5 -5.400684253083288e-5 9.05522507836218e-5 7.955255264995324e-5 -0.0001119138063059932 0.00010610237646493419 0.0001558681366603927 9.293415192395642e-5 5.5631019365943226e-5 -0.00010620995030158966 -2.065705242977925e-5 4.288319958398091e-6 -4.4210035795176075e-6 -9.850583168564221e-5 -0.00015115602886190617 0.00011411800793019213 0.00015363377738857585 6.877386172881538e-5 2.0368842031431386e-5 6.382416600708039e-5; -0.00010513176624090221 -0.0001877368917399287 -9.437693318484502e-5 -9.148455085830836e-5 1.3676673524447614e-5 4.518120540010734e-6 1.857206240804262e-5 8.048818535324242e-6 1.8777875602086845e-5 2.324963731500643e-5 -5.16316935596249e-5 4.1830273921209924e-5 5.494556325571891e-6 -0.00012860353984302722 1.5438034615216777e-5 -0.00013147940574578416 8.054466940796101e-5 0.00013066611479140673 -1.8945494439831718e-5 1.3569606898658322e-5 -4.46281950922356e-5 -0.00015510508040331416 7.667679944410435e-5 3.8037721032593666e-5 -6.149933272405312e-5 -0.00021104194404922508 6.302744231494965e-6 -7.159817999359804e-5 -4.944840787227073e-5 8.327464860707092e-7 0.0002231444819421307 0.0002553133531995858; -7.2283388272155905e-6 2.722382842877191e-5 5.0930302595601064e-5 -1.0694262609819732e-5 6.280987864557794e-5 -4.605580769747811e-5 4.2324067098901186e-5 7.694948099015532e-6 0.0001148676382490637 8.255210551398114e-5 -0.00017159093121334718 6.880247375194167e-5 2.0049212171222093e-5 -0.00014076792293962512 -0.00013398057498306546 4.112127126973926e-5 -0.00013743485041230814 4.963281745403111e-5 9.394095824064163e-5 7.618140799423923e-5 0.00015965587218489394 -1.1585628353702002e-5 -2.0508699972193198e-5 -6.250923624065142e-5 -4.414733492921583e-5 -4.9840502551875816e-5 3.948032818699371e-5 8.877112842128994e-5 -8.173314281510579e-6 6.712680616148056e-5 -8.899494431715219e-5 7.713899677584136e-5; 7.750173317517014e-5 0.00010934572564925656 8.947374662872063e-5 -5.5008792697698246e-5 8.042209883475167e-5 -1.3494926134670283e-5 0.00017223579270554356 2.900323561238126e-5 -2.943717590918761e-5 2.9057499704287667e-5 9.786604714368985e-5 -0.00014855869012630059 -8.872758117017765e-5 2.098586337428957e-5 -0.00011779671258505164 -0.00016965975301085995 2.82408498596483e-5 -9.657458504177682e-5 0.00012578760724768292 9.476210179760715e-5 -5.90721312628778e-5 0.00014593281302855904 -0.00013165760101497054 0.00022205852364909405 0.00010484841259217218 1.5791097029215373e-5 -0.00010875994412439496 -4.8176093697340964e-5 3.9657094747743556e-5 -3.378454785073521e-5 -0.00012235968388362204 0.00013681164345958767; -8.201800651919584e-6 4.7446393013689544e-5 -6.13738410303845e-6 1.77218597078362e-5 0.00017802834992342446 3.881567391960892e-5 -7.57390873199424e-5 -0.00012658093981866823 8.48557036211608e-5 2.3222140176418035e-6 0.00013732133158062497 -8.001935128542705e-5 -0.00013222843625538132 -0.00012182455173881885 -7.949086938008971e-5 -3.581476920355727e-5 6.968283634212566e-5 9.0499729426069e-5 -1.7588711839773214e-5 -8.04526491132659e-5 -3.6859109227793443e-5 0.00016558256248989098 -0.00016646526563402723 0.0001673993545542277 0.0001747765643945684 0.00018823699866932746 0.00010553186513296108 -4.08799270966779e-5 -4.6350348552934286e-5 -1.2383139788124406e-5 0.00010703883964613993 -0.0001199606260210379; 4.494935317993585e-5 -2.446244390524156e-5 9.372918141133387e-6 0.00012850609026133366 7.188356219773437e-5 7.283123384910682e-5 -4.0446098426065196e-5 -2.436138994893304e-5 -8.592416195489573e-6 2.0359357661044444e-5 4.978438344766667e-5 -6.0953374047705025e-5 6.932894346988827e-7 -3.16722103140224e-5 -7.011770095926752e-5 -2.0417191551708226e-5 -4.1076389168330866e-5 -4.7857075985704476e-5 -9.880097378994387e-5 2.362749044766022e-5 -4.205471806711374e-5 1.6322665471642415e-5 0.0001426127168837088 0.00013266611902724044 6.929456032926414e-6 -4.5729091214190174e-5 3.211060946556983e-5 7.92802061208611e-5 0.00013082144549331844 -0.0002473619347416672 -0.00012790120264765507 7.965616940270456e-5; 4.3820417528481946e-5 -0.00014698111535230856 -0.00015100482177635857 -7.129133645626077e-5 -3.644393781846805e-5 5.197465598634018e-5 0.00018467595019196873 0.00014821547175108583 2.8169276089692633e-5 -0.0001874591593089907 -0.00017924020848417957 0.0001547395901131038 -9.860862100446205e-5 0.0001987438541679113 0.00016385234867505837 8.873445277153476e-5 -4.107089752628955e-5 -3.649126792278876e-5 4.777263766903511e-5 -9.329032526998053e-5 4.29226516704542e-5 -0.0001809233121033224 3.547330434833568e-5 0.00022714025781773425 0.00010421835268665283 -4.315266177704839e-5 3.9955869039338985e-5 0.0001044976184917989 9.135572878810213e-6 -7.312037395727102e-5 9.221821038104381e-5 7.184725346328369e-5; -7.631237754594158e-5 -7.183189113599618e-6 -4.1591162203000504e-5 9.817545428184748e-5 0.00012651602223293692 -0.00014746439633647992 3.217103722664788e-7 -0.0001224048632638563 4.9403418911498854e-5 -4.5114119034148466e-5 -1.45986518463521e-5 -7.177061847794898e-6 8.061230395930637e-5 0.00017957802405823677 7.735804460486884e-5 -0.00012713232941216518 0.00020494365666771081 -1.0195820736916409e-5 1.6911462394152757e-5 -3.686448185809849e-5 0.0001585840050621083 0.00013527518380677324 7.530223005693721e-5 0.00010308812265527255 -0.0001737474107857738 0.00014145114759692889 -6.979255927482824e-5 -3.963815329819502e-5 1.96362885117918e-5 -7.068567579600933e-5 -2.871900001529577e-5 -6.251072095363167e-6; -7.222531988608318e-5 1.116569543731449e-5 -3.286528138639119e-5 0.0002295446801922766 7.627343890410834e-5 -5.109754696103234e-5 -0.00010970577116901717 8.201070610584271e-5 -7.74106474950437e-5 -7.980071225980952e-5 -1.101283405176919e-6 5.575651536267358e-5 6.588438278914779e-5 7.525959241821632e-5 -1.2004661616166708e-5 -4.8773598823099053e-5 2.305573886181113e-5 -4.602303582444965e-5 -4.96815940504522e-6 0.00011863479900560631 0.00017901642874269022 0.0001607249332351167 9.247669819661089e-5 -9.150384269968252e-5 4.1142441628389505e-5 0.0001563711893736868 -0.00010167894928096883 6.925592330436051e-5 -5.5452527735245305e-5 4.040453947303161e-5 0.00011200228342707543 -0.00011220058061960031; -0.00010283235628157065 -0.0001547148380663435 0.00014495282499099113 -9.072540291838145e-5 -7.632759247227963e-5 -0.0001333497743356849 2.4439751138356488e-5 -0.00010721024177051001 -6.321280256399503e-5 -7.729687371777388e-5 -7.61623845786951e-5 -0.0001988389371430657 -0.00016910789003612788 -3.598322604683258e-5 3.625014556439128e-5 -7.825107917906529e-5 8.528490576811938e-6 -0.00010884611714873083 7.8378012009321e-6 -0.00010150100155754061 -8.17295398914251e-5 -1.0420544134307787e-5 4.596277408984542e-5 -8.976897828996505e-5 -4.513033621498257e-6 -9.964010991409246e-5 -0.00011344748367575243 0.0001458355587207035 -5.208822353180404e-5 5.428903801346182e-5 0.00019604519598631173 3.980717934298606e-5; 5.9825172940141934e-5 5.0348409132626195e-5 5.477665382701016e-6 -2.0542397293444034e-6 -6.323737281368521e-5 -4.8622284083327706e-5 -0.00011941535441897401 -4.0641540834838746e-5 -8.583142582088219e-5 0.000139293833200414 -0.0001143905053310423 -0.00013915093833807382 -0.00014296666878967337 0.00010366364338674853 -7.928399529068354e-5 -0.00020206401924829082 -7.535532014903979e-5 0.00015240227119772793 7.674002452527565e-5 0.00014761341051903012 0.00014421911808435692 3.348751241114416e-5 1.682554764680121e-5 1.0268448278433385e-5 0.00013107319435344412 3.2621935873488247e-6 -0.00013815592203050297 -4.964528736185791e-5 -0.0001111691769607675 -7.343349236057664e-5 3.749554278451423e-5 8.600991466787181e-5; 5.523098121108003e-5 2.6071309598581823e-5 1.6036970969242912e-5 2.7024862042698747e-5 9.314439926583025e-5 2.6179712272095703e-5 5.61824909300311e-5 -5.231913195983794e-5 -7.195637767382311e-5 -2.7531237669801345e-5 0.0002191676522261238 -0.00010245240441237012 -5.54104639832049e-6 -8.825406434425417e-5 -2.479603057963421e-5 -9.645301356158985e-5 -0.00010328662933262302 3.166170250099425e-5 -9.347713234546686e-5 0.00014923837994046038 -0.0001193567968084651 0.0001693524882784815 -9.153091554707802e-5 -5.386044854276163e-6 7.109028052127748e-5 4.3987800431741054e-5 -0.0001025327091565567 -0.00015290380719761417 -9.628258153043834e-5 -0.00010028558057912161 2.0545105194277135e-5 -7.99604985741191e-5; -0.00011971851249063535 -4.2990487184757204e-5 0.00017804466138532406 0.00014053567379651507 6.47950214048265e-5 -9.205966538812479e-5 -5.7844973110509995e-5 1.784953333896883e-5 -5.0391791259222964e-6 -1.3138755587893435e-5 -6.37829274946573e-6 8.606900397377939e-7 7.878326092085222e-5 7.521923494694977e-5 -0.00012573185135208065 1.53581144846559e-5 -1.0461312325241479e-5 -1.820762967641012e-5 9.384513346647815e-5 1.8302448967555755e-5 0.0001450939302745563 9.248686497503858e-5 -9.002076744165499e-5 5.87626741040254e-5 -0.00013418987559217868 0.00011504760137231948 -8.743142228376338e-6 6.201408683939599e-6 1.716215179530916e-5 0.00017906910166548575 7.010938812218614e-5 -5.033827669048801e-5; -0.00018500865337273275 9.71754468742059e-6 5.972091310218435e-5 -1.0533130646531797e-5 -0.0002189003695300394 -4.1456498022591005e-5 -0.00021680087740855728 -0.00017134025767761153 5.619806904838389e-5 -0.0001096510728737012 5.040677058537299e-5 -3.110986801652347e-5 4.390370156879e-5 -9.558953536061802e-5 -3.290066308427308e-5 0.00015825221123784663 -0.00016250688133617516 -3.359810000130238e-5 -0.0001645552380275131 -2.7663732564859244e-5 9.294985136627878e-5 2.6135884015505434e-5 -0.00013216840818376364 -0.00013440241820963648 -7.237215106461113e-5 0.000143100815309712 9.876094043398569e-5 -7.10288201141305e-5 5.654515769243896e-5 1.8280631026984343e-5 -0.00016845360880574661 -0.00012377537279712177; 0.00011000125091627175 0.00019965765973443584 0.00012536835715996525 4.663131626562422e-5 5.687788366462939e-5 0.0001989658907882816 -6.869932392368385e-6 0.0001863807396625727 6.755759973434776e-5 -5.04694449295529e-5 -0.00017553045613335257 -2.5963510367953017e-5 7.627692544021768e-5 -1.4438810965184969e-5 0.00010213897480981782 7.347403382060052e-6 -0.00010084238014992596 -0.00016858238227319633 5.368175193554085e-5 5.6149804052034935e-5 -3.267272675168743e-6 0.00013040513881838267 0.00012556255246864526 0.00016773833927120218 -2.1786497698015804e-5 -5.398139323683224e-5 0.00013870154948794993 3.0414992991165948e-5 3.3442358494956634e-6 4.656258757000259e-5 -0.0001484797990519327 3.823365700787073e-5; 0.00022994697596084672 4.4292353632815565e-5 -0.00012591833561523594 -4.1734280020275383e-5 -5.257608419486848e-7 8.321244044132254e-5 3.964775336575922e-5 -0.0001586510644839497 0.00016916297121574454 -1.2200318596253705e-5 -1.345742847799873e-6 4.2496116148729714e-5 -0.00020813469214694814 -0.00015231465127648984 8.287976910728832e-5 -5.888579667875966e-5 0.00013201008354449042 -2.064866217473112e-5 -0.00032677188770244026 -0.00011089490922963956 0.00010208233819944343 2.705910068548058e-5 0.00013497216954497557 5.721568763042562e-5 -9.202913693947874e-5 4.677143973899326e-5 -0.00010365268295495449 0.0001832740943799344 -9.842189516271096e-5 -6.017670799260733e-5 5.892386056152399e-5 -4.169338913848358e-5; 2.3656569340914534e-5 -9.044533148336356e-7 -3.3392088302572564e-6 9.754950139136835e-5 3.855586970854493e-5 2.528261671938654e-5 -3.102172000981268e-6 9.883827318463103e-6 -9.138078615836902e-5 4.074677696149583e-5 2.9914893333225934e-5 0.00012762577813515315 -0.0001263973289828115 -2.895599840160575e-5 4.650345641818851e-5 -4.438542361125471e-6 -5.3321790743609934e-5 4.01331445242205e-5 0.00011631693911437917 1.5951968692715885e-5 -0.0001520939740079796 -7.19033968711663e-5 -0.00010128852312520914 -0.00010909069982192677 -6.129780098165418e-5 -0.00010390293476706528 -7.960785018116253e-5 6.825138103847473e-5 -0.00016038265761056841 7.074015144422439e-5 -3.5533831520697356e-5 6.484071769915391e-5; 3.139413802874498e-5 6.5919514842742256e-6 -7.4044455862627e-5 7.408074071162796e-5 -6.753518773012502e-5 2.2724687193369647e-5 0.0001571890641632259 8.905440682307752e-5 3.0819530190177044e-5 4.71258754868788e-6 -2.742650570604615e-6 5.248338112023105e-5 -6.100137415461989e-5 -0.00010542448234748017 0.00010816713966658237 0.00015688636977455322 0.00015760578735961802 -0.00012297753364087059 -0.0001461864268943121 -9.315764709145258e-5 -6.534294897693413e-5 4.233468923025007e-6 -5.537828855569762e-5 0.0001032099351565931 -0.0001940356853022615 0.000204449362906138 5.2865419826677616e-5 0.0001816581387235393 5.341191700307769e-5 0.0001204306570859396 3.11468448371189e-6 -0.00014910437693588978; -0.00010299839281305515 2.908903430542687e-5 -0.00011410644450771386 -0.00013854552757075532 3.1362946716376733e-6 -0.00026100620974864504 -0.00010582171902606928 5.2135915629750413e-5 -0.00017787073969921413 -0.00013401327536522156 9.576737416827601e-5 0.00016019766686403746 -0.00020700472172427328 8.224198014398029e-5 8.524243226896415e-5 2.016290223729147e-5 1.0125801472531256e-5 -0.00022800734090223327 0.0002578437538064781 -5.731425930244616e-5 -9.255834678732643e-5 -6.889375108955479e-6 -0.000174465038562958 -0.00014112580041944394 4.775722536244055e-6 2.335048107815405e-5 1.2303062666977866e-5 -3.442252504133979e-5 -0.0001360840420060391 0.00010797074852197686 6.155224169523712e-5 -0.00010310004521707477; -5.5444548458237e-6 3.9909405065893534e-5 7.028215525458328e-5 0.00015267909377832418 9.7249066063856e-5 2.0160644387522437e-5 -9.211214735153404e-5 0.0002092216372398058 -0.00017133131866324417 -6.580480209998114e-5 0.00021196761274729352 0.000143747535664778 -4.8751143433460585e-5 0.00011873240048547027 -6.195498378699459e-5 6.399737951828634e-5 0.0001093363670657381 4.199361403829686e-5 8.060155862724527e-5 -0.000148131156558909 -2.813206726609582e-5 9.495591705451363e-5 0.00010680114694652747 0.00014409018961262935 0.00011146416993827012 -4.894107139301004e-5 9.074924212433368e-5 1.7696457623536414e-5 0.00013222822501808034 8.989320388315916e-5 -0.00012864972183927224 1.6300277754970882e-5; -9.412936374438403e-5 -5.350023816585526e-5 -0.00012078722825403311 4.063605779070458e-5 0.00014675596143921005 1.076282573909534e-5 -3.524659218978118e-5 -7.055305645929527e-5 0.00012693626745008938 -0.00015543533841277455 0.00013269662854132298 -1.575738492842231e-5 -7.261194873730104e-5 4.619727228406477e-5 1.79293383988479e-5 -1.1585071617573386e-5 -2.409980693495699e-5 0.00010056871364929662 -0.00016141508417798455 -2.300122828468667e-5 -2.3645206922210396e-5 -2.310065242649489e-5 -0.00012094428707508553 -0.0001505369600200878 0.00013671724995935046 0.00011112254837631294 6.035492883516601e-5 0.00010633187598420077 -5.622878411681213e-5 -0.0001053436662479209 1.1134215172075372e-5 6.330926957876484e-5; -0.0002068806926528479 -0.000114636014708652 0.00019582515311952796 -0.00012626234013993272 -0.00011377611293397966 9.75250448990869e-5 4.6732679826106005e-5 6.563477987498566e-6 0.00030037760813314405 0.00012956907527580965 -8.02163041353784e-5 9.545907311433899e-5 0.00018533838816976739 1.7556922020229161e-6 2.8801111389764282e-5 0.00017805735193717487 5.349074203269561e-7 -1.2937812365740195e-5 7.144104407341103e-6 3.8036880929154574e-5 -0.0002926965236238831 -6.781380137368971e-5 0.00010332414318865203 7.956958221795611e-5 -0.00014948953970277357 1.323142247741604e-5 0.00013450007906243028 -4.799427473161209e-5 3.334345171054504e-5 -1.874356088052398e-5 0.0001562256082851823 -5.677030197517962e-5; -5.594457003561147e-5 -2.352137859367132e-5 0.0002633136452777189 -0.0001301625172937414 4.931882243448522e-5 6.065883529566588e-5 5.009740446888542e-5 0.00014887939208613755 -0.00010970766130873333 -3.6817205241334816e-5 2.3555786115081576e-5 -0.00011541685059006651 0.00018116291788367888 -9.449522009623861e-5 -0.00022555081638080734 2.283432034795159e-5 -2.907205512259536e-6 7.945768332767852e-5 -6.635097203349439e-5 -2.013005474970041e-5 -0.00011717549320116174 9.337825643791512e-5 -5.599645125136491e-5 3.117867412123232e-5 9.377339186806514e-5 -0.00013127958051433174 -2.8956433438964992e-5 -5.458790953077129e-5 0.0002119800471836569 -2.0326667495340303e-5 0.00016455131002197674 -6.913456424177947e-5], bias = [-1.220469019178618e-9, 2.8532450398334086e-10, -3.4629826072242666e-9, -1.6443579663715684e-10, -9.761623438133123e-10, -4.212982270814579e-9, 1.8518517971854745e-9, -1.5882287012861711e-9, -6.834609670582937e-10, 1.321184655727297e-9, -1.8039195999889193e-9, 1.0040003085858506e-9, 1.6669209454132599e-9, 1.6305553633001868e-9, 4.5518606273430097e-10, 2.0614667402934943e-9, 1.8749643307299174e-9, 2.9440030019100997e-9, -5.641998928511834e-9, -1.2030056079168487e-9, -1.711619016807388e-9, 2.502435203878771e-9, -6.007778891547735e-9, 5.033555277265573e-9, -7.732152450999191e-10, -1.5615445956254514e-9, 2.040730300760766e-9, -5.153688478794741e-9, 5.594236038203783e-9, -8.74890377622727e-10, 2.2999901708803867e-9, 4.936145495844469e-10]), layer_4 = (weight = [-0.0006402204701238931 -0.0006803431875735442 -0.0006795700841799984 -0.0006538367768310498 -0.0008900125295571859 -0.0006758478231493276 -0.0005740249792230504 -0.0006017205950910349 -0.000675338065283163 -0.0006060364751937468 -0.0006567267267397231 -0.0006563590628753845 -0.0007399387780720531 -0.0007400839357609118 -0.0007790695150374049 -0.0008988191355273558 -0.0006420583226883104 -0.0006688585540892138 -0.0006464597186350906 -0.00071846999102066 -0.0007467126036940512 -0.0007135869610834132 -0.0005468262081399156 -0.0007141615969195783 -0.0006424021983555022 -0.0007809062644734776 -0.0005980802404369942 -0.0006577502572859882 -0.000894160702026299 -0.0007213157849403031 -0.0006981063968149626 -0.0006483263807096775; 0.00034355068046015787 0.0002386511765071695 0.000337033410796739 0.00026636997297442376 0.0002980195306373009 0.00019945843609900252 0.00030758203717790906 0.00011944280507707006 9.637480926944121e-5 5.2251036843761005e-5 0.00040166784558388826 0.00029648562432332495 0.00020969219814465 0.00021941519569972795 0.00045521406824864963 3.745218608668266e-5 0.0003417685303185467 0.00020405115709021536 0.00020985820751356525 0.00034060928003688796 0.00027727415797229447 0.00020158364774521762 0.0002740368883519699 0.00028336454710834846 0.00011034269711375005 5.825672333882821e-5 0.000351536783235513 0.0002036908906652421 0.0001838041446996321 0.0001158744318460888 0.00021409881989620848 0.00020689936083340878], bias = [-0.0007051026435639656, 0.00023963413948182829]))Visualizing the Results
Let us now plot the loss over time
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Iteration", ylabel="Loss")
lines!(ax, losses; linewidth=4, alpha=0.75)
scatter!(ax, 1:length(losses), losses; marker=:circle, markersize=12, strokewidth=2)
fig
endFinally let us visualize the results
prob_nn = ODEProblem(ODE_model, u0, tspan, res.u)
soln_nn = Array(solve(prob_nn, RK4(); u0, p=res.u, saveat=tsteps, dt, adaptive=false))
waveform_nn_trained = first(
compute_waveform(dt_data, soln_nn, mass_ratio, ode_model_params)
)
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")
l1 = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
s1 = scatter!(
ax, tsteps, waveform; marker=:circle, alpha=0.5, strokewidth=2, markersize=12
)
l2 = lines!(ax, tsteps, waveform_nn; linewidth=2, alpha=0.75)
s2 = scatter!(
ax, tsteps, waveform_nn; marker=:circle, alpha=0.5, strokewidth=2, markersize=12
)
l3 = lines!(ax, tsteps, waveform_nn_trained; linewidth=2, alpha=0.75)
s3 = scatter!(
ax,
tsteps,
waveform_nn_trained;
marker=:circle,
alpha=0.5,
strokewidth=2,
markersize=12,
)
axislegend(
ax,
[[l1, s1], [l2, s2], [l3, s3]],
["Waveform Data", "Waveform Neural Net (Untrained)", "Waveform Neural Net"];
position=:lb,
)
fig
endAppendix
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
endJulia Version 1.12.7
Commit 6d172b025e4 (2026-08-15 08:05 UTC)
Build Info:
Official https://julialang.org release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 4 × AMD EPYC 7763 64-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-18.1.7 (ORCJIT, znver3)
GC: Built with stock GC
Threads: 4 default, 1 interactive, 4 GC (on 4 virtual cores)
Environment:
JULIA_DEBUG = Literate
LD_LIBRARY_PATH =
JULIA_NUM_THREADS = 4
JULIA_CPU_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0This page was generated using Literate.jl.