Training a Neural ODE to Model Gravitational Waveforms

This code is adapted from Astroinformatics/ScientificMachineLearning

The code has been minimally adapted from Keith et. al. 2021 which originally used Flux.jl

Package Imports

using Lux, ComponentArrays, LineSearches, AMDGPU, LuxCUDA, OrdinaryDiffEq, Optimization,
      OptimizationOptimJL, Printf, Random, SciMLSensitivity
using CairoMakie

CUDA.allowscalar(false)

┌ Warning: Module SymbolicsPreallocationToolsExt with build ID ffffffff-ffff-ffff-000d-a7359cfbf7eb is missing from the cache.
│ This may mean SymbolicsPreallocationToolsExt [d479e226-fb54-5ebe-a75e-a7af7f39127f] does not support precompilation but is imported by a module that does.
└ @ Base loading.jl:1948
┌ Error: Error during loading of extension SymbolicsPreallocationToolsExt of Symbolics, use `Base.retry_load_extensions()` to retry.
│   exception =
│    1-element ExceptionStack:
│    Declaring __precompile__(false) is not allowed in files that are being precompiled.
│    Stacktrace:
│      [1] _require(pkg::Base.PkgId, env::Nothing)
│        @ Base ./loading.jl:1952
│      [2] __require_prelocked(uuidkey::Base.PkgId, env::Nothing)
│        @ Base ./loading.jl:1812
│      [3] #invoke_in_world#3
│        @ ./essentials.jl:926 [inlined]
│      [4] invoke_in_world
│        @ ./essentials.jl:923 [inlined]
│      [5] _require_prelocked
│        @ ./loading.jl:1803 [inlined]
│      [6] _require_prelocked
│        @ ./loading.jl:1802 [inlined]
│      [7] run_extension_callbacks(extid::Base.ExtensionId)
│        @ Base ./loading.jl:1295
│      [8] run_extension_callbacks(pkgid::Base.PkgId)
│        @ Base ./loading.jl:1330
│      [9] run_package_callbacks(modkey::Base.PkgId)
│        @ Base ./loading.jl:1164
│     [10] __require_prelocked(uuidkey::Base.PkgId, env::String)
│        @ Base ./loading.jl:1819
│     [11] #invoke_in_world#3
│        @ ./essentials.jl:926 [inlined]
│     [12] invoke_in_world
│        @ ./essentials.jl:923 [inlined]
│     [13] _require_prelocked(uuidkey::Base.PkgId, env::String)
│        @ Base ./loading.jl:1803
│     [14] macro expansion
│        @ ./loading.jl:1790 [inlined]
│     [15] macro expansion
│        @ ./lock.jl:267 [inlined]
│     [16] __require(into::Module, mod::Symbol)
│        @ Base ./loading.jl:1753
│     [17] #invoke_in_world#3
│        @ ./essentials.jl:926 [inlined]
│     [18] invoke_in_world
│        @ ./essentials.jl:923 [inlined]
│     [19] require(into::Module, mod::Symbol)
│        @ Base ./loading.jl:1746
│     [20] include
│        @ ./Base.jl:495 [inlined]
│     [21] include_package_for_output(pkg::Base.PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, concrete_deps::Vector{Pair{Base.PkgId, UInt128}}, source::String)
│        @ Base ./loading.jl:2222
│     [22] top-level scope
│        @ stdin:3
│     [23] eval
│        @ ./boot.jl:385 [inlined]
│     [24] include_string(mapexpr::typeof(identity), mod::Module, code::String, filename::String)
│        @ Base ./loading.jl:2076
│     [25] include_string
│        @ ./loading.jl:2086 [inlined]
│     [26] exec_options(opts::Base.JLOptions)
│        @ Base ./client.jl:316
│     [27] _start()
│        @ Base ./client.jl:552
└ @ Base loading.jl:1301

Define some Utility Functions

Tip

This section can be skipped. It defines functions to simulate the model, however, from a scientific machine learning perspective, isn't super relevant.

We need a very crude 2-body path. Assume the 1-body motion is a newtonian 2-body position vector $r=r_1-r_2$ and use Newtonian formulas to get $r_1$, $r_2$ (e.g. Theoretical Mechanics of Particles and Continua 4.3)

function one2two(path, m₁, m₂)
    M = m₁ + m₂
    r₁ = m₂ / M .* path
    r₂ = -m₁ / M .* path
    return r₁, r₂
end

one2two (generic function with 1 method)

Next we define a function to perform the change of variables: $(\chi (t),\varphi (t))\mapsto (x(t),y(t))$

@views function soln2orbit(soln, model_params=nothing)
    @assert size(soln, 1) ∈ [2, 4] "size(soln,1) must be either 2 or 4"

    if size(soln, 1) == 2
        χ = soln[1, :]
        ϕ = soln[2, :]

        @assert length(model_params)==3 "model_params must have length 3 when size(soln,2) = 2"
        p, M, e = model_params
    else
        χ = soln[1, :]
        ϕ = soln[2, :]
        p = soln[3, :]
        e = soln[4, :]
    end

    r = p ./ (1 .+ e .* cos.(χ))
    x = r .* cos.(ϕ)
    y = r .* sin.(ϕ)

    orbit = vcat(x', y')
    return orbit
end

soln2orbit (generic function with 2 methods)

This function uses second-order one-sided difference stencils at the endpoints; see https://doi.org/10.1090/S0025-5718-1988-0935077-0

function d_dt(v::AbstractVector, dt)
    a = -3 / 2 * v[1] + 2 * v[2] - 1 / 2 * v[3]
    b = (v[3:end] .- v[1:(end - 2)]) / 2
    c = 3 / 2 * v[end] - 2 * v[end - 1] + 1 / 2 * v[end - 2]
    return [a; b; c] / dt
end

d_dt (generic function with 1 method)

This function uses second-order one-sided difference stencils at the endpoints; see https://doi.org/10.1090/S0025-5718-1988-0935077-0

function d2_dt2(v::AbstractVector, dt)
    a = 2 * v[1] - 5 * v[2] + 4 * v[3] - v[4]
    b = v[1:(end - 2)] .- 2 * v[2:(end - 1)] .+ v[3:end]
    c = 2 * v[end] - 5 * v[end - 1] + 4 * v[end - 2] - v[end - 3]
    return [a; b; c] / (dt^2)
end

d2_dt2 (generic function with 1 method)

Now we define a function to compute the trace-free moment tensor from the orbit

function orbit2tensor(orbit, component, mass=1)
    x = orbit[1, :]
    y = orbit[2, :]

    Ixx = x .^ 2
    Iyy = y .^ 2
    Ixy = x .* y
    trace = Ixx .+ Iyy

    if component[1] == 1 && component[2] == 1
        tmp = Ixx .- trace ./ 3
    elseif component[1] == 2 && component[2] == 2
        tmp = Iyy .- trace ./ 3
    else
        tmp = Ixy
    end

    return mass .* tmp
end

function h_22_quadrupole_components(dt, orbit, component, mass=1)
    mtensor = orbit2tensor(orbit, component, mass)
    mtensor_ddot = d2_dt2(mtensor, dt)
    return 2 * mtensor_ddot
end

function h_22_quadrupole(dt, orbit, mass=1)
    h11 = h_22_quadrupole_components(dt, orbit, (1, 1), mass)
    h22 = h_22_quadrupole_components(dt, orbit, (2, 2), mass)
    h12 = h_22_quadrupole_components(dt, orbit, (1, 2), mass)
    return h11, h12, h22
end

function h_22_strain_one_body(dt::T, orbit) where {T}
    h11, h12, h22 = h_22_quadrupole(dt, orbit)

    h₊ = h11 - h22
    hₓ = T(2) * h12

    scaling_const = √(T(π) / 5)
    return scaling_const * h₊, -scaling_const * hₓ
end

function h_22_quadrupole_two_body(dt, orbit1, mass1, orbit2, mass2)
    h11_1, h12_1, h22_1 = h_22_quadrupole(dt, orbit1, mass1)
    h11_2, h12_2, h22_2 = h_22_quadrupole(dt, orbit2, mass2)
    h11 = h11_1 + h11_2
    h12 = h12_1 + h12_2
    h22 = h22_1 + h22_2
    return h11, h12, h22
end

function h_22_strain_two_body(dt::T, orbit1, mass1, orbit2, mass2) where {T}
    # compute (2,2) mode strain from orbits of BH 1 of mass1 and BH2 of mass 2

    @assert abs(mass1 + mass2 - 1.0)<1e-12 "Masses do not sum to unity"

    h11, h12, h22 = h_22_quadrupole_two_body(dt, orbit1, mass1, orbit2, mass2)

    h₊ = h11 - h22
    hₓ = T(2) * h12

    scaling_const = √(T(π) / 5)
    return scaling_const * h₊, -scaling_const * hₓ
end

function compute_waveform(dt::T, soln, mass_ratio, model_params=nothing) where {T}
    @assert mass_ratio≤1 "mass_ratio must be <= 1"
    @assert mass_ratio≥0 "mass_ratio must be non-negative"

    orbit = soln2orbit(soln, model_params)
    if mass_ratio > 0
        m₂ = inv(T(1) + mass_ratio)
        m₁ = mass_ratio * m₂

        orbit₁, orbit₂ = one2two(orbit, m₁, m₂)
        waveform = h_22_strain_two_body(dt, orbit₁, m₁, orbit₂, m₂)
    else
        waveform = h_22_strain_one_body(dt, orbit)
    end
    return waveform
end

compute_waveform (generic function with 2 methods)

Simulating the True Model

RelativisticOrbitModel defines system of odes which describes motion of point like particle in schwarzschild background, uses

$$u[1]=\chi$$$$u[2]=\varphi$$

where, $p$, $M$, and $e$ are constants

function RelativisticOrbitModel(u, (p, M, e), t)
    χ, ϕ = u

    numer = (p - 2 - 2 * e * cos(χ)) * (1 + e * cos(χ))^2
    denom = sqrt((p - 2)^2 - 4 * e^2)

    χ̇ = numer * sqrt(p - 6 - 2 * e * cos(χ)) / (M * (p^2) * denom)
    ϕ̇ = numer / (M * (p^(3 / 2)) * denom)

    return [χ̇, ϕ̇]
end

mass_ratio = 0.0         # test particle
u0 = Float64[π, 0.0]     # initial conditions
datasize = 250
tspan = (0.0f0, 6.0f4)   # timespace for GW waveform
tsteps = range(tspan[1], tspan[2]; length=datasize)  # time at each timestep
dt_data = tsteps[2] - tsteps[1]
dt = 100.0
const ode_model_params = [100.0, 1.0, 0.5]; # p, M, e

Let's simulate the true model and plot the results using OrdinaryDiffEq.jl

prob = ODEProblem(RelativisticOrbitModel, u0, tspan, ode_model_params)
soln = Array(solve(prob, RK4(); saveat=tsteps, dt, adaptive=false))
waveform = first(compute_waveform(dt_data, soln, mass_ratio, ode_model_params))

begin
    fig = Figure()
    ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")

    l = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
    s = scatter!(ax, tsteps, waveform; marker=:circle, markersize=12, alpha=0.5)

    axislegend(ax, [[l, s]], ["Waveform Data"])

    fig
end

Defiing a Neural Network Model

Next, we define the neural network model that takes 1 input (time) and has two outputs. We'll make a function ODE_model that takes the initial conditions, neural network parameters and a time as inputs and returns the derivatives.

It is typically never recommended to use globals but incase you do use them, make sure to mark them as const.

We will deviate from the standard Neural Network initialization and use WeightInitializers.jl,

const nn = Chain(Base.Fix1(broadcast, cos),
    Dense(1 => 32, cos; init_weight=truncated_normal(; std=1e-4)),
    Dense(32 => 32, cos; init_weight=truncated_normal(; std=1e-4)),
    Dense(32 => 2; init_weight=truncated_normal(; std=1e-4)))
ps, st = Lux.setup(Xoshiro(), nn)

((layer_1 = NamedTuple(), layer_2 = (weight = Float32[0.00016373771; -0.0001723072; -3.0804506f-5; 0.00017129372; 0.00021004312; 6.2614825f-5; 8.9672634f-5; 5.0600815f-5; 0.00018971424; -0.00021288611; -0.00011350648; 4.709769f-6; 0.00012735366; -0.00018767703; 4.5327357f-5; -0.00013070507; -8.721254f-5; 0.00010652612; -8.485286f-5; 0.00012322523; 0.000192678; -5.7727066f-5; -2.2805214f-5; -5.551938f-5; 1.6263419f-5; 0.00014814515; -6.412835f-5; 4.765311f-5; -0.00010560792; -9.525769f-5; 5.173929f-5; -0.00013154293;;], bias = Float32[0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0;;]), layer_3 = (weight = Float32[-0.00010694874 -8.890326f-5 4.034049f-5 5.319966f-6 -0.0003459722 -0.00012620965 0.00022975777 0.00018096282 8.400319f-5 6.917889f-5 -3.9769933f-5 -6.2100604f-5 -0.00011184003 9.6614785f-5 -5.3859194f-6 3.6702386f-5 -0.00012370253 2.4659794f-5 -0.00012399025 -0.00011506463 0.000101669626 -0.00018179906 -5.3396947f-5 0.00016046135 4.7632868f-5 -6.860522f-5 -7.310127f-5 -8.233278f-6 7.3894867f-6 -4.1679396f-5 2.7466624f-5 0.000113972674; 0.00030453491 6.3051475f-6 3.8891092f-5 3.9777533f-5 8.981667f-5 -1.650299f-5 -6.580113f-5 -1.8318224f-5 0.00011751977 1.5250294f-5 -0.00022252346 8.8024564f-5 0.00010238377 -5.6422723f-6 -2.3755638f-5 -1.3139059f-5 9.0867925f-6 1.6610093f-5 9.422604f-5 0.00014709994 -5.7532994f-5 -6.441986f-5 -5.1155934f-5 7.715527f-5 5.1059496f-6 -1.660976f-5 -9.2718015f-5 -9.988419f-6 -3.2302432f-5 -0.000120697754 0.00014143944 0.00015510482; 0.00011625534 6.553586f-6 0.00014954538 -0.00010076313 -0.00014538519 0.00014416037 -9.3846924f-5 0.0001521419 0.00015364122 0.00010508916 5.3120737f-5 4.93332f-5 0.00016876367 -0.000108838314 -0.00013233365 1.739391f-5 4.6043544f-5 2.767919f-5 -0.00010448473 -1.0520376f-5 -0.00011460128 -9.2696166f-5 -1.41573855f-5 4.9000782f-6 6.8920475f-5 -3.91867f-6 8.030558f-5 -0.00011824741 -3.896603f-5 -0.00014922203 5.5022854f-5 7.5014934f-5; -0.00011940578 -5.5499528f-5 -0.00018923677 -0.00011896671 3.229365f-5 7.64244f-5 0.00020838124 7.9978796f-5 -5.397553f-5 0.00012022381 -8.960315f-6 -7.3842125f-6 0.00010486508 -0.00010914301 -8.5673186f-5 1.1095491f-5 6.867036f-5 -1.0970863f-5 0.000121570956 -7.279764f-5 -5.336332f-5 -5.8746635f-5 3.8865375f-5 -5.9877617f-5 -5.8712463f-5 0.000111406494 -0.00014943103 -6.213235f-5 -3.092008f-5 9.206083f-5 0.00015571497 8.0263264f-5; -5.677299f-5 0.00015016178 -5.9317306f-5 -2.3020097f-5 9.4310526f-5 0.00012410061 -0.0001703463 5.6306213f-5 -2.947607f-5 4.2232306f-5 -3.123399f-5 -4.98881f-5 -0.000107062726 -6.2932006f-5 9.269813f-5 -4.4522763f-5 -3.730372f-5 -4.48639f-5 -5.4368324f-5 2.3852926f-5 6.4334694f-5 3.4735083f-5 -3.2913405f-5 3.3898345f-6 -4.482059f-5 -6.290806f-6 7.493171f-5 -6.850766f-5 -0.00021639057 6.1299506f-5 -9.043545f-5 -9.122096f-5; -4.7735062f-5 7.372723f-5 -8.437444f-5 5.6277604f-5 -0.00012333917 -5.0172937f-5 4.0936367f-5 -7.8092606f-5 -9.555358f-5 0.00017332393 0.00012738032 5.1394483f-5 -5.5089753f-5 -3.3467266f-5 -0.00021093969 -0.00010417776 2.482426f-5 -2.9269506f-5 5.2413285f-5 -3.0916573f-5 0.00011387007 -1.9529893f-5 -6.560572f-5 0.000100560916 4.7062313f-5 -6.358637f-5 2.3830804f-5 -7.128756f-5 8.863318f-5 7.5094285f-5 -0.00013946221 -3.4949233f-5; 0.00020351606 3.893417f-5 3.610237f-5 1.9937765f-5 1.4817893f-5 9.76573f-5 -9.97998f-6 -0.00024097721 -2.1636848f-5 -1.4534239f-5 0.00011973061 4.1390795f-5 8.2372004f-5 -0.0001049126 -0.00012869941 6.5387256f-5 -6.6009314f-5 -1.0663849f-5 -0.00010138918 -9.3947765f-6 -9.580858f-5 3.7678255f-5 3.656603f-5 -2.6882842f-6 5.3785254f-5 0.0001238681 -1.034295f-5 -1.098684f-6 -9.389047f-5 -0.00017239501 -0.00013694122 4.602454f-6; 0.00013789826 0.00014513262 0.000111287765 -8.448727f-5 -0.00011332749 2.0951205f-5 -0.0001624396 5.0393224f-5 2.867179f-5 -2.0645064f-5 -0.00015781043 1.9534202f-5 9.886214f-6 6.8089544f-6 -0.0001456766 -9.868171f-5 6.68243f-5 -4.0461066f-5 4.45389f-5 -4.7815145f-5 0.00010242379 -6.231554f-5 -1.6329159f-5 7.824833f-5 -3.736544f-5 1.602686f-7 5.5147313f-5 -1.3319667f-5 -0.0001687736 0.00019208192 5.991068f-5 0.00017845622; 4.9510265f-5 -1.5327298f-5 -0.00016002465 -2.0942902f-5 8.7282795f-5 3.8725797f-5 2.8012386f-5 1.22668025f-5 -6.8224326f-5 -4.3408476f-5 -3.1815096f-6 3.3622695f-5 -3.302833f-5 0.00012200777 -7.754287f-5 -6.265148f-5 -5.5428747f-5 2.3648132f-5 8.877604f-6 2.9252866f-5 -8.7985056f-5 8.313225f-5 -0.00013218315 -7.551062f-5 0.0001618332 -7.7373385f-5 -2.8199927f-5 -2.4497674f-6 1.9357245f-5 -4.163317f-5 0.00019518248 0.0001935601; 2.433431f-5 -1.80121f-5 -0.00011198838 8.6451146f-5 -3.8411246f-5 0.00012686057 0.00016552999 5.1299572f-5 -7.1273476f-5 -8.310552f-5 0.00010924213 5.8726826f-5 -9.073325f-5 -0.00015873526 -0.00010254181 -0.00020244425 -7.4885786f-5 8.424628f-6 7.637193f-5 9.1504255f-5 -0.00014364537 8.964831f-5 1.6997014f-5 -0.00013082444 5.9737282f-5 7.246278f-5 -0.00014698753 -0.00023377838 -1.0035749f-5 -2.9793886f-5 2.7507615f-5 4.3875727f-5; 6.295204f-5 -0.00012645223 -9.073207f-5 -9.134207f-5 -0.00013391025 1.4404124f-5 -2.2943688f-5 -9.225641f-5 -3.7829846f-5 -1.726343f-5 -8.736765f-5 8.946875f-5 -4.267641f-5 -6.563267f-5 -7.8331f-5 2.8248937f-6 4.299665f-6 -0.00016002195 -1.6294669f-5 -6.586075f-5 -5.190408f-5 6.375257f-5 -0.00011367948 6.460186f-5 9.781469f-6 2.5922047f-5 0.00023445956 -3.727989f-5 -5.314336f-5 0.00021869637 3.5302357f-5 2.1888722f-5; -0.000106479085 -6.876478f-5 0.00019171844 -4.9209626f-5 4.9593717f-5 -0.00015652983 -8.026312f-5 -2.9821611f-5 -9.87901f-5 -7.181417f-5 0.00012342396 2.743319f-5 0.00014545949 -0.00012906383 2.861567f-5 9.0593654f-5 -5.268863f-5 4.3677603f-5 2.4622597f-5 -0.00023605213 6.230097f-5 -8.785521f-5 3.5168796f-5 -4.999212f-5 6.468893f-5 -0.00019670057 -0.00026613649 -7.319912f-5 4.916333f-5 -7.0723385f-5 -7.8692065f-5 -3.230481f-5; -9.1939735f-5 -1.6640237f-5 0.00010852261 -0.00021214472 -1.9275476f-5 -9.5116707f-7 0.00014113034 8.2653525f-5 0.00020127883 -0.000115029514 4.069467f-5 8.89875f-5 -0.00010638105 -9.7692864f-5 6.4430635f-5 0.00011508062 4.8970114f-6 -7.6409335f-5 -8.466746f-5 0.00011398435 2.7418762f-5 -7.377621f-5 -0.00017452131 -2.3060356f-5 -0.0001037233 1.328866f-5 8.8480534f-5 -0.00019720718 -0.00015424828 2.5711988f-6 -1.8169487f-5 -7.690731f-5; 0.00010071401 -1.1918058f-5 7.8102916f-5 -5.126684f-5 -5.5605036f-5 0.00012966264 4.704469f-5 -1.911685f-6 -0.00016081629 7.443157f-5 -2.8829321f-5 2.078657f-5 -3.9926428f-5 -0.00015637693 5.8183585f-5 8.129123f-6 -6.343689f-5 0.00012118094 -0.00019924379 -0.00017191946 -2.4149676f-5 -2.4014007f-5 0.0001851281 0.000102005906 0.00017451268 5.9854774f-5 -2.8574721f-5 0.00013421563 -0.00017002028 -0.000111996495 -6.650551f-5 0.00020709957; -3.2903347f-6 0.0002050334 8.881042f-5 4.4837907f-5 -8.706873f-5 0.00011339434 7.982011f-5 0.0001645412 5.518259f-5 0.00020737527 -5.3931723f-5 -9.531843f-5 6.496321f-5 -4.1690135f-5 -1.0833746f-5 0.00021134574 6.1453415f-5 -4.9207665f-5 7.2564267f-6 9.413437f-5 -9.474841f-5 8.993446f-5 0.00012307779 -5.661606f-5 1.999269f-6 0.00015999116 -0.0001265607 -4.4987217f-5 9.240926f-5 -1.0582938f-5 0.00016095539 -2.2632876f-5; -5.4059396f-6 9.307573f-7 -4.6411045f-5 -2.8202412f-5 -8.13997f-5 -0.00013091946 6.3623776f-7 -0.00012179059 7.9399135f-5 -0.00015905943 -2.1291167f-5 -0.00010595171 -3.889293f-5 2.0163132f-5 8.406657f-5 3.3078162f-5 0.00023594857 -3.543401f-5 -4.5019227f-5 0.00012131354 -7.913775f-5 -0.0002683046 3.6860776f-5 1.0231894f-5 0.00016550101 -2.7606007f-5 3.803275f-5 6.83925f-5 -5.2836975f-5 0.00018663882 -3.4741748f-5 3.769375f-5; -0.00015594886 4.8037167f-5 -0.00010172898 5.5528188f-5 0.00015005047 -0.00013480285 -0.000104488696 -0.00016123282 -2.1036569f-5 0.00011135856 -0.00018874714 -8.785194f-6 -5.8102927f-5 -8.021641f-5 -0.00010484486 2.6035601f-5 7.688937f-5 -0.00010625598 0.00010102279 3.9147162f-5 -3.434012f-5 -5.9647184f-5 -2.4965651f-5 7.452568f-5 5.6762216f-5 -0.0001740291 0.0001065781 -4.282217f-5 9.151989f-5 -0.0001406439 -9.995751f-5 8.420006f-5; 3.5486217f-5 2.9863619f-5 -8.21412f-5 0.00016356552 -0.00022909862 9.313924f-5 6.062999f-5 -7.669441f-5 -6.686192f-5 -9.0385365f-5 -5.120943f-5 0.0001440759 -0.00013841537 -9.821521f-6 -0.00019120588 0.00019957662 0.0001263879 -5.5368677f-5 0.00019844071 5.5987293f-5 4.6416073f-5 -0.0001209487 7.6879376f-5 4.467185f-5 0.00017564639 5.1972373f-5 6.4457905f-5 -0.00018492909 -0.00010645703 8.909723f-5 7.649297f-5 -3.2112388f-5; 4.1868683f-5 -3.071039f-5 -0.00012943913 -1.7546145f-5 -0.0001273104 8.333437f-5 9.897768f-5 -0.00010930006 7.982587f-5 9.190587f-5 4.683601f-5 -3.7059394f-6 -5.7105112f-6 -1.4753802f-5 -0.00016252455 8.3223f-6 5.4450487f-5 0.000121560675 -9.79022f-6 0.00016530394 -1.7137936f-6 1.1979617f-5 5.5393357f-5 -0.00010121554 -1.0409351f-6 -1.1542158f-5 0.00014295371 2.1651042f-5 -6.6681474f-5 7.069651f-5 -4.2798016f-5 2.781102f-5; -2.5328267f-5 0.00013122658 -2.8057064f-5 3.5820147f-5 -4.5062254f-5 4.926186f-5 -0.00019823725 -4.6172136f-5 -5.958349f-5 3.762732f-5 3.24441f-5 -0.00014681062 -0.0001468335 9.054843f-5 4.9134283f-5 4.0992876f-5 -0.00017358409 -8.598959f-5 -0.0002176851 -8.217607f-5 3.7704318f-5 8.0970585f-6 9.9800825f-5 3.334106f-5 -0.00012251509 0.000107615 0.00018649087 2.0906065f-5 -8.663797f-5 5.149301f-5 0.00012691421 -0.00013912188; -2.0074278f-5 -2.3845516f-5 -3.065287f-5 -0.00012635987 9.219768f-6 0.00013077643 -9.2900234f-5 -1.3494813f-5 0.0001372153 -3.9675615f-5 -9.96337f-5 0.00015899795 5.3383676f-5 -6.003343f-6 9.440587f-5 -3.2299274f-6 3.2692442f-5 -1.0895862f-5 7.907242f-5 7.4859985f-5 0.00014270199 2.2861785f-5 -6.975755f-5 0.00012045709 9.296696f-5 0.00011554436 6.941035f-5 8.2505096f-5 0.000163678 -4.1085837f-6 3.128219f-6 -6.332239f-5; 0.0001661218 6.2489057f-6 -5.3486707f-5 3.9137292f-5 6.509948f-5 -9.512635f-5 -9.428642f-5 -0.000118572716 0.00014290473 -7.9414596f-5 -0.00019487942 0.0001827206 -5.5861463f-5 -5.2715955f-6 -3.0493698f-6 -2.6186834f-5 -6.1136045f-5 -5.5037813f-6 -9.7537695f-6 4.1693548f-7 -0.00012961749 4.2347918f-5 0.00020345293 0.000114076254 -0.00023870979 -0.0001485775 0.00013385707 5.2688418f-5 -6.1267274f-5 -3.9199367f-5 -0.00013906955 0.00012998455; 9.748159f-5 7.2743904f-5 7.5776676f-5 0.000118198 0.0001266223 -0.00017699145 0.0002883184 -0.00010134479 0.00014936297 0.00014090768 -8.454715f-5 4.7592104f-5 -6.198127f-5 3.1363437f-5 6.61888f-5 3.844453f-5 -8.462849f-6 -8.225017f-5 -5.4944136f-5 3.9931154f-5 1.5003369f-5 -5.9531485f-5 8.2539984f-5 -3.4913894f-5 0.00010703965 -4.802087f-6 0.0001909797 -5.0932857f-5 3.843382f-5 2.3095157f-5 -8.697749f-5 -7.696679f-5; -2.107221f-5 2.3589016f-6 -0.00012510591 -0.000102401085 0.00011408393 0.00022276308 0.0001561087 -7.659035f-5 -3.3878165f-5 1.9540705f-5 -2.3315562f-5 -8.697153f-5 4.1945503f-5 3.233793f-5 -4.6819336f-5 1.5409978f-5 -0.0001972973 0.00018785658 -7.939498f-6 9.238694f-7 -0.000104465355 -0.00014348625 7.517659f-5 -2.4652245f-5 -2.2343618f-5 -8.922004f-6 -9.284893f-5 -0.00012740293 9.0998525f-5 5.9873328f-5 -1.6327189f-5 0.00010489988; -0.0002752276 -9.361787f-5 -6.2432635f-5 3.768976f-5 -3.859818f-6 0.00019726853 7.318292f-5 7.193804f-5 0.00013892686 0.00012156338 4.557716f-5 -7.630119f-5 6.769982f-5 0.000105161795 4.9561833f-5 6.198458f-5 2.0436564f-5 -0.00013073158 -4.1752337f-5 7.820146f-5 0.00012737399 8.448752f-5 1.5097722f-5 -9.294738f-5 -0.00019184595 2.6141379f-5 1.8201745f-5 -6.220067f-5 0.00013549704 0.00010391265 2.7703181f-5 0.00017388909; 4.955789f-5 -1.6406475f-5 -5.0154296f-5 -1.937346f-5 -1.5793543f-5 -7.847255f-5 4.5386507f-5 1.0034457f-5 -4.2361394f-6 6.561596f-5 -4.8612637f-5 2.3592456f-6 9.391162f-5 -2.6184702f-5 0.00010917904 -2.1974382f-5 -0.00014429049 -3.230766f-5 5.8238067f-5 -7.907766f-5 -8.5969354f-5 2.0308906f-5 0.0001380361 4.4162818f-5 5.9456226f-5 -4.602802f-5 -9.34629f-5 -3.4706668f-6 5.722024f-5 -4.5002063f-5 0.00010185955 0.00010244141; -3.823507f-5 -5.6128418f-5 -0.00012824523 6.524803f-5 0.00018336061 -5.0329676f-5 8.167387f-5 0.00011664406 1.2073418f-5 0.00012642126 -3.9144663f-5 1.8425127f-6 -0.00015535025 3.2487256f-5 9.3268f-5 -6.258532f-5 5.2429656f-5 2.5912716f-5 0.00010690065 -3.4070003f-5 3.248595f-5 -0.000112967915 5.1772102f-5 -0.00011433688 0.00014917938 2.7192077f-6 -9.985258f-5 -8.96928f-5 -8.6771515f-6 -6.761547f-5 -0.000109043875 1.910393f-5; 1.7649729f-6 3.34299f-5 4.340704f-5 -1.8753555f-5 8.069256f-5 -6.2340085f-5 8.3690924f-5 -0.00019233294 0.00022725601 8.3491206f-5 6.004536f-5 5.2688283f-5 0.0001332725 -7.7350103f-7 -7.67233f-5 -4.517462f-5 1.0317836f-5 0.00012960017 -0.00010631081 8.633271f-6 0.00018181496 -1.59308f-5 -4.7386187f-5 -5.8881436f-5 7.1132665f-5 9.685048f-6 -1.8066767f-5 0.000101606165 -3.7635837f-5 -0.00013329691 -0.00024898196 7.7767065f-5; 0.00010300002 -7.4887386f-5 -0.00024371967 1.4090536f-5 0.00017816323 -0.00019117848 2.8733284f-5 -3.529498f-5 4.0042596f-5 -5.4634023f-5 -0.00012753948 -6.0610157f-5 1.3021068f-5 5.9232923f-5 -0.00016325977 -2.1715277f-6 6.0687066f-6 -7.5999866f-5 -9.461311f-5 2.9688694f-5 -7.364525f-5 -4.114431f-6 0.0001211141 2.7937951f-5 9.048701f-5 0.00013239527 -6.243748f-5 7.184595f-5 9.416184f-5 -8.989481f-5 -8.209577f-6 -3.936139f-5; -6.116025f-5 -9.474167f-5 7.152702f-5 -0.00023639023 5.3568005f-5 0.00025000368 -0.00029104005 5.8974354f-5 -0.0002365924 -0.00010004452 -5.584017f-5 -9.0054596f-7 4.0760413f-5 -2.326736f-5 0.00012463781 -0.00012320584 1.3917735f-5 -5.0355076f-5 -6.516802f-5 -6.065718f-5 -3.4001896f-5 -9.058502f-5 0.00025379244 7.222445f-5 -4.7186873f-5 -0.000100693665 -7.383852f-5 -3.634956f-5 4.8962334f-5 -0.00010655424 -1.9349349f-5 -1.5070884f-5; -2.26786f-5 5.8219834f-5 -7.442795f-5 0.00013767368 0.00012771568 -0.000101841295 6.1440915f-5 1.8778735f-5 -0.00012474964 -0.00018880224 3.735041f-5 4.0341634f-5 5.9921284f-5 0.0001263154 5.9236f-5 8.165418f-5 -2.7490255f-6 7.735728f-5 -9.3523784f-5 -0.00022587477 -3.815074f-5 -5.6857603f-5 2.4990713f-5 1.0730445f-5 -9.985394f-5 0.00015063582 -7.7813995f-5 3.227031f-5 -0.0001369399 2.3901055f-5 5.1079074f-5 -6.808405f-5; 6.210516f-5 -3.7043394f-6 5.1657098f-5 -3.217545f-5 -8.932678f-5 0.00011483903 7.5106596f-5 0.00017566794 0.00025532523 4.776594f-5 -2.2731063f-5 5.8554393f-5 0.000115292176 6.4017404f-5 0.00012503787 0.00013153971 -3.784825f-5 -3.4302582f-6 -5.289436f-5 -7.206925f-5 -4.525488f-5 0.0002540609 -3.53406f-5 -2.263856f-5 0.00011529421 0.00010437796 -0.00010650316 -1.4229225f-5 6.2855586f-5 -1.7171007f-5 -0.00018311127 1.811043f-5], bias = Float32[0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0;;]), layer_4 = (weight = Float32[0.00010784265 -2.9762747f-5 -7.8600206f-5 7.9807105f-5 -7.634957f-5 -6.577673f-5 -8.4594656f-5 3.1334122f-5 3.4221703f-6 0.00010743183 -3.846182f-5 3.8278165f-5 -6.133932f-5 8.387087f-5 -0.00012547821 -1.9419589f-5 -0.00015134379 8.456145f-5 -7.910615f-5 -5.0588602f-5 -0.00014900714 -6.848079f-5 -2.5551368f-5 -6.5414664f-5 -0.00018350958 -0.00017597667 -5.6173212f-5 -5.565551f-5 7.2790346f-5 -6.84759f-6 0.00010433266 -2.9287146f-6; 3.5102785f-5 1.0631198f-5 -0.00014356662 -2.3903502f-5 0.00017306309 3.4179764f-5 0.0001335161 0.000146983 0.0002251323 -5.2635332f-5 -0.000105182655 5.1612664f-5 6.279592f-5 5.8770356f-6 6.8182504f-5 0.00018816267 -4.965534f-5 4.3108616f-5 -5.015717f-5 6.4668675f-5 2.4073197f-5 3.6509155f-5 0.00018315403 1.227862f-5 8.699771f-5 6.216788f-5 -1.6548398f-5 -9.4985975f-5 -7.5790454f-6 7.798897f-5 -7.616071f-5 -0.00010732112], bias = Float32[0.0; 0.0;;])), (layer_1 = NamedTuple(), layer_2 = NamedTuple(), layer_3 = NamedTuple(), layer_4 = NamedTuple()))

Similar to most DL frameworks, Lux defaults to using Float32, however, in this case we need Float64

const params = ComponentArray{Float64}(ps)

const nn_model = StatefulLuxLayer(nn, st)

StatefulLuxLayer{true}(
    Chain(
        layer_1 = WrappedFunction{:direct_call}(Base.Fix1{typeof(broadcast), typeof(cos)}(broadcast, cos)),
        layer_2 = Dense(1 => 32, cos),  # 64 parameters
        layer_3 = Dense(32 => 32, cos),  # 1_056 parameters
        layer_4 = Dense(32 => 2),       # 66 parameters
    ),
)         # Total: 1_186 parameters,
          #        plus 0 states.

Now we define a system of odes which describes motion of point like particle with Newtonian physics, uses

$$u[1]=\chi$$$$u[2]=\varphi$$

where, $p$, $M$, and $e$ are constants

function ODE_model(u, nn_params, t)
    χ, ϕ = u
    p, M, e = ode_model_params

    # In this example we know that `st` is am empty NamedTuple hence we can safely ignore
    # it, however, in general, we should use `st` to store the state of the neural network.
    y = 1 .+ nn_model([first(u)], nn_params)

    numer = (1 + e * cos(χ))^2
    denom = M * (p^(3 / 2))

    χ̇ = (numer / denom) * y[1]
    ϕ̇ = (numer / denom) * y[2]

    return [χ̇, ϕ̇]
end

ODE_model (generic function with 1 method)

Let us now simulate the neural network model and plot the results. We'll use the untrained neural network parameters to simulate the model.

prob_nn = ODEProblem(ODE_model, u0, tspan, params)
soln_nn = Array(solve(prob_nn, RK4(); u0, p=params, saveat=tsteps, dt, adaptive=false))
waveform_nn = first(compute_waveform(dt_data, soln_nn, mass_ratio, ode_model_params))

begin
    fig = Figure()
    ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")

    l1 = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
    s1 = scatter!(
        ax, tsteps, waveform; marker=:circle, markersize=12, alpha=0.5, strokewidth=2)

    l2 = lines!(ax, tsteps, waveform_nn; linewidth=2, alpha=0.75)
    s2 = scatter!(
        ax, tsteps, waveform_nn; marker=:circle, markersize=12, alpha=0.5, strokewidth=2)

    axislegend(ax, [[l1, s1], [l2, s2]],
        ["Waveform Data", "Waveform Neural Net (Untrained)"]; position=:lb)

    fig
end

Setting Up for Training the Neural Network

Next, we define the objective (loss) function to be minimized when training the neural differential equations.

const mseloss = MSELoss()

function loss(θ)
    pred = Array(solve(prob_nn, RK4(); u0, p=θ, saveat=tsteps, dt, adaptive=false))
    pred_waveform = first(compute_waveform(dt_data, pred, mass_ratio, ode_model_params))
    return mseloss(waveform, pred_waveform), pred_waveform
end

loss (generic function with 1 method)

Warmup the loss function

loss(params)

(0.0006987510079413579, [-0.024279214682950235, -0.023493411903372372, -0.022707609123794412, -0.021379909742243967, -0.01948262163614706, -0.01697555664426333, -0.01380596841353563, -0.00990560958380139, -0.005193026333551311, 0.00042674910688446523, 0.007052935609584813, 0.014767990892920383, 0.023589935186109592, 0.03334403762460476, 0.043385032104678534, 0.051941072922356085, 0.05472700248260987, 0.04268038225383832, 0.002261025315810013, -0.06606395264760874, -0.1105504301877244, -0.07662440241175528, -0.006609194054202847, 0.03939456429759882, 0.05479315354660446, 0.05323460574234514, 0.04488726235832278, 0.034680825741757314, 0.024662105433552396, 0.015583042266398044, 0.007648359607275285, 0.0008483812249534275, -0.004904234102555666, -0.009715428565414273, -0.013686964885317435, -0.016905617175273214, -0.01944487119820609, -0.021362104262045503, -0.022700911374103402, -0.023492118590135145, -0.023754849611451186, -0.023496950580281634, -0.02271637765650105, -0.021398832126899463, -0.019520096159548934, -0.017043682142129826, -0.013920242130325887, -0.010087168382259122, -0.005468042464107061, 2.5516835515041767e-5, 0.006485998844874764, 0.013993637339772661, 0.022567874457828357, 0.03206416518184268, 0.041924369021480994, 0.05061374082662128, 0.05440593991737983, 0.04527048398131244, 0.010179513903034641, -0.05510741863440177, -0.10828918363025952, -0.08638420354092197, -0.01638141250392585, 0.03532546912153892, 0.05454835397755129, 0.054473641403675346, 0.046427515894992756, 0.036075338662653826, 0.025787367154011462, 0.01644062794093563, 0.008273895969835047, 0.0012930573474073473, -0.004602427751705485, -0.009515347416762422, -0.013563441470767323, -0.01683230718863553, -0.019407547799902333, -0.021344767935321578, -0.02269649587003526, -0.023493024368429948, -0.02375873765887814, -0.023503480655773717, -0.022726601360307557, -0.021418758632353128, -0.019558309522709014, -0.01710893550116989, -0.014031201080070797, -0.010259705332622432, -0.0057313546446795234, -0.0003556178143047562, 0.005946022970330864, 0.013257345651009951, 0.02159338326783427, 0.030840031358121626, 0.040507401626849675, 0.04926890251082551, 0.05388060956058237, 0.04725301488632783, 0.017133986605053077, -0.044118021562471944, -0.10388466796029765, -0.09494740028226595, -0.026929969666167328, 0.030383556307769368, 0.05392587080983015, 0.055633705879647895, 0.04799984406112667, 0.03752807338183102, 0.02697068343317593, 0.01733983715168724, 0.008936017708968508, 0.0017569583149370375, -0.004281362087740555, -0.00930876116871291, -0.013431502180592882, -0.016759049423947858, -0.019367531635559335, -0.021330059492456904, -0.022692755465921567, -0.023496949047032607, -0.023765142766651967, -0.023511792349942617, -0.022740034393929347, -0.021439818612854086, -0.01959586406253758, -0.017173440501695763, -0.014137128154943354, -0.01042577226781109, -0.00598178690955261, -0.0007195333558319343, 0.005432804670918187, 0.012555620031911388, 0.020665904883785453, 0.02966782037378566, 0.03913743845824375, 0.04792005054370085, 0.05319010840296443, 0.04871442179689489, 0.023145121077945516, -0.033394473217953434, -0.09757960753005261, -0.10192538061909714, -0.03807404377410431, 0.024488167740480127, 0.052853854717698434, 0.05668339887577469, 0.04959720771761246, 0.039042166087881275, 0.028207990798597853, 0.01829160430864556, 0.009629240424686375, 0.0022493594075639364, -0.003949900828929886, -0.009086071654413143, -0.01329665243255961, -0.016680476362199607, -0.019330387213819963, -0.021313125108435537, -0.022692203618353567, -0.023502787446419596, -0.023774089880786938, -0.0235232123846774, -0.022754699494696447, -0.021463282782931855, -0.01963143458566602, -0.017238922745756856, -0.014238383316638457, -0.010583950093267068, -0.0062218734091160155, -0.0010678399305575426, 0.004945173533394233, 0.01188867949700287, 0.019780251442125113, 0.02854687982483881, 0.03781489399890422, 0.04657690907011834, 0.05237382641800536, 0.04972612914137448, 0.02827231376491445, -0.023193670836498075, -0.0896813421587656, -0.1069971481008573, -0.049564159060107894, 0.017584866388381616, 0.051241594856381883, 0.05758957955227995, 0.051209775057789814, 0.04061340775215224, 0.029513121373386533, 0.01928801018057323, 0.010361239224370407, 0.0027640769070432993, -0.0035926405061932256, -0.008857896000770563, -0.01315434322019679, -0.01660090980168504, -0.019287280016649824, -0.02130037216902835, -0.0226928193331354, -0.02351109855304639, -0.02378554505306226, -0.02353694680774821, -0.022771672940322563, -0.021486131264653735, -0.019672358252752393, -0.01729935937220057, -0.014337171091955496, -0.010736683300382912, -0.006450379625645341, -0.001399006930909863, 0.00447965547229894, 0.011251982369253467, 0.018937655762732673, 0.027475285369982526, 0.0365392127843562, 0.04524907361505584, 0.051458486865206515, 0.050359381081313, 0.032578645709190106, -0.01368251910589828, -0.08057143092542023, -0.10993078765971787, -0.06106874001171594, 0.009628042381171095, 0.049005960130269824, 0.058305384006354634, 0.05282506389622571, 0.04224622197950377, 0.030877856702003468, 0.02033960485764291, 0.011132422541900487, 0.003308988378921086, -0.0032211487125325382, -0.008615685988323568, -0.013004850301319055, -0.01651642952019183, -0.01924615738575899, -0.021286709942063612, -0.022695152541065165, -0.023521653773736877, -0.023799744959782616, -0.023553150872709903, -0.022790840352117966, -0.02151189910763354, -0.01971096265959731, -0.01736021260581242, -0.014432747642448614, -0.010883431171146199, -0.006669049291926244, -0.0024546674127063637])

Now let us define a callback function to store the loss over time

const losses = Float64[]

function callback(θ, l, pred_waveform)
    push!(losses, l)
    @printf "Training %10s Iteration: %5d %10s Loss: %.10f\n" "" length(losses) "" l
    return false
end

callback (generic function with 1 method)

Training the Neural Network

Training uses the BFGS optimizers. This seems to give good results because the Newtonian model seems to give a very good initial guess

adtype = Optimization.AutoZygote()
optf = Optimization.OptimizationFunction((x, p) -> loss(x), adtype)
optprob = Optimization.OptimizationProblem(optf, params)
res = Optimization.solve(
    optprob, BFGS(; initial_stepnorm=0.01, linesearch=LineSearches.BackTracking());
    callback, maxiters=1000)

retcode: Success
u: ComponentVector{Float64}(layer_1 = Float64[], layer_2 = (weight = [0.00016373771359212697; -0.00017230720550287515; -3.0804505513515323e-5; 0.0001712937228148803; 0.00021004311565775424; 6.261482485570014e-5; 8.967263420345262e-5; 5.060081457486376e-5; 0.00018971423560287803; -0.00021288610878400505; -0.00011350648128427565; 4.709769200417213e-6; 0.00012735366181004792; -0.0001876770256785676; 4.532735692919232e-5; -0.0001307050697505474; -8.721253834664822e-5; 0.00010652611672412604; -8.485285798087716e-5; 0.00012322522525209934; 0.00019267799507360905; -5.7727065723156556e-5; -2.2805214030086063e-5; -5.551938011194579e-5; 1.626341872906778e-5; 0.00014814514725003392; -6.412834773072973e-5; 4.765310950460844e-5; -0.00010560791997704655; -9.525768837193027e-5; 5.1739290938712656e-5; -0.0001315429253736511;;], bias = [0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0;;]), layer_3 = (weight = [-0.00010694874072214589 -8.890325989341363e-5 4.034048834000714e-5 5.3199660214886535e-6 -0.0003459721920080483 -0.00012620964844245464 0.00022975777392275631 0.0001809628156479448 8.400319347856566e-5 6.917888822499663e-5 -3.976993320975453e-5 -6.210060382727534e-5 -0.00011184003233211115 9.661478543421254e-5 -5.38591939402977e-6 3.670238584163599e-5 -0.00012370252807158977 2.4659793780301698e-5 -0.00012399024853948504 -0.00011506462760735303 0.00010166962601942942 -0.00018179905600845814 -5.339694689610042e-5 0.00016046134987846017 4.763286779052578e-5 -6.860522262286395e-5 -7.310126966331154e-5 -8.233278094849084e-6 7.389486654574284e-6 -4.167939550825395e-5 2.7466623578220606e-5 0.00011397267371648923; 0.0003045349149033427 6.305147508101072e-6 3.889109211741015e-5 3.977753294748254e-5 8.981666906038299e-5 -1.650299054745119e-5 -6.580112676601857e-5 -1.8318223737878725e-5 0.00011751976853702217 1.5250294381985441e-5 -0.00022252346388995647 8.802456432022154e-5 0.00010238376853521913 -5.642272299155593e-6 -2.3755637812428176e-5 -1.3139058864908293e-5 9.086792488233186e-6 1.661009264353197e-5 9.422603761777282e-5 0.00014709994138684124 -5.753299410571344e-5 -6.441985897254199e-5 -5.1155933761037886e-5 7.715527317486703e-5 5.1059496399830095e-6 -1.6609759768471122e-5 -9.271801536669955e-5 -9.988419151341077e-6 -3.2302432373398915e-5 -0.0001206977540277876 0.0001414394355379045 0.0001551048189867288; 0.00011625533807091415 6.553585990332067e-6 0.00014954537618905306 -0.00010076312901219353 -0.00014538518735207617 0.00014416036719921976 -9.384692384628579e-5 0.000152141903527081 0.0001536412164568901 0.0001050891587510705 5.3120736993150786e-5 4.933320087729953e-5 0.00016876366862561554 -0.00010883831419050694 -0.00013233364734333009 1.739390972943511e-5 4.604354398907162e-5 2.7679190679918975e-5 -0.00010448473040014505 -1.052037623594515e-5 -0.0001146012800745666 -9.269616566598415e-5 -1.4157385521684773e-5 4.900078238279093e-6 6.892047531437129e-5 -3.918669790436979e-6 8.03055809228681e-5 -0.0001182474079541862 -3.8966030842857435e-5 -0.00014922203263267875 5.502285421243869e-5 7.501493382733315e-5; -0.00011940577678615227 -5.5499527661595494e-5 -0.0001892367727123201 -0.00011896670912392437 3.2293650292558596e-5 7.642440323252231e-5 0.00020838124328292906 7.997879583854228e-5 -5.397553104558028e-5 0.00012022381270071492 -8.960315426520538e-6 -7.384212494798703e-6 0.00010486508108442649 -0.0001091430094675161 -8.567318582208827e-5 1.1095490663137753e-5 6.867035699542612e-5 -1.0970862604153808e-5 0.00012157095625298098 -7.279763667611405e-5 -5.336332105798647e-5 -5.874663474969566e-5 3.886537524522282e-5 -5.9877616877201945e-5 -5.8712463214760646e-5 0.00011140649439767003 -0.00014943102723918855 -6.213234883034602e-5 -3.092008046223782e-5 9.206082904711366e-5 0.00015571496624033898 8.026326395338401e-5; -5.6772991229081526e-5 0.00015016178076621145 -5.93173062952701e-5 -2.3020096705295146e-5 9.431052603758872e-5 0.00012410061026457697 -0.00017034630582202226 5.630621308228001e-5 -2.947607026726473e-5 4.223230644129217e-5 -3.123399073956534e-5 -4.988810178474523e-5 -0.00010706272587412968 -6.293200567597523e-5 9.269813017453998e-5 -4.452276334632188e-5 -3.73037182725966e-5 -4.486390025704168e-5 -5.4368323617381975e-5 2.3852926460676827e-5 6.433469388866797e-5 3.4735083318082616e-5 -3.291340544819832e-5 3.3898345463967416e-6 -4.482059011934325e-5 -6.290806140896166e-6 7.493171142414212e-5 -6.850765930721536e-5 -0.0002163905737688765 6.129950634203851e-5 -9.043545287568122e-5 -9.122095798375085e-5; -4.773506225319579e-5 7.372722757281736e-5 -8.437444193987176e-5 5.627760401694104e-5 -0.00012333916674833745 -5.017293733544648e-5 4.093636744073592e-5 -7.809260569047183e-5 -9.555357974022627e-5 0.0001733239332679659 0.0001273803209187463 5.1394483307376504e-5 -5.50897530047223e-5 -3.346726589370519e-5 -0.00021093968825880438 -0.00010417775774840266 2.48242595262127e-5 -2.9269505830598064e-5 5.241328472038731e-5 -3.0916573450667784e-5 0.00011387006816221401 -1.9529892597347498e-5 -6.560571637237445e-5 0.00010056091559818015 4.7062312660273165e-5 -6.358636892400682e-5 2.3830803911550902e-5 -7.128756260499358e-5 8.863318362273276e-5 7.509428542107344e-5 -0.00013946220860816538 -3.494923294056207e-5; 0.0002035160578088835 3.8934169424464926e-5 3.6102370359003544e-5 1.9937764591304585e-5 1.4817893315921538e-5 9.765729919308797e-5 -9.979979950003326e-6 -0.0002409772132523358 -2.1636848032358103e-5 -1.4534239198837895e-5 0.00011973061191383749 4.139079464948736e-5 8.23720038169995e-5 -0.00010491260036360472 -0.00012869940837845206 6.538725574500859e-5 -6.60093137412332e-5 -1.066384902514983e-5 -0.00010138918150914833 -9.394776498083957e-6 -9.580858022673056e-5 3.7678255466744304e-5 3.656603075796738e-5 -2.688284212126746e-6 5.3785253840032965e-5 0.00012386809976305813 -1.0342950190533884e-5 -1.098684037970088e-6 -9.389047045260668e-5 -0.00017239501175936311 -0.00013694122026208788 4.602453827828867e-6; 0.00013789825607091188 0.00014513262431137264 0.00011128776532132179 -8.44872702145949e-5 -0.00011332749272696674 2.095120544254314e-5 -0.00016243959544226527 5.0393224228173494e-5 2.8671789550571702e-5 -2.0645064068958163e-5 -0.00015781042748130858 1.9534201783244498e-5 9.886213774734642e-6 6.808954367443221e-6 -0.00014567660400643945 -9.86817103694193e-5 6.68243010295555e-5 -4.046106550958939e-5 4.453890142031014e-5 -4.78151450806763e-5 0.00010242379357805476 -6.231554289115593e-5 -1.632915882510133e-5 7.82483330112882e-5 -3.7365440221037716e-5 1.6026859839257668e-7 5.51473131054081e-5 -1.3319667232281063e-5 -0.00016877359303180128 0.0001920819195220247 5.9910678828600794e-5 0.00017845621914602816; 4.951026494381949e-5 -1.5327297660405748e-5 -0.00016002464690245688 -2.0942901755915955e-5 8.728279499337077e-5 3.8725796912331134e-5 2.801238588290289e-5 1.2266802514204755e-5 -6.822432624176145e-5 -4.340847590356134e-5 -3.1815095553611172e-6 3.3622694900259376e-5 -3.3028329198714346e-5 0.00012200776836834848 -7.75428707129322e-5 -6.265148113016039e-5 -5.542874714592472e-5 2.364813190069981e-5 8.877604159351904e-6 2.9252865715534426e-5 -8.798505587037653e-5 8.313224680023268e-5 -0.00013218315143603832 -7.551062299171463e-5 0.000161833202582784 -7.737338455626741e-5 -2.8199927328387275e-5 -2.449767407597392e-6 1.9357245037099347e-5 -4.163317134953104e-5 0.0001951824815478176 0.0001935601030709222; 2.4334309273399413e-5 -1.8012100554187782e-5 -0.00011198838183190674 8.645114576211199e-5 -3.841124635073356e-5 0.00012686057016253471 0.00016552998567931354 5.129957207827829e-5 -7.127347635105252e-5 -8.310552220791578e-5 0.00010924212983809412 5.8726825955091044e-5 -9.073325054487213e-5 -0.0001587352599017322 -0.00010254180961055681 -0.00020244425104465336 -7.48857855796814e-5 8.424627594649792e-6 7.637192902620882e-5 9.15042546694167e-5 -0.00014364537491928786 8.96483106771484e-5 1.6997013517539017e-5 -0.0001308244391111657 5.973728184471838e-5 7.24627825547941e-5 -0.0001469875278417021 -0.0002337783807888627 -1.003574925562134e-5 -2.9793885914841667e-5 2.7507614504429512e-5 4.387572698760778e-5; 6.295203638728708e-5 -0.0001264522288693115 -9.073207183973864e-5 -9.134207357419655e-5 -0.00013391024549491704 1.4404124158318155e-5 -2.2943688236409798e-5 -9.225640678778291e-5 -3.782984640565701e-5 -1.7263429981539957e-5 -8.736764721106738e-5 8.946874731918797e-5 -4.267640906618908e-5 -6.563266651937738e-5 -7.833100244170055e-5 2.8248937269381713e-6 4.299664851714624e-6 -0.00016002195479813963 -1.6294668967020698e-5 -6.58607532386668e-5 -5.190408046473749e-5 6.375257362378761e-5 -0.00011367948172846809 6.460185977630317e-5 9.781469088920858e-6 2.5922046916093677e-5 0.00023445955594070256 -3.727988951141015e-5 -5.314336158335209e-5 0.0002186963683925569 3.5302356991451234e-5 2.1888721676077694e-5; -0.00010647908493410796 -6.876477709738538e-5 0.00019171844178345054 -4.920962601318024e-5 4.959371653967537e-5 -0.00015652982983738184 -8.026311843423173e-5 -2.9821610951330513e-5 -9.87901003099978e-5 -7.181416731327772e-5 0.00012342396075837314 2.7433190552983433e-5 0.00014545948943123221 -0.00012906383199151605 2.8615670089493506e-5 9.059365402208641e-5 -5.268862878438085e-5 4.367760266177356e-5 2.462259726598859e-5 -0.00023605213209521025 6.230096914805472e-5 -8.785520913079381e-5 3.5168795875506476e-5 -4.99921188747976e-5 6.46889311610721e-5 -0.00019670056644827127 -0.00026613648515194654 -7.319911674130708e-5 4.916332909488119e-5 -7.07233848515898e-5 -7.869206456234679e-5 -3.230481161153875e-5; -9.19397352845408e-5 -1.6640236935927533e-5 0.00010852261038962752 -0.0002121447178069502 -1.927547600644175e-5 -9.511670668871375e-7 0.00014113033830653876 8.265352516900748e-5 0.00020127883180975914 -0.00011502951383590698 4.069467104272917e-5 8.898750093067065e-5 -0.00010638104868121445 -9.76928640739061e-5 6.44306346657686e-5 0.00011508062016218901 4.897011422144715e-6 -7.640933472430333e-5 -8.466745930491015e-5 0.00011398435162845999 2.7418762329034507e-5 -7.377620931947604e-5 -0.00017452130850870162 -2.3060356397763826e-5 -0.00010372330143582076 1.3288659829413518e-5 8.848053403198719e-5 -0.00019720717682503164 -0.00015424827870447189 2.57119882007828e-6 -1.8169486793340184e-5 -7.690730853937566e-5; 0.00010071400902234018 -1.1918057680304628e-5 7.810291572241113e-5 -5.1266841182950884e-5 -5.560503632295877e-5 0.00012966264330316335 4.704469029093161e-5 -1.9116851035505533e-6 -0.00016081628564279526 7.443156937370077e-5 -2.8829321308876388e-5 2.078657053061761e-5 -3.992642814409919e-5 -0.0001563769328640774 5.8183584769722074e-5 8.129122761602048e-6 -6.343689165078104e-5 0.0001211809430969879 -0.00019924379012081772 -0.00017191945516970009 -2.4149676391971298e-5 -2.401400706730783e-5 0.00018512809765525162 0.00010200590622844175 0.0001745126792229712 5.9854774008272216e-5 -2.857472100004088e-5 0.00013421563198789954 -0.00017002028471324593 -0.00011199649452464655 -6.650551222264767e-5 0.00020709956879727542; -3.290334689154406e-6 0.00020503340056166053 8.881041867425665e-5 4.483790689846501e-5 -8.706872904440388e-5 0.00011339433694956824 7.982010720297694e-5 0.00016454119759146124 5.518259058590047e-5 0.00020737526938319206 -5.3931722504785284e-5 -9.531842806609347e-5 6.496321293525398e-5 -4.169013482169248e-5 -1.083374627341982e-5 0.00021134574490133673 6.145341467345133e-5 -4.920766514260322e-5 7.256426670210203e-6 9.413436782779172e-5 -9.47484077187255e-5 8.993445953819901e-5 0.00012307778524700552 -5.661605973728001e-5 1.9992689885839354e-6 0.0001599911629455164 -0.00012656069884542376 -4.4987216824665666e-5 9.240926010534167e-5 -1.0582937647996005e-5 0.00016095538740046322 -2.2632875698036514e-5; -5.405939646152547e-6 9.307573236583266e-7 -4.641104533220641e-5 -2.820241206791252e-5 -8.139970304910094e-5 -0.00013091946311760694 6.362377575896971e-7 -0.00012179058830952272 7.939913484733552e-5 -0.00015905943291727453 -2.1291167286108248e-5 -0.00010595170897431672 -3.8892929296707734e-5 2.0163131921435706e-5 8.40665670693852e-5 3.307816223241389e-5 0.00023594856611452997 -3.5434011806501076e-5 -4.5019227400189266e-5 0.00012131354014854878 -7.913775334600359e-5 -0.0002683046041056514 3.6860776162939146e-5 1.0231893611489795e-5 0.00016550101281609386 -2.7606007279246114e-5 3.803275103564374e-5 6.839250272605568e-5 -5.283697464619763e-5 0.00018663881928659976 -3.474174809525721e-5 3.769374961848371e-5; -0.00015594885917380452 4.80371672892943e-5 -0.00010172898328164592 5.552818765863776e-5 0.00015005047316662967 -0.00013480284542310983 -0.00010448869579704478 -0.00016123281966429204 -2.1036568796262145e-5 0.00011135856038890779 -0.00018874714442063123 -8.785194040683564e-6 -5.8102927141590044e-5 -8.021640678634867e-5 -0.00010484486119821668 2.6035600967588834e-5 7.68893733038567e-5 -0.00010625598224578425 0.00010102279338752851 3.9147162169683725e-5 -3.434011887293309e-5 -5.964718366158195e-5 -2.4965651391539723e-5 7.452567660948262e-5 5.676221553585492e-5 -0.00017402910452801734 0.00010657810344127938 -4.282216832507402e-5 9.151989070232958e-5 -0.0001406438968842849 -9.99575131572783e-5 8.420005906373262e-5; 3.548621680238284e-5 2.9863618692616e-5 -8.214120316551998e-5 0.0001635655207792297 -0.00022909861581865698 9.313924238085747e-5 6.062999091227539e-5 -7.669440674362704e-5 -6.686191773042083e-5 -9.038536518346518e-5 -5.120943023939617e-5 0.0001440758933313191 -0.00013841537293046713 -9.821521416597534e-6 -0.00019120587967336178 0.00019957662152592093 0.0001263878948520869 -5.536867683986202e-5 0.00019844071357510984 5.598729330813512e-5 4.6416073018917814e-5 -0.00012094870180590078 7.687937613809481e-5 4.4671851355815306e-5 0.0001756463898345828 5.1972372602904215e-5 6.445790495490655e-5 -0.00018492908566258848 -0.00010645703150657937 8.909722964745015e-5 7.649297185707837e-5 -3.211238799849525e-5; 4.1868683183565736e-5 -3.071039100177586e-5 -0.00012943912588525563 -1.754614459059667e-5 -0.000127310398966074 8.33343729027547e-5 9.897768177324906e-5 -0.00010930006101261824 7.982586976140738e-5 9.190587297780439e-5 4.683600855059922e-5 -3.7059394344396424e-6 -5.710511231882265e-6 -1.4753802133782301e-5 -0.00016252454952336848 8.32230034575332e-6 5.445048736874014e-5 0.00012156067532487214 -9.790220246941317e-6 0.0001653039362281561 -1.7137936083599925e-6 1.1979616829194129e-5 5.539335688808933e-5 -0.00010121554078068584 -1.0409351034468273e-6 -1.1542158063093666e-5 0.00014295370783656836 2.165104160667397e-5 -6.668147398158908e-5 7.0696507464163e-5 -4.2798015783773735e-5 2.7811020117951557e-5; -2.532826692913659e-5 0.00013122658128850162 -2.8057063900632784e-5 3.582014687708579e-5 -4.506225377554074e-5 4.9261860112892464e-5 -0.00019823724869638681 -4.617213562596589e-5 -5.958348992862739e-5 3.762732012546621e-5 3.244409890612587e-5 -0.00014681062020827085 -0.00014683349581900984 9.054842666955665e-5 4.913428347208537e-5 4.099287616554648e-5 -0.00017358409240841866 -8.598958811489865e-5 -0.00021768509759567678 -8.217606955440715e-5 3.770431794691831e-5 8.097058525891043e-6 9.980082541005686e-5 3.334106077090837e-5 -0.00012251509178895503 0.00010761500016087666 0.00018649086996447295 2.0906065401504748e-5 -8.663797052577138e-5 5.149301068740897e-5 0.00012691420852206647 -0.00013912188296671957; -2.00742779270513e-5 -2.384551589784678e-5 -3.0652870918856934e-5 -0.00012635986786335707 9.219767889590003e-6 0.00013077643234282732 -9.290023444918916e-5 -1.3494813174474984e-5 0.00013721530558541417 -3.967561497120187e-5 -9.963369666365907e-5 0.00015899795107543468 5.3383675549412146e-5 -6.003343060001498e-6 9.440587018616498e-5 -3.229927415304701e-6 3.269244189141318e-5 -1.0895862033066805e-5 7.907242252258584e-5 7.48599850339815e-5 0.00014270198880694807 2.286178460053634e-5 -6.975755241001025e-5 0.00012045708717778325 9.296696225646883e-5 0.0001155443605966866 6.941035098861903e-5 8.250509563367814e-5 0.00016367800708394498 -4.108583652850939e-6 3.128219077552785e-6 -6.332238990580663e-5; 0.000166121797519736 6.248905719985487e-6 -5.348670674720779e-5 3.9137292333180085e-5 6.50994770694524e-5 -9.512635006103665e-5 -9.428642079001293e-5 -0.00011857271601911634 0.00014290472608990967 -7.941459625726566e-5 -0.00019487942336127162 0.00018272059969604015 -5.5861462897155434e-5 -5.271595455269562e-6 -3.049369752261555e-6 -2.618683356558904e-5 -6.113604467827827e-5 -5.503781267179875e-6 -9.753769518283661e-6 4.169354781424772e-7 -0.00012961748871020973 4.2347917769802734e-5 0.00020345293160062283 0.00011407625424908474 -0.00023870979202911258 -0.00014857749920338392 0.00013385707279667258 5.268841778161004e-5 -6.126727384980768e-5 -3.919936716556549e-5 -0.00013906955427955836 0.00012998454621993005; 9.748159209266305e-5 7.274390372913331e-5 7.577667565783486e-5 0.00011819799692602828 0.00012662229710258543 -0.0001769914524629712 0.000288318406092003 -0.0001013447908917442 0.00014936296793166548 0.00014090767945162952 -8.454715134575963e-5 4.7592104237992316e-5 -6.198127084644511e-5 3.136343730147928e-5 6.618879706365988e-5 3.8444530218839645e-5 -8.462849109491799e-6 -8.225016790675e-5 -5.494413562701084e-5 3.99311538785696e-5 1.5003369298938196e-5 -5.953148502158001e-5 8.253998385043815e-5 -3.491389361442998e-5 0.00010703964653657749 -4.802087005373323e-6 0.00019097969925496727 -5.093285653856583e-5 3.843382000923157e-5 2.3095157303032465e-5 -8.697748853592202e-5 -7.696678949287161e-5; -2.1072210074635223e-5 2.358901610932662e-6 -0.00012510591477621347 -0.00010240108531434089 0.0001140839303843677 0.00022276307572610676 0.0001561086974106729 -7.65903532737866e-5 -3.387816468602978e-5 1.9540704670362175e-5 -2.3315562430070713e-5 -8.6971529526636e-5 4.1945502744056284e-5 3.2337931770598516e-5 -4.6819335693726316e-5 1.540997800475452e-5 -0.00019729729683604091 0.0001878565817605704 -7.939497663755901e-6 9.238693792212871e-7 -0.00010446535452501848 -0.00014348624972626567 7.517659105360508e-5 -2.4652244974276982e-5 -2.234361818409525e-5 -8.922003871703055e-6 -9.284893167205155e-5 -0.00012740293459501117 9.099852468352765e-5 5.9873327700188383e-5 -1.632718885957729e-5 0.00010489988198969513; -0.00027522759046405554 -9.361786942463368e-5 -6.243263487704098e-5 3.768975875573233e-5 -3.859818207274657e-6 0.00019726852769963443 7.318292045965791e-5 7.19380404916592e-5 0.000138926858198829 0.00012156338198110461 4.557715874398127e-5 -7.630119216628373e-5 6.769981700927019e-5 0.00010516179463593289 4.956183329341002e-5 6.198458140715957e-5 2.0436564227566123e-5 -0.0001307315833400935 -4.1752336983336136e-5 7.820146129233763e-5 0.00012737399083562195 8.448751759715378e-5 1.5097722098289523e-5 -9.294738265452906e-5 -0.0001918459456646815 2.6141378839383833e-5 1.8201744751422666e-5 -6.22006700723432e-5 0.00013549704453907907 0.00010391265095677227 2.770318133116234e-5 0.00017388908599969; 4.955788972438313e-5 -1.6406474969699048e-5 -5.015429633203894e-5 -1.9373459508642554e-5 -1.5793542843312025e-5 -7.847254892112687e-5 4.5386506826616824e-5 1.003445686365012e-5 -4.23613937527989e-6 6.561596092069522e-5 -4.86126373289153e-5 2.3592456273036078e-6 9.391162166139111e-5 -2.6184701710008085e-5 0.00010917904000962153 -2.1974381525069475e-5 -0.00014429049042519182 -3.2307660148944706e-5 5.823806714033708e-5 -7.907766121206805e-5 -8.596935367677361e-5 2.030890573223587e-5 0.0001380361063638702 4.416281808516942e-5 5.9456226153997704e-5 -4.602802073350176e-5 -9.346289880340919e-5 -3.4706667975115124e-6 5.7220240705646574e-5 -4.500206341617741e-5 0.00010185955034103245 0.0001024414086714387; -3.823506995104253e-5 -5.6128417782019824e-5 -0.0001282452285522595 6.524803029606119e-5 0.00018336060747969896 -5.0329676014371216e-5 8.167386840796098e-5 0.00011664406338240951 1.207341756526148e-5 0.00012642126239370555 -3.914466287824325e-5 1.8425126881993492e-6 -0.00015535025158897042 3.24872562487144e-5 9.326799772679806e-5 -6.258532084757462e-5 5.242965562501922e-5 2.5912715500453487e-5 0.00010690064664231613 -3.407000258448534e-5 3.2485950214322656e-5 -0.0001129679149016738 5.177210186957382e-5 -0.00011433687905082479 0.0001491793809691444 2.7192077141080517e-6 -9.985257929656655e-5 -8.969280315795913e-5 -8.67715152708115e-6 -6.761546683264896e-5 -0.00010904387454502285 1.9103930753772147e-5; 1.7649729215918342e-6 3.342990021337755e-5 4.340703890193254e-5 -1.875355519587174e-5 8.069256000453606e-5 -6.234008469618857e-5 8.369092392968014e-5 -0.00019233294005971402 0.00022725600865669549 8.349120616912842e-5 6.0045360442018136e-5 5.268828317639418e-5 0.00013327249325811863 -7.735010285614408e-7 -7.672329957131296e-5 -4.517462002695538e-5 1.0317836313333828e-5 0.00012960017193108797 -0.00010631080658640712 8.633271136204712e-6 0.00018181496125180274 -1.593080014572479e-5 -4.738618736155331e-5 -5.8881436416413635e-5 7.113266474334523e-5 9.685048098617699e-6 -1.8066766642732546e-5 0.00010160616511711851 -3.7635836633853614e-5 -0.00013329691137187183 -0.0002489819598849863 7.776706479489803e-5; 0.00010300002031726763 -7.488738629035652e-5 -0.0002437196671962738 1.409053584211506e-5 0.00017816323088482022 -0.00019117847841698676 2.8733284125337377e-5 -3.529497917043045e-5 4.0042596083367243e-5 -5.463402339955792e-5 -0.00012753947521559894 -6.0610156651819125e-5 1.3021068298257887e-5 5.923292337683961e-5 -0.00016325977048836648 -2.171527739847079e-6 6.068706625228515e-6 -7.599986565764993e-5 -9.461311128688976e-5 2.968869375763461e-5 -7.364524935837835e-5 -4.114430794288637e-6 0.0001211140988743864 2.793795101752039e-5 9.048701031133533e-5 0.00013239527470432222 -6.243748066481203e-5 7.184594869613647e-5 9.416184184374288e-5 -8.989481284515932e-5 -8.209576662920881e-6 -3.936138818971813e-5; -6.116025178926066e-5 -9.474167018197477e-5 7.152702164603397e-5 -0.00023639023129362613 5.356800465960987e-5 0.0002500036789570004 -0.00029104005079716444 5.8974354033125564e-5 -0.00023659240105189383 -0.0001000445190584287 -5.5840169807197526e-5 -9.00545956028509e-7 4.0760412957752123e-5 -2.3267359210876748e-5 0.00012463780876714736 -0.00012320584210101515 1.3917734577262308e-5 -5.035507638240233e-5 -6.516801659017801e-5 -6.065718116587959e-5 -3.400189598323777e-5 -9.058501746039838e-5 0.0002537924447096884 7.222445128718391e-5 -4.718687341664918e-5 -0.00010069366544485092 -7.38385206204839e-5 -3.6349560105009004e-5 4.896233440376818e-5 -0.00010655423830030486 -1.9349348804098554e-5 -1.5070883819134906e-5; -2.2678599634673446e-5 5.8219833590555936e-5 -7.442795322276652e-5 0.0001376736763631925 0.00012771568435709924 -0.00010184129496337846 6.144091457827017e-5 1.8778735466185026e-5 -0.00012474964023567736 -0.00018880223797168583 3.735041173058562e-5 4.0341634303331375e-5 5.9921283536823466e-5 0.00012631539721041918 5.923600110691041e-5 8.1654179666657e-5 -2.749025497905677e-6 7.735728286206722e-5 -9.352378401672468e-5 -0.00022587476996704936 -3.815074160229415e-5 -5.6857603340176865e-5 2.4990713427541777e-5 1.0730444955697749e-5 -9.98539399006404e-5 0.00015063582395669073 -7.781399472150952e-5 3.2270309020532295e-5 -0.0001369398960378021 2.3901055101305246e-5 5.107907418278046e-5 -6.80840530549176e-5; 6.210515857674181e-5 -3.7043394058855483e-6 5.1657098083524033e-5 -3.217544872313738e-5 -8.932677883422002e-5 0.00011483902926556766 7.510659634135664e-5 0.00017566794122103602 0.00025532522704452276 4.776594141731039e-5 -2.2731062927050516e-5 5.855439303559251e-5 0.000115292175905779 6.401740392902866e-5 0.00012503787002060562 0.00013153970940038562 -3.7848250940442085e-5 -3.4302581752854167e-6 -5.28943601239007e-5 -7.206924783531576e-5 -4.525488111539744e-5 0.00025406089844182134 -3.534059942467138e-5 -2.2638560039922595e-5 0.00011529421317391098 0.00010437796299811453 -0.00010650316107785329 -1.4229224689188413e-5 6.285558629315346e-5 -1.7171007129945792e-5 -0.00018311127496417612 1.811042966437526e-5], bias = [0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0; 0.0;;]), layer_4 = (weight = [0.00010784265032270923 -2.9762746635242365e-5 -7.860020559746772e-5 7.98071050667204e-5 -7.634957000846043e-5 -6.577673048013821e-5 -8.459465607302263e-5 3.1334122468251735e-5 3.4221702662762254e-6 0.0001074318279279396 -3.846182153210975e-5 3.827816544799134e-5 -6.133932038210332e-5 8.387087291339412e-5 -0.000125478210975416 -1.9419589079916477e-5 -0.000151343789184466 8.45614486024715e-5 -7.910614658612758e-5 -5.0588601880008355e-5 -0.00014900714450050145 -6.848078919574618e-5 -2.5551367798470892e-5 -6.541466427734122e-5 -0.00018350957543589175 -0.00017597667465452105 -5.6173212215071544e-5 -5.565550964092836e-5 7.279034616658464e-5 -6.8475901571218856e-6 0.00010433266288600862 -2.92871459350863e-6; 3.5102784750051796e-5 1.0631198165356182e-5 -0.00014356661995407194 -2.3903501642053016e-5 0.00017306309018749744 3.41797640430741e-5 0.00013351610687095672 0.00014698300219606608 0.00022513230214826763 -5.263533239485696e-5 -0.00010518265480641276 5.1612663810374215e-5 6.279592344071716e-5 5.87703561905073e-6 6.818250403739512e-5 0.00018816266674548388 -4.9655340262688696e-5 4.3108615500386804e-5 -5.015717033529654e-5 6.466867489507422e-5 2.4073196982499212e-5 3.650915459729731e-5 0.00018315402849111706 1.2278619578864891e-5 8.699770842213184e-5 6.216787733137608e-5 -1.6548397979931906e-5 -9.49859750107862e-5 -7.579045359307202e-6 7.79889669502154e-5 -7.616070797666907e-5 -0.00010732111695688218], bias = [0.0; 0.0;;]))

Visualizing the Results

Let us now plot the loss over time

begin
    fig = Figure()
    ax = CairoMakie.Axis(fig[1, 1]; xlabel="Iteration", ylabel="Loss")

    lines!(ax, losses; linewidth=4, alpha=0.75)
    scatter!(ax, 1:length(losses), losses; marker=:circle, markersize=12, strokewidth=2)

    fig
end

Finally let us visualize the results

prob_nn = ODEProblem(ODE_model, u0, tspan, res.u)
soln_nn = Array(solve(prob_nn, RK4(); u0, p=res.u, saveat=tsteps, dt, adaptive=false))
waveform_nn_trained = first(compute_waveform(
    dt_data, soln_nn, mass_ratio, ode_model_params))

begin
    fig = Figure()
    ax = CairoMakie.Axis(fig[1, 1]; xlabel="Time", ylabel="Waveform")

    l1 = lines!(ax, tsteps, waveform; linewidth=2, alpha=0.75)
    s1 = scatter!(
        ax, tsteps, waveform; marker=:circle, alpha=0.5, strokewidth=2, markersize=12)

    l2 = lines!(ax, tsteps, waveform_nn; linewidth=2, alpha=0.75)
    s2 = scatter!(
        ax, tsteps, waveform_nn; marker=:circle, alpha=0.5, strokewidth=2, markersize=12)

    l3 = lines!(ax, tsteps, waveform_nn_trained; linewidth=2, alpha=0.75)
    s3 = scatter!(ax, tsteps, waveform_nn_trained; marker=:circle,
        alpha=0.5, strokewidth=2, markersize=12)

    axislegend(ax, [[l1, s1], [l2, s2], [l3, s3]],
        ["Waveform Data", "Waveform Neural Net (Untrained)", "Waveform Neural Net"];
        position=:lb)

    fig
end

Appendix

using InteractiveUtils
InteractiveUtils.versioninfo()

if @isdefined(LuxDeviceUtils)
    if @isdefined(CUDA) && LuxDeviceUtils.functional(LuxCUDADevice)
        println()
        CUDA.versioninfo()
    end

    if @isdefined(AMDGPU) && LuxDeviceUtils.functional(LuxAMDGPUDevice)
        println()
        AMDGPU.versioninfo()
    end
end

Julia Version 1.10.4
Commit 48d4fd48430 (2024-06-04 10:41 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 48 × AMD EPYC 7402 24-Core Processor
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, znver2)
Threads: 4 default, 0 interactive, 2 GC (on 2 virtual cores)
Environment:
  JULIA_CPU_THREADS = 2
  JULIA_DEPOT_PATH = /root/.cache/julia-buildkite-plugin/depots/01872db4-8c79-43af-ab7d-12abac4f24f6
  LD_LIBRARY_PATH = /usr/local/nvidia/lib:/usr/local/nvidia/lib64
  JULIA_PKG_SERVER = 
  JULIA_NUM_THREADS = 4
  JULIA_CUDA_HARD_MEMORY_LIMIT = 100%
  JULIA_PKG_PRECOMPILE_AUTO = 0
  JULIA_DEBUG = Literate

CUDA runtime 12.5, artifact installation
CUDA driver 12.5
NVIDIA driver 555.42.6

CUDA libraries: 
- CUBLAS: 12.5.3
- CURAND: 10.3.6
- CUFFT: 11.2.3
- CUSOLVER: 11.6.3
- CUSPARSE: 12.5.1
- CUPTI: 2024.2.1 (API 23.0.0)
- NVML: 12.0.0+555.42.6

Julia packages: 
- CUDA: 5.4.3
- CUDA_Driver_jll: 0.9.2+0
- CUDA_Runtime_jll: 0.14.1+0

Toolchain:
- Julia: 1.10.4
- LLVM: 15.0.7

Environment:
- JULIA_CUDA_HARD_MEMORY_LIMIT: 100%

1 device:
  0: NVIDIA A100-PCIE-40GB MIG 1g.5gb (sm_80, 4.735 GiB / 4.750 GiB available)

---

This page was generated using Literate.jl.