Training a PINN on 2D PDE
In this tutorial we will go over using a PINN to solve 2D PDEs. We will be using the system from NeuralPDE Tutorials. However, we will be using our custom loss function and use nested AD capabilities of Lux.jl.
This is a demonstration of Lux.jl. For serious use cases of PINNs, please refer to the package: NeuralPDE.jl.
Package Imports
using Lux,
Optimisers,
Random,
Printf,
Statistics,
MLUtils,
OnlineStats,
CairoMakie,
Reactant,
Enzyme
const xdev = reactant_device(; force=true)
const cdev = cpu_device()Problem Definition
Since Lux supports efficient nested AD upto 2nd order, we will rewrite the problem with first order derivatives, so that we can compute the gradients of the loss using 2nd order AD.
Define the Neural Networks
All the networks take 3 input variables and output a scalar value. Here, we will define a wrapper over the 3 networks, so that we can train them using Training.TrainState.
struct PINN{M} <: AbstractLuxWrapperLayer{:model}
model::M
end
function PINN(; hidden_dims::Int=32)
return PINN(
Chain(
Dense(3 => hidden_dims, tanh),
Dense(hidden_dims => hidden_dims, tanh),
Dense(hidden_dims => hidden_dims, tanh),
Dense(hidden_dims => 1),
),
)
endDefine the Loss Functions
We will define a custom loss function to compute the loss using 2nd order AD. For that, first we'll need to define the derivatives of our model:
function ∂u_∂t(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][3, :]
end
function ∂u_∂x(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][1, :]
end
function ∂u_∂y(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][2, :]
end
function ∂²u_∂x²(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ ∂u_∂x, Enzyme.Const(model), xyt)[2][1, :]
end
function ∂²u_∂y²(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ ∂u_∂y, Enzyme.Const(model), xyt)[2][2, :]
endWe will use the following loss function
function physics_informed_loss_function(model::StatefulLuxLayer, xyt::AbstractArray)
return mean(abs2, ∂u_∂t(model, xyt) .- ∂²u_∂x²(model, xyt) .- ∂²u_∂y²(model, xyt))
endAdditionally, we need to compute the loss with respect to the boundary conditions.
function mse_loss_function(
model::StatefulLuxLayer, target::AbstractArray, xyt::AbstractArray
)
return MSELoss()(model(xyt), target)
end
function loss_function(model, ps, st, (xyt, target_data, xyt_bc, target_bc))
smodel = StatefulLuxLayer(model, ps, st)
physics_loss = physics_informed_loss_function(smodel, xyt)
data_loss = mse_loss_function(smodel, target_data, xyt)
bc_loss = mse_loss_function(smodel, target_bc, xyt_bc)
loss = physics_loss + data_loss + bc_loss
return loss, smodel.st, (; physics_loss, data_loss, bc_loss)
endGenerate the Data
We will generate some random data to train the model on. We will take data on a square spatial and temporal domain
analytical_solution(x, y, t) = @. exp(x + y) * cos(x + y + 4t)
analytical_solution(xyt) = analytical_solution(xyt[1, :], xyt[2, :], xyt[3, :])begin
grid_len = 16
grid = range(0.0f0, 2.0f0; length=grid_len)
xyt = stack([[elem...] for elem in vec(collect(Iterators.product(grid, grid, grid)))])
target_data = reshape(analytical_solution(xyt), 1, :)
bc_len = 512
x = collect(range(0.0f0, 2.0f0; length=bc_len))
y = collect(range(0.0f0, 2.0f0; length=bc_len))
t = collect(range(0.0f0, 2.0f0; length=bc_len))
xyt_bc = hcat(
stack((x, y, zeros(Float32, bc_len)); dims=1),
stack((zeros(Float32, bc_len), y, t); dims=1),
stack((ones(Float32, bc_len) .* 2, y, t); dims=1),
stack((x, zeros(Float32, bc_len), t); dims=1),
stack((x, ones(Float32, bc_len) .* 2, t); dims=1),
)
target_bc = reshape(analytical_solution(xyt_bc), 1, :)
min_target_bc, max_target_bc = extrema(target_bc)
min_data, max_data = extrema(target_data)
min_pde_val, max_pde_val = min(min_data, min_target_bc), max(max_data, max_target_bc)
xyt = (xyt .- minimum(xyt)) ./ (maximum(xyt) .- minimum(xyt))
xyt_bc = (xyt_bc .- minimum(xyt_bc)) ./ (maximum(xyt_bc) .- minimum(xyt_bc))
target_bc = (target_bc .- min_pde_val) ./ (max_pde_val - min_pde_val)
target_data = (target_data .- min_pde_val) ./ (max_pde_val - min_pde_val)
endTraining
function train_model(
xyt,
target_data,
xyt_bc,
target_bc;
seed::Int=0,
maxiters::Int=50000,
hidden_dims::Int=128,
)
rng = Random.default_rng()
Random.seed!(rng, seed)
pinn = PINN(; hidden_dims)
ps, st = Lux.setup(rng, pinn) |> xdev
bc_dataloader =
DataLoader((xyt_bc, target_bc); batchsize=128, shuffle=true, partial=false) |> xdev
pde_dataloader =
DataLoader((xyt, target_data); batchsize=128, shuffle=true, partial=false) |> xdev
train_state = Training.TrainState(pinn, ps, st, Adam(0.005f0))
lr = i -> i < 5000 ? 0.005f0 : (i < 10000 ? 0.0005f0 : 0.00005f0)
total_loss_tracker, physics_loss_tracker, data_loss_tracker, bc_loss_tracker = ntuple(
_ -> OnlineStats.CircBuff(Float32, 32; rev=true), 4
)
iter = 1
for ((xyt_batch, target_data_batch), (xyt_bc_batch, target_bc_batch)) in
zip(Iterators.cycle(pde_dataloader), Iterators.cycle(bc_dataloader))
Optimisers.adjust!(train_state, lr(iter))
_, loss, stats, train_state = Training.single_train_step!(
AutoEnzyme(),
loss_function,
(xyt_batch, target_data_batch, xyt_bc_batch, target_bc_batch),
train_state;
return_gradients=Val(false),
)
fit!(total_loss_tracker, Float32(loss))
fit!(physics_loss_tracker, Float32(stats.physics_loss))
fit!(data_loss_tracker, Float32(stats.data_loss))
fit!(bc_loss_tracker, Float32(stats.bc_loss))
mean_loss = mean(OnlineStats.value(total_loss_tracker))
mean_physics_loss = mean(OnlineStats.value(physics_loss_tracker))
mean_data_loss = mean(OnlineStats.value(data_loss_tracker))
mean_bc_loss = mean(OnlineStats.value(bc_loss_tracker))
isnan(loss) && throw(ArgumentError("NaN Loss Detected"))
if iter % 1000 == 1 || iter == maxiters
@printf(
"Iteration: [%6d/%6d] \t Loss: %.9f (%.9f) \t Physics Loss: %.9f \
(%.9f) \t Data Loss: %.9f (%.9f) \t BC \
Loss: %.9f (%.9f)\n",
iter,
maxiters,
loss,
mean_loss,
stats.physics_loss,
mean_physics_loss,
stats.data_loss,
mean_data_loss,
stats.bc_loss,
mean_bc_loss
)
end
iter += 1
iter ≥ maxiters && break
end
return StatefulLuxLayer(pinn, cdev(train_state.parameters), cdev(train_state.states))
end
trained_model = train_model(xyt, target_data, xyt_bc, target_bc)WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1761574507.847845 3920505 service.cc:158] XLA service 0x2d259e90 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
I0000 00:00:1761574507.847931 3920505 service.cc:166] StreamExecutor device (0): Quadro RTX 5000, Compute Capability 7.5
I0000 00:00:1761574507.849017 3920505 se_gpu_pjrt_client.cc:1339] Using BFC allocator.
I0000 00:00:1761574507.849061 3920505 gpu_helpers.cc:136] XLA backend allocating 12526534656 bytes on device 0 for BFCAllocator.
I0000 00:00:1761574507.849110 3920505 gpu_helpers.cc:177] XLA backend will use up to 4175511552 bytes on device 0 for CollectiveBFCAllocator.
I0000 00:00:1761574507.858514 3920505 cuda_dnn.cc:463] Loaded cuDNN version 91200
Iteration: [ 1/ 50000] Loss: 20.523933411 (20.523933411) Physics Loss: 16.931318283 (16.931318283) Data Loss: 2.007483006 (2.007483006) BC Loss: 1.585133314 (1.585133314)
Iteration: [ 1001/ 50000] Loss: 0.017368635 (0.019241156) Physics Loss: 0.000384367 (0.000523641) Data Loss: 0.005318496 (0.007538575) BC Loss: 0.011665772 (0.011178941)
Iteration: [ 2001/ 50000] Loss: 0.015431631 (0.018665718) Physics Loss: 0.001248533 (0.001662089) Data Loss: 0.004322525 (0.006408243) BC Loss: 0.009860573 (0.010595389)
Iteration: [ 3001/ 50000] Loss: 0.015749749 (0.015216012) Physics Loss: 0.000569918 (0.001279069) Data Loss: 0.004014887 (0.004232429) BC Loss: 0.011164944 (0.009704513)
Iteration: [ 4001/ 50000] Loss: 0.009719813 (0.008716348) Physics Loss: 0.002388457 (0.003383033) Data Loss: 0.003175432 (0.002104753) BC Loss: 0.004155925 (0.003228562)
Iteration: [ 5001/ 50000] Loss: 0.005180652 (0.005663842) Physics Loss: 0.001528659 (0.002126877) Data Loss: 0.001560176 (0.001513253) BC Loss: 0.002091817 (0.002023713)
Iteration: [ 6001/ 50000] Loss: 0.001401311 (0.001579032) Physics Loss: 0.000423297 (0.000411505) Data Loss: 0.000627589 (0.000790539) BC Loss: 0.000350425 (0.000376988)
Iteration: [ 7001/ 50000] Loss: 0.001472101 (0.000994279) Physics Loss: 0.000400396 (0.000312601) Data Loss: 0.000957632 (0.000519736) BC Loss: 0.000114073 (0.000161942)
Iteration: [ 8001/ 50000] Loss: 0.000760192 (0.000831841) Physics Loss: 0.000370746 (0.000340470) Data Loss: 0.000310544 (0.000397757) BC Loss: 0.000078903 (0.000093615)
Iteration: [ 9001/ 50000] Loss: 0.001032158 (0.000814782) Physics Loss: 0.000305266 (0.000424161) Data Loss: 0.000596162 (0.000318210) BC Loss: 0.000130729 (0.000072411)
Iteration: [ 10001/ 50000] Loss: 0.000632554 (0.000672352) Physics Loss: 0.000197734 (0.000301190) Data Loss: 0.000360583 (0.000308381) BC Loss: 0.000074237 (0.000062781)
Iteration: [ 11001/ 50000] Loss: 0.000362428 (0.000382574) Physics Loss: 0.000125533 (0.000072012) Data Loss: 0.000192466 (0.000272181) BC Loss: 0.000044429 (0.000038381)
Iteration: [ 12001/ 50000] Loss: 0.000258560 (0.000365691) Physics Loss: 0.000048193 (0.000078385) Data Loss: 0.000165178 (0.000250273) BC Loss: 0.000045189 (0.000037033)
Iteration: [ 13001/ 50000] Loss: 0.000314376 (0.000335677) Physics Loss: 0.000067176 (0.000073715) Data Loss: 0.000208000 (0.000227852) BC Loss: 0.000039200 (0.000034110)
Iteration: [ 14001/ 50000] Loss: 0.000382924 (0.000343026) Physics Loss: 0.000065185 (0.000075548) Data Loss: 0.000285065 (0.000237004) BC Loss: 0.000032675 (0.000030475)
Iteration: [ 15001/ 50000] Loss: 0.000279566 (0.000290738) Physics Loss: 0.000080205 (0.000059520) Data Loss: 0.000163266 (0.000201267) BC Loss: 0.000036095 (0.000029950)
Iteration: [ 16001/ 50000] Loss: 0.000244786 (0.000304263) Physics Loss: 0.000062074 (0.000070639) Data Loss: 0.000145659 (0.000204590) BC Loss: 0.000037052 (0.000029034)
Iteration: [ 17001/ 50000] Loss: 0.000418657 (0.000296086) Physics Loss: 0.000090505 (0.000062406) Data Loss: 0.000302952 (0.000205329) BC Loss: 0.000025200 (0.000028352)
Iteration: [ 18001/ 50000] Loss: 0.000220707 (0.000285989) Physics Loss: 0.000047528 (0.000060282) Data Loss: 0.000141771 (0.000198929) BC Loss: 0.000031409 (0.000026778)
Iteration: [ 19001/ 50000] Loss: 0.000249162 (0.000288640) Physics Loss: 0.000087009 (0.000061922) Data Loss: 0.000143188 (0.000200855) BC Loss: 0.000018965 (0.000025863)
Iteration: [ 20001/ 50000] Loss: 0.000320152 (0.000264607) Physics Loss: 0.000063959 (0.000056454) Data Loss: 0.000234710 (0.000184159) BC Loss: 0.000021483 (0.000023994)
Iteration: [ 21001/ 50000] Loss: 0.000303469 (0.000246624) Physics Loss: 0.000043297 (0.000047528) Data Loss: 0.000229889 (0.000174633) BC Loss: 0.000030283 (0.000024463)
Iteration: [ 22001/ 50000] Loss: 0.000162006 (0.000247801) Physics Loss: 0.000026536 (0.000053575) Data Loss: 0.000110369 (0.000170920) BC Loss: 0.000025100 (0.000023307)
Iteration: [ 23001/ 50000] Loss: 0.000247045 (0.000261523) Physics Loss: 0.000048755 (0.000057545) Data Loss: 0.000175695 (0.000180751) BC Loss: 0.000022595 (0.000023227)
Iteration: [ 24001/ 50000] Loss: 0.000297020 (0.000247546) Physics Loss: 0.000056966 (0.000049992) Data Loss: 0.000219789 (0.000173168) BC Loss: 0.000020265 (0.000024385)
Iteration: [ 25001/ 50000] Loss: 0.000187981 (0.000232632) Physics Loss: 0.000020291 (0.000039371) Data Loss: 0.000148261 (0.000170987) BC Loss: 0.000019428 (0.000022274)
Iteration: [ 26001/ 50000] Loss: 0.000245584 (0.000246094) Physics Loss: 0.000063731 (0.000056586) Data Loss: 0.000160293 (0.000167173) BC Loss: 0.000021560 (0.000022334)
Iteration: [ 27001/ 50000] Loss: 0.000262562 (0.000255008) Physics Loss: 0.000080459 (0.000062649) Data Loss: 0.000155585 (0.000168712) BC Loss: 0.000026518 (0.000023647)
Iteration: [ 28001/ 50000] Loss: 0.000209314 (0.000222538) Physics Loss: 0.000036939 (0.000039556) Data Loss: 0.000153870 (0.000161040) BC Loss: 0.000018506 (0.000021942)
Iteration: [ 29001/ 50000] Loss: 0.000215636 (0.000230862) Physics Loss: 0.000033974 (0.000047063) Data Loss: 0.000149528 (0.000162132) BC Loss: 0.000032133 (0.000021668)
Iteration: [ 30001/ 50000] Loss: 0.000208729 (0.000227855) Physics Loss: 0.000026107 (0.000042861) Data Loss: 0.000158081 (0.000164112) BC Loss: 0.000024540 (0.000020882)
Iteration: [ 31001/ 50000] Loss: 0.000265276 (0.000218171) Physics Loss: 0.000029673 (0.000038187) Data Loss: 0.000215964 (0.000159851) BC Loss: 0.000019639 (0.000020134)
Iteration: [ 32001/ 50000] Loss: 0.000224519 (0.000216053) Physics Loss: 0.000061066 (0.000040443) Data Loss: 0.000145471 (0.000154796) BC Loss: 0.000017982 (0.000020814)
Iteration: [ 33001/ 50000] Loss: 0.000206821 (0.000215371) Physics Loss: 0.000050756 (0.000041650) Data Loss: 0.000134251 (0.000152131) BC Loss: 0.000021815 (0.000021589)
Iteration: [ 34001/ 50000] Loss: 0.000203319 (0.000204159) Physics Loss: 0.000040416 (0.000038230) Data Loss: 0.000144833 (0.000146631) BC Loss: 0.000018069 (0.000019298)
Iteration: [ 35001/ 50000] Loss: 0.000150320 (0.000211646) Physics Loss: 0.000026159 (0.000041630) Data Loss: 0.000100420 (0.000150200) BC Loss: 0.000023740 (0.000019816)
Iteration: [ 36001/ 50000] Loss: 0.000174350 (0.000207113) Physics Loss: 0.000039072 (0.000039784) Data Loss: 0.000118376 (0.000148643) BC Loss: 0.000016902 (0.000018687)
Iteration: [ 37001/ 50000] Loss: 0.000296712 (0.000198252) Physics Loss: 0.000058570 (0.000031682) Data Loss: 0.000217866 (0.000147088) BC Loss: 0.000020276 (0.000019482)
Iteration: [ 38001/ 50000] Loss: 0.000258811 (0.000219698) Physics Loss: 0.000036458 (0.000048734) Data Loss: 0.000198889 (0.000153059) BC Loss: 0.000023463 (0.000017905)
Iteration: [ 39001/ 50000] Loss: 0.000189867 (0.000198078) Physics Loss: 0.000050881 (0.000032471) Data Loss: 0.000124566 (0.000146880) BC Loss: 0.000014420 (0.000018727)
Iteration: [ 40001/ 50000] Loss: 0.000195661 (0.000199009) Physics Loss: 0.000045046 (0.000033782) Data Loss: 0.000128464 (0.000145595) BC Loss: 0.000022151 (0.000019631)
Iteration: [ 41001/ 50000] Loss: 0.000171645 (0.000210088) Physics Loss: 0.000031580 (0.000042804) Data Loss: 0.000117591 (0.000148266) BC Loss: 0.000022474 (0.000019017)
Iteration: [ 42001/ 50000] Loss: 0.000176154 (0.000196715) Physics Loss: 0.000022212 (0.000029369) Data Loss: 0.000139413 (0.000148044) BC Loss: 0.000014529 (0.000019301)
Iteration: [ 43001/ 50000] Loss: 0.000194663 (0.000201880) Physics Loss: 0.000024540 (0.000037395) Data Loss: 0.000153372 (0.000144963) BC Loss: 0.000016751 (0.000019522)
Iteration: [ 44001/ 50000] Loss: 0.000176828 (0.000196648) Physics Loss: 0.000017554 (0.000031234) Data Loss: 0.000144832 (0.000144287) BC Loss: 0.000014442 (0.000021126)
Iteration: [ 45001/ 50000] Loss: 0.000310105 (0.000226800) Physics Loss: 0.000076172 (0.000045668) Data Loss: 0.000203600 (0.000156299) BC Loss: 0.000030333 (0.000024834)
Iteration: [ 46001/ 50000] Loss: 0.000232240 (0.000204458) Physics Loss: 0.000054119 (0.000046867) Data Loss: 0.000159402 (0.000139673) BC Loss: 0.000018719 (0.000017919)
Iteration: [ 47001/ 50000] Loss: 0.000194370 (0.000194374) Physics Loss: 0.000034218 (0.000036009) Data Loss: 0.000142445 (0.000140484) BC Loss: 0.000017708 (0.000017882)
Iteration: [ 48001/ 50000] Loss: 0.000194524 (0.000188704) Physics Loss: 0.000034514 (0.000031860) Data Loss: 0.000140071 (0.000138887) BC Loss: 0.000019939 (0.000017957)
Iteration: [ 49001/ 50000] Loss: 0.000170939 (0.000190115) Physics Loss: 0.000027584 (0.000029413) Data Loss: 0.000119772 (0.000142421) BC Loss: 0.000023583 (0.000018281)Visualizing the Results
ts, xs, ys = 0.0f0:0.05f0:2.0f0, 0.0f0:0.02f0:2.0f0, 0.0f0:0.02f0:2.0f0
grid = stack([[elem...] for elem in vec(collect(Iterators.product(xs, ys, ts)))])
u_real = reshape(analytical_solution(grid), length(xs), length(ys), length(ts))
grid_normalized = (grid .- minimum(grid)) ./ (maximum(grid) .- minimum(grid))
u_pred = reshape(trained_model(grid_normalized), length(xs), length(ys), length(ts))
u_pred = u_pred .* (max_pde_val - min_pde_val) .+ min_pde_val
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="x", ylabel="y")
errs = [abs.(u_pred[:, :, i] .- u_real[:, :, i]) for i in 1:length(ts)]
Colorbar(fig[1, 2]; limits=extrema(stack(errs)))
CairoMakie.record(fig, "pinn_nested_ad.gif", 1:length(ts); framerate=10) do i
ax.title = "Abs. Predictor Error | Time: $(ts[i])"
err = errs[i]
contour!(ax, xs, ys, err; levels=10, linewidth=2)
heatmap!(ax, xs, ys, err)
return fig
end
fig
endAppendix
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
endJulia Version 1.11.7
Commit f2b3dbda30a (2025-09-08 12:10 UTC)
Build Info:
Official https://julialang.org/ release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 48 × AMD EPYC 7402 24-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-16.0.6 (ORCJIT, znver2)
Threads: 48 default, 0 interactive, 24 GC (on 2 virtual cores)
Environment:
JULIA_CPU_THREADS = 2
JULIA_DEPOT_PATH = /root/.cache/julia-buildkite-plugin/depots/01872db4-8c79-43af-ab7d-12abac4f24f6
LD_LIBRARY_PATH = /usr/local/nvidia/lib:/usr/local/nvidia/lib64
JULIA_PKG_SERVER =
JULIA_NUM_THREADS = 48
JULIA_CUDA_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0
JULIA_DEBUG = LiterateThis page was generated using Literate.jl.