Training a PINN on 2D PDE
In this tutorial we will go over using a PINN to solve 2D PDEs. We will be using the system from NeuralPDE Tutorials. However, we will be using our custom loss function and use nested AD capabilities of Lux.jl.
This is a demonstration of Lux.jl. For serious use cases of PINNs, please refer to the package: NeuralPDE.jl.
Package Imports
using Lux,
Optimisers,
Random,
Printf,
Statistics,
MLUtils,
OnlineStats,
CairoMakie,
Reactant,
Enzyme
const xdev = reactant_device(; force=true)
const cdev = cpu_device()
Problem Definition
Since Lux supports efficient nested AD upto 2nd order, we will rewrite the problem with first order derivatives, so that we can compute the gradients of the loss using 2nd order AD.
Define the Neural Networks
All the networks take 3 input variables and output a scalar value. Here, we will define a wrapper over the 3 networks, so that we can train them using Training.TrainState
.
struct PINN{M} <: AbstractLuxWrapperLayer{:model}
model::M
end
function PINN(; hidden_dims::Int=32)
return PINN(
Chain(
Dense(3 => hidden_dims, tanh),
Dense(hidden_dims => hidden_dims, tanh),
Dense(hidden_dims => hidden_dims, tanh),
Dense(hidden_dims => 1),
),
)
end
Define the Loss Functions
We will define a custom loss function to compute the loss using 2nd order AD. For that, first we'll need to define the derivatives of our model:
function ∂u_∂t(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][3, :]
end
function ∂u_∂x(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][1, :]
end
function ∂u_∂y(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][2, :]
end
function ∂²u_∂x²(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ ∂u_∂x, Enzyme.Const(model), xyt)[2][1, :]
end
function ∂²u_∂y²(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ ∂u_∂y, Enzyme.Const(model), xyt)[2][2, :]
end
We will use the following loss function
function physics_informed_loss_function(model::StatefulLuxLayer, xyt::AbstractArray)
return mean(abs2, ∂u_∂t(model, xyt) .- ∂²u_∂x²(model, xyt) .- ∂²u_∂y²(model, xyt))
end
Additionally, we need to compute the loss with respect to the boundary conditions.
function mse_loss_function(
model::StatefulLuxLayer, target::AbstractArray, xyt::AbstractArray
)
return MSELoss()(model(xyt), target)
end
function loss_function(model, ps, st, (xyt, target_data, xyt_bc, target_bc))
smodel = StatefulLuxLayer(model, ps, st)
physics_loss = physics_informed_loss_function(smodel, xyt)
data_loss = mse_loss_function(smodel, target_data, xyt)
bc_loss = mse_loss_function(smodel, target_bc, xyt_bc)
loss = physics_loss + data_loss + bc_loss
return loss, smodel.st, (; physics_loss, data_loss, bc_loss)
end
Generate the Data
We will generate some random data to train the model on. We will take data on a square spatial and temporal domain
analytical_solution(x, y, t) = @. exp(x + y) * cos(x + y + 4t)
analytical_solution(xyt) = analytical_solution(xyt[1, :], xyt[2, :], xyt[3, :])
begin
grid_len = 16
grid = range(0.0f0, 2.0f0; length=grid_len)
xyt = stack([[elem...] for elem in vec(collect(Iterators.product(grid, grid, grid)))])
target_data = reshape(analytical_solution(xyt), 1, :)
bc_len = 512
x = collect(range(0.0f0, 2.0f0; length=bc_len))
y = collect(range(0.0f0, 2.0f0; length=bc_len))
t = collect(range(0.0f0, 2.0f0; length=bc_len))
xyt_bc = hcat(
stack((x, y, zeros(Float32, bc_len)); dims=1),
stack((zeros(Float32, bc_len), y, t); dims=1),
stack((ones(Float32, bc_len) .* 2, y, t); dims=1),
stack((x, zeros(Float32, bc_len), t); dims=1),
stack((x, ones(Float32, bc_len) .* 2, t); dims=1),
)
target_bc = reshape(analytical_solution(xyt_bc), 1, :)
min_target_bc, max_target_bc = extrema(target_bc)
min_data, max_data = extrema(target_data)
min_pde_val, max_pde_val = min(min_data, min_target_bc), max(max_data, max_target_bc)
xyt = (xyt .- minimum(xyt)) ./ (maximum(xyt) .- minimum(xyt))
xyt_bc = (xyt_bc .- minimum(xyt_bc)) ./ (maximum(xyt_bc) .- minimum(xyt_bc))
target_bc = (target_bc .- min_pde_val) ./ (max_pde_val - min_pde_val)
target_data = (target_data .- min_pde_val) ./ (max_pde_val - min_pde_val)
end
Training
function train_model(
xyt,
target_data,
xyt_bc,
target_bc;
seed::Int=0,
maxiters::Int=50000,
hidden_dims::Int=128,
)
rng = Random.default_rng()
Random.seed!(rng, seed)
pinn = PINN(; hidden_dims)
ps, st = Lux.setup(rng, pinn) |> xdev
bc_dataloader =
DataLoader((xyt_bc, target_bc); batchsize=128, shuffle=true, partial=false) |> xdev
pde_dataloader =
DataLoader((xyt, target_data); batchsize=128, shuffle=true, partial=false) |> xdev
train_state = Training.TrainState(pinn, ps, st, Adam(0.005f0))
lr = i -> i < 5000 ? 0.005f0 : (i < 10000 ? 0.0005f0 : 0.00005f0)
total_loss_tracker, physics_loss_tracker, data_loss_tracker, bc_loss_tracker = ntuple(
_ -> OnlineStats.CircBuff(Float32, 32; rev=true), 4
)
iter = 1
for ((xyt_batch, target_data_batch), (xyt_bc_batch, target_bc_batch)) in
zip(Iterators.cycle(pde_dataloader), Iterators.cycle(bc_dataloader))
Optimisers.adjust!(train_state, lr(iter))
_, loss, stats, train_state = Training.single_train_step!(
AutoEnzyme(),
loss_function,
(xyt_batch, target_data_batch, xyt_bc_batch, target_bc_batch),
train_state;
return_gradients=Val(false),
)
fit!(total_loss_tracker, Float32(loss))
fit!(physics_loss_tracker, Float32(stats.physics_loss))
fit!(data_loss_tracker, Float32(stats.data_loss))
fit!(bc_loss_tracker, Float32(stats.bc_loss))
mean_loss = mean(OnlineStats.value(total_loss_tracker))
mean_physics_loss = mean(OnlineStats.value(physics_loss_tracker))
mean_data_loss = mean(OnlineStats.value(data_loss_tracker))
mean_bc_loss = mean(OnlineStats.value(bc_loss_tracker))
isnan(loss) && throw(ArgumentError("NaN Loss Detected"))
if iter % 1000 == 1 || iter == maxiters
@printf(
"Iteration: [%6d/%6d] \t Loss: %.9f (%.9f) \t Physics Loss: %.9f \
(%.9f) \t Data Loss: %.9f (%.9f) \t BC \
Loss: %.9f (%.9f)\n",
iter,
maxiters,
loss,
mean_loss,
stats.physics_loss,
mean_physics_loss,
stats.data_loss,
mean_data_loss,
stats.bc_loss,
mean_bc_loss
)
end
iter += 1
iter ≥ maxiters && break
end
return StatefulLuxLayer(pinn, cdev(train_state.parameters), cdev(train_state.states))
end
trained_model = train_model(xyt, target_data, xyt_bc, target_bc)
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1757110970.975315 2397803 service.cc:163] XLA service 0x481ee490 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
I0000 00:00:1757110970.975393 2397803 service.cc:171] StreamExecutor device (0): Quadro RTX 5000, Compute Capability 7.5
I0000 00:00:1757110970.975399 2397803 service.cc:171] StreamExecutor device (1): Quadro RTX 5000, Compute Capability 7.5
I0000 00:00:1757110970.980372 2397803 se_gpu_pjrt_client.cc:1338] Using BFC allocator.
I0000 00:00:1757110970.980477 2397803 gpu_helpers.cc:136] XLA backend allocating 12526534656 bytes on device 0 for BFCAllocator.
I0000 00:00:1757110970.980564 2397803 gpu_helpers.cc:136] XLA backend allocating 12526534656 bytes on device 1 for BFCAllocator.
I0000 00:00:1757110970.980584 2397803 gpu_helpers.cc:177] XLA backend will use up to 4175511552 bytes on device 0 for CollectiveBFCAllocator.
I0000 00:00:1757110970.980603 2397803 gpu_helpers.cc:177] XLA backend will use up to 4175511552 bytes on device 1 for CollectiveBFCAllocator.
I0000 00:00:1757110970.992549 2397803 cuda_dnn.cc:463] Loaded cuDNN version 91200
Iteration: [ 1/ 50000] Loss: 20.523933411 (20.523933411) Physics Loss: 16.931318283 (16.931318283) Data Loss: 2.007483006 (2.007483006) BC Loss: 1.585133314 (1.585133314)
Iteration: [ 1001/ 50000] Loss: 0.017368661 (0.019241149) Physics Loss: 0.000384373 (0.000523629) Data Loss: 0.005318505 (0.007538578) BC Loss: 0.011665784 (0.011178941)
Iteration: [ 2001/ 50000] Loss: 0.015431685 (0.018665671) Physics Loss: 0.001248567 (0.001662043) Data Loss: 0.004322537 (0.006408241) BC Loss: 0.009860582 (0.010595389)
Iteration: [ 3001/ 50000] Loss: 0.015749779 (0.015216095) Physics Loss: 0.000569924 (0.001279109) Data Loss: 0.004014906 (0.004232453) BC Loss: 0.011164947 (0.009704536)
Iteration: [ 4001/ 50000] Loss: 0.009719368 (0.008715330) Physics Loss: 0.002388043 (0.003382138) Data Loss: 0.003175628 (0.002104700) BC Loss: 0.004155698 (0.003228491)
Iteration: [ 5001/ 50000] Loss: 0.006572966 (0.006051276) Physics Loss: 0.001518856 (0.002297103) Data Loss: 0.001734660 (0.001557001) BC Loss: 0.003319450 (0.002197171)
Iteration: [ 6001/ 50000] Loss: 0.001203572 (0.001477470) Physics Loss: 0.000318373 (0.000374529) Data Loss: 0.000597942 (0.000762007) BC Loss: 0.000287257 (0.000340935)
Iteration: [ 7001/ 50000] Loss: 0.001361690 (0.000975937) Physics Loss: 0.000289049 (0.000295807) Data Loss: 0.000964172 (0.000537330) BC Loss: 0.000108469 (0.000142800)
Iteration: [ 8001/ 50000] Loss: 0.001223069 (0.000799660) Physics Loss: 0.000818447 (0.000328481) Data Loss: 0.000329079 (0.000391040) BC Loss: 0.000075543 (0.000080139)
Iteration: [ 9001/ 50000] Loss: 0.002254438 (0.002575407) Physics Loss: 0.001366787 (0.001639548) Data Loss: 0.000716488 (0.000556985) BC Loss: 0.000171162 (0.000378874)
Iteration: [ 10001/ 50000] Loss: 0.000587171 (0.000591648) Physics Loss: 0.000196131 (0.000229374) Data Loss: 0.000321169 (0.000307114) BC Loss: 0.000069871 (0.000055161)
Iteration: [ 11001/ 50000] Loss: 0.000367381 (0.000379070) Physics Loss: 0.000135983 (0.000066894) Data Loss: 0.000184252 (0.000274867) BC Loss: 0.000047147 (0.000037309)
Iteration: [ 12001/ 50000] Loss: 0.000264205 (0.000360175) Physics Loss: 0.000049809 (0.000070430) Data Loss: 0.000170174 (0.000251477) BC Loss: 0.000044222 (0.000038268)
Iteration: [ 13001/ 50000] Loss: 0.000310069 (0.000331328) Physics Loss: 0.000060866 (0.000068280) Data Loss: 0.000209690 (0.000228723) BC Loss: 0.000039512 (0.000034325)
Iteration: [ 14001/ 50000] Loss: 0.000403482 (0.000338823) Physics Loss: 0.000074766 (0.000071470) Data Loss: 0.000294049 (0.000236756) BC Loss: 0.000034667 (0.000030597)
Iteration: [ 15001/ 50000] Loss: 0.000244624 (0.000284788) Physics Loss: 0.000043479 (0.000050475) Data Loss: 0.000166692 (0.000202296) BC Loss: 0.000034454 (0.000032017)
Iteration: [ 16001/ 50000] Loss: 0.000224490 (0.000290654) Physics Loss: 0.000043120 (0.000058712) Data Loss: 0.000147436 (0.000202430) BC Loss: 0.000033933 (0.000029512)
Iteration: [ 17001/ 50000] Loss: 0.000415414 (0.000300979) Physics Loss: 0.000091245 (0.000068379) Data Loss: 0.000299402 (0.000203778) BC Loss: 0.000024767 (0.000028823)
Iteration: [ 18001/ 50000] Loss: 0.000218020 (0.000289040) Physics Loss: 0.000041157 (0.000063033) Data Loss: 0.000142338 (0.000197336) BC Loss: 0.000034526 (0.000028671)
Iteration: [ 19001/ 50000] Loss: 0.000237335 (0.000282468) Physics Loss: 0.000079303 (0.000059545) Data Loss: 0.000138319 (0.000197726) BC Loss: 0.000019714 (0.000025196)
Iteration: [ 20001/ 50000] Loss: 0.000301725 (0.000253947) Physics Loss: 0.000050562 (0.000049033) Data Loss: 0.000230317 (0.000181449) BC Loss: 0.000020845 (0.000023465)
Iteration: [ 21001/ 50000] Loss: 0.000293685 (0.000237799) Physics Loss: 0.000040557 (0.000042509) Data Loss: 0.000226891 (0.000171598) BC Loss: 0.000026237 (0.000023691)
Iteration: [ 22001/ 50000] Loss: 0.000160168 (0.000236668) Physics Loss: 0.000026102 (0.000045009) Data Loss: 0.000107784 (0.000168821) BC Loss: 0.000026282 (0.000022838)
Iteration: [ 23001/ 50000] Loss: 0.000240116 (0.000254437) Physics Loss: 0.000045024 (0.000053803) Data Loss: 0.000171133 (0.000178459) BC Loss: 0.000023960 (0.000022175)
Iteration: [ 24001/ 50000] Loss: 0.000288947 (0.000244835) Physics Loss: 0.000050358 (0.000048882) Data Loss: 0.000220293 (0.000171023) BC Loss: 0.000018295 (0.000024931)
Iteration: [ 25001/ 50000] Loss: 0.000196178 (0.000222719) Physics Loss: 0.000034494 (0.000032817) Data Loss: 0.000142552 (0.000168641) BC Loss: 0.000019132 (0.000021261)
Iteration: [ 26001/ 50000] Loss: 0.000235113 (0.000240319) Physics Loss: 0.000057308 (0.000053108) Data Loss: 0.000156987 (0.000164729) BC Loss: 0.000020818 (0.000022482)
Iteration: [ 27001/ 50000] Loss: 0.000248998 (0.000231835) Physics Loss: 0.000066480 (0.000043987) Data Loss: 0.000157504 (0.000165717) BC Loss: 0.000025014 (0.000022132)
Iteration: [ 28001/ 50000] Loss: 0.000207978 (0.000212856) Physics Loss: 0.000036753 (0.000032777) Data Loss: 0.000153864 (0.000158607) BC Loss: 0.000017362 (0.000021472)
Iteration: [ 29001/ 50000] Loss: 0.000195244 (0.000227428) Physics Loss: 0.000022110 (0.000046830) Data Loss: 0.000143451 (0.000159385) BC Loss: 0.000029684 (0.000021213)
Iteration: [ 30001/ 50000] Loss: 0.000202901 (0.000223159) Physics Loss: 0.000021556 (0.000039438) Data Loss: 0.000155354 (0.000162553) BC Loss: 0.000025992 (0.000021168)
Iteration: [ 31001/ 50000] Loss: 0.000259623 (0.000212104) Physics Loss: 0.000028857 (0.000034059) Data Loss: 0.000211938 (0.000158377) BC Loss: 0.000018828 (0.000019668)
Iteration: [ 32001/ 50000] Loss: 0.000209357 (0.000205690) Physics Loss: 0.000042520 (0.000032734) Data Loss: 0.000146363 (0.000153002) BC Loss: 0.000020474 (0.000019954)
Iteration: [ 33001/ 50000] Loss: 0.000193965 (0.000205770) Physics Loss: 0.000033881 (0.000035243) Data Loss: 0.000137588 (0.000149592) BC Loss: 0.000022497 (0.000020935)
Iteration: [ 34001/ 50000] Loss: 0.000192018 (0.000194181) Physics Loss: 0.000028130 (0.000029067) Data Loss: 0.000142240 (0.000145605) BC Loss: 0.000021647 (0.000019510)
Iteration: [ 35001/ 50000] Loss: 0.000133685 (0.000204517) Physics Loss: 0.000014262 (0.000037587) Data Loss: 0.000097358 (0.000148151) BC Loss: 0.000022065 (0.000018779)
Iteration: [ 36001/ 50000] Loss: 0.000166764 (0.000205503) Physics Loss: 0.000031166 (0.000038654) Data Loss: 0.000117131 (0.000147762) BC Loss: 0.000018468 (0.000019087)
Iteration: [ 37001/ 50000] Loss: 0.000308157 (0.000195143) Physics Loss: 0.000068980 (0.000029073) Data Loss: 0.000220309 (0.000146627) BC Loss: 0.000018868 (0.000019443)
Iteration: [ 38001/ 50000] Loss: 0.000239020 (0.000196551) Physics Loss: 0.000023837 (0.000027279) Data Loss: 0.000191654 (0.000151912) BC Loss: 0.000023530 (0.000017360)
Iteration: [ 39001/ 50000] Loss: 0.000173764 (0.000199541) Physics Loss: 0.000035406 (0.000033822) Data Loss: 0.000122925 (0.000147050) BC Loss: 0.000015433 (0.000018669)
Iteration: [ 40001/ 50000] Loss: 0.000171057 (0.000192756) Physics Loss: 0.000020721 (0.000028046) Data Loss: 0.000129040 (0.000145425) BC Loss: 0.000021296 (0.000019284)
Iteration: [ 41001/ 50000] Loss: 0.000159438 (0.000191454) Physics Loss: 0.000019348 (0.000025182) Data Loss: 0.000122243 (0.000148193) BC Loss: 0.000017846 (0.000018078)
Iteration: [ 42001/ 50000] Loss: 0.000185340 (0.000193629) Physics Loss: 0.000032512 (0.000027948) Data Loss: 0.000137283 (0.000147058) BC Loss: 0.000015545 (0.000018623)
Iteration: [ 43001/ 50000] Loss: 0.000187346 (0.000193299) Physics Loss: 0.000019203 (0.000027983) Data Loss: 0.000153204 (0.000145236) BC Loss: 0.000014939 (0.000020080)
Iteration: [ 44001/ 50000] Loss: 0.000168027 (0.000188327) Physics Loss: 0.000009133 (0.000022478) Data Loss: 0.000143137 (0.000144525) BC Loss: 0.000015757 (0.000021324)
Iteration: [ 45001/ 50000] Loss: 0.000276085 (0.000208336) Physics Loss: 0.000047352 (0.000032902) Data Loss: 0.000201373 (0.000152718) BC Loss: 0.000027360 (0.000022716)
Iteration: [ 46001/ 50000] Loss: 0.000235316 (0.000201868) Physics Loss: 0.000055009 (0.000043864) Data Loss: 0.000160512 (0.000139937) BC Loss: 0.000019796 (0.000018067)
Iteration: [ 47001/ 50000] Loss: 0.000178999 (0.000188161) Physics Loss: 0.000023969 (0.000029823) Data Loss: 0.000137819 (0.000140310) BC Loss: 0.000017211 (0.000018028)
Iteration: [ 48001/ 50000] Loss: 0.000185548 (0.000178888) Physics Loss: 0.000024271 (0.000021332) Data Loss: 0.000142044 (0.000139339) BC Loss: 0.000019233 (0.000018217)
Iteration: [ 49001/ 50000] Loss: 0.000181234 (0.000209413) Physics Loss: 0.000035708 (0.000048259) Data Loss: 0.000123530 (0.000142605) BC Loss: 0.000021996 (0.000018549)
Visualizing the Results
ts, xs, ys = 0.0f0:0.05f0:2.0f0, 0.0f0:0.02f0:2.0f0, 0.0f0:0.02f0:2.0f0
grid = stack([[elem...] for elem in vec(collect(Iterators.product(xs, ys, ts)))])
u_real = reshape(analytical_solution(grid), length(xs), length(ys), length(ts))
grid_normalized = (grid .- minimum(grid)) ./ (maximum(grid) .- minimum(grid))
u_pred = reshape(trained_model(grid_normalized), length(xs), length(ys), length(ts))
u_pred = u_pred .* (max_pde_val - min_pde_val) .+ min_pde_val
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="x", ylabel="y")
errs = [abs.(u_pred[:, :, i] .- u_real[:, :, i]) for i in 1:length(ts)]
Colorbar(fig[1, 2]; limits=extrema(stack(errs)))
CairoMakie.record(fig, "pinn_nested_ad.gif", 1:length(ts); framerate=10) do i
ax.title = "Abs. Predictor Error | Time: $(ts[i])"
err = errs[i]
contour!(ax, xs, ys, err; levels=10, linewidth=2)
heatmap!(ax, xs, ys, err)
return fig
end
fig
end
Appendix
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
end
Julia Version 1.11.6
Commit 9615af0f269 (2025-07-09 12:58 UTC)
Build Info:
Official https://julialang.org/ release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 48 × AMD EPYC 7402 24-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-16.0.6 (ORCJIT, znver2)
Threads: 48 default, 0 interactive, 24 GC (on 2 virtual cores)
Environment:
JULIA_CPU_THREADS = 2
JULIA_DEPOT_PATH = /root/.cache/julia-buildkite-plugin/depots/01872db4-8c79-43af-ab7d-12abac4f24f6
LD_LIBRARY_PATH = /usr/local/nvidia/lib:/usr/local/nvidia/lib64
JULIA_PKG_SERVER =
JULIA_NUM_THREADS = 48
JULIA_CUDA_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0
JULIA_DEBUG = Literate
This page was generated using Literate.jl.