Training a PINN on 2D PDE
In this tutorial we will go over using a PINN to solve 2D PDEs. We will be using the system from NeuralPDE Tutorials. However, we will be using our custom loss function and use nested AD capabilities of Lux.jl.
This is a demonstration of Lux.jl. For serious usecases of PINNs, please refer to the package: NeuralPDE.jl.
Package Imports
julia
using Lux,
Optimisers,
Random,
Printf,
Statistics,
MLUtils,
OnlineStats,
CairoMakie,
Reactant,
Enzyme
const xdev = reactant_device(; force=true)
const cdev = cpu_device()(::MLDataDevices.CPUDevice) (generic function with 1 method)Problem Definition
Since Lux supports efficient nested AD upto 2nd order, we will rewrite the problem with first order derivatives, so that we can compute the gradients of the loss using 2nd order AD.
Define the Neural Networks
All the networks take 3 input variables and output a scalar value. Here, we will define a a wrapper over the 3 networks, so that we can train them using Training.TrainState.
julia
struct PINN{M} <: AbstractLuxWrapperLayer{:model}
model::M
end
function PINN(; hidden_dims::Int=32)
return PINN(
Chain(
Dense(3 => hidden_dims, tanh),
Dense(hidden_dims => hidden_dims, tanh),
Dense(hidden_dims => hidden_dims, tanh),
Dense(hidden_dims => 1),
),
)
endMain.var"##230".PINNDefine the Loss Functions
We will define a custom loss function to compute the loss using 2nd order AD. We will use the following loss function
julia
function ∂u_∂t(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][3, :]
end
function ∂u_∂x(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][1, :]
end
function ∂u_∂y(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ model, xyt)[1][2, :]
end
function ∂²u_∂x²(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ ∂u_∂x, Enzyme.Const(model), xyt)[2][1, :]
end
function ∂²u_∂y²(model::StatefulLuxLayer, xyt::AbstractArray)
return Enzyme.gradient(Enzyme.Reverse, sum ∘ ∂u_∂y, Enzyme.Const(model), xyt)[2][2, :]
end
function physics_informed_loss_function(model::StatefulLuxLayer, xyt::AbstractArray)
return mean(abs2, ∂u_∂t(model, xyt) .- ∂²u_∂x²(model, xyt) .- ∂²u_∂y²(model, xyt))
endphysics_informed_loss_function (generic function with 1 method)Additionally, we need to compute the loss wrt the boundary conditions.
julia
function mse_loss_function(
model::StatefulLuxLayer, target::AbstractArray, xyt::AbstractArray
)
return MSELoss()(model(xyt), target)
end
function loss_function(model, ps, st, (xyt, target_data, xyt_bc, target_bc))
smodel = StatefulLuxLayer{true}(model, ps, st)
physics_loss = physics_informed_loss_function(smodel, xyt)
data_loss = mse_loss_function(smodel, target_data, xyt)
bc_loss = mse_loss_function(smodel, target_bc, xyt_bc)
loss = physics_loss + data_loss + bc_loss
return loss, smodel.st, (; physics_loss, data_loss, bc_loss)
endloss_function (generic function with 1 method)Generate the Data
We will generate some random data to train the model on. We will take data on a square spatial and temporal domain
julia
analytical_solution(x, y, t) = @. exp(x + y) * cos(x + y + 4t)
analytical_solution(xyt) = analytical_solution(xyt[1, :], xyt[2, :], xyt[3, :])
begin
grid_len = 16
grid = range(0.0f0, 2.0f0; length=grid_len)
xyt = stack([[elem...] for elem in vec(collect(Iterators.product(grid, grid, grid)))])
target_data = reshape(analytical_solution(xyt), 1, :)
bc_len = 512
x = collect(range(0.0f0, 2.0f0; length=bc_len))
y = collect(range(0.0f0, 2.0f0; length=bc_len))
t = collect(range(0.0f0, 2.0f0; length=bc_len))
xyt_bc = hcat(
stack((x, y, zeros(Float32, bc_len)); dims=1),
stack((zeros(Float32, bc_len), y, t); dims=1),
stack((ones(Float32, bc_len) .* 2, y, t); dims=1),
stack((x, zeros(Float32, bc_len), t); dims=1),
stack((x, ones(Float32, bc_len) .* 2, t); dims=1),
)
target_bc = reshape(analytical_solution(xyt_bc), 1, :)
min_target_bc, max_target_bc = extrema(target_bc)
min_data, max_data = extrema(target_data)
min_pde_val, max_pde_val = min(min_data, min_target_bc), max(max_data, max_target_bc)
xyt = (xyt .- minimum(xyt)) ./ (maximum(xyt) .- minimum(xyt))
xyt_bc = (xyt_bc .- minimum(xyt_bc)) ./ (maximum(xyt_bc) .- minimum(xyt_bc))
target_bc = (target_bc .- min_pde_val) ./ (max_pde_val - min_pde_val)
target_data = (target_data .- min_pde_val) ./ (max_pde_val - min_pde_val)
endTraining
julia
function train_model(
xyt,
target_data,
xyt_bc,
target_bc;
seed::Int=0,
maxiters::Int=50000,
hidden_dims::Int=128,
)
rng = Random.default_rng()
Random.seed!(rng, seed)
pinn = PINN(; hidden_dims)
ps, st = Lux.setup(rng, pinn) |> xdev
bc_dataloader =
DataLoader((xyt_bc, target_bc); batchsize=128, shuffle=true, partial=false) |> xdev
pde_dataloader =
DataLoader((xyt, target_data); batchsize=128, shuffle=true, partial=false) |> xdev
train_state = Training.TrainState(pinn, ps, st, Adam(0.005f0))
lr = i -> i < 5000 ? 0.005f0 : (i < 10000 ? 0.0005f0 : 0.00005f0)
total_loss_tracker, physics_loss_tracker, data_loss_tracker, bc_loss_tracker = ntuple(
_ -> OnlineStats.CircBuff(Float32, 32; rev=true), 4
)
iter = 1
for ((xyt_batch, target_data_batch), (xyt_bc_batch, target_bc_batch)) in
zip(Iterators.cycle(pde_dataloader), Iterators.cycle(bc_dataloader))
Optimisers.adjust!(train_state, lr(iter))
_, loss, stats, train_state = Training.single_train_step!(
AutoEnzyme(),
loss_function,
(xyt_batch, target_data_batch, xyt_bc_batch, target_bc_batch),
train_state;
return_gradients=Val(false),
)
fit!(total_loss_tracker, Float32(loss))
fit!(physics_loss_tracker, Float32(stats.physics_loss))
fit!(data_loss_tracker, Float32(stats.data_loss))
fit!(bc_loss_tracker, Float32(stats.bc_loss))
mean_loss = mean(OnlineStats.value(total_loss_tracker))
mean_physics_loss = mean(OnlineStats.value(physics_loss_tracker))
mean_data_loss = mean(OnlineStats.value(data_loss_tracker))
mean_bc_loss = mean(OnlineStats.value(bc_loss_tracker))
isnan(loss) && throw(ArgumentError("NaN Loss Detected"))
if iter % 1000 == 1 || iter == maxiters
@printf "Iteration: [%6d/%6d] \t Loss: %.9f (%.9f) \t Physics Loss: %.9f \
(%.9f) \t Data Loss: %.9f (%.9f) \t BC \
Loss: %.9f (%.9f)\n" iter maxiters loss mean_loss stats.physics_loss mean_physics_loss stats.data_loss mean_data_loss stats.bc_loss mean_bc_loss
end
iter += 1
iter ≥ maxiters && break
end
return StatefulLuxLayer{true}(
pinn, cdev(train_state.parameters), cdev(train_state.states)
)
end
trained_model = train_model(xyt, target_data, xyt_bc, target_bc)2025-05-23 22:26:32.139622: I external/xla/xla/service/service.cc:152] XLA service 0x3c0a3d70 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2025-05-23 22:26:32.139673: I external/xla/xla/service/service.cc:160] StreamExecutor device (0): NVIDIA A100-PCIE-40GB MIG 1g.5gb, Compute Capability 8.0
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1748039192.140470 707462 se_gpu_pjrt_client.cc:1026] Using BFC allocator.
I0000 00:00:1748039192.140571 707462 gpu_helpers.cc:136] XLA backend allocating 3825205248 bytes on device 0 for BFCAllocator.
I0000 00:00:1748039192.140605 707462 gpu_helpers.cc:177] XLA backend will use up to 1275068416 bytes on device 0 for CollectiveBFCAllocator.
I0000 00:00:1748039192.155615 707462 cuda_dnn.cc:529] Loaded cuDNN version 90400
2025-05-23 22:32:55.110871: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_34', 72 bytes spill stores, 72 bytes spill loads
2025-05-23 22:32:55.577767: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_45', 32 bytes spill stores, 32 bytes spill loads
2025-05-23 22:32:56.363304: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_26', 76 bytes spill stores, 76 bytes spill loads
2025-05-23 22:32:56.518876: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_34', 268 bytes spill stores, 268 bytes spill loads
2025-05-23 22:32:56.692460: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_34', 68 bytes spill stores, 68 bytes spill loads
2025-05-23 22:32:57.942451: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_26', 4 bytes spill stores, 4 bytes spill loads
2025-05-23 22:33:04.300159: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_222', 164 bytes spill stores, 164 bytes spill loads
2025-05-23 22:33:04.362335: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_222', 600 bytes spill stores, 420 bytes spill loads
2025-05-23 22:33:04.727371: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_222', 4 bytes spill stores, 4 bytes spill loads
2025-05-23 22:33:05.072542: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_34', 200 bytes spill stores, 200 bytes spill loads
2025-05-23 22:33:05.833589: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 4 bytes spill stores, 4 bytes spill loads
2025-05-23 22:33:06.735711: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 164 bytes spill stores, 164 bytes spill loads
2025-05-23 22:33:07.053838: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_34', 340 bytes spill stores, 340 bytes spill loads
2025-05-23 22:33:07.658644: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 4 bytes spill stores, 4 bytes spill loads
2025-05-23 22:33:07.804163: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 600 bytes spill stores, 420 bytes spill loads
2025-05-23 22:33:07.884054: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_222', 20 bytes spill stores, 20 bytes spill loads
E0000 00:00:1748039587.995736 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039587.995785 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
E0000 00:00:1748039587.995789 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 33.0107
E0000 00:00:1748039587.995793 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 33.1252
E0000 00:00:1748039587.995796 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 32.0789
E0000 00:00:1748039587.995799 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 35.7839
E0000 00:00:1748039587.995803 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 33.4785
E0000 00:00:1748039587.995806 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 34.4481
E0000 00:00:1748039587.995810 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 35.7397
E0000 00:00:1748039587.995813 707462 buffer_comparator.cc:145] Difference at 25: 0, expected 35.0979
2025-05-23 22:33:07.995823: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039587.998298 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039587.998314 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
E0000 00:00:1748039587.998317 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 33.0107
E0000 00:00:1748039587.998320 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 33.1252
E0000 00:00:1748039587.998323 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 32.0789
E0000 00:00:1748039587.998326 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 35.7839
E0000 00:00:1748039587.998329 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 33.4785
E0000 00:00:1748039587.998331 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 34.4481
E0000 00:00:1748039587.998334 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 35.7397
E0000 00:00:1748039587.998337 707462 buffer_comparator.cc:145] Difference at 25: 0, expected 35.0979
2025-05-23 22:33:07.998342: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.000464 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039588.000476 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
E0000 00:00:1748039588.000479 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 33.0107
E0000 00:00:1748039588.000482 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 33.1252
E0000 00:00:1748039588.000485 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 32.0789
E0000 00:00:1748039588.000488 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 35.7839
E0000 00:00:1748039588.000490 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 33.4785
E0000 00:00:1748039588.000493 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 34.4481
E0000 00:00:1748039588.000496 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 35.7397
E0000 00:00:1748039588.000499 707462 buffer_comparator.cc:145] Difference at 25: 0, expected 35.0979
2025-05-23 22:33:08.000504: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.002676 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039588.002695 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
E0000 00:00:1748039588.002699 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 33.0107
E0000 00:00:1748039588.002702 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 33.1252
E0000 00:00:1748039588.002705 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 32.0789
E0000 00:00:1748039588.002707 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 35.7839
E0000 00:00:1748039588.002710 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 33.4785
E0000 00:00:1748039588.002713 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 34.4481
E0000 00:00:1748039588.002716 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 35.7397
E0000 00:00:1748039588.002719 707462 buffer_comparator.cc:145] Difference at 25: 0, expected 35.0979
2025-05-23 22:33:08.002723: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.004849 707462 buffer_comparator.cc:145] Difference at 2048: 0, expected 34.6692
E0000 00:00:1748039588.004863 707462 buffer_comparator.cc:145] Difference at 2049: 0, expected 34.434
E0000 00:00:1748039588.004866 707462 buffer_comparator.cc:145] Difference at 2050: 0, expected 33.9942
E0000 00:00:1748039588.004869 707462 buffer_comparator.cc:145] Difference at 2051: 0, expected 30.2626
E0000 00:00:1748039588.004872 707462 buffer_comparator.cc:145] Difference at 2052: 0, expected 31.5452
E0000 00:00:1748039588.004875 707462 buffer_comparator.cc:145] Difference at 2053: 0, expected 31.8868
E0000 00:00:1748039588.004878 707462 buffer_comparator.cc:145] Difference at 2054: 0, expected 33.2951
E0000 00:00:1748039588.004881 707462 buffer_comparator.cc:145] Difference at 2055: 0, expected 31.1197
E0000 00:00:1748039588.004884 707462 buffer_comparator.cc:145] Difference at 2056: 0, expected 34.0554
E0000 00:00:1748039588.004886 707462 buffer_comparator.cc:145] Difference at 2057: 0, expected 31.1344
2025-05-23 22:33:08.004891: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.007002 707462 buffer_comparator.cc:145] Difference at 4096: 0, expected 35.3358
E0000 00:00:1748039588.007017 707462 buffer_comparator.cc:145] Difference at 4097: 0, expected 36.142
E0000 00:00:1748039588.007021 707462 buffer_comparator.cc:145] Difference at 4098: 0, expected 38.8935
E0000 00:00:1748039588.007024 707462 buffer_comparator.cc:145] Difference at 4099: 0, expected 34.3662
E0000 00:00:1748039588.007026 707462 buffer_comparator.cc:145] Difference at 4100: 0, expected 33.2298
E0000 00:00:1748039588.007029 707462 buffer_comparator.cc:145] Difference at 4101: 0, expected 33.6911
E0000 00:00:1748039588.007032 707462 buffer_comparator.cc:145] Difference at 4102: 0, expected 36.6112
E0000 00:00:1748039588.007035 707462 buffer_comparator.cc:145] Difference at 4103: 0, expected 32.0137
E0000 00:00:1748039588.007038 707462 buffer_comparator.cc:145] Difference at 4104: 0, expected 36.6626
E0000 00:00:1748039588.007041 707462 buffer_comparator.cc:145] Difference at 4105: 0, expected 32.5371
2025-05-23 22:33:08.007045: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.009173 707462 buffer_comparator.cc:145] Difference at 4112: 0, expected 33.1318
E0000 00:00:1748039588.009187 707462 buffer_comparator.cc:145] Difference at 4113: 0, expected 35.2935
E0000 00:00:1748039588.009191 707462 buffer_comparator.cc:145] Difference at 4114: 0, expected 34.0675
E0000 00:00:1748039588.009194 707462 buffer_comparator.cc:145] Difference at 4115: 0, expected 33.6177
E0000 00:00:1748039588.009197 707462 buffer_comparator.cc:145] Difference at 4116: 0, expected 35.2139
E0000 00:00:1748039588.009200 707462 buffer_comparator.cc:145] Difference at 4117: 0, expected 35.2738
E0000 00:00:1748039588.009203 707462 buffer_comparator.cc:145] Difference at 4118: 0, expected 35.4778
E0000 00:00:1748039588.009205 707462 buffer_comparator.cc:145] Difference at 4119: 0, expected 36.1507
E0000 00:00:1748039588.009208 707462 buffer_comparator.cc:145] Difference at 4120: 0, expected 35.4548
E0000 00:00:1748039588.009211 707462 buffer_comparator.cc:145] Difference at 4121: 0, expected 34.2895
2025-05-23 22:33:08.009216: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.011370 707462 buffer_comparator.cc:145] Difference at 0: 40.49, expected 35.1785
E0000 00:00:1748039588.011386 707462 buffer_comparator.cc:145] Difference at 1: 43.6533, expected 37.7175
E0000 00:00:1748039588.011390 707462 buffer_comparator.cc:145] Difference at 3: 38.0606, expected 33.665
E0000 00:00:1748039588.011393 707462 buffer_comparator.cc:145] Difference at 6: 43.1473, expected 36.7561
E0000 00:00:1748039588.011396 707462 buffer_comparator.cc:145] Difference at 9: 39.753, expected 30.2847
E0000 00:00:1748039588.011399 707462 buffer_comparator.cc:145] Difference at 11: 40.933, expected 36.0152
E0000 00:00:1748039588.011402 707462 buffer_comparator.cc:145] Difference at 12: 42.6871, expected 34.5496
E0000 00:00:1748039588.011405 707462 buffer_comparator.cc:145] Difference at 14: 41.3934, expected 35.5656
E0000 00:00:1748039588.011408 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039588.011411 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
2025-05-23 22:33:08.011416: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.013524 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039588.013538 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
E0000 00:00:1748039588.013542 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 33.0107
E0000 00:00:1748039588.013545 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 33.1252
E0000 00:00:1748039588.013548 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 32.0789
E0000 00:00:1748039588.013550 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 35.7839
E0000 00:00:1748039588.013553 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 33.4785
E0000 00:00:1748039588.013556 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 34.4481
E0000 00:00:1748039588.013559 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 35.7397
E0000 00:00:1748039588.013562 707462 buffer_comparator.cc:145] Difference at 25: 0, expected 35.0979
2025-05-23 22:33:08.013567: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.015694 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.0052
E0000 00:00:1748039588.015708 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 36.0408
E0000 00:00:1748039588.015712 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 33.0107
E0000 00:00:1748039588.015715 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 33.1252
E0000 00:00:1748039588.015718 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 32.0789
E0000 00:00:1748039588.015720 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 35.7839
E0000 00:00:1748039588.015724 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 33.4785
E0000 00:00:1748039588.015727 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 34.4481
E0000 00:00:1748039588.015730 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 35.7397
E0000 00:00:1748039588.015733 707462 buffer_comparator.cc:145] Difference at 25: 0, expected 35.0979
2025-05-23 22:33:08.015737: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.017845 707462 buffer_comparator.cc:145] Difference at 32: 0, expected 32.3706
E0000 00:00:1748039588.017859 707462 buffer_comparator.cc:145] Difference at 33: 0, expected 32.7046
E0000 00:00:1748039588.017862 707462 buffer_comparator.cc:145] Difference at 34: 0, expected 31.9291
E0000 00:00:1748039588.017865 707462 buffer_comparator.cc:145] Difference at 35: 0, expected 33.8781
E0000 00:00:1748039588.017868 707462 buffer_comparator.cc:145] Difference at 36: 0, expected 34.4514
E0000 00:00:1748039588.017871 707462 buffer_comparator.cc:145] Difference at 37: 0, expected 37.604
E0000 00:00:1748039588.017874 707462 buffer_comparator.cc:145] Difference at 38: 0, expected 33.8942
E0000 00:00:1748039588.017877 707462 buffer_comparator.cc:145] Difference at 39: 0, expected 34.3168
E0000 00:00:1748039588.017880 707462 buffer_comparator.cc:145] Difference at 40: 0, expected 32.06
E0000 00:00:1748039588.017883 707462 buffer_comparator.cc:145] Difference at 41: 0, expected 34.9762
2025-05-23 22:33:08.017887: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.020007 707462 buffer_comparator.cc:145] Difference at 32: 0, expected 32.3706
E0000 00:00:1748039588.020020 707462 buffer_comparator.cc:145] Difference at 33: 0, expected 32.7046
E0000 00:00:1748039588.020023 707462 buffer_comparator.cc:145] Difference at 34: 0, expected 31.9291
E0000 00:00:1748039588.020026 707462 buffer_comparator.cc:145] Difference at 35: 0, expected 33.8781
E0000 00:00:1748039588.020029 707462 buffer_comparator.cc:145] Difference at 36: 0, expected 34.4514
E0000 00:00:1748039588.020032 707462 buffer_comparator.cc:145] Difference at 37: 0, expected 37.604
E0000 00:00:1748039588.020035 707462 buffer_comparator.cc:145] Difference at 38: 0, expected 33.8942
E0000 00:00:1748039588.020038 707462 buffer_comparator.cc:145] Difference at 39: 0, expected 34.3168
E0000 00:00:1748039588.020041 707462 buffer_comparator.cc:145] Difference at 40: 0, expected 32.06
E0000 00:00:1748039588.020044 707462 buffer_comparator.cc:145] Difference at 41: 0, expected 34.9762
2025-05-23 22:33:08.020048: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.022166 707462 buffer_comparator.cc:145] Difference at 64: 0, expected 36.4939
E0000 00:00:1748039588.022180 707462 buffer_comparator.cc:145] Difference at 65: 0, expected 32.8155
E0000 00:00:1748039588.022184 707462 buffer_comparator.cc:145] Difference at 66: 0, expected 34.2602
E0000 00:00:1748039588.022186 707462 buffer_comparator.cc:145] Difference at 67: 0, expected 35.1865
E0000 00:00:1748039588.022189 707462 buffer_comparator.cc:145] Difference at 68: 0, expected 34.2647
E0000 00:00:1748039588.022192 707462 buffer_comparator.cc:145] Difference at 69: 0, expected 35.3053
E0000 00:00:1748039588.022195 707462 buffer_comparator.cc:145] Difference at 70: 0, expected 38.3918
E0000 00:00:1748039588.022198 707462 buffer_comparator.cc:145] Difference at 71: 0, expected 34.9764
E0000 00:00:1748039588.022201 707462 buffer_comparator.cc:145] Difference at 72: 0, expected 32.9802
E0000 00:00:1748039588.022204 707462 buffer_comparator.cc:145] Difference at 73: 0, expected 34.8911
2025-05-23 22:33:08.022209: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.024338 707462 buffer_comparator.cc:145] Difference at 64: 0, expected 36.4939
E0000 00:00:1748039588.024353 707462 buffer_comparator.cc:145] Difference at 65: 0, expected 32.8155
E0000 00:00:1748039588.024356 707462 buffer_comparator.cc:145] Difference at 66: 0, expected 34.2602
E0000 00:00:1748039588.024359 707462 buffer_comparator.cc:145] Difference at 67: 0, expected 35.1865
E0000 00:00:1748039588.024362 707462 buffer_comparator.cc:145] Difference at 68: 0, expected 34.2647
E0000 00:00:1748039588.024365 707462 buffer_comparator.cc:145] Difference at 69: 0, expected 35.3053
E0000 00:00:1748039588.024368 707462 buffer_comparator.cc:145] Difference at 70: 0, expected 38.3918
E0000 00:00:1748039588.024371 707462 buffer_comparator.cc:145] Difference at 71: 0, expected 34.9764
E0000 00:00:1748039588.024373 707462 buffer_comparator.cc:145] Difference at 72: 0, expected 32.9802
E0000 00:00:1748039588.024376 707462 buffer_comparator.cc:145] Difference at 73: 0, expected 34.8911
2025-05-23 22:33:08.024381: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.026533 707462 buffer_comparator.cc:145] Difference at 8192: 0, expected 30.4354
E0000 00:00:1748039588.026551 707462 buffer_comparator.cc:145] Difference at 8193: 0, expected 31.9843
E0000 00:00:1748039588.026554 707462 buffer_comparator.cc:145] Difference at 8194: 0, expected 32.9993
E0000 00:00:1748039588.026557 707462 buffer_comparator.cc:145] Difference at 8195: 0, expected 30.0135
E0000 00:00:1748039588.026560 707462 buffer_comparator.cc:145] Difference at 8196: 0, expected 28.7962
E0000 00:00:1748039588.026563 707462 buffer_comparator.cc:145] Difference at 8197: 0, expected 28.9198
E0000 00:00:1748039588.026566 707462 buffer_comparator.cc:145] Difference at 8198: 0, expected 31.4091
E0000 00:00:1748039588.026569 707462 buffer_comparator.cc:145] Difference at 8199: 0, expected 29.0414
E0000 00:00:1748039588.026572 707462 buffer_comparator.cc:145] Difference at 8200: 0, expected 31.2657
E0000 00:00:1748039588.026575 707462 buffer_comparator.cc:145] Difference at 8201: 0, expected 28.8774
2025-05-23 22:33:08.026579: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.028721 707462 buffer_comparator.cc:145] Difference at 8192: 0, expected 30.4354
E0000 00:00:1748039588.028737 707462 buffer_comparator.cc:145] Difference at 8193: 0, expected 31.9843
E0000 00:00:1748039588.028740 707462 buffer_comparator.cc:145] Difference at 8194: 0, expected 32.9993
E0000 00:00:1748039588.028743 707462 buffer_comparator.cc:145] Difference at 8195: 0, expected 30.0135
E0000 00:00:1748039588.028746 707462 buffer_comparator.cc:145] Difference at 8196: 0, expected 28.7962
E0000 00:00:1748039588.028748 707462 buffer_comparator.cc:145] Difference at 8197: 0, expected 28.9198
E0000 00:00:1748039588.028751 707462 buffer_comparator.cc:145] Difference at 8198: 0, expected 31.4091
E0000 00:00:1748039588.028754 707462 buffer_comparator.cc:145] Difference at 8199: 0, expected 29.0414
E0000 00:00:1748039588.028757 707462 buffer_comparator.cc:145] Difference at 8200: 0, expected 31.2657
E0000 00:00:1748039588.028760 707462 buffer_comparator.cc:145] Difference at 8201: 0, expected 28.8774
2025-05-23 22:33:08.028764: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.030910 707462 buffer_comparator.cc:145] Difference at 8192: 0, expected 30.4354
E0000 00:00:1748039588.030923 707462 buffer_comparator.cc:145] Difference at 8193: 0, expected 31.9843
E0000 00:00:1748039588.030927 707462 buffer_comparator.cc:145] Difference at 8194: 0, expected 32.9993
E0000 00:00:1748039588.030930 707462 buffer_comparator.cc:145] Difference at 8195: 0, expected 30.0135
E0000 00:00:1748039588.030933 707462 buffer_comparator.cc:145] Difference at 8196: 0, expected 28.7962
E0000 00:00:1748039588.030936 707462 buffer_comparator.cc:145] Difference at 8197: 0, expected 28.9198
E0000 00:00:1748039588.030938 707462 buffer_comparator.cc:145] Difference at 8198: 0, expected 31.4091
E0000 00:00:1748039588.030941 707462 buffer_comparator.cc:145] Difference at 8199: 0, expected 29.0414
E0000 00:00:1748039588.030944 707462 buffer_comparator.cc:145] Difference at 8200: 0, expected 31.2657
E0000 00:00:1748039588.030947 707462 buffer_comparator.cc:145] Difference at 8201: 0, expected 28.8774
2025-05-23 22:33:08.030952: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.039318 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.039334 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.8763
E0000 00:00:1748039588.039338 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 32.9469
E0000 00:00:1748039588.039341 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 34.7741
E0000 00:00:1748039588.039344 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 32.8701
E0000 00:00:1748039588.039346 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 31.0828
E0000 00:00:1748039588.039349 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 31.1895
E0000 00:00:1748039588.039352 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 31.3422
E0000 00:00:1748039588.039355 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 32.9441
E0000 00:00:1748039588.039358 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 34.7409
2025-05-23 22:33:08.039363: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.043894 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.043909 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.8763
E0000 00:00:1748039588.043913 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 32.9469
E0000 00:00:1748039588.043916 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 34.7741
E0000 00:00:1748039588.043918 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 32.8701
E0000 00:00:1748039588.043921 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 31.0828
E0000 00:00:1748039588.043924 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 31.1895
E0000 00:00:1748039588.043927 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 31.3422
E0000 00:00:1748039588.043930 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 32.9441
E0000 00:00:1748039588.043933 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 34.7409
2025-05-23 22:33:08.043937: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.048498 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.048514 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.8763
E0000 00:00:1748039588.048517 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 32.9469
E0000 00:00:1748039588.048520 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 34.7741
E0000 00:00:1748039588.048523 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 32.8701
E0000 00:00:1748039588.048526 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 31.0828
E0000 00:00:1748039588.048529 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 31.1895
E0000 00:00:1748039588.048532 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 31.3422
E0000 00:00:1748039588.048535 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 32.9441
E0000 00:00:1748039588.048538 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 34.7409
2025-05-23 22:33:08.048543: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.053084 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.053101 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.8763
E0000 00:00:1748039588.053104 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 32.9469
E0000 00:00:1748039588.053107 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 34.7741
E0000 00:00:1748039588.053110 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 32.8701
E0000 00:00:1748039588.053113 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 31.0828
E0000 00:00:1748039588.053116 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 31.1895
E0000 00:00:1748039588.053119 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 31.3422
E0000 00:00:1748039588.053122 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 32.9441
E0000 00:00:1748039588.053124 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 34.7409
2025-05-23 22:33:08.053129: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.057655 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.057673 707462 buffer_comparator.cc:145] Difference at 16: 0, expected 32.8763
E0000 00:00:1748039588.057676 707462 buffer_comparator.cc:145] Difference at 17: 0, expected 32.9469
E0000 00:00:1748039588.057679 707462 buffer_comparator.cc:145] Difference at 18: 0, expected 34.7741
E0000 00:00:1748039588.057682 707462 buffer_comparator.cc:145] Difference at 19: 0, expected 32.8701
E0000 00:00:1748039588.057685 707462 buffer_comparator.cc:145] Difference at 20: 0, expected 31.0828
E0000 00:00:1748039588.057688 707462 buffer_comparator.cc:145] Difference at 21: 0, expected 31.1895
E0000 00:00:1748039588.057691 707462 buffer_comparator.cc:145] Difference at 22: 0, expected 31.3422
E0000 00:00:1748039588.057694 707462 buffer_comparator.cc:145] Difference at 23: 0, expected 32.9441
E0000 00:00:1748039588.057697 707462 buffer_comparator.cc:145] Difference at 24: 0, expected 34.7409
2025-05-23 22:33:08.057701: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.062226 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.062242 707462 buffer_comparator.cc:145] Difference at 32: 0, expected 35.5022
E0000 00:00:1748039588.062245 707462 buffer_comparator.cc:145] Difference at 33: 0, expected 31.2274
E0000 00:00:1748039588.062248 707462 buffer_comparator.cc:145] Difference at 34: 0, expected 31.5749
E0000 00:00:1748039588.062251 707462 buffer_comparator.cc:145] Difference at 35: 0, expected 32.6706
E0000 00:00:1748039588.062254 707462 buffer_comparator.cc:145] Difference at 36: 0, expected 33.8859
E0000 00:00:1748039588.062257 707462 buffer_comparator.cc:145] Difference at 37: 0, expected 37.0505
E0000 00:00:1748039588.062259 707462 buffer_comparator.cc:145] Difference at 38: 0, expected 35.3452
E0000 00:00:1748039588.062262 707462 buffer_comparator.cc:145] Difference at 39: 0, expected 37.8185
E0000 00:00:1748039588.062265 707462 buffer_comparator.cc:145] Difference at 40: 0, expected 34.3828
2025-05-23 22:33:08.062271: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.066811 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.066827 707462 buffer_comparator.cc:145] Difference at 32: 0, expected 35.5022
E0000 00:00:1748039588.066831 707462 buffer_comparator.cc:145] Difference at 33: 0, expected 31.2274
E0000 00:00:1748039588.066833 707462 buffer_comparator.cc:145] Difference at 34: 0, expected 31.5749
E0000 00:00:1748039588.066836 707462 buffer_comparator.cc:145] Difference at 35: 0, expected 32.6706
E0000 00:00:1748039588.066839 707462 buffer_comparator.cc:145] Difference at 36: 0, expected 33.8859
E0000 00:00:1748039588.066842 707462 buffer_comparator.cc:145] Difference at 37: 0, expected 37.0505
E0000 00:00:1748039588.066845 707462 buffer_comparator.cc:145] Difference at 38: 0, expected 35.3452
E0000 00:00:1748039588.066848 707462 buffer_comparator.cc:145] Difference at 39: 0, expected 37.8185
E0000 00:00:1748039588.066851 707462 buffer_comparator.cc:145] Difference at 40: 0, expected 34.3828
2025-05-23 22:33:08.066855: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.071422 707462 buffer_comparator.cc:145] Difference at 1: 36.1679, expected 32.0834
E0000 00:00:1748039588.071436 707462 buffer_comparator.cc:145] Difference at 3: 41.4303, expected 32.9274
E0000 00:00:1748039588.071440 707462 buffer_comparator.cc:145] Difference at 8: 38.423, expected 34.022
E0000 00:00:1748039588.071443 707462 buffer_comparator.cc:145] Difference at 9: 36.6361, expected 32.0218
E0000 00:00:1748039588.071446 707462 buffer_comparator.cc:145] Difference at 14: 38.9704, expected 34.8082
E0000 00:00:1748039588.071449 707462 buffer_comparator.cc:145] Difference at 15: 37.8988, expected 33.5914
E0000 00:00:1748039588.071452 707462 buffer_comparator.cc:145] Difference at 16: 39.2657, expected 32.8763
E0000 00:00:1748039588.071455 707462 buffer_comparator.cc:145] Difference at 19: 37.1267, expected 32.8701
E0000 00:00:1748039588.071458 707462 buffer_comparator.cc:145] Difference at 20: 36.0826, expected 31.0828
E0000 00:00:1748039588.071461 707462 buffer_comparator.cc:145] Difference at 21: 36.7797, expected 31.1895
2025-05-23 22:33:08.071466: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.076023 707462 buffer_comparator.cc:145] Difference at 5: 31.0411, expected 34.8896
E0000 00:00:1748039588.076038 707462 buffer_comparator.cc:145] Difference at 20: 35.018, expected 31.0828
E0000 00:00:1748039588.076041 707462 buffer_comparator.cc:145] Difference at 21: 35.8637, expected 31.1895
E0000 00:00:1748039588.076044 707462 buffer_comparator.cc:145] Difference at 32: 30.5306, expected 35.5022
E0000 00:00:1748039588.076047 707462 buffer_comparator.cc:145] Difference at 37: 32.2286, expected 37.0505
E0000 00:00:1748039588.076050 707462 buffer_comparator.cc:145] Difference at 43: 29.5364, expected 35.6413
E0000 00:00:1748039588.076053 707462 buffer_comparator.cc:145] Difference at 50: 36.6745, expected 31.735
E0000 00:00:1748039588.076056 707462 buffer_comparator.cc:145] Difference at 56: 35.3037, expected 31.5996
E0000 00:00:1748039588.076059 707462 buffer_comparator.cc:145] Difference at 59: 28.7437, expected 34.669
E0000 00:00:1748039588.076062 707462 buffer_comparator.cc:145] Difference at 61: 35.5649, expected 31.5182
2025-05-23 22:33:08.076067: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.080620 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.080646 707462 buffer_comparator.cc:145] Difference at 39: 31.6271, expected 37.8185
E0000 00:00:1748039588.080649 707462 buffer_comparator.cc:145] Difference at 42: 29.8285, expected 33.6959
E0000 00:00:1748039588.080652 707462 buffer_comparator.cc:145] Difference at 48: 34.6791, expected 30.4053
E0000 00:00:1748039588.080655 707462 buffer_comparator.cc:145] Difference at 51: 29.7902, expected 34.2193
E0000 00:00:1748039588.080658 707462 buffer_comparator.cc:145] Difference at 52: 34.3219, expected 30.2305
E0000 00:00:1748039588.080661 707462 buffer_comparator.cc:145] Difference at 58: 29.7302, expected 35.0407
E0000 00:00:1748039588.080664 707462 buffer_comparator.cc:145] Difference at 62: 29.6881, expected 33.2153
E0000 00:00:1748039588.080667 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.080671 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
2025-05-23 22:33:08.080675: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.085254 707462 buffer_comparator.cc:145] Difference at 23: 36.8341, expected 32.9441
E0000 00:00:1748039588.085269 707462 buffer_comparator.cc:145] Difference at 29: 31.2701, expected 36.1798
E0000 00:00:1748039588.085272 707462 buffer_comparator.cc:145] Difference at 52: 35.5891, expected 30.2305
E0000 00:00:1748039588.085275 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.085279 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
E0000 00:00:1748039588.085282 707462 buffer_comparator.cc:145] Difference at 66: 0.495432, expected 34.1747
E0000 00:00:1748039588.085285 707462 buffer_comparator.cc:145] Difference at 67: 0.285409, expected 36.6246
E0000 00:00:1748039588.085288 707462 buffer_comparator.cc:145] Difference at 68: 0.166012, expected 33.8422
E0000 00:00:1748039588.085290 707462 buffer_comparator.cc:145] Difference at 69: 0.0715962, expected 34.0494
E0000 00:00:1748039588.085294 707462 buffer_comparator.cc:145] Difference at 70: 0.325998, expected 31.1267
2025-05-23 22:33:08.085298: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.090003 707462 buffer_comparator.cc:145] Difference at 23: 36.8341, expected 32.9441
E0000 00:00:1748039588.090019 707462 buffer_comparator.cc:145] Difference at 29: 31.2701, expected 36.1798
E0000 00:00:1748039588.090022 707462 buffer_comparator.cc:145] Difference at 52: 35.5891, expected 30.2305
E0000 00:00:1748039588.090025 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.090028 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
E0000 00:00:1748039588.090032 707462 buffer_comparator.cc:145] Difference at 66: 0.495432, expected 34.1747
E0000 00:00:1748039588.090034 707462 buffer_comparator.cc:145] Difference at 67: 0.285409, expected 36.6246
E0000 00:00:1748039588.090037 707462 buffer_comparator.cc:145] Difference at 68: 0.166012, expected 33.8422
E0000 00:00:1748039588.090040 707462 buffer_comparator.cc:145] Difference at 69: 0.0715962, expected 34.0494
E0000 00:00:1748039588.090043 707462 buffer_comparator.cc:145] Difference at 70: 0.325998, expected 31.1267
2025-05-23 22:33:08.090048: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.094612 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.094633 707462 buffer_comparator.cc:145] Difference at 39: 31.6271, expected 37.8185
E0000 00:00:1748039588.094636 707462 buffer_comparator.cc:145] Difference at 42: 29.8285, expected 33.6959
E0000 00:00:1748039588.094640 707462 buffer_comparator.cc:145] Difference at 48: 34.6791, expected 30.4053
E0000 00:00:1748039588.094643 707462 buffer_comparator.cc:145] Difference at 51: 29.7902, expected 34.2193
E0000 00:00:1748039588.094646 707462 buffer_comparator.cc:145] Difference at 52: 34.3219, expected 30.2305
E0000 00:00:1748039588.094649 707462 buffer_comparator.cc:145] Difference at 58: 29.7302, expected 35.0407
E0000 00:00:1748039588.094652 707462 buffer_comparator.cc:145] Difference at 62: 29.6881, expected 33.2153
E0000 00:00:1748039588.094655 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.094658 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
2025-05-23 22:33:08.094663: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.099235 707462 buffer_comparator.cc:145] Difference at 23: 36.8341, expected 32.9441
E0000 00:00:1748039588.099250 707462 buffer_comparator.cc:145] Difference at 29: 31.2701, expected 36.1798
E0000 00:00:1748039588.099253 707462 buffer_comparator.cc:145] Difference at 52: 35.5891, expected 30.2305
E0000 00:00:1748039588.099256 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.099259 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
E0000 00:00:1748039588.099262 707462 buffer_comparator.cc:145] Difference at 66: 0.495432, expected 34.1747
E0000 00:00:1748039588.099265 707462 buffer_comparator.cc:145] Difference at 67: 0.285409, expected 36.6246
E0000 00:00:1748039588.099268 707462 buffer_comparator.cc:145] Difference at 68: 0.166012, expected 33.8422
E0000 00:00:1748039588.099271 707462 buffer_comparator.cc:145] Difference at 69: 0.0715962, expected 34.0494
E0000 00:00:1748039588.099274 707462 buffer_comparator.cc:145] Difference at 70: 0.325998, expected 31.1267
2025-05-23 22:33:08.099279: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.103841 707462 buffer_comparator.cc:145] Difference at 23: 36.8341, expected 32.9441
E0000 00:00:1748039588.103855 707462 buffer_comparator.cc:145] Difference at 29: 31.2701, expected 36.1798
E0000 00:00:1748039588.103859 707462 buffer_comparator.cc:145] Difference at 52: 35.5891, expected 30.2305
E0000 00:00:1748039588.103862 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.103865 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
E0000 00:00:1748039588.103868 707462 buffer_comparator.cc:145] Difference at 66: 0.495432, expected 34.1747
E0000 00:00:1748039588.103871 707462 buffer_comparator.cc:145] Difference at 67: 0.285409, expected 36.6246
E0000 00:00:1748039588.103874 707462 buffer_comparator.cc:145] Difference at 68: 0.166012, expected 33.8422
E0000 00:00:1748039588.103877 707462 buffer_comparator.cc:145] Difference at 69: 0.0715962, expected 34.0494
E0000 00:00:1748039588.103880 707462 buffer_comparator.cc:145] Difference at 70: 0.325998, expected 31.1267
2025-05-23 22:33:08.103884: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.108495 707462 buffer_comparator.cc:145] Difference at 3: 37.1279, expected 32.9274
E0000 00:00:1748039588.108511 707462 buffer_comparator.cc:145] Difference at 15: 39.8297, expected 33.5914
E0000 00:00:1748039588.108514 707462 buffer_comparator.cc:145] Difference at 16: 41.0514, expected 32.8763
E0000 00:00:1748039588.108518 707462 buffer_comparator.cc:145] Difference at 19: 37.7434, expected 32.8701
E0000 00:00:1748039588.108521 707462 buffer_comparator.cc:145] Difference at 20: 38.115, expected 31.0828
E0000 00:00:1748039588.108524 707462 buffer_comparator.cc:145] Difference at 21: 37.5197, expected 31.1895
E0000 00:00:1748039588.108528 707462 buffer_comparator.cc:145] Difference at 30: 34.2314, expected 30.5264
E0000 00:00:1748039588.108531 707462 buffer_comparator.cc:145] Difference at 34: 36.523, expected 31.5749
E0000 00:00:1748039588.108534 707462 buffer_comparator.cc:145] Difference at 39: 31.1668, expected 37.8185
E0000 00:00:1748039588.108537 707462 buffer_comparator.cc:145] Difference at 48: 38.3567, expected 30.4053
2025-05-23 22:33:08.108541: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.113111 707462 buffer_comparator.cc:145] Difference at 23: 36.8341, expected 32.9441
E0000 00:00:1748039588.113125 707462 buffer_comparator.cc:145] Difference at 29: 31.2701, expected 36.1798
E0000 00:00:1748039588.113129 707462 buffer_comparator.cc:145] Difference at 52: 35.5891, expected 30.2305
E0000 00:00:1748039588.113132 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.113135 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
E0000 00:00:1748039588.113138 707462 buffer_comparator.cc:145] Difference at 66: 0.495432, expected 34.1747
E0000 00:00:1748039588.113141 707462 buffer_comparator.cc:145] Difference at 67: 0.285409, expected 36.6246
E0000 00:00:1748039588.113144 707462 buffer_comparator.cc:145] Difference at 68: 0.166012, expected 33.8422
E0000 00:00:1748039588.113147 707462 buffer_comparator.cc:145] Difference at 69: 0.0715962, expected 34.0494
E0000 00:00:1748039588.113150 707462 buffer_comparator.cc:145] Difference at 70: 0.325998, expected 31.1267
2025-05-23 22:33:08.113155: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.117698 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.117713 707462 buffer_comparator.cc:145] Difference at 39: 31.6271, expected 37.8185
E0000 00:00:1748039588.117716 707462 buffer_comparator.cc:145] Difference at 42: 29.8285, expected 33.6959
E0000 00:00:1748039588.117719 707462 buffer_comparator.cc:145] Difference at 48: 34.6791, expected 30.4053
E0000 00:00:1748039588.117722 707462 buffer_comparator.cc:145] Difference at 51: 29.7902, expected 34.2193
E0000 00:00:1748039588.117725 707462 buffer_comparator.cc:145] Difference at 52: 34.3219, expected 30.2305
E0000 00:00:1748039588.117728 707462 buffer_comparator.cc:145] Difference at 58: 29.7302, expected 35.0407
E0000 00:00:1748039588.117731 707462 buffer_comparator.cc:145] Difference at 62: 29.6881, expected 33.2153
E0000 00:00:1748039588.117734 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.117737 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
2025-05-23 22:33:08.117742: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.122315 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.122334 707462 buffer_comparator.cc:145] Difference at 39: 31.6271, expected 37.8185
E0000 00:00:1748039588.122337 707462 buffer_comparator.cc:145] Difference at 42: 29.8285, expected 33.6959
E0000 00:00:1748039588.122340 707462 buffer_comparator.cc:145] Difference at 48: 34.6791, expected 30.4053
E0000 00:00:1748039588.122343 707462 buffer_comparator.cc:145] Difference at 51: 29.7902, expected 34.2193
E0000 00:00:1748039588.122346 707462 buffer_comparator.cc:145] Difference at 52: 34.3219, expected 30.2305
E0000 00:00:1748039588.122349 707462 buffer_comparator.cc:145] Difference at 58: 29.7302, expected 35.0407
E0000 00:00:1748039588.122353 707462 buffer_comparator.cc:145] Difference at 62: 29.6881, expected 33.2153
E0000 00:00:1748039588.122356 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.122359 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
2025-05-23 22:33:08.122364: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.126930 707462 buffer_comparator.cc:145] Difference at 23: 36.8341, expected 32.9441
E0000 00:00:1748039588.126945 707462 buffer_comparator.cc:145] Difference at 29: 31.2701, expected 36.1798
E0000 00:00:1748039588.126948 707462 buffer_comparator.cc:145] Difference at 52: 35.5891, expected 30.2305
E0000 00:00:1748039588.126951 707462 buffer_comparator.cc:145] Difference at 64: 0.915925, expected 34.4632
E0000 00:00:1748039588.126954 707462 buffer_comparator.cc:145] Difference at 65: 0.934271, expected 35.0461
E0000 00:00:1748039588.126957 707462 buffer_comparator.cc:145] Difference at 66: 0.495432, expected 34.1747
E0000 00:00:1748039588.126960 707462 buffer_comparator.cc:145] Difference at 67: 0.285409, expected 36.6246
E0000 00:00:1748039588.126963 707462 buffer_comparator.cc:145] Difference at 68: 0.166012, expected 33.8422
E0000 00:00:1748039588.126966 707462 buffer_comparator.cc:145] Difference at 69: 0.0715962, expected 34.0494
E0000 00:00:1748039588.126969 707462 buffer_comparator.cc:145] Difference at 70: 0.325998, expected 31.1267
2025-05-23 22:33:08.126974: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.131578 707462 buffer_comparator.cc:145] Difference at 1: 36.6464, expected 32.0834
E0000 00:00:1748039588.131592 707462 buffer_comparator.cc:145] Difference at 21: 36.9561, expected 31.1895
E0000 00:00:1748039588.131596 707462 buffer_comparator.cc:145] Difference at 22: 35.8185, expected 31.3422
E0000 00:00:1748039588.131599 707462 buffer_comparator.cc:145] Difference at 33: 36.1795, expected 31.2274
E0000 00:00:1748039588.131602 707462 buffer_comparator.cc:145] Difference at 35: 38.2464, expected 32.6706
E0000 00:00:1748039588.131605 707462 buffer_comparator.cc:145] Difference at 45: 29.3617, expected 34.2598
E0000 00:00:1748039588.131608 707462 buffer_comparator.cc:145] Difference at 48: 33.9342, expected 30.4053
E0000 00:00:1748039588.131611 707462 buffer_comparator.cc:145] Difference at 52: 34.8898, expected 30.2305
E0000 00:00:1748039588.131614 707462 buffer_comparator.cc:145] Difference at 54: 39.4298, expected 33.4086
E0000 00:00:1748039588.131617 707462 buffer_comparator.cc:145] Difference at 63: 39.2801, expected 34.5611
2025-05-23 22:33:08.131622: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.136373 707462 buffer_comparator.cc:145] Difference at 3: 36.8014, expected 32.9274
E0000 00:00:1748039588.136387 707462 buffer_comparator.cc:145] Difference at 9: 36.0916, expected 32.0218
E0000 00:00:1748039588.136390 707462 buffer_comparator.cc:145] Difference at 16: 38.425, expected 32.8763
E0000 00:00:1748039588.136394 707462 buffer_comparator.cc:145] Difference at 20: 35.7997, expected 31.0828
E0000 00:00:1748039588.136397 707462 buffer_comparator.cc:145] Difference at 34: 35.6762, expected 31.5749
E0000 00:00:1748039588.136400 707462 buffer_comparator.cc:145] Difference at 48: 35.5051, expected 30.4053
E0000 00:00:1748039588.136403 707462 buffer_comparator.cc:145] Difference at 52: 33.7408, expected 30.2305
E0000 00:00:1748039588.136406 707462 buffer_comparator.cc:145] Difference at 57: 38.4488, expected 33.7507
E0000 00:00:1748039588.136409 707462 buffer_comparator.cc:145] Difference at 67: 42.4147, expected 36.6246
E0000 00:00:1748039588.136413 707462 buffer_comparator.cc:145] Difference at 70: 36.3766, expected 31.1267
2025-05-23 22:33:08.136417: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.140987 707462 buffer_comparator.cc:145] Difference at 3: 36.8014, expected 32.9274
E0000 00:00:1748039588.141001 707462 buffer_comparator.cc:145] Difference at 9: 36.0916, expected 32.0218
E0000 00:00:1748039588.141004 707462 buffer_comparator.cc:145] Difference at 16: 38.425, expected 32.8763
E0000 00:00:1748039588.141008 707462 buffer_comparator.cc:145] Difference at 20: 35.7997, expected 31.0828
E0000 00:00:1748039588.141011 707462 buffer_comparator.cc:145] Difference at 34: 35.6762, expected 31.5749
E0000 00:00:1748039588.141014 707462 buffer_comparator.cc:145] Difference at 48: 35.5051, expected 30.4053
E0000 00:00:1748039588.141017 707462 buffer_comparator.cc:145] Difference at 52: 33.7408, expected 30.2305
E0000 00:00:1748039588.141020 707462 buffer_comparator.cc:145] Difference at 57: 38.4488, expected 33.7507
E0000 00:00:1748039588.141023 707462 buffer_comparator.cc:145] Difference at 67: 42.4147, expected 36.6246
E0000 00:00:1748039588.141026 707462 buffer_comparator.cc:145] Difference at 70: 36.3766, expected 31.1267
2025-05-23 22:33:08.141031: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.145617 707462 buffer_comparator.cc:145] Difference at 3: 40.9573, expected 32.9274
E0000 00:00:1748039588.145642 707462 buffer_comparator.cc:145] Difference at 20: 35.2578, expected 31.0828
E0000 00:00:1748039588.145646 707462 buffer_comparator.cc:145] Difference at 21: 35.8832, expected 31.1895
E0000 00:00:1748039588.145649 707462 buffer_comparator.cc:145] Difference at 22: 39.5176, expected 31.3422
E0000 00:00:1748039588.145652 707462 buffer_comparator.cc:145] Difference at 23: 37.0378, expected 32.9441
E0000 00:00:1748039588.145655 707462 buffer_comparator.cc:145] Difference at 26: 42.8329, expected 35.311
E0000 00:00:1748039588.145658 707462 buffer_comparator.cc:145] Difference at 30: 36.9447, expected 30.5264
E0000 00:00:1748039588.145661 707462 buffer_comparator.cc:145] Difference at 32: 40.373, expected 35.5022
E0000 00:00:1748039588.145664 707462 buffer_comparator.cc:145] Difference at 33: 36.2629, expected 31.2274
E0000 00:00:1748039588.145667 707462 buffer_comparator.cc:145] Difference at 34: 37.1722, expected 31.5749
2025-05-23 22:33:08.145671: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.150227 707462 buffer_comparator.cc:145] Difference at 4: 28.6004, expected 32.6935
E0000 00:00:1748039588.150242 707462 buffer_comparator.cc:145] Difference at 39: 31.6271, expected 37.8185
E0000 00:00:1748039588.150245 707462 buffer_comparator.cc:145] Difference at 42: 29.8285, expected 33.6959
E0000 00:00:1748039588.150248 707462 buffer_comparator.cc:145] Difference at 48: 34.6791, expected 30.4053
E0000 00:00:1748039588.150251 707462 buffer_comparator.cc:145] Difference at 51: 29.7902, expected 34.2193
E0000 00:00:1748039588.150254 707462 buffer_comparator.cc:145] Difference at 52: 34.3219, expected 30.2305
E0000 00:00:1748039588.150257 707462 buffer_comparator.cc:145] Difference at 58: 29.7302, expected 35.0407
E0000 00:00:1748039588.150260 707462 buffer_comparator.cc:145] Difference at 62: 29.6881, expected 33.2153
E0000 00:00:1748039588.150263 707462 buffer_comparator.cc:145] Difference at 70: 36.6794, expected 31.1267
E0000 00:00:1748039588.150266 707462 buffer_comparator.cc:145] Difference at 73: 34.609, expected 29.7019
2025-05-23 22:33:08.150271: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.153017 707462 buffer_comparator.cc:145] Difference at 16: -nan, expected 34.4785
E0000 00:00:1748039588.153028 707462 buffer_comparator.cc:145] Difference at 17: -nan, expected 37.4173
E0000 00:00:1748039588.153031 707462 buffer_comparator.cc:145] Difference at 18: -nan, expected 31.9632
E0000 00:00:1748039588.153034 707462 buffer_comparator.cc:145] Difference at 19: -nan, expected 32.4228
E0000 00:00:1748039588.153037 707462 buffer_comparator.cc:145] Difference at 20: -nan, expected 35.0304
E0000 00:00:1748039588.153039 707462 buffer_comparator.cc:145] Difference at 21: -nan, expected 34.3462
E0000 00:00:1748039588.153042 707462 buffer_comparator.cc:145] Difference at 22: -nan, expected 32.7297
E0000 00:00:1748039588.153045 707462 buffer_comparator.cc:145] Difference at 23: -nan, expected 33.394
E0000 00:00:1748039588.153048 707462 buffer_comparator.cc:145] Difference at 24: -nan, expected 36.6392
E0000 00:00:1748039588.153050 707462 buffer_comparator.cc:145] Difference at 25: -nan, expected 33.2879
2025-05-23 22:33:08.153055: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.155131 707462 buffer_comparator.cc:145] Difference at 16: -nan, expected 34.4785
E0000 00:00:1748039588.155142 707462 buffer_comparator.cc:145] Difference at 17: -nan, expected 37.4173
E0000 00:00:1748039588.155145 707462 buffer_comparator.cc:145] Difference at 18: -nan, expected 31.9632
E0000 00:00:1748039588.155147 707462 buffer_comparator.cc:145] Difference at 19: -nan, expected 32.4228
E0000 00:00:1748039588.155150 707462 buffer_comparator.cc:145] Difference at 20: -nan, expected 35.0304
E0000 00:00:1748039588.155153 707462 buffer_comparator.cc:145] Difference at 21: -nan, expected 34.3462
E0000 00:00:1748039588.155155 707462 buffer_comparator.cc:145] Difference at 22: -nan, expected 32.7297
E0000 00:00:1748039588.155158 707462 buffer_comparator.cc:145] Difference at 23: -nan, expected 33.394
E0000 00:00:1748039588.155161 707462 buffer_comparator.cc:145] Difference at 24: -nan, expected 36.6392
E0000 00:00:1748039588.155164 707462 buffer_comparator.cc:145] Difference at 25: -nan, expected 33.2879
2025-05-23 22:33:08.155168: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.157219 707462 buffer_comparator.cc:145] Difference at 16: -nan, expected 34.4785
E0000 00:00:1748039588.157230 707462 buffer_comparator.cc:145] Difference at 17: -nan, expected 37.4173
E0000 00:00:1748039588.157233 707462 buffer_comparator.cc:145] Difference at 18: -nan, expected 31.9632
E0000 00:00:1748039588.157236 707462 buffer_comparator.cc:145] Difference at 19: -nan, expected 32.4228
E0000 00:00:1748039588.157238 707462 buffer_comparator.cc:145] Difference at 20: -nan, expected 35.0304
E0000 00:00:1748039588.157241 707462 buffer_comparator.cc:145] Difference at 21: -nan, expected 34.3462
E0000 00:00:1748039588.157244 707462 buffer_comparator.cc:145] Difference at 22: -nan, expected 32.7297
E0000 00:00:1748039588.157247 707462 buffer_comparator.cc:145] Difference at 23: -nan, expected 33.394
E0000 00:00:1748039588.157249 707462 buffer_comparator.cc:145] Difference at 24: -nan, expected 36.6392
E0000 00:00:1748039588.157252 707462 buffer_comparator.cc:145] Difference at 25: -nan, expected 33.2879
2025-05-23 22:33:08.157257: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.159320 707462 buffer_comparator.cc:145] Difference at 32: -nan, expected 33.5174
E0000 00:00:1748039588.159331 707462 buffer_comparator.cc:145] Difference at 33: -nan, expected 32.8146
E0000 00:00:1748039588.159336 707462 buffer_comparator.cc:145] Difference at 34: -nan, expected 34.3622
E0000 00:00:1748039588.159339 707462 buffer_comparator.cc:145] Difference at 35: -nan, expected 30.8134
E0000 00:00:1748039588.159341 707462 buffer_comparator.cc:145] Difference at 36: -nan, expected 33.9446
E0000 00:00:1748039588.159344 707462 buffer_comparator.cc:145] Difference at 37: -nan, expected 35.4711
E0000 00:00:1748039588.159347 707462 buffer_comparator.cc:145] Difference at 38: -nan, expected 34.5312
E0000 00:00:1748039588.159349 707462 buffer_comparator.cc:145] Difference at 39: -nan, expected 36.127
E0000 00:00:1748039588.159352 707462 buffer_comparator.cc:145] Difference at 40: -nan, expected 33.3523
E0000 00:00:1748039588.159355 707462 buffer_comparator.cc:145] Difference at 41: -nan, expected 34.9963
2025-05-23 22:33:08.159359: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.161435 707462 buffer_comparator.cc:145] Difference at 64: -nan, expected 30.326
E0000 00:00:1748039588.161446 707462 buffer_comparator.cc:145] Difference at 65: -nan, expected 33.1404
E0000 00:00:1748039588.161449 707462 buffer_comparator.cc:145] Difference at 66: -nan, expected 34.4443
E0000 00:00:1748039588.161452 707462 buffer_comparator.cc:145] Difference at 67: -nan, expected 32.8887
E0000 00:00:1748039588.161455 707462 buffer_comparator.cc:145] Difference at 68: -nan, expected 34.4676
E0000 00:00:1748039588.161457 707462 buffer_comparator.cc:145] Difference at 69: -nan, expected 32.0586
E0000 00:00:1748039588.161460 707462 buffer_comparator.cc:145] Difference at 70: -nan, expected 34.49
E0000 00:00:1748039588.161463 707462 buffer_comparator.cc:145] Difference at 71: -nan, expected 32.0229
E0000 00:00:1748039588.161465 707462 buffer_comparator.cc:145] Difference at 72: -nan, expected 32.6764
E0000 00:00:1748039588.161468 707462 buffer_comparator.cc:145] Difference at 73: -nan, expected 31.6891
2025-05-23 22:33:08.161473: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.163534 707462 buffer_comparator.cc:145] Difference at 64: -nan, expected 30.326
E0000 00:00:1748039588.163545 707462 buffer_comparator.cc:145] Difference at 65: -nan, expected 33.1404
E0000 00:00:1748039588.163548 707462 buffer_comparator.cc:145] Difference at 66: -nan, expected 34.4443
E0000 00:00:1748039588.163551 707462 buffer_comparator.cc:145] Difference at 67: -nan, expected 32.8887
E0000 00:00:1748039588.163554 707462 buffer_comparator.cc:145] Difference at 68: -nan, expected 34.4676
E0000 00:00:1748039588.163557 707462 buffer_comparator.cc:145] Difference at 69: -nan, expected 32.0586
E0000 00:00:1748039588.163559 707462 buffer_comparator.cc:145] Difference at 70: -nan, expected 34.49
E0000 00:00:1748039588.163562 707462 buffer_comparator.cc:145] Difference at 71: -nan, expected 32.0229
E0000 00:00:1748039588.163565 707462 buffer_comparator.cc:145] Difference at 72: -nan, expected 32.6764
E0000 00:00:1748039588.163567 707462 buffer_comparator.cc:145] Difference at 73: -nan, expected 31.6891
2025-05-23 22:33:08.163572: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1179] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1748039588.165622 707462 buffer_comparator.cc:145] Difference at 64: -nan, expected 30.326
E0000 00:00:1748039588.165639 707462 buffer_comparator.cc:145] Difference at 65: -nan, expected 33.1404
E0000 00:00:1748039588.165642 707462 buffer_comparator.cc:145] Difference at 66: -nan, expected 34.4443
E0000 00:00:1748039588.165645 707462 buffer_comparator.cc:145] Difference at 67: -nan, expected 32.8887
Iteration: [ 1/ 50000] Loss: 20.529172897 (20.529172897) Physics Loss: 16.935272217 (16.935272217) Data Loss: 2.008198738 (2.008198738) BC Loss: 1.585703492 (1.585703492)
Iteration: [ 1001/ 50000] Loss: 0.017321292 (0.019188093) Physics Loss: 0.000387124 (0.000483649) Data Loss: 0.005288614 (0.007527556) BC Loss: 0.011645554 (0.011176892)
Iteration: [ 2001/ 50000] Loss: 0.015881859 (0.018626312) Physics Loss: 0.001433536 (0.001500272) Data Loss: 0.004478142 (0.006470782) BC Loss: 0.009970182 (0.010655257)
Iteration: [ 3001/ 50000] Loss: 0.015988765 (0.015200405) Physics Loss: 0.000754316 (0.001274648) Data Loss: 0.004122454 (0.004274714) BC Loss: 0.011111995 (0.009651039)
Iteration: [ 4001/ 50000] Loss: 0.010084770 (0.009846112) Physics Loss: 0.004493770 (0.004005726) Data Loss: 0.002406818 (0.002175212) BC Loss: 0.003184181 (0.003665175)
Iteration: [ 5001/ 50000] Loss: 0.006814518 (0.006231492) Physics Loss: 0.002749096 (0.003049052) Data Loss: 0.002047949 (0.001410708) BC Loss: 0.002017472 (0.001771732)
Iteration: [ 6001/ 50000] Loss: 0.000958037 (0.001114675) Physics Loss: 0.000273446 (0.000262883) Data Loss: 0.000520235 (0.000666318) BC Loss: 0.000164357 (0.000185475)
Iteration: [ 7001/ 50000] Loss: 0.001299945 (0.000882932) Physics Loss: 0.000330739 (0.000318865) Data Loss: 0.000871611 (0.000459761) BC Loss: 0.000097594 (0.000104306)
Iteration: [ 8001/ 50000] Loss: 0.001464393 (0.000967458) Physics Loss: 0.001072832 (0.000473990) Data Loss: 0.000297506 (0.000389055) BC Loss: 0.000094055 (0.000104413)
Iteration: [ 9001/ 50000] Loss: 0.001841821 (0.002833713) Physics Loss: 0.000678029 (0.001821128) Data Loss: 0.000698891 (0.000460439) BC Loss: 0.000464901 (0.000552146)
Iteration: [ 10001/ 50000] Loss: 0.000793718 (0.000768279) Physics Loss: 0.000450070 (0.000394656) Data Loss: 0.000272823 (0.000307522) BC Loss: 0.000070825 (0.000066100)
Iteration: [ 11001/ 50000] Loss: 0.000307545 (0.000371781) Physics Loss: 0.000070551 (0.000059241) Data Loss: 0.000192187 (0.000274836) BC Loss: 0.000044806 (0.000037704)
Iteration: [ 12001/ 50000] Loss: 0.000264240 (0.000347334) Physics Loss: 0.000050129 (0.000059885) Data Loss: 0.000172895 (0.000249727) BC Loss: 0.000041215 (0.000037723)
Iteration: [ 13001/ 50000] Loss: 0.000277749 (0.000325745) Physics Loss: 0.000040338 (0.000064447) Data Loss: 0.000203687 (0.000226932) BC Loss: 0.000033723 (0.000034366)
Iteration: [ 14001/ 50000] Loss: 0.000360874 (0.000330477) Physics Loss: 0.000045172 (0.000062879) Data Loss: 0.000278050 (0.000236518) BC Loss: 0.000037652 (0.000031080)
Iteration: [ 15001/ 50000] Loss: 0.000282735 (0.000295349) Physics Loss: 0.000088833 (0.000064575) Data Loss: 0.000159011 (0.000198782) BC Loss: 0.000034890 (0.000031992)
Iteration: [ 16001/ 50000] Loss: 0.000205527 (0.000284188) Physics Loss: 0.000037767 (0.000053886) Data Loss: 0.000135716 (0.000200887) BC Loss: 0.000032044 (0.000029415)
Iteration: [ 17001/ 50000] Loss: 0.000384842 (0.000286053) Physics Loss: 0.000066238 (0.000056009) Data Loss: 0.000295485 (0.000203404) BC Loss: 0.000023119 (0.000026640)
Iteration: [ 18001/ 50000] Loss: 0.000215725 (0.000272987) Physics Loss: 0.000052321 (0.000052928) Data Loss: 0.000131431 (0.000193379) BC Loss: 0.000031974 (0.000026680)
Iteration: [ 19001/ 50000] Loss: 0.000215173 (0.000277970) Physics Loss: 0.000058122 (0.000052191) Data Loss: 0.000135308 (0.000199108) BC Loss: 0.000021743 (0.000026670)
Iteration: [ 20001/ 50000] Loss: 0.000297623 (0.000262847) Physics Loss: 0.000054696 (0.000057994) Data Loss: 0.000222282 (0.000179757) BC Loss: 0.000020645 (0.000025096)
Iteration: [ 21001/ 50000] Loss: 0.000283539 (0.000241687) Physics Loss: 0.000034155 (0.000045711) Data Loss: 0.000224215 (0.000171413) BC Loss: 0.000025168 (0.000024563)
Iteration: [ 22001/ 50000] Loss: 0.000171136 (0.000247036) Physics Loss: 0.000038278 (0.000056392) Data Loss: 0.000105568 (0.000167438) BC Loss: 0.000027290 (0.000023206)
Iteration: [ 23001/ 50000] Loss: 0.000240798 (0.000255637) Physics Loss: 0.000053050 (0.000055502) Data Loss: 0.000165620 (0.000176228) BC Loss: 0.000022128 (0.000023907)
Iteration: [ 24001/ 50000] Loss: 0.000283106 (0.000249614) Physics Loss: 0.000042711 (0.000053635) Data Loss: 0.000216142 (0.000170252) BC Loss: 0.000024254 (0.000025726)
Iteration: [ 25001/ 50000] Loss: 0.000199778 (0.000223985) Physics Loss: 0.000030719 (0.000033137) Data Loss: 0.000146854 (0.000169124) BC Loss: 0.000022205 (0.000021725)
Iteration: [ 26001/ 50000] Loss: 0.000204676 (0.000249734) Physics Loss: 0.000028904 (0.000058925) Data Loss: 0.000157492 (0.000166216) BC Loss: 0.000018280 (0.000024594)
Iteration: [ 27001/ 50000] Loss: 0.000220342 (0.000229080) Physics Loss: 0.000039189 (0.000041789) Data Loss: 0.000157917 (0.000165389) BC Loss: 0.000023235 (0.000021902)
Iteration: [ 28001/ 50000] Loss: 0.000193007 (0.000219532) Physics Loss: 0.000026483 (0.000039869) Data Loss: 0.000146609 (0.000157922) BC Loss: 0.000019916 (0.000021740)
Iteration: [ 29001/ 50000] Loss: 0.000215370 (0.000228163) Physics Loss: 0.000041442 (0.000047834) Data Loss: 0.000140694 (0.000157771) BC Loss: 0.000033234 (0.000022558)
Iteration: [ 30001/ 50000] Loss: 0.000227084 (0.000220160) Physics Loss: 0.000048764 (0.000039706) Data Loss: 0.000157700 (0.000160400) BC Loss: 0.000020620 (0.000020055)
Iteration: [ 31001/ 50000] Loss: 0.000247517 (0.000215346) Physics Loss: 0.000018236 (0.000037168) Data Loss: 0.000211377 (0.000158035) BC Loss: 0.000017903 (0.000020143)
Iteration: [ 32001/ 50000] Loss: 0.000201217 (0.000200759) Physics Loss: 0.000025999 (0.000027192) Data Loss: 0.000154025 (0.000153511) BC Loss: 0.000021193 (0.000020057)
Iteration: [ 33001/ 50000] Loss: 0.000176313 (0.000193139) Physics Loss: 0.000020468 (0.000024745) Data Loss: 0.000134218 (0.000148609) BC Loss: 0.000021627 (0.000019786)
Iteration: [ 34001/ 50000] Loss: 0.000196718 (0.000192766) Physics Loss: 0.000034154 (0.000028387) Data Loss: 0.000143433 (0.000144866) BC Loss: 0.000019131 (0.000019513)
Iteration: [ 35001/ 50000] Loss: 0.000151887 (0.000201741) Physics Loss: 0.000037206 (0.000034578) Data Loss: 0.000095043 (0.000147770) BC Loss: 0.000019638 (0.000019394)
Iteration: [ 36001/ 50000] Loss: 0.000154521 (0.000195857) Physics Loss: 0.000022864 (0.000027813) Data Loss: 0.000116497 (0.000148621) BC Loss: 0.000015160 (0.000019423)
Iteration: [ 37001/ 50000] Loss: 0.000272683 (0.000194823) Physics Loss: 0.000034293 (0.000030298) Data Loss: 0.000218445 (0.000145534) BC Loss: 0.000019945 (0.000018991)
Iteration: [ 38001/ 50000] Loss: 0.000236056 (0.000201835) Physics Loss: 0.000027764 (0.000032658) Data Loss: 0.000188621 (0.000151346) BC Loss: 0.000019671 (0.000017831)
Iteration: [ 39001/ 50000] Loss: 0.000171183 (0.000191931) Physics Loss: 0.000028466 (0.000026645) Data Loss: 0.000126124 (0.000146002) BC Loss: 0.000016593 (0.000019284)
Iteration: [ 40001/ 50000] Loss: 0.000182631 (0.000212775) Physics Loss: 0.000036312 (0.000047015) Data Loss: 0.000128745 (0.000145980) BC Loss: 0.000017575 (0.000019780)
Iteration: [ 41001/ 50000] Loss: 0.000159033 (0.000190222) Physics Loss: 0.000020863 (0.000024467) Data Loss: 0.000119687 (0.000147972) BC Loss: 0.000018483 (0.000017783)
Iteration: [ 42001/ 50000] Loss: 0.000176350 (0.000188812) Physics Loss: 0.000024680 (0.000023249) Data Loss: 0.000133921 (0.000146800) BC Loss: 0.000017750 (0.000018763)
Iteration: [ 43001/ 50000] Loss: 0.000197619 (0.000194697) Physics Loss: 0.000029880 (0.000025560) Data Loss: 0.000151151 (0.000147349) BC Loss: 0.000016588 (0.000021788)
Iteration: [ 44001/ 50000] Loss: 0.000173947 (0.000191413) Physics Loss: 0.000018872 (0.000025112) Data Loss: 0.000140941 (0.000143911) BC Loss: 0.000014134 (0.000022390)
Iteration: [ 45001/ 50000] Loss: 0.000260985 (0.000199877) Physics Loss: 0.000028050 (0.000027113) Data Loss: 0.000207982 (0.000152018) BC Loss: 0.000024953 (0.000020745)
Iteration: [ 46001/ 50000] Loss: 0.000200032 (0.000192333) Physics Loss: 0.000031079 (0.000034827) Data Loss: 0.000148447 (0.000139151) BC Loss: 0.000020505 (0.000018354)
Iteration: [ 47001/ 50000] Loss: 0.000169360 (0.000185530) Physics Loss: 0.000014163 (0.000027930) Data Loss: 0.000135304 (0.000139651) BC Loss: 0.000019893 (0.000017949)
Iteration: [ 48001/ 50000] Loss: 0.000178327 (0.000180882) Physics Loss: 0.000017509 (0.000022763) Data Loss: 0.000142409 (0.000138938) BC Loss: 0.000018409 (0.000019181)
Iteration: [ 49001/ 50000] Loss: 0.000167815 (0.000191479) Physics Loss: 0.000028613 (0.000030012) Data Loss: 0.000120989 (0.000142755) BC Loss: 0.000018213 (0.000018711)Visualizing the Results
julia
ts, xs, ys = 0.0f0:0.05f0:2.0f0, 0.0f0:0.02f0:2.0f0, 0.0f0:0.02f0:2.0f0
grid = stack([[elem...] for elem in vec(collect(Iterators.product(xs, ys, ts)))])
u_real = reshape(analytical_solution(grid), length(xs), length(ys), length(ts))
grid_normalized = (grid .- minimum(grid)) ./ (maximum(grid) .- minimum(grid))
u_pred = reshape(trained_model(grid_normalized), length(xs), length(ys), length(ts))
u_pred = u_pred .* (max_pde_val - min_pde_val) .+ min_pde_val
begin
fig = Figure()
ax = CairoMakie.Axis(fig[1, 1]; xlabel="x", ylabel="y")
errs = [abs.(u_pred[:, :, i] .- u_real[:, :, i]) for i in 1:length(ts)]
Colorbar(fig[1, 2]; limits=extrema(stack(errs)))
CairoMakie.record(fig, "pinn_nested_ad.gif", 1:length(ts); framerate=10) do i
ax.title = "Abs. Predictor Error | Time: $(ts[i])"
err = errs[i]
contour!(ax, xs, ys, err; levels=10, linewidth=2)
heatmap!(ax, xs, ys, err)
return fig
end
fig
end
Appendix
julia
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
endJulia Version 1.11.5
Commit 760b2e5b739 (2025-04-14 06:53 UTC)
Build Info:
Official https://julialang.org/ release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 48 × AMD EPYC 7402 24-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-16.0.6 (ORCJIT, znver2)
Threads: 48 default, 0 interactive, 24 GC (on 2 virtual cores)
Environment:
JULIA_CPU_THREADS = 2
LD_LIBRARY_PATH = /usr/local/nvidia/lib:/usr/local/nvidia/lib64
JULIA_PKG_SERVER =
JULIA_NUM_THREADS = 48
JULIA_CUDA_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0
JULIA_DEBUG = Literate
JULIA_DEPOT_PATH = /root/.cache/julia-buildkite-plugin/depots/01872db4-8c79-43af-ab7d-12abac4f24f6This page was generated using Literate.jl.