Convolutional VAE for MNIST using Reactant
Convolutional variational autoencoder (CVAE) implementation in MLX using MNIST. This is based on the CVAE implementation in MLX.
julia
using Lux,
Reactant,
MLDatasets,
Random,
Statistics,
Enzyme,
MLUtils,
DataAugmentation,
ConcreteStructs,
OneHotArrays,
ImageShow,
Images,
Printf,
Optimisers
const xdev = reactant_device(; force=true)
const cdev = cpu_device()(::MLDataDevices.CPUDevice) (generic function with 1 method)Model Definition
First we will define the encoder.It maps the input to a normal distribution in latent space and sample a latent vector from that distribution.
julia
function cvae_encoder(
rng=Random.default_rng();
num_latent_dims::Int,
image_shape::Dims{3},
max_num_filters::Int,
)
flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
return @compact(;
embed=Chain(
Chain(
Conv((3, 3), image_shape[3] => max_num_filters ÷ 4; stride=2, pad=1),
BatchNorm(max_num_filters ÷ 4, leakyrelu),
),
Chain(
Conv((3, 3), max_num_filters ÷ 4 => max_num_filters ÷ 2; stride=2, pad=1),
BatchNorm(max_num_filters ÷ 2, leakyrelu),
),
Chain(
Conv((3, 3), max_num_filters ÷ 2 => max_num_filters; stride=2, pad=1),
BatchNorm(max_num_filters, leakyrelu),
),
FlattenLayer(),
),
proj_mu=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
proj_log_var=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
rng
) do x
y = embed(x)
μ = proj_mu(y)
logσ² = proj_log_var(y)
T = eltype(logσ²)
logσ² = clamp.(logσ², -T(20.0f0), T(10.0f0))
σ = exp.(logσ² .* T(0.5))
# Generate a tensor of random values from a normal distribution
rng = Lux.replicate(rng)
ϵ = randn_like(rng, σ)
# Reparameterization trick to brackpropagate through sampling
z = ϵ .* σ .+ μ
@return z, μ, logσ²
end
endcvae_encoder (generic function with 2 methods)Similarly we define the decoder.
julia
function cvae_decoder(; num_latent_dims::Int, image_shape::Dims{3}, max_num_filters::Int)
flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
return @compact(;
linear=Dense(num_latent_dims, flattened_dim),
upchain=Chain(
Chain(
Upsample(2),
Conv((3, 3), max_num_filters => max_num_filters ÷ 2; stride=1, pad=1),
BatchNorm(max_num_filters ÷ 2, leakyrelu),
),
Chain(
Upsample(2),
Conv((3, 3), max_num_filters ÷ 2 => max_num_filters ÷ 4; stride=1, pad=1),
BatchNorm(max_num_filters ÷ 4, leakyrelu),
),
Chain(
Upsample(2),
Conv(
(3, 3), max_num_filters ÷ 4 => image_shape[3], sigmoid; stride=1, pad=1
),
),
),
max_num_filters
) do x
y = linear(x)
img = reshape(y, image_shape[1] ÷ 8, image_shape[2] ÷ 8, max_num_filters, :)
@return upchain(img)
end
end
@concrete struct CVAE <: Lux.AbstractLuxContainerLayer{(:encoder, :decoder)}
encoder <: Lux.AbstractLuxLayer
decoder <: Lux.AbstractLuxLayer
end
function CVAE(
rng=Random.default_rng();
num_latent_dims::Int,
image_shape::Dims{3},
max_num_filters::Int,
)
decoder = cvae_decoder(; num_latent_dims, image_shape, max_num_filters)
encoder = cvae_encoder(rng; num_latent_dims, image_shape, max_num_filters)
return CVAE(encoder, decoder)
end
function (cvae::CVAE)(x, ps, st)
(z, μ, logσ²), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
return (x_rec, μ, logσ²), (; encoder=st_enc, decoder=st_dec)
end
function encode(cvae::CVAE, x, ps, st)
(z, _, _), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
return z, (; encoder=st_enc, st.decoder)
end
function decode(cvae::CVAE, z, ps, st)
x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
return x_rec, (; decoder=st_dec, st.encoder)
enddecode (generic function with 1 method)Loading MNIST
julia
@concrete struct TensorDataset
dataset
transform
total_samples::Int
end
Base.length(ds::TensorDataset) = ds.total_samples
function Base.getindex(ds::TensorDataset, idxs::Union{Vector{<:Integer},AbstractRange})
img = Image.(eachslice(convert2image(ds.dataset, idxs); dims=3))
return stack(parent ∘ itemdata ∘ Base.Fix1(apply, ds.transform), img)
end
function loadmnist(batchsize, image_size::Dims{2})
# Load MNIST: Only 1500 for demonstration purposes on CI
train_dataset = MNIST(; split=:train)
N = parse(Bool, get(ENV, "CI", "false")) ? 1500 : length(train_dataset)
train_transform = ScaleKeepAspect(image_size) |> ImageToTensor()
trainset = TensorDataset(train_dataset, train_transform, N)
trainloader = DataLoader(trainset; batchsize, shuffle=true, partial=false)
return trainloader
endloadmnist (generic function with 1 method)Helper Functions
Generate an Image Grid from a list of images
julia
function create_image_grid(imgs::AbstractArray, grid_rows::Int, grid_cols::Int)
total_images = grid_rows * grid_cols
imgs = map(eachslice(imgs[:, :, :, 1:total_images]; dims=4)) do img
cimg = if size(img, 3) == 1
colorview(Gray, view(img, :, :, 1))
else
colorview(RGB, permutedims(img, (3, 1, 2)))
end
return cimg'
end
return create_image_grid(imgs, grid_rows, grid_cols)
end
function create_image_grid(images::Vector, grid_rows::Int, grid_cols::Int)
# Check if the number of images matches the grid
total_images = grid_rows * grid_cols
@assert length(images) == total_images
# Get the size of a single image (assuming all images are the same size)
img_height, img_width = size(images[1])
# Create a blank grid canvas
grid_height = img_height * grid_rows
grid_width = img_width * grid_cols
grid_canvas = similar(images[1], grid_height, grid_width)
# Place each image in the correct position on the canvas
for idx in 1:total_images
row = div(idx - 1, grid_cols) + 1
col = mod(idx - 1, grid_cols) + 1
start_row = (row - 1) * img_height + 1
start_col = (col - 1) * img_width + 1
grid_canvas[start_row:(start_row + img_height - 1), start_col:(start_col + img_width - 1)] .= images[idx]
end
return grid_canvas
end
function loss_function(model, ps, st, X)
(y, μ, logσ²), st = model(X, ps, st)
reconstruction_loss = MSELoss(; agg=sum)(y, X)
kldiv_loss = -sum(1 .+ logσ² .- μ .^ 2 .- exp.(logσ²)) / 2
loss = reconstruction_loss + kldiv_loss
return loss, st, (; y, μ, logσ², reconstruction_loss, kldiv_loss)
end
function generate_images(
model, ps, st; num_samples::Int=128, num_latent_dims::Int, decode_compiled=nothing
)
z = get_device((ps, st))(randn(Float32, num_latent_dims, num_samples))
if decode_compiled === nothing
images, _ = decode(model, z, ps, Lux.testmode(st))
else
images, _ = decode_compiled(model, z, ps, Lux.testmode(st))
images = cpu_device()(images)
end
return create_image_grid(images, 8, num_samples ÷ 8)
end
function reconstruct_images(model, ps, st, X)
(recon, _, _), _ = model(X, ps, Lux.testmode(st))
recon = cpu_device()(recon)
return create_image_grid(recon, 8, size(X, ndims(X)) ÷ 8)
endreconstruct_images (generic function with 1 method)Training the Model
julia
function main(;
batchsize=128,
image_size=(64, 64),
num_latent_dims=8,
max_num_filters=64,
seed=0,
epochs=50,
weight_decay=1.0e-5,
learning_rate=1.0e-3,
num_samples=batchsize,
)
rng = Xoshiro()
Random.seed!(rng, seed)
cvae = CVAE(rng; num_latent_dims, image_shape=(image_size..., 1), max_num_filters)
ps, st = xdev(Lux.setup(rng, cvae))
z = xdev(randn(Float32, num_latent_dims, num_samples))
decode_compiled = @compile decode(cvae, z, ps, Lux.testmode(st))
x = xdev(randn(Float32, image_size..., 1, batchsize))
cvae_compiled = @compile cvae(x, ps, Lux.testmode(st))
train_dataloader = xdev(loadmnist(batchsize, image_size))
opt = AdamW(; eta=learning_rate, lambda=weight_decay)
train_state = Training.TrainState(cvae, ps, st, opt)
@printf "Total Trainable Parameters: %0.4f M\n" (Lux.parameterlength(ps) / 1.0e6)
is_vscode = isdefined(Main, :VSCodeServer)
empty_row, model_img_full = nothing, nothing
for epoch in 1:epochs
loss_total = 0.0f0
total_samples = 0
start_time = time()
for (i, X) in enumerate(train_dataloader)
(_, loss, _, train_state) = Training.single_train_step!(
AutoEnzyme(), loss_function, X, train_state; return_gradients=Val(false)
)
loss_total += loss
total_samples += size(X, ndims(X))
if i % 250 == 0 || i == length(train_dataloader)
throughput = total_samples / (time() - start_time)
@printf "Epoch %d, Iter %d, Loss: %.7f, Throughput: %.6f im/s\n" epoch i loss throughput
end
end
total_time = time() - start_time
train_loss = loss_total / length(train_dataloader)
throughput = total_samples / total_time
@printf "Epoch %d, Train Loss: %.7f, Time: %.4fs, Throughput: %.6f im/s\n" epoch train_loss total_time throughput
if is_vscode || epoch == epochs
recon_images = reconstruct_images(
cvae_compiled,
train_state.parameters,
train_state.states,
first(train_dataloader),
)
gen_images = generate_images(
cvae,
train_state.parameters,
train_state.states;
num_samples,
num_latent_dims,
decode_compiled,
)
if empty_row === nothing
empty_row = similar(gen_images, image_size[1], size(gen_images, 2))
fill!(empty_row, 0)
end
model_img_full = vcat(recon_images, empty_row, gen_images)
is_vscode && display(model_img_full)
end
end
return model_img_full
end
img = main()2025-04-16 04:03:02.785737: I external/xla/xla/service/service.cc:152] XLA service 0x147c47c0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2025-04-16 04:03:02.785775: I external/xla/xla/service/service.cc:160] StreamExecutor device (0): NVIDIA A100-PCIE-40GB MIG 1g.5gb, Compute Capability 8.0
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1744776182.786614 151949 se_gpu_pjrt_client.cc:1040] Using BFC allocator.
I0000 00:00:1744776182.786709 151949 gpu_helpers.cc:136] XLA backend allocating 3825205248 bytes on device 0 for BFCAllocator.
I0000 00:00:1744776182.786767 151949 gpu_helpers.cc:177] XLA backend will use up to 1275068416 bytes on device 0 for CollectiveBFCAllocator.
I0000 00:00:1744776182.800152 151949 cuda_dnn.cc:529] Loaded cuDNN version 90400
┌ Warning: `training` is set to `Val{false}()` but is being used within an autodiff call (gradient, jacobian, etc...). This might lead to incorrect results. If you are using a `Lux.jl` model, set it to training mode using `LuxCore.trainmode`.
└ @ LuxLib.Utils /var/lib/buildkite-agent/builds/gpuci-1/julialang/lux-dot-jl/lib/LuxLib/src/utils.jl:344
E0000 00:00:1744776249.218734 151949 buffer_comparator.cc:156] Difference at 16: 0, expected 1.69914
E0000 00:00:1744776249.218785 151949 buffer_comparator.cc:156] Difference at 17: 0, expected 1.20048
E0000 00:00:1744776249.218792 151949 buffer_comparator.cc:156] Difference at 18: 0, expected 1.89606
E0000 00:00:1744776249.218799 151949 buffer_comparator.cc:156] Difference at 19: 0, expected 1.32316
E0000 00:00:1744776249.218805 151949 buffer_comparator.cc:156] Difference at 20: 0, expected 1.85572
E0000 00:00:1744776249.218811 151949 buffer_comparator.cc:156] Difference at 21: 0, expected 0.944347
E0000 00:00:1744776249.218818 151949 buffer_comparator.cc:156] Difference at 22: 0, expected 1.86911
E0000 00:00:1744776249.218824 151949 buffer_comparator.cc:156] Difference at 23: 0, expected 0.58457
E0000 00:00:1744776249.218831 151949 buffer_comparator.cc:156] Difference at 24: 0, expected 2.7184
E0000 00:00:1744776249.218837 151949 buffer_comparator.cc:156] Difference at 25: 0, expected 2.16503
2025-04-16 04:04:09.218852: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.222350 151949 buffer_comparator.cc:156] Difference at 16: 0, expected 1.69914
E0000 00:00:1744776249.222367 151949 buffer_comparator.cc:156] Difference at 17: 0, expected 1.20048
E0000 00:00:1744776249.222372 151949 buffer_comparator.cc:156] Difference at 18: 0, expected 1.89606
E0000 00:00:1744776249.222375 151949 buffer_comparator.cc:156] Difference at 19: 0, expected 1.32316
E0000 00:00:1744776249.222379 151949 buffer_comparator.cc:156] Difference at 20: 0, expected 1.85572
E0000 00:00:1744776249.222383 151949 buffer_comparator.cc:156] Difference at 21: 0, expected 0.944347
E0000 00:00:1744776249.222388 151949 buffer_comparator.cc:156] Difference at 22: 0, expected 1.86911
E0000 00:00:1744776249.222391 151949 buffer_comparator.cc:156] Difference at 23: 0, expected 0.58457
E0000 00:00:1744776249.222395 151949 buffer_comparator.cc:156] Difference at 24: 0, expected 2.7184
E0000 00:00:1744776249.222399 151949 buffer_comparator.cc:156] Difference at 25: 0, expected 2.16503
2025-04-16 04:04:09.222406: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.225441 151949 buffer_comparator.cc:156] Difference at 64: 0, expected 1.60438
E0000 00:00:1744776249.225456 151949 buffer_comparator.cc:156] Difference at 65: 0, expected 1.28593
E0000 00:00:1744776249.225460 151949 buffer_comparator.cc:156] Difference at 66: 0, expected 2.03265
E0000 00:00:1744776249.225464 151949 buffer_comparator.cc:156] Difference at 67: 0, expected 2.28838
E0000 00:00:1744776249.225468 151949 buffer_comparator.cc:156] Difference at 68: 0, expected 1.76697
E0000 00:00:1744776249.225472 151949 buffer_comparator.cc:156] Difference at 69: 0, expected 1.33634
E0000 00:00:1744776249.225476 151949 buffer_comparator.cc:156] Difference at 70: 0, expected 1.96458
E0000 00:00:1744776249.225480 151949 buffer_comparator.cc:156] Difference at 71: 0, expected 1.16919
E0000 00:00:1744776249.225484 151949 buffer_comparator.cc:156] Difference at 72: 0, expected 0.893586
E0000 00:00:1744776249.225488 151949 buffer_comparator.cc:156] Difference at 73: 0, expected 2.3101
2025-04-16 04:04:09.225494: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.228597 151949 buffer_comparator.cc:156] Difference at 2048: 0, expected 1.77658
E0000 00:00:1744776249.228613 151949 buffer_comparator.cc:156] Difference at 2049: 0, expected 1.05138
E0000 00:00:1744776249.228618 151949 buffer_comparator.cc:156] Difference at 2050: 0, expected 1.81989
E0000 00:00:1744776249.228622 151949 buffer_comparator.cc:156] Difference at 2051: 0, expected 2.39565
E0000 00:00:1744776249.228626 151949 buffer_comparator.cc:156] Difference at 2052: 0, expected 1.54422
E0000 00:00:1744776249.228631 151949 buffer_comparator.cc:156] Difference at 2053: 0, expected 2.20321
E0000 00:00:1744776249.228635 151949 buffer_comparator.cc:156] Difference at 2054: 0, expected 1.94268
E0000 00:00:1744776249.228639 151949 buffer_comparator.cc:156] Difference at 2055: 0, expected 2.41167
E0000 00:00:1744776249.228643 151949 buffer_comparator.cc:156] Difference at 2056: 0, expected 2.09568
E0000 00:00:1744776249.228647 151949 buffer_comparator.cc:156] Difference at 2057: 0, expected 1.1478
2025-04-16 04:04:09.228653: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.231683 151949 buffer_comparator.cc:156] Difference at 2064: 0, expected 1.87707
E0000 00:00:1744776249.231695 151949 buffer_comparator.cc:156] Difference at 2065: 0, expected 0.928597
E0000 00:00:1744776249.231698 151949 buffer_comparator.cc:156] Difference at 2066: 0, expected 2.285
E0000 00:00:1744776249.231701 151949 buffer_comparator.cc:156] Difference at 2067: 0, expected 1.20842
E0000 00:00:1744776249.231704 151949 buffer_comparator.cc:156] Difference at 2068: 0, expected 1.71371
E0000 00:00:1744776249.231707 151949 buffer_comparator.cc:156] Difference at 2069: 0, expected 1.03264
E0000 00:00:1744776249.231709 151949 buffer_comparator.cc:156] Difference at 2070: 0, expected 1.51429
E0000 00:00:1744776249.231712 151949 buffer_comparator.cc:156] Difference at 2071: 0, expected 0.671343
E0000 00:00:1744776249.231715 151949 buffer_comparator.cc:156] Difference at 2072: 0, expected 2.95498
E0000 00:00:1744776249.231718 151949 buffer_comparator.cc:156] Difference at 2073: 0, expected 2.32851
2025-04-16 04:04:09.231722: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.234599 151949 buffer_comparator.cc:156] Difference at 2080: 0, expected 1.44171
E0000 00:00:1744776249.234611 151949 buffer_comparator.cc:156] Difference at 2081: 0, expected 2.15664
E0000 00:00:1744776249.234614 151949 buffer_comparator.cc:156] Difference at 2082: 0, expected 2.50291
E0000 00:00:1744776249.234617 151949 buffer_comparator.cc:156] Difference at 2083: 0, expected 1.60771
E0000 00:00:1744776249.234620 151949 buffer_comparator.cc:156] Difference at 2084: 0, expected 2.0311
E0000 00:00:1744776249.234623 151949 buffer_comparator.cc:156] Difference at 2085: 0, expected 1.65865
E0000 00:00:1744776249.234626 151949 buffer_comparator.cc:156] Difference at 2086: 0, expected 1.72254
E0000 00:00:1744776249.234628 151949 buffer_comparator.cc:156] Difference at 2087: 0, expected 2.04439
E0000 00:00:1744776249.234631 151949 buffer_comparator.cc:156] Difference at 2088: 0, expected 1.54755
E0000 00:00:1744776249.234634 151949 buffer_comparator.cc:156] Difference at 2089: 0, expected 0.955993
2025-04-16 04:04:09.234639: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.237515 151949 buffer_comparator.cc:156] Difference at 2112: 0, expected 1.48116
E0000 00:00:1744776249.237526 151949 buffer_comparator.cc:156] Difference at 2113: 0, expected 1.4478
E0000 00:00:1744776249.237529 151949 buffer_comparator.cc:156] Difference at 2114: 0, expected 2.13734
E0000 00:00:1744776249.237532 151949 buffer_comparator.cc:156] Difference at 2115: 0, expected 2.19489
E0000 00:00:1744776249.237535 151949 buffer_comparator.cc:156] Difference at 2116: 0, expected 1.82644
E0000 00:00:1744776249.237538 151949 buffer_comparator.cc:156] Difference at 2117: 0, expected 1.36935
E0000 00:00:1744776249.237540 151949 buffer_comparator.cc:156] Difference at 2118: 0, expected 1.67303
E0000 00:00:1744776249.237543 151949 buffer_comparator.cc:156] Difference at 2119: 0, expected 1.26478
E0000 00:00:1744776249.237548 151949 buffer_comparator.cc:156] Difference at 2120: 0, expected 1.00439
E0000 00:00:1744776249.237551 151949 buffer_comparator.cc:156] Difference at 2121: 0, expected 2.64095
2025-04-16 04:04:09.237555: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.240451 151949 buffer_comparator.cc:156] Difference at 4096: 0, expected 2.5269
E0000 00:00:1744776249.240463 151949 buffer_comparator.cc:156] Difference at 4097: 0, expected 1.96414
E0000 00:00:1744776249.240466 151949 buffer_comparator.cc:156] Difference at 4098: 0, expected 3.84871
E0000 00:00:1744776249.240469 151949 buffer_comparator.cc:156] Difference at 4099: 0, expected 4.06422
E0000 00:00:1744776249.240472 151949 buffer_comparator.cc:156] Difference at 4100: 0, expected 1.84843
E0000 00:00:1744776249.240474 151949 buffer_comparator.cc:156] Difference at 4101: 0, expected 3.06637
E0000 00:00:1744776249.240477 151949 buffer_comparator.cc:156] Difference at 4102: 0, expected 3.26933
E0000 00:00:1744776249.240480 151949 buffer_comparator.cc:156] Difference at 4103: 0, expected 2.61708
E0000 00:00:1744776249.240482 151949 buffer_comparator.cc:156] Difference at 4104: 0, expected 3.42482
E0000 00:00:1744776249.240485 151949 buffer_comparator.cc:156] Difference at 4105: 0, expected 2.29469
2025-04-16 04:04:09.240490: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.243383 151949 buffer_comparator.cc:156] Difference at 4128: 0, expected 2.63461
E0000 00:00:1744776249.243395 151949 buffer_comparator.cc:156] Difference at 4129: 0, expected 2.9239
E0000 00:00:1744776249.243398 151949 buffer_comparator.cc:156] Difference at 4130: 0, expected 2.40995
E0000 00:00:1744776249.243400 151949 buffer_comparator.cc:156] Difference at 4131: 0, expected 2.61607
E0000 00:00:1744776249.243403 151949 buffer_comparator.cc:156] Difference at 4132: 0, expected 3.62419
E0000 00:00:1744776249.243406 151949 buffer_comparator.cc:156] Difference at 4133: 0, expected 3.10893
E0000 00:00:1744776249.243409 151949 buffer_comparator.cc:156] Difference at 4134: 0, expected 2.56351
E0000 00:00:1744776249.243411 151949 buffer_comparator.cc:156] Difference at 4135: 0, expected 2.19994
E0000 00:00:1744776249.243414 151949 buffer_comparator.cc:156] Difference at 4136: 0, expected 2.37682
E0000 00:00:1744776249.243417 151949 buffer_comparator.cc:156] Difference at 4137: 0, expected 2.50883
2025-04-16 04:04:09.243421: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.246319 151949 buffer_comparator.cc:156] Difference at 4128: 0, expected 2.63461
E0000 00:00:1744776249.246331 151949 buffer_comparator.cc:156] Difference at 4129: 0, expected 2.9239
E0000 00:00:1744776249.246334 151949 buffer_comparator.cc:156] Difference at 4130: 0, expected 2.40995
E0000 00:00:1744776249.246336 151949 buffer_comparator.cc:156] Difference at 4131: 0, expected 2.61607
E0000 00:00:1744776249.246339 151949 buffer_comparator.cc:156] Difference at 4132: 0, expected 3.62419
E0000 00:00:1744776249.246342 151949 buffer_comparator.cc:156] Difference at 4133: 0, expected 3.10893
E0000 00:00:1744776249.246345 151949 buffer_comparator.cc:156] Difference at 4134: 0, expected 2.56351
E0000 00:00:1744776249.246347 151949 buffer_comparator.cc:156] Difference at 4135: 0, expected 2.19994
E0000 00:00:1744776249.246350 151949 buffer_comparator.cc:156] Difference at 4136: 0, expected 2.37682
E0000 00:00:1744776249.246353 151949 buffer_comparator.cc:156] Difference at 4137: 0, expected 2.50883
2025-04-16 04:04:09.246357: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.249235 151949 buffer_comparator.cc:156] Difference at 4128: 0, expected 2.63461
E0000 00:00:1744776249.249247 151949 buffer_comparator.cc:156] Difference at 4129: 0, expected 2.9239
E0000 00:00:1744776249.249250 151949 buffer_comparator.cc:156] Difference at 4130: 0, expected 2.40995
E0000 00:00:1744776249.249252 151949 buffer_comparator.cc:156] Difference at 4131: 0, expected 2.61607
E0000 00:00:1744776249.249255 151949 buffer_comparator.cc:156] Difference at 4132: 0, expected 3.62419
E0000 00:00:1744776249.249258 151949 buffer_comparator.cc:156] Difference at 4133: 0, expected 3.10893
E0000 00:00:1744776249.249261 151949 buffer_comparator.cc:156] Difference at 4134: 0, expected 2.56351
E0000 00:00:1744776249.249263 151949 buffer_comparator.cc:156] Difference at 4135: 0, expected 2.19994
E0000 00:00:1744776249.249266 151949 buffer_comparator.cc:156] Difference at 4136: 0, expected 2.37682
E0000 00:00:1744776249.249269 151949 buffer_comparator.cc:156] Difference at 4137: 0, expected 2.50883
2025-04-16 04:04:09.249273: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.252136 151949 buffer_comparator.cc:156] Difference at 4160: 0, expected 2.86435
E0000 00:00:1744776249.252148 151949 buffer_comparator.cc:156] Difference at 4161: 0, expected 2.0664
E0000 00:00:1744776249.252151 151949 buffer_comparator.cc:156] Difference at 4162: 0, expected 2.61642
E0000 00:00:1744776249.252154 151949 buffer_comparator.cc:156] Difference at 4163: 0, expected 3.35871
E0000 00:00:1744776249.252157 151949 buffer_comparator.cc:156] Difference at 4164: 0, expected 3.1527
E0000 00:00:1744776249.252160 151949 buffer_comparator.cc:156] Difference at 4165: 0, expected 1.96457
E0000 00:00:1744776249.252162 151949 buffer_comparator.cc:156] Difference at 4166: 0, expected 2.94612
E0000 00:00:1744776249.252165 151949 buffer_comparator.cc:156] Difference at 4167: 0, expected 1.54296
E0000 00:00:1744776249.252168 151949 buffer_comparator.cc:156] Difference at 4168: 0, expected 1.32996
E0000 00:00:1744776249.252171 151949 buffer_comparator.cc:156] Difference at 4169: 0, expected 3.04802
2025-04-16 04:04:09.252175: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.255071 151949 buffer_comparator.cc:156] Difference at 8192: 0, expected 1.98786
E0000 00:00:1744776249.255083 151949 buffer_comparator.cc:156] Difference at 8193: 0, expected 1.72939
E0000 00:00:1744776249.255086 151949 buffer_comparator.cc:156] Difference at 8194: 0, expected 3.03127
E0000 00:00:1744776249.255088 151949 buffer_comparator.cc:156] Difference at 8195: 0, expected 3.3803
E0000 00:00:1744776249.255091 151949 buffer_comparator.cc:156] Difference at 8196: 0, expected 1.7692
E0000 00:00:1744776249.255094 151949 buffer_comparator.cc:156] Difference at 8197: 0, expected 2.87459
E0000 00:00:1744776249.255097 151949 buffer_comparator.cc:156] Difference at 8198: 0, expected 2.74368
E0000 00:00:1744776249.255100 151949 buffer_comparator.cc:156] Difference at 8199: 0, expected 2.30403
E0000 00:00:1744776249.255102 151949 buffer_comparator.cc:156] Difference at 8200: 0, expected 3.07071
E0000 00:00:1744776249.255105 151949 buffer_comparator.cc:156] Difference at 8201: 0, expected 1.9239
2025-04-16 04:04:09.255110: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.258006 151949 buffer_comparator.cc:156] Difference at 8224: 0, expected 2.18642
E0000 00:00:1744776249.258018 151949 buffer_comparator.cc:156] Difference at 8225: 0, expected 2.28379
E0000 00:00:1744776249.258022 151949 buffer_comparator.cc:156] Difference at 8226: 0, expected 2.33258
E0000 00:00:1744776249.258027 151949 buffer_comparator.cc:156] Difference at 8227: 0, expected 2.12912
E0000 00:00:1744776249.258030 151949 buffer_comparator.cc:156] Difference at 8228: 0, expected 2.99415
E0000 00:00:1744776249.258032 151949 buffer_comparator.cc:156] Difference at 8229: 0, expected 2.27319
E0000 00:00:1744776249.258035 151949 buffer_comparator.cc:156] Difference at 8230: 0, expected 2.165
E0000 00:00:1744776249.258038 151949 buffer_comparator.cc:156] Difference at 8231: 0, expected 2.33242
E0000 00:00:1744776249.258041 151949 buffer_comparator.cc:156] Difference at 8232: 0, expected 1.81753
E0000 00:00:1744776249.258043 151949 buffer_comparator.cc:156] Difference at 8233: 0, expected 1.754
2025-04-16 04:04:09.258053: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.260950 151949 buffer_comparator.cc:156] Difference at 8256: 0, expected 2.10464
E0000 00:00:1744776249.260962 151949 buffer_comparator.cc:156] Difference at 8257: 0, expected 2.00542
E0000 00:00:1744776249.260965 151949 buffer_comparator.cc:156] Difference at 8258: 0, expected 2.33829
E0000 00:00:1744776249.260967 151949 buffer_comparator.cc:156] Difference at 8259: 0, expected 2.68564
E0000 00:00:1744776249.260970 151949 buffer_comparator.cc:156] Difference at 8260: 0, expected 2.75266
E0000 00:00:1744776249.260973 151949 buffer_comparator.cc:156] Difference at 8261: 0, expected 2.02724
E0000 00:00:1744776249.260976 151949 buffer_comparator.cc:156] Difference at 8262: 0, expected 1.93574
E0000 00:00:1744776249.260978 151949 buffer_comparator.cc:156] Difference at 8263: 0, expected 1.335
E0000 00:00:1744776249.260981 151949 buffer_comparator.cc:156] Difference at 8264: 0, expected 1.15464
E0000 00:00:1744776249.260984 151949 buffer_comparator.cc:156] Difference at 8265: 0, expected 3.13129
2025-04-16 04:04:09.260989: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.263908 151949 buffer_comparator.cc:156] Difference at 16384: 0, expected 2.13072
E0000 00:00:1744776249.263920 151949 buffer_comparator.cc:156] Difference at 16385: 0, expected 1.18098
E0000 00:00:1744776249.263923 151949 buffer_comparator.cc:156] Difference at 16386: 0, expected 2.76235
E0000 00:00:1744776249.263926 151949 buffer_comparator.cc:156] Difference at 16387: 0, expected 2.87006
E0000 00:00:1744776249.263929 151949 buffer_comparator.cc:156] Difference at 16388: 0, expected 1.48471
E0000 00:00:1744776249.263931 151949 buffer_comparator.cc:156] Difference at 16389: 0, expected 2.14624
E0000 00:00:1744776249.263934 151949 buffer_comparator.cc:156] Difference at 16390: 0, expected 2.04824
E0000 00:00:1744776249.263937 151949 buffer_comparator.cc:156] Difference at 16391: 0, expected 2.47653
E0000 00:00:1744776249.263940 151949 buffer_comparator.cc:156] Difference at 16392: 0, expected 2.26896
E0000 00:00:1744776249.263942 151949 buffer_comparator.cc:156] Difference at 16393: 0, expected 1.36576
2025-04-16 04:04:09.263947: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.266866 151949 buffer_comparator.cc:156] Difference at 16384: 0, expected 2.13072
E0000 00:00:1744776249.266878 151949 buffer_comparator.cc:156] Difference at 16385: 0, expected 1.18098
E0000 00:00:1744776249.266881 151949 buffer_comparator.cc:156] Difference at 16386: 0, expected 2.76235
E0000 00:00:1744776249.266884 151949 buffer_comparator.cc:156] Difference at 16387: 0, expected 2.87006
E0000 00:00:1744776249.266886 151949 buffer_comparator.cc:156] Difference at 16388: 0, expected 1.48471
E0000 00:00:1744776249.266889 151949 buffer_comparator.cc:156] Difference at 16389: 0, expected 2.14624
E0000 00:00:1744776249.266893 151949 buffer_comparator.cc:156] Difference at 16390: 0, expected 2.04824
E0000 00:00:1744776249.266896 151949 buffer_comparator.cc:156] Difference at 16391: 0, expected 2.47653
E0000 00:00:1744776249.266899 151949 buffer_comparator.cc:156] Difference at 16392: 0, expected 2.26896
E0000 00:00:1744776249.266902 151949 buffer_comparator.cc:156] Difference at 16393: 0, expected 1.36576
2025-04-16 04:04:09.266906: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.269823 151949 buffer_comparator.cc:156] Difference at 16448: 0, expected 2.04485
E0000 00:00:1744776249.269834 151949 buffer_comparator.cc:156] Difference at 16449: 0, expected 1.38058
E0000 00:00:1744776249.269837 151949 buffer_comparator.cc:156] Difference at 16450: 0, expected 1.77153
E0000 00:00:1744776249.269840 151949 buffer_comparator.cc:156] Difference at 16451: 0, expected 2.60773
E0000 00:00:1744776249.269843 151949 buffer_comparator.cc:156] Difference at 16452: 0, expected 2.24942
E0000 00:00:1744776249.269846 151949 buffer_comparator.cc:156] Difference at 16453: 0, expected 1.32792
E0000 00:00:1744776249.269848 151949 buffer_comparator.cc:156] Difference at 16454: 0, expected 2.7744
E0000 00:00:1744776249.269851 151949 buffer_comparator.cc:156] Difference at 16455: 0, expected 1.26898
E0000 00:00:1744776249.269854 151949 buffer_comparator.cc:156] Difference at 16456: 0, expected 1.06744
E0000 00:00:1744776249.269857 151949 buffer_comparator.cc:156] Difference at 16457: 0, expected 1.99063
2025-04-16 04:04:09.269861: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776249.272845 151949 buffer_comparator.cc:156] Difference at 32768: 0, expected 2.02216
E0000 00:00:1744776249.272857 151949 buffer_comparator.cc:156] Difference at 32769: 0, expected 1.27847
E0000 00:00:1744776249.272860 151949 buffer_comparator.cc:156] Difference at 32770: 0, expected 2.25127
E0000 00:00:1744776249.272863 151949 buffer_comparator.cc:156] Difference at 32771: 0, expected 2.96132
E0000 00:00:1744776249.272866 151949 buffer_comparator.cc:156] Difference at 32772: 0, expected 1.65598
E0000 00:00:1744776249.272868 151949 buffer_comparator.cc:156] Difference at 32773: 0, expected 2.09882
E0000 00:00:1744776249.272871 151949 buffer_comparator.cc:156] Difference at 32774: 0, expected 2.35327
E0000 00:00:1744776249.272874 151949 buffer_comparator.cc:156] Difference at 32775: 0, expected 2.14737
E0000 00:00:1744776249.272876 151949 buffer_comparator.cc:156] Difference at 32776: 0, expected 2.35827
E0000 00:00:1744776249.272879 151949 buffer_comparator.cc:156] Difference at 32777: 0, expected 1.49831
2025-04-16 04:04:09.272884: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
2025-04-16 04:04:25.481226: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 236 bytes spill stores, 236 bytes spill loads
2025-04-16 04:04:26.510153: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 192 bytes spill stores, 192 bytes spill loads
2025-04-16 04:04:26.575510: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 180 bytes spill stores, 180 bytes spill loads
2025-04-16 04:04:26.657282: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 4620 bytes spill stores, 4616 bytes spill loads
2025-04-16 04:04:27.489993: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 3940 bytes spill stores, 3928 bytes spill loads
E0000 00:00:1744776267.524518 151949 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1744776267.524568 151949 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1744776267.524573 151949 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1744776267.524577 151949 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1744776267.524582 151949 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1744776267.524587 151949 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1744776267.524591 151949 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1744776267.524595 151949 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1744776267.524600 151949 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1744776267.524604 151949 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-04-16 04:04:27.524616: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.526915 151949 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1744776267.526932 151949 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1744776267.526936 151949 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1744776267.526941 151949 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1744776267.526945 151949 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1744776267.526949 151949 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1744776267.526954 151949 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1744776267.526958 151949 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1744776267.526962 151949 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1744776267.526967 151949 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-04-16 04:04:27.526973: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.529292 151949 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1744776267.529308 151949 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1744776267.529313 151949 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1744776267.529317 151949 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1744776267.529322 151949 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1744776267.529326 151949 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1744776267.529331 151949 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1744776267.529335 151949 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1744776267.529339 151949 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1744776267.529343 151949 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-04-16 04:04:27.529352: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.531667 151949 buffer_comparator.cc:156] Difference at 512: 711.971, expected 1037.38
E0000 00:00:1744776267.531679 151949 buffer_comparator.cc:156] Difference at 513: 712.779, expected 1037.38
E0000 00:00:1744776267.531682 151949 buffer_comparator.cc:156] Difference at 514: 725.127, expected 1037.27
E0000 00:00:1744776267.531686 151949 buffer_comparator.cc:156] Difference at 515: 698.019, expected 1030.23
E0000 00:00:1744776267.531689 151949 buffer_comparator.cc:156] Difference at 516: 707.664, expected 1029.96
E0000 00:00:1744776267.531692 151949 buffer_comparator.cc:156] Difference at 517: 711.597, expected 1023.46
E0000 00:00:1744776267.531695 151949 buffer_comparator.cc:156] Difference at 518: 720.495, expected 1036.27
E0000 00:00:1744776267.531698 151949 buffer_comparator.cc:156] Difference at 519: 719.076, expected 1042.91
E0000 00:00:1744776267.531701 151949 buffer_comparator.cc:156] Difference at 520: 711.292, expected 1031.19
E0000 00:00:1744776267.531704 151949 buffer_comparator.cc:156] Difference at 521: 693.751, expected 1022.88
2025-04-16 04:04:27.531709: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.533873 151949 buffer_comparator.cc:156] Difference at 512: 351.538, expected 1037.38
E0000 00:00:1744776267.533884 151949 buffer_comparator.cc:156] Difference at 513: 348.221, expected 1037.38
E0000 00:00:1744776267.533887 151949 buffer_comparator.cc:156] Difference at 514: 361.824, expected 1037.27
E0000 00:00:1744776267.533891 151949 buffer_comparator.cc:156] Difference at 515: 352.926, expected 1030.23
E0000 00:00:1744776267.533894 151949 buffer_comparator.cc:156] Difference at 516: 344.916, expected 1029.96
E0000 00:00:1744776267.533897 151949 buffer_comparator.cc:156] Difference at 517: 356.516, expected 1023.46
E0000 00:00:1744776267.533900 151949 buffer_comparator.cc:156] Difference at 518: 354.126, expected 1036.27
E0000 00:00:1744776267.533903 151949 buffer_comparator.cc:156] Difference at 519: 357.665, expected 1042.91
E0000 00:00:1744776267.533906 151949 buffer_comparator.cc:156] Difference at 520: 359.549, expected 1031.19
E0000 00:00:1744776267.533909 151949 buffer_comparator.cc:156] Difference at 521: 342.212, expected 1022.88
2025-04-16 04:04:27.533914: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.536129 151949 buffer_comparator.cc:156] Difference at 1024: 344.602, expected 1028.47
E0000 00:00:1744776267.536141 151949 buffer_comparator.cc:156] Difference at 1025: 338.216, expected 1025.84
E0000 00:00:1744776267.536144 151949 buffer_comparator.cc:156] Difference at 1026: 352.085, expected 1041.27
E0000 00:00:1744776267.536147 151949 buffer_comparator.cc:156] Difference at 1027: 347.813, expected 1022.34
E0000 00:00:1744776267.536150 151949 buffer_comparator.cc:156] Difference at 1028: 342.989, expected 1025.87
E0000 00:00:1744776267.536153 151949 buffer_comparator.cc:156] Difference at 1029: 351.344, expected 1021.14
E0000 00:00:1744776267.536156 151949 buffer_comparator.cc:156] Difference at 1030: 340.434, expected 1030.37
E0000 00:00:1744776267.536160 151949 buffer_comparator.cc:156] Difference at 1031: 355.013, expected 1039.95
E0000 00:00:1744776267.536163 151949 buffer_comparator.cc:156] Difference at 1032: 350.737, expected 1032.45
E0000 00:00:1744776267.536166 151949 buffer_comparator.cc:156] Difference at 1033: 340.266, expected 1019.38
2025-04-16 04:04:27.536171: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.538324 151949 buffer_comparator.cc:156] Difference at 38: 925.654, expected 1048.8
E0000 00:00:1744776267.538336 151949 buffer_comparator.cc:156] Difference at 44: 914.794, expected 1041.72
E0000 00:00:1744776267.538340 151949 buffer_comparator.cc:156] Difference at 64: 1158.54, expected 1032.49
E0000 00:00:1744776267.538343 151949 buffer_comparator.cc:156] Difference at 68: 925.055, expected 1037.91
E0000 00:00:1744776267.538347 151949 buffer_comparator.cc:156] Difference at 96: 1212.46, expected 1026.76
E0000 00:00:1744776267.538350 151949 buffer_comparator.cc:156] Difference at 109: 1176.82, expected 1029.18
E0000 00:00:1744776267.538353 151949 buffer_comparator.cc:156] Difference at 162: 923.662, expected 1032.2
E0000 00:00:1744776267.538357 151949 buffer_comparator.cc:156] Difference at 166: 919.618, expected 1027.87
E0000 00:00:1744776267.538360 151949 buffer_comparator.cc:156] Difference at 172: 920.648, expected 1025.22
E0000 00:00:1744776267.538363 151949 buffer_comparator.cc:156] Difference at 208: 1155.74, expected 1016.78
2025-04-16 04:04:27.538368: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.540661 151949 buffer_comparator.cc:156] Difference at 1056: 937.724, expected 1049.14
E0000 00:00:1744776267.540673 151949 buffer_comparator.cc:156] Difference at 1057: 939.844, expected 1047.51
E0000 00:00:1744776267.540676 151949 buffer_comparator.cc:156] Difference at 1061: 932.553, expected 1037.02
E0000 00:00:1744776267.540680 151949 buffer_comparator.cc:156] Difference at 1062: 945.623, expected 1051.04
E0000 00:00:1744776267.540683 151949 buffer_comparator.cc:156] Difference at 1067: 936.361, expected 1046.17
E0000 00:00:1744776267.540686 151949 buffer_comparator.cc:156] Difference at 1071: 926.359, expected 1048.51
E0000 00:00:1744776267.540689 151949 buffer_comparator.cc:156] Difference at 1104: 922.036, expected 1028.34
E0000 00:00:1744776267.540692 151949 buffer_comparator.cc:156] Difference at 1109: 920.805, expected 1023.27
E0000 00:00:1744776267.540695 151949 buffer_comparator.cc:156] Difference at 1119: 926.194, expected 1035.14
E0000 00:00:1744776267.540698 151949 buffer_comparator.cc:156] Difference at 1137: 940.954, expected 1046.87
2025-04-16 04:04:27.540703: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.543012 151949 buffer_comparator.cc:156] Difference at 1056: 937.724, expected 1049.14
E0000 00:00:1744776267.543024 151949 buffer_comparator.cc:156] Difference at 1057: 939.844, expected 1047.51
E0000 00:00:1744776267.543027 151949 buffer_comparator.cc:156] Difference at 1061: 932.553, expected 1037.02
E0000 00:00:1744776267.543031 151949 buffer_comparator.cc:156] Difference at 1062: 945.623, expected 1051.04
E0000 00:00:1744776267.543034 151949 buffer_comparator.cc:156] Difference at 1067: 936.361, expected 1046.17
E0000 00:00:1744776267.543037 151949 buffer_comparator.cc:156] Difference at 1071: 926.359, expected 1048.51
E0000 00:00:1744776267.543040 151949 buffer_comparator.cc:156] Difference at 1104: 922.036, expected 1028.34
E0000 00:00:1744776267.543043 151949 buffer_comparator.cc:156] Difference at 1109: 920.805, expected 1023.27
E0000 00:00:1744776267.543046 151949 buffer_comparator.cc:156] Difference at 1119: 926.194, expected 1035.14
E0000 00:00:1744776267.543049 151949 buffer_comparator.cc:156] Difference at 1137: 940.954, expected 1046.87
2025-04-16 04:04:27.543054: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.545289 151949 buffer_comparator.cc:156] Difference at 1056: 937.724, expected 1049.14
E0000 00:00:1744776267.545302 151949 buffer_comparator.cc:156] Difference at 1057: 939.844, expected 1047.51
E0000 00:00:1744776267.545305 151949 buffer_comparator.cc:156] Difference at 1061: 932.553, expected 1037.02
E0000 00:00:1744776267.545309 151949 buffer_comparator.cc:156] Difference at 1062: 945.623, expected 1051.04
E0000 00:00:1744776267.545312 151949 buffer_comparator.cc:156] Difference at 1067: 936.361, expected 1046.17
E0000 00:00:1744776267.545315 151949 buffer_comparator.cc:156] Difference at 1071: 926.359, expected 1048.51
E0000 00:00:1744776267.545318 151949 buffer_comparator.cc:156] Difference at 1104: 922.036, expected 1028.34
E0000 00:00:1744776267.545321 151949 buffer_comparator.cc:156] Difference at 1109: 920.805, expected 1023.27
E0000 00:00:1744776267.545324 151949 buffer_comparator.cc:156] Difference at 1119: 926.194, expected 1035.14
E0000 00:00:1744776267.545327 151949 buffer_comparator.cc:156] Difference at 1137: 940.954, expected 1046.87
2025-04-16 04:04:27.545332: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.547530 151949 buffer_comparator.cc:156] Difference at 1056: 937.724, expected 1049.14
E0000 00:00:1744776267.547542 151949 buffer_comparator.cc:156] Difference at 1057: 939.844, expected 1047.51
E0000 00:00:1744776267.547545 151949 buffer_comparator.cc:156] Difference at 1061: 932.553, expected 1037.02
E0000 00:00:1744776267.547548 151949 buffer_comparator.cc:156] Difference at 1062: 945.623, expected 1051.04
E0000 00:00:1744776267.547551 151949 buffer_comparator.cc:156] Difference at 1067: 936.361, expected 1046.17
E0000 00:00:1744776267.547555 151949 buffer_comparator.cc:156] Difference at 1071: 926.359, expected 1048.51
E0000 00:00:1744776267.547558 151949 buffer_comparator.cc:156] Difference at 1104: 922.036, expected 1028.34
E0000 00:00:1744776267.547561 151949 buffer_comparator.cc:156] Difference at 1109: 920.805, expected 1023.27
E0000 00:00:1744776267.547564 151949 buffer_comparator.cc:156] Difference at 1119: 926.194, expected 1035.14
E0000 00:00:1744776267.547567 151949 buffer_comparator.cc:156] Difference at 1137: 940.954, expected 1046.87
2025-04-16 04:04:27.547572: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.565896 151949 buffer_comparator.cc:156] Difference at 16: -nan, expected 1.69914
E0000 00:00:1744776267.565909 151949 buffer_comparator.cc:156] Difference at 17: -nan, expected 1.20048
E0000 00:00:1744776267.565913 151949 buffer_comparator.cc:156] Difference at 18: -nan, expected 1.89606
E0000 00:00:1744776267.565915 151949 buffer_comparator.cc:156] Difference at 19: -nan, expected 1.32316
E0000 00:00:1744776267.565918 151949 buffer_comparator.cc:156] Difference at 20: -nan, expected 1.85572
E0000 00:00:1744776267.565921 151949 buffer_comparator.cc:156] Difference at 21: -nan, expected 0.944347
E0000 00:00:1744776267.565924 151949 buffer_comparator.cc:156] Difference at 22: -nan, expected 1.86911
E0000 00:00:1744776267.565926 151949 buffer_comparator.cc:156] Difference at 23: -nan, expected 0.58457
E0000 00:00:1744776267.565929 151949 buffer_comparator.cc:156] Difference at 24: -nan, expected 2.7184
E0000 00:00:1744776267.565932 151949 buffer_comparator.cc:156] Difference at 25: -nan, expected 2.16503
2025-04-16 04:04:27.565937: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.569074 151949 buffer_comparator.cc:156] Difference at 16: -nan, expected 1.69914
E0000 00:00:1744776267.569085 151949 buffer_comparator.cc:156] Difference at 17: -nan, expected 1.20048
E0000 00:00:1744776267.569088 151949 buffer_comparator.cc:156] Difference at 18: -nan, expected 1.89606
E0000 00:00:1744776267.569092 151949 buffer_comparator.cc:156] Difference at 19: -nan, expected 1.32316
E0000 00:00:1744776267.569095 151949 buffer_comparator.cc:156] Difference at 20: -nan, expected 1.85572
E0000 00:00:1744776267.569098 151949 buffer_comparator.cc:156] Difference at 21: -nan, expected 0.944347
E0000 00:00:1744776267.569101 151949 buffer_comparator.cc:156] Difference at 22: -nan, expected 1.86911
E0000 00:00:1744776267.569103 151949 buffer_comparator.cc:156] Difference at 23: -nan, expected 0.58457
E0000 00:00:1744776267.569106 151949 buffer_comparator.cc:156] Difference at 24: -nan, expected 2.7184
E0000 00:00:1744776267.569109 151949 buffer_comparator.cc:156] Difference at 25: -nan, expected 2.16503
2025-04-16 04:04:27.569113: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.572184 151949 buffer_comparator.cc:156] Difference at 64: -nan, expected 1.60438
E0000 00:00:1744776267.572195 151949 buffer_comparator.cc:156] Difference at 65: -nan, expected 1.28593
E0000 00:00:1744776267.572198 151949 buffer_comparator.cc:156] Difference at 66: -nan, expected 2.03265
E0000 00:00:1744776267.572201 151949 buffer_comparator.cc:156] Difference at 67: -nan, expected 2.28838
E0000 00:00:1744776267.572204 151949 buffer_comparator.cc:156] Difference at 68: -nan, expected 1.76697
E0000 00:00:1744776267.572206 151949 buffer_comparator.cc:156] Difference at 69: -nan, expected 1.33634
E0000 00:00:1744776267.572209 151949 buffer_comparator.cc:156] Difference at 70: -nan, expected 1.96458
E0000 00:00:1744776267.572211 151949 buffer_comparator.cc:156] Difference at 71: -nan, expected 1.16919
E0000 00:00:1744776267.572214 151949 buffer_comparator.cc:156] Difference at 72: -nan, expected 0.893586
E0000 00:00:1744776267.572217 151949 buffer_comparator.cc:156] Difference at 73: -nan, expected 2.3101
2025-04-16 04:04:27.572221: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.575316 151949 buffer_comparator.cc:156] Difference at 2048: -nan, expected 1.77658
E0000 00:00:1744776267.575332 151949 buffer_comparator.cc:156] Difference at 2049: -nan, expected 1.05138
E0000 00:00:1744776267.575335 151949 buffer_comparator.cc:156] Difference at 2050: -nan, expected 1.81989
E0000 00:00:1744776267.575338 151949 buffer_comparator.cc:156] Difference at 2051: -nan, expected 2.39565
E0000 00:00:1744776267.575340 151949 buffer_comparator.cc:156] Difference at 2052: -nan, expected 1.54422
E0000 00:00:1744776267.575343 151949 buffer_comparator.cc:156] Difference at 2053: -nan, expected 2.20321
E0000 00:00:1744776267.575345 151949 buffer_comparator.cc:156] Difference at 2054: -nan, expected 1.94268
E0000 00:00:1744776267.575348 151949 buffer_comparator.cc:156] Difference at 2055: -nan, expected 2.41167
E0000 00:00:1744776267.575351 151949 buffer_comparator.cc:156] Difference at 2056: -nan, expected 2.09568
E0000 00:00:1744776267.575353 151949 buffer_comparator.cc:156] Difference at 2057: -nan, expected 1.1478
2025-04-16 04:04:27.575358: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.578457 151949 buffer_comparator.cc:156] Difference at 2064: -nan, expected 1.87707
E0000 00:00:1744776267.578476 151949 buffer_comparator.cc:156] Difference at 2065: -nan, expected 0.928597
E0000 00:00:1744776267.578479 151949 buffer_comparator.cc:156] Difference at 2066: -nan, expected 2.285
E0000 00:00:1744776267.578482 151949 buffer_comparator.cc:156] Difference at 2067: -nan, expected 1.20842
E0000 00:00:1744776267.578484 151949 buffer_comparator.cc:156] Difference at 2068: -nan, expected 1.71371
E0000 00:00:1744776267.578487 151949 buffer_comparator.cc:156] Difference at 2069: -nan, expected 1.03264
E0000 00:00:1744776267.578492 151949 buffer_comparator.cc:156] Difference at 2070: -nan, expected 1.51429
E0000 00:00:1744776267.578495 151949 buffer_comparator.cc:156] Difference at 2071: -nan, expected 0.671343
E0000 00:00:1744776267.578497 151949 buffer_comparator.cc:156] Difference at 2072: -nan, expected 2.95498
E0000 00:00:1744776267.578500 151949 buffer_comparator.cc:156] Difference at 2073: -nan, expected 2.32851
2025-04-16 04:04:27.578504: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.581453 151949 buffer_comparator.cc:156] Difference at 2080: -nan, expected 1.44171
E0000 00:00:1744776267.581468 151949 buffer_comparator.cc:156] Difference at 2081: -nan, expected 2.15664
E0000 00:00:1744776267.581471 151949 buffer_comparator.cc:156] Difference at 2082: -nan, expected 2.50291
E0000 00:00:1744776267.581473 151949 buffer_comparator.cc:156] Difference at 2083: -nan, expected 1.60771
E0000 00:00:1744776267.581476 151949 buffer_comparator.cc:156] Difference at 2084: -nan, expected 2.0311
E0000 00:00:1744776267.581479 151949 buffer_comparator.cc:156] Difference at 2085: -nan, expected 1.65865
E0000 00:00:1744776267.581481 151949 buffer_comparator.cc:156] Difference at 2086: -nan, expected 1.72254
E0000 00:00:1744776267.581484 151949 buffer_comparator.cc:156] Difference at 2087: -nan, expected 2.04439
E0000 00:00:1744776267.581487 151949 buffer_comparator.cc:156] Difference at 2088: -nan, expected 1.54755
E0000 00:00:1744776267.581489 151949 buffer_comparator.cc:156] Difference at 2089: -nan, expected 0.955993
2025-04-16 04:04:27.581494: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.584419 151949 buffer_comparator.cc:156] Difference at 2112: -nan, expected 1.48116
E0000 00:00:1744776267.584433 151949 buffer_comparator.cc:156] Difference at 2113: -nan, expected 1.4478
E0000 00:00:1744776267.584436 151949 buffer_comparator.cc:156] Difference at 2114: -nan, expected 2.13734
E0000 00:00:1744776267.584438 151949 buffer_comparator.cc:156] Difference at 2115: -nan, expected 2.19489
E0000 00:00:1744776267.584441 151949 buffer_comparator.cc:156] Difference at 2116: -nan, expected 1.82644
E0000 00:00:1744776267.584444 151949 buffer_comparator.cc:156] Difference at 2117: -nan, expected 1.36935
E0000 00:00:1744776267.584446 151949 buffer_comparator.cc:156] Difference at 2118: -nan, expected 1.67303
E0000 00:00:1744776267.584449 151949 buffer_comparator.cc:156] Difference at 2119: -nan, expected 1.26478
E0000 00:00:1744776267.584451 151949 buffer_comparator.cc:156] Difference at 2120: -nan, expected 1.00439
E0000 00:00:1744776267.584454 151949 buffer_comparator.cc:156] Difference at 2121: -nan, expected 2.64095
2025-04-16 04:04:27.584458: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.587350 151949 buffer_comparator.cc:156] Difference at 4096: -nan, expected 2.5269
E0000 00:00:1744776267.587364 151949 buffer_comparator.cc:156] Difference at 4097: -nan, expected 1.96414
E0000 00:00:1744776267.587367 151949 buffer_comparator.cc:156] Difference at 4098: -nan, expected 3.84871
E0000 00:00:1744776267.587369 151949 buffer_comparator.cc:156] Difference at 4099: -nan, expected 4.06422
E0000 00:00:1744776267.587372 151949 buffer_comparator.cc:156] Difference at 4100: -nan, expected 1.84843
E0000 00:00:1744776267.587375 151949 buffer_comparator.cc:156] Difference at 4101: -nan, expected 3.06637
E0000 00:00:1744776267.587377 151949 buffer_comparator.cc:156] Difference at 4102: -nan, expected 3.26933
E0000 00:00:1744776267.587380 151949 buffer_comparator.cc:156] Difference at 4103: -nan, expected 2.61708
E0000 00:00:1744776267.587383 151949 buffer_comparator.cc:156] Difference at 4104: -nan, expected 3.42482
E0000 00:00:1744776267.587387 151949 buffer_comparator.cc:156] Difference at 4105: -nan, expected 2.29469
2025-04-16 04:04:27.587391: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.590295 151949 buffer_comparator.cc:156] Difference at 4128: -nan, expected 2.63461
E0000 00:00:1744776267.590308 151949 buffer_comparator.cc:156] Difference at 4129: -nan, expected 2.9239
E0000 00:00:1744776267.590311 151949 buffer_comparator.cc:156] Difference at 4130: -nan, expected 2.40995
E0000 00:00:1744776267.590314 151949 buffer_comparator.cc:156] Difference at 4131: -nan, expected 2.61607
E0000 00:00:1744776267.590316 151949 buffer_comparator.cc:156] Difference at 4132: -nan, expected 3.62419
E0000 00:00:1744776267.590319 151949 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.10893
E0000 00:00:1744776267.590321 151949 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.56351
E0000 00:00:1744776267.590324 151949 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.19994
E0000 00:00:1744776267.590327 151949 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.37682
E0000 00:00:1744776267.590329 151949 buffer_comparator.cc:156] Difference at 4137: -nan, expected 2.50883
2025-04-16 04:04:27.590334: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.593232 151949 buffer_comparator.cc:156] Difference at 4128: -nan, expected 2.63461
E0000 00:00:1744776267.593246 151949 buffer_comparator.cc:156] Difference at 4129: -nan, expected 2.9239
E0000 00:00:1744776267.593249 151949 buffer_comparator.cc:156] Difference at 4130: -nan, expected 2.40995
E0000 00:00:1744776267.593252 151949 buffer_comparator.cc:156] Difference at 4131: -nan, expected 2.61607
E0000 00:00:1744776267.593255 151949 buffer_comparator.cc:156] Difference at 4132: -nan, expected 3.62419
E0000 00:00:1744776267.593257 151949 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.10893
E0000 00:00:1744776267.593260 151949 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.56351
E0000 00:00:1744776267.593262 151949 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.19994
E0000 00:00:1744776267.593265 151949 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.37682
E0000 00:00:1744776267.593268 151949 buffer_comparator.cc:156] Difference at 4137: -nan, expected 2.50883
2025-04-16 04:04:27.593272: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.596171 151949 buffer_comparator.cc:156] Difference at 4128: -nan, expected 2.63461
E0000 00:00:1744776267.596184 151949 buffer_comparator.cc:156] Difference at 4129: -nan, expected 2.9239
E0000 00:00:1744776267.596187 151949 buffer_comparator.cc:156] Difference at 4130: -nan, expected 2.40995
E0000 00:00:1744776267.596190 151949 buffer_comparator.cc:156] Difference at 4131: -nan, expected 2.61607
E0000 00:00:1744776267.596192 151949 buffer_comparator.cc:156] Difference at 4132: -nan, expected 3.62419
E0000 00:00:1744776267.596195 151949 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.10893
E0000 00:00:1744776267.596198 151949 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.56351
E0000 00:00:1744776267.596200 151949 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.19994
E0000 00:00:1744776267.596203 151949 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.37682
E0000 00:00:1744776267.596205 151949 buffer_comparator.cc:156] Difference at 4137: -nan, expected 2.50883
2025-04-16 04:04:27.596210: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.599110 151949 buffer_comparator.cc:156] Difference at 4160: -nan, expected 2.86435
E0000 00:00:1744776267.599127 151949 buffer_comparator.cc:156] Difference at 4161: -nan, expected 2.0664
E0000 00:00:1744776267.599129 151949 buffer_comparator.cc:156] Difference at 4162: -nan, expected 2.61642
E0000 00:00:1744776267.599132 151949 buffer_comparator.cc:156] Difference at 4163: -nan, expected 3.35871
E0000 00:00:1744776267.599135 151949 buffer_comparator.cc:156] Difference at 4164: -nan, expected 3.1527
E0000 00:00:1744776267.599137 151949 buffer_comparator.cc:156] Difference at 4165: -nan, expected 1.96457
E0000 00:00:1744776267.599140 151949 buffer_comparator.cc:156] Difference at 4166: -nan, expected 2.94612
E0000 00:00:1744776267.599143 151949 buffer_comparator.cc:156] Difference at 4167: -nan, expected 1.54296
E0000 00:00:1744776267.599145 151949 buffer_comparator.cc:156] Difference at 4168: -nan, expected 1.32996
E0000 00:00:1744776267.599148 151949 buffer_comparator.cc:156] Difference at 4169: -nan, expected 3.04802
2025-04-16 04:04:27.599152: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.602112 151949 buffer_comparator.cc:156] Difference at 8192: -nan, expected 1.98786
E0000 00:00:1744776267.602126 151949 buffer_comparator.cc:156] Difference at 8193: -nan, expected 1.72939
E0000 00:00:1744776267.602129 151949 buffer_comparator.cc:156] Difference at 8194: -nan, expected 3.03127
E0000 00:00:1744776267.602131 151949 buffer_comparator.cc:156] Difference at 8195: -nan, expected 3.3803
E0000 00:00:1744776267.602134 151949 buffer_comparator.cc:156] Difference at 8196: -nan, expected 1.7692
E0000 00:00:1744776267.602137 151949 buffer_comparator.cc:156] Difference at 8197: -nan, expected 2.87459
E0000 00:00:1744776267.602139 151949 buffer_comparator.cc:156] Difference at 8198: -nan, expected 2.74368
E0000 00:00:1744776267.602142 151949 buffer_comparator.cc:156] Difference at 8199: -nan, expected 2.30403
E0000 00:00:1744776267.602145 151949 buffer_comparator.cc:156] Difference at 8200: -nan, expected 3.07071
E0000 00:00:1744776267.602147 151949 buffer_comparator.cc:156] Difference at 8201: -nan, expected 1.9239
2025-04-16 04:04:27.602152: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.605055 151949 buffer_comparator.cc:156] Difference at 8224: -nan, expected 2.18642
E0000 00:00:1744776267.605069 151949 buffer_comparator.cc:156] Difference at 8225: -nan, expected 2.28379
E0000 00:00:1744776267.605071 151949 buffer_comparator.cc:156] Difference at 8226: -nan, expected 2.33258
E0000 00:00:1744776267.605074 151949 buffer_comparator.cc:156] Difference at 8227: -nan, expected 2.12912
E0000 00:00:1744776267.605077 151949 buffer_comparator.cc:156] Difference at 8228: -nan, expected 2.99415
E0000 00:00:1744776267.605080 151949 buffer_comparator.cc:156] Difference at 8229: -nan, expected 2.27319
E0000 00:00:1744776267.605083 151949 buffer_comparator.cc:156] Difference at 8230: -nan, expected 2.165
E0000 00:00:1744776267.605086 151949 buffer_comparator.cc:156] Difference at 8231: -nan, expected 2.33242
E0000 00:00:1744776267.605089 151949 buffer_comparator.cc:156] Difference at 8232: -nan, expected 1.81753
E0000 00:00:1744776267.605092 151949 buffer_comparator.cc:156] Difference at 8233: -nan, expected 1.754
2025-04-16 04:04:27.605097: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.608013 151949 buffer_comparator.cc:156] Difference at 8256: -nan, expected 2.10464
E0000 00:00:1744776267.608029 151949 buffer_comparator.cc:156] Difference at 8257: -nan, expected 2.00542
E0000 00:00:1744776267.608034 151949 buffer_comparator.cc:156] Difference at 8258: -nan, expected 2.33829
E0000 00:00:1744776267.608036 151949 buffer_comparator.cc:156] Difference at 8259: -nan, expected 2.68564
E0000 00:00:1744776267.608039 151949 buffer_comparator.cc:156] Difference at 8260: -nan, expected 2.75266
E0000 00:00:1744776267.608042 151949 buffer_comparator.cc:156] Difference at 8261: -nan, expected 2.02724
E0000 00:00:1744776267.608044 151949 buffer_comparator.cc:156] Difference at 8262: -nan, expected 1.93574
E0000 00:00:1744776267.608047 151949 buffer_comparator.cc:156] Difference at 8263: -nan, expected 1.335
E0000 00:00:1744776267.608050 151949 buffer_comparator.cc:156] Difference at 8264: -nan, expected 1.15464
E0000 00:00:1744776267.608052 151949 buffer_comparator.cc:156] Difference at 8265: -nan, expected 3.13129
2025-04-16 04:04:27.608057: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.610996 151949 buffer_comparator.cc:156] Difference at 16384: -nan, expected 2.13072
E0000 00:00:1744776267.611011 151949 buffer_comparator.cc:156] Difference at 16385: -nan, expected 1.18098
E0000 00:00:1744776267.611014 151949 buffer_comparator.cc:156] Difference at 16386: -nan, expected 2.76235
E0000 00:00:1744776267.611016 151949 buffer_comparator.cc:156] Difference at 16387: -nan, expected 2.87006
E0000 00:00:1744776267.611019 151949 buffer_comparator.cc:156] Difference at 16388: -nan, expected 1.48471
E0000 00:00:1744776267.611022 151949 buffer_comparator.cc:156] Difference at 16389: -nan, expected 2.14624
E0000 00:00:1744776267.611024 151949 buffer_comparator.cc:156] Difference at 16390: -nan, expected 2.04824
E0000 00:00:1744776267.611027 151949 buffer_comparator.cc:156] Difference at 16391: -nan, expected 2.47653
E0000 00:00:1744776267.611029 151949 buffer_comparator.cc:156] Difference at 16392: -nan, expected 2.26896
E0000 00:00:1744776267.611032 151949 buffer_comparator.cc:156] Difference at 16393: -nan, expected 1.36576
2025-04-16 04:04:27.611036: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.613970 151949 buffer_comparator.cc:156] Difference at 16384: -nan, expected 2.13072
E0000 00:00:1744776267.613984 151949 buffer_comparator.cc:156] Difference at 16385: -nan, expected 1.18098
E0000 00:00:1744776267.613986 151949 buffer_comparator.cc:156] Difference at 16386: -nan, expected 2.76235
E0000 00:00:1744776267.613989 151949 buffer_comparator.cc:156] Difference at 16387: -nan, expected 2.87006
E0000 00:00:1744776267.613992 151949 buffer_comparator.cc:156] Difference at 16388: -nan, expected 1.48471
E0000 00:00:1744776267.613994 151949 buffer_comparator.cc:156] Difference at 16389: -nan, expected 2.14624
E0000 00:00:1744776267.613997 151949 buffer_comparator.cc:156] Difference at 16390: -nan, expected 2.04824
E0000 00:00:1744776267.613999 151949 buffer_comparator.cc:156] Difference at 16391: -nan, expected 2.47653
E0000 00:00:1744776267.614002 151949 buffer_comparator.cc:156] Difference at 16392: -nan, expected 2.26896
E0000 00:00:1744776267.614005 151949 buffer_comparator.cc:156] Difference at 16393: -nan, expected 1.36576
2025-04-16 04:04:27.614009: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.616965 151949 buffer_comparator.cc:156] Difference at 16448: -nan, expected 2.04485
E0000 00:00:1744776267.616980 151949 buffer_comparator.cc:156] Difference at 16449: -nan, expected 1.38058
E0000 00:00:1744776267.616983 151949 buffer_comparator.cc:156] Difference at 16450: -nan, expected 1.77153
E0000 00:00:1744776267.616986 151949 buffer_comparator.cc:156] Difference at 16451: -nan, expected 2.60773
E0000 00:00:1744776267.616990 151949 buffer_comparator.cc:156] Difference at 16452: -nan, expected 2.24942
E0000 00:00:1744776267.616993 151949 buffer_comparator.cc:156] Difference at 16453: -nan, expected 1.32792
E0000 00:00:1744776267.616995 151949 buffer_comparator.cc:156] Difference at 16454: -nan, expected 2.7744
E0000 00:00:1744776267.616998 151949 buffer_comparator.cc:156] Difference at 16455: -nan, expected 1.26898
E0000 00:00:1744776267.617001 151949 buffer_comparator.cc:156] Difference at 16456: -nan, expected 1.06744
E0000 00:00:1744776267.617003 151949 buffer_comparator.cc:156] Difference at 16457: -nan, expected 1.99063
2025-04-16 04:04:27.617008: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776267.619996 151949 buffer_comparator.cc:156] Difference at 32768: -nan, expected 2.02216
E0000 00:00:1744776267.620010 151949 buffer_comparator.cc:156] Difference at 32769: -nan, expected 1.27847
E0000 00:00:1744776267.620013 151949 buffer_comparator.cc:156] Difference at 32770: -nan, expected 2.25127
E0000 00:00:1744776267.620016 151949 buffer_comparator.cc:156] Difference at 32771: -nan, expected 2.96132
E0000 00:00:1744776267.620018 151949 buffer_comparator.cc:156] Difference at 32772: -nan, expected 1.65598
E0000 00:00:1744776267.620021 151949 buffer_comparator.cc:156] Difference at 32773: -nan, expected 2.09882
E0000 00:00:1744776267.620023 151949 buffer_comparator.cc:156] Difference at 32774: -nan, expected 2.35327
E0000 00:00:1744776267.620026 151949 buffer_comparator.cc:156] Difference at 32775: -nan, expected 2.14737
E0000 00:00:1744776267.620029 151949 buffer_comparator.cc:156] Difference at 32776: -nan, expected 2.35827
E0000 00:00:1744776267.620031 151949 buffer_comparator.cc:156] Difference at 32777: -nan, expected 1.49831
2025-04-16 04:04:27.620036: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
Total Trainable Parameters: 0.1493 M
2025-04-16 04:05:50.192038: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 4572 bytes spill stores, 4564 bytes spill loads
2025-04-16 04:05:50.328089: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 180 bytes spill stores, 180 bytes spill loads
2025-04-16 04:05:50.415957: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 192 bytes spill stores, 192 bytes spill loads
2025-04-16 04:05:50.498834: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_886', 3212 bytes spill stores, 3236 bytes spill loads
2025-04-16 04:05:50.656376: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 3932 bytes spill stores, 3920 bytes spill loads
2025-04-16 04:05:51.450976: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 232 bytes spill stores, 232 bytes spill loads
2025-04-16 04:05:52.527855: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_886', 2732 bytes spill stores, 2752 bytes spill loads
E0000 00:00:1744776354.508982 151949 buffer_comparator.cc:156] Difference at 16: nan, expected 1026.62
E0000 00:00:1744776354.509042 151949 buffer_comparator.cc:156] Difference at 17: nan, expected 1040.31
E0000 00:00:1744776354.509055 151949 buffer_comparator.cc:156] Difference at 18: nan, expected 1023.91
E0000 00:00:1744776354.509063 151949 buffer_comparator.cc:156] Difference at 19: nan, expected 1055.64
E0000 00:00:1744776354.509069 151949 buffer_comparator.cc:156] Difference at 20: nan, expected 1021.6
E0000 00:00:1744776354.509076 151949 buffer_comparator.cc:156] Difference at 21: nan, expected 1036.76
E0000 00:00:1744776354.509082 151949 buffer_comparator.cc:156] Difference at 22: nan, expected 1036.42
E0000 00:00:1744776354.509089 151949 buffer_comparator.cc:156] Difference at 23: nan, expected 1028.95
E0000 00:00:1744776354.509095 151949 buffer_comparator.cc:156] Difference at 24: nan, expected 1042.97
E0000 00:00:1744776354.509102 151949 buffer_comparator.cc:156] Difference at 25: nan, expected 1033.85
2025-04-16 04:05:54.509117: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.511641 151949 buffer_comparator.cc:156] Difference at 16: nan, expected 1026.62
E0000 00:00:1744776354.511668 151949 buffer_comparator.cc:156] Difference at 17: nan, expected 1040.31
E0000 00:00:1744776354.511677 151949 buffer_comparator.cc:156] Difference at 18: nan, expected 1023.91
E0000 00:00:1744776354.511684 151949 buffer_comparator.cc:156] Difference at 19: nan, expected 1055.64
E0000 00:00:1744776354.511690 151949 buffer_comparator.cc:156] Difference at 20: nan, expected 1021.6
E0000 00:00:1744776354.511697 151949 buffer_comparator.cc:156] Difference at 21: nan, expected 1036.76
E0000 00:00:1744776354.511703 151949 buffer_comparator.cc:156] Difference at 22: nan, expected 1036.42
E0000 00:00:1744776354.511710 151949 buffer_comparator.cc:156] Difference at 23: nan, expected 1028.95
E0000 00:00:1744776354.511716 151949 buffer_comparator.cc:156] Difference at 24: nan, expected 1042.97
E0000 00:00:1744776354.511723 151949 buffer_comparator.cc:156] Difference at 25: nan, expected 1033.85
2025-04-16 04:05:54.511733: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.514258 151949 buffer_comparator.cc:156] Difference at 16: nan, expected 1026.62
E0000 00:00:1744776354.514288 151949 buffer_comparator.cc:156] Difference at 17: nan, expected 1040.31
E0000 00:00:1744776354.514296 151949 buffer_comparator.cc:156] Difference at 18: nan, expected 1023.91
E0000 00:00:1744776354.514303 151949 buffer_comparator.cc:156] Difference at 19: nan, expected 1055.64
E0000 00:00:1744776354.514310 151949 buffer_comparator.cc:156] Difference at 20: nan, expected 1021.6
E0000 00:00:1744776354.514317 151949 buffer_comparator.cc:156] Difference at 21: nan, expected 1036.76
E0000 00:00:1744776354.514323 151949 buffer_comparator.cc:156] Difference at 22: nan, expected 1036.42
E0000 00:00:1744776354.514330 151949 buffer_comparator.cc:156] Difference at 23: nan, expected 1028.95
E0000 00:00:1744776354.514336 151949 buffer_comparator.cc:156] Difference at 24: nan, expected 1042.97
E0000 00:00:1744776354.514343 151949 buffer_comparator.cc:156] Difference at 25: nan, expected 1033.85
2025-04-16 04:05:54.514353: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.516584 151949 buffer_comparator.cc:156] Difference at 32: nan, expected 1037.37
E0000 00:00:1744776354.516595 151949 buffer_comparator.cc:156] Difference at 33: nan, expected 1035.2
E0000 00:00:1744776354.516599 151949 buffer_comparator.cc:156] Difference at 34: nan, expected 1045.91
E0000 00:00:1744776354.516602 151949 buffer_comparator.cc:156] Difference at 35: nan, expected 1018
E0000 00:00:1744776354.516605 151949 buffer_comparator.cc:156] Difference at 36: nan, expected 1041.06
E0000 00:00:1744776354.516608 151949 buffer_comparator.cc:156] Difference at 37: nan, expected 1034.34
E0000 00:00:1744776354.516611 151949 buffer_comparator.cc:156] Difference at 38: nan, expected 1028.43
E0000 00:00:1744776354.516614 151949 buffer_comparator.cc:156] Difference at 39: nan, expected 1044.85
E0000 00:00:1744776354.516617 151949 buffer_comparator.cc:156] Difference at 40: nan, expected 1002.92
E0000 00:00:1744776354.516620 151949 buffer_comparator.cc:156] Difference at 41: nan, expected 1052.35
2025-04-16 04:05:54.516624: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.518793 151949 buffer_comparator.cc:156] Difference at 32: 820.595, expected 1037.37
E0000 00:00:1744776354.518805 151949 buffer_comparator.cc:156] Difference at 33: 816.623, expected 1035.2
E0000 00:00:1744776354.518809 151949 buffer_comparator.cc:156] Difference at 34: 838.89, expected 1045.91
E0000 00:00:1744776354.518812 151949 buffer_comparator.cc:156] Difference at 35: 814.264, expected 1018
E0000 00:00:1744776354.518816 151949 buffer_comparator.cc:156] Difference at 36: 826.931, expected 1041.06
E0000 00:00:1744776354.518819 151949 buffer_comparator.cc:156] Difference at 37: 821.224, expected 1034.34
E0000 00:00:1744776354.518822 151949 buffer_comparator.cc:156] Difference at 38: 816.876, expected 1028.43
E0000 00:00:1744776354.518825 151949 buffer_comparator.cc:156] Difference at 39: 829.33, expected 1044.85
E0000 00:00:1744776354.518829 151949 buffer_comparator.cc:156] Difference at 40: 814.723, expected 1002.92
E0000 00:00:1744776354.518832 151949 buffer_comparator.cc:156] Difference at 41: 827.582, expected 1052.35
2025-04-16 04:05:54.518837: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.521150 151949 buffer_comparator.cc:156] Difference at 64: 824.662, expected 1028.46
E0000 00:00:1744776354.521162 151949 buffer_comparator.cc:156] Difference at 65: 822.425, expected 1030.32
E0000 00:00:1744776354.521166 151949 buffer_comparator.cc:156] Difference at 66: 827.578, expected 1049.13
E0000 00:00:1744776354.521170 151949 buffer_comparator.cc:156] Difference at 67: 808.591, expected 1011.57
E0000 00:00:1744776354.521175 151949 buffer_comparator.cc:156] Difference at 68: 832.873, expected 1052.12
E0000 00:00:1744776354.521179 151949 buffer_comparator.cc:156] Difference at 69: 818.422, expected 1028.34
E0000 00:00:1744776354.521182 151949 buffer_comparator.cc:156] Difference at 70: 821.002, expected 1027.96
E0000 00:00:1744776354.521185 151949 buffer_comparator.cc:156] Difference at 71: 829.276, expected 1035.51
E0000 00:00:1744776354.521188 151949 buffer_comparator.cc:156] Difference at 72: 800.076, expected 1018.47
E0000 00:00:1744776354.521191 151949 buffer_comparator.cc:156] Difference at 73: 836.836, expected 1058.93
2025-04-16 04:05:54.521196: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.523343 151949 buffer_comparator.cc:156] Difference at 4: 1158.54, expected 1032.49
E0000 00:00:1744776354.523356 151949 buffer_comparator.cc:156] Difference at 6: 1212.46, expected 1026.75
E0000 00:00:1744776354.523360 151949 buffer_comparator.cc:156] Difference at 13: 1155.74, expected 1016.78
E0000 00:00:1744776354.523363 151949 buffer_comparator.cc:156] Difference at 15: 1182.13, expected 1029.67
E0000 00:00:1744776354.523366 151949 buffer_comparator.cc:156] Difference at 19: 1214.79, expected 1055.64
E0000 00:00:1744776354.523369 151949 buffer_comparator.cc:156] Difference at 21: 1182.77, expected 1036.76
E0000 00:00:1744776354.523373 151949 buffer_comparator.cc:156] Difference at 28: 1150.36, expected 1028.72
E0000 00:00:1744776354.523376 151949 buffer_comparator.cc:156] Difference at 35: 1149.11, expected 1018
E0000 00:00:1744776354.523379 151949 buffer_comparator.cc:156] Difference at 43: 1192.41, expected 1024.68
E0000 00:00:1744776354.523382 151949 buffer_comparator.cc:156] Difference at 45: 1132.55, expected 1015.8
2025-04-16 04:05:54.523387: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.525694 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 1028.46
E0000 00:00:1744776354.525705 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 1030.32
E0000 00:00:1744776354.525709 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 1049.13
E0000 00:00:1744776354.525712 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 1011.57
E0000 00:00:1744776354.525715 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 1052.12
E0000 00:00:1744776354.525718 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 1028.34
E0000 00:00:1744776354.525721 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 1027.96
E0000 00:00:1744776354.525724 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 1035.51
E0000 00:00:1744776354.525727 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 1018.47
E0000 00:00:1744776354.525729 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 1058.93
2025-04-16 04:05:54.525734: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.528061 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 1028.46
E0000 00:00:1744776354.528073 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 1030.32
E0000 00:00:1744776354.528077 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 1049.13
E0000 00:00:1744776354.528080 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 1011.57
E0000 00:00:1744776354.528082 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 1052.12
E0000 00:00:1744776354.528085 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 1028.34
E0000 00:00:1744776354.528088 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 1027.96
E0000 00:00:1744776354.528093 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 1035.51
E0000 00:00:1744776354.528096 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 1018.47
E0000 00:00:1744776354.528099 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 1058.93
2025-04-16 04:05:54.528103: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.530348 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 1028.46
E0000 00:00:1744776354.530359 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 1030.32
E0000 00:00:1744776354.530363 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 1049.13
E0000 00:00:1744776354.530366 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 1011.57
E0000 00:00:1744776354.530369 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 1052.12
E0000 00:00:1744776354.530372 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 1028.34
E0000 00:00:1744776354.530374 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 1027.96
E0000 00:00:1744776354.530377 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 1035.51
E0000 00:00:1744776354.530380 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 1018.47
E0000 00:00:1744776354.530383 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 1058.93
2025-04-16 04:05:54.530388: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.532584 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 1028.46
E0000 00:00:1744776354.532596 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 1030.32
E0000 00:00:1744776354.532600 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 1049.13
E0000 00:00:1744776354.532602 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 1011.57
E0000 00:00:1744776354.532605 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 1052.12
E0000 00:00:1744776354.532608 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 1028.34
E0000 00:00:1744776354.532611 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 1027.96
E0000 00:00:1744776354.532614 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 1035.51
E0000 00:00:1744776354.532617 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 1018.47
E0000 00:00:1744776354.532620 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 1058.93
2025-04-16 04:05:54.532624: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.549916 151949 buffer_comparator.cc:156] Difference at 16: -nan, expected 2.37843
E0000 00:00:1744776354.549929 151949 buffer_comparator.cc:156] Difference at 17: -nan, expected 2.03294
E0000 00:00:1744776354.549933 151949 buffer_comparator.cc:156] Difference at 18: -nan, expected 1.81258
E0000 00:00:1744776354.549936 151949 buffer_comparator.cc:156] Difference at 19: -nan, expected 1.88877
E0000 00:00:1744776354.549939 151949 buffer_comparator.cc:156] Difference at 20: -nan, expected 1.82906
E0000 00:00:1744776354.549942 151949 buffer_comparator.cc:156] Difference at 21: -nan, expected 2.60748
E0000 00:00:1744776354.549944 151949 buffer_comparator.cc:156] Difference at 22: -nan, expected 1.56132
E0000 00:00:1744776354.549947 151949 buffer_comparator.cc:156] Difference at 23: -nan, expected 2.04635
E0000 00:00:1744776354.549950 151949 buffer_comparator.cc:156] Difference at 24: -nan, expected 1.78507
E0000 00:00:1744776354.549953 151949 buffer_comparator.cc:156] Difference at 25: -nan, expected 2.05368
2025-04-16 04:05:54.549959: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.553093 151949 buffer_comparator.cc:156] Difference at 16: -nan, expected 2.37843
E0000 00:00:1744776354.553104 151949 buffer_comparator.cc:156] Difference at 17: -nan, expected 2.03294
E0000 00:00:1744776354.553108 151949 buffer_comparator.cc:156] Difference at 18: -nan, expected 1.81258
E0000 00:00:1744776354.553111 151949 buffer_comparator.cc:156] Difference at 19: -nan, expected 1.88877
E0000 00:00:1744776354.553114 151949 buffer_comparator.cc:156] Difference at 20: -nan, expected 1.82906
E0000 00:00:1744776354.553116 151949 buffer_comparator.cc:156] Difference at 21: -nan, expected 2.60748
E0000 00:00:1744776354.553119 151949 buffer_comparator.cc:156] Difference at 22: -nan, expected 1.56132
E0000 00:00:1744776354.553122 151949 buffer_comparator.cc:156] Difference at 23: -nan, expected 2.04635
E0000 00:00:1744776354.553124 151949 buffer_comparator.cc:156] Difference at 24: -nan, expected 1.78507
E0000 00:00:1744776354.553127 151949 buffer_comparator.cc:156] Difference at 25: -nan, expected 2.05368
2025-04-16 04:05:54.553132: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.556333 151949 buffer_comparator.cc:156] Difference at 64: -nan, expected 1.48598
E0000 00:00:1744776354.556345 151949 buffer_comparator.cc:156] Difference at 65: -nan, expected 2.07857
E0000 00:00:1744776354.556348 151949 buffer_comparator.cc:156] Difference at 66: -nan, expected 1.26324
E0000 00:00:1744776354.556351 151949 buffer_comparator.cc:156] Difference at 67: -nan, expected 1.68463
E0000 00:00:1744776354.556354 151949 buffer_comparator.cc:156] Difference at 68: -nan, expected 1.99749
E0000 00:00:1744776354.556357 151949 buffer_comparator.cc:156] Difference at 69: -nan, expected 1.67603
E0000 00:00:1744776354.556359 151949 buffer_comparator.cc:156] Difference at 70: -nan, expected 2.47978
E0000 00:00:1744776354.556362 151949 buffer_comparator.cc:156] Difference at 71: -nan, expected 1.62348
E0000 00:00:1744776354.556365 151949 buffer_comparator.cc:156] Difference at 72: -nan, expected 1.64724
E0000 00:00:1744776354.556367 151949 buffer_comparator.cc:156] Difference at 73: -nan, expected 2.03763
2025-04-16 04:05:54.556372: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.559470 151949 buffer_comparator.cc:156] Difference at 2048: -nan, expected 1.73277
E0000 00:00:1744776354.559482 151949 buffer_comparator.cc:156] Difference at 2049: -nan, expected 2.13006
E0000 00:00:1744776354.559485 151949 buffer_comparator.cc:156] Difference at 2050: -nan, expected 2.23299
E0000 00:00:1744776354.559488 151949 buffer_comparator.cc:156] Difference at 2051: -nan, expected 1.71049
E0000 00:00:1744776354.559491 151949 buffer_comparator.cc:156] Difference at 2052: -nan, expected 1.91874
E0000 00:00:1744776354.559494 151949 buffer_comparator.cc:156] Difference at 2053: -nan, expected 2.10385
E0000 00:00:1744776354.559496 151949 buffer_comparator.cc:156] Difference at 2054: -nan, expected 1.4864
E0000 00:00:1744776354.559499 151949 buffer_comparator.cc:156] Difference at 2055: -nan, expected 1.20659
E0000 00:00:1744776354.559502 151949 buffer_comparator.cc:156] Difference at 2056: -nan, expected 1.09857
E0000 00:00:1744776354.559505 151949 buffer_comparator.cc:156] Difference at 2057: -nan, expected 1.95037
2025-04-16 04:05:54.559509: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.562404 151949 buffer_comparator.cc:156] Difference at 2064: -nan, expected 2.07054
E0000 00:00:1744776354.562418 151949 buffer_comparator.cc:156] Difference at 2065: -nan, expected 1.96733
E0000 00:00:1744776354.562421 151949 buffer_comparator.cc:156] Difference at 2066: -nan, expected 1.96903
E0000 00:00:1744776354.562424 151949 buffer_comparator.cc:156] Difference at 2067: -nan, expected 1.58632
E0000 00:00:1744776354.562427 151949 buffer_comparator.cc:156] Difference at 2068: -nan, expected 1.71271
E0000 00:00:1744776354.562430 151949 buffer_comparator.cc:156] Difference at 2069: -nan, expected 2.52547
E0000 00:00:1744776354.562432 151949 buffer_comparator.cc:156] Difference at 2070: -nan, expected 1.66897
E0000 00:00:1744776354.562435 151949 buffer_comparator.cc:156] Difference at 2071: -nan, expected 2.21203
E0000 00:00:1744776354.562438 151949 buffer_comparator.cc:156] Difference at 2072: -nan, expected 1.8923
E0000 00:00:1744776354.562441 151949 buffer_comparator.cc:156] Difference at 2073: -nan, expected 2.03053
2025-04-16 04:05:54.562445: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.565319 151949 buffer_comparator.cc:156] Difference at 2080: -nan, expected 1.15173
E0000 00:00:1744776354.565330 151949 buffer_comparator.cc:156] Difference at 2081: -nan, expected 2.23549
E0000 00:00:1744776354.565334 151949 buffer_comparator.cc:156] Difference at 2082: -nan, expected 1.57531
E0000 00:00:1744776354.565337 151949 buffer_comparator.cc:156] Difference at 2083: -nan, expected 2.56993
E0000 00:00:1744776354.565339 151949 buffer_comparator.cc:156] Difference at 2084: -nan, expected 2.12358
E0000 00:00:1744776354.565342 151949 buffer_comparator.cc:156] Difference at 2085: -nan, expected 1.4598
E0000 00:00:1744776354.565345 151949 buffer_comparator.cc:156] Difference at 2086: -nan, expected 2.02148
E0000 00:00:1744776354.565348 151949 buffer_comparator.cc:156] Difference at 2087: -nan, expected 2.01461
E0000 00:00:1744776354.565350 151949 buffer_comparator.cc:156] Difference at 2088: -nan, expected 1.30566
E0000 00:00:1744776354.565353 151949 buffer_comparator.cc:156] Difference at 2089: -nan, expected 1.95103
2025-04-16 04:05:54.565358: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.568238 151949 buffer_comparator.cc:156] Difference at 2112: -nan, expected 1.3194
E0000 00:00:1744776354.568250 151949 buffer_comparator.cc:156] Difference at 2113: -nan, expected 2.17972
E0000 00:00:1744776354.568253 151949 buffer_comparator.cc:156] Difference at 2114: -nan, expected 0.977965
E0000 00:00:1744776354.568257 151949 buffer_comparator.cc:156] Difference at 2115: -nan, expected 1.81951
E0000 00:00:1744776354.568259 151949 buffer_comparator.cc:156] Difference at 2116: -nan, expected 1.99945
E0000 00:00:1744776354.568262 151949 buffer_comparator.cc:156] Difference at 2117: -nan, expected 1.60803
E0000 00:00:1744776354.568265 151949 buffer_comparator.cc:156] Difference at 2118: -nan, expected 2.17071
E0000 00:00:1744776354.568268 151949 buffer_comparator.cc:156] Difference at 2119: -nan, expected 1.78929
E0000 00:00:1744776354.568270 151949 buffer_comparator.cc:156] Difference at 2120: -nan, expected 1.71855
E0000 00:00:1744776354.568273 151949 buffer_comparator.cc:156] Difference at 2121: -nan, expected 1.74983
2025-04-16 04:05:54.568278: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.571175 151949 buffer_comparator.cc:156] Difference at 4096: -nan, expected 3.95136
E0000 00:00:1744776354.571187 151949 buffer_comparator.cc:156] Difference at 4097: -nan, expected 3.05755
E0000 00:00:1744776354.571191 151949 buffer_comparator.cc:156] Difference at 4098: -nan, expected 2.39692
E0000 00:00:1744776354.571193 151949 buffer_comparator.cc:156] Difference at 4099: -nan, expected 3.12597
E0000 00:00:1744776354.571198 151949 buffer_comparator.cc:156] Difference at 4100: -nan, expected 2.07176
E0000 00:00:1744776354.571201 151949 buffer_comparator.cc:156] Difference at 4101: -nan, expected 3.0179
E0000 00:00:1744776354.571204 151949 buffer_comparator.cc:156] Difference at 4102: -nan, expected 2.32025
E0000 00:00:1744776354.571207 151949 buffer_comparator.cc:156] Difference at 4103: -nan, expected 1.89592
E0000 00:00:1744776354.571209 151949 buffer_comparator.cc:156] Difference at 4104: -nan, expected 2.29819
E0000 00:00:1744776354.571212 151949 buffer_comparator.cc:156] Difference at 4105: -nan, expected 2.81816
2025-04-16 04:05:54.571217: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.574126 151949 buffer_comparator.cc:156] Difference at 4128: -nan, expected 1.89634
E0000 00:00:1744776354.574138 151949 buffer_comparator.cc:156] Difference at 4129: -nan, expected 3.44874
E0000 00:00:1744776354.574141 151949 buffer_comparator.cc:156] Difference at 4130: -nan, expected 3.00624
E0000 00:00:1744776354.574144 151949 buffer_comparator.cc:156] Difference at 4131: -nan, expected 4.18145
E0000 00:00:1744776354.574147 151949 buffer_comparator.cc:156] Difference at 4132: -nan, expected 2.41143
E0000 00:00:1744776354.574149 151949 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.24339
E0000 00:00:1744776354.574152 151949 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.63852
E0000 00:00:1744776354.574155 151949 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.78792
E0000 00:00:1744776354.574158 151949 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.28124
E0000 00:00:1744776354.574160 151949 buffer_comparator.cc:156] Difference at 4137: -nan, expected 3.05469
2025-04-16 04:05:54.574165: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.577069 151949 buffer_comparator.cc:156] Difference at 4128: -nan, expected 1.89634
E0000 00:00:1744776354.577080 151949 buffer_comparator.cc:156] Difference at 4129: -nan, expected 3.44874
E0000 00:00:1744776354.577084 151949 buffer_comparator.cc:156] Difference at 4130: -nan, expected 3.00624
E0000 00:00:1744776354.577087 151949 buffer_comparator.cc:156] Difference at 4131: -nan, expected 4.18145
E0000 00:00:1744776354.577089 151949 buffer_comparator.cc:156] Difference at 4132: -nan, expected 2.41143
E0000 00:00:1744776354.577092 151949 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.24339
E0000 00:00:1744776354.577095 151949 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.63852
E0000 00:00:1744776354.577098 151949 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.78792
E0000 00:00:1744776354.577100 151949 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.28124
E0000 00:00:1744776354.577103 151949 buffer_comparator.cc:156] Difference at 4137: -nan, expected 3.05469
2025-04-16 04:05:54.577108: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.580193 151949 buffer_comparator.cc:156] Difference at 4128: -nan, expected 1.89634
E0000 00:00:1744776354.580204 151949 buffer_comparator.cc:156] Difference at 4129: -nan, expected 3.44874
E0000 00:00:1744776354.580208 151949 buffer_comparator.cc:156] Difference at 4130: -nan, expected 3.00624
E0000 00:00:1744776354.580211 151949 buffer_comparator.cc:156] Difference at 4131: -nan, expected 4.18145
E0000 00:00:1744776354.580213 151949 buffer_comparator.cc:156] Difference at 4132: -nan, expected 2.41143
E0000 00:00:1744776354.580216 151949 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.24339
E0000 00:00:1744776354.580219 151949 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.63852
E0000 00:00:1744776354.580223 151949 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.78792
E0000 00:00:1744776354.580226 151949 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.28124
E0000 00:00:1744776354.580229 151949 buffer_comparator.cc:156] Difference at 4137: -nan, expected 3.05469
2025-04-16 04:05:54.580234: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.583123 151949 buffer_comparator.cc:156] Difference at 4160: -nan, expected 2.85659
E0000 00:00:1744776354.583135 151949 buffer_comparator.cc:156] Difference at 4161: -nan, expected 3.05156
E0000 00:00:1744776354.583138 151949 buffer_comparator.cc:156] Difference at 4162: -nan, expected 1.92304
E0000 00:00:1744776354.583141 151949 buffer_comparator.cc:156] Difference at 4163: -nan, expected 2.29572
E0000 00:00:1744776354.583144 151949 buffer_comparator.cc:156] Difference at 4164: -nan, expected 2.87811
E0000 00:00:1744776354.583147 151949 buffer_comparator.cc:156] Difference at 4165: -nan, expected 2.18636
E0000 00:00:1744776354.583149 151949 buffer_comparator.cc:156] Difference at 4166: -nan, expected 3.87149
E0000 00:00:1744776354.583152 151949 buffer_comparator.cc:156] Difference at 4167: -nan, expected 2.91653
E0000 00:00:1744776354.583155 151949 buffer_comparator.cc:156] Difference at 4168: -nan, expected 3.14416
E0000 00:00:1744776354.583157 151949 buffer_comparator.cc:156] Difference at 4169: -nan, expected 3.40267
2025-04-16 04:05:54.583162: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.586071 151949 buffer_comparator.cc:156] Difference at 8192: -nan, expected 3.30245
E0000 00:00:1744776354.586082 151949 buffer_comparator.cc:156] Difference at 8193: -nan, expected 2.31814
E0000 00:00:1744776354.586086 151949 buffer_comparator.cc:156] Difference at 8194: -nan, expected 2.06803
E0000 00:00:1744776354.586089 151949 buffer_comparator.cc:156] Difference at 8195: -nan, expected 2.58822
E0000 00:00:1744776354.586092 151949 buffer_comparator.cc:156] Difference at 8196: -nan, expected 2.05602
E0000 00:00:1744776354.586094 151949 buffer_comparator.cc:156] Difference at 8197: -nan, expected 2.65508
E0000 00:00:1744776354.586097 151949 buffer_comparator.cc:156] Difference at 8198: -nan, expected 2.08415
E0000 00:00:1744776354.586100 151949 buffer_comparator.cc:156] Difference at 8199: -nan, expected 1.49923
E0000 00:00:1744776354.586102 151949 buffer_comparator.cc:156] Difference at 8200: -nan, expected 1.93301
E0000 00:00:1744776354.586105 151949 buffer_comparator.cc:156] Difference at 8201: -nan, expected 2.54077
2025-04-16 04:05:54.586110: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.589008 151949 buffer_comparator.cc:156] Difference at 8224: -nan, expected 1.54037
E0000 00:00:1744776354.589020 151949 buffer_comparator.cc:156] Difference at 8225: -nan, expected 3.09375
E0000 00:00:1744776354.589023 151949 buffer_comparator.cc:156] Difference at 8226: -nan, expected 2.52357
E0000 00:00:1744776354.589026 151949 buffer_comparator.cc:156] Difference at 8227: -nan, expected 3.50565
E0000 00:00:1744776354.589029 151949 buffer_comparator.cc:156] Difference at 8228: -nan, expected 2.15432
E0000 00:00:1744776354.589032 151949 buffer_comparator.cc:156] Difference at 8229: -nan, expected 2.41339
E0000 00:00:1744776354.589035 151949 buffer_comparator.cc:156] Difference at 8230: -nan, expected 2.09712
E0000 00:00:1744776354.589037 151949 buffer_comparator.cc:156] Difference at 8231: -nan, expected 2.18512
E0000 00:00:1744776354.589040 151949 buffer_comparator.cc:156] Difference at 8232: -nan, expected 1.76097
E0000 00:00:1744776354.589043 151949 buffer_comparator.cc:156] Difference at 8233: -nan, expected 2.34615
2025-04-16 04:05:54.589049: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.591950 151949 buffer_comparator.cc:156] Difference at 8256: -nan, expected 2.44267
E0000 00:00:1744776354.591961 151949 buffer_comparator.cc:156] Difference at 8257: -nan, expected 2.40642
E0000 00:00:1744776354.591965 151949 buffer_comparator.cc:156] Difference at 8258: -nan, expected 1.67472
E0000 00:00:1744776354.591968 151949 buffer_comparator.cc:156] Difference at 8259: -nan, expected 1.80386
E0000 00:00:1744776354.591971 151949 buffer_comparator.cc:156] Difference at 8260: -nan, expected 2.40018
E0000 00:00:1744776354.591973 151949 buffer_comparator.cc:156] Difference at 8261: -nan, expected 1.56439
E0000 00:00:1744776354.591976 151949 buffer_comparator.cc:156] Difference at 8262: -nan, expected 2.89522
E0000 00:00:1744776354.591979 151949 buffer_comparator.cc:156] Difference at 8263: -nan, expected 2.47338
E0000 00:00:1744776354.591982 151949 buffer_comparator.cc:156] Difference at 8264: -nan, expected 2.87187
E0000 00:00:1744776354.591984 151949 buffer_comparator.cc:156] Difference at 8265: -nan, expected 2.84699
2025-04-16 04:05:54.591989: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.594958 151949 buffer_comparator.cc:156] Difference at 16384: -nan, expected 2.42136
E0000 00:00:1744776354.594969 151949 buffer_comparator.cc:156] Difference at 16385: -nan, expected 2.7064
E0000 00:00:1744776354.594973 151949 buffer_comparator.cc:156] Difference at 16386: -nan, expected 1.98959
E0000 00:00:1744776354.594976 151949 buffer_comparator.cc:156] Difference at 16387: -nan, expected 2.13567
E0000 00:00:1744776354.594979 151949 buffer_comparator.cc:156] Difference at 16388: -nan, expected 1.89129
E0000 00:00:1744776354.594981 151949 buffer_comparator.cc:156] Difference at 16389: -nan, expected 2.3481
E0000 00:00:1744776354.594984 151949 buffer_comparator.cc:156] Difference at 16390: -nan, expected 1.58654
E0000 00:00:1744776354.594987 151949 buffer_comparator.cc:156] Difference at 16391: -nan, expected 1.65543
E0000 00:00:1744776354.594989 151949 buffer_comparator.cc:156] Difference at 16392: -nan, expected 1.13108
E0000 00:00:1744776354.594992 151949 buffer_comparator.cc:156] Difference at 16393: -nan, expected 1.974
2025-04-16 04:05:54.594997: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.597960 151949 buffer_comparator.cc:156] Difference at 16384: -nan, expected 2.42136
E0000 00:00:1744776354.597971 151949 buffer_comparator.cc:156] Difference at 16385: -nan, expected 2.7064
E0000 00:00:1744776354.597975 151949 buffer_comparator.cc:156] Difference at 16386: -nan, expected 1.98959
E0000 00:00:1744776354.597978 151949 buffer_comparator.cc:156] Difference at 16387: -nan, expected 2.13567
E0000 00:00:1744776354.597980 151949 buffer_comparator.cc:156] Difference at 16388: -nan, expected 1.89129
E0000 00:00:1744776354.597983 151949 buffer_comparator.cc:156] Difference at 16389: -nan, expected 2.3481
E0000 00:00:1744776354.597986 151949 buffer_comparator.cc:156] Difference at 16390: -nan, expected 1.58654
E0000 00:00:1744776354.597989 151949 buffer_comparator.cc:156] Difference at 16391: -nan, expected 1.65543
E0000 00:00:1744776354.597991 151949 buffer_comparator.cc:156] Difference at 16392: -nan, expected 1.13108
E0000 00:00:1744776354.597994 151949 buffer_comparator.cc:156] Difference at 16393: -nan, expected 1.974
2025-04-16 04:05:54.597999: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.600944 151949 buffer_comparator.cc:156] Difference at 16448: -nan, expected 2.11561
E0000 00:00:1744776354.600970 151949 buffer_comparator.cc:156] Difference at 16449: -nan, expected 2.68627
E0000 00:00:1744776354.600974 151949 buffer_comparator.cc:156] Difference at 16450: -nan, expected 1.22243
E0000 00:00:1744776354.600977 151949 buffer_comparator.cc:156] Difference at 16451: -nan, expected 2.23608
E0000 00:00:1744776354.600980 151949 buffer_comparator.cc:156] Difference at 16452: -nan, expected 2.2239
E0000 00:00:1744776354.600982 151949 buffer_comparator.cc:156] Difference at 16453: -nan, expected 2.10314
E0000 00:00:1744776354.600985 151949 buffer_comparator.cc:156] Difference at 16454: -nan, expected 3.03131
E0000 00:00:1744776354.600988 151949 buffer_comparator.cc:156] Difference at 16455: -nan, expected 2.28758
E0000 00:00:1744776354.600991 151949 buffer_comparator.cc:156] Difference at 16456: -nan, expected 1.88236
E0000 00:00:1744776354.600993 151949 buffer_comparator.cc:156] Difference at 16457: -nan, expected 2.34062
2025-04-16 04:05:54.600998: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.604003 151949 buffer_comparator.cc:156] Difference at 32768: -nan, expected 2.21854
E0000 00:00:1744776354.604015 151949 buffer_comparator.cc:156] Difference at 32769: -nan, expected 2.45025
E0000 00:00:1744776354.604018 151949 buffer_comparator.cc:156] Difference at 32770: -nan, expected 1.95889
E0000 00:00:1744776354.604021 151949 buffer_comparator.cc:156] Difference at 32771: -nan, expected 1.83328
E0000 00:00:1744776354.604024 151949 buffer_comparator.cc:156] Difference at 32772: -nan, expected 1.51117
E0000 00:00:1744776354.604027 151949 buffer_comparator.cc:156] Difference at 32773: -nan, expected 2.1262
E0000 00:00:1744776354.604029 151949 buffer_comparator.cc:156] Difference at 32774: -nan, expected 1.50435
E0000 00:00:1744776354.604032 151949 buffer_comparator.cc:156] Difference at 32775: -nan, expected 1.66723
E0000 00:00:1744776354.604035 151949 buffer_comparator.cc:156] Difference at 32776: -nan, expected 1.5487
E0000 00:00:1744776354.604038 151949 buffer_comparator.cc:156] Difference at 32777: -nan, expected 1.95205
2025-04-16 04:05:54.604042: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.623712 151949 buffer_comparator.cc:156] Difference at 16: 2.37843, expected 33.6617
E0000 00:00:1744776354.623732 151949 buffer_comparator.cc:156] Difference at 17: 2.03294, expected 32.8116
E0000 00:00:1744776354.623739 151949 buffer_comparator.cc:156] Difference at 18: 1.81258, expected 33.3818
E0000 00:00:1744776354.623743 151949 buffer_comparator.cc:156] Difference at 19: 1.88877, expected 29.6073
E0000 00:00:1744776354.623748 151949 buffer_comparator.cc:156] Difference at 20: 1.82906, expected 32.4568
E0000 00:00:1744776354.623752 151949 buffer_comparator.cc:156] Difference at 21: 2.60748, expected 34.803
E0000 00:00:1744776354.623756 151949 buffer_comparator.cc:156] Difference at 22: 1.56132, expected 29.9968
E0000 00:00:1744776354.623760 151949 buffer_comparator.cc:156] Difference at 23: 2.04635, expected 31.9787
E0000 00:00:1744776354.623765 151949 buffer_comparator.cc:156] Difference at 24: 1.78507, expected 33.4708
E0000 00:00:1744776354.623769 151949 buffer_comparator.cc:156] Difference at 25: 2.05368, expected 30.1226
2025-04-16 04:05:54.623776: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.626071 151949 buffer_comparator.cc:156] Difference at 16: 2.37843, expected 33.6617
E0000 00:00:1744776354.626088 151949 buffer_comparator.cc:156] Difference at 17: 2.03294, expected 32.8116
E0000 00:00:1744776354.626095 151949 buffer_comparator.cc:156] Difference at 18: 1.81258, expected 33.3818
E0000 00:00:1744776354.626102 151949 buffer_comparator.cc:156] Difference at 19: 1.88877, expected 29.6073
E0000 00:00:1744776354.626106 151949 buffer_comparator.cc:156] Difference at 20: 1.82906, expected 32.4568
E0000 00:00:1744776354.626111 151949 buffer_comparator.cc:156] Difference at 21: 2.60748, expected 34.803
E0000 00:00:1744776354.626115 151949 buffer_comparator.cc:156] Difference at 22: 1.56132, expected 29.9968
E0000 00:00:1744776354.626120 151949 buffer_comparator.cc:156] Difference at 23: 2.04635, expected 31.9787
E0000 00:00:1744776354.626124 151949 buffer_comparator.cc:156] Difference at 24: 1.78507, expected 33.4708
E0000 00:00:1744776354.626129 151949 buffer_comparator.cc:156] Difference at 25: 2.05368, expected 30.1226
2025-04-16 04:05:54.626136: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.628421 151949 buffer_comparator.cc:156] Difference at 16: 2.37843, expected 33.6617
E0000 00:00:1744776354.628437 151949 buffer_comparator.cc:156] Difference at 17: 2.03294, expected 32.8116
E0000 00:00:1744776354.628442 151949 buffer_comparator.cc:156] Difference at 18: 1.81258, expected 33.3818
E0000 00:00:1744776354.628447 151949 buffer_comparator.cc:156] Difference at 19: 1.88877, expected 29.6073
E0000 00:00:1744776354.628451 151949 buffer_comparator.cc:156] Difference at 20: 1.82906, expected 32.4568
E0000 00:00:1744776354.628455 151949 buffer_comparator.cc:156] Difference at 21: 2.60748, expected 34.803
E0000 00:00:1744776354.628459 151949 buffer_comparator.cc:156] Difference at 22: 1.56132, expected 29.9968
E0000 00:00:1744776354.628463 151949 buffer_comparator.cc:156] Difference at 23: 2.04635, expected 31.9787
E0000 00:00:1744776354.628467 151949 buffer_comparator.cc:156] Difference at 24: 1.78507, expected 33.4708
E0000 00:00:1744776354.628472 151949 buffer_comparator.cc:156] Difference at 25: 2.05368, expected 30.1226
2025-04-16 04:05:54.628478: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.630746 151949 buffer_comparator.cc:156] Difference at 32: 1.33492, expected 32.5446
E0000 00:00:1744776354.630762 151949 buffer_comparator.cc:156] Difference at 33: 2.11146, expected 29.9209
E0000 00:00:1744776354.630768 151949 buffer_comparator.cc:156] Difference at 34: 1.96094, expected 34.9027
E0000 00:00:1744776354.630772 151949 buffer_comparator.cc:156] Difference at 35: 2.66507, expected 30.7275
E0000 00:00:1744776354.630776 151949 buffer_comparator.cc:156] Difference at 36: 2.10381, expected 27.0867
E0000 00:00:1744776354.630781 151949 buffer_comparator.cc:156] Difference at 37: 1.78908, expected 32.4467
E0000 00:00:1744776354.630785 151949 buffer_comparator.cc:156] Difference at 38: 2.11969, expected 33.3911
E0000 00:00:1744776354.630789 151949 buffer_comparator.cc:156] Difference at 39: 2.02591, expected 30.7366
E0000 00:00:1744776354.630793 151949 buffer_comparator.cc:156] Difference at 40: 1.48234, expected 34.7622
E0000 00:00:1744776354.630798 151949 buffer_comparator.cc:156] Difference at 41: 2.06967, expected 31.7114
2025-04-16 04:05:54.630804: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.633065 151949 buffer_comparator.cc:156] Difference at 32: 1.33492, expected 32.5446
E0000 00:00:1744776354.633081 151949 buffer_comparator.cc:156] Difference at 33: 2.11146, expected 29.9209
E0000 00:00:1744776354.633087 151949 buffer_comparator.cc:156] Difference at 34: 1.96094, expected 34.9027
E0000 00:00:1744776354.633092 151949 buffer_comparator.cc:156] Difference at 35: 2.66507, expected 30.7275
E0000 00:00:1744776354.633097 151949 buffer_comparator.cc:156] Difference at 36: 2.10381, expected 27.0867
E0000 00:00:1744776354.633103 151949 buffer_comparator.cc:156] Difference at 37: 1.78908, expected 32.4467
E0000 00:00:1744776354.633108 151949 buffer_comparator.cc:156] Difference at 38: 2.11969, expected 33.3911
E0000 00:00:1744776354.633113 151949 buffer_comparator.cc:156] Difference at 39: 2.02591, expected 30.7366
E0000 00:00:1744776354.633117 151949 buffer_comparator.cc:156] Difference at 40: 1.48234, expected 34.7622
E0000 00:00:1744776354.633121 151949 buffer_comparator.cc:156] Difference at 41: 2.06967, expected 31.7114
2025-04-16 04:05:54.633128: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.635389 151949 buffer_comparator.cc:156] Difference at 32: 1.33492, expected 32.5446
E0000 00:00:1744776354.635401 151949 buffer_comparator.cc:156] Difference at 33: 2.11146, expected 29.9209
E0000 00:00:1744776354.635405 151949 buffer_comparator.cc:156] Difference at 34: 1.96094, expected 34.9027
E0000 00:00:1744776354.635408 151949 buffer_comparator.cc:156] Difference at 35: 2.66507, expected 30.7275
E0000 00:00:1744776354.635411 151949 buffer_comparator.cc:156] Difference at 36: 2.10381, expected 27.0867
E0000 00:00:1744776354.635414 151949 buffer_comparator.cc:156] Difference at 37: 1.78908, expected 32.4467
E0000 00:00:1744776354.635417 151949 buffer_comparator.cc:156] Difference at 38: 2.11969, expected 33.3911
E0000 00:00:1744776354.635420 151949 buffer_comparator.cc:156] Difference at 39: 2.02591, expected 30.7366
E0000 00:00:1744776354.635423 151949 buffer_comparator.cc:156] Difference at 40: 1.48234, expected 34.7622
E0000 00:00:1744776354.635426 151949 buffer_comparator.cc:156] Difference at 41: 2.06967, expected 31.7114
2025-04-16 04:05:54.635431: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.637589 151949 buffer_comparator.cc:156] Difference at 64: 1.48598, expected 30.5369
E0000 00:00:1744776354.637601 151949 buffer_comparator.cc:156] Difference at 65: 2.07857, expected 32.0282
E0000 00:00:1744776354.637605 151949 buffer_comparator.cc:156] Difference at 66: 1.26324, expected 31.1887
E0000 00:00:1744776354.637608 151949 buffer_comparator.cc:156] Difference at 67: 1.68463, expected 33.8384
E0000 00:00:1744776354.637611 151949 buffer_comparator.cc:156] Difference at 68: 1.99749, expected 29.9969
E0000 00:00:1744776354.637614 151949 buffer_comparator.cc:156] Difference at 69: 1.67603, expected 30.1031
E0000 00:00:1744776354.637617 151949 buffer_comparator.cc:156] Difference at 70: 2.47978, expected 33.9939
E0000 00:00:1744776354.637620 151949 buffer_comparator.cc:156] Difference at 71: 1.62348, expected 32.8713
E0000 00:00:1744776354.637623 151949 buffer_comparator.cc:156] Difference at 72: 1.64724, expected 31.3445
E0000 00:00:1744776354.637626 151949 buffer_comparator.cc:156] Difference at 73: 2.03763, expected 30.0438
2025-04-16 04:05:54.637631: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.639820 151949 buffer_comparator.cc:156] Difference at 5: 33.9431, expected 29.9465
E0000 00:00:1744776354.639833 151949 buffer_comparator.cc:156] Difference at 35: 34.5119, expected 30.7275
E0000 00:00:1744776354.639837 151949 buffer_comparator.cc:156] Difference at 45: 35.7689, expected 31.9191
E0000 00:00:1744776354.639840 151949 buffer_comparator.cc:156] Difference at 56: 34.1441, expected 30.5837
E0000 00:00:1744776354.639844 151949 buffer_comparator.cc:156] Difference at 57: 36.2798, expected 31.843
E0000 00:00:1744776354.639847 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 30.5369
E0000 00:00:1744776354.639850 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 32.0282
E0000 00:00:1744776354.639855 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 31.1887
E0000 00:00:1744776354.639858 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 33.8384
E0000 00:00:1744776354.639861 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 29.9969
2025-04-16 04:05:54.639865: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.642029 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 30.5369
E0000 00:00:1744776354.642040 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 32.0282
E0000 00:00:1744776354.642044 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 31.1887
E0000 00:00:1744776354.642047 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 33.8384
E0000 00:00:1744776354.642055 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 29.9969
E0000 00:00:1744776354.642058 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 30.1031
E0000 00:00:1744776354.642061 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 33.9939
E0000 00:00:1744776354.642064 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 32.8713
E0000 00:00:1744776354.642067 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 31.3445
E0000 00:00:1744776354.642070 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 30.0438
2025-04-16 04:05:54.642074: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.644240 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 30.5369
E0000 00:00:1744776354.644251 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 32.0282
E0000 00:00:1744776354.644255 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 31.1887
E0000 00:00:1744776354.644258 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 33.8384
E0000 00:00:1744776354.644261 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 29.9969
E0000 00:00:1744776354.644264 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 30.1031
E0000 00:00:1744776354.644267 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 33.9939
E0000 00:00:1744776354.644269 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 32.8713
E0000 00:00:1744776354.644272 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 31.3445
E0000 00:00:1744776354.644275 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 30.0438
2025-04-16 04:05:54.644280: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.646455 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 30.5369
E0000 00:00:1744776354.646466 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 32.0282
E0000 00:00:1744776354.646470 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 31.1887
E0000 00:00:1744776354.646473 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 33.8384
E0000 00:00:1744776354.646476 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 29.9969
E0000 00:00:1744776354.646479 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 30.1031
E0000 00:00:1744776354.646482 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 33.9939
E0000 00:00:1744776354.646485 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 32.8713
E0000 00:00:1744776354.646488 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 31.3445
E0000 00:00:1744776354.646490 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 30.0438
2025-04-16 04:05:54.646497: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.648659 151949 buffer_comparator.cc:156] Difference at 64: nan, expected 30.5369
E0000 00:00:1744776354.648670 151949 buffer_comparator.cc:156] Difference at 65: nan, expected 32.0282
E0000 00:00:1744776354.648674 151949 buffer_comparator.cc:156] Difference at 66: nan, expected 31.1887
E0000 00:00:1744776354.648677 151949 buffer_comparator.cc:156] Difference at 67: nan, expected 33.8384
E0000 00:00:1744776354.648680 151949 buffer_comparator.cc:156] Difference at 68: nan, expected 29.9969
E0000 00:00:1744776354.648683 151949 buffer_comparator.cc:156] Difference at 69: nan, expected 30.1031
E0000 00:00:1744776354.648686 151949 buffer_comparator.cc:156] Difference at 70: nan, expected 33.9939
E0000 00:00:1744776354.648689 151949 buffer_comparator.cc:156] Difference at 71: nan, expected 32.8713
E0000 00:00:1744776354.648692 151949 buffer_comparator.cc:156] Difference at 72: nan, expected 31.3445
E0000 00:00:1744776354.648694 151949 buffer_comparator.cc:156] Difference at 73: nan, expected 30.0438
2025-04-16 04:05:54.648699: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.650852 151949 buffer_comparator.cc:156] Difference at 128: nan, expected 29.2775
E0000 00:00:1744776354.650864 151949 buffer_comparator.cc:156] Difference at 129: nan, expected 31.6099
E0000 00:00:1744776354.650868 151949 buffer_comparator.cc:156] Difference at 130: nan, expected 33.0989
E0000 00:00:1744776354.650871 151949 buffer_comparator.cc:156] Difference at 131: nan, expected 31.0273
E0000 00:00:1744776354.650874 151949 buffer_comparator.cc:156] Difference at 132: nan, expected 31.3552
E0000 00:00:1744776354.650877 151949 buffer_comparator.cc:156] Difference at 133: nan, expected 32.8032
E0000 00:00:1744776354.650880 151949 buffer_comparator.cc:156] Difference at 134: nan, expected 27.0109
E0000 00:00:1744776354.650883 151949 buffer_comparator.cc:156] Difference at 135: nan, expected 30.9195
E0000 00:00:1744776354.650886 151949 buffer_comparator.cc:156] Difference at 136: nan, expected 33.0022
E0000 00:00:1744776354.650889 151949 buffer_comparator.cc:156] Difference at 137: nan, expected 29.5814
2025-04-16 04:05:54.650893: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.653057 151949 buffer_comparator.cc:156] Difference at 128: nan, expected 29.2775
E0000 00:00:1744776354.653069 151949 buffer_comparator.cc:156] Difference at 129: nan, expected 31.6099
E0000 00:00:1744776354.653072 151949 buffer_comparator.cc:156] Difference at 130: nan, expected 33.0989
E0000 00:00:1744776354.653075 151949 buffer_comparator.cc:156] Difference at 131: nan, expected 31.0273
E0000 00:00:1744776354.653078 151949 buffer_comparator.cc:156] Difference at 132: nan, expected 31.3552
E0000 00:00:1744776354.653081 151949 buffer_comparator.cc:156] Difference at 133: nan, expected 32.8032
E0000 00:00:1744776354.653084 151949 buffer_comparator.cc:156] Difference at 134: nan, expected 27.0109
E0000 00:00:1744776354.653087 151949 buffer_comparator.cc:156] Difference at 135: nan, expected 30.9195
E0000 00:00:1744776354.653090 151949 buffer_comparator.cc:156] Difference at 136: nan, expected 33.0022
E0000 00:00:1744776354.653093 151949 buffer_comparator.cc:156] Difference at 137: nan, expected 29.5814
2025-04-16 04:05:54.653098: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.655276 151949 buffer_comparator.cc:156] Difference at 128: nan, expected 29.2775
E0000 00:00:1744776354.655289 151949 buffer_comparator.cc:156] Difference at 129: nan, expected 31.6099
E0000 00:00:1744776354.655293 151949 buffer_comparator.cc:156] Difference at 130: nan, expected 33.0989
E0000 00:00:1744776354.655296 151949 buffer_comparator.cc:156] Difference at 131: nan, expected 31.0273
E0000 00:00:1744776354.655299 151949 buffer_comparator.cc:156] Difference at 132: nan, expected 31.3552
E0000 00:00:1744776354.655302 151949 buffer_comparator.cc:156] Difference at 133: nan, expected 32.8032
E0000 00:00:1744776354.655305 151949 buffer_comparator.cc:156] Difference at 134: nan, expected 27.0109
E0000 00:00:1744776354.655307 151949 buffer_comparator.cc:156] Difference at 135: nan, expected 30.9195
E0000 00:00:1744776354.655310 151949 buffer_comparator.cc:156] Difference at 136: nan, expected 33.0022
E0000 00:00:1744776354.655313 151949 buffer_comparator.cc:156] Difference at 137: nan, expected 29.5814
2025-04-16 04:05:54.655318: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.657518 151949 buffer_comparator.cc:156] Difference at 1: 28.3852, expected 32.3281
E0000 00:00:1744776354.657529 151949 buffer_comparator.cc:156] Difference at 2: 24.9609, expected 31.1329
E0000 00:00:1744776354.657534 151949 buffer_comparator.cc:156] Difference at 3: 29.3608, expected 35.2904
E0000 00:00:1744776354.657537 151949 buffer_comparator.cc:156] Difference at 5: 24.7285, expected 29.9465
E0000 00:00:1744776354.657540 151949 buffer_comparator.cc:156] Difference at 6: 26.7573, expected 30.7693
E0000 00:00:1744776354.657543 151949 buffer_comparator.cc:156] Difference at 8: 23.0127, expected 29.9683
E0000 00:00:1744776354.657546 151949 buffer_comparator.cc:156] Difference at 9: 28.8364, expected 32.4294
E0000 00:00:1744776354.657549 151949 buffer_comparator.cc:156] Difference at 15: 25.0733, expected 28.9349
E0000 00:00:1744776354.657553 151949 buffer_comparator.cc:156] Difference at 18: 27.5223, expected 33.3818
E0000 00:00:1744776354.657556 151949 buffer_comparator.cc:156] Difference at 30: 22.8423, expected 29.9586
2025-04-16 04:05:54.657561: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.659749 151949 buffer_comparator.cc:156] Difference at 128: nan, expected 29.2775
E0000 00:00:1744776354.659761 151949 buffer_comparator.cc:156] Difference at 129: nan, expected 31.6099
E0000 00:00:1744776354.659765 151949 buffer_comparator.cc:156] Difference at 130: nan, expected 33.0989
E0000 00:00:1744776354.659768 151949 buffer_comparator.cc:156] Difference at 131: nan, expected 31.0273
E0000 00:00:1744776354.659771 151949 buffer_comparator.cc:156] Difference at 132: nan, expected 31.3552
E0000 00:00:1744776354.659774 151949 buffer_comparator.cc:156] Difference at 133: nan, expected 32.8032
E0000 00:00:1744776354.659777 151949 buffer_comparator.cc:156] Difference at 134: nan, expected 27.0109
E0000 00:00:1744776354.659780 151949 buffer_comparator.cc:156] Difference at 135: nan, expected 30.9195
E0000 00:00:1744776354.659783 151949 buffer_comparator.cc:156] Difference at 136: nan, expected 33.0022
E0000 00:00:1744776354.659785 151949 buffer_comparator.cc:156] Difference at 137: nan, expected 29.5814
2025-04-16 04:05:54.659790: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.661974 151949 buffer_comparator.cc:156] Difference at 256: nan, expected 29.5007
E0000 00:00:1744776354.661985 151949 buffer_comparator.cc:156] Difference at 257: nan, expected 30.0471
E0000 00:00:1744776354.661989 151949 buffer_comparator.cc:156] Difference at 258: nan, expected 28.5151
E0000 00:00:1744776354.661992 151949 buffer_comparator.cc:156] Difference at 259: nan, expected 30.821
E0000 00:00:1744776354.661997 151949 buffer_comparator.cc:156] Difference at 260: nan, expected 32.8143
E0000 00:00:1744776354.662000 151949 buffer_comparator.cc:156] Difference at 261: nan, expected 31.8055
E0000 00:00:1744776354.662003 151949 buffer_comparator.cc:156] Difference at 262: nan, expected 29.1327
E0000 00:00:1744776354.662006 151949 buffer_comparator.cc:156] Difference at 263: nan, expected 31.8548
E0000 00:00:1744776354.662008 151949 buffer_comparator.cc:156] Difference at 264: nan, expected 30.0785
E0000 00:00:1744776354.662011 151949 buffer_comparator.cc:156] Difference at 265: nan, expected 31.9735
2025-04-16 04:05:54.662016: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.664217 151949 buffer_comparator.cc:156] Difference at 256: nan, expected 29.5007
E0000 00:00:1744776354.664229 151949 buffer_comparator.cc:156] Difference at 257: nan, expected 30.0471
E0000 00:00:1744776354.664233 151949 buffer_comparator.cc:156] Difference at 258: nan, expected 28.5151
E0000 00:00:1744776354.664236 151949 buffer_comparator.cc:156] Difference at 259: nan, expected 30.821
E0000 00:00:1744776354.664239 151949 buffer_comparator.cc:156] Difference at 260: nan, expected 32.8143
E0000 00:00:1744776354.664242 151949 buffer_comparator.cc:156] Difference at 261: nan, expected 31.8055
E0000 00:00:1744776354.664245 151949 buffer_comparator.cc:156] Difference at 262: nan, expected 29.1327
E0000 00:00:1744776354.664247 151949 buffer_comparator.cc:156] Difference at 263: nan, expected 31.8548
E0000 00:00:1744776354.664250 151949 buffer_comparator.cc:156] Difference at 264: nan, expected 30.0785
E0000 00:00:1744776354.664253 151949 buffer_comparator.cc:156] Difference at 265: nan, expected 31.9735
2025-04-16 04:05:54.664258: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.666448 151949 buffer_comparator.cc:156] Difference at 256: nan, expected 29.5007
E0000 00:00:1744776354.666459 151949 buffer_comparator.cc:156] Difference at 257: nan, expected 30.0471
E0000 00:00:1744776354.666463 151949 buffer_comparator.cc:156] Difference at 258: nan, expected 28.5151
E0000 00:00:1744776354.666466 151949 buffer_comparator.cc:156] Difference at 259: nan, expected 30.821
E0000 00:00:1744776354.666469 151949 buffer_comparator.cc:156] Difference at 260: nan, expected 32.8143
E0000 00:00:1744776354.666472 151949 buffer_comparator.cc:156] Difference at 261: nan, expected 31.8055
E0000 00:00:1744776354.666475 151949 buffer_comparator.cc:156] Difference at 262: nan, expected 29.1327
E0000 00:00:1744776354.666478 151949 buffer_comparator.cc:156] Difference at 263: nan, expected 31.8548
E0000 00:00:1744776354.666481 151949 buffer_comparator.cc:156] Difference at 264: nan, expected 30.0785
E0000 00:00:1744776354.666484 151949 buffer_comparator.cc:156] Difference at 265: nan, expected 31.9735
2025-04-16 04:05:54.666488: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1744776354.668977 151949 buffer_comparator.cc:156] Difference at 16: nan, expected 1022.04
E0000 00:00:1744776354.668989 151949 buffer_comparator.cc:156] Difference at 17: nan, expected 1034.86
E0000 00:00:1744776354.668993 151949 buffer_comparator.cc:156] Difference at 18: nan, expected 1026.77
E0000 00:00:1744776354.668996 151949 buffer_comparator.cc:156] Difference at 19: nan, expected 1046.95
E0000 00:00:1744776354.668999 151949 buffer_comparator.cc:156] Difference at 20: nan, expected 1032.02
E0000 00:00:1744776354.669002 151949 buffer_comparator.cc:156] Difference at 21: nan, expected 1015.56
Epoch 1, Iter 11, Loss: 61748.1796875, Throughput: 16.288409 im/s
Epoch 1, Train Loss: 75623.9687500, Time: 86.8414s, Throughput: 16.213467 im/s
Epoch 2, Iter 11, Loss: 54222.2734375, Throughput: 1745.471171 im/s
Epoch 2, Train Loss: 56387.1132812, Time: 0.8070s, Throughput: 1744.699215 im/s
Epoch 3, Iter 11, Loss: 50404.4453125, Throughput: 1746.651308 im/s
Epoch 3, Train Loss: 52353.8867188, Time: 0.8065s, Throughput: 1745.914955 im/s
Epoch 4, Iter 11, Loss: 53238.5859375, Throughput: 1759.155507 im/s
Epoch 4, Train Loss: 52346.3125000, Time: 0.8007s, Throughput: 1758.391298 im/s
Epoch 5, Iter 11, Loss: 47479.6796875, Throughput: 1762.566651 im/s
Epoch 5, Train Loss: 51204.1250000, Time: 0.7992s, Throughput: 1761.810513 im/s
Epoch 6, Iter 11, Loss: 46805.1445312, Throughput: 1765.326953 im/s
Epoch 6, Train Loss: 49098.9531250, Time: 0.7979s, Throughput: 1764.527848 im/s
Epoch 7, Iter 11, Loss: 48482.8320312, Throughput: 1764.010794 im/s
Epoch 7, Train Loss: 47691.8242188, Time: 0.7985s, Throughput: 1763.292902 im/s
Epoch 8, Iter 11, Loss: 48303.0351562, Throughput: 1762.180614 im/s
Epoch 8, Train Loss: 47924.4140625, Time: 0.7994s, Throughput: 1761.409046 im/s
Epoch 9, Iter 11, Loss: 47113.5390625, Throughput: 1764.685502 im/s
Epoch 9, Train Loss: 49434.6914062, Time: 0.7982s, Throughput: 1763.949147 im/s
Epoch 10, Iter 11, Loss: 51160.1328125, Throughput: 1767.571476 im/s
Epoch 10, Train Loss: 50709.1406250, Time: 0.7969s, Throughput: 1766.857556 im/s
Epoch 11, Iter 11, Loss: 48083.2343750, Throughput: 1751.239323 im/s
Epoch 11, Train Loss: 51386.0976562, Time: 0.8043s, Throughput: 1750.529711 im/s
Epoch 12, Iter 11, Loss: 51323.5351562, Throughput: 1765.740767 im/s
Epoch 12, Train Loss: 51336.9101562, Time: 0.7977s, Throughput: 1764.997202 im/s
Epoch 13, Iter 11, Loss: 50187.0117188, Throughput: 1764.384455 im/s
Epoch 13, Train Loss: 50785.3750000, Time: 0.7983s, Throughput: 1763.673107 im/s
Epoch 14, Iter 11, Loss: 51163.8320312, Throughput: 1765.205063 im/s
Epoch 14, Train Loss: 50554.8125000, Time: 0.7980s, Throughput: 1764.510450 im/s
Epoch 15, Iter 11, Loss: 47059.9570312, Throughput: 1764.999312 im/s
Epoch 15, Train Loss: 49766.3906250, Time: 0.7980s, Throughput: 1764.304861 im/s
Epoch 16, Iter 11, Loss: 48550.0468750, Throughput: 1760.237220 im/s
Epoch 16, Train Loss: 49859.3242188, Time: 0.8002s, Throughput: 1759.511387 im/s
Epoch 17, Iter 11, Loss: 50406.3359375, Throughput: 1740.562328 im/s
Epoch 17, Train Loss: 49381.8476562, Time: 0.8093s, Throughput: 1739.788044 im/s
Epoch 18, Iter 11, Loss: 48031.6093750, Throughput: 1764.603772 im/s
Epoch 18, Train Loss: 50179.8984375, Time: 0.7983s, Throughput: 1763.823232 im/s
Epoch 19, Iter 11, Loss: 50312.8125000, Throughput: 1753.156123 im/s
Epoch 19, Train Loss: 50467.4140625, Time: 0.8034s, Throughput: 1752.462118 im/s
Epoch 20, Iter 11, Loss: 49187.8867188, Throughput: 1767.149929 im/s
Epoch 20, Train Loss: 50374.5117188, Time: 0.7971s, Throughput: 1766.409931 im/s
Epoch 21, Iter 11, Loss: 50245.1562500, Throughput: 1766.595930 im/s
Epoch 21, Train Loss: 48894.0664062, Time: 0.7973s, Throughput: 1765.915536 im/s
Epoch 22, Iter 11, Loss: 47582.4101562, Throughput: 1766.206012 im/s
Epoch 22, Train Loss: 47668.1835938, Time: 0.7975s, Throughput: 1765.519585 im/s
Epoch 23, Iter 11, Loss: 47651.3750000, Throughput: 1717.054339 im/s
Epoch 23, Train Loss: 47031.1132812, Time: 0.8204s, Throughput: 1716.221021 im/s
Epoch 24, Iter 11, Loss: 47673.0703125, Throughput: 1764.720833 im/s
Epoch 24, Train Loss: 46980.5585938, Time: 0.7982s, Throughput: 1764.011321 im/s
Epoch 25, Iter 11, Loss: 45814.9414062, Throughput: 1764.592172 im/s
Epoch 25, Train Loss: 46642.9843750, Time: 0.7983s, Throughput: 1763.807428 im/s
Epoch 26, Iter 11, Loss: 45145.2890625, Throughput: 1763.675213 im/s
Epoch 26, Train Loss: 46100.7539062, Time: 0.7987s, Throughput: 1762.922334 im/s
Epoch 27, Iter 11, Loss: 48507.0234375, Throughput: 1747.485490 im/s
Epoch 27, Train Loss: 46227.5234375, Time: 0.8061s, Throughput: 1746.763417 im/s
Epoch 28, Iter 11, Loss: 47332.0859375, Throughput: 1752.464718 im/s
Epoch 28, Train Loss: 47393.5585938, Time: 0.8038s, Throughput: 1751.727613 im/s
Epoch 29, Iter 11, Loss: 48674.9843750, Throughput: 1754.357623 im/s
Epoch 29, Train Loss: 48682.9726562, Time: 0.8029s, Throughput: 1753.669436 im/s
Epoch 30, Iter 11, Loss: 49575.1171875, Throughput: 1763.953362 im/s
Epoch 30, Train Loss: 49717.5156250, Time: 0.7986s, Throughput: 1763.164975 im/s
Epoch 31, Iter 11, Loss: 49916.1523438, Throughput: 1763.474032 im/s
Epoch 31, Train Loss: 49096.3593750, Time: 0.7987s, Throughput: 1762.754472 im/s
Epoch 32, Iter 11, Loss: 44313.8632812, Throughput: 1764.048733 im/s
Epoch 32, Train Loss: 48043.5156250, Time: 0.7985s, Throughput: 1763.308170 im/s
Epoch 33, Iter 11, Loss: 49775.8242188, Throughput: 1746.237098 im/s
Epoch 33, Train Loss: 47734.3007812, Time: 0.8066s, Throughput: 1745.520698 im/s
Epoch 34, Iter 11, Loss: 48231.7578125, Throughput: 1746.235032 im/s
Epoch 34, Train Loss: 47501.4726562, Time: 0.8066s, Throughput: 1745.505737 im/s
Epoch 35, Iter 11, Loss: 44300.6132812, Throughput: 1750.751825 im/s
Epoch 35, Train Loss: 47404.3085938, Time: 0.8045s, Throughput: 1750.076836 im/s
Epoch 36, Iter 11, Loss: 47484.1367188, Throughput: 1765.550727 im/s
Epoch 36, Train Loss: 47710.1289062, Time: 0.7978s, Throughput: 1764.837910 im/s
Epoch 37, Iter 11, Loss: 45540.6171875, Throughput: 1764.643318 im/s
Epoch 37, Train Loss: 47661.6093750, Time: 0.7982s, Throughput: 1763.889612 im/s
Epoch 38, Iter 11, Loss: 47924.8593750, Throughput: 1746.540764 im/s
Epoch 38, Train Loss: 48133.5390625, Time: 0.8065s, Throughput: 1745.830309 im/s
Epoch 39, Iter 11, Loss: 46567.2968750, Throughput: 1766.739682 im/s
Epoch 39, Train Loss: 48197.2734375, Time: 0.7973s, Throughput: 1765.917648 im/s
Epoch 40, Iter 11, Loss: 47002.2656250, Throughput: 1741.102171 im/s
Epoch 40, Train Loss: 48227.9882812, Time: 0.8089s, Throughput: 1740.529497 im/s
Epoch 41, Iter 11, Loss: 48914.1171875, Throughput: 1768.013338 im/s
Epoch 41, Train Loss: 47780.4218750, Time: 0.7966s, Throughput: 1767.414364 im/s
Epoch 42, Iter 11, Loss: 49229.3320312, Throughput: 1736.794317 im/s
Epoch 42, Train Loss: 48551.2656250, Time: 0.8110s, Throughput: 1736.048897 im/s
Epoch 43, Iter 11, Loss: 49029.1015625, Throughput: 1767.764598 im/s
Epoch 43, Train Loss: 49679.6367188, Time: 0.7968s, Throughput: 1767.017212 im/s
Epoch 44, Iter 11, Loss: 49599.3710938, Throughput: 1746.490662 im/s
Epoch 44, Train Loss: 50178.9531250, Time: 0.8065s, Throughput: 1745.881921 im/s
Epoch 45, Iter 11, Loss: 50038.8242188, Throughput: 1754.888849 im/s
Epoch 45, Train Loss: 50510.2890625, Time: 0.8026s, Throughput: 1754.291959 im/s
Epoch 46, Iter 11, Loss: 52251.3945312, Throughput: 1767.827041 im/s
Epoch 46, Train Loss: 50267.4257812, Time: 0.7968s, Throughput: 1767.068499 im/s
Epoch 47, Iter 11, Loss: 52837.6992188, Throughput: 1764.495161 im/s
Epoch 47, Train Loss: 50733.5273438, Time: 0.7983s, Throughput: 1763.765813 im/s
Epoch 48, Iter 11, Loss: 50252.1562500, Throughput: 1765.902862 im/s
Epoch 48, Train Loss: 50524.8710938, Time: 0.7976s, Throughput: 1765.298458 im/s
Epoch 49, Iter 11, Loss: 49137.9101562, Throughput: 1755.418309 im/s
Epoch 49, Train Loss: 49330.2617188, Time: 0.8025s, Throughput: 1754.554644 im/s
Epoch 50, Iter 11, Loss: 47361.1718750, Throughput: 1767.316514 im/s
Epoch 50, Train Loss: 47985.0117188, Time: 0.7971s, Throughput: 1766.465410 im/s
Appendix
julia
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
endJulia Version 1.11.5
Commit 760b2e5b739 (2025-04-14 06:53 UTC)
Build Info:
Official https://julialang.org/ release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 48 × AMD EPYC 7402 24-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-16.0.6 (ORCJIT, znver2)
Threads: 48 default, 0 interactive, 24 GC (on 2 virtual cores)
Environment:
JULIA_CPU_THREADS = 2
LD_LIBRARY_PATH = /usr/local/nvidia/lib:/usr/local/nvidia/lib64
JULIA_PKG_SERVER =
JULIA_NUM_THREADS = 48
JULIA_CUDA_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0
JULIA_DEBUG = Literate
JULIA_DEPOT_PATH = /root/.cache/julia-buildkite-plugin/depots/01872db4-8c79-43af-ab7d-12abac4f24f6This page was generated using Literate.jl.