Convolutional VAE for MNIST using Reactant

Convolutional variational autoencoder (CVAE) implementation in MLX using MNIST. This is based on the CVAE implementation in MLX.

using Lux,
    Reactant,
    MLDatasets,
    Random,
    Statistics,
    Enzyme,
    MLUtils,
    DataAugmentation,
    ConcreteStructs,
    OneHotArrays,
    ImageShow,
    Images,
    Printf,
    Optimisers

const xdev = reactant_device(; force=true)
const cdev = cpu_device()

(::MLDataDevices.CPUDevice) (generic function with 1 method)

Model Definition

First we will define the encoder.It maps the input to a normal distribution in latent space and sample a latent vector from that distribution.

function cvae_encoder(
    rng=Random.default_rng();
    num_latent_dims::Int,
    image_shape::Dims{3},
    max_num_filters::Int,
)
    flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
    return @compact(;
        embed=Chain(
            Chain(
                Conv((3, 3), image_shape[3] => max_num_filters ÷ 4; stride=2, pad=1),
                BatchNorm(max_num_filters ÷ 4, leakyrelu),
            ),
            Chain(
                Conv((3, 3), max_num_filters ÷ 4 => max_num_filters ÷ 2; stride=2, pad=1),
                BatchNorm(max_num_filters ÷ 2, leakyrelu),
            ),
            Chain(
                Conv((3, 3), max_num_filters ÷ 2 => max_num_filters; stride=2, pad=1),
                BatchNorm(max_num_filters, leakyrelu),
            ),
            FlattenLayer(),
        ),
        proj_mu=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
        proj_log_var=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
        rng
    ) do x
        y = embed(x)

        μ = proj_mu(y)
        logσ² = proj_log_var(y)

        T = eltype(logσ²)
        logσ² = clamp.(logσ², -T(20.0f0), T(10.0f0))
        σ = exp.(logσ² .* T(0.5))

        # Generate a tensor of random values from a normal distribution
        rng = Lux.replicate(rng)
        ϵ = randn_like(rng, σ)

        # Reparameterization trick to brackpropagate through sampling
        z = ϵ .* σ .+ μ

        @return z, μ, logσ²
    end
end

cvae_encoder (generic function with 2 methods)

Similarly we define the decoder.

function cvae_decoder(; num_latent_dims::Int, image_shape::Dims{3}, max_num_filters::Int)
    flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
    return @compact(;
        linear=Dense(num_latent_dims, flattened_dim),
        upchain=Chain(
            Chain(
                Upsample(2),
                Conv((3, 3), max_num_filters => max_num_filters ÷ 2; stride=1, pad=1),
                BatchNorm(max_num_filters ÷ 2, leakyrelu),
            ),
            Chain(
                Upsample(2),
                Conv((3, 3), max_num_filters ÷ 2 => max_num_filters ÷ 4; stride=1, pad=1),
                BatchNorm(max_num_filters ÷ 4, leakyrelu),
            ),
            Chain(
                Upsample(2),
                Conv(
                    (3, 3), max_num_filters ÷ 4 => image_shape[3], sigmoid; stride=1, pad=1
                ),
            ),
        ),
        max_num_filters
    ) do x
        y = linear(x)
        img = reshape(y, image_shape[1] ÷ 8, image_shape[2] ÷ 8, max_num_filters, :)
        @return upchain(img)
    end
end

@concrete struct CVAE <: Lux.AbstractLuxContainerLayer{(:encoder, :decoder)}
    encoder <: Lux.AbstractLuxLayer
    decoder <: Lux.AbstractLuxLayer
end

function CVAE(
    rng=Random.default_rng();
    num_latent_dims::Int,
    image_shape::Dims{3},
    max_num_filters::Int,
)
    decoder = cvae_decoder(; num_latent_dims, image_shape, max_num_filters)
    encoder = cvae_encoder(rng; num_latent_dims, image_shape, max_num_filters)
    return CVAE(encoder, decoder)
end

function (cvae::CVAE)(x, ps, st)
    (z, μ, logσ²), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
    x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
    return (x_rec, μ, logσ²), (; encoder=st_enc, decoder=st_dec)
end

function encode(cvae::CVAE, x, ps, st)
    (z, _, _), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
    return z, (; encoder=st_enc, st.decoder)
end

function decode(cvae::CVAE, z, ps, st)
    x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
    return x_rec, (; decoder=st_dec, st.encoder)
end

decode (generic function with 1 method)

Loading MNIST

@concrete struct TensorDataset
    dataset
    transform
    total_samples::Int
end

Base.length(ds::TensorDataset) = ds.total_samples

function Base.getindex(ds::TensorDataset, idxs::Union{Vector{<:Integer},AbstractRange})
    img = Image.(eachslice(convert2image(ds.dataset, idxs); dims=3))
    return stack(parent ∘ itemdata ∘ Base.Fix1(apply, ds.transform), img)
end

function loadmnist(batchsize, image_size::Dims{2})
    # Load MNIST: Only 1500 for demonstration purposes on CI
    train_dataset = MNIST(; split=:train)
    N = parse(Bool, get(ENV, "CI", "false")) ? 1500 : length(train_dataset)

    train_transform = ScaleKeepAspect(image_size) |> ImageToTensor()
    trainset = TensorDataset(train_dataset, train_transform, N)
    trainloader = DataLoader(trainset; batchsize, shuffle=true, partial=false)

    return trainloader
end

loadmnist (generic function with 1 method)

Helper Functions

Generate an Image Grid from a list of images

function create_image_grid(imgs::AbstractArray, grid_rows::Int, grid_cols::Int)
    total_images = grid_rows * grid_cols
    imgs = map(eachslice(imgs[:, :, :, 1:total_images]; dims=4)) do img
        cimg = if size(img, 3) == 1
            colorview(Gray, view(img, :, :, 1))
        else
            colorview(RGB, permutedims(img, (3, 1, 2)))
        end
        return cimg'
    end
    return create_image_grid(imgs, grid_rows, grid_cols)
end

function create_image_grid(images::Vector, grid_rows::Int, grid_cols::Int)
    # Check if the number of images matches the grid
    total_images = grid_rows * grid_cols
    @assert length(images) == total_images

    # Get the size of a single image (assuming all images are the same size)
    img_height, img_width = size(images[1])

    # Create a blank grid canvas
    grid_height = img_height * grid_rows
    grid_width = img_width * grid_cols
    grid_canvas = similar(images[1], grid_height, grid_width)

    # Place each image in the correct position on the canvas
    for idx in 1:total_images
        row = div(idx - 1, grid_cols) + 1
        col = mod(idx - 1, grid_cols) + 1

        start_row = (row - 1) * img_height + 1
        start_col = (col - 1) * img_width + 1

        grid_canvas[start_row:(start_row + img_height - 1), start_col:(start_col + img_width - 1)] .= images[idx]
    end

    return grid_canvas
end

function loss_function(model, ps, st, X)
    (y, μ, logσ²), st = model(X, ps, st)
    reconstruction_loss = MSELoss(; agg=sum)(y, X)
    kldiv_loss = -sum(1 .+ logσ² .- μ .^ 2 .- exp.(logσ²)) / 2
    loss = reconstruction_loss + kldiv_loss
    return loss, st, (; y, μ, logσ², reconstruction_loss, kldiv_loss)
end

function generate_images(
    model, ps, st; num_samples::Int=128, num_latent_dims::Int, decode_compiled=nothing
)
    z = get_device((ps, st))(randn(Float32, num_latent_dims, num_samples))
    if decode_compiled === nothing
        images, _ = decode(model, z, ps, Lux.testmode(st))
    else
        images, _ = decode_compiled(model, z, ps, Lux.testmode(st))
        images = cpu_device()(images)
    end
    return create_image_grid(images, 8, num_samples ÷ 8)
end

function reconstruct_images(model, ps, st, X)
    (recon, _, _), _ = model(X, ps, Lux.testmode(st))
    recon = cpu_device()(recon)
    return create_image_grid(recon, 8, size(X, ndims(X)) ÷ 8)
end

reconstruct_images (generic function with 1 method)

Training the Model

function main(;
    batchsize=128,
    image_size=(64, 64),
    num_latent_dims=8,
    max_num_filters=64,
    seed=0,
    epochs=50,
    weight_decay=1.0e-5,
    learning_rate=1.0e-3,
    num_samples=batchsize,
)
    rng = Xoshiro()
    Random.seed!(rng, seed)

    cvae = CVAE(rng; num_latent_dims, image_shape=(image_size..., 1), max_num_filters)
    ps, st = xdev(Lux.setup(rng, cvae))

    z = xdev(randn(Float32, num_latent_dims, num_samples))
    decode_compiled = @compile decode(cvae, z, ps, Lux.testmode(st))
    x = xdev(randn(Float32, image_size..., 1, batchsize))
    cvae_compiled = @compile cvae(x, ps, Lux.testmode(st))

    train_dataloader = xdev(loadmnist(batchsize, image_size))

    opt = AdamW(; eta=learning_rate, lambda=weight_decay)

    train_state = Training.TrainState(cvae, ps, st, opt)

    @printf "Total Trainable Parameters: %0.4f M\n" (Lux.parameterlength(ps) / 1.0e6)

    is_vscode = isdefined(Main, :VSCodeServer)
    empty_row, model_img_full = nothing, nothing

    for epoch in 1:epochs
        loss_total = 0.0f0
        total_samples = 0

        start_time = time()
        for (i, X) in enumerate(train_dataloader)
            (_, loss, _, train_state) = Training.single_train_step!(
                AutoEnzyme(), loss_function, X, train_state; return_gradients=Val(false)
            )

            loss_total += loss
            total_samples += size(X, ndims(X))

            if i % 250 == 0 || i == length(train_dataloader)
                throughput = total_samples / (time() - start_time)
                @printf "Epoch %d, Iter %d, Loss: %.7f, Throughput: %.6f im/s\n" epoch i loss throughput
            end
        end
        total_time = time() - start_time

        train_loss = loss_total / length(train_dataloader)
        throughput = total_samples / total_time
        @printf "Epoch %d, Train Loss: %.7f, Time: %.4fs, Throughput: %.6f im/s\n" epoch train_loss total_time throughput

        if is_vscode || epoch == epochs
            recon_images = reconstruct_images(
                cvae_compiled,
                train_state.parameters,
                train_state.states,
                first(train_dataloader),
            )
            gen_images = generate_images(
                cvae,
                train_state.parameters,
                train_state.states;
                num_samples,
                num_latent_dims,
                decode_compiled,
            )
            if empty_row === nothing
                empty_row = similar(gen_images, image_size[1], size(gen_images, 2))
                fill!(empty_row, 0)
            end
            model_img_full = vcat(recon_images, empty_row, gen_images)
            is_vscode && display(model_img_full)
        end
    end

    return model_img_full
end

img = main()

2025-03-28 04:58:57.987638: I external/xla/xla/service/service.cc:152] XLA service 0x75630b0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2025-03-28 04:58:57.987694: I external/xla/xla/service/service.cc:160]   StreamExecutor device (0): NVIDIA A100-PCIE-40GB MIG 1g.5gb, Compute Capability 8.0
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1743137937.988638 3656567 se_gpu_pjrt_client.cc:1039] Using BFC allocator.
I0000 00:00:1743137937.988714 3656567 gpu_helpers.cc:136] XLA backend allocating 3825205248 bytes on device 0 for BFCAllocator.
I0000 00:00:1743137937.988757 3656567 gpu_helpers.cc:177] XLA backend will use up to 1275068416 bytes on device 0 for CollectiveBFCAllocator.
I0000 00:00:1743137938.002231 3656567 cuda_dnn.cc:529] Loaded cuDNN version 90400
┌ Warning: `training` is set to `Val{false}()` but is being used within an autodiff call (gradient, jacobian, etc...). This might lead to incorrect results. If you are using a `Lux.jl` model, set it to training mode using `LuxCore.trainmode`.
└ @ LuxLib.Utils /var/lib/buildkite-agent/builds/gpuci-3/julialang/lux-dot-jl/lib/LuxLib/src/utils.jl:344
E0000 00:00:1743137998.916820 3656567 buffer_comparator.cc:156] Difference at 16: 0, expected 1.69914
E0000 00:00:1743137998.916899 3656567 buffer_comparator.cc:156] Difference at 17: 0, expected 1.20048
E0000 00:00:1743137998.916907 3656567 buffer_comparator.cc:156] Difference at 18: 0, expected 1.89606
E0000 00:00:1743137998.916913 3656567 buffer_comparator.cc:156] Difference at 19: 0, expected 1.32316
E0000 00:00:1743137998.916919 3656567 buffer_comparator.cc:156] Difference at 20: 0, expected 1.85572
E0000 00:00:1743137998.916926 3656567 buffer_comparator.cc:156] Difference at 21: 0, expected 0.944347
E0000 00:00:1743137998.916933 3656567 buffer_comparator.cc:156] Difference at 22: 0, expected 1.86911
E0000 00:00:1743137998.916940 3656567 buffer_comparator.cc:156] Difference at 23: 0, expected 0.58457
E0000 00:00:1743137998.916946 3656567 buffer_comparator.cc:156] Difference at 24: 0, expected 2.7184
E0000 00:00:1743137998.916953 3656567 buffer_comparator.cc:156] Difference at 25: 0, expected 2.16503
2025-03-28 04:59:58.916972: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.920684 3656567 buffer_comparator.cc:156] Difference at 16: 0, expected 1.69914
E0000 00:00:1743137998.920751 3656567 buffer_comparator.cc:156] Difference at 17: 0, expected 1.20048
E0000 00:00:1743137998.920759 3656567 buffer_comparator.cc:156] Difference at 18: 0, expected 1.89606
E0000 00:00:1743137998.920765 3656567 buffer_comparator.cc:156] Difference at 19: 0, expected 1.32316
E0000 00:00:1743137998.920771 3656567 buffer_comparator.cc:156] Difference at 20: 0, expected 1.85572
E0000 00:00:1743137998.920777 3656567 buffer_comparator.cc:156] Difference at 21: 0, expected 0.944347
E0000 00:00:1743137998.920785 3656567 buffer_comparator.cc:156] Difference at 22: 0, expected 1.86911
E0000 00:00:1743137998.920791 3656567 buffer_comparator.cc:156] Difference at 23: 0, expected 0.58457
E0000 00:00:1743137998.920797 3656567 buffer_comparator.cc:156] Difference at 24: 0, expected 2.7184
E0000 00:00:1743137998.920804 3656567 buffer_comparator.cc:156] Difference at 25: 0, expected 2.16503
2025-03-28 04:59:58.920819: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.924175 3656567 buffer_comparator.cc:156] Difference at 64: 0, expected 1.60438
E0000 00:00:1743137998.924222 3656567 buffer_comparator.cc:156] Difference at 65: 0, expected 1.28593
E0000 00:00:1743137998.924225 3656567 buffer_comparator.cc:156] Difference at 66: 0, expected 2.03265
E0000 00:00:1743137998.924228 3656567 buffer_comparator.cc:156] Difference at 67: 0, expected 2.28838
E0000 00:00:1743137998.924231 3656567 buffer_comparator.cc:156] Difference at 68: 0, expected 1.76697
E0000 00:00:1743137998.924234 3656567 buffer_comparator.cc:156] Difference at 69: 0, expected 1.33634
E0000 00:00:1743137998.924237 3656567 buffer_comparator.cc:156] Difference at 70: 0, expected 1.96458
E0000 00:00:1743137998.924239 3656567 buffer_comparator.cc:156] Difference at 71: 0, expected 1.16919
E0000 00:00:1743137998.924242 3656567 buffer_comparator.cc:156] Difference at 72: 0, expected 0.893586
E0000 00:00:1743137998.924245 3656567 buffer_comparator.cc:156] Difference at 73: 0, expected 2.3101
2025-03-28 04:59:58.924256: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.927340 3656567 buffer_comparator.cc:156] Difference at 2048: 0, expected 1.77658
E0000 00:00:1743137998.927379 3656567 buffer_comparator.cc:156] Difference at 2049: 0, expected 1.05138
E0000 00:00:1743137998.927383 3656567 buffer_comparator.cc:156] Difference at 2050: 0, expected 1.81989
E0000 00:00:1743137998.927386 3656567 buffer_comparator.cc:156] Difference at 2051: 0, expected 2.39565
E0000 00:00:1743137998.927388 3656567 buffer_comparator.cc:156] Difference at 2052: 0, expected 1.54422
E0000 00:00:1743137998.927395 3656567 buffer_comparator.cc:156] Difference at 2053: 0, expected 2.20321
E0000 00:00:1743137998.927397 3656567 buffer_comparator.cc:156] Difference at 2054: 0, expected 1.94268
E0000 00:00:1743137998.927400 3656567 buffer_comparator.cc:156] Difference at 2055: 0, expected 2.41167
E0000 00:00:1743137998.927403 3656567 buffer_comparator.cc:156] Difference at 2056: 0, expected 2.09568
E0000 00:00:1743137998.927406 3656567 buffer_comparator.cc:156] Difference at 2057: 0, expected 1.1478
2025-03-28 04:59:58.927414: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.930441 3656567 buffer_comparator.cc:156] Difference at 2064: 0, expected 1.87707
E0000 00:00:1743137998.930473 3656567 buffer_comparator.cc:156] Difference at 2065: 0, expected 0.928597
E0000 00:00:1743137998.930477 3656567 buffer_comparator.cc:156] Difference at 2066: 0, expected 2.285
E0000 00:00:1743137998.930480 3656567 buffer_comparator.cc:156] Difference at 2067: 0, expected 1.20842
E0000 00:00:1743137998.930483 3656567 buffer_comparator.cc:156] Difference at 2068: 0, expected 1.71371
E0000 00:00:1743137998.930485 3656567 buffer_comparator.cc:156] Difference at 2069: 0, expected 1.03264
E0000 00:00:1743137998.930488 3656567 buffer_comparator.cc:156] Difference at 2070: 0, expected 1.51429
E0000 00:00:1743137998.930491 3656567 buffer_comparator.cc:156] Difference at 2071: 0, expected 0.671343
E0000 00:00:1743137998.930494 3656567 buffer_comparator.cc:156] Difference at 2072: 0, expected 2.95498
E0000 00:00:1743137998.930497 3656567 buffer_comparator.cc:156] Difference at 2073: 0, expected 2.32851
2025-03-28 04:59:58.930504: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.933456 3656567 buffer_comparator.cc:156] Difference at 2080: 0, expected 1.44171
E0000 00:00:1743137998.933484 3656567 buffer_comparator.cc:156] Difference at 2081: 0, expected 2.15664
E0000 00:00:1743137998.933487 3656567 buffer_comparator.cc:156] Difference at 2082: 0, expected 2.50291
E0000 00:00:1743137998.933490 3656567 buffer_comparator.cc:156] Difference at 2083: 0, expected 1.60771
E0000 00:00:1743137998.933493 3656567 buffer_comparator.cc:156] Difference at 2084: 0, expected 2.0311
E0000 00:00:1743137998.933496 3656567 buffer_comparator.cc:156] Difference at 2085: 0, expected 1.65865
E0000 00:00:1743137998.933499 3656567 buffer_comparator.cc:156] Difference at 2086: 0, expected 1.72254
E0000 00:00:1743137998.933502 3656567 buffer_comparator.cc:156] Difference at 2087: 0, expected 2.04439
E0000 00:00:1743137998.933504 3656567 buffer_comparator.cc:156] Difference at 2088: 0, expected 1.54755
E0000 00:00:1743137998.933507 3656567 buffer_comparator.cc:156] Difference at 2089: 0, expected 0.955993
2025-03-28 04:59:58.933514: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.936501 3656567 buffer_comparator.cc:156] Difference at 2112: 0, expected 1.48116
E0000 00:00:1743137998.936536 3656567 buffer_comparator.cc:156] Difference at 2113: 0, expected 1.4478
E0000 00:00:1743137998.936539 3656567 buffer_comparator.cc:156] Difference at 2114: 0, expected 2.13734
E0000 00:00:1743137998.936542 3656567 buffer_comparator.cc:156] Difference at 2115: 0, expected 2.19489
E0000 00:00:1743137998.936545 3656567 buffer_comparator.cc:156] Difference at 2116: 0, expected 1.82644
E0000 00:00:1743137998.936548 3656567 buffer_comparator.cc:156] Difference at 2117: 0, expected 1.36935
E0000 00:00:1743137998.936550 3656567 buffer_comparator.cc:156] Difference at 2118: 0, expected 1.67303
E0000 00:00:1743137998.936553 3656567 buffer_comparator.cc:156] Difference at 2119: 0, expected 1.26478
E0000 00:00:1743137998.936558 3656567 buffer_comparator.cc:156] Difference at 2120: 0, expected 1.00439
E0000 00:00:1743137998.936561 3656567 buffer_comparator.cc:156] Difference at 2121: 0, expected 2.64095
2025-03-28 04:59:58.936567: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.939576 3656567 buffer_comparator.cc:156] Difference at 4096: 0, expected 2.5269
E0000 00:00:1743137998.939620 3656567 buffer_comparator.cc:156] Difference at 4097: 0, expected 1.96414
E0000 00:00:1743137998.939624 3656567 buffer_comparator.cc:156] Difference at 4098: 0, expected 3.84871
E0000 00:00:1743137998.939627 3656567 buffer_comparator.cc:156] Difference at 4099: 0, expected 4.06422
E0000 00:00:1743137998.939629 3656567 buffer_comparator.cc:156] Difference at 4100: 0, expected 1.84843
E0000 00:00:1743137998.939632 3656567 buffer_comparator.cc:156] Difference at 4101: 0, expected 3.06637
E0000 00:00:1743137998.939635 3656567 buffer_comparator.cc:156] Difference at 4102: 0, expected 3.26933
E0000 00:00:1743137998.939638 3656567 buffer_comparator.cc:156] Difference at 4103: 0, expected 2.61708
E0000 00:00:1743137998.939641 3656567 buffer_comparator.cc:156] Difference at 4104: 0, expected 3.42482
E0000 00:00:1743137998.939643 3656567 buffer_comparator.cc:156] Difference at 4105: 0, expected 2.29469
2025-03-28 04:59:58.939651: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.942899 3656567 buffer_comparator.cc:156] Difference at 4128: 0, expected 2.63461
E0000 00:00:1743137998.942933 3656567 buffer_comparator.cc:156] Difference at 4129: 0, expected 2.9239
E0000 00:00:1743137998.942937 3656567 buffer_comparator.cc:156] Difference at 4130: 0, expected 2.40995
E0000 00:00:1743137998.942940 3656567 buffer_comparator.cc:156] Difference at 4131: 0, expected 2.61607
E0000 00:00:1743137998.942942 3656567 buffer_comparator.cc:156] Difference at 4132: 0, expected 3.62419
E0000 00:00:1743137998.942945 3656567 buffer_comparator.cc:156] Difference at 4133: 0, expected 3.10893
E0000 00:00:1743137998.942948 3656567 buffer_comparator.cc:156] Difference at 4134: 0, expected 2.56351
E0000 00:00:1743137998.942951 3656567 buffer_comparator.cc:156] Difference at 4135: 0, expected 2.19994
E0000 00:00:1743137998.942953 3656567 buffer_comparator.cc:156] Difference at 4136: 0, expected 2.37682
E0000 00:00:1743137998.942956 3656567 buffer_comparator.cc:156] Difference at 4137: 0, expected 2.50883
2025-03-28 04:59:58.942964: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.945959 3656567 buffer_comparator.cc:156] Difference at 4128: 0, expected 2.63461
E0000 00:00:1743137998.945987 3656567 buffer_comparator.cc:156] Difference at 4129: 0, expected 2.9239
E0000 00:00:1743137998.945990 3656567 buffer_comparator.cc:156] Difference at 4130: 0, expected 2.40995
E0000 00:00:1743137998.945993 3656567 buffer_comparator.cc:156] Difference at 4131: 0, expected 2.61607
E0000 00:00:1743137998.945996 3656567 buffer_comparator.cc:156] Difference at 4132: 0, expected 3.62419
E0000 00:00:1743137998.945999 3656567 buffer_comparator.cc:156] Difference at 4133: 0, expected 3.10893
E0000 00:00:1743137998.946002 3656567 buffer_comparator.cc:156] Difference at 4134: 0, expected 2.56351
E0000 00:00:1743137998.946004 3656567 buffer_comparator.cc:156] Difference at 4135: 0, expected 2.19994
E0000 00:00:1743137998.946007 3656567 buffer_comparator.cc:156] Difference at 4136: 0, expected 2.37682
E0000 00:00:1743137998.946010 3656567 buffer_comparator.cc:156] Difference at 4137: 0, expected 2.50883
2025-03-28 04:59:58.946017: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.949010 3656567 buffer_comparator.cc:156] Difference at 4128: 0, expected 2.63461
E0000 00:00:1743137998.949043 3656567 buffer_comparator.cc:156] Difference at 4129: 0, expected 2.9239
E0000 00:00:1743137998.949046 3656567 buffer_comparator.cc:156] Difference at 4130: 0, expected 2.40995
E0000 00:00:1743137998.949049 3656567 buffer_comparator.cc:156] Difference at 4131: 0, expected 2.61607
E0000 00:00:1743137998.949052 3656567 buffer_comparator.cc:156] Difference at 4132: 0, expected 3.62419
E0000 00:00:1743137998.949055 3656567 buffer_comparator.cc:156] Difference at 4133: 0, expected 3.10893
E0000 00:00:1743137998.949057 3656567 buffer_comparator.cc:156] Difference at 4134: 0, expected 2.56351
E0000 00:00:1743137998.949060 3656567 buffer_comparator.cc:156] Difference at 4135: 0, expected 2.19994
E0000 00:00:1743137998.949063 3656567 buffer_comparator.cc:156] Difference at 4136: 0, expected 2.37682
E0000 00:00:1743137998.949066 3656567 buffer_comparator.cc:156] Difference at 4137: 0, expected 2.50883
2025-03-28 04:59:58.949073: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.952073 3656567 buffer_comparator.cc:156] Difference at 4160: 0, expected 2.86435
E0000 00:00:1743137998.952114 3656567 buffer_comparator.cc:156] Difference at 4161: 0, expected 2.0664
E0000 00:00:1743137998.952117 3656567 buffer_comparator.cc:156] Difference at 4162: 0, expected 2.61642
E0000 00:00:1743137998.952120 3656567 buffer_comparator.cc:156] Difference at 4163: 0, expected 3.35871
E0000 00:00:1743137998.952123 3656567 buffer_comparator.cc:156] Difference at 4164: 0, expected 3.1527
E0000 00:00:1743137998.952125 3656567 buffer_comparator.cc:156] Difference at 4165: 0, expected 1.96457
E0000 00:00:1743137998.952128 3656567 buffer_comparator.cc:156] Difference at 4166: 0, expected 2.94612
E0000 00:00:1743137998.952131 3656567 buffer_comparator.cc:156] Difference at 4167: 0, expected 1.54296
E0000 00:00:1743137998.952134 3656567 buffer_comparator.cc:156] Difference at 4168: 0, expected 1.32996
E0000 00:00:1743137998.952137 3656567 buffer_comparator.cc:156] Difference at 4169: 0, expected 3.04802
2025-03-28 04:59:58.952145: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.955250 3656567 buffer_comparator.cc:156] Difference at 8192: 0, expected 1.98786
E0000 00:00:1743137998.955295 3656567 buffer_comparator.cc:156] Difference at 8193: 0, expected 1.72939
E0000 00:00:1743137998.955299 3656567 buffer_comparator.cc:156] Difference at 8194: 0, expected 3.03127
E0000 00:00:1743137998.955302 3656567 buffer_comparator.cc:156] Difference at 8195: 0, expected 3.3803
E0000 00:00:1743137998.955304 3656567 buffer_comparator.cc:156] Difference at 8196: 0, expected 1.7692
E0000 00:00:1743137998.955307 3656567 buffer_comparator.cc:156] Difference at 8197: 0, expected 2.87459
E0000 00:00:1743137998.955310 3656567 buffer_comparator.cc:156] Difference at 8198: 0, expected 2.74368
E0000 00:00:1743137998.955313 3656567 buffer_comparator.cc:156] Difference at 8199: 0, expected 2.30403
E0000 00:00:1743137998.955316 3656567 buffer_comparator.cc:156] Difference at 8200: 0, expected 3.07071
E0000 00:00:1743137998.955318 3656567 buffer_comparator.cc:156] Difference at 8201: 0, expected 1.9239
2025-03-28 04:59:58.955327: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.958416 3656567 buffer_comparator.cc:156] Difference at 8224: 0, expected 2.18642
E0000 00:00:1743137998.958450 3656567 buffer_comparator.cc:156] Difference at 8225: 0, expected 2.28379
E0000 00:00:1743137998.958453 3656567 buffer_comparator.cc:156] Difference at 8226: 0, expected 2.33258
E0000 00:00:1743137998.958458 3656567 buffer_comparator.cc:156] Difference at 8227: 0, expected 2.12912
E0000 00:00:1743137998.958461 3656567 buffer_comparator.cc:156] Difference at 8228: 0, expected 2.99415
E0000 00:00:1743137998.958463 3656567 buffer_comparator.cc:156] Difference at 8229: 0, expected 2.27319
E0000 00:00:1743137998.958466 3656567 buffer_comparator.cc:156] Difference at 8230: 0, expected 2.165
E0000 00:00:1743137998.958469 3656567 buffer_comparator.cc:156] Difference at 8231: 0, expected 2.33242
E0000 00:00:1743137998.958472 3656567 buffer_comparator.cc:156] Difference at 8232: 0, expected 1.81753
E0000 00:00:1743137998.958474 3656567 buffer_comparator.cc:156] Difference at 8233: 0, expected 1.754
2025-03-28 04:59:58.958482: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.961492 3656567 buffer_comparator.cc:156] Difference at 8256: 0, expected 2.10464
E0000 00:00:1743137998.961531 3656567 buffer_comparator.cc:156] Difference at 8257: 0, expected 2.00542
E0000 00:00:1743137998.961534 3656567 buffer_comparator.cc:156] Difference at 8258: 0, expected 2.33829
E0000 00:00:1743137998.961537 3656567 buffer_comparator.cc:156] Difference at 8259: 0, expected 2.68564
E0000 00:00:1743137998.961540 3656567 buffer_comparator.cc:156] Difference at 8260: 0, expected 2.75266
E0000 00:00:1743137998.961543 3656567 buffer_comparator.cc:156] Difference at 8261: 0, expected 2.02724
E0000 00:00:1743137998.961546 3656567 buffer_comparator.cc:156] Difference at 8262: 0, expected 1.93574
E0000 00:00:1743137998.961548 3656567 buffer_comparator.cc:156] Difference at 8263: 0, expected 1.335
E0000 00:00:1743137998.961551 3656567 buffer_comparator.cc:156] Difference at 8264: 0, expected 1.15464
E0000 00:00:1743137998.961554 3656567 buffer_comparator.cc:156] Difference at 8265: 0, expected 3.13129
2025-03-28 04:59:58.961562: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.964614 3656567 buffer_comparator.cc:156] Difference at 16384: 0, expected 2.13072
E0000 00:00:1743137998.964635 3656567 buffer_comparator.cc:156] Difference at 16385: 0, expected 1.18098
E0000 00:00:1743137998.964638 3656567 buffer_comparator.cc:156] Difference at 16386: 0, expected 2.76235
E0000 00:00:1743137998.964641 3656567 buffer_comparator.cc:156] Difference at 16387: 0, expected 2.87006
E0000 00:00:1743137998.964643 3656567 buffer_comparator.cc:156] Difference at 16388: 0, expected 1.48471
E0000 00:00:1743137998.964646 3656567 buffer_comparator.cc:156] Difference at 16389: 0, expected 2.14624
E0000 00:00:1743137998.964649 3656567 buffer_comparator.cc:156] Difference at 16390: 0, expected 2.04824
E0000 00:00:1743137998.964652 3656567 buffer_comparator.cc:156] Difference at 16391: 0, expected 2.47653
E0000 00:00:1743137998.964655 3656567 buffer_comparator.cc:156] Difference at 16392: 0, expected 2.26896
E0000 00:00:1743137998.964657 3656567 buffer_comparator.cc:156] Difference at 16393: 0, expected 1.36576
2025-03-28 04:59:58.964663: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.967592 3656567 buffer_comparator.cc:156] Difference at 16384: 0, expected 2.13072
E0000 00:00:1743137998.967608 3656567 buffer_comparator.cc:156] Difference at 16385: 0, expected 1.18098
E0000 00:00:1743137998.967611 3656567 buffer_comparator.cc:156] Difference at 16386: 0, expected 2.76235
E0000 00:00:1743137998.967614 3656567 buffer_comparator.cc:156] Difference at 16387: 0, expected 2.87006
E0000 00:00:1743137998.967617 3656567 buffer_comparator.cc:156] Difference at 16388: 0, expected 1.48471
E0000 00:00:1743137998.967620 3656567 buffer_comparator.cc:156] Difference at 16389: 0, expected 2.14624
E0000 00:00:1743137998.967625 3656567 buffer_comparator.cc:156] Difference at 16390: 0, expected 2.04824
E0000 00:00:1743137998.967627 3656567 buffer_comparator.cc:156] Difference at 16391: 0, expected 2.47653
E0000 00:00:1743137998.967630 3656567 buffer_comparator.cc:156] Difference at 16392: 0, expected 2.26896
E0000 00:00:1743137998.967633 3656567 buffer_comparator.cc:156] Difference at 16393: 0, expected 1.36576
2025-03-28 04:59:58.967638: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.970574 3656567 buffer_comparator.cc:156] Difference at 16448: 0, expected 2.04485
E0000 00:00:1743137998.970590 3656567 buffer_comparator.cc:156] Difference at 16449: 0, expected 1.38058
E0000 00:00:1743137998.970593 3656567 buffer_comparator.cc:156] Difference at 16450: 0, expected 1.77153
E0000 00:00:1743137998.970596 3656567 buffer_comparator.cc:156] Difference at 16451: 0, expected 2.60773
E0000 00:00:1743137998.970599 3656567 buffer_comparator.cc:156] Difference at 16452: 0, expected 2.24942
E0000 00:00:1743137998.970601 3656567 buffer_comparator.cc:156] Difference at 16453: 0, expected 1.32792
E0000 00:00:1743137998.970604 3656567 buffer_comparator.cc:156] Difference at 16454: 0, expected 2.7744
E0000 00:00:1743137998.970607 3656567 buffer_comparator.cc:156] Difference at 16455: 0, expected 1.26898
E0000 00:00:1743137998.970610 3656567 buffer_comparator.cc:156] Difference at 16456: 0, expected 1.06744
E0000 00:00:1743137998.970612 3656567 buffer_comparator.cc:156] Difference at 16457: 0, expected 1.99063
2025-03-28 04:59:58.970618: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743137998.973600 3656567 buffer_comparator.cc:156] Difference at 32768: 0, expected 2.02216
E0000 00:00:1743137998.973615 3656567 buffer_comparator.cc:156] Difference at 32769: 0, expected 1.27847
E0000 00:00:1743137998.973618 3656567 buffer_comparator.cc:156] Difference at 32770: 0, expected 2.25127
E0000 00:00:1743137998.973621 3656567 buffer_comparator.cc:156] Difference at 32771: 0, expected 2.96132
E0000 00:00:1743137998.973624 3656567 buffer_comparator.cc:156] Difference at 32772: 0, expected 1.65598
E0000 00:00:1743137998.973626 3656567 buffer_comparator.cc:156] Difference at 32773: 0, expected 2.09882
E0000 00:00:1743137998.973629 3656567 buffer_comparator.cc:156] Difference at 32774: 0, expected 2.35327
E0000 00:00:1743137998.973632 3656567 buffer_comparator.cc:156] Difference at 32775: 0, expected 2.14737
E0000 00:00:1743137998.973635 3656567 buffer_comparator.cc:156] Difference at 32776: 0, expected 2.35827
E0000 00:00:1743137998.973637 3656567 buffer_comparator.cc:156] Difference at 32777: 0, expected 1.49831
2025-03-28 04:59:58.973642: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
2025-03-28 05:00:16.374314: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 336 bytes spill stores, 336 bytes spill loads

2025-03-28 05:00:16.706813: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 3744 bytes spill stores, 3732 bytes spill loads

E0000 00:00:1743138016.715354 3656567 buffer_comparator.cc:156] Difference at 16: -nan, expected 1.69914
E0000 00:00:1743138016.715452 3656567 buffer_comparator.cc:156] Difference at 17: -nan, expected 1.20048
E0000 00:00:1743138016.715464 3656567 buffer_comparator.cc:156] Difference at 18: -nan, expected 1.89606
E0000 00:00:1743138016.715471 3656567 buffer_comparator.cc:156] Difference at 19: -nan, expected 1.32316
E0000 00:00:1743138016.715477 3656567 buffer_comparator.cc:156] Difference at 20: -nan, expected 1.85572
E0000 00:00:1743138016.715485 3656567 buffer_comparator.cc:156] Difference at 21: -nan, expected 0.944347
E0000 00:00:1743138016.715492 3656567 buffer_comparator.cc:156] Difference at 22: -nan, expected 1.86911
E0000 00:00:1743138016.715498 3656567 buffer_comparator.cc:156] Difference at 23: -nan, expected 0.58457
E0000 00:00:1743138016.715505 3656567 buffer_comparator.cc:156] Difference at 24: -nan, expected 2.7184
E0000 00:00:1743138016.715511 3656567 buffer_comparator.cc:156] Difference at 25: -nan, expected 2.16503
2025-03-28 05:00:16.715530: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.719169 3656567 buffer_comparator.cc:156] Difference at 16: -nan, expected 1.69914
E0000 00:00:1743138016.719216 3656567 buffer_comparator.cc:156] Difference at 17: -nan, expected 1.20048
E0000 00:00:1743138016.719220 3656567 buffer_comparator.cc:156] Difference at 18: -nan, expected 1.89606
E0000 00:00:1743138016.719222 3656567 buffer_comparator.cc:156] Difference at 19: -nan, expected 1.32316
E0000 00:00:1743138016.719225 3656567 buffer_comparator.cc:156] Difference at 20: -nan, expected 1.85572
E0000 00:00:1743138016.719228 3656567 buffer_comparator.cc:156] Difference at 21: -nan, expected 0.944347
E0000 00:00:1743138016.719231 3656567 buffer_comparator.cc:156] Difference at 22: -nan, expected 1.86911
E0000 00:00:1743138016.719234 3656567 buffer_comparator.cc:156] Difference at 23: -nan, expected 0.58457
E0000 00:00:1743138016.719236 3656567 buffer_comparator.cc:156] Difference at 24: -nan, expected 2.7184
E0000 00:00:1743138016.719239 3656567 buffer_comparator.cc:156] Difference at 25: -nan, expected 2.16503
2025-03-28 05:00:16.719249: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.722381 3656567 buffer_comparator.cc:156] Difference at 64: -nan, expected 1.60438
E0000 00:00:1743138016.722426 3656567 buffer_comparator.cc:156] Difference at 65: -nan, expected 1.28593
E0000 00:00:1743138016.722430 3656567 buffer_comparator.cc:156] Difference at 66: -nan, expected 2.03265
E0000 00:00:1743138016.722432 3656567 buffer_comparator.cc:156] Difference at 67: -nan, expected 2.28838
E0000 00:00:1743138016.722435 3656567 buffer_comparator.cc:156] Difference at 68: -nan, expected 1.76697
E0000 00:00:1743138016.722438 3656567 buffer_comparator.cc:156] Difference at 69: -nan, expected 1.33634
E0000 00:00:1743138016.722441 3656567 buffer_comparator.cc:156] Difference at 70: -nan, expected 1.96458
E0000 00:00:1743138016.722443 3656567 buffer_comparator.cc:156] Difference at 71: -nan, expected 1.16919
E0000 00:00:1743138016.722446 3656567 buffer_comparator.cc:156] Difference at 72: -nan, expected 0.893586
E0000 00:00:1743138016.722449 3656567 buffer_comparator.cc:156] Difference at 73: -nan, expected 2.3101
2025-03-28 05:00:16.722459: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.725548 3656567 buffer_comparator.cc:156] Difference at 2048: -nan, expected 1.77658
E0000 00:00:1743138016.725595 3656567 buffer_comparator.cc:156] Difference at 2049: -nan, expected 1.05138
E0000 00:00:1743138016.725599 3656567 buffer_comparator.cc:156] Difference at 2050: -nan, expected 1.81989
E0000 00:00:1743138016.725602 3656567 buffer_comparator.cc:156] Difference at 2051: -nan, expected 2.39565
E0000 00:00:1743138016.725605 3656567 buffer_comparator.cc:156] Difference at 2052: -nan, expected 1.54422
E0000 00:00:1743138016.725608 3656567 buffer_comparator.cc:156] Difference at 2053: -nan, expected 2.20321
E0000 00:00:1743138016.725610 3656567 buffer_comparator.cc:156] Difference at 2054: -nan, expected 1.94268
E0000 00:00:1743138016.725613 3656567 buffer_comparator.cc:156] Difference at 2055: -nan, expected 2.41167
E0000 00:00:1743138016.725620 3656567 buffer_comparator.cc:156] Difference at 2056: -nan, expected 2.09568
E0000 00:00:1743138016.725623 3656567 buffer_comparator.cc:156] Difference at 2057: -nan, expected 1.1478
2025-03-28 05:00:16.725632: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.728775 3656567 buffer_comparator.cc:156] Difference at 2064: -nan, expected 1.87707
E0000 00:00:1743138016.728815 3656567 buffer_comparator.cc:156] Difference at 2065: -nan, expected 0.928597
E0000 00:00:1743138016.728819 3656567 buffer_comparator.cc:156] Difference at 2066: -nan, expected 2.285
E0000 00:00:1743138016.728822 3656567 buffer_comparator.cc:156] Difference at 2067: -nan, expected 1.20842
E0000 00:00:1743138016.728825 3656567 buffer_comparator.cc:156] Difference at 2068: -nan, expected 1.71371
E0000 00:00:1743138016.728827 3656567 buffer_comparator.cc:156] Difference at 2069: -nan, expected 1.03264
E0000 00:00:1743138016.728830 3656567 buffer_comparator.cc:156] Difference at 2070: -nan, expected 1.51429
E0000 00:00:1743138016.728833 3656567 buffer_comparator.cc:156] Difference at 2071: -nan, expected 0.671343
E0000 00:00:1743138016.728836 3656567 buffer_comparator.cc:156] Difference at 2072: -nan, expected 2.95498
E0000 00:00:1743138016.728838 3656567 buffer_comparator.cc:156] Difference at 2073: -nan, expected 2.32851
2025-03-28 05:00:16.728847: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.731911 3656567 buffer_comparator.cc:156] Difference at 2080: -nan, expected 1.44171
E0000 00:00:1743138016.731964 3656567 buffer_comparator.cc:156] Difference at 2081: -nan, expected 2.15664
E0000 00:00:1743138016.731968 3656567 buffer_comparator.cc:156] Difference at 2082: -nan, expected 2.50291
E0000 00:00:1743138016.731971 3656567 buffer_comparator.cc:156] Difference at 2083: -nan, expected 1.60771
E0000 00:00:1743138016.731973 3656567 buffer_comparator.cc:156] Difference at 2084: -nan, expected 2.0311
E0000 00:00:1743138016.731976 3656567 buffer_comparator.cc:156] Difference at 2085: -nan, expected 1.65865
E0000 00:00:1743138016.731979 3656567 buffer_comparator.cc:156] Difference at 2086: -nan, expected 1.72254
E0000 00:00:1743138016.731981 3656567 buffer_comparator.cc:156] Difference at 2087: -nan, expected 2.04439
E0000 00:00:1743138016.731984 3656567 buffer_comparator.cc:156] Difference at 2088: -nan, expected 1.54755
E0000 00:00:1743138016.731987 3656567 buffer_comparator.cc:156] Difference at 2089: -nan, expected 0.955993
2025-03-28 05:00:16.731997: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.735126 3656567 buffer_comparator.cc:156] Difference at 2112: -nan, expected 1.48116
E0000 00:00:1743138016.735163 3656567 buffer_comparator.cc:156] Difference at 2113: -nan, expected 1.4478
E0000 00:00:1743138016.735167 3656567 buffer_comparator.cc:156] Difference at 2114: -nan, expected 2.13734
E0000 00:00:1743138016.735170 3656567 buffer_comparator.cc:156] Difference at 2115: -nan, expected 2.19489
E0000 00:00:1743138016.735173 3656567 buffer_comparator.cc:156] Difference at 2116: -nan, expected 1.82644
E0000 00:00:1743138016.735175 3656567 buffer_comparator.cc:156] Difference at 2117: -nan, expected 1.36935
E0000 00:00:1743138016.735178 3656567 buffer_comparator.cc:156] Difference at 2118: -nan, expected 1.67303
E0000 00:00:1743138016.735181 3656567 buffer_comparator.cc:156] Difference at 2119: -nan, expected 1.26478
E0000 00:00:1743138016.735183 3656567 buffer_comparator.cc:156] Difference at 2120: -nan, expected 1.00439
E0000 00:00:1743138016.735186 3656567 buffer_comparator.cc:156] Difference at 2121: -nan, expected 2.64095
2025-03-28 05:00:16.735194: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.738264 3656567 buffer_comparator.cc:156] Difference at 4096: -nan, expected 2.5269
E0000 00:00:1743138016.738299 3656567 buffer_comparator.cc:156] Difference at 4097: -nan, expected 1.96414
E0000 00:00:1743138016.738303 3656567 buffer_comparator.cc:156] Difference at 4098: -nan, expected 3.84871
E0000 00:00:1743138016.738305 3656567 buffer_comparator.cc:156] Difference at 4099: -nan, expected 4.06422
E0000 00:00:1743138016.738308 3656567 buffer_comparator.cc:156] Difference at 4100: -nan, expected 1.84843
E0000 00:00:1743138016.738311 3656567 buffer_comparator.cc:156] Difference at 4101: -nan, expected 3.06637
E0000 00:00:1743138016.738313 3656567 buffer_comparator.cc:156] Difference at 4102: -nan, expected 3.26933
E0000 00:00:1743138016.738316 3656567 buffer_comparator.cc:156] Difference at 4103: -nan, expected 2.61708
E0000 00:00:1743138016.738319 3656567 buffer_comparator.cc:156] Difference at 4104: -nan, expected 3.42482
E0000 00:00:1743138016.738321 3656567 buffer_comparator.cc:156] Difference at 4105: -nan, expected 2.29469
2025-03-28 05:00:16.738329: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.741342 3656567 buffer_comparator.cc:156] Difference at 4128: -nan, expected 2.63461
E0000 00:00:1743138016.741378 3656567 buffer_comparator.cc:156] Difference at 4129: -nan, expected 2.9239
E0000 00:00:1743138016.741382 3656567 buffer_comparator.cc:156] Difference at 4130: -nan, expected 2.40995
E0000 00:00:1743138016.741384 3656567 buffer_comparator.cc:156] Difference at 4131: -nan, expected 2.61607
E0000 00:00:1743138016.741387 3656567 buffer_comparator.cc:156] Difference at 4132: -nan, expected 3.62419
E0000 00:00:1743138016.741390 3656567 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.10893
E0000 00:00:1743138016.741392 3656567 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.56351
E0000 00:00:1743138016.741395 3656567 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.19994
E0000 00:00:1743138016.741398 3656567 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.37682
E0000 00:00:1743138016.741400 3656567 buffer_comparator.cc:156] Difference at 4137: -nan, expected 2.50883
2025-03-28 05:00:16.741407: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.744475 3656567 buffer_comparator.cc:156] Difference at 4128: -nan, expected 2.63461
E0000 00:00:1743138016.744522 3656567 buffer_comparator.cc:156] Difference at 4129: -nan, expected 2.9239
E0000 00:00:1743138016.744526 3656567 buffer_comparator.cc:156] Difference at 4130: -nan, expected 2.40995
E0000 00:00:1743138016.744529 3656567 buffer_comparator.cc:156] Difference at 4131: -nan, expected 2.61607
E0000 00:00:1743138016.744531 3656567 buffer_comparator.cc:156] Difference at 4132: -nan, expected 3.62419
E0000 00:00:1743138016.744534 3656567 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.10893
E0000 00:00:1743138016.744537 3656567 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.56351
E0000 00:00:1743138016.744539 3656567 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.19994
E0000 00:00:1743138016.744542 3656567 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.37682
E0000 00:00:1743138016.744545 3656567 buffer_comparator.cc:156] Difference at 4137: -nan, expected 2.50883
2025-03-28 05:00:16.744553: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.747638 3656567 buffer_comparator.cc:156] Difference at 4128: -nan, expected 2.63461
E0000 00:00:1743138016.747682 3656567 buffer_comparator.cc:156] Difference at 4129: -nan, expected 2.9239
E0000 00:00:1743138016.747689 3656567 buffer_comparator.cc:156] Difference at 4130: -nan, expected 2.40995
E0000 00:00:1743138016.747691 3656567 buffer_comparator.cc:156] Difference at 4131: -nan, expected 2.61607
E0000 00:00:1743138016.747694 3656567 buffer_comparator.cc:156] Difference at 4132: -nan, expected 3.62419
E0000 00:00:1743138016.747697 3656567 buffer_comparator.cc:156] Difference at 4133: -nan, expected 3.10893
E0000 00:00:1743138016.747699 3656567 buffer_comparator.cc:156] Difference at 4134: -nan, expected 2.56351
E0000 00:00:1743138016.747702 3656567 buffer_comparator.cc:156] Difference at 4135: -nan, expected 2.19994
E0000 00:00:1743138016.747705 3656567 buffer_comparator.cc:156] Difference at 4136: -nan, expected 2.37682
E0000 00:00:1743138016.747707 3656567 buffer_comparator.cc:156] Difference at 4137: -nan, expected 2.50883
2025-03-28 05:00:16.747715: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.750828 3656567 buffer_comparator.cc:156] Difference at 4160: -nan, expected 2.86435
E0000 00:00:1743138016.750876 3656567 buffer_comparator.cc:156] Difference at 4161: -nan, expected 2.0664
E0000 00:00:1743138016.750880 3656567 buffer_comparator.cc:156] Difference at 4162: -nan, expected 2.61642
E0000 00:00:1743138016.750882 3656567 buffer_comparator.cc:156] Difference at 4163: -nan, expected 3.35871
E0000 00:00:1743138016.750885 3656567 buffer_comparator.cc:156] Difference at 4164: -nan, expected 3.1527
E0000 00:00:1743138016.750888 3656567 buffer_comparator.cc:156] Difference at 4165: -nan, expected 1.96457
E0000 00:00:1743138016.750890 3656567 buffer_comparator.cc:156] Difference at 4166: -nan, expected 2.94612
E0000 00:00:1743138016.750893 3656567 buffer_comparator.cc:156] Difference at 4167: -nan, expected 1.54296
E0000 00:00:1743138016.750895 3656567 buffer_comparator.cc:156] Difference at 4168: -nan, expected 1.32996
E0000 00:00:1743138016.750898 3656567 buffer_comparator.cc:156] Difference at 4169: -nan, expected 3.04802
2025-03-28 05:00:16.750908: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.754261 3656567 buffer_comparator.cc:156] Difference at 8192: -nan, expected 1.98786
E0000 00:00:1743138016.754312 3656567 buffer_comparator.cc:156] Difference at 8193: -nan, expected 1.72939
E0000 00:00:1743138016.754315 3656567 buffer_comparator.cc:156] Difference at 8194: -nan, expected 3.03127
E0000 00:00:1743138016.754318 3656567 buffer_comparator.cc:156] Difference at 8195: -nan, expected 3.3803
E0000 00:00:1743138016.754321 3656567 buffer_comparator.cc:156] Difference at 8196: -nan, expected 1.7692
E0000 00:00:1743138016.754323 3656567 buffer_comparator.cc:156] Difference at 8197: -nan, expected 2.87459
E0000 00:00:1743138016.754326 3656567 buffer_comparator.cc:156] Difference at 8198: -nan, expected 2.74368
E0000 00:00:1743138016.754329 3656567 buffer_comparator.cc:156] Difference at 8199: -nan, expected 2.30403
E0000 00:00:1743138016.754332 3656567 buffer_comparator.cc:156] Difference at 8200: -nan, expected 3.07071
E0000 00:00:1743138016.754334 3656567 buffer_comparator.cc:156] Difference at 8201: -nan, expected 1.9239
2025-03-28 05:00:16.754344: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.757440 3656567 buffer_comparator.cc:156] Difference at 8224: -nan, expected 2.18642
E0000 00:00:1743138016.757475 3656567 buffer_comparator.cc:156] Difference at 8225: -nan, expected 2.28379
E0000 00:00:1743138016.757478 3656567 buffer_comparator.cc:156] Difference at 8226: -nan, expected 2.33258
E0000 00:00:1743138016.757481 3656567 buffer_comparator.cc:156] Difference at 8227: -nan, expected 2.12912
E0000 00:00:1743138016.757483 3656567 buffer_comparator.cc:156] Difference at 8228: -nan, expected 2.99415
E0000 00:00:1743138016.757489 3656567 buffer_comparator.cc:156] Difference at 8229: -nan, expected 2.27319
E0000 00:00:1743138016.757492 3656567 buffer_comparator.cc:156] Difference at 8230: -nan, expected 2.165
E0000 00:00:1743138016.757495 3656567 buffer_comparator.cc:156] Difference at 8231: -nan, expected 2.33242
E0000 00:00:1743138016.757497 3656567 buffer_comparator.cc:156] Difference at 8232: -nan, expected 1.81753
E0000 00:00:1743138016.757500 3656567 buffer_comparator.cc:156] Difference at 8233: -nan, expected 1.754
2025-03-28 05:00:16.757508: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.760518 3656567 buffer_comparator.cc:156] Difference at 8256: -nan, expected 2.10464
E0000 00:00:1743138016.760554 3656567 buffer_comparator.cc:156] Difference at 8257: -nan, expected 2.00542
E0000 00:00:1743138016.760557 3656567 buffer_comparator.cc:156] Difference at 8258: -nan, expected 2.33829
E0000 00:00:1743138016.760560 3656567 buffer_comparator.cc:156] Difference at 8259: -nan, expected 2.68564
E0000 00:00:1743138016.760563 3656567 buffer_comparator.cc:156] Difference at 8260: -nan, expected 2.75266
E0000 00:00:1743138016.760565 3656567 buffer_comparator.cc:156] Difference at 8261: -nan, expected 2.02724
E0000 00:00:1743138016.760568 3656567 buffer_comparator.cc:156] Difference at 8262: -nan, expected 1.93574
E0000 00:00:1743138016.760571 3656567 buffer_comparator.cc:156] Difference at 8263: -nan, expected 1.335
E0000 00:00:1743138016.760573 3656567 buffer_comparator.cc:156] Difference at 8264: -nan, expected 1.15464
E0000 00:00:1743138016.760576 3656567 buffer_comparator.cc:156] Difference at 8265: -nan, expected 3.13129
2025-03-28 05:00:16.760584: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.763630 3656567 buffer_comparator.cc:156] Difference at 16384: -nan, expected 2.13072
E0000 00:00:1743138016.763671 3656567 buffer_comparator.cc:156] Difference at 16385: -nan, expected 1.18098
E0000 00:00:1743138016.763675 3656567 buffer_comparator.cc:156] Difference at 16386: -nan, expected 2.76235
E0000 00:00:1743138016.763678 3656567 buffer_comparator.cc:156] Difference at 16387: -nan, expected 2.87006
E0000 00:00:1743138016.763680 3656567 buffer_comparator.cc:156] Difference at 16388: -nan, expected 1.48471
E0000 00:00:1743138016.763683 3656567 buffer_comparator.cc:156] Difference at 16389: -nan, expected 2.14624
E0000 00:00:1743138016.763686 3656567 buffer_comparator.cc:156] Difference at 16390: -nan, expected 2.04824
E0000 00:00:1743138016.763688 3656567 buffer_comparator.cc:156] Difference at 16391: -nan, expected 2.47653
E0000 00:00:1743138016.763691 3656567 buffer_comparator.cc:156] Difference at 16392: -nan, expected 2.26896
E0000 00:00:1743138016.763694 3656567 buffer_comparator.cc:156] Difference at 16393: -nan, expected 1.36576
2025-03-28 05:00:16.763702: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.766842 3656567 buffer_comparator.cc:156] Difference at 16384: -nan, expected 2.13072
E0000 00:00:1743138016.766881 3656567 buffer_comparator.cc:156] Difference at 16385: -nan, expected 1.18098
E0000 00:00:1743138016.766885 3656567 buffer_comparator.cc:156] Difference at 16386: -nan, expected 2.76235
E0000 00:00:1743138016.766888 3656567 buffer_comparator.cc:156] Difference at 16387: -nan, expected 2.87006
E0000 00:00:1743138016.766890 3656567 buffer_comparator.cc:156] Difference at 16388: -nan, expected 1.48471
E0000 00:00:1743138016.766893 3656567 buffer_comparator.cc:156] Difference at 16389: -nan, expected 2.14624
E0000 00:00:1743138016.766896 3656567 buffer_comparator.cc:156] Difference at 16390: -nan, expected 2.04824
E0000 00:00:1743138016.766901 3656567 buffer_comparator.cc:156] Difference at 16391: -nan, expected 2.47653
E0000 00:00:1743138016.766904 3656567 buffer_comparator.cc:156] Difference at 16392: -nan, expected 2.26896
E0000 00:00:1743138016.766906 3656567 buffer_comparator.cc:156] Difference at 16393: -nan, expected 1.36576
2025-03-28 05:00:16.766915: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.770035 3656567 buffer_comparator.cc:156] Difference at 16448: -nan, expected 2.04485
E0000 00:00:1743138016.770091 3656567 buffer_comparator.cc:156] Difference at 16449: -nan, expected 1.38058
E0000 00:00:1743138016.770095 3656567 buffer_comparator.cc:156] Difference at 16450: -nan, expected 1.77153
E0000 00:00:1743138016.770097 3656567 buffer_comparator.cc:156] Difference at 16451: -nan, expected 2.60773
E0000 00:00:1743138016.770100 3656567 buffer_comparator.cc:156] Difference at 16452: -nan, expected 2.24942
E0000 00:00:1743138016.770103 3656567 buffer_comparator.cc:156] Difference at 16453: -nan, expected 1.32792
E0000 00:00:1743138016.770105 3656567 buffer_comparator.cc:156] Difference at 16454: -nan, expected 2.7744
E0000 00:00:1743138016.770108 3656567 buffer_comparator.cc:156] Difference at 16455: -nan, expected 1.26898
E0000 00:00:1743138016.770111 3656567 buffer_comparator.cc:156] Difference at 16456: -nan, expected 1.06744
E0000 00:00:1743138016.770113 3656567 buffer_comparator.cc:156] Difference at 16457: -nan, expected 1.99063
2025-03-28 05:00:16.770121: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.773254 3656567 buffer_comparator.cc:156] Difference at 32768: -nan, expected 2.02216
E0000 00:00:1743138016.773302 3656567 buffer_comparator.cc:156] Difference at 32769: -nan, expected 1.27847
E0000 00:00:1743138016.773306 3656567 buffer_comparator.cc:156] Difference at 32770: -nan, expected 2.25127
E0000 00:00:1743138016.773308 3656567 buffer_comparator.cc:156] Difference at 32771: -nan, expected 2.96132
E0000 00:00:1743138016.773311 3656567 buffer_comparator.cc:156] Difference at 32772: -nan, expected 1.65598
E0000 00:00:1743138016.773314 3656567 buffer_comparator.cc:156] Difference at 32773: -nan, expected 2.09882
E0000 00:00:1743138016.773317 3656567 buffer_comparator.cc:156] Difference at 32774: -nan, expected 2.35327
E0000 00:00:1743138016.773319 3656567 buffer_comparator.cc:156] Difference at 32775: -nan, expected 2.14737
E0000 00:00:1743138016.773322 3656567 buffer_comparator.cc:156] Difference at 32776: -nan, expected 2.35827
E0000 00:00:1743138016.773324 3656567 buffer_comparator.cc:156] Difference at 32777: -nan, expected 1.49831
2025-03-28 05:00:16.773334: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.782762 3656567 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1743138016.782821 3656567 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1743138016.782825 3656567 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1743138016.782829 3656567 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1743138016.782832 3656567 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1743138016.782835 3656567 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1743138016.782838 3656567 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1743138016.782841 3656567 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1743138016.782844 3656567 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1743138016.782850 3656567 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-03-28 05:00:16.782861: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.785139 3656567 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1743138016.785153 3656567 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1743138016.785157 3656567 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1743138016.785160 3656567 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1743138016.785163 3656567 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1743138016.785166 3656567 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1743138016.785169 3656567 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1743138016.785172 3656567 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1743138016.785175 3656567 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1743138016.785178 3656567 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-03-28 05:00:16.785184: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.787382 3656567 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1743138016.787396 3656567 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1743138016.787399 3656567 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1743138016.787402 3656567 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1743138016.787406 3656567 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1743138016.787409 3656567 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1743138016.787412 3656567 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1743138016.787415 3656567 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1743138016.787418 3656567 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1743138016.787421 3656567 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-03-28 05:00:16.787426: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.789602 3656567 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1743138016.789616 3656567 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1743138016.789620 3656567 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1743138016.789623 3656567 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1743138016.789626 3656567 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1743138016.789629 3656567 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1743138016.789632 3656567 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1743138016.789635 3656567 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1743138016.789638 3656567 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
E0000 00:00:1743138016.789641 3656567 buffer_comparator.cc:156] Difference at 265: 699.558, expected 1029.49
2025-03-28 05:00:16.789647: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.791878 3656567 buffer_comparator.cc:156] Difference at 226: 1167.19, expected 1045.8
E0000 00:00:1743138016.791898 3656567 buffer_comparator.cc:156] Difference at 256: 701.466, expected 1026.63
E0000 00:00:1743138016.791902 3656567 buffer_comparator.cc:156] Difference at 257: 721.513, expected 1036.99
E0000 00:00:1743138016.791905 3656567 buffer_comparator.cc:156] Difference at 258: 713.145, expected 1046.75
E0000 00:00:1743138016.791908 3656567 buffer_comparator.cc:156] Difference at 259: 705.587, expected 1025.83
E0000 00:00:1743138016.791911 3656567 buffer_comparator.cc:156] Difference at 260: 710.736, expected 1034.15
E0000 00:00:1743138016.791914 3656567 buffer_comparator.cc:156] Difference at 261: 720.361, expected 1039.47
E0000 00:00:1743138016.791917 3656567 buffer_comparator.cc:156] Difference at 262: 715.077, expected 1033.55
E0000 00:00:1743138016.791920 3656567 buffer_comparator.cc:156] Difference at 263: 725.583, expected 1041.94
E0000 00:00:1743138016.791924 3656567 buffer_comparator.cc:156] Difference at 264: 714.73, expected 1026.16
2025-03-28 05:00:16.791930: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.794178 3656567 buffer_comparator.cc:156] Difference at 512: 711.971, expected 1037.38
E0000 00:00:1743138016.794205 3656567 buffer_comparator.cc:156] Difference at 513: 712.779, expected 1037.38
E0000 00:00:1743138016.794208 3656567 buffer_comparator.cc:156] Difference at 514: 725.127, expected 1037.27
E0000 00:00:1743138016.794212 3656567 buffer_comparator.cc:156] Difference at 515: 698.019, expected 1030.23
E0000 00:00:1743138016.794215 3656567 buffer_comparator.cc:156] Difference at 516: 707.664, expected 1029.96
E0000 00:00:1743138016.794218 3656567 buffer_comparator.cc:156] Difference at 517: 711.597, expected 1023.46
E0000 00:00:1743138016.794221 3656567 buffer_comparator.cc:156] Difference at 518: 720.495, expected 1036.27
E0000 00:00:1743138016.794224 3656567 buffer_comparator.cc:156] Difference at 519: 719.076, expected 1042.91
E0000 00:00:1743138016.794227 3656567 buffer_comparator.cc:156] Difference at 520: 711.292, expected 1031.19
E0000 00:00:1743138016.794230 3656567 buffer_comparator.cc:156] Difference at 521: 693.751, expected 1022.88
2025-03-28 05:00:16.794237: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.796540 3656567 buffer_comparator.cc:156] Difference at 512: 351.538, expected 1037.38
E0000 00:00:1743138016.796574 3656567 buffer_comparator.cc:156] Difference at 513: 348.221, expected 1037.38
E0000 00:00:1743138016.796577 3656567 buffer_comparator.cc:156] Difference at 514: 361.824, expected 1037.27
E0000 00:00:1743138016.796581 3656567 buffer_comparator.cc:156] Difference at 515: 352.926, expected 1030.23
E0000 00:00:1743138016.796584 3656567 buffer_comparator.cc:156] Difference at 516: 344.916, expected 1029.96
E0000 00:00:1743138016.796587 3656567 buffer_comparator.cc:156] Difference at 517: 356.516, expected 1023.46
E0000 00:00:1743138016.796590 3656567 buffer_comparator.cc:156] Difference at 518: 354.126, expected 1036.27
E0000 00:00:1743138016.796593 3656567 buffer_comparator.cc:156] Difference at 519: 357.665, expected 1042.91
E0000 00:00:1743138016.796596 3656567 buffer_comparator.cc:156] Difference at 520: 359.549, expected 1031.19
E0000 00:00:1743138016.796599 3656567 buffer_comparator.cc:156] Difference at 521: 342.212, expected 1022.88
2025-03-28 05:00:16.796608: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.798949 3656567 buffer_comparator.cc:156] Difference at 512: 351.538, expected 1037.38
E0000 00:00:1743138016.798986 3656567 buffer_comparator.cc:156] Difference at 513: 348.221, expected 1037.38
E0000 00:00:1743138016.798989 3656567 buffer_comparator.cc:156] Difference at 514: 361.824, expected 1037.27
E0000 00:00:1743138016.798993 3656567 buffer_comparator.cc:156] Difference at 515: 352.926, expected 1030.23
E0000 00:00:1743138016.798996 3656567 buffer_comparator.cc:156] Difference at 516: 344.916, expected 1029.96
E0000 00:00:1743138016.798999 3656567 buffer_comparator.cc:156] Difference at 517: 356.516, expected 1023.46
E0000 00:00:1743138016.799002 3656567 buffer_comparator.cc:156] Difference at 518: 354.126, expected 1036.27
E0000 00:00:1743138016.799005 3656567 buffer_comparator.cc:156] Difference at 519: 357.665, expected 1042.91
E0000 00:00:1743138016.799008 3656567 buffer_comparator.cc:156] Difference at 520: 359.549, expected 1031.19
E0000 00:00:1743138016.799011 3656567 buffer_comparator.cc:156] Difference at 521: 342.212, expected 1022.88
2025-03-28 05:00:16.799019: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.801363 3656567 buffer_comparator.cc:156] Difference at 1024: 344.602, expected 1028.47
E0000 00:00:1743138016.801398 3656567 buffer_comparator.cc:156] Difference at 1025: 338.216, expected 1025.84
E0000 00:00:1743138016.801401 3656567 buffer_comparator.cc:156] Difference at 1026: 352.085, expected 1041.27
E0000 00:00:1743138016.801404 3656567 buffer_comparator.cc:156] Difference at 1027: 347.813, expected 1022.34
E0000 00:00:1743138016.801407 3656567 buffer_comparator.cc:156] Difference at 1028: 342.989, expected 1025.87
E0000 00:00:1743138016.801411 3656567 buffer_comparator.cc:156] Difference at 1029: 351.344, expected 1021.14
E0000 00:00:1743138016.801414 3656567 buffer_comparator.cc:156] Difference at 1030: 340.434, expected 1030.37
E0000 00:00:1743138016.801417 3656567 buffer_comparator.cc:156] Difference at 1031: 355.013, expected 1039.95
E0000 00:00:1743138016.801420 3656567 buffer_comparator.cc:156] Difference at 1032: 350.737, expected 1032.45
E0000 00:00:1743138016.801423 3656567 buffer_comparator.cc:156] Difference at 1033: 340.266, expected 1019.38
2025-03-28 05:00:16.801431: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.803751 3656567 buffer_comparator.cc:156] Difference at 38: 924.94, expected 1048.8
E0000 00:00:1743138016.803779 3656567 buffer_comparator.cc:156] Difference at 44: 914.212, expected 1041.72
E0000 00:00:1743138016.803783 3656567 buffer_comparator.cc:156] Difference at 64: 1157.7, expected 1032.49
E0000 00:00:1743138016.803786 3656567 buffer_comparator.cc:156] Difference at 68: 924.491, expected 1037.91
E0000 00:00:1743138016.803790 3656567 buffer_comparator.cc:156] Difference at 96: 1211.63, expected 1026.76
E0000 00:00:1743138016.803793 3656567 buffer_comparator.cc:156] Difference at 109: 1176.07, expected 1029.18
E0000 00:00:1743138016.803797 3656567 buffer_comparator.cc:156] Difference at 162: 923.061, expected 1032.2
E0000 00:00:1743138016.803800 3656567 buffer_comparator.cc:156] Difference at 166: 918.968, expected 1027.87
E0000 00:00:1743138016.803803 3656567 buffer_comparator.cc:156] Difference at 172: 920.028, expected 1025.22
E0000 00:00:1743138016.803807 3656567 buffer_comparator.cc:156] Difference at 208: 1155.01, expected 1016.78
2025-03-28 05:00:16.803816: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.806148 3656567 buffer_comparator.cc:156] Difference at 1024: nan, expected 1028.47
E0000 00:00:1743138016.806170 3656567 buffer_comparator.cc:156] Difference at 1025: nan, expected 1025.84
E0000 00:00:1743138016.806173 3656567 buffer_comparator.cc:156] Difference at 1026: nan, expected 1041.27
E0000 00:00:1743138016.806179 3656567 buffer_comparator.cc:156] Difference at 1027: nan, expected 1022.34
E0000 00:00:1743138016.806182 3656567 buffer_comparator.cc:156] Difference at 1028: nan, expected 1025.87
E0000 00:00:1743138016.806185 3656567 buffer_comparator.cc:156] Difference at 1029: nan, expected 1021.14
E0000 00:00:1743138016.806187 3656567 buffer_comparator.cc:156] Difference at 1030: nan, expected 1030.37
E0000 00:00:1743138016.806190 3656567 buffer_comparator.cc:156] Difference at 1031: nan, expected 1039.95
E0000 00:00:1743138016.806193 3656567 buffer_comparator.cc:156] Difference at 1032: nan, expected 1032.45
E0000 00:00:1743138016.806196 3656567 buffer_comparator.cc:156] Difference at 1033: nan, expected 1019.38
2025-03-28 05:00:16.806202: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.808527 3656567 buffer_comparator.cc:156] Difference at 1024: nan, expected 1028.47
E0000 00:00:1743138016.808546 3656567 buffer_comparator.cc:156] Difference at 1025: nan, expected 1025.84
E0000 00:00:1743138016.808549 3656567 buffer_comparator.cc:156] Difference at 1026: nan, expected 1041.27
E0000 00:00:1743138016.808552 3656567 buffer_comparator.cc:156] Difference at 1027: nan, expected 1022.34
E0000 00:00:1743138016.808555 3656567 buffer_comparator.cc:156] Difference at 1028: nan, expected 1025.87
E0000 00:00:1743138016.808558 3656567 buffer_comparator.cc:156] Difference at 1029: nan, expected 1021.14
E0000 00:00:1743138016.808561 3656567 buffer_comparator.cc:156] Difference at 1030: nan, expected 1030.37
E0000 00:00:1743138016.808563 3656567 buffer_comparator.cc:156] Difference at 1031: nan, expected 1039.95
E0000 00:00:1743138016.808566 3656567 buffer_comparator.cc:156] Difference at 1032: nan, expected 1032.45
E0000 00:00:1743138016.808569 3656567 buffer_comparator.cc:156] Difference at 1033: nan, expected 1019.38
2025-03-28 05:00:16.808575: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.810807 3656567 buffer_comparator.cc:156] Difference at 1024: nan, expected 1028.47
E0000 00:00:1743138016.810825 3656567 buffer_comparator.cc:156] Difference at 1025: nan, expected 1025.84
E0000 00:00:1743138016.810829 3656567 buffer_comparator.cc:156] Difference at 1026: nan, expected 1041.27
E0000 00:00:1743138016.810833 3656567 buffer_comparator.cc:156] Difference at 1027: nan, expected 1022.34
E0000 00:00:1743138016.810836 3656567 buffer_comparator.cc:156] Difference at 1028: nan, expected 1025.87
E0000 00:00:1743138016.810839 3656567 buffer_comparator.cc:156] Difference at 1029: nan, expected 1021.14
E0000 00:00:1743138016.810842 3656567 buffer_comparator.cc:156] Difference at 1030: nan, expected 1030.37
E0000 00:00:1743138016.810844 3656567 buffer_comparator.cc:156] Difference at 1031: nan, expected 1039.95
E0000 00:00:1743138016.810847 3656567 buffer_comparator.cc:156] Difference at 1032: nan, expected 1032.45
E0000 00:00:1743138016.810850 3656567 buffer_comparator.cc:156] Difference at 1033: nan, expected 1019.38
2025-03-28 05:00:16.810855: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138016.813081 3656567 buffer_comparator.cc:156] Difference at 1024: nan, expected 1028.47
E0000 00:00:1743138016.813097 3656567 buffer_comparator.cc:156] Difference at 1025: nan, expected 1025.84
E0000 00:00:1743138016.813100 3656567 buffer_comparator.cc:156] Difference at 1026: nan, expected 1041.27
E0000 00:00:1743138016.813103 3656567 buffer_comparator.cc:156] Difference at 1027: nan, expected 1022.34
E0000 00:00:1743138016.813106 3656567 buffer_comparator.cc:156] Difference at 1028: nan, expected 1025.87
Total Trainable Parameters: 0.1493 M
2025-03-28 05:01:27.997757: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 336 bytes spill stores, 336 bytes spill loads

2025-03-28 05:01:29.749151: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot', 3700 bytes spill stores, 3688 bytes spill loads

2025-03-28 05:01:30.192718: I external/xla/xla/stream_executor/cuda/subprocess_compilation.cc:346] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_751', 56 bytes spill stores, 344 bytes spill loads

E0000 00:00:1743138091.865109 3656567 buffer_comparator.cc:156] Difference at 256: -nan, expected 35.6174
E0000 00:00:1743138091.865210 3656567 buffer_comparator.cc:156] Difference at 257: -nan, expected 35.1987
E0000 00:00:1743138091.865235 3656567 buffer_comparator.cc:156] Difference at 258: -nan, expected 29.9844
E0000 00:00:1743138091.865239 3656567 buffer_comparator.cc:156] Difference at 259: -nan, expected 30.8692
E0000 00:00:1743138091.865244 3656567 buffer_comparator.cc:156] Difference at 260: -nan, expected 31.6219
E0000 00:00:1743138091.865248 3656567 buffer_comparator.cc:156] Difference at 261: -nan, expected 30.9767
E0000 00:00:1743138091.865253 3656567 buffer_comparator.cc:156] Difference at 262: -nan, expected 30.1212
E0000 00:00:1743138091.865257 3656567 buffer_comparator.cc:156] Difference at 263: -nan, expected 33.229
E0000 00:00:1743138091.865262 3656567 buffer_comparator.cc:156] Difference at 264: -nan, expected 34.0365
E0000 00:00:1743138091.865266 3656567 buffer_comparator.cc:156] Difference at 265: -nan, expected 33.1295
2025-03-28 05:01:31.865281: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.867870 3656567 buffer_comparator.cc:156] Difference at 256: -nan, expected 35.6174
E0000 00:00:1743138091.867889 3656567 buffer_comparator.cc:156] Difference at 257: -nan, expected 35.1987
E0000 00:00:1743138091.867895 3656567 buffer_comparator.cc:156] Difference at 258: -nan, expected 29.9844
E0000 00:00:1743138091.867899 3656567 buffer_comparator.cc:156] Difference at 259: -nan, expected 30.8692
E0000 00:00:1743138091.867904 3656567 buffer_comparator.cc:156] Difference at 260: -nan, expected 31.6219
E0000 00:00:1743138091.867908 3656567 buffer_comparator.cc:156] Difference at 261: -nan, expected 30.9767
E0000 00:00:1743138091.867913 3656567 buffer_comparator.cc:156] Difference at 262: -nan, expected 30.1212
E0000 00:00:1743138091.867917 3656567 buffer_comparator.cc:156] Difference at 263: -nan, expected 33.229
E0000 00:00:1743138091.867921 3656567 buffer_comparator.cc:156] Difference at 264: -nan, expected 34.0365
E0000 00:00:1743138091.867926 3656567 buffer_comparator.cc:156] Difference at 265: -nan, expected 33.1295
2025-03-28 05:01:31.867933: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.870352 3656567 buffer_comparator.cc:156] Difference at 256: -nan, expected 35.6174
E0000 00:00:1743138091.870371 3656567 buffer_comparator.cc:156] Difference at 257: -nan, expected 35.1987
E0000 00:00:1743138091.870376 3656567 buffer_comparator.cc:156] Difference at 258: -nan, expected 29.9844
E0000 00:00:1743138091.870381 3656567 buffer_comparator.cc:156] Difference at 259: -nan, expected 30.8692
E0000 00:00:1743138091.870385 3656567 buffer_comparator.cc:156] Difference at 260: -nan, expected 31.6219
E0000 00:00:1743138091.870390 3656567 buffer_comparator.cc:156] Difference at 261: -nan, expected 30.9767
E0000 00:00:1743138091.870394 3656567 buffer_comparator.cc:156] Difference at 262: -nan, expected 30.1212
E0000 00:00:1743138091.870399 3656567 buffer_comparator.cc:156] Difference at 263: -nan, expected 33.229
E0000 00:00:1743138091.870403 3656567 buffer_comparator.cc:156] Difference at 264: -nan, expected 34.0365
E0000 00:00:1743138091.870410 3656567 buffer_comparator.cc:156] Difference at 265: -nan, expected 33.1295
2025-03-28 05:01:31.870417: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.872797 3656567 buffer_comparator.cc:156] Difference at 512: -nan, expected 32.1819
E0000 00:00:1743138091.872814 3656567 buffer_comparator.cc:156] Difference at 513: -nan, expected 31.0574
E0000 00:00:1743138091.872818 3656567 buffer_comparator.cc:156] Difference at 514: -nan, expected 27.3018
E0000 00:00:1743138091.872821 3656567 buffer_comparator.cc:156] Difference at 515: -nan, expected 29
E0000 00:00:1743138091.872824 3656567 buffer_comparator.cc:156] Difference at 516: -nan, expected 28.9931
E0000 00:00:1743138091.872827 3656567 buffer_comparator.cc:156] Difference at 517: -nan, expected 26.8922
E0000 00:00:1743138091.872830 3656567 buffer_comparator.cc:156] Difference at 518: -nan, expected 30.7309
E0000 00:00:1743138091.872833 3656567 buffer_comparator.cc:156] Difference at 519: -nan, expected 30.711
E0000 00:00:1743138091.872836 3656567 buffer_comparator.cc:156] Difference at 520: -nan, expected 32.7655
E0000 00:00:1743138091.872839 3656567 buffer_comparator.cc:156] Difference at 521: -nan, expected 31.5494
2025-03-28 05:01:31.872844: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.875338 3656567 buffer_comparator.cc:156] Difference at 512: -nan, expected 32.1819
E0000 00:00:1743138091.875353 3656567 buffer_comparator.cc:156] Difference at 513: -nan, expected 31.0574
E0000 00:00:1743138091.875358 3656567 buffer_comparator.cc:156] Difference at 514: -nan, expected 27.3018
E0000 00:00:1743138091.875360 3656567 buffer_comparator.cc:156] Difference at 515: -nan, expected 29
E0000 00:00:1743138091.875363 3656567 buffer_comparator.cc:156] Difference at 516: -nan, expected 28.9931
E0000 00:00:1743138091.875366 3656567 buffer_comparator.cc:156] Difference at 517: -nan, expected 26.8922
E0000 00:00:1743138091.875369 3656567 buffer_comparator.cc:156] Difference at 518: -nan, expected 30.7309
E0000 00:00:1743138091.875372 3656567 buffer_comparator.cc:156] Difference at 519: -nan, expected 30.711
E0000 00:00:1743138091.875375 3656567 buffer_comparator.cc:156] Difference at 520: -nan, expected 32.7655
E0000 00:00:1743138091.875378 3656567 buffer_comparator.cc:156] Difference at 521: -nan, expected 31.5494
2025-03-28 05:01:31.875383: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.877685 3656567 buffer_comparator.cc:156] Difference at 512: -nan, expected 32.1819
E0000 00:00:1743138091.877701 3656567 buffer_comparator.cc:156] Difference at 513: -nan, expected 31.0574
E0000 00:00:1743138091.877705 3656567 buffer_comparator.cc:156] Difference at 514: -nan, expected 27.3018
E0000 00:00:1743138091.877708 3656567 buffer_comparator.cc:156] Difference at 515: -nan, expected 29
E0000 00:00:1743138091.877711 3656567 buffer_comparator.cc:156] Difference at 516: -nan, expected 28.9931
E0000 00:00:1743138091.877714 3656567 buffer_comparator.cc:156] Difference at 517: -nan, expected 26.8922
E0000 00:00:1743138091.877717 3656567 buffer_comparator.cc:156] Difference at 518: -nan, expected 30.7309
E0000 00:00:1743138091.877719 3656567 buffer_comparator.cc:156] Difference at 519: -nan, expected 30.711
E0000 00:00:1743138091.877722 3656567 buffer_comparator.cc:156] Difference at 520: -nan, expected 32.7655
E0000 00:00:1743138091.877725 3656567 buffer_comparator.cc:156] Difference at 521: -nan, expected 31.5494
2025-03-28 05:01:31.877730: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.880077 3656567 buffer_comparator.cc:156] Difference at 1024: -nan, expected 31.9461
E0000 00:00:1743138091.880091 3656567 buffer_comparator.cc:156] Difference at 1025: -nan, expected 32.6886
E0000 00:00:1743138091.880095 3656567 buffer_comparator.cc:156] Difference at 1026: -nan, expected 26.3857
E0000 00:00:1743138091.880098 3656567 buffer_comparator.cc:156] Difference at 1027: -nan, expected 30.2342
E0000 00:00:1743138091.880101 3656567 buffer_comparator.cc:156] Difference at 1028: -nan, expected 29.0966
E0000 00:00:1743138091.880104 3656567 buffer_comparator.cc:156] Difference at 1029: -nan, expected 28.3209
E0000 00:00:1743138091.880107 3656567 buffer_comparator.cc:156] Difference at 1030: -nan, expected 29.9067
E0000 00:00:1743138091.880110 3656567 buffer_comparator.cc:156] Difference at 1031: -nan, expected 29.6903
E0000 00:00:1743138091.880113 3656567 buffer_comparator.cc:156] Difference at 1032: -nan, expected 31.1917
E0000 00:00:1743138091.880116 3656567 buffer_comparator.cc:156] Difference at 1033: -nan, expected 30.8328
2025-03-28 05:01:31.880120: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.882394 3656567 buffer_comparator.cc:156] Difference at 1024: -nan, expected 31.9461
E0000 00:00:1743138091.882408 3656567 buffer_comparator.cc:156] Difference at 1025: -nan, expected 32.6886
E0000 00:00:1743138091.882412 3656567 buffer_comparator.cc:156] Difference at 1026: -nan, expected 26.3857
E0000 00:00:1743138091.882415 3656567 buffer_comparator.cc:156] Difference at 1027: -nan, expected 30.2342
E0000 00:00:1743138091.882418 3656567 buffer_comparator.cc:156] Difference at 1028: -nan, expected 29.0966
E0000 00:00:1743138091.882421 3656567 buffer_comparator.cc:156] Difference at 1029: -nan, expected 28.3209
E0000 00:00:1743138091.882424 3656567 buffer_comparator.cc:156] Difference at 1030: -nan, expected 29.9067
E0000 00:00:1743138091.882427 3656567 buffer_comparator.cc:156] Difference at 1031: -nan, expected 29.6903
E0000 00:00:1743138091.882430 3656567 buffer_comparator.cc:156] Difference at 1032: -nan, expected 31.1917
E0000 00:00:1743138091.882432 3656567 buffer_comparator.cc:156] Difference at 1033: -nan, expected 30.8328
2025-03-28 05:01:31.882437: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.884731 3656567 buffer_comparator.cc:156] Difference at 1024: -nan, expected 31.9461
E0000 00:00:1743138091.884745 3656567 buffer_comparator.cc:156] Difference at 1025: -nan, expected 32.6886
E0000 00:00:1743138091.884749 3656567 buffer_comparator.cc:156] Difference at 1026: -nan, expected 26.3857
E0000 00:00:1743138091.884752 3656567 buffer_comparator.cc:156] Difference at 1027: -nan, expected 30.2342
E0000 00:00:1743138091.884755 3656567 buffer_comparator.cc:156] Difference at 1028: -nan, expected 29.0966
E0000 00:00:1743138091.884757 3656567 buffer_comparator.cc:156] Difference at 1029: -nan, expected 28.3209
E0000 00:00:1743138091.884760 3656567 buffer_comparator.cc:156] Difference at 1030: -nan, expected 29.9067
E0000 00:00:1743138091.884763 3656567 buffer_comparator.cc:156] Difference at 1031: -nan, expected 29.6903
E0000 00:00:1743138091.884766 3656567 buffer_comparator.cc:156] Difference at 1032: -nan, expected 31.1917
E0000 00:00:1743138091.884769 3656567 buffer_comparator.cc:156] Difference at 1033: -nan, expected 30.8328
2025-03-28 05:01:31.884774: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.887055 3656567 buffer_comparator.cc:156] Difference at 1024: -nan, expected 31.9461
E0000 00:00:1743138091.887068 3656567 buffer_comparator.cc:156] Difference at 1025: -nan, expected 32.6886
E0000 00:00:1743138091.887073 3656567 buffer_comparator.cc:156] Difference at 1026: -nan, expected 26.3857
E0000 00:00:1743138091.887076 3656567 buffer_comparator.cc:156] Difference at 1027: -nan, expected 30.2342
E0000 00:00:1743138091.887079 3656567 buffer_comparator.cc:156] Difference at 1028: -nan, expected 29.0966
E0000 00:00:1743138091.887082 3656567 buffer_comparator.cc:156] Difference at 1029: -nan, expected 28.3209
E0000 00:00:1743138091.887085 3656567 buffer_comparator.cc:156] Difference at 1030: -nan, expected 29.9067
E0000 00:00:1743138091.887088 3656567 buffer_comparator.cc:156] Difference at 1031: -nan, expected 29.6903
E0000 00:00:1743138091.887091 3656567 buffer_comparator.cc:156] Difference at 1032: -nan, expected 31.1917
E0000 00:00:1743138091.887093 3656567 buffer_comparator.cc:156] Difference at 1033: -nan, expected 30.8328
2025-03-28 05:01:31.887098: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.889377 3656567 buffer_comparator.cc:156] Difference at 1024: -nan, expected 31.9461
E0000 00:00:1743138091.889392 3656567 buffer_comparator.cc:156] Difference at 1025: -nan, expected 32.6886
E0000 00:00:1743138091.889395 3656567 buffer_comparator.cc:156] Difference at 1026: -nan, expected 26.3857
E0000 00:00:1743138091.889398 3656567 buffer_comparator.cc:156] Difference at 1027: -nan, expected 30.2342
E0000 00:00:1743138091.889401 3656567 buffer_comparator.cc:156] Difference at 1028: -nan, expected 29.0966
E0000 00:00:1743138091.889404 3656567 buffer_comparator.cc:156] Difference at 1029: -nan, expected 28.3209
E0000 00:00:1743138091.889407 3656567 buffer_comparator.cc:156] Difference at 1030: -nan, expected 29.9067
E0000 00:00:1743138091.889410 3656567 buffer_comparator.cc:156] Difference at 1031: -nan, expected 29.6903
E0000 00:00:1743138091.889413 3656567 buffer_comparator.cc:156] Difference at 1032: -nan, expected 31.1917
E0000 00:00:1743138091.889416 3656567 buffer_comparator.cc:156] Difference at 1033: -nan, expected 30.8328
2025-03-28 05:01:31.889420: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.891709 3656567 buffer_comparator.cc:156] Difference at 1024: -nan, expected 31.9461
E0000 00:00:1743138091.891723 3656567 buffer_comparator.cc:156] Difference at 1025: -nan, expected 32.6886
E0000 00:00:1743138091.891727 3656567 buffer_comparator.cc:156] Difference at 1026: -nan, expected 26.3857
E0000 00:00:1743138091.891730 3656567 buffer_comparator.cc:156] Difference at 1027: -nan, expected 30.2342
E0000 00:00:1743138091.891733 3656567 buffer_comparator.cc:156] Difference at 1028: -nan, expected 29.0966
E0000 00:00:1743138091.891736 3656567 buffer_comparator.cc:156] Difference at 1029: -nan, expected 28.3209
E0000 00:00:1743138091.891739 3656567 buffer_comparator.cc:156] Difference at 1030: -nan, expected 29.9067
E0000 00:00:1743138091.891742 3656567 buffer_comparator.cc:156] Difference at 1031: -nan, expected 29.6903
E0000 00:00:1743138091.891744 3656567 buffer_comparator.cc:156] Difference at 1032: -nan, expected 31.1917
E0000 00:00:1743138091.891747 3656567 buffer_comparator.cc:156] Difference at 1033: -nan, expected 30.8328
2025-03-28 05:01:31.891752: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.894058 3656567 buffer_comparator.cc:156] Difference at 2048: -nan, expected 34.1171
E0000 00:00:1743138091.894075 3656567 buffer_comparator.cc:156] Difference at 2049: -nan, expected 32.4667
E0000 00:00:1743138091.894079 3656567 buffer_comparator.cc:156] Difference at 2050: -nan, expected 26.1563
E0000 00:00:1743138091.894082 3656567 buffer_comparator.cc:156] Difference at 2051: -nan, expected 28.6045
E0000 00:00:1743138091.894085 3656567 buffer_comparator.cc:156] Difference at 2052: -nan, expected 29.7198
E0000 00:00:1743138091.894090 3656567 buffer_comparator.cc:156] Difference at 2053: -nan, expected 27.7798
E0000 00:00:1743138091.894093 3656567 buffer_comparator.cc:156] Difference at 2054: -nan, expected 30.2458
E0000 00:00:1743138091.894096 3656567 buffer_comparator.cc:156] Difference at 2055: -nan, expected 29.5776
E0000 00:00:1743138091.894098 3656567 buffer_comparator.cc:156] Difference at 2056: -nan, expected 31.5941
E0000 00:00:1743138091.894101 3656567 buffer_comparator.cc:156] Difference at 2057: -nan, expected 31.3044
2025-03-28 05:01:31.894107: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.896439 3656567 buffer_comparator.cc:156] Difference at 2048: -nan, expected 34.1171
E0000 00:00:1743138091.896453 3656567 buffer_comparator.cc:156] Difference at 2049: -nan, expected 32.4667
E0000 00:00:1743138091.896457 3656567 buffer_comparator.cc:156] Difference at 2050: -nan, expected 26.1563
E0000 00:00:1743138091.896460 3656567 buffer_comparator.cc:156] Difference at 2051: -nan, expected 28.6045
E0000 00:00:1743138091.896463 3656567 buffer_comparator.cc:156] Difference at 2052: -nan, expected 29.7198
E0000 00:00:1743138091.896466 3656567 buffer_comparator.cc:156] Difference at 2053: -nan, expected 27.7798
E0000 00:00:1743138091.896469 3656567 buffer_comparator.cc:156] Difference at 2054: -nan, expected 30.2458
E0000 00:00:1743138091.896472 3656567 buffer_comparator.cc:156] Difference at 2055: -nan, expected 29.5776
E0000 00:00:1743138091.896475 3656567 buffer_comparator.cc:156] Difference at 2056: -nan, expected 31.5941
E0000 00:00:1743138091.896478 3656567 buffer_comparator.cc:156] Difference at 2057: -nan, expected 31.3044
2025-03-28 05:01:31.896482: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.898796 3656567 buffer_comparator.cc:156] Difference at 2048: -nan, expected 34.1171
E0000 00:00:1743138091.898811 3656567 buffer_comparator.cc:156] Difference at 2049: -nan, expected 32.4667
E0000 00:00:1743138091.898815 3656567 buffer_comparator.cc:156] Difference at 2050: -nan, expected 26.1563
E0000 00:00:1743138091.898818 3656567 buffer_comparator.cc:156] Difference at 2051: -nan, expected 28.6045
E0000 00:00:1743138091.898821 3656567 buffer_comparator.cc:156] Difference at 2052: -nan, expected 29.7198
E0000 00:00:1743138091.898824 3656567 buffer_comparator.cc:156] Difference at 2053: -nan, expected 27.7798
E0000 00:00:1743138091.898827 3656567 buffer_comparator.cc:156] Difference at 2054: -nan, expected 30.2458
E0000 00:00:1743138091.898830 3656567 buffer_comparator.cc:156] Difference at 2055: -nan, expected 29.5776
E0000 00:00:1743138091.898833 3656567 buffer_comparator.cc:156] Difference at 2056: -nan, expected 31.5941
E0000 00:00:1743138091.898836 3656567 buffer_comparator.cc:156] Difference at 2057: -nan, expected 31.3044
2025-03-28 05:01:31.898841: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.901169 3656567 buffer_comparator.cc:156] Difference at 14: 35.7463, expected 31.9276
E0000 00:00:1743138091.901185 3656567 buffer_comparator.cc:156] Difference at 33: 24.9115, expected 30.7605
E0000 00:00:1743138091.901190 3656567 buffer_comparator.cc:156] Difference at 34: 19.8613, expected 26.0849
E0000 00:00:1743138091.901193 3656567 buffer_comparator.cc:156] Difference at 35: 21.3149, expected 26.4869
E0000 00:00:1743138091.901196 3656567 buffer_comparator.cc:156] Difference at 36: 21.7164, expected 27.19
E0000 00:00:1743138091.901199 3656567 buffer_comparator.cc:156] Difference at 38: 23.1726, expected 27.0763
E0000 00:00:1743138091.901203 3656567 buffer_comparator.cc:156] Difference at 39: 22.3473, expected 27.6611
E0000 00:00:1743138091.901207 3656567 buffer_comparator.cc:156] Difference at 40: 26.6328, expected 32.7606
E0000 00:00:1743138091.901211 3656567 buffer_comparator.cc:156] Difference at 41: 22.0337, expected 28.8157
E0000 00:00:1743138091.901214 3656567 buffer_comparator.cc:156] Difference at 42: 24.6832, expected 27.946
2025-03-28 05:01:31.901219: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.903523 3656567 buffer_comparator.cc:156] Difference at 14: 35.7463, expected 31.9276
E0000 00:00:1743138091.903538 3656567 buffer_comparator.cc:156] Difference at 33: 24.9115, expected 30.7605
E0000 00:00:1743138091.903542 3656567 buffer_comparator.cc:156] Difference at 34: 19.8613, expected 26.0849
E0000 00:00:1743138091.903545 3656567 buffer_comparator.cc:156] Difference at 35: 21.3149, expected 26.4869
E0000 00:00:1743138091.903549 3656567 buffer_comparator.cc:156] Difference at 36: 21.7164, expected 27.19
E0000 00:00:1743138091.903552 3656567 buffer_comparator.cc:156] Difference at 38: 23.1726, expected 27.0763
E0000 00:00:1743138091.903555 3656567 buffer_comparator.cc:156] Difference at 39: 22.3473, expected 27.6611
E0000 00:00:1743138091.903558 3656567 buffer_comparator.cc:156] Difference at 40: 26.6328, expected 32.7606
E0000 00:00:1743138091.903561 3656567 buffer_comparator.cc:156] Difference at 41: 22.0337, expected 28.8157
E0000 00:00:1743138091.903564 3656567 buffer_comparator.cc:156] Difference at 42: 24.6832, expected 27.946
2025-03-28 05:01:31.903569: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.905846 3656567 buffer_comparator.cc:156] Difference at 2048: nan, expected 34.1171
E0000 00:00:1743138091.905860 3656567 buffer_comparator.cc:156] Difference at 2049: nan, expected 32.4667
E0000 00:00:1743138091.905864 3656567 buffer_comparator.cc:156] Difference at 2050: nan, expected 26.1563
E0000 00:00:1743138091.905867 3656567 buffer_comparator.cc:156] Difference at 2051: nan, expected 28.6045
E0000 00:00:1743138091.905870 3656567 buffer_comparator.cc:156] Difference at 2052: nan, expected 29.7198
E0000 00:00:1743138091.905873 3656567 buffer_comparator.cc:156] Difference at 2053: nan, expected 27.7798
E0000 00:00:1743138091.905875 3656567 buffer_comparator.cc:156] Difference at 2054: nan, expected 30.2458
E0000 00:00:1743138091.905878 3656567 buffer_comparator.cc:156] Difference at 2055: nan, expected 29.5776
E0000 00:00:1743138091.905881 3656567 buffer_comparator.cc:156] Difference at 2056: nan, expected 31.5941
E0000 00:00:1743138091.905884 3656567 buffer_comparator.cc:156] Difference at 2057: nan, expected 31.3044
2025-03-28 05:01:31.905889: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.908167 3656567 buffer_comparator.cc:156] Difference at 4096: nan, expected 35.3139
E0000 00:00:1743138091.908181 3656567 buffer_comparator.cc:156] Difference at 4097: nan, expected 33.723
E0000 00:00:1743138091.908184 3656567 buffer_comparator.cc:156] Difference at 4098: nan, expected 30.5046
E0000 00:00:1743138091.908187 3656567 buffer_comparator.cc:156] Difference at 4099: nan, expected 31.9991
E0000 00:00:1743138091.908190 3656567 buffer_comparator.cc:156] Difference at 4100: nan, expected 32.3158
E0000 00:00:1743138091.908193 3656567 buffer_comparator.cc:156] Difference at 4101: nan, expected 31.243
E0000 00:00:1743138091.908196 3656567 buffer_comparator.cc:156] Difference at 4102: nan, expected 32.8456
E0000 00:00:1743138091.908199 3656567 buffer_comparator.cc:156] Difference at 4103: nan, expected 34.6195
E0000 00:00:1743138091.908202 3656567 buffer_comparator.cc:156] Difference at 4104: nan, expected 35.1943
E0000 00:00:1743138091.908205 3656567 buffer_comparator.cc:156] Difference at 4105: nan, expected 31.1287
2025-03-28 05:01:31.908211: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.910475 3656567 buffer_comparator.cc:156] Difference at 4096: nan, expected 35.3139
E0000 00:00:1743138091.910488 3656567 buffer_comparator.cc:156] Difference at 4097: nan, expected 33.723
E0000 00:00:1743138091.910492 3656567 buffer_comparator.cc:156] Difference at 4098: nan, expected 30.5046
E0000 00:00:1743138091.910495 3656567 buffer_comparator.cc:156] Difference at 4099: nan, expected 31.9991
E0000 00:00:1743138091.910498 3656567 buffer_comparator.cc:156] Difference at 4100: nan, expected 32.3158
E0000 00:00:1743138091.910501 3656567 buffer_comparator.cc:156] Difference at 4101: nan, expected 31.243
E0000 00:00:1743138091.910504 3656567 buffer_comparator.cc:156] Difference at 4102: nan, expected 32.8456
E0000 00:00:1743138091.910507 3656567 buffer_comparator.cc:156] Difference at 4103: nan, expected 34.6195
E0000 00:00:1743138091.910509 3656567 buffer_comparator.cc:156] Difference at 4104: nan, expected 35.1943
E0000 00:00:1743138091.910512 3656567 buffer_comparator.cc:156] Difference at 4105: nan, expected 31.1287
2025-03-28 05:01:31.910517: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.912801 3656567 buffer_comparator.cc:156] Difference at 4096: nan, expected 35.3139
E0000 00:00:1743138091.912816 3656567 buffer_comparator.cc:156] Difference at 4097: nan, expected 33.723
E0000 00:00:1743138091.912820 3656567 buffer_comparator.cc:156] Difference at 4098: nan, expected 30.5046
E0000 00:00:1743138091.912823 3656567 buffer_comparator.cc:156] Difference at 4099: nan, expected 31.9991
E0000 00:00:1743138091.912826 3656567 buffer_comparator.cc:156] Difference at 4100: nan, expected 32.3158
E0000 00:00:1743138091.912829 3656567 buffer_comparator.cc:156] Difference at 4101: nan, expected 31.243
E0000 00:00:1743138091.912832 3656567 buffer_comparator.cc:156] Difference at 4102: nan, expected 32.8456
E0000 00:00:1743138091.912835 3656567 buffer_comparator.cc:156] Difference at 4103: nan, expected 34.6195
E0000 00:00:1743138091.912837 3656567 buffer_comparator.cc:156] Difference at 4104: nan, expected 35.1943
E0000 00:00:1743138091.912840 3656567 buffer_comparator.cc:156] Difference at 4105: nan, expected 31.1287
2025-03-28 05:01:31.912845: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.919798 3656567 buffer_comparator.cc:156] Difference at 16: 34.1914, expected 3.67967
E0000 00:00:1743138091.919856 3656567 buffer_comparator.cc:156] Difference at 17: 35.9985, expected 2.49503
E0000 00:00:1743138091.919864 3656567 buffer_comparator.cc:156] Difference at 18: 32.5913, expected 2.74918
E0000 00:00:1743138091.919867 3656567 buffer_comparator.cc:156] Difference at 19: 31.4616, expected 2.07467
E0000 00:00:1743138091.919870 3656567 buffer_comparator.cc:156] Difference at 20: 30.6486, expected 1.64784
E0000 00:00:1743138091.919874 3656567 buffer_comparator.cc:156] Difference at 21: 33.8629, expected 1.20572
E0000 00:00:1743138091.919877 3656567 buffer_comparator.cc:156] Difference at 22: 34.2281, expected 2.92035
E0000 00:00:1743138091.919880 3656567 buffer_comparator.cc:156] Difference at 23: 35.4912, expected 2.59559
E0000 00:00:1743138091.919883 3656567 buffer_comparator.cc:156] Difference at 24: 36.1501, expected 2.08935
E0000 00:00:1743138091.919886 3656567 buffer_comparator.cc:156] Difference at 25: 33.4235, expected 1.7094
2025-03-28 05:01:31.919896: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.922895 3656567 buffer_comparator.cc:156] Difference at 16: 34.1914, expected 3.67967
E0000 00:00:1743138091.922928 3656567 buffer_comparator.cc:156] Difference at 17: 35.9985, expected 2.49503
E0000 00:00:1743138091.922933 3656567 buffer_comparator.cc:156] Difference at 18: 32.5913, expected 2.74918
E0000 00:00:1743138091.922936 3656567 buffer_comparator.cc:156] Difference at 19: 31.4616, expected 2.07467
E0000 00:00:1743138091.922940 3656567 buffer_comparator.cc:156] Difference at 20: 30.6486, expected 1.64784
E0000 00:00:1743138091.922943 3656567 buffer_comparator.cc:156] Difference at 21: 33.8629, expected 1.20572
E0000 00:00:1743138091.922946 3656567 buffer_comparator.cc:156] Difference at 22: 34.2281, expected 2.92035
E0000 00:00:1743138091.922949 3656567 buffer_comparator.cc:156] Difference at 23: 35.4912, expected 2.59559
E0000 00:00:1743138091.922952 3656567 buffer_comparator.cc:156] Difference at 24: 36.1501, expected 2.08935
E0000 00:00:1743138091.922955 3656567 buffer_comparator.cc:156] Difference at 25: 33.4235, expected 1.7094
2025-03-28 05:01:31.922962: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.925892 3656567 buffer_comparator.cc:156] Difference at 64: 31.6261, expected 1.95636
E0000 00:00:1743138091.925917 3656567 buffer_comparator.cc:156] Difference at 65: 32.5711, expected 2.45016
E0000 00:00:1743138091.925922 3656567 buffer_comparator.cc:156] Difference at 66: 26.0687, expected 2.54367
E0000 00:00:1743138091.925925 3656567 buffer_comparator.cc:156] Difference at 67: 28.248, expected 2.35805
E0000 00:00:1743138091.925928 3656567 buffer_comparator.cc:156] Difference at 68: 29.7137, expected 2.35103
E0000 00:00:1743138091.925931 3656567 buffer_comparator.cc:156] Difference at 69: 29.2431, expected 2.02228
E0000 00:00:1743138091.925934 3656567 buffer_comparator.cc:156] Difference at 70: 29.1362, expected 2.32733
E0000 00:00:1743138091.925938 3656567 buffer_comparator.cc:156] Difference at 71: 31.4437, expected 2.48089
E0000 00:00:1743138091.925941 3656567 buffer_comparator.cc:156] Difference at 72: 31.0749, expected 2.31803
E0000 00:00:1743138091.925944 3656567 buffer_comparator.cc:156] Difference at 73: 29.4565, expected 2.33896
2025-03-28 05:01:31.925950: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.928894 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.928918 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.928923 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.928926 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.928929 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.928932 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.928936 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.928939 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.928942 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.928945 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.928951: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.931855 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.931877 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.931881 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.931886 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.931889 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.931893 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.931896 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.931899 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.931902 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.931905 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.931911: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.934828 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.934851 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.934856 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.934859 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.934862 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.934865 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.934868 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.934871 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.934874 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.934877 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.934883: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.937794 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.937815 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.937820 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.937823 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.937826 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.937829 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.937832 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.937835 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.937839 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.937842 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.937847: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.940804 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.940826 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.940831 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.940834 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.940837 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.940842 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.940846 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.940849 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.940852 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.940855 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.940861: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.943803 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.943819 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.943823 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.943826 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.943830 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.943833 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.943836 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.943839 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.943842 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.943845 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.943849: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.946792 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.946808 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.946812 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.946815 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.946818 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.946821 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.946824 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.946827 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.946830 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.946833 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.946838: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.949748 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.949766 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.949771 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.949774 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.949777 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.949780 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.949783 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.949788 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.949791 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.949794 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.949800: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.952741 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.952762 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.952767 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.952770 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.952773 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.952776 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.952779 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.952782 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.952785 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.952788 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.952794: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.955878 3656567 buffer_comparator.cc:156] Difference at 128: 36.554, expected 1.66001
E0000 00:00:1743138091.955898 3656567 buffer_comparator.cc:156] Difference at 129: 33.3869, expected 2.1534
E0000 00:00:1743138091.955903 3656567 buffer_comparator.cc:156] Difference at 130: 29.9477, expected 3.04883
E0000 00:00:1743138091.955906 3656567 buffer_comparator.cc:156] Difference at 131: 29.5455, expected 1.95862
E0000 00:00:1743138091.955909 3656567 buffer_comparator.cc:156] Difference at 132: 30.3501, expected 1.61748
E0000 00:00:1743138091.955912 3656567 buffer_comparator.cc:156] Difference at 133: 29.7812, expected 2.64003
E0000 00:00:1743138091.955915 3656567 buffer_comparator.cc:156] Difference at 134: 30.7704, expected 2.44252
E0000 00:00:1743138091.955918 3656567 buffer_comparator.cc:156] Difference at 135: 32.8145, expected 3.17516
E0000 00:00:1743138091.955921 3656567 buffer_comparator.cc:156] Difference at 136: 33.7876, expected 3.17344
E0000 00:00:1743138091.955924 3656567 buffer_comparator.cc:156] Difference at 137: 32.6281, expected 1.32327
2025-03-28 05:01:31.955930: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.958857 3656567 buffer_comparator.cc:156] Difference at 256: 33.8549, expected 2.24226
E0000 00:00:1743138091.958876 3656567 buffer_comparator.cc:156] Difference at 257: 34.9547, expected 1.76633
E0000 00:00:1743138091.958881 3656567 buffer_comparator.cc:156] Difference at 258: 25.9137, expected 2.52916
E0000 00:00:1743138091.958884 3656567 buffer_comparator.cc:156] Difference at 259: 27.69, expected 3.12706
E0000 00:00:1743138091.958887 3656567 buffer_comparator.cc:156] Difference at 260: 30.4811, expected 2.61699
E0000 00:00:1743138091.958890 3656567 buffer_comparator.cc:156] Difference at 261: 29.9915, expected 2.01735
E0000 00:00:1743138091.958893 3656567 buffer_comparator.cc:156] Difference at 262: 32.71, expected 2.64784
E0000 00:00:1743138091.958896 3656567 buffer_comparator.cc:156] Difference at 263: 33.0606, expected 1.73141
E0000 00:00:1743138091.958899 3656567 buffer_comparator.cc:156] Difference at 264: 31.7765, expected 0.907326
E0000 00:00:1743138091.958905 3656567 buffer_comparator.cc:156] Difference at 265: 30.1216, expected 2.93559
2025-03-28 05:01:31.958910: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.961814 3656567 buffer_comparator.cc:156] Difference at 256: 33.8549, expected 2.24226
E0000 00:00:1743138091.961828 3656567 buffer_comparator.cc:156] Difference at 257: 34.9547, expected 1.76633
E0000 00:00:1743138091.961832 3656567 buffer_comparator.cc:156] Difference at 258: 25.9137, expected 2.52916
E0000 00:00:1743138091.961835 3656567 buffer_comparator.cc:156] Difference at 259: 27.69, expected 3.12706
E0000 00:00:1743138091.961839 3656567 buffer_comparator.cc:156] Difference at 260: 30.4811, expected 2.61699
E0000 00:00:1743138091.961842 3656567 buffer_comparator.cc:156] Difference at 261: 29.9915, expected 2.01735
E0000 00:00:1743138091.961845 3656567 buffer_comparator.cc:156] Difference at 262: 32.71, expected 2.64784
E0000 00:00:1743138091.961848 3656567 buffer_comparator.cc:156] Difference at 263: 33.0606, expected 1.73141
E0000 00:00:1743138091.961851 3656567 buffer_comparator.cc:156] Difference at 264: 31.7765, expected 0.907326
E0000 00:00:1743138091.961854 3656567 buffer_comparator.cc:156] Difference at 265: 30.1216, expected 2.93559
2025-03-28 05:01:31.961859: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.964764 3656567 buffer_comparator.cc:156] Difference at 256: 33.8549, expected 2.24226
E0000 00:00:1743138091.964784 3656567 buffer_comparator.cc:156] Difference at 257: 34.9547, expected 1.76633
E0000 00:00:1743138091.964789 3656567 buffer_comparator.cc:156] Difference at 258: 25.9137, expected 2.52916
E0000 00:00:1743138091.964792 3656567 buffer_comparator.cc:156] Difference at 259: 27.69, expected 3.12706
E0000 00:00:1743138091.964795 3656567 buffer_comparator.cc:156] Difference at 260: 30.4811, expected 2.61699
E0000 00:00:1743138091.964798 3656567 buffer_comparator.cc:156] Difference at 261: 29.9915, expected 2.01735
E0000 00:00:1743138091.964801 3656567 buffer_comparator.cc:156] Difference at 262: 32.71, expected 2.64784
E0000 00:00:1743138091.964804 3656567 buffer_comparator.cc:156] Difference at 263: 33.0606, expected 1.73141
E0000 00:00:1743138091.964808 3656567 buffer_comparator.cc:156] Difference at 264: 31.7765, expected 0.907326
E0000 00:00:1743138091.964811 3656567 buffer_comparator.cc:156] Difference at 265: 30.1216, expected 2.93559
2025-03-28 05:01:31.964817: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.967764 3656567 buffer_comparator.cc:156] Difference at 256: 33.8549, expected 2.24226
E0000 00:00:1743138091.967784 3656567 buffer_comparator.cc:156] Difference at 257: 34.9547, expected 1.76633
E0000 00:00:1743138091.967789 3656567 buffer_comparator.cc:156] Difference at 258: 25.9137, expected 2.52916
E0000 00:00:1743138091.967792 3656567 buffer_comparator.cc:156] Difference at 259: 27.69, expected 3.12706
E0000 00:00:1743138091.967795 3656567 buffer_comparator.cc:156] Difference at 260: 30.4811, expected 2.61699
E0000 00:00:1743138091.967798 3656567 buffer_comparator.cc:156] Difference at 261: 29.9915, expected 2.01735
E0000 00:00:1743138091.967801 3656567 buffer_comparator.cc:156] Difference at 262: 32.71, expected 2.64784
E0000 00:00:1743138091.967804 3656567 buffer_comparator.cc:156] Difference at 263: 33.0606, expected 1.73141
E0000 00:00:1743138091.967807 3656567 buffer_comparator.cc:156] Difference at 264: 31.7765, expected 0.907326
E0000 00:00:1743138091.967810 3656567 buffer_comparator.cc:156] Difference at 265: 30.1216, expected 2.93559
2025-03-28 05:01:31.967816: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.970761 3656567 buffer_comparator.cc:156] Difference at 256: 33.8549, expected 2.24226
E0000 00:00:1743138091.970779 3656567 buffer_comparator.cc:156] Difference at 257: 34.9547, expected 1.76633
E0000 00:00:1743138091.970783 3656567 buffer_comparator.cc:156] Difference at 258: 25.9137, expected 2.52916
E0000 00:00:1743138091.970786 3656567 buffer_comparator.cc:156] Difference at 259: 27.69, expected 3.12706
E0000 00:00:1743138091.970789 3656567 buffer_comparator.cc:156] Difference at 260: 30.4811, expected 2.61699
E0000 00:00:1743138091.970792 3656567 buffer_comparator.cc:156] Difference at 261: 29.9915, expected 2.01735
E0000 00:00:1743138091.970795 3656567 buffer_comparator.cc:156] Difference at 262: 32.71, expected 2.64784
E0000 00:00:1743138091.970798 3656567 buffer_comparator.cc:156] Difference at 263: 33.0606, expected 1.73141
E0000 00:00:1743138091.970801 3656567 buffer_comparator.cc:156] Difference at 264: 31.7765, expected 0.907326
E0000 00:00:1743138091.970805 3656567 buffer_comparator.cc:156] Difference at 265: 30.1216, expected 2.93559
2025-03-28 05:01:31.970811: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.973934 3656567 buffer_comparator.cc:156] Difference at 256: 33.8549, expected 2.24226
E0000 00:00:1743138091.973957 3656567 buffer_comparator.cc:156] Difference at 257: 34.9547, expected 1.76633
E0000 00:00:1743138091.973961 3656567 buffer_comparator.cc:156] Difference at 258: 25.9137, expected 2.52916
E0000 00:00:1743138091.973964 3656567 buffer_comparator.cc:156] Difference at 259: 27.69, expected 3.12706
E0000 00:00:1743138091.973967 3656567 buffer_comparator.cc:156] Difference at 260: 30.4811, expected 2.61699
E0000 00:00:1743138091.973970 3656567 buffer_comparator.cc:156] Difference at 261: 29.9915, expected 2.01735
E0000 00:00:1743138091.973973 3656567 buffer_comparator.cc:156] Difference at 262: 32.71, expected 2.64784
E0000 00:00:1743138091.973976 3656567 buffer_comparator.cc:156] Difference at 263: 33.0606, expected 1.73141
E0000 00:00:1743138091.973979 3656567 buffer_comparator.cc:156] Difference at 264: 31.7765, expected 0.907326
E0000 00:00:1743138091.973983 3656567 buffer_comparator.cc:156] Difference at 265: 30.1216, expected 2.93559
2025-03-28 05:01:31.973988: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.977110 3656567 buffer_comparator.cc:156] Difference at 33: 907.216, expected 1025.71
E0000 00:00:1743138091.977128 3656567 buffer_comparator.cc:156] Difference at 34: 915.936, expected 1042.56
E0000 00:00:1743138091.977132 3656567 buffer_comparator.cc:156] Difference at 35: 914.632, expected 1022.9
E0000 00:00:1743138091.977136 3656567 buffer_comparator.cc:156] Difference at 36: 904.6, expected 1023.97
E0000 00:00:1743138091.977139 3656567 buffer_comparator.cc:156] Difference at 37: 927.217, expected 1044.19
E0000 00:00:1743138091.977143 3656567 buffer_comparator.cc:156] Difference at 38: 908.499, expected 1026.18
E0000 00:00:1743138091.977146 3656567 buffer_comparator.cc:156] Difference at 41: 912.473, expected 1023.32
E0000 00:00:1743138091.977149 3656567 buffer_comparator.cc:156] Difference at 42: 925.365, expected 1041.29
E0000 00:00:1743138091.977152 3656567 buffer_comparator.cc:156] Difference at 43: 901.683, expected 1026.34
E0000 00:00:1743138091.977155 3656567 buffer_comparator.cc:156] Difference at 45: 905.59, expected 1018.47
2025-03-28 05:01:31.977161: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.979345 3656567 buffer_comparator.cc:156] Difference at 16: 1299.42, expected 1022.04
E0000 00:00:1743138091.979360 3656567 buffer_comparator.cc:156] Difference at 17: 1311.07, expected 1034.86
E0000 00:00:1743138091.979364 3656567 buffer_comparator.cc:156] Difference at 18: 1260.22, expected 1026.77
E0000 00:00:1743138091.979368 3656567 buffer_comparator.cc:156] Difference at 19: 1270.33, expected 1046.95
E0000 00:00:1743138091.979371 3656567 buffer_comparator.cc:156] Difference at 20: 1280.44, expected 1032.02
E0000 00:00:1743138091.979374 3656567 buffer_comparator.cc:156] Difference at 21: 1261.99, expected 1015.56
E0000 00:00:1743138091.979377 3656567 buffer_comparator.cc:156] Difference at 22: 1298.95, expected 1050.44
E0000 00:00:1743138091.979380 3656567 buffer_comparator.cc:156] Difference at 23: 1279, expected 1032.61
E0000 00:00:1743138091.979384 3656567 buffer_comparator.cc:156] Difference at 24: 1305.72, expected 1024.78
E0000 00:00:1743138091.979387 3656567 buffer_comparator.cc:156] Difference at 25: 1284.61, expected 1040.24
2025-03-28 05:01:31.979392: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.981614 3656567 buffer_comparator.cc:156] Difference at 16: 1299.42, expected 1022.04
E0000 00:00:1743138091.981627 3656567 buffer_comparator.cc:156] Difference at 17: 1311.07, expected 1034.86
E0000 00:00:1743138091.981632 3656567 buffer_comparator.cc:156] Difference at 18: 1260.22, expected 1026.77
E0000 00:00:1743138091.981635 3656567 buffer_comparator.cc:156] Difference at 19: 1270.33, expected 1046.95
E0000 00:00:1743138091.981638 3656567 buffer_comparator.cc:156] Difference at 20: 1280.44, expected 1032.02
E0000 00:00:1743138091.981641 3656567 buffer_comparator.cc:156] Difference at 21: 1261.99, expected 1015.56
E0000 00:00:1743138091.981644 3656567 buffer_comparator.cc:156] Difference at 22: 1298.95, expected 1050.44
E0000 00:00:1743138091.981647 3656567 buffer_comparator.cc:156] Difference at 23: 1279, expected 1032.61
E0000 00:00:1743138091.981651 3656567 buffer_comparator.cc:156] Difference at 24: 1305.72, expected 1024.78
E0000 00:00:1743138091.981654 3656567 buffer_comparator.cc:156] Difference at 25: 1284.61, expected 1040.24
2025-03-28 05:01:31.981659: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.983889 3656567 buffer_comparator.cc:156] Difference at 16: 1299.42, expected 1022.04
E0000 00:00:1743138091.983901 3656567 buffer_comparator.cc:156] Difference at 17: 1311.07, expected 1034.86
E0000 00:00:1743138091.983905 3656567 buffer_comparator.cc:156] Difference at 18: 1260.22, expected 1026.77
E0000 00:00:1743138091.983908 3656567 buffer_comparator.cc:156] Difference at 19: 1270.33, expected 1046.95
E0000 00:00:1743138091.983911 3656567 buffer_comparator.cc:156] Difference at 20: 1280.44, expected 1032.02
E0000 00:00:1743138091.983915 3656567 buffer_comparator.cc:156] Difference at 21: 1261.99, expected 1015.56
E0000 00:00:1743138091.983918 3656567 buffer_comparator.cc:156] Difference at 22: 1298.95, expected 1050.44
E0000 00:00:1743138091.983921 3656567 buffer_comparator.cc:156] Difference at 23: 1279, expected 1032.61
E0000 00:00:1743138091.983924 3656567 buffer_comparator.cc:156] Difference at 24: 1305.72, expected 1024.78
E0000 00:00:1743138091.983927 3656567 buffer_comparator.cc:156] Difference at 25: 1284.61, expected 1040.24
2025-03-28 05:01:31.983932: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.986126 3656567 buffer_comparator.cc:156] Difference at 16: 1299.42, expected 1022.04
E0000 00:00:1743138091.986138 3656567 buffer_comparator.cc:156] Difference at 17: 1311.07, expected 1034.86
E0000 00:00:1743138091.986143 3656567 buffer_comparator.cc:156] Difference at 18: 1260.22, expected 1026.77
E0000 00:00:1743138091.986148 3656567 buffer_comparator.cc:156] Difference at 19: 1270.33, expected 1046.95
E0000 00:00:1743138091.986151 3656567 buffer_comparator.cc:156] Difference at 20: 1280.44, expected 1032.02
E0000 00:00:1743138091.986154 3656567 buffer_comparator.cc:156] Difference at 21: 1261.99, expected 1015.56
E0000 00:00:1743138091.986157 3656567 buffer_comparator.cc:156] Difference at 22: 1298.95, expected 1050.44
E0000 00:00:1743138091.986160 3656567 buffer_comparator.cc:156] Difference at 23: 1279, expected 1032.61
E0000 00:00:1743138091.986164 3656567 buffer_comparator.cc:156] Difference at 24: 1305.72, expected 1024.78
E0000 00:00:1743138091.986167 3656567 buffer_comparator.cc:156] Difference at 25: 1284.61, expected 1040.24
2025-03-28 05:01:31.986172: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.988360 3656567 buffer_comparator.cc:156] Difference at 33: 907.216, expected 1025.71
E0000 00:00:1743138091.988373 3656567 buffer_comparator.cc:156] Difference at 34: 915.936, expected 1042.56
E0000 00:00:1743138091.988377 3656567 buffer_comparator.cc:156] Difference at 35: 914.632, expected 1022.9
E0000 00:00:1743138091.988380 3656567 buffer_comparator.cc:156] Difference at 36: 904.6, expected 1023.97
E0000 00:00:1743138091.988384 3656567 buffer_comparator.cc:156] Difference at 37: 927.217, expected 1044.19
E0000 00:00:1743138091.988387 3656567 buffer_comparator.cc:156] Difference at 38: 908.499, expected 1026.18
E0000 00:00:1743138091.988390 3656567 buffer_comparator.cc:156] Difference at 41: 912.473, expected 1023.32
E0000 00:00:1743138091.988393 3656567 buffer_comparator.cc:156] Difference at 42: 925.365, expected 1041.29
E0000 00:00:1743138091.988396 3656567 buffer_comparator.cc:156] Difference at 43: 901.683, expected 1026.34
E0000 00:00:1743138091.988399 3656567 buffer_comparator.cc:156] Difference at 45: 905.59, expected 1018.47
2025-03-28 05:01:31.988405: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.990610 3656567 buffer_comparator.cc:156] Difference at 33: 907.216, expected 1025.71
E0000 00:00:1743138091.990622 3656567 buffer_comparator.cc:156] Difference at 34: 915.936, expected 1042.56
E0000 00:00:1743138091.990627 3656567 buffer_comparator.cc:156] Difference at 35: 914.632, expected 1022.9
E0000 00:00:1743138091.990630 3656567 buffer_comparator.cc:156] Difference at 36: 904.6, expected 1023.97
E0000 00:00:1743138091.990633 3656567 buffer_comparator.cc:156] Difference at 37: 927.217, expected 1044.19
E0000 00:00:1743138091.990637 3656567 buffer_comparator.cc:156] Difference at 38: 908.499, expected 1026.18
E0000 00:00:1743138091.990640 3656567 buffer_comparator.cc:156] Difference at 41: 912.473, expected 1023.32
E0000 00:00:1743138091.990643 3656567 buffer_comparator.cc:156] Difference at 42: 925.365, expected 1041.29
E0000 00:00:1743138091.990646 3656567 buffer_comparator.cc:156] Difference at 43: 901.683, expected 1026.34
E0000 00:00:1743138091.990649 3656567 buffer_comparator.cc:156] Difference at 45: 905.59, expected 1018.47
2025-03-28 05:01:31.990654: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.992848 3656567 buffer_comparator.cc:156] Difference at 33: 907.216, expected 1025.71
E0000 00:00:1743138091.992860 3656567 buffer_comparator.cc:156] Difference at 34: 915.936, expected 1042.56
E0000 00:00:1743138091.992865 3656567 buffer_comparator.cc:156] Difference at 35: 914.632, expected 1022.9
E0000 00:00:1743138091.992868 3656567 buffer_comparator.cc:156] Difference at 36: 904.6, expected 1023.97
E0000 00:00:1743138091.992871 3656567 buffer_comparator.cc:156] Difference at 37: 927.217, expected 1044.19
E0000 00:00:1743138091.992876 3656567 buffer_comparator.cc:156] Difference at 38: 908.499, expected 1026.18
E0000 00:00:1743138091.992879 3656567 buffer_comparator.cc:156] Difference at 41: 912.473, expected 1023.32
E0000 00:00:1743138091.992883 3656567 buffer_comparator.cc:156] Difference at 42: 925.365, expected 1041.29
E0000 00:00:1743138091.992886 3656567 buffer_comparator.cc:156] Difference at 43: 901.683, expected 1026.34
E0000 00:00:1743138091.992889 3656567 buffer_comparator.cc:156] Difference at 45: 905.59, expected 1018.47
2025-03-28 05:01:31.992894: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.995075 3656567 buffer_comparator.cc:156] Difference at 32: 838.525, expected 1038.15
E0000 00:00:1743138091.995088 3656567 buffer_comparator.cc:156] Difference at 33: 809.246, expected 1025.71
E0000 00:00:1743138091.995092 3656567 buffer_comparator.cc:156] Difference at 34: 828.77, expected 1042.56
E0000 00:00:1743138091.995096 3656567 buffer_comparator.cc:156] Difference at 35: 823.487, expected 1022.9
E0000 00:00:1743138091.995099 3656567 buffer_comparator.cc:156] Difference at 36: 809.228, expected 1023.97
E0000 00:00:1743138091.995102 3656567 buffer_comparator.cc:156] Difference at 37: 836.592, expected 1044.19
E0000 00:00:1743138091.995105 3656567 buffer_comparator.cc:156] Difference at 38: 814.645, expected 1026.18
E0000 00:00:1743138091.995108 3656567 buffer_comparator.cc:156] Difference at 39: 831.27, expected 1032.04
E0000 00:00:1743138091.995112 3656567 buffer_comparator.cc:156] Difference at 40: 823.074, expected 1022.55
E0000 00:00:1743138091.995115 3656567 buffer_comparator.cc:156] Difference at 41: 821.106, expected 1023.32
2025-03-28 05:01:31.995120: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.997331 3656567 buffer_comparator.cc:156] Difference at 32: 838.525, expected 1038.15
E0000 00:00:1743138091.997343 3656567 buffer_comparator.cc:156] Difference at 33: 809.246, expected 1025.71
E0000 00:00:1743138091.997348 3656567 buffer_comparator.cc:156] Difference at 34: 828.77, expected 1042.56
E0000 00:00:1743138091.997351 3656567 buffer_comparator.cc:156] Difference at 35: 823.487, expected 1022.9
E0000 00:00:1743138091.997354 3656567 buffer_comparator.cc:156] Difference at 36: 809.228, expected 1023.97
E0000 00:00:1743138091.997358 3656567 buffer_comparator.cc:156] Difference at 37: 836.592, expected 1044.19
E0000 00:00:1743138091.997361 3656567 buffer_comparator.cc:156] Difference at 38: 814.645, expected 1026.18
E0000 00:00:1743138091.997364 3656567 buffer_comparator.cc:156] Difference at 39: 831.27, expected 1032.04
E0000 00:00:1743138091.997367 3656567 buffer_comparator.cc:156] Difference at 40: 823.074, expected 1022.55
E0000 00:00:1743138091.997370 3656567 buffer_comparator.cc:156] Difference at 41: 821.106, expected 1023.32
2025-03-28 05:01:31.997375: E external/xla/xla/service/gpu/autotuning/gemm_fusion_autotuner.cc:1137] Results do not match the reference. This is likely a bug/unexpected loss of precision.
E0000 00:00:1743138091.999580 3656567 buffer_comparator.cc:156] Difference at 64: 833.692, expected 1038.16
E0000 00:00:1743138091.999593 3656567 buffer_comparator.cc:156] Difference at 65: 816.426, expected 1024.5
E0000 00:00:1743138091.999597 3656567 buffer_comparator.cc:156] Difference at 66: 838.443, expected 1030.77
E0000 00:00:1743138091.999600 3656567 buffer_comparator.cc:156] Difference at 67: 812.344, expected 1024.87
E0000 00:00:1743138091.999604 3656567 buffer_comparator.cc:156] Difference at 68: 821.788, expected 1032.37
E0000 00:00:1743138091.999607 3656567 buffer_comparator.cc:156] Difference at 69: 834.317, expected 1034.29
Epoch 1, Iter 11, Loss: 63375.9921875, Throughput: 15.719986 im/s
Epoch 1, Train Loss: 77095.1484375, Time: 90.0104s, Throughput: 15.642636 im/s
Epoch 2, Iter 11, Loss: 50906.2890625, Throughput: 1670.470713 im/s
Epoch 2, Train Loss: 55020.3710938, Time: 0.8432s, Throughput: 1669.831173 im/s
Epoch 3, Iter 11, Loss: 49555.8085938, Throughput: 1723.680203 im/s
Epoch 3, Train Loss: 48615.8398438, Time: 0.8172s, Throughput: 1722.992242 im/s
Epoch 4, Iter 11, Loss: 51103.5429688, Throughput: 1784.747643 im/s
Epoch 4, Train Loss: 48448.6367188, Time: 0.7892s, Throughput: 1784.058049 im/s
Epoch 5, Iter 11, Loss: 48584.9140625, Throughput: 1804.487900 im/s
Epoch 5, Train Loss: 47869.2656250, Time: 0.7806s, Throughput: 1803.814925 im/s
Epoch 6, Iter 11, Loss: 50431.5585938, Throughput: 1713.314243 im/s
Epoch 6, Train Loss: 48570.4960938, Time: 0.8221s, Throughput: 1712.787022 im/s
Epoch 7, Iter 11, Loss: 54826.8203125, Throughput: 1699.239182 im/s
Epoch 7, Train Loss: 51799.3242188, Time: 0.8288s, Throughput: 1698.741108 im/s
Epoch 8, Iter 11, Loss: 49647.7539062, Throughput: 1674.794991 im/s
Epoch 8, Train Loss: 52314.4765625, Time: 0.8412s, Throughput: 1673.858885 im/s
Epoch 9, Iter 11, Loss: 53270.0078125, Throughput: 1718.565356 im/s
Epoch 9, Train Loss: 51823.7890625, Time: 0.8197s, Throughput: 1717.701593 im/s
Epoch 10, Iter 11, Loss: 54488.1484375, Throughput: 1706.952186 im/s
Epoch 10, Train Loss: 51807.9726562, Time: 0.8252s, Throughput: 1706.170047 im/s
Epoch 11, Iter 11, Loss: 54787.4921875, Throughput: 1687.850622 im/s
Epoch 11, Train Loss: 54414.3515625, Time: 0.8344s, Throughput: 1687.361130 im/s
Epoch 12, Iter 11, Loss: 58954.5507812, Throughput: 1765.107984 im/s
Epoch 12, Train Loss: 56662.0234375, Time: 0.7980s, Throughput: 1764.468801 im/s
Epoch 13, Iter 11, Loss: 55746.8710938, Throughput: 1723.726489 im/s
Epoch 13, Train Loss: 56369.5625000, Time: 0.8172s, Throughput: 1723.003302 im/s
Epoch 14, Iter 11, Loss: 59782.0156250, Throughput: 1807.842059 im/s
Epoch 14, Train Loss: 57231.1132812, Time: 0.7792s, Throughput: 1807.046585 im/s
Epoch 15, Iter 11, Loss: 57410.4804688, Throughput: 1793.463697 im/s
Epoch 15, Train Loss: 58504.5468750, Time: 0.7853s, Throughput: 1792.853889 im/s
Epoch 16, Iter 11, Loss: 55743.3632812, Throughput: 1819.911492 im/s
Epoch 16, Train Loss: 55788.0468750, Time: 0.7740s, Throughput: 1819.203988 im/s
Epoch 17, Iter 11, Loss: 50695.4414062, Throughput: 1822.641819 im/s
Epoch 17, Train Loss: 52138.7109375, Time: 0.7728s, Throughput: 1821.927695 im/s
Epoch 18, Iter 11, Loss: 49952.2539062, Throughput: 1763.709977 im/s
Epoch 18, Train Loss: 51119.5585938, Time: 0.7987s, Throughput: 1762.816035 im/s
Epoch 19, Iter 11, Loss: 51504.4921875, Throughput: 1797.282773 im/s
Epoch 19, Train Loss: 51380.8593750, Time: 0.7837s, Throughput: 1796.564877 im/s
Epoch 20, Iter 11, Loss: 51466.4218750, Throughput: 1797.443052 im/s
Epoch 20, Train Loss: 52255.8789062, Time: 0.7836s, Throughput: 1796.819055 im/s
Epoch 21, Iter 11, Loss: 51493.8125000, Throughput: 1807.482405 im/s
Epoch 21, Train Loss: 52989.4218750, Time: 0.7794s, Throughput: 1806.508184 im/s
Epoch 22, Iter 11, Loss: 53444.3632812, Throughput: 1804.607556 im/s
Epoch 22, Train Loss: 53696.8789062, Time: 0.7806s, Throughput: 1803.851840 im/s
Epoch 23, Iter 11, Loss: 55722.9531250, Throughput: 1803.389132 im/s
Epoch 23, Train Loss: 54629.5976562, Time: 0.7811s, Throughput: 1802.491385 im/s
Epoch 24, Iter 11, Loss: 52160.1835938, Throughput: 1655.543432 im/s
Epoch 24, Train Loss: 54914.9414062, Time: 0.8508s, Throughput: 1654.829479 im/s
Epoch 25, Iter 11, Loss: 55281.5312500, Throughput: 1645.515313 im/s
Epoch 25, Train Loss: 54699.4414062, Time: 0.8561s, Throughput: 1644.639123 im/s
Epoch 26, Iter 11, Loss: 51048.0546875, Throughput: 1658.660202 im/s
Epoch 26, Train Loss: 52725.9492188, Time: 0.8493s, Throughput: 1657.802537 im/s
Epoch 27, Iter 11, Loss: 51788.9023438, Throughput: 1676.240128 im/s
Epoch 27, Train Loss: 50630.7617188, Time: 0.8403s, Throughput: 1675.628017 im/s
Epoch 28, Iter 11, Loss: 49227.5625000, Throughput: 1768.357985 im/s
Epoch 28, Train Loss: 49613.2851562, Time: 0.7965s, Throughput: 1767.654011 im/s
Epoch 29, Iter 11, Loss: 48816.1992188, Throughput: 1790.810793 im/s
Epoch 29, Train Loss: 49028.7617188, Time: 0.7866s, Throughput: 1789.988998 im/s
Epoch 30, Iter 11, Loss: 48322.5781250, Throughput: 1788.091010 im/s
Epoch 30, Train Loss: 48843.9609375, Time: 0.7877s, Throughput: 1787.423715 im/s
Epoch 31, Iter 11, Loss: 50652.1406250, Throughput: 1762.333115 im/s
Epoch 31, Train Loss: 49366.9609375, Time: 0.7992s, Throughput: 1761.691210 im/s
Epoch 32, Iter 11, Loss: 49091.5234375, Throughput: 1788.924610 im/s
Epoch 32, Train Loss: 50045.7148438, Time: 0.7874s, Throughput: 1788.268064 im/s
Epoch 33, Iter 11, Loss: 49996.5546875, Throughput: 1789.565911 im/s
Epoch 33, Train Loss: 50317.4375000, Time: 0.7871s, Throughput: 1788.836284 im/s
Epoch 34, Iter 11, Loss: 51249.7617188, Throughput: 1805.586352 im/s
Epoch 34, Train Loss: 50129.2617188, Time: 0.7801s, Throughput: 1804.781278 im/s
Epoch 35, Iter 11, Loss: 51550.6210938, Throughput: 1797.665739 im/s
Epoch 35, Train Loss: 50280.9101562, Time: 0.7837s, Throughput: 1796.669819 im/s
Epoch 36, Iter 11, Loss: 49536.4218750, Throughput: 1794.806190 im/s
Epoch 36, Train Loss: 49714.8515625, Time: 0.7849s, Throughput: 1793.904977 im/s
Epoch 37, Iter 11, Loss: 50257.3750000, Throughput: 1771.138594 im/s
Epoch 37, Train Loss: 49828.1484375, Time: 0.7953s, Throughput: 1770.309809 im/s
Epoch 38, Iter 11, Loss: 51409.1562500, Throughput: 1726.844942 im/s
Epoch 38, Train Loss: 49800.4414062, Time: 0.8156s, Throughput: 1726.232661 im/s
Epoch 39, Iter 11, Loss: 47600.9648438, Throughput: 1772.420199 im/s
Epoch 39, Train Loss: 49620.0000000, Time: 0.7947s, Throughput: 1771.757637 im/s
Epoch 40, Iter 11, Loss: 49235.3046875, Throughput: 1781.453297 im/s
Epoch 40, Train Loss: 48847.5859375, Time: 0.7907s, Throughput: 1780.783966 im/s
Epoch 41, Iter 11, Loss: 49184.9140625, Throughput: 1804.860154 im/s
Epoch 41, Train Loss: 50112.8750000, Time: 0.7804s, Throughput: 1804.159342 im/s
Epoch 42, Iter 11, Loss: 47691.5390625, Throughput: 1781.823096 im/s
Epoch 42, Train Loss: 49193.9335938, Time: 0.7905s, Throughput: 1781.198613 im/s
Epoch 43, Iter 11, Loss: 44883.6757812, Throughput: 1803.297170 im/s
Epoch 43, Train Loss: 46925.9570312, Time: 0.7811s, Throughput: 1802.526596 im/s
Epoch 44, Iter 11, Loss: 45107.3203125, Throughput: 1775.463915 im/s
Epoch 44, Train Loss: 45845.3085938, Time: 0.7934s, Throughput: 1774.750006 im/s
Epoch 45, Iter 11, Loss: 46143.3515625, Throughput: 1800.124985 im/s
Epoch 45, Train Loss: 44982.1171875, Time: 0.7825s, Throughput: 1799.458001 im/s
Epoch 46, Iter 11, Loss: 45278.7968750, Throughput: 1757.956325 im/s
Epoch 46, Train Loss: 45519.1367188, Time: 0.8012s, Throughput: 1757.420625 im/s
Epoch 47, Iter 11, Loss: 45474.1679688, Throughput: 1790.710335 im/s
Epoch 47, Train Loss: 45288.9414062, Time: 0.7868s, Throughput: 1789.447699 im/s
Epoch 48, Iter 11, Loss: 44007.6914062, Throughput: 1802.572261 im/s
Epoch 48, Train Loss: 44610.2500000, Time: 0.7814s, Throughput: 1801.817699 im/s
Epoch 49, Iter 11, Loss: 43499.2460938, Throughput: 1802.761551 im/s
Epoch 49, Train Loss: 44174.6679688, Time: 0.7813s, Throughput: 1802.092062 im/s
Epoch 50, Iter 11, Loss: 41312.5585938, Throughput: 1711.019894 im/s
Epoch 50, Train Loss: 43917.8984375, Time: 0.8232s, Throughput: 1710.469312 im/s

---

Appendix

using InteractiveUtils
InteractiveUtils.versioninfo()

if @isdefined(MLDataDevices)
    if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
        println()
        CUDA.versioninfo()
    end

    if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
        println()
        AMDGPU.versioninfo()
    end
end

Julia Version 1.11.4
Commit 8561cc3d68d (2025-03-10 11:36 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 48 × AMD EPYC 7402 24-Core Processor
  WORD_SIZE: 64
  LLVM: libLLVM-16.0.6 (ORCJIT, znver2)
Threads: 48 default, 0 interactive, 24 GC (on 2 virtual cores)
Environment:
  JULIA_CPU_THREADS = 2
  LD_LIBRARY_PATH = /usr/local/nvidia/lib:/usr/local/nvidia/lib64
  JULIA_PKG_SERVER = 
  JULIA_NUM_THREADS = 48
  JULIA_CUDA_HARD_MEMORY_LIMIT = 100%
  JULIA_PKG_PRECOMPILE_AUTO = 0
  JULIA_DEBUG = Literate
  JULIA_DEPOT_PATH = /root/.cache/julia-buildkite-plugin/depots/01872db4-8c79-43af-ab7d-12abac4f24f6

---

This page was generated using Literate.jl.