Skip to content

Convolutional VAE for MNIST ​

Convolutional variational autoencoder (CVAE) implementation in MLX using MNIST. This is based on the CVAE implementation in MLX.

julia
using Lux,
    Reactant,
    MLDatasets,
    Random,
    Statistics,
    Enzyme,
    MLUtils,
    DataAugmentation,
    ConcreteStructs,
    OneHotArrays,
    ImageShow,
    Images,
    Printf,
    Optimisers

const xdev = reactant_device(; force=true)
const cdev = cpu_device()

const IN_VSCODE = isdefined(Main, :VSCodeServer)
false

Model Definition ​

First we will define the encoder.It maps the input to a normal distribution in latent space and sample a latent vector from that distribution.

julia
function cvae_encoder(
    rng=Random.default_rng();
    num_latent_dims::Int,
    image_shape::Dims{3},
    max_num_filters::Int,
)
    flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
    return @compact(;
        embed=Chain(
            Chain(
                Conv((3, 3), image_shape[3] => max_num_filters ÷ 4; stride=2, pad=1),
                BatchNorm(max_num_filters ÷ 4, leakyrelu),
            ),
            Chain(
                Conv((3, 3), max_num_filters ÷ 4 => max_num_filters ÷ 2; stride=2, pad=1),
                BatchNorm(max_num_filters ÷ 2, leakyrelu),
            ),
            Chain(
                Conv((3, 3), max_num_filters ÷ 2 => max_num_filters; stride=2, pad=1),
                BatchNorm(max_num_filters, leakyrelu),
            ),
            FlattenLayer(),
        ),
        proj_mu=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
        proj_log_var=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
        rng
    ) do x
        y = embed(x)

        μ = proj_mu(y)
        logσ² = proj_log_var(y)

        T = eltype(logσ²)
        logσ² = clamp.(logσ², -T(20.0f0), T(10.0f0))
        σ = exp.(logσ² .* T(0.5))

        # Generate a tensor of random values from a normal distribution
        ϵ = randn_like(Lux.replicate(rng), σ)

        # Reparameterization trick to backpropagate through sampling
        z = ϵ .* σ .+ μ

        @return z, μ, logσ²
    end
end

Similarly we define the decoder.

julia
function cvae_decoder(; num_latent_dims::Int, image_shape::Dims{3}, max_num_filters::Int)
    flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
    return @compact(;
        linear=Dense(num_latent_dims, flattened_dim),
        upchain=Chain(
            Chain(
                Upsample(2),
                Conv((3, 3), max_num_filters => max_num_filters ÷ 2; stride=1, pad=1),
                BatchNorm(max_num_filters ÷ 2, leakyrelu),
            ),
            Chain(
                Upsample(2),
                Conv((3, 3), max_num_filters ÷ 2 => max_num_filters ÷ 4; stride=1, pad=1),
                BatchNorm(max_num_filters ÷ 4, leakyrelu),
            ),
            Chain(
                Upsample(2),
                Conv(
                    (3, 3), max_num_filters ÷ 4 => image_shape[3], sigmoid; stride=1, pad=1
                ),
            ),
        ),
        max_num_filters
    ) do x
        y = linear(x)
        img = reshape(y, image_shape[1] ÷ 8, image_shape[2] ÷ 8, max_num_filters, :)
        @return upchain(img)
    end
end

@concrete struct CVAE <: AbstractLuxContainerLayer{(:encoder, :decoder)}
    encoder <: AbstractLuxLayer
    decoder <: AbstractLuxLayer
end

function CVAE(
    rng=Random.default_rng();
    num_latent_dims::Int,
    image_shape::Dims{3},
    max_num_filters::Int,
)
    decoder = cvae_decoder(; num_latent_dims, image_shape, max_num_filters)
    encoder = cvae_encoder(rng; num_latent_dims, image_shape, max_num_filters)
    return CVAE(encoder, decoder)
end

function (cvae::CVAE)(x, ps, st)
    (z, μ, logσ²), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
    x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
    return (x_rec, μ, logσ²), (; encoder=st_enc, decoder=st_dec)
end

function encode(cvae::CVAE, x, ps, st)
    (z, _, _), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
    return z, (; encoder=st_enc, st.decoder)
end

function decode(cvae::CVAE, z, ps, st)
    x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
    return x_rec, (; decoder=st_dec, st.encoder)
end

Loading MNIST ​

julia
@concrete struct TensorDataset
    dataset
    transform
    total_samples::Int
end

Base.length(ds::TensorDataset) = ds.total_samples

function Base.getindex(ds::TensorDataset, idxs::Union{Vector{<:Integer},AbstractRange})
    img = Image.(eachslice(convert2image(ds.dataset, idxs); dims=3))
    return stack(parent ∘ itemdata ∘ Base.Fix1(apply, ds.transform), img)
end

function loadmnist(batchsize, image_size::Dims{2})
    # Load MNIST: Only 1500 for demonstration purposes on CI
    train_dataset = MNIST(; split=:train)
    N = parse(Bool, get(ENV, "CI", "false")) ? 5000 : length(train_dataset)

    train_transform = ScaleKeepAspect(image_size) |> ImageToTensor()
    trainset = TensorDataset(train_dataset, train_transform, N)
    trainloader = DataLoader(trainset; batchsize, shuffle=true, partial=false)

    return trainloader
end

Helper Functions ​

Generate an Image Grid from a list of images

julia
function create_image_grid(imgs::AbstractArray, grid_rows::Int, grid_cols::Int)
    total_images = grid_rows * grid_cols
    imgs = map(eachslice(imgs[:, :, :, 1:total_images]; dims=4)) do img
        cimg = if size(img, 3) == 1
            colorview(Gray, view(img, :, :, 1))
        else
            colorview(RGB, permutedims(img, (3, 1, 2)))
        end
        return cimg'
    end
    return create_image_grid(imgs, grid_rows, grid_cols)
end

function create_image_grid(images::Vector, grid_rows::Int, grid_cols::Int)
    # Check if the number of images matches the grid
    total_images = grid_rows * grid_cols
    @assert length(images) == total_images

    # Get the size of a single image (assuming all images are the same size)
    img_height, img_width = size(images[1])

    # Create a blank grid canvas
    grid_height = img_height * grid_rows
    grid_width = img_width * grid_cols
    grid_canvas = similar(images[1], grid_height, grid_width)

    # Place each image in the correct position on the canvas
    for idx in 1:total_images
        row = div(idx - 1, grid_cols) + 1
        col = mod(idx - 1, grid_cols) + 1

        start_row = (row - 1) * img_height + 1
        start_col = (col - 1) * img_width + 1

        grid_canvas[start_row:(start_row + img_height - 1), start_col:(start_col + img_width - 1)] .= images[idx]
    end

    return grid_canvas
end

function loss_function(model, ps, st, X)
    (y, μ, logσ²), st = model(X, ps, st)
    reconstruction_loss = MSELoss(; agg=sum)(y, X)
    kldiv_loss = -sum(1 .+ logσ² .- μ .^ 2 .- exp.(logσ²)) / 2
    loss = reconstruction_loss + kldiv_loss
    return loss, st, (; y, μ, logσ², reconstruction_loss, kldiv_loss)
end

function generate_images(
    model, ps, st; num_samples::Int=128, num_latent_dims::Int, decode_compiled=nothing
)
    z = get_device((ps, st))(randn(Float32, num_latent_dims, num_samples))
    if decode_compiled === nothing
        images, _ = decode(model, z, ps, Lux.testmode(st))
    else
        images, _ = decode_compiled(model, z, ps, Lux.testmode(st))
        images = cpu_device()(images)
    end
    return create_image_grid(images, 8, num_samples ÷ 8)
end

function reconstruct_images(model, ps, st, X)
    (recon, _, _), _ = model(X, ps, Lux.testmode(st))
    recon = cpu_device()(recon)
    return create_image_grid(recon, 8, size(X, ndims(X)) ÷ 8)
end
reconstruct_images (generic function with 1 method)

Training the Model ​

julia
function main(;
    batchsize=128,
    image_size=(64, 64),
    num_latent_dims=8,
    max_num_filters=64,
    seed=0,
    epochs=50,
    weight_decay=1.0e-5,
    learning_rate=1.0e-3,
    num_samples=batchsize,
)
    rng = Xoshiro()
    Random.seed!(rng, seed)

    cvae = CVAE(rng; num_latent_dims, image_shape=(image_size..., 1), max_num_filters)
    ps, st = Lux.setup(rng, cvae) |> xdev

    z = xdev(randn(Float32, num_latent_dims, num_samples))
    decode_compiled = @compile decode(cvae, z, ps, Lux.testmode(st))
    x = randn(Float32, image_size..., 1, batchsize) |> xdev
    cvae_compiled = @compile cvae(x, ps, Lux.testmode(st))

    train_dataloader = loadmnist(batchsize, image_size) |> xdev

    opt = AdamW(; eta=learning_rate, lambda=weight_decay)

    train_state = Training.TrainState(cvae, ps, st, opt)

    @printf "Total Trainable Parameters: %0.4f M\n" (Lux.parameterlength(ps) / 1.0e6)

    empty_row, model_img_full = nothing, nothing

    for epoch in 1:epochs
        loss_total = 0.0f0
        total_samples = 0

        start_time = time()
        for (i, X) in enumerate(train_dataloader)
            (_, loss, _, train_state) = Training.single_train_step!(
                AutoEnzyme(), loss_function, X, train_state; return_gradients=Val(false)
            )

            loss_total += loss
            total_samples += size(X, ndims(X))

            if i % 250 == 0 || i == length(train_dataloader)
                throughput = total_samples / (time() - start_time)
                @printf "Epoch %d, Iter %d, Loss: %.7f, Throughput: %.6f im/s\n" epoch i loss throughput
            end
        end
        total_time = time() - start_time

        train_loss = loss_total / length(train_dataloader)
        throughput = total_samples / total_time
        @printf "Epoch %d, Train Loss: %.7f, Time: %.4fs, Throughput: %.6f im/s\n" epoch train_loss total_time throughput

        if IN_VSCODE || epoch == epochs
            recon_images = reconstruct_images(
                cvae_compiled,
                train_state.parameters,
                train_state.states,
                first(train_dataloader),
            )
            gen_images = generate_images(
                cvae,
                train_state.parameters,
                train_state.states;
                num_samples,
                num_latent_dims,
                decode_compiled,
            )
            if empty_row === nothing
                empty_row = similar(gen_images, image_size[1], size(gen_images, 2))
                fill!(empty_row, 0)
            end
            model_img_full = vcat(recon_images, empty_row, gen_images)
            IN_VSCODE && display(model_img_full)
        end
    end

    return model_img_full
end

img = main()
Total Trainable Parameters: 0.1493 M
Epoch 1, Iter 39, Loss: 23846.0800781, Throughput: 19.587155 im/s
Epoch 1, Train Loss: 39866.9921875, Time: 255.0645s, Throughput: 19.571521 im/s
Epoch 2, Iter 39, Loss: 18317.3593750, Throughput: 121.739823 im/s
Epoch 2, Train Loss: 20259.1738281, Time: 41.0057s, Throughput: 121.739226 im/s
Epoch 3, Iter 39, Loss: 15486.2207031, Throughput: 121.372391 im/s
Epoch 3, Train Loss: 16591.0039062, Time: 41.1298s, Throughput: 121.371869 im/s
Epoch 4, Iter 39, Loss: 14585.2880859, Throughput: 121.768782 im/s
Epoch 4, Train Loss: 14954.7119141, Time: 40.9959s, Throughput: 121.768206 im/s
Epoch 5, Iter 39, Loss: 13737.6796875, Throughput: 121.910287 im/s
Epoch 5, Train Loss: 13940.6875000, Time: 40.9483s, Throughput: 121.909746 im/s
Epoch 6, Iter 39, Loss: 12636.5683594, Throughput: 121.856587 im/s
Epoch 6, Train Loss: 13357.7197266, Time: 40.9664s, Throughput: 121.855977 im/s
Epoch 7, Iter 39, Loss: 12773.5019531, Throughput: 121.535011 im/s
Epoch 7, Train Loss: 12848.6445312, Time: 41.0748s, Throughput: 121.534487 im/s
Epoch 8, Iter 39, Loss: 12041.8261719, Throughput: 122.823548 im/s
Epoch 8, Train Loss: 12463.4531250, Time: 40.6438s, Throughput: 122.823016 im/s
Epoch 9, Iter 39, Loss: 12466.2421875, Throughput: 122.132490 im/s
Epoch 9, Train Loss: 12216.8505859, Time: 40.8739s, Throughput: 122.131859 im/s
Epoch 10, Iter 39, Loss: 11602.1269531, Throughput: 121.914828 im/s
Epoch 10, Train Loss: 11962.2431641, Time: 40.9468s, Throughput: 121.914238 im/s
Epoch 11, Iter 39, Loss: 12200.1298828, Throughput: 121.976819 im/s
Epoch 11, Train Loss: 11691.0664062, Time: 40.9260s, Throughput: 121.976348 im/s
Epoch 12, Iter 39, Loss: 11366.2666016, Throughput: 122.640530 im/s
Epoch 12, Train Loss: 11493.3652344, Time: 40.7045s, Throughput: 122.639948 im/s
Epoch 13, Iter 39, Loss: 11514.2792969, Throughput: 123.022536 im/s
Epoch 13, Train Loss: 11343.0205078, Time: 40.5781s, Throughput: 123.022042 im/s
Epoch 14, Iter 39, Loss: 11082.2753906, Throughput: 122.605894 im/s
Epoch 14, Train Loss: 11217.0722656, Time: 40.7160s, Throughput: 122.605334 im/s
Epoch 15, Iter 39, Loss: 11479.7675781, Throughput: 122.483552 im/s
Epoch 15, Train Loss: 11093.4785156, Time: 40.7567s, Throughput: 122.482929 im/s
Epoch 16, Iter 39, Loss: 11529.2119141, Throughput: 121.661294 im/s
Epoch 16, Train Loss: 11013.6035156, Time: 41.0321s, Throughput: 121.660807 im/s
Epoch 17, Iter 39, Loss: 10781.3339844, Throughput: 121.871468 im/s
Epoch 17, Train Loss: 10866.3369141, Time: 40.9614s, Throughput: 121.870953 im/s
Epoch 18, Iter 39, Loss: 11129.9687500, Throughput: 121.938777 im/s
Epoch 18, Train Loss: 10683.4619141, Time: 40.9388s, Throughput: 121.938205 im/s
Epoch 19, Iter 39, Loss: 10319.9716797, Throughput: 120.694626 im/s
Epoch 19, Train Loss: 10676.1503906, Time: 41.3608s, Throughput: 120.694101 im/s
Epoch 20, Iter 39, Loss: 10695.1142578, Throughput: 121.219524 im/s
Epoch 20, Train Loss: 10646.6298828, Time: 41.1817s, Throughput: 121.218914 im/s
Epoch 21, Iter 39, Loss: 10079.3212891, Throughput: 122.199267 im/s
Epoch 21, Train Loss: 10510.7890625, Time: 40.8515s, Throughput: 122.198632 im/s
Epoch 22, Iter 39, Loss: 11049.0820312, Throughput: 121.725203 im/s
Epoch 22, Train Loss: 10438.9775391, Time: 41.0106s, Throughput: 121.724642 im/s
Epoch 23, Iter 39, Loss: 10176.1826172, Throughput: 122.139018 im/s
Epoch 23, Train Loss: 10382.8505859, Time: 40.8716s, Throughput: 122.138463 im/s
Epoch 24, Iter 39, Loss: 9941.4765625, Throughput: 122.146917 im/s
Epoch 24, Train Loss: 10279.0322266, Time: 40.8690s, Throughput: 122.146337 im/s
Epoch 25, Iter 39, Loss: 9824.2568359, Throughput: 122.443273 im/s
Epoch 25, Train Loss: 10228.0947266, Time: 40.7701s, Throughput: 122.442660 im/s
Epoch 26, Iter 39, Loss: 10845.0839844, Throughput: 122.486770 im/s
Epoch 26, Train Loss: 10240.6416016, Time: 40.7556s, Throughput: 122.486217 im/s
Epoch 27, Iter 39, Loss: 9947.7949219, Throughput: 122.768700 im/s
Epoch 27, Train Loss: 10102.8535156, Time: 40.6620s, Throughput: 122.768247 im/s
Epoch 28, Iter 39, Loss: 9051.8398438, Throughput: 122.983370 im/s
Epoch 28, Train Loss: 10083.4501953, Time: 40.5910s, Throughput: 122.982909 im/s
Epoch 29, Iter 39, Loss: 9523.9003906, Throughput: 123.126380 im/s
Epoch 29, Train Loss: 9987.7255859, Time: 40.5448s, Throughput: 123.123125 im/s
Epoch 30, Iter 39, Loss: 10510.9277344, Throughput: 123.056568 im/s
Epoch 30, Train Loss: 9984.9130859, Time: 40.5669s, Throughput: 123.055998 im/s
Epoch 31, Iter 39, Loss: 10216.2070312, Throughput: 123.038461 im/s
Epoch 31, Train Loss: 9917.6591797, Time: 40.5728s, Throughput: 123.037967 im/s
Epoch 32, Iter 39, Loss: 10001.7207031, Throughput: 123.323570 im/s
Epoch 32, Train Loss: 9886.6894531, Time: 40.4791s, Throughput: 123.322967 im/s
Epoch 33, Iter 39, Loss: 10453.1757812, Throughput: 123.375689 im/s
Epoch 33, Train Loss: 9884.3857422, Time: 40.4620s, Throughput: 123.375015 im/s
Epoch 34, Iter 39, Loss: 10158.9199219, Throughput: 123.044379 im/s
Epoch 34, Train Loss: 9786.7851562, Time: 40.5709s, Throughput: 123.043802 im/s
Epoch 35, Iter 39, Loss: 9256.8574219, Throughput: 123.279102 im/s
Epoch 35, Train Loss: 9765.9423828, Time: 40.4936s, Throughput: 123.278637 im/s
Epoch 36, Iter 39, Loss: 9274.2177734, Throughput: 121.612296 im/s
Epoch 36, Train Loss: 9761.7685547, Time: 41.0487s, Throughput: 121.611738 im/s
Epoch 37, Iter 39, Loss: 9990.4677734, Throughput: 122.871505 im/s
Epoch 37, Train Loss: 9671.7285156, Time: 40.6280s, Throughput: 122.870918 im/s
Epoch 38, Iter 39, Loss: 9844.3056641, Throughput: 122.653726 im/s
Epoch 38, Train Loss: 9668.6611328, Time: 40.7001s, Throughput: 122.653138 im/s
Epoch 39, Iter 39, Loss: 10520.4306641, Throughput: 122.536358 im/s
Epoch 39, Train Loss: 9669.6611328, Time: 40.7392s, Throughput: 122.535636 im/s
Epoch 40, Iter 39, Loss: 9477.6777344, Throughput: 122.573842 im/s
Epoch 40, Train Loss: 9604.8798828, Time: 40.7267s, Throughput: 122.573240 im/s
Epoch 41, Iter 39, Loss: 10123.9238281, Throughput: 122.183935 im/s
Epoch 41, Train Loss: 9539.9218750, Time: 40.8566s, Throughput: 122.183322 im/s
Epoch 42, Iter 39, Loss: 9578.7070312, Throughput: 121.647416 im/s
Epoch 42, Train Loss: 9503.5654297, Time: 41.0368s, Throughput: 121.646814 im/s
Epoch 43, Iter 39, Loss: 9753.9648438, Throughput: 121.209976 im/s
Epoch 43, Train Loss: 9459.3681641, Time: 41.1849s, Throughput: 121.209335 im/s
Epoch 44, Iter 39, Loss: 9222.5996094, Throughput: 121.786048 im/s
Epoch 44, Train Loss: 9453.6005859, Time: 40.9901s, Throughput: 121.785448 im/s
Epoch 45, Iter 39, Loss: 9053.7500000, Throughput: 121.139025 im/s
Epoch 45, Train Loss: 9413.6181641, Time: 41.2091s, Throughput: 121.138387 im/s
Epoch 46, Iter 39, Loss: 9158.6074219, Throughput: 121.648205 im/s
Epoch 46, Train Loss: 9410.7968750, Time: 41.0366s, Throughput: 121.647558 im/s
Epoch 47, Iter 39, Loss: 9009.3662109, Throughput: 121.219621 im/s
Epoch 47, Train Loss: 9472.4160156, Time: 41.1817s, Throughput: 121.219036 im/s
Epoch 48, Iter 39, Loss: 9605.7919922, Throughput: 122.147850 im/s
Epoch 48, Train Loss: 9402.3750000, Time: 40.8687s, Throughput: 122.147201 im/s
Epoch 49, Iter 39, Loss: 9159.3593750, Throughput: 121.892850 im/s
Epoch 49, Train Loss: 9350.1582031, Time: 40.9542s, Throughput: 121.892296 im/s
Epoch 50, Iter 39, Loss: 9168.4052734, Throughput: 121.971315 im/s
Epoch 50, Train Loss: 9276.9794922, Time: 40.9278s, Throughput: 121.970790 im/s

Appendix ​

julia
using InteractiveUtils
InteractiveUtils.versioninfo()

if @isdefined(MLDataDevices)
    if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
        println()
        CUDA.versioninfo()
    end

    if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
        println()
        AMDGPU.versioninfo()
    end
end
Julia Version 1.12.7
Commit 6d172b025e4 (2026-08-15 08:05 UTC)
Build Info:
  Official https://julialang.org release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 4 × Intel(R) Xeon(R) 6975P-C
  WORD_SIZE: 64
  LLVM: libLLVM-18.1.7 (ORCJIT, graniterapids)
  GC: Built with stock GC
Threads: 4 default, 1 interactive, 4 GC (on 4 virtual cores)
Environment:
  JULIA_DEBUG = Literate
  LD_LIBRARY_PATH = 
  JULIA_NUM_THREADS = 4
  JULIA_CPU_HARD_MEMORY_LIMIT = 100%
  JULIA_PKG_PRECOMPILE_AUTO = 0

This page was generated using Literate.jl.