Convolutional VAE for MNIST
Convolutional variational autoencoder (CVAE) implementation in MLX using MNIST. This is based on the CVAE implementation in MLX.
julia
using Lux,
Reactant,
MLDatasets,
Random,
Statistics,
Enzyme,
MLUtils,
DataAugmentation,
ConcreteStructs,
OneHotArrays,
ImageShow,
Images,
Printf,
Optimisers
const xdev = reactant_device(; force=true)
const cdev = cpu_device()
const IN_VSCODE = isdefined(Main, :VSCodeServer)falseModel Definition
First we will define the encoder.It maps the input to a normal distribution in latent space and sample a latent vector from that distribution.
julia
function cvae_encoder(
rng=Random.default_rng();
num_latent_dims::Int,
image_shape::Dims{3},
max_num_filters::Int,
)
flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
return @compact(;
embed=Chain(
Chain(
Conv((3, 3), image_shape[3] => max_num_filters ÷ 4; stride=2, pad=1),
BatchNorm(max_num_filters ÷ 4, leakyrelu),
),
Chain(
Conv((3, 3), max_num_filters ÷ 4 => max_num_filters ÷ 2; stride=2, pad=1),
BatchNorm(max_num_filters ÷ 2, leakyrelu),
),
Chain(
Conv((3, 3), max_num_filters ÷ 2 => max_num_filters; stride=2, pad=1),
BatchNorm(max_num_filters, leakyrelu),
),
FlattenLayer(),
),
proj_mu=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
proj_log_var=Dense(flattened_dim, num_latent_dims; init_bias=zeros32),
rng
) do x
y = embed(x)
μ = proj_mu(y)
logσ² = proj_log_var(y)
T = eltype(logσ²)
logσ² = clamp.(logσ², -T(20.0f0), T(10.0f0))
σ = exp.(logσ² .* T(0.5))
# Generate a tensor of random values from a normal distribution
ϵ = randn_like(Lux.replicate(rng), σ)
# Reparameterization trick to backpropagate through sampling
z = ϵ .* σ .+ μ
@return z, μ, logσ²
end
endSimilarly we define the decoder.
julia
function cvae_decoder(; num_latent_dims::Int, image_shape::Dims{3}, max_num_filters::Int)
flattened_dim = prod(image_shape[1:2] .÷ 8) * max_num_filters
return @compact(;
linear=Dense(num_latent_dims, flattened_dim),
upchain=Chain(
Chain(
Upsample(2),
Conv((3, 3), max_num_filters => max_num_filters ÷ 2; stride=1, pad=1),
BatchNorm(max_num_filters ÷ 2, leakyrelu),
),
Chain(
Upsample(2),
Conv((3, 3), max_num_filters ÷ 2 => max_num_filters ÷ 4; stride=1, pad=1),
BatchNorm(max_num_filters ÷ 4, leakyrelu),
),
Chain(
Upsample(2),
Conv(
(3, 3), max_num_filters ÷ 4 => image_shape[3], sigmoid; stride=1, pad=1
),
),
),
max_num_filters
) do x
y = linear(x)
img = reshape(y, image_shape[1] ÷ 8, image_shape[2] ÷ 8, max_num_filters, :)
@return upchain(img)
end
end
@concrete struct CVAE <: AbstractLuxContainerLayer{(:encoder, :decoder)}
encoder <: AbstractLuxLayer
decoder <: AbstractLuxLayer
end
function CVAE(
rng=Random.default_rng();
num_latent_dims::Int,
image_shape::Dims{3},
max_num_filters::Int,
)
decoder = cvae_decoder(; num_latent_dims, image_shape, max_num_filters)
encoder = cvae_encoder(rng; num_latent_dims, image_shape, max_num_filters)
return CVAE(encoder, decoder)
end
function (cvae::CVAE)(x, ps, st)
(z, μ, logσ²), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
return (x_rec, μ, logσ²), (; encoder=st_enc, decoder=st_dec)
end
function encode(cvae::CVAE, x, ps, st)
(z, _, _), st_enc = cvae.encoder(x, ps.encoder, st.encoder)
return z, (; encoder=st_enc, st.decoder)
end
function decode(cvae::CVAE, z, ps, st)
x_rec, st_dec = cvae.decoder(z, ps.decoder, st.decoder)
return x_rec, (; decoder=st_dec, st.encoder)
endLoading MNIST
julia
@concrete struct TensorDataset
dataset
transform
total_samples::Int
end
Base.length(ds::TensorDataset) = ds.total_samples
function Base.getindex(ds::TensorDataset, idxs::Union{Vector{<:Integer},AbstractRange})
img = Image.(eachslice(convert2image(ds.dataset, idxs); dims=3))
return stack(parent ∘ itemdata ∘ Base.Fix1(apply, ds.transform), img)
end
function loadmnist(batchsize, image_size::Dims{2})
# Load MNIST: Only 1500 for demonstration purposes on CI
train_dataset = MNIST(; split=:train)
N = parse(Bool, get(ENV, "CI", "false")) ? 5000 : length(train_dataset)
train_transform = ScaleKeepAspect(image_size) |> ImageToTensor()
trainset = TensorDataset(train_dataset, train_transform, N)
trainloader = DataLoader(trainset; batchsize, shuffle=true, partial=false)
return trainloader
endHelper Functions
Generate an Image Grid from a list of images
julia
function create_image_grid(imgs::AbstractArray, grid_rows::Int, grid_cols::Int)
total_images = grid_rows * grid_cols
imgs = map(eachslice(imgs[:, :, :, 1:total_images]; dims=4)) do img
cimg = if size(img, 3) == 1
colorview(Gray, view(img, :, :, 1))
else
colorview(RGB, permutedims(img, (3, 1, 2)))
end
return cimg'
end
return create_image_grid(imgs, grid_rows, grid_cols)
end
function create_image_grid(images::Vector, grid_rows::Int, grid_cols::Int)
# Check if the number of images matches the grid
total_images = grid_rows * grid_cols
@assert length(images) == total_images
# Get the size of a single image (assuming all images are the same size)
img_height, img_width = size(images[1])
# Create a blank grid canvas
grid_height = img_height * grid_rows
grid_width = img_width * grid_cols
grid_canvas = similar(images[1], grid_height, grid_width)
# Place each image in the correct position on the canvas
for idx in 1:total_images
row = div(idx - 1, grid_cols) + 1
col = mod(idx - 1, grid_cols) + 1
start_row = (row - 1) * img_height + 1
start_col = (col - 1) * img_width + 1
grid_canvas[start_row:(start_row + img_height - 1), start_col:(start_col + img_width - 1)] .= images[idx]
end
return grid_canvas
end
function loss_function(model, ps, st, X)
(y, μ, logσ²), st = model(X, ps, st)
reconstruction_loss = MSELoss(; agg=sum)(y, X)
kldiv_loss = -sum(1 .+ logσ² .- μ .^ 2 .- exp.(logσ²)) / 2
loss = reconstruction_loss + kldiv_loss
return loss, st, (; y, μ, logσ², reconstruction_loss, kldiv_loss)
end
function generate_images(
model, ps, st; num_samples::Int=128, num_latent_dims::Int, decode_compiled=nothing
)
z = get_device((ps, st))(randn(Float32, num_latent_dims, num_samples))
if decode_compiled === nothing
images, _ = decode(model, z, ps, Lux.testmode(st))
else
images, _ = decode_compiled(model, z, ps, Lux.testmode(st))
images = cpu_device()(images)
end
return create_image_grid(images, 8, num_samples ÷ 8)
end
function reconstruct_images(model, ps, st, X)
(recon, _, _), _ = model(X, ps, Lux.testmode(st))
recon = cpu_device()(recon)
return create_image_grid(recon, 8, size(X, ndims(X)) ÷ 8)
endreconstruct_images (generic function with 1 method)Training the Model
julia
function main(;
batchsize=128,
image_size=(64, 64),
num_latent_dims=8,
max_num_filters=64,
seed=0,
epochs=50,
weight_decay=1.0e-5,
learning_rate=1.0e-3,
num_samples=batchsize,
)
rng = Xoshiro()
Random.seed!(rng, seed)
cvae = CVAE(rng; num_latent_dims, image_shape=(image_size..., 1), max_num_filters)
ps, st = Lux.setup(rng, cvae) |> xdev
z = xdev(randn(Float32, num_latent_dims, num_samples))
decode_compiled = @compile decode(cvae, z, ps, Lux.testmode(st))
x = randn(Float32, image_size..., 1, batchsize) |> xdev
cvae_compiled = @compile cvae(x, ps, Lux.testmode(st))
train_dataloader = loadmnist(batchsize, image_size) |> xdev
opt = AdamW(; eta=learning_rate, lambda=weight_decay)
train_state = Training.TrainState(cvae, ps, st, opt)
@printf "Total Trainable Parameters: %0.4f M\n" (Lux.parameterlength(ps) / 1.0e6)
empty_row, model_img_full = nothing, nothing
for epoch in 1:epochs
loss_total = 0.0f0
total_samples = 0
start_time = time()
for (i, X) in enumerate(train_dataloader)
(_, loss, _, train_state) = Training.single_train_step!(
AutoEnzyme(), loss_function, X, train_state; return_gradients=Val(false)
)
loss_total += loss
total_samples += size(X, ndims(X))
if i % 250 == 0 || i == length(train_dataloader)
throughput = total_samples / (time() - start_time)
@printf "Epoch %d, Iter %d, Loss: %.7f, Throughput: %.6f im/s\n" epoch i loss throughput
end
end
total_time = time() - start_time
train_loss = loss_total / length(train_dataloader)
throughput = total_samples / total_time
@printf "Epoch %d, Train Loss: %.7f, Time: %.4fs, Throughput: %.6f im/s\n" epoch train_loss total_time throughput
if IN_VSCODE || epoch == epochs
recon_images = reconstruct_images(
cvae_compiled,
train_state.parameters,
train_state.states,
first(train_dataloader),
)
gen_images = generate_images(
cvae,
train_state.parameters,
train_state.states;
num_samples,
num_latent_dims,
decode_compiled,
)
if empty_row === nothing
empty_row = similar(gen_images, image_size[1], size(gen_images, 2))
fill!(empty_row, 0)
end
model_img_full = vcat(recon_images, empty_row, gen_images)
IN_VSCODE && display(model_img_full)
end
end
return model_img_full
end
img = main()Total Trainable Parameters: 0.1493 M
Epoch 1, Iter 39, Loss: 23513.9609375, Throughput: 26.752703 im/s
Epoch 1, Train Loss: 39492.6250000, Time: 186.9502s, Throughput: 26.702297 im/s
Epoch 2, Iter 39, Loss: 18142.1171875, Throughput: 87.024593 im/s
Epoch 2, Train Loss: 19986.1035156, Time: 57.3633s, Throughput: 87.024347 im/s
Epoch 3, Iter 39, Loss: 15799.9121094, Throughput: 86.613955 im/s
Epoch 3, Train Loss: 16585.5585938, Time: 57.6352s, Throughput: 86.613706 im/s
Epoch 4, Iter 39, Loss: 14923.4814453, Throughput: 86.393171 im/s
Epoch 4, Train Loss: 15043.1982422, Time: 57.7825s, Throughput: 86.392893 im/s
Epoch 5, Iter 39, Loss: 14868.5400391, Throughput: 85.581033 im/s
Epoch 5, Train Loss: 14026.1542969, Time: 58.3309s, Throughput: 85.580777 im/s
Epoch 6, Iter 39, Loss: 12978.7822266, Throughput: 86.708010 im/s
Epoch 6, Train Loss: 13453.4375000, Time: 57.5727s, Throughput: 86.707773 im/s
Epoch 7, Iter 39, Loss: 11856.0214844, Throughput: 86.223106 im/s
Epoch 7, Train Loss: 12906.7167969, Time: 57.8965s, Throughput: 86.222870 im/s
Epoch 8, Iter 39, Loss: 12546.0585938, Throughput: 87.544182 im/s
Epoch 8, Train Loss: 12469.6523438, Time: 57.0228s, Throughput: 87.543922 im/s
Epoch 9, Iter 39, Loss: 12761.9003906, Throughput: 86.265060 im/s
Epoch 9, Train Loss: 12315.0078125, Time: 57.8683s, Throughput: 86.264842 im/s
Epoch 10, Iter 39, Loss: 12340.8320312, Throughput: 85.318685 im/s
Epoch 10, Train Loss: 11989.8193359, Time: 58.5102s, Throughput: 85.318450 im/s
Epoch 11, Iter 39, Loss: 11165.0957031, Throughput: 87.309611 im/s
Epoch 11, Train Loss: 11774.3359375, Time: 57.1760s, Throughput: 87.309393 im/s
Epoch 12, Iter 39, Loss: 11965.0058594, Throughput: 87.824223 im/s
Epoch 12, Train Loss: 11636.6621094, Time: 56.8410s, Throughput: 87.823999 im/s
Epoch 13, Iter 39, Loss: 10602.0917969, Throughput: 88.197210 im/s
Epoch 13, Train Loss: 11419.1455078, Time: 56.6006s, Throughput: 88.196955 im/s
Epoch 14, Iter 39, Loss: 11326.6416016, Throughput: 87.680478 im/s
Epoch 14, Train Loss: 11210.7080078, Time: 56.9342s, Throughput: 87.680236 im/s
Epoch 15, Iter 39, Loss: 10195.6103516, Throughput: 86.608439 im/s
Epoch 15, Train Loss: 11145.2333984, Time: 57.6389s, Throughput: 86.608186 im/s
Epoch 16, Iter 39, Loss: 10841.6035156, Throughput: 86.832940 im/s
Epoch 16, Train Loss: 11032.9550781, Time: 57.4899s, Throughput: 86.832718 im/s
Epoch 17, Iter 39, Loss: 10860.6123047, Throughput: 87.366787 im/s
Epoch 17, Train Loss: 10957.2656250, Time: 57.1386s, Throughput: 87.366504 im/s
Epoch 18, Iter 39, Loss: 11013.8808594, Throughput: 88.042749 im/s
Epoch 18, Train Loss: 10853.8798828, Time: 56.6999s, Throughput: 88.042502 im/s
Epoch 19, Iter 39, Loss: 11080.2998047, Throughput: 86.631450 im/s
Epoch 19, Train Loss: 10735.3134766, Time: 57.6236s, Throughput: 86.631236 im/s
Epoch 20, Iter 39, Loss: 9922.8261719, Throughput: 87.669016 im/s
Epoch 20, Train Loss: 10613.6357422, Time: 56.9416s, Throughput: 87.668792 im/s
Epoch 21, Iter 39, Loss: 10871.5253906, Throughput: 86.802561 im/s
Epoch 21, Train Loss: 10496.3173828, Time: 57.5100s, Throughput: 86.802303 im/s
Epoch 22, Iter 39, Loss: 10455.6562500, Throughput: 87.482942 im/s
Epoch 22, Train Loss: 10461.7880859, Time: 57.0627s, Throughput: 87.482654 im/s
Epoch 23, Iter 39, Loss: 11126.2626953, Throughput: 86.230469 im/s
Epoch 23, Train Loss: 10434.4082031, Time: 57.8915s, Throughput: 86.230243 im/s
Epoch 24, Iter 39, Loss: 10048.9648438, Throughput: 87.233030 im/s
Epoch 24, Train Loss: 10320.3056641, Time: 57.2262s, Throughput: 87.232776 im/s
Epoch 25, Iter 39, Loss: 11053.2929688, Throughput: 87.961835 im/s
Epoch 25, Train Loss: 10286.4550781, Time: 56.7520s, Throughput: 87.961596 im/s
Epoch 26, Iter 39, Loss: 10829.7080078, Throughput: 85.166064 im/s
Epoch 26, Train Loss: 10178.8906250, Time: 58.6150s, Throughput: 85.165861 im/s
Epoch 27, Iter 39, Loss: 9870.8447266, Throughput: 87.506759 im/s
Epoch 27, Train Loss: 10127.2607422, Time: 57.0472s, Throughput: 87.506512 im/s
Epoch 28, Iter 39, Loss: 10151.1181641, Throughput: 86.776991 im/s
Epoch 28, Train Loss: 10128.9843750, Time: 57.5269s, Throughput: 86.776747 im/s
Epoch 29, Iter 39, Loss: 9964.6103516, Throughput: 87.551304 im/s
Epoch 29, Train Loss: 10126.3447266, Time: 57.0182s, Throughput: 87.551063 im/s
Epoch 30, Iter 39, Loss: 10787.0351562, Throughput: 86.845780 im/s
Epoch 30, Train Loss: 10012.7255859, Time: 57.4814s, Throughput: 86.845546 im/s
Epoch 31, Iter 39, Loss: 9698.0722656, Throughput: 85.754185 im/s
Epoch 31, Train Loss: 9969.8867188, Time: 58.2131s, Throughput: 85.753954 im/s
Epoch 32, Iter 39, Loss: 9462.9365234, Throughput: 86.361606 im/s
Epoch 32, Train Loss: 9894.8867188, Time: 57.8036s, Throughput: 86.361360 im/s
Epoch 33, Iter 39, Loss: 8922.9355469, Throughput: 86.371637 im/s
Epoch 33, Train Loss: 9848.8281250, Time: 57.7969s, Throughput: 86.371404 im/s
Epoch 34, Iter 39, Loss: 10100.4375000, Throughput: 87.444263 im/s
Epoch 34, Train Loss: 9867.1181641, Time: 57.0879s, Throughput: 87.444023 im/s
Epoch 35, Iter 39, Loss: 10800.0820312, Throughput: 86.518105 im/s
Epoch 35, Train Loss: 9806.1015625, Time: 57.6991s, Throughput: 86.517882 im/s
Epoch 36, Iter 39, Loss: 9964.9003906, Throughput: 86.210455 im/s
Epoch 36, Train Loss: 9871.9296875, Time: 57.9050s, Throughput: 86.210219 im/s
Epoch 37, Iter 39, Loss: 10225.5537109, Throughput: 87.233863 im/s
Epoch 37, Train Loss: 9816.9169922, Time: 57.2256s, Throughput: 87.233628 im/s
Epoch 38, Iter 39, Loss: 9651.7402344, Throughput: 86.706740 im/s
Epoch 38, Train Loss: 9645.4267578, Time: 57.5735s, Throughput: 86.706510 im/s
Epoch 39, Iter 39, Loss: 10247.3427734, Throughput: 87.024690 im/s
Epoch 39, Train Loss: 9606.2744141, Time: 57.3632s, Throughput: 87.024413 im/s
Epoch 40, Iter 39, Loss: 10015.5078125, Throughput: 87.893203 im/s
Epoch 40, Train Loss: 9678.2685547, Time: 56.7964s, Throughput: 87.892966 im/s
Epoch 41, Iter 39, Loss: 9671.9589844, Throughput: 87.869908 im/s
Epoch 41, Train Loss: 9580.9326172, Time: 56.8114s, Throughput: 87.869682 im/s
Epoch 42, Iter 39, Loss: 9940.6621094, Throughput: 88.293938 im/s
Epoch 42, Train Loss: 9538.1367188, Time: 56.5386s, Throughput: 88.293717 im/s
Epoch 43, Iter 39, Loss: 9907.2812500, Throughput: 87.635772 im/s
Epoch 43, Train Loss: 9490.8271484, Time: 56.9632s, Throughput: 87.635540 im/s
Epoch 44, Iter 39, Loss: 9767.4453125, Throughput: 88.338721 im/s
Epoch 44, Train Loss: 9511.9042969, Time: 56.5099s, Throughput: 88.338483 im/s
Epoch 45, Iter 39, Loss: 8839.3603516, Throughput: 87.200951 im/s
Epoch 45, Train Loss: 9494.9492188, Time: 57.2472s, Throughput: 87.200729 im/s
Epoch 46, Iter 39, Loss: 9461.7509766, Throughput: 86.791319 im/s
Epoch 46, Train Loss: 9400.0419922, Time: 57.5174s, Throughput: 86.791112 im/s
Epoch 47, Iter 39, Loss: 9731.1152344, Throughput: 86.063339 im/s
Epoch 47, Train Loss: 9411.8320312, Time: 58.0040s, Throughput: 86.063100 im/s
Epoch 48, Iter 39, Loss: 9216.5390625, Throughput: 85.950036 im/s
Epoch 48, Train Loss: 9449.3642578, Time: 58.0804s, Throughput: 85.949799 im/s
Epoch 49, Iter 39, Loss: 8865.0185547, Throughput: 87.021554 im/s
Epoch 49, Train Loss: 9332.3076172, Time: 57.3652s, Throughput: 87.021328 im/s
Epoch 50, Iter 39, Loss: 9178.2890625, Throughput: 87.224222 im/s
Epoch 50, Train Loss: 9304.5761719, Time: 57.2320s, Throughput: 87.223993 im/s
Appendix
julia
using InteractiveUtils
InteractiveUtils.versioninfo()
if @isdefined(MLDataDevices)
if @isdefined(CUDA) && MLDataDevices.functional(CUDADevice)
println()
CUDA.versioninfo()
end
if @isdefined(AMDGPU) && MLDataDevices.functional(AMDGPUDevice)
println()
AMDGPU.versioninfo()
end
endJulia Version 1.11.8
Commit cf1da5e20e3 (2025-11-06 17:49 UTC)
Build Info:
Official https://julialang.org/ release
Platform Info:
OS: Linux (x86_64-linux-gnu)
CPU: 4 × AMD EPYC 7763 64-Core Processor
WORD_SIZE: 64
LLVM: libLLVM-16.0.6 (ORCJIT, znver3)
Threads: 4 default, 0 interactive, 2 GC (on 4 virtual cores)
Environment:
JULIA_DEBUG = Literate
LD_LIBRARY_PATH =
JULIA_NUM_THREADS = 4
JULIA_CPU_HARD_MEMORY_LIMIT = 100%
JULIA_PKG_PRECOMPILE_AUTO = 0This page was generated using Literate.jl.