FRAPPE v2 — image

@inproceedings{jacobellis2026frappe,
  title={FRAPPE: Full Input, Residual Output Autoencoding with Projection Pursuit Encoder},
  author={Jacobellis, Dan and Yadwadkar, Neeraja J.},
  note={under review},
  year={2026},
  url={https://ut-sysml.github.io/FRAPPE}
}

Project webpage · GitHub

The differences between FRAPPE v1 and v2:

• Support for 1d and 3d signals (e.g. audio and video). This notebook is the image instance (dim=2, 3 channels); see FRAPPE.ipynb for the v1 image tutorial and FRAPPE_v2_audio.ipynb for the v2 stereo-audio instance.

• Training occurs in channel groups. This gives fewer operating points to choose from (v1 allows truncation at every channel; v2 only at group boundaries) but makes training significantly faster.

The codec (round-5 campaign run 5a, exported at 30 channels / 9 operating points) is loaded from the Hugging Face hub through the compressors.frappe_v2_image public API — there is no local checkpoint or training code in this notebook.

import io, json, glob
import torch, numpy as np, matplotlib.pyplot as plt
import PIL.Image, datasets
from torchvision.transforms.v2.functional import pil_to_tensor, to_pil_image
from compressors.frappe_v2_image import load_all_codecs, encode_latents, decode_latents
from compressors.frappe_v2_image.quantize import srgb_to_linear

Load the codec

device = 'cuda:2'
config, models = load_all_codecs(device=device)          # one MergedAutoencoder per operating point

ops = list(config.cumulative_channels)                    # channel-truncation points (group boundaries)
cum = [0] + ops
group_sizes = [cum[i + 1] - cum[i] for i in range(len(ops))]
ps_groups   = sorted(set(config.ps), reverse=True)
max_ps      = max(config.ps)
n_trained   = max(ops)
model       = models[n_trained]                           # full-rate model (all channels)

print(f'modality      = {config.modality}  (dim={config.dim})')
print(f'channels      = {config.input_channels}')
print(f'ps ladder     = {config.ps}')
print(f'group_sizes   = {group_sizes}')
print(f'boundaries    = {ops}  (cumulative channels)')
print(f'decoder ps    = {config.decoder_ps}   decoder_dim = {config.decoder_dim}')
for s, (ps_s, start, end) in enumerate(model.scale_groups):
    print(f'  scale {s+1}: ps={ps_s:>2}  channels {start+1}-{end}  (latent rate 1/{ps_s}^2 per pixel per channel)')

modality      = image  (dim=2)
channels      = 3
ps ladder     = [32, 32, 32, 16, 16, 16, 16, 16, 16, 8, 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2]
group_sizes   = [3, 3, 3, 3, 3, 3, 3, 3, 6]
boundaries    = [3, 6, 9, 12, 15, 18, 21, 24, 30]  (cumulative channels)
decoder ps    = 8   decoder_dim = 768
  scale 1: ps=32  channels 1-3  (latent rate 1/32^2 per pixel per channel)
  scale 2: ps=16  channels 4-9  (latent rate 1/16^2 per pixel per channel)
  scale 3: ps= 8  channels 10-15  (latent rate 1/8^2 per pixel per channel)
  scale 4: ps= 4  channels 16-21  (latent rate 1/4^2 per pixel per channel)
  scale 5: ps= 2  channels 22-30  (latent rate 1/2^2 per pixel per channel)

Analysis filterbank

One learned ps×ps projection per latent channel. The top row of each panel is the impulse response (the RGB filter taps, normalized to ±4σ per scale); the bottom row is the magnitude response (2-D DFT magnitude, averaged over the three input channels, zero frequency at the center). Coarse scales learn low-frequency color/luma averages; fine scales learn oriented high-frequency detail.

N = 65                                                     # DFT size for the magnitude response
for s, (ps_s, start, end) in enumerate(model.scale_groups):
    W = model.encoders[s][0].weight.data.cpu()             # (C_group, 3, ps, ps)
    n_g = end - start
    sigma = W.std().item()
    fig, axes = plt.subplots(2, n_g, figsize=(1.4 * n_g, 3.0), dpi=120)
    axes = axes.reshape(2, n_g)
    for j in range(n_g):
        f = W[j]                                           # (3, ps, ps)
        axes[0, j].imshow((f.permute(1, 2, 0) / (4 * sigma) + 0.5).clamp(0, 1).numpy())
        H = np.fft.fftshift(np.abs(np.fft.fft2(f.mean(0).numpy(), s=(N, N))))
        axes[1, j].imshow(H, cmap='inferno')
        axes[0, j].set_title(f'ch{start + j + 1}', fontsize=8)
        for ax in (axes[0, j], axes[1, j]):
            ax.axis('off')
    axes[0, 0].set_title(f'ch{start + 1}\nimpulse', fontsize=8)
    axes[1, 0].text(-0.25, 0.5, '|H(f)|', transform=axes[1, 0].transAxes,
                    rotation=90, va='center', fontsize=8)
    fig.suptitle(f'scale ps={ps_s}  (channels {start + 1}-{end})', fontsize=10)
    plt.tight_layout()
    plt.show()

Load an example image

A Kodak image, mapped to the codec convention [-1, 1] (x / 127.5 - 1), cropped to a multiple of the coarsest patch size max(ps) = 32 (a no-op for Kodak’s 768×512). The published config has linear_input=false, so pixels are fed in sRGB; the srgb_to_linear guard mirrors the v1 recipe.

dataset = datasets.load_dataset('danjacobellis/kodak', split='validation')
img = dataset[22]['image'].convert('RGB')
x = pil_to_tensor(img).to(torch.float).to(device).unsqueeze(0) / 127.5 - 1.0
x = x[..., :max_ps * (x.shape[2] // max_ps), :max_ps * (x.shape[3] // max_ps)]  # crop to multiple of max(ps)
x_in = srgb_to_linear(x) if getattr(config, 'linear_input', False) else x
n_pixels = x.shape[2] * x.shape[3]
x_01 = x / 2 + 0.5
print(f'input {tuple(x.shape)}  ({n_pixels} pixels),  range [{x.min():.2f}, {x.max():.2f}]')
display(to_pil_image(x_01[0].cpu().clamp(0, 1)))

input (1, 3, 512, 768)  (393216 pixels),  range [-1.00, 1.00]

Analysis transform

with torch.no_grad():
    latents = model.encode(x_in)        # list of (1, C_scale, H_s, W_s), companded but not yet rounded
n_latent_values = sum(z.numel() for z in latents)
print(f'{x_in.numel()} input values -> {n_latent_values} latent values '
      f'({n_latent_values / x_in.numel():.2f}x)')
for s, z in enumerate(latents):
    print(f'  scale {s+1} (ps={model.scale_groups[s][0]:>2}): latent {tuple(z.shape)}  '
          f'range [{z.min():.1f}, {z.max():.1f}]')

1179648 input values -> 1079424 latent values (0.92x)
  scale 1 (ps=32): latent (1, 3, 16, 24)  range [-14.6, 12.3]
  scale 2 (ps=16): latent (1, 6, 32, 48)  range [-12.6, 11.5]
  scale 3 (ps= 8): latent (1, 6, 64, 96)  range [-14.1, 13.1]
  scale 4 (ps= 4): latent (1, 6, 128, 192)  range [-13.4, 14.9]
  scale 5 (ps= 2): latent (1, 9, 256, 384)  range [-17.1, 15.8]

Companding

ramp = torch.linspace(-600, 600, 2401, device=device)
plt.figure(figsize=(5.5, 3.5), dpi=130)
colors = plt.cm.viridis(np.linspace(0, 1, len(model.scale_groups)))
for s, (ps_s, start, end) in enumerate(model.scale_groups):
    C = end - start
    with torch.no_grad():
        curves = model.encoders[s][1](ramp.view(1, 1, -1, 1).expand(1, C, -1, 1))   # softsign compander
    plt.plot(ramp.cpu(), curves[0, :, :, 0].cpu().T, lw=0.5, color=colors[s])
    plt.plot([], [], color=colors[s], label=f'ps={ps_s}')   # legend proxy
plt.axhline(127, ls=':', c='k', lw=0.6); plt.axhline(-127, ls=':', c='k', lw=0.6)
plt.xlabel('latent value'); plt.ylabel('companded value')
plt.title('softsign compander curves (one per latent channel)')
plt.legend(fontsize=7); plt.tight_layout(); plt.show()

Rounding

latents_q = [z.round().clamp(-127, 127) for z in latents]
z_c = torch.cat([z.flatten() for z in latents])
z_q = torch.cat([z.flatten() for z in latents_q])
qsnr = 10 * (z_c.pow(2).mean() / (z_c - z_q).pow(2).mean()).log10().item()
print(f'integer latent range:    [{int(z_q.min())}, {int(z_q.max())}]')
print(f'latent quantization SNR: {qsnr:.2f} dB')

plt.figure(figsize=(5, 2), dpi=120)
plt.hist(z_q.cpu().numpy(), range=(-127.5, 127.5), bins=255, width=0.85)
plt.xlim([-15, 15]); plt.title('histogram of integer latents (zoomed)')
plt.xlabel('value'); plt.ylabel('count'); plt.tight_layout(); plt.show()

latents_q = [z.to(torch.int8).cpu() for z in latents_q]

integer latent range:    [-17, 16]
latent quantization SNR: 23.62 dB

Stacking latents and entropy coding using a lossless image codec

encode_latents (from compressors.frappe_v2_image) reshapes each per-scale int8 latent (1, C, H, W) to a (C·H, W) grayscale image, shifts to uint8, and saves it as JPEG-LS; the per-scale streams are length-prefixed so the blob is self-describing.

blob = encode_latents(latents_q)
sizes = [len(encode_latents([z])) - 4 for z in latents_q]   # JPEG-LS payload bytes per scale (minus 4-byte prefix)

for s, (z, sz) in enumerate(zip(latents_q, sizes)):
    print(f'scale {s+1} (ps={model.scale_groups[s][0]:>2}, {z.shape[1]}ch x {z.shape[2]}x{z.shape[3]}):  {sz:>8,} bytes')
    z_2d = z[0].reshape(z.shape[1] * z.shape[2], z.shape[3])
    display(to_pil_image((z_2d.long() + 127).to(torch.uint8)))

scale 1 (ps=32, 3ch x 16x24):       580 bytes

scale 2 (ps=16, 6ch x 32x48):     2,319 bytes

scale 3 (ps= 8, 6ch x 64x96):     5,988 bytes

scale 4 (ps= 4, 6ch x 128x192):    23,137 bytes

scale 5 (ps= 2, 9ch x 256x384):   133,154 bytes

n_bits = len(blob) * 8
bpp = n_bits / n_pixels
CR = 24.0 / bpp
bits_per_latent_value = n_bits / n_latent_values
print(f'compressed size:       {len(blob):,} bytes')
print(f'rate:                  {bpp:.4f} bpp')
print(f'bits per latent value: {bits_per_latent_value:.3f}')
print(f'compression ratio:     {CR:.1f}x  (vs 24-bit RGB,  {n_trained} ch)')

compressed size:       165,198 bytes
rate:                  3.3610 bpp
bits per latent value: 1.224
compression ratio:     7.1x  (vs 24-bit RGB,  30 ch)

Decode latents

latents_dec = decode_latents(blob, model.scale_groups)
assert all(torch.equal(a.cpu(), b.cpu()) for a, b in zip(latents_dec, latents_q)), 'entropy round-trip mismatch'
print('entropy round-trip: latents recovered exactly from the byte blob ✓')

with torch.no_grad():
    xhat = model.decode([z.to(device) for z in latents_dec]).clamp(-1, 1)
xhat_01 = xhat / 2 + 0.5

psnr = -10 * torch.nn.functional.mse_loss(x_01, xhat_01).log10().item()
print(f'bpp  = {bpp:.4f}')
print(f'PSNR = {psnr:.2f} dB   ({n_trained} ch)')
display(to_pil_image(xhat_01[0].cpu().clamp(0, 1)))

entropy round-trip: latents recovered exactly from the byte blob ✓
bpp  = 3.3610
PSNR = 43.20 dB   (30 ch)

Reconstructions at each operating point (channel truncation at group boundaries)

Each operating point re-encodes with its own encoder stack and decodes with its own merged decoder. A fixed 256×256 crop of the original (left) and the reconstruction (right) is shown side by side per point.

ch, cw = 256, 256
ct = (x.shape[2] - ch) // 2
cl = (x.shape[3] - cw) // 2
orig_crop = to_pil_image(x_01[0, :, ct:ct+ch, cl:cl+cw].cpu().clamp(0, 1))

rd_img = []
for n_ch in ops:
    partial = models[n_ch]
    with torch.no_grad():
        lq = [z.round().clamp(-127, 127).to(torch.int8).cpu() for z in partial.encode(x_in)]
    blob_g = encode_latents(lq)
    ld = decode_latents(blob_g, partial.scale_groups)
    with torch.no_grad():
        xh = partial.decode([z.to(device) for z in ld]).clamp(-1, 1)
    bpp_g = len(blob_g) * 8 / n_pixels
    psnr_g = -10 * torch.nn.functional.mse_loss(x_01, xh / 2 + 0.5).log10().item()
    rd_img.append((bpp_g, psnr_g, n_ch))
    print(f'{n_ch:>2} ch:  bpp={bpp_g:.4f}   CR={24 / bpp_g:7.1f}x   PSNR={psnr_g:.2f} dB')
    recon_crop = to_pil_image((xh[0, :, ct:ct+ch, cl:cl+cw] / 2 + 0.5).cpu().clamp(0, 1))
    pair = PIL.Image.new('RGB', (2 * cw + 4, ch), 'white')
    pair.paste(orig_crop, (0, 0)); pair.paste(recon_crop, (cw + 4, 0))
    display(pair)

 3 ch:  bpp=0.0119   CR= 2019.9x   PSNR=23.50 dB

 6 ch:  bpp=0.0426   CR=  563.9x   PSNR=26.28 dB

 9 ch:  bpp=0.0591   CR=  405.8x   PSNR=27.23 dB

12 ch:  bpp=0.1399   CR=  171.5x   PSNR=30.76 dB

15 ch:  bpp=0.1811   CR=  132.6x   PSNR=32.26 dB

18 ch:  bpp=0.4585   CR=   52.3x   PSNR=35.64 dB

21 ch:  bpp=0.6519   CR=   36.8x   PSNR=37.04 dB

24 ch:  bpp=1.5809   CR=   15.2x   PSNR=40.83 dB

30 ch:  bpp=3.3610   CR=    7.1x   PSNR=43.20 dB

Rate–distortion (this image vs full Kodak validation set)

The single image above shows the workflow; the operating curve below is the full-validation average over all 24 Kodak images, read from results/frappe_v2_image/rate_distortion_*.json (produced by python -m compressors.frappe_v2_image.evaluate_rate_distortion). The demo image is a single sample, so its curve sits off the average — the 24-image mean is the codec’s reported operating point. AVIF and FRAPPE v1 (whose top operating point 99l reaches 32.28 dB at 0.94 bpp) are overlaid from the same Kodak JSONs used by image_compressors.ipynb — v2 5a clearly extends the high-rate end.

rd_path = sorted(glob.glob('results/frappe_v2_image/rate_distortion_*.json'))[-1]
rd = json.load(open(rd_path))
qs = rd['channel_counts']
val_bpp  = [rd['results'][str(q)]['mean']['bpp'] for q in qs]
val_psnr = [rd['results'][str(q)]['mean']['PSNR_dB'] for q in qs]

def anchor(path):
    d = json.load(open(path))
    key = 'channel_counts' if 'channel_counts' in d else 'quality_values'
    pts = [(d['results'][str(q)]['mean']['bpp'], d['results'][str(q)]['mean']['PSNR_dB']) for q in d[key]]
    return [p[0] for p in pts], [p[1] for p in pts]

avif_bpp, avif_psnr = anchor('results/avif/rate_distortion_1777321273.json')
v1_bpp, v1_psnr = anchor('results/frappe/rate_distortion_1777315303.json')

plt.figure(figsize=(6.5, 4), dpi=120)
plt.semilogx(val_bpp, val_psnr, 'o-', label=f"FRAPPE v2 5a — full Kodak ({rd['n_images']} images)")
plt.semilogx([p[0] for p in rd_img], [p[1] for p in rd_img], 's--', alpha=0.6,
             label='FRAPPE v2 5a — demo image (#23)')
plt.semilogx(v1_bpp, v1_psnr, '-', color='tab:green', alpha=0.7, label='FRAPPE v1 (99l)')
plt.semilogx(avif_bpp, avif_psnr, '-', color='black', alpha=0.7, label='AVIF')
for q, bv, psv in zip(qs, val_bpp, val_psnr):
    plt.annotate(f'{q}ch', (bv, psv), fontsize=7, xytext=(3, -8), textcoords='offset points')
plt.xlabel('bpp'); plt.ylabel('PSNR (dB)')
plt.title('FRAPPE v2 image — PSNR vs bpp (Kodak)')
plt.grid(True, alpha=0.3); plt.legend(fontsize=8); plt.tight_layout(); plt.show()

print(f'Full-validation average (from {rd_path.split("/")[-1]}):')
for q, bv, psv in zip(qs, val_bpp, val_psnr):
    print(f'  {q:>2} ch:  bpp={bv:.4f}   CR={24 / bv:7.1f}x   PSNR={psv:.2f} dB')

Full-validation average (from rate_distortion_1782987823.json):
   3 ch:  bpp=0.0109   CR= 2195.5x   PSNR=21.51 dB
   6 ch:  bpp=0.0439   CR=  546.4x   PSNR=24.03 dB
   9 ch:  bpp=0.0697   CR=  344.2x   PSNR=25.02 dB
  12 ch:  bpp=0.1708   CR=  140.5x   PSNR=27.34 dB
  15 ch:  bpp=0.2614   CR=   91.8x   PSNR=28.72 dB
  18 ch:  bpp=0.6512   CR=   36.9x   PSNR=31.77 dB
  21 ch:  bpp=0.9338   CR=   25.7x   PSNR=33.66 dB
  24 ch:  bpp=2.2864   CR=   10.5x   PSNR=38.38 dB
  30 ch:  bpp=4.8747   CR=    4.9x   PSNR=41.82 dB