FRAPPE v3 — stereo audio#
@inproceedings{jacobellis2026frappe,
title={FRAPPE: Full Input, Residual Output Autoencoding with Projection Pursuit Encoder},
author={Jacobellis, Dan and Yadwadkar, Neeraja J.},
note={Asilomar 2026},
year={2026},
url={https://ut-sysml.github.io/FRAPPE}
}
What changes from v2 to v3:
Entropy coding switches from per-scale JPEG-LS to the lawcoder — an analytic two-law conditional GGD model, 6 scalars per channel, from which Huffman tables and zero-run-mode parameters are generated at load. No fitted tables persist.
Stream layout is
all_detail— NO global stream; every macroregion (16384 samples ≈ 372 ms at 44.1 kHz) is coded independently with the detail template (block reset, run mode, one byte-pad per region). Each region decodable from its own bytes alone.SC6 companding (softsign, bits=6,
affine='bounded') replaces v2’s SC8 (±127) — the coder alphabet [-31, 31] is a structural guarantee.Per-group decoder configs (multidec) — later groups may use deeper decoders.
The codec is loaded from the Hugging Face hub (danjacobellis/FRAPPEv3, audio/ subfolder) through the compressors.frappe_v3_audio public API.
import json, glob
import torch, numpy as np, matplotlib.pyplot as plt
import torchaudio
from IPython.display import Audio, display
from datasets import load_dataset
from torchvision.transforms import ToPILImage
from huggingface_hub import hf_hub_download
from compressors.frappe_v3_audio import load_all_codecs, encode_to_latents
from compressors._frappev3.entropy_coding import rom_facts
Load the codec#
device = 'cpu'
config, models, coder = load_all_codecs(device=device)
ops = list(config.cumulative_channels)
cum = [0] + ops
group_sizes = [cum[i + 1] - cum[i] for i in range(len(ops))]
ps_groups = list(config.ps)
max_ps = max(config.ps)
model = models[ops[-1]]
rom = rom_facts(coder.blob)
print(f'modality = {config.modality}')
print(f'stream mode = {coder.mode} macroregion = {coder.macroregion} samples')
print(f'channels = {config.input_channels}')
print(f'ps per group = {ps_groups} group_sizes = {group_sizes}')
print(f'boundaries = {ops} (cumulative channels = the coder\'s operating points)')
for s, (ps_s, start, end) in enumerate(model.scale_groups):
print(f' scale {s + 1}: ps={ps_s:>4} channels {start + 1}-{end} (latent rate 1/{ps_s})')
print(f'\nentropy coder: alphabet [-31, 31] persistent model = '
f'{rom["model_scalars"]} scalars ({rom["blob_bytes_f32"]} B as f32) for {rom["n_channels"]} channels;')
print(f'generated at load: {rom["generated_tables_per_channel"]} tables/channel, '
f'run mode on {rom["run_mode_channels"]} channels')
modality = audio
stream mode = all_detail macroregion = 16384 samples
channels = 2
ps per group = [256, 256, 128, 64, 32, 16, 8] group_sizes = [4, 5, 6, 6, 12, 9, 9]
boundaries = [4, 9, 15, 21, 33, 42, 51] (cumulative channels = the coder's operating points)
scale 1: ps= 256 channels 1-9 (latent rate 1/256)
scale 2: ps= 128 channels 10-15 (latent rate 1/128)
scale 3: ps= 64 channels 16-21 (latent rate 1/64)
scale 4: ps= 32 channels 22-33 (latent rate 1/32)
scale 5: ps= 16 channels 34-42 (latent rate 1/16)
scale 6: ps= 8 channels 43-51 (latent rate 1/8)
entropy coder: alphabet [-31, 31] persistent model = 459 scalars (1836 B as f32) for 51 channels;
generated at load: 41 tables/channel, run mode on 51 channels
Analysis filterbank#
def chunk_sizes(n, max_per=6):
k = max(1, -(-n // max_per))
return [n // k + (1 if i < n % k else 0) for i in range(k)]
N = 3001
fs_hz = 44100
freqs = np.fft.fftshift(np.fft.fftfreq(N)) * fs_hz
panels = []
for s, (ps_s, start, end) in enumerate(model.scale_groups):
W = model.encoders[s][0].weight.data.cpu().numpy().mean(1)
off = 0
for sz in chunk_sizes(end - start):
panels.append((ps_s, start + off + 1, start + off + sz, W[off:off + sz]))
off += sz
fig, axes = plt.subplots(len(panels), 2, figsize=(10, 1.7 * len(panels)), dpi=120,
constrained_layout=True)
axes = np.atleast_2d(axes)
for (ax_imp, ax_mag), (ps_s, ch_lo, ch_hi, filt) in zip(axes, panels):
t_ms = np.arange(filt.shape[1]) / fs_hz * 1e3
for c, h in enumerate(filt, start=ch_lo):
H = np.fft.fftshift(np.fft.fft(h, N))
ax_imp.plot(t_ms, h, lw=1.0, label=f'ch{c}')
ax_mag.plot(freqs, np.abs(H), lw=1.0, label=f'ch{c}')
ax_imp.set_title(f'ps={ps_s} channels {ch_lo}-{ch_hi} (impulse response)', fontsize=9)
ax_mag.set_title(f'ps={ps_s} channels {ch_lo}-{ch_hi}', fontsize=9)
ax_mag.set_xlim(-fs_hz / 2, fs_hz / 2)
ax_imp.set_ylabel(r'$h(t)$')
ax_mag.set_ylabel(r'$|H(f)|$')
ax_mag.legend(fontsize=7, loc='upper right', framealpha=0.6, ncol=2)
axes[-1, 0].set_xlabel('time (ms)')
axes[-1, 1].set_xlabel('frequency (Hz)')
fig.suptitle('analysis filterbank — impulse (left) and magnitude (right) response per filter', fontsize=11)
plt.show()
Load an example clip#
The danjacobellis/music dataset (private due to copyright) holds 24 stereo music tracks. The source files are lossy MP3 but at near-lossless rates. We use a five-second stereo 44.1 kHz excerpt, normalized to [-1, 1], cropped to a whole number of macroregions.
CLIP_INDEX = 16
START_SECONDS = 150.0
DURATION_SECONDS = 5.0
ds = load_dataset("danjacobellis/music", split="train")
samples = ds[CLIP_INDEX]["audio"].get_all_samples()
fs = samples.sample_rate
i0 = round(START_SECONDS * fs)
x = samples.data[:, i0 : i0 + round(DURATION_SECONDS * fs)].to(torch.float)
x = x - x.mean()
x = x / (x.abs().max() + 1e-8)
unit = max(coder.macroregion, max_ps)
L = unit * (x.shape[-1] // unit)
x = x[..., :L]
n_samples = x.shape[-1]
x_in = x.unsqueeze(0).to(device)
print(f'fs = {fs} Hz')
print(f'clip = {tuple(x.shape)} ({n_samples / fs:.1f} s), range [{x.min():.2f}, {x.max():.2f}]')
print(f'model in = {tuple(x_in.shape)} ({n_samples // coder.macroregion} macroregions of '
f'{coder.macroregion} samples = {coder.macroregion / fs * 1e3:.0f} ms each)')
fs = 44100 Hz
clip = (2, 212992) (4.8 s), range [-1.00, 0.95]
model in = (1, 2, 212992) (13 macroregions of 16384 samples = 372 ms each)
mel = torchaudio.transforms.MelSpectrogram(sample_rate=fs, n_fft=4096, hop_length=512)
def make_spectrogram(x_1ch):
S = mel(x_1ch).log()
S = ((S - S.mean()) / (3 * S.std()) + 0.5).clamp(0, 1)
return ToPILImage()(S.flip(0))
for ch, name in enumerate(['left', 'right']):
print(f'{name} channel')
display(make_spectrogram(x[ch]))
Audio(x.numpy(), rate=fs)
left channel
right channel
Analysis transform#
with torch.no_grad():
latents = model.encode(x_in)
n_latent_values = sum(z.numel() for z in latents)
print(f'{x_in.numel()} input samples -> {n_latent_values} latent values '
f'({n_latent_values / x_in.numel():.2f}x)')
for s, z in enumerate(latents):
print(f' scale {s + 1} (ps={model.scale_groups[s][0]:>4}): latent {tuple(z.shape)} '
f'range [{z.min():.1f}, {z.max():.1f}]')
425984 input samples -> 476736 latent values (1.12x)
scale 1 (ps= 256): latent (1, 9, 832) range [-5.6, 4.8]
scale 2 (ps= 128): latent (1, 6, 1664) range [-4.6, 4.3]
scale 3 (ps= 64): latent (1, 6, 3328) range [-11.2, 11.5]
scale 4 (ps= 32): latent (1, 12, 6656) range [-12.8, 11.4]
scale 5 (ps= 16): latent (1, 9, 13312) range [-2.2, 2.3]
scale 6 (ps= 8): latent (1, 9, 26624) range [-15.1, 14.5]
Companding#
Softsign compander curves per latent channel (SC6: bits=6, affine='bounded'). The softsign asymptote makes the coder’s alphabet bound ±31 (dotted lines) structural.
ramp = torch.linspace(-600, 600, 2401, device=device)
plt.figure(figsize=(5.5, 3.5), dpi=130)
colors = plt.cm.viridis(np.linspace(0, 1, len(model.scale_groups)))
for s, (ps_s, start, end) in enumerate(model.scale_groups):
Cg = end - start
with torch.no_grad():
curves = model.encoders[s][1](ramp.view(1, 1, -1).expand(1, Cg, -1))
plt.plot(ramp.cpu(), curves[0].cpu().T, lw=0.5, color=colors[s])
plt.plot([], [], color=colors[s], label=f'ps={ps_s}')
plt.axhline(31, ls=':', c='k', lw=0.6); plt.axhline(-31, ls=':', c='k', lw=0.6)
plt.xlabel('latent value'); plt.ylabel('companded value')
plt.title('softsign compander curves (dotted: coder alphabet bound +/-31)')
plt.legend(fontsize=7); plt.tight_layout(); plt.show()
Rounding (to the coder’s [-31, 31] grid)#
z_c = torch.cat([z.flatten() for z in latents])
n_clip = int((z_c.round().abs() > 31).sum())
latents_q = [z.round().clamp(-31, 31) for z in latents]
z_q = torch.cat([z.flatten() for z in latents_q])
qsnr = 10 * (z_c.pow(2).mean() / (z_c - z_q).pow(2).mean()).log10().item()
print(f'integer latent range: [{int(z_q.min())}, {int(z_q.max())}] (clipped: {n_clip}/{z_c.numel()})')
print(f'latent quantization SNR: {qsnr:.2f} dB')
plt.figure(figsize=(5, 2), dpi=120)
plt.hist(z_q.cpu().numpy(), range=(-31.5, 31.5), bins=63, width=0.85)
plt.xlim([-15, 15]); plt.title('histogram of integer latents (zoomed)')
plt.xlabel('value'); plt.ylabel('count'); plt.tight_layout(); plt.show()
latents_np = [z[0].to(torch.int8).cpu().numpy() for z in latents_q]
integer latent range: [-15, 14] (clipped: 0/476736)
latent quantization SNR: 15.91 dB
Quantized latents (prior to entropy coding)#
One image per (macroregion, channel group) — the units the coder actually works in — region-major, groups in channel order within each region. Each pixel is one latent value, no resizing or interpolation: height = the group’s channel count, width = the group’s latent samples in the region (macroregion / ps), the coder alphabet [-31, 31] mapped linearly to [0, 255] (0 → mid-gray).
import PIL.ImageOps
n_regions = n_samples // coder.macroregion
cum_ch = [0] + list(np.cumsum(config.group_sizes))
print(f'{n_regions} macroregions, latents resized 8x per dimension for display (nearest neighbor)')
for u in range(min(n_regions, 3)):
for g in range(len(ops)):
lo, hi = cum_ch[g], cum_ch[g + 1]
s, (ps_g, start, end) = next((s, sg) for s, sg in enumerate(model.scale_groups)
if sg[1] <= lo < sg[2])
f = coder.macroregion // ps_g
z = latents_np[s][lo - start : hi - start, u * f : (u + 1) * f]
img = PIL.Image.fromarray(((z.astype(np.int16) + 31) * 255 // 62).astype(np.uint8), 'L')
img = PIL.ImageOps.autocontrast(img)
print(f'region {u}, group {g+1} (ch{lo+1}-{hi}, ps={ps_g}): {z.shape[0]}ch x {z.shape[1]} samples')
display(img.resize((8 * img.width, 8 * img.height), PIL.Image.Resampling.NEAREST))
13 macroregions, latents resized 8x per dimension for display (nearest neighbor)
region 0, group 1 (ch1-4, ps=256): 4ch x 64 samples
region 0, group 2 (ch5-9, ps=256): 5ch x 64 samples
region 0, group 3 (ch10-15, ps=128): 6ch x 128 samples
region 0, group 4 (ch16-21, ps=64): 6ch x 256 samples
region 0, group 5 (ch22-33, ps=32): 12ch x 512 samples
region 0, group 6 (ch34-42, ps=16): 9ch x 1024 samples
region 0, group 7 (ch43-51, ps=8): 9ch x 2048 samples
region 1, group 1 (ch1-4, ps=256): 4ch x 64 samples
region 1, group 2 (ch5-9, ps=256): 5ch x 64 samples
region 1, group 3 (ch10-15, ps=128): 6ch x 128 samples
region 1, group 4 (ch16-21, ps=64): 6ch x 256 samples
region 1, group 5 (ch22-33, ps=32): 12ch x 512 samples
region 1, group 6 (ch34-42, ps=16): 9ch x 1024 samples
region 1, group 7 (ch43-51, ps=8): 9ch x 2048 samples
region 2, group 1 (ch1-4, ps=256): 4ch x 64 samples
region 2, group 2 (ch5-9, ps=256): 5ch x 64 samples
region 2, group 3 (ch10-15, ps=128): 6ch x 128 samples
region 2, group 4 (ch16-21, ps=64): 6ch x 256 samples
region 2, group 5 (ch22-33, ps=32): 12ch x 512 samples
region 2, group 6 (ch34-42, ps=16): 9ch x 1024 samples
region 2, group 7 (ch43-51, ps=8): 9ch x 2048 samples
All-detail lawcoder entropy coding#
coder.encode produces one stream per macroregion and nothing else — in all_detail mode there is no global stream. Every channel is coded per-region with the detail template: block-reset prediction, causal statistic mapped through the two laws to a generated canonical GGD-Huffman table, and zero-run mode. Each region’s stream is byte-padded once, and the stream for operating point n is the literal bit-prefix covering channels ≤ n.
enc = coder.encode(latents_np)
assert enc.gblob == b''
region_bytes = (enc.prefix[:, -1] + 7) // 8
print(f'{len(enc.streams)} macroregion streams, one byte-pad each; at max detail '
f'{int(region_bytes.sum()):,} bytes total, per region min/median/max = '
f'{int(region_bytes.min())}/{int(np.median(region_bytes))}/{int(region_bytes.max())}')
print(f'\n{"op":>4} {"stream bits":>12} {"bits/pair":>10} {"kbps":>8} {"CR":>8} {"bits/latent":>12}')
for n in ops:
bits = coder.op_bits(enc, n)
n_lat_n = sum(min(max(n - start, 0), end - start) * z.shape[1]
for z, (ps_s, start, end) in zip(latents_np, model.scale_groups))
kbps_n = bits / (n_samples / fs) / 1000
print(f'{n:>4} {bits:>12,} {bits / n_samples:>10.4f} {kbps_n:>8.2f} '
f'{32 * n_samples / bits:>7.1f}x {bits / n_lat_n:>12.3f}')
13 macroregion streams, one byte-pad each; at max detail 107,629 bytes total, per region min/median/max = 6034/8933/9741
op stream bits bits/pair kbps CR bits/latent
4 8,456 0.0397 1.75 806.0x 2.541
9 18,104 0.0850 3.75 376.5x 2.418
15 40,912 0.1921 8.47 166.6x 2.342
21 104,376 0.4900 21.61 65.3x 2.788
33 333,144 1.5641 68.98 20.5x 2.840
42 438,712 2.0598 90.84 15.5x 1.850
51 861,032 4.0426 178.28 7.9x 1.806
Round-trip verification#
coder.verify decodes every macroregion from its own truncated bytes alone — no global input, no cross-region input, state zero-initialized — at every operating point’s literal bit-prefix, and asserts the latents bit-exact.
coder.verify(enc, latents_np, ops)
print(f'round trip verified at ops {ops}: every macroregion decoded from its own truncated')
print('bytes alone (state zero-initialized); op streams are literal bit-prefixes')
round trip verified at ops [4, 9, 15, 21, 33, 42, 51]: every macroregion decoded from its own truncated
bytes alone (state zero-initialized); op streams are literal bit-prefixes
Decoding#
The full-rate reconstruction from the integer latents.
with torch.no_grad():
x_hat = model.decode([torch.from_numpy(z).unsqueeze(0).to(device) for z in latents_np]).clamp(-1, 1)
x_hat = x_hat[0].float().cpu()
psnr = -10 * torch.nn.functional.mse_loss(x / 2 + 0.5, x_hat / 2 + 0.5).log10().item()
bits_full = coder.op_bits(enc, ops[-1])
print(f'rate = {bits_full / n_samples:.4f} bits/pair ({bits_full / (n_samples / fs) / 1000:.2f} kbps)')
print(f'PSNR = {psnr:.2f} dB ({ops[-1]} ch)')
display(make_spectrogram(x_hat[0]))
Audio(x_hat.numpy(), rate=fs)
rate = 4.0426 bits/pair (178.28 kbps)
PSNR = 46.08 dB (51 ch)
Per-macroregion rate profile#
Each macroregion carries its own byte-padded stream, so the rate allocation over time is directly visible — and any region could be transmitted, dropped, or upgraded independently.
t_edges = np.arange(len(enc.streams) + 1) * coder.macroregion / fs
fig, axes = plt.subplots(2, 1, figsize=(8, 4), dpi=120, sharex=True,
gridspec_kw={'height_ratios': [1, 1.6]})
t_sig = np.arange(n_samples) / fs
axes[0].plot(t_sig, x[0].numpy(), lw=0.3, color='0.4')
axes[0].set_ylabel('amplitude'); axes[0].set_title('input (left channel)', fontsize=9)
for n in ops:
rb = 8 * ((enc.prefix[:, n - 1] + 7) // 8)
axes[1].stairs(rb / (coder.macroregion / fs) / 1000, t_edges, label=f'{n} ch')
axes[1].set_xlabel('time (s)'); axes[1].set_ylabel('kbps')
axes[1].set_title(f'per-macroregion rate ({coder.macroregion} samples each)', fontsize=9)
axes[1].legend(fontsize=8); axes[1].grid(True, alpha=0.3)
plt.tight_layout(); plt.show()
Reconstructions at each operating point#
One encode serves every operating point: the latents are channel-prefix-truncated at each group boundary and decoded with that point’s own merged decoder; the rate is the same encode’s literal per-region bit-prefix.
td_len = 1000
td_start = n_samples // 2
td_sl = slice(td_start, td_start + td_len)
td_t = np.arange(td_len) / fs * 1e3
print('time-domain colors: ch1 original = blue, ch1 reconstructed = green, '
'ch2 original = red, ch2 reconstructed = purple')
print('original (channel 0)')
display(make_spectrogram(x[0]))
rd_clip = []
for n in ops:
partial = models[n]
lts = [torch.from_numpy(latents_np[s][:end - start]).unsqueeze(0).to(device)
for s, (ps_s, start, end) in enumerate(partial.scale_groups)]
with torch.no_grad():
xh = partial.decode(lts).clamp(-1, 1)[0].float().cpu()
bits_n = coder.op_bits(enc, n)
kbps_n = bits_n / (n_samples / fs) / 1000
psnr_n = -10 * torch.nn.functional.mse_loss(x / 2 + 0.5, xh / 2 + 0.5).log10().item()
rd_clip.append((bits_n / n_samples, psnr_n, n, xh))
print(f'{n:>2} ch: {bits_n / n_samples:.4f} bits/pair {kbps_n:8.2f} kbps '
f'CR={32 * n_samples / bits_n:8.1f}x PSNR={psnr_n:.2f} dB')
plt.figure(figsize=(4, 2), dpi=120)
plt.plot(td_t, x[0, td_sl].numpy(), alpha=0.5, c='b')
plt.plot(td_t, xh[0, td_sl].numpy(), alpha=0.5, c='g')
plt.plot(td_t, x[1, td_sl].numpy(), alpha=0.5, c='r')
plt.plot(td_t, xh[1, td_sl].numpy(), alpha=0.5, c='purple')
plt.xlabel('time (ms)'); plt.tight_layout(); plt.show()
display(make_spectrogram(xh[0]))
display(Audio(xh.numpy(), rate=fs))
time-domain colors: ch1 original = blue, ch1 reconstructed = green, ch2 original = red, ch2 reconstructed = purple
original (channel 0)
4 ch: 0.0397 bits/pair 1.75 kbps CR= 806.0x PSNR=23.13 dB
9 ch: 0.0850 bits/pair 3.75 kbps CR= 376.5x PSNR=25.22 dB
15 ch: 0.1921 bits/pair 8.47 kbps CR= 166.6x PSNR=29.01 dB
21 ch: 0.4900 bits/pair 21.61 kbps CR= 65.3x PSNR=33.11 dB
33 ch: 1.5641 bits/pair 68.98 kbps CR= 20.5x PSNR=42.25 dB
42 ch: 2.0598 bits/pair 90.84 kbps CR= 15.5x PSNR=42.56 dB
51 ch: 4.0426 bits/pair 178.28 kbps CR= 7.9x PSNR=46.08 dB
Rate-distortion (published validation average vs v2 baseline)#
The v3 curve is from audio/rd.json on the hub (the trainer’s final-pass round-trip-verified Kodak/musdb validation average). The v2 baseline (run B1, 7 operating points) is from results/frappe_v2_audio/.
rd3_path = hf_hub_download(repo_id='danjacobellis/FRAPPEv3', filename='audio/rd.json')
rd3 = json.load(open(rd3_path))
v3_bps = [p['bpp'] for p in rd3['rd_curve']]
v3_psnr = [p['psnr'] for p in rd3['rd_curve']]
v3_ops = rd3['ops']
rd2_path = sorted(glob.glob('results/frappe_v2_audio/rate_distortion_*.json'))[-1]
rd2 = json.load(open(rd2_path))
v2_qs = rd2['quality_values']
pcm_kbps = 16 * config.input_channels * fs / 1000
v2_kbps = [pcm_kbps / rd2['results'][str(q)]['mean']['CR'] for q in v2_qs]
v2_psnr = [rd2['results'][str(q)]['mean']['PSNR_dB'] for q in v2_qs]
v2_bps = [k * 1000 / fs for k in v2_kbps]
kbps_of = lambda bps_list: [b * fs / 1000 for b in bps_list]
plt.figure(figsize=(6.5, 4), dpi=120)
plt.semilogx(kbps_of(v2_bps), v2_psnr, '^-', color='black', ms=5, lw=1.2, alpha=0.8,
label='FRAPPE v2 audio + JPEG-LS (published B1)')
plt.semilogx(kbps_of(v3_bps), v3_psnr, 'o-', color='tab:blue',
label='FRAPPE v3 audio — musdb validation (verified streams)')
plt.semilogx(kbps_of([p[0] for p in rd_clip]), [p[1] for p in rd_clip], 's--', color='tab:blue',
alpha=0.5, label=f'v3 — demo clip (#{CLIP_INDEX})')
for n, bv, psv in zip(v2_qs, kbps_of(v2_bps), v2_psnr):
plt.annotate(f'{n}ch', (bv, psv), fontsize=7, color='black',
xytext=(3, 3), textcoords='offset points')
for n, bv, psv in zip(v3_ops, kbps_of(v3_bps), v3_psnr):
plt.annotate(f'{n}ch', (bv, psv), fontsize=7, color='tab:blue',
xytext=(3, -9), textcoords='offset points')
plt.xlabel('kbps'); plt.ylabel('PSNR (dB)')
plt.title('FRAPPE v3 audio vs v2 + JPEG-LS — musdb')
plt.grid(True, alpha=0.3); plt.legend(fontsize=8); plt.tight_layout(); plt.show()
print('v3 musdb validation average (rd.json, verified streams):')
for n, bv, psv in zip(v3_ops, v3_bps, v3_psnr):
print(f' {n:>2} ch: {bv:.6f} bits/pair ({bv * fs / 1000:8.3f} kbps) PSNR={psv:.3f} dB')
v3 musdb validation average (rd.json, verified streams):
4 ch: 0.032468 bits/pair ( 1.432 kbps) PSNR=28.059 dB
9 ch: 0.057352 bits/pair ( 2.529 kbps) PSNR=29.797 dB
15 ch: 0.107535 bits/pair ( 4.742 kbps) PSNR=31.293 dB
21 ch: 0.324187 bits/pair ( 14.297 kbps) PSNR=33.863 dB
33 ch: 1.095220 bits/pair ( 48.299 kbps) PSNR=39.268 dB
42 ch: 1.777679 bits/pair ( 78.396 kbps) PSNR=41.522 dB
51 ch: 3.226965 bits/pair ( 142.309 kbps) PSNR=44.577 dB
Spatial quality (SSDR / SRDR) — FRAPPE v3 vs MP3 / Opus#
Encode the same clip with MP3 and Opus at a low target bitrate via torchcodec, and compare waveform PSNR and the spatial-quality metrics SSDR / SRDR (Watcharasupat & Lerch 2024, via compressors.spatial_audio_quality) against every v3 operating point. SSDR captures per-channel distortion (gain errors, noise); SRDR captures interchannel spatial distortion (delay, leakage).
from torchcodec.encoders import AudioEncoder
from torchcodec.decoders import AudioDecoder
from compressors.audio_eval import align_length
from compressors.spatial_audio_quality import ssdr_srdr
def frappe_psnr(ref, est):
return -10 * torch.nn.functional.mse_loss(ref / 2 + 0.5, est / 2 + 0.5).log10().item()
def spatial_row(name, kbps_pt, est):
est = est.float().cpu()
m = ssdr_srdr(x, est, fs)
return (name, kbps_pt, frappe_psnr(x, est), m['SSDR'], m['SRDR'])
target_bit_rate = 1000
enc_tc = AudioEncoder(x, sample_rate=fs)
op_blob = enc_tc.to_tensor(format='opus', bit_rate=target_bit_rate, sample_rate=48_000)
mp_blob = enc_tc.to_tensor(format='mp3', bit_rate=target_bit_rate)
x_opus = align_length(AudioDecoder(op_blob.numpy().tobytes(), sample_rate=fs).get_all_samples().data, n_samples)
x_mp3 = align_length(AudioDecoder(mp_blob.numpy().tobytes(), sample_rate=fs).get_all_samples().data, n_samples)
op_kbps = 8 * len(op_blob) / 1000 / (n_samples / fs)
mp_kbps = 8 * len(mp_blob) / 1000 / (n_samples / fs)
rows = [spatial_row('Opus', op_kbps, x_opus), spatial_row('MP3', mp_kbps, x_mp3)]
rows += [spatial_row(f'FRAPPE v3 {nch}ch', bps * fs / 1000, xh)
for (bps, ps, nch, xh) in rd_clip]
print(f'{"codec":>16} {"kbps":>8} {"PSNR":>7} {"SSDR":>7} {"SRDR":>7}')
print('-' * 49)
for name, kb, p, ssdr, srdr in rows:
print(f'{name:>16} {kb:8.1f} {p:7.2f} {ssdr:7.2f} {srdr:7.2f}')
print('\ntime-domain colors: ch1 original = blue, ch1 reconstructed = green, '
'ch2 original = red, ch2 reconstructed = purple')
for name, kbps_pt, est in [('Opus', op_kbps, x_opus), ('MP3', mp_kbps, x_mp3)]:
est = est.float().cpu()
print(f'{name} ({kbps_pt:.1f} kbps)')
plt.figure(figsize=(4, 2), dpi=120)
plt.plot(td_t, x[0, td_sl].numpy(), alpha=0.5, c='b')
plt.plot(td_t, est[0, td_sl].numpy(), alpha=0.5, c='g')
plt.plot(td_t, x[1, td_sl].numpy(), alpha=0.5, c='r')
plt.plot(td_t, est[1, td_sl].numpy(), alpha=0.5, c='purple')
plt.title(f'{name} ({kbps_pt:.1f} kbps) \u2014 time domain', fontsize=9)
plt.xlabel('time (ms)'); plt.tight_layout(); plt.show()
display(make_spectrogram(est[0]))
display(Audio(est.numpy(), rate=fs))
codec kbps PSNR SSDR SRDR
-------------------------------------------------
Opus 5.6 18.42 1.61 -11.49
MP3 32.6 34.16 21.57 13.33
FRAPPE v3 4ch 1.8 23.13 3.45 0.69
FRAPPE v3 9ch 3.7 25.22 7.28 3.39
FRAPPE v3 15ch 8.5 29.01 15.93 7.74
FRAPPE v3 21ch 21.6 33.11 21.67 12.40
FRAPPE v3 33ch 69.0 42.25 41.32 21.28
FRAPPE v3 42ch 90.8 42.56 42.25 21.57
FRAPPE v3 51ch 178.3 46.08 50.80 25.10
time-domain colors: ch1 original = blue, ch1 reconstructed = green, ch2 original = red, ch2 reconstructed = purple
Opus (5.6 kbps)
/home/dan/g/lib/python3.14/site-packages/torchcodec/encoders/_audio_encoder.py:125: UserWarning: The given buffer is not writable, and PyTorch does not support non-writable tensors. This means you can write to the underlying (supposedly non-writable) buffer using the tensor. You may want to copy the buffer to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/utils/tensor_new.cpp:1556.)
return torch.frombuffer(buf.getvalue(), dtype=torch.uint8).clone()
MP3 (32.6 kbps)
