diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c2f5101..3417c678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ `Image.renderText(font, 32, 'Hi', {color: 0xff0000ff, strokeWidth: 2})`, `new Frame(10, 10, {duration: 50})`, `animation.encodeGIF({quality: 80})`. Options can be left out or added without counting positions; the positional arguments keep working until 4.0. +- **Smaller animations**: `encodeGIF()`, `encodeAPNG()` and `encodeWebP()` store each frame as the area that changed + since the frame before, with unchanged pixels transparent where that compresses better, and merge identical + frames. Re-encoding 762 decoded animations (which have every frame as a full image) took 51% (GIF), 63% (APNG) and + 54% (WebP) of the bytes in total; the less moves, the smaller a file gets, ones where everything moves stay about + the same. Large animations encode faster, small ones up to about twice as slow. The files show the same animation as + before, but don't decode to the same frames anymore: `{optimize: false}` stores the frames as they are. - **Metadata in every format**: `encodeJPEG()`, `encodeWebP()`, `encodeTGA()` and, for animations, `encodeGIF()`, `encodeAPNG()` and `encodeWebP()` take the same metadata as `encode()` (now called `Metadata`), as the `metadata` option: `image.encodeJPEG({quality: 80, metadata: {author: 'me'}})`. Each format stores diff --git a/src/animation.ts b/src/animation.ts index 3f6a4ecd..6199bdac 100644 --- a/src/animation.ts +++ b/src/animation.ts @@ -5,6 +5,7 @@ import { BinaryData, view } from './util/mem.ts'; import { Image, ResizeMode, ResizeOptions } from './image.ts'; import { Frame } from './frame.ts'; import { ImageType } from './image-type.ts'; +import { optimize, OptimizeOptions, OptimizedFrame } from './util/optimize.ts'; import { Metadata, resolveMetadata, pngText, exifData, gifComments } from './metadata.ts'; import { isOptions } from './util/options.ts'; @@ -37,6 +38,12 @@ export interface AnimationDecodeOptions { export interface AnimationEncodeOptions { /** Metadata to include, see {@link Metadata} */ metadata?: Metadata; + /** + * Whether to store each frame as the area that changed since the frame before and merge identical frames, which + * usually makes files a lot smaller. The file shows the same animation either way, but only decodes to the same + * frames without it, which also encodes faster when little changes. Default: `true` + */ + optimize?: boolean; } /** Options of {@link Animation.encodeGIF} */ @@ -163,73 +170,92 @@ export class Animation extends Array { } /** - * Encodes the animation as GIF + * Encodes the animation as GIF. Like all encoders, it stores each frame as the area that changed since the frame + * before and merges identical frames (unless the `optimize` option is false), so the file decodes to what the + * animation shows, but not necessarily to the same frames. Semi-transparent pixels become opaque or transparent + * after the frames are composited * @param options See {@link GIFEncodeOptions} */ encodeGIF(options?: GIFEncodeOptions): Promise; /** - * Encodes the animation as GIF + * Encodes the animation as GIF. Like all encoders, it stores each frame as the area that changed since the frame + * before and merges identical frames (unless the `optimize` option is false), so the file decodes to what the + * animation shows, but not necessarily to the same frames. Semi-transparent pixels become opaque or transparent + * after the frames are composited * @param quality 1-100, scales the palette size from 8 colors (1) to 256 colors (90 and above) */ encodeGIF(quality?: number): Promise; async encodeGIF(qualityOrOptions: number | GIFEncodeOptions = 95): Promise { - const { quality = 95, metadata = {} } = isOptions(qualityOrOptions) ? qualityOrOptions : { quality: qualityOrOptions }; + const { quality = 95, metadata = {}, optimize = true } = isOptions(qualityOrOptions) ? qualityOrOptions : { quality: qualityOrOptions }; this.__validate__(); - const frames: gifCodec.GIFFrame[] = []; - - for (const frame of this) - frames.push({ + const frames = optimize + ? this.__optimized__({ replace: false, even: false, alpha: 'frame', palette: 256, cover: true }).map(f => ({ ...f, dispose: f.dispose ? 3 : 1 })) + : Array.from(this, frame => ({ x: frame.xOffset, y: frame.yOffset, width: frame.width, height: frame.height, - // gifs store centiseconds; rounding keeps e.g. 16 ms (60 fps) from becoming 10 ms - delay: Math.round(frame.duration / 10), + duration: frame.duration, dispose: frame.__disposal_id__, pixels: new Uint8Array(frame.bitmap.buffer, frame.bitmap.byteOffset, frame.bitmap.byteLength) - }); + })); + // gifs store centiseconds; rounding keeps e.g. 16 ms (60 fps) from becoming 10 ms + const gifFrames = frames.map(({ duration, ...f }) => ({ ...f, delay: Math.round(duration / 10) })); - return gifCodec.encode({ width: this.width, height: this.height, loopCount: this.loopCount, frames }, quality, gifComments(resolveMetadata(metadata))); + return gifCodec.encode({ width: this.width, height: this.height, loopCount: this.loopCount, frames: gifFrames }, quality, gifComments(resolveMetadata(metadata))); } /** * Encodes the animation as APNG (animated PNG, lossless and with full alpha). Viewers without APNG support show - * the first frame + * the first frame. Each frame is stored as the area that changed since the frame before, identical frames are + * merged (unless the `optimize` option is false): the file shows exactly the same animation, but doesn't + * necessarily decode to the same frames * @param options See {@link APNGEncodeOptions} */ encodeAPNG(options?: APNGEncodeOptions): Promise; /** * Encodes the animation as APNG (animated PNG, lossless and with full alpha). Viewers without APNG support show - * the first frame + * the first frame. Each frame is stored as the area that changed since the frame before, identical frames are + * merged (unless the `optimize` option is false): the file shows exactly the same animation, but doesn't + * necessarily decode to the same frames * @param compression The zlib compression level (0-9), higher is smaller but slower */ encodeAPNG(compression?: number): Promise; async encodeAPNG(compressionOrOptions: number | APNGEncodeOptions = 1): Promise { - const { compression = 1, metadata = {} } = isOptions(compressionOrOptions) ? compressionOrOptions : { compression: compressionOrOptions }; + const { compression = 1, metadata = {}, optimize = true } = isOptions(compressionOrOptions) ? compressionOrOptions : { compression: compressionOrOptions }; this.__validate__(); if (!this.length) throw new Error('An animation needs at least one frame'); - return png.encodeAnimation(this.__apngFrames__(), this.width, this.height, toPlays(this.loopCount), compression, pngText(resolveMetadata(metadata))); + const frames = optimize + ? this.__optimized__({ replace: true, even: false, alpha: frames => png.alphaCost(frames.map(f => f.pixels)) }).map(f => ({ ...f, dispose: f.dispose ? 1 : 0 })) + : this.__apngFrames__().map(({ blend: _, ...f }) => f); // blended, the encoder's default + return png.encodeAnimation(frames, this.width, this.height, toPlays(this.loopCount), compression, pngText(resolveMetadata(metadata))); } /** - * Encodes the animation as lossless animated WebP + * Encodes the animation as lossless animated WebP. Each frame is stored as the area that changed since the frame + * before, identical frames are merged (unless the `optimize` option is false): the file shows exactly the same + * animation, but doesn't necessarily decode to the same frames * @param options See {@link AnimationEncodeOptions} */ - async encodeWebP({ metadata = {} }: AnimationEncodeOptions = {}): Promise { + async encodeWebP({ metadata = {}, optimize = true }: AnimationEncodeOptions = {}): Promise { this.__validate__(); if (!this.length) throw new Error('An animation needs at least one frame'); - const width = this.width, height = this.height; - const exif = exifData(resolveMetadata(metadata)); + const frames = optimize ? this.__optimized__({ replace: true, even: true, alpha: 'frame' }) : this.__webpFrames__(); + return webp.encodeAnimation(frames, this.width, this.height, toPlays(this.loopCount), exifData(resolveMetadata(metadata))); + } + + /** @internal the frames as they are, in the WebP codec's terms */ + __webpFrames__(): webp.WebPEncodeFrame[] { + const width = this.width, height = this.height; // WebP knows no "previous" disposal and stores offsets halved: such animations are stored as full frames const fullFrames = this.some(f => f.disposalMode === 'previous' || (f.disposalMode === 'background' && (f.xOffset & 1 || f.yOffset & 1))); - if (fullFrames) { - const canvases = png.compositeAnimation({ width, height, frames: this.__apngFrames__() }); - return webp.encodeAnimation(canvases.map(c => ({ x: 0, y: 0, width, height, duration: c.duration, dispose: true, pixels: c.pixels })), width, height, toPlays(this.loopCount), exif); - } + if (fullFrames) + return png.compositeAnimation({ width, height, frames: this.__apngFrames__() }) + .map(c => ({ x: 0, y: 0, width, height, duration: c.duration, dispose: true, pixels: c.pixels })); - const frames = Array.from(this, frame => { + return Array.from(this, frame => { let { xOffset: x, yOffset: y, width: w, height: h } = frame; let pixels = new Uint8Array(frame.bitmap.buffer, frame.bitmap.byteOffset, frame.bitmap.byteLength); // odd offsets: one transparent pixel more on the left/top (blending makes it invisible) @@ -242,7 +268,15 @@ export class Animation extends Array { } return { x, y, width: w, height: h, duration: frame.duration, dispose: frame.disposalMode === 'background', pixels }; }); - return webp.encodeAnimation(frames, width, height, toPlays(this.loopCount), exif); + } + + /** + * The frames as the encoders store them: each frame only covers what changed since the one before (see + * util/optimize.ts) + * @internal + */ + __optimized__(options: OptimizeOptions): OptimizedFrame[] { + return optimize(this.__apngFrames__(), this.width, this.height, options); } /** @internal frames in the APNG codec's terms (alpha blended, disposal 0 none, 1 background, 2 previous) */ diff --git a/src/codecs/gif.ts b/src/codecs/gif.ts index dbd3e6dc..39626783 100644 --- a/src/codecs/gif.ts +++ b/src/codecs/gif.ts @@ -64,8 +64,13 @@ export function encode({ width, height, loopCount = -1, frames }: Omit frame.dispose === 3); + for (const [i, frame] of frames.entries()) { + const transparent = frame.dispose === 3 || (i === 0 && restores); + const { palette, indices, transparentIndex } = quantize(frame.pixels, frame.width, { quality, transparent }); const entries = palette.length / 3; const bits = Math.max(1, Math.ceil(Math.log2(entries))); diff --git a/src/codecs/png.ts b/src/codecs/png.ts index 22fcf1dc..709a8d0a 100644 --- a/src/codecs/png.ts +++ b/src/codecs/png.ts @@ -155,6 +155,37 @@ interface PixelFormat { * Picks the smallest lossless format that fits all the given RGBA images: a palette for up to 256 colors, * otherwise greyscale or RGB, with alpha only if a pixel isn't opaque */ +/** + * How much more an opaque image costs per pixel with transparent pixels than without them, given the pixel formats + * (palette, grey or truecolor) both get: e.g. 4/3 for RGBA instead of RGB, 4 for RGBA instead of a palette when the + * transparent pixels are the 257th color, 2 for 8 instead of 4 bit palette indices when they're the 17th + * @param images RGBA, only the transparent pixels aren't opaque + */ +export function alphaCost(images: Uint8Array[]): number { + let grey = true; + const colors = new Set(); + // more than 256 colors and not grey: truecolor either way, nothing else can change + scan: for (const rgba of images) { + const pixels = new Uint32Array(rgba.buffer, rgba.byteOffset, rgba.length >> 2); + let previous = -1; + for (let i = 0; i < pixels.length; i++) { + const value = pixels[i]; + if (value === previous) continue; + previous = value; + const o = i * 4; + if (rgba[o + 3] === 0) continue; + if (rgba[o] !== rgba[o + 1] || rgba[o] !== rgba[o + 2]) grey = false; + if (colors.size <= 256) colors.add(value); + else if (!grey) break scan; + } + } + // bytes per pixel: palette 1/8 to 1 (by its bit depth), grey 1 (with alpha 2), truecolor 3 (with alpha 4) + const palette = (entries: number): number => (entries <= 2 ? 1 : entries <= 4 ? 2 : entries <= 16 ? 4 : 8) / 8; + const transparent = colors.size < 256 ? palette(colors.size + 1) : grey ? 2 : 4; + const opaque = colors.size <= 256 ? palette(colors.size) : grey ? 1 : 3; + return transparent / opaque; +} + function pixelFormat(images: Uint8Array[]): PixelFormat { let opaque = true, grey = true; @@ -273,6 +304,8 @@ export interface APNGEncodeFrame { duration: number; /** 0 none, 1 background (transparent), 2 previous */ dispose: number; + /** whether the frame is alpha blended onto the canvas, otherwise it replaces its area. Default: true */ + blend?: boolean; /** RGBA */ pixels: Uint8Array; } @@ -337,7 +370,7 @@ export async function encodeAnimation(frames: APNGEncodeFrame[], width: number, if (ms <= 65535) { v.setUint16(20, ms); v.setUint16(22, 1000); } else { v.setUint16(20, Math.min(65535, Math.round(ms / 10))); v.setUint16(22, 100); } fctl[24] = f.dispose; - fctl[25] = 1; // APNG_BLEND_OP_OVER + fctl[25] = f.blend === false ? 0 : 1; // APNG_BLEND_OP_SOURCE or APNG_BLEND_OP_OVER chunks.push(chunk(fcTL, fctl)); const data = await compressRows(format.convert(images[i], f.width, f.height), f.width, f.height, channels, depth, level, filterRows); diff --git a/src/codecs/quantize.ts b/src/codecs/quantize.ts index 39ceed58..0852ad1d 100644 --- a/src/codecs/quantize.ts +++ b/src/codecs/quantize.ts @@ -13,6 +13,8 @@ const WR = 3, WG = 4, WB = 2; export interface QuantizeOptions { maxColors?: number; quality?: number; + /** whether to have a transparent index even if no pixel is transparent */ + transparent?: boolean; } export interface Quantized { @@ -59,9 +61,9 @@ interface Box { error: number; } -export function quantize(pixels: Uint8Array, width: number, { maxColors = 256, quality = 95 }: QuantizeOptions = {}): Quantized { +export function quantize(pixels: Uint8Array, width: number, { maxColors = 256, quality = 95, transparent = false }: QuantizeOptions = {}): Quantized { const length = pixels.length >> 2; - let hasTransparency = false; + let hasTransparency = transparent; for (let i = 3; i < pixels.length; i += 4) if (pixels[i] < 128) { hasTransparency = true; diff --git a/src/codecs/webp.ts b/src/codecs/webp.ts index 878d55ac..673e0d3e 100644 --- a/src/codecs/webp.ts +++ b/src/codecs/webp.ts @@ -327,7 +327,9 @@ export interface WebPEncodeFrame { duration: number; /** whether the frame's area is cleared after it was shown */ dispose: boolean; - /** RGBA, alpha blended onto the canvas */ + /** whether the frame is alpha blended onto the canvas, otherwise it replaces its area. Default: true */ + blend?: boolean; + /** RGBA */ pixels: Uint8Array; } @@ -340,7 +342,11 @@ export function encodeAnimation(frames: WebPEncodeFrame[], width: number, height if (!frames.length) throw new Error('WebP: an animation needs at least one frame'); if (width > 1 << 24 || height > 1 << 24) throw new RangeError('WebP: canvas too large'); - const alpha = frames.some(f => hasAlpha(f.pixels)); + // transparency also comes from the canvas: where the first frame doesn't cover it, and where frames are disposed + // (decoders like libwebp take the canvas as opaque without the flag) + const first = frames[0]; + const alpha = first.x > 0 || first.y > 0 || first.width < width || first.height < height + || frames.some(f => f.dispose || hasAlpha(f.pixels)); const vp8x = new Uint8Array(10); vp8x[0] = 2 | (alpha ? 16 : 0) | (exif ? 8 : 0); // animation, alpha, exif @@ -361,7 +367,7 @@ export function encodeAnimation(frames: WebPEncodeFrame[], width: number, height set24(header, 6, f.width - 1); set24(header, 9, f.height - 1); set24(header, 12, Math.min(0xffffff, Math.max(0, Math.round(f.duration)))); - header[15] = f.dispose ? 1 : 0; // blending (bit 1 unset) and disposal + header[15] = (f.blend === false ? 2 : 0) | (f.dispose ? 1 : 0); // no blending, disposal const body = new Uint8Array(16 + bitstream.length); body.set(header); body.set(bitstream, 16); diff --git a/src/util/optimize.ts b/src/util/optimize.ts new file mode 100644 index 00000000..bb09f3ed --- /dev/null +++ b/src/util/optimize.ts @@ -0,0 +1,368 @@ +// Frame optimization for the animation encoders: the frames are composited, and each is stored as the rectangle that +// changed since the frame before, with the pixels that didn't change transparent (they compress better, and GIFs keep +// their palette for the others). Consecutive identical frames become one. The animation looks the same as before + +// a pixel's alpha bits, read as one 32-bit word (whatever the byte order): 0 transparent, ALPHA opaque +const ALPHA = new Int32Array(new Uint8Array([0, 0, 0, 255]).buffer)[0]; + +export interface InputFrame { + x: number; + y: number; + width: number; + height: number; + /** milliseconds */ + duration: number; + /** 0 none, 1 background (transparent), 2 previous */ + dispose: number; + /** RGBA, alpha blended onto the canvas */ + pixels: Uint8Array; +} + +export interface OptimizedFrame { + x: number; + y: number; + width: number; + height: number; + /** milliseconds */ + duration: number; + /** whether the frame's area is cleared (to transparent) after it was shown */ + dispose: boolean; + /** whether the frame is alpha blended onto the canvas (otherwise it replaces its area) */ + blend: boolean; + /** RGBA */ + pixels: Uint8Array; +} + +export interface OptimizeOptions { + /** + * Whether frames can replace their area instead of being blended onto it (APNG, WebP). GIFs can't: pixels that get + * more transparent are cleared by the disposal of the frame before instead + */ + replace: boolean; + /** Whether frame offsets have to be even (WebP stores them halved) */ + even: boolean; + /** + * What the transparent pixels of sparse frames cost when they would be opaque otherwise. "frame" if each frame + * stores an alpha channel (WebP) or transparent palette index (GIF) only if it needs one. If all frames share one + * pixel format (APNG): a function that tells how much more an opaque animation costs per pixel with its frames + * (the transparent pixels are the sparse frames' unchanged ones) than without the transparent pixels + */ + alpha: 'frame' | ((frames: OptimizedFrame[]) => number); + /** + * The palette size if each frame has one (GIF: 256): a frame stays dense if its colors fit, but its sparse pixels + * (the transparent ones are a color more) don't, so they'd lose colors + */ + palette?: number; + /** + * Whether the first frame has to cover the canvas (GIF: decoders disagree on what's outside it, browsers show it + * transparent, others the background color) + */ + cover?: boolean; +} + +// what transparent pixels make an opaque frame cost: an alpha channel, 4 channels instead of 3 (or a palette entry +// less in GIFs, which costs about as much) +const ALPHA_COST = 4 / 3; + +/** + * Composites the frames and stores each as the area that changed + * @param width The canvas width + * @param height The canvas height + */ +export function optimize(frames: InputFrame[], width: number, height: number, { replace, even, alpha, palette, cover }: OptimizeOptions): OptimizedFrame[] { + const size = width * height * 4; + // the input animation's canvas, and what the optimized animation shows after its last frame (and its disposal) + const canvas = new Uint8Array(size), canvas32 = new Int32Array(canvas.buffer); + const shown = new Uint8Array(size), shown32 = new Int32Array(shown.buffer); + const output: OptimizedFrame[] = []; + + const state: State = { canvas32, shown32, width, replace }; + + // for a shared pixel format: whether the animation is opaque when all frames are stored dense, and the changes + // from pixel to pixel all dense and as chosen + let opaque = true, denseTotal = 0, chosenTotal = 0; + + // stores what the canvas shows now as the next frame + const add = (duration: number): void => { + let changed = bounds(state, { x: 0, y: 0, width, height }); + if (!changed) { + if (output.length) output[output.length - 1].duration += duration; + else { + output.push(empty(duration)); + opaque = false; + } + return; + } + if (even) changed = evenArea(changed); + let area = sparse(state, changed); + let blend = true; + if (area.unblendable && replace) blend = false; + else if (area.unblendable) { + // GIF: the frame before clears an area covering these pixels (it grows by transparent, unchanged pixels + // if needed), then the frame is blended onto the cleared area + const last = output[output.length - 1]; + grow(last, union(last, area.unblendable)); + last.dispose = true; + clear(shown, width, last); + changed = bounds(state, { x: 0, y: 0, width, height }); + // the pixels only became transparent: clearing them is all the frame shows, it only takes its time + if (!changed) return void output.push(empty(duration)); + if (even) changed = evenArea(changed); + area = sparse(state, changed); + } + + // blended frames: the changed pixels, the others transparent ("sparse"). The area as it is ("dense") + // compresses better when the unchanged pixels are scattered (e.g. in videos): choose the one with fewer + // changes from pixel to pixel, which estimates the compressed size well and costs a lot less than compressing + // both. GIFs blend the dense area too (its unchanged pixels are the same or transparent in both), the others + // replace the area with it + const { x, y, width: w, height: h } = changed; + const pixels = area.pixels; + if (!output.length && (w !== width || h !== height)) opaque = false; + if (!area.denseOpaque) opaque = false; + const sparseCost = alpha === 'frame' && area.denseOpaque ? area.sparseChanges * ALPHA_COST : area.sparseChanges; + const dense = !blend || area.denseChanges < sparseCost + || (palette !== undefined && colors(new Int32Array(pixels.buffer), w, { x: 0, y: 0, width: w, height: h }, palette + 1) > palette + && colors(canvas32, width, changed, palette + 1) <= palette); + denseTotal += area.denseChanges; + chosenTotal += dense ? area.denseChanges : area.sparseChanges; + if (dense) { + for (let row = 0; row < h; row++) pixels.set(canvas.subarray(((y + row) * width + x) * 4, ((y + row) * width + x + w) * 4), row * w * 4); + if (replace) blend = false; + } + for (let row = 0; row < h; row++) { + const start = ((y + row) * width + x) * 4; + shown.set(canvas.subarray(start, start + w * 4), start); + } + output.push({ x, y, width: w, height: h, duration, dispose: false, blend, pixels }); + }; + + // the area of the canvas that is all transparent (frames are copied onto it): decoded animations clear each full + // frame before the next + let cleared: Rect | null = { x: 0, y: 0, width, height }; + for (const f of frames) { + const saved = f.dispose === 2 ? canvas.slice() : null; + const area = clip(f, width, height); + if (cleared && contains(cleared, area)) copy(canvas, width, f, area); + else draw(canvas, width, height, f); + cleared = null; + add(f.duration); + if (f.dispose === 1) { + clear(canvas, width, area); + cleared = area; + } else if (saved) canvas.set(saved); + } + + // an opaque animation with sparse frames needs transparency in its pixel format, for every frame: when that costs + // more than storing them dense, the unchanged pixels are filled in again + if (typeof alpha === 'function' && opaque && output.some(f => f.blend) && denseTotal < chosenTotal * alpha(output)) { + shown.fill(0); + for (const f of output) { + const pixels32 = new Int32Array(f.pixels.buffer, f.pixels.byteOffset, f.width * f.height); + for (let row = 0, k = 0; row < f.height; row++, k += f.width) { + const start = (f.y + row) * width + f.x; + if (!f.blend) shown32.set(pixels32.subarray(k, k + f.width), start); + else + for (let i = k, p = start, end = k + f.width; i < end; i++, p++) { + if ((pixels32[i] & ALPHA) === 0) pixels32[i] = shown32[p]; + else shown32[p] = pixels32[i]; + } + } + f.blend = false; + } + } + + if (cover && output.length) grow(output[0], { x: 0, y: 0, width, height }); + return output; +} + +// how many colors an area of an image has, counted up to a limit +function colors(image32: Int32Array, width: number, { x, y, width: w, height: h }: Rect, limit: number): number { + const seen = new Set(); + for (let row = y; row < y + h; row++) { + let previous: number | undefined; + for (let p = row * width + x, end = p + w; p < end; p++) { + const value = image32[p]; + if (value === previous) continue; + previous = value; + seen.add(value); + if (seen.size >= limit) return seen.size; + } + } + return seen.size; +} + +// a frame that doesn't change anything +const empty = (duration: number): OptimizedFrame => ({ x: 0, y: 0, width: 1, height: 1, duration, dispose: false, blend: true, pixels: new Uint8Array(4) }); + +interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +function clip(r: Rect, width: number, height: number): Rect { + const x = Math.max(0, r.x), y = Math.max(0, r.y); + return { x, y, width: Math.max(0, Math.min(width, r.x + r.width) - x), height: Math.max(0, Math.min(height, r.y + r.height) - y) }; +} + +function contains(a: Rect, b: Rect): boolean { + return b.x >= a.x && b.y >= a.y && b.x + b.width <= a.x + a.width && b.y + b.height <= a.y + a.height; +} + +function union(a: Rect, b: Rect): Rect { + const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y); + return { x, y, width: Math.max(a.x + a.width, b.x + b.width) - x, height: Math.max(a.y + a.height, b.y + b.height) - y }; +} + +interface State { + canvas32: Int32Array; + shown32: Int32Array; + width: number; + replace: boolean; +} + +// the area grown to even offsets (WebP stores them halved) +function evenArea({ x, y, width, height }: Rect): Rect { + return { x: x & ~1, y: y & ~1, width: width + (x & 1), height: height + (y & 1) }; +} + +interface SparseArea { + /** RGBA */ + pixels: Uint8Array; + sparseChanges: number; + denseChanges: number; + unblendable: Rect | null; + /** whether the area as it is has no transparent pixels */ + denseOpaque: boolean; +} + +/** + * The area's changed pixels, the others transparent, and how many times the color changes from pixel to pixel in that + * and in the area as it is. Also finds the area of the changed pixels that blending onto what's shown can't produce: + * blending gives exactly opaque pixels, for GIFs (which only know opaque and transparent) also anything on a + * transparent pixel + */ +function sparse({ canvas32, shown32, width, replace }: State, { x, y, width: w, height: h }: Rect): SparseArea { + const pixels = new Uint8Array(w * h * 4), pixels32 = new Int32Array(pixels.buffer); + let sparseChanges = 0, denseChanges = 0, lastSparse = -1, lastDense = -1; + let left = w, right = -1, top = h, bottom = -1, translucent = 0; + for (let row = 0, k = 0; row < h; row++) { + const start = (y + row) * width + x; + for (let i = 0, p = start; i < w; i++, p++, k++) { + const value = canvas32[p], before = shown32[p]; + // unchanged: the same, or transparent in both + if (value === before || ((value | before) & ALPHA) === 0) { + if (lastSparse !== 0) sparseChanges++; + lastSparse = 0; + } else { + pixels32[k] = value; + if (value !== lastSparse) sparseChanges++; + lastSparse = value; + if ((value & ALPHA) !== ALPHA && (replace || (before & ALPHA) !== 0)) { + if (i < left) left = i; + if (i > right) right = i; + if (row < top) top = row; + bottom = row; + } + } + if (value !== lastDense) { + denseChanges++; + translucent |= ~value & ALPHA; + } + lastDense = value; + } + } + const unblendable: Rect | null = right < 0 ? null : { x: x + left, y: y + top, width: right - left + 1, height: bottom - top + 1 }; + return { pixels, sparseChanges, denseChanges, unblendable, denseOpaque: translucent === 0 }; +} + +/** + * The bounding box of the changed pixels within an area. Searches from the edges inwards, so it stops early when much + * changed + */ +function bounds({ canvas32, shown32, width }: State, area: Rect): Rect | null { + const x0 = area.x, x1 = area.x + area.width, y0 = area.y, y1 = area.y + area.height; + const test = (p: number): boolean => { + const c = canvas32[p], s = shown32[p]; + return c !== s && ((c | s) & ALPHA) !== 0; + }; + const rowHas = (y: number, from: number, to: number): number => { + for (let x = from, p = y * width + from; x < to; x++, p++) if (test(p)) return x; + return -1; + }; + + let top = y0, first = -1; + for (; top < y1; top++) if ((first = rowHas(top, x0, x1)) >= 0) break; + if (top === y1) return null; + let bottom = y1 - 1; + while (bottom > top && rowHas(bottom, x0, x1) < 0) bottom--; + + // the columns: only the parts of the rows outside the bounds found so far + let left = first, right = first; + for (let y = top; y <= bottom; y++) { + if (left > x0) { + const x = rowHas(y, x0, left); + if (x >= 0) left = x; + } + for (let x = x1 - 1, p = y * width + x; x > right; x--, p--) + if (test(p)) { + right = x; + break; + } + } + return { x: left, y: top, width: right - left + 1, height: bottom - top + 1 }; +} + +// alpha blends a frame onto the canvas, like APNG's "over" (and GIF's transparency) +function draw(canvas: Uint8Array, width: number, height: number, f: InputFrame): void { + const { x: x0, y: y0, width: w, height: h } = clip(f, width, height); + const pixels = f.pixels.byteOffset % 4 ? f.pixels.slice() : f.pixels; + const src = new Int32Array(pixels.buffer, pixels.byteOffset, f.width * f.height); + const dst = new Int32Array(canvas.buffer, canvas.byteOffset, width * height); + for (let y = y0; y < y0 + h; y++) + for (let x = x0, s = (y - f.y) * f.width + x - f.x, d = y * width + x; x < x0 + w; x++, s++, d++) { + // opaque pixels replace the canvas: runs of them are copied at once (much faster) + let n = 0; + while (n < x0 + w - x && (src[s + n] & ALPHA) === ALPHA) n++; + if (n) { + if (n >= 16) dst.set(src.subarray(s, s + n), d); + else for (let i = 0; i < n; i++) dst[d + i] = src[s + i]; + x += n - 1; s += n - 1; d += n - 1; + continue; + } + const value = src[s], alpha = value & ALPHA; + if (alpha === 0) continue; + if (alpha === ALPHA || (dst[d] & ALPHA) === 0) dst[d] = value; + else { + const sa = pixels[s * 4 + 3], da = canvas[d * 4 + 3]; + const u = sa * 255, v = (255 - sa) * da, al = u + v; + for (let c = 0; c < 3; c++) canvas[d * 4 + c] = (pixels[s * 4 + c] * u + canvas[d * 4 + c] * v) / al | 0; + canvas[d * 4 + 3] = al / 255 | 0; + } + } +} + +// draws a frame onto a transparent area: blending only copies its pixels (transparent ones as 0, like draw()) +function copy(canvas: Uint8Array, width: number, f: InputFrame, { x, y, width: w, height: h }: Rect): void { + const canvas32 = new Int32Array(canvas.buffer, canvas.byteOffset, width * (y + h)); + for (let row = y; row < y + h; row++) { + const s = ((row - f.y) * f.width + x - f.x) * 4, d = row * width + x; + canvas.set(f.pixels.subarray(s, s + w * 4), d * 4); + for (let p = d; p < d + w; p++) if ((canvas32[p] & ALPHA) === 0) canvas32[p] = 0; + } +} + +function clear(canvas: Uint8Array, width: number, r: Rect): void { + for (let y = r.y; y < r.y + r.height; y++) canvas.fill(0, (y * width + r.x) * 4, (y * width + r.x + r.width) * 4); +} + +// grows a frame to a larger area, the new pixels are transparent +function grow(frame: OptimizedFrame, area: Rect): void { + if (area.x === frame.x && area.y === frame.y && area.width === frame.width && area.height === frame.height) return; + const pixels = new Uint8Array(area.width * area.height * 4); + for (let row = 0; row < frame.height; row++) + pixels.set(frame.pixels.subarray(row * frame.width * 4, (row + 1) * frame.width * 4), ((frame.y - area.y + row) * area.width + frame.x - area.x) * 4); + Object.assign(frame, area, { pixels }); +} diff --git a/tests/animation-optimize.js b/tests/animation-optimize.js new file mode 100644 index 00000000..43a670a8 --- /dev/null +++ b/tests/animation-optimize.js @@ -0,0 +1,165 @@ +const {Image, Frame, Animation} = require('..'); + +const panic = message => { + console.error(message); + process.exit(1); +}; + +// what an animation shows: its composited frames, consecutive identical ones merged (transparent pixels compare equal +// whatever their color) +async function timeline(animation) { + const decoded = animation instanceof Animation ? animation : await Animation.decode(animation); + const result = []; + for (const frame of decoded) { + const pixels = Buffer.from(frame.bitmap); + for (let i = 0; i < pixels.length; i += 4) if (pixels[i + 3] === 0) pixels.fill(0, i, i + 4); + const last = result[result.length - 1]; + if (last && last.pixels.equals(pixels)) last.duration += frame.duration; + else result.push({pixels, duration: frame.duration}); + } + return result; +} + +// the input animation's composited frames, as APNG decoders composite them (blending "over", disposal) +async function composite(animation) { + const png = await import('../dist/codecs/png.js'); + const {width, height} = animation; + const canvases = png.compositeAnimation({width, height, frames: animation.__apngFrames__()}); + return new Animation(canvases.map(({pixels, duration}) => { + const frame = new Frame(width, height, duration); + frame.bitmap.set(pixels); + return frame; + })); +} + +const same = (a, b) => a.length === b.length && a.every((f, i) => f.pixels.equals(b[i].pixels) && f.duration === b[i].duration); + +// GIF only has opaque and transparent pixels and a palette: compare transparency exactly, colors on average +function gifMatches(expected, actual) { + if (expected.length !== actual.length) return `${actual.length} frames instead of ${expected.length}`; + for (let k = 0; k < expected.length; k++) { + const a = expected[k].pixels, b = actual[k].pixels; + let error = 0, opaque = 0; + for (let i = 0; i < a.length; i += 4) { + if ((a[i + 3] >= 128) !== (b[i + 3] === 255)) return `frame ${k}: transparency differs at pixel ${i / 4}`; + if (!b[i + 3]) continue; + error += Math.abs(a[i] - b[i]) + Math.abs(a[i + 1] - b[i + 1]) + Math.abs(a[i + 2] - b[i + 2]); + opaque++; + } + if (opaque && error / opaque / 3 > 8) return `frame ${k}: colors differ by ${(error / opaque / 3).toFixed(1)} on average`; + } + return null; +} + +async function check(name, animation) { + const expected = await timeline(await composite(animation)); + for (const format of ['APNG', 'WebP']) { + const actual = await timeline(await animation[`encode${format}`]()); + if (!same(expected, actual)) panic(`${name}: ${format} doesn't show the same animation`); + } + // GIF: opaque/transparent only, compare with the animation's transparency rounded (gifMatches) + const error = gifMatches(expected, await timeline(await animation.encodeGIF())); + if (error) panic(`${name}: GIF ${error}`); +} + +// an antialiased disc +const disc = (size, color) => new Image(size, size).fill((x, y) => { + const d = Math.hypot(x - size / 2 + 0.5, y - size / 2 + 0.5) - size / 2 + 1; + return d <= 0 ? color : d >= 1 ? 0 : (color & ~255) | Math.round((1 - d) * (color & 255)); +}); + +(async () => { + // a sticker: an antialiased disc moving over a transparent background (pixels become transparent again) + await check('moving sticker', new Animation(Array.from({length: 8}, (_, i) => + Frame.from(new Image(48, 32).composite(disc(16, 0xff8800ff), i * 4, 4 + (i & 1)), 60)))); + + // fading out: pixels get more transparent + await check('fade out', new Animation(Array.from({length: 6}, (_, i) => + Frame.from(new Image(20, 20).drawBox(4, 4, 12, 12, 0x3366ff00 | 255 - i * 50), 100)))); + + // frames with offsets and disposal modes, odd offsets (WebP stores them halved) + await check('offsets and disposal', new Animation([ + Frame.from(new Image(30, 30).fill(0x202020ff), 100), + new Frame(5, 7, 100, 3, 9, 'previous').fill(0xff0000ff), + new Frame(9, 3, 100, 11, 1, 'background').fill(0x00ff0080), + new Frame(4, 4, 100, 21, 21, 'keep').fill(0x0000ffff), + new Frame(30, 30, 100, 0, 0, 'background').fill(0), + new Frame(1, 1, 100, 29, 29).fill(0xffffffff) + ])); + + // a frame that only clears: GIF can't blend pixels to transparent, the frame before clears its area instead + await check('cleared frame', new Animation([ + Frame.from(new Image(12, 12).drawBox(2, 2, 6, 6, 0xff0000ff), 100, 0, 0, 'background'), + new Frame(12, 12, 100), + Frame.from(new Image(12, 12).drawBox(4, 4, 6, 6, 0x00ff00ff), 100) + ])); + + // an empty first frame, and identical frames, which become one with their durations added up (the empty last + // frame doesn't clear the canvas, so it shows the same too) + { + const dot = Frame.from(new Image(10, 10).setPixelAt(5, 5, 0xffffffff), 100); + const animation = new Animation([new Frame(10, 10, 40), dot, dot.clone(), Frame.from(dot, 25), new Frame(10, 10, 30)]); + await check('empty and identical frames', animation); + const decoded = await Animation.decode(await animation.encodeAPNG()); + if (decoded.length !== 2 || decoded[0].duration !== 40 || decoded[1].duration !== 255) panic(`identical frames: ${decoded.length} frames, durations ${Array.from(decoded, f => f.duration)}`); + } + + // an opaque animation whose frames are smaller sparse, but not by enough to pay for an alpha channel in every APNG + // frame: they're filled in again + { + const background = new Image(40, 20).fill((x, y) => Image.rgbaToColor(y * 6, 100, 200 - y * 4, 255)); + const animation = new Animation(Array.from({length: 6}, (_, i) => Frame.from(background.clone() + .drawBox(1, 1, 6, 6, Image.rgbaToColor(255, i * 40, 0, 255)) + .drawBox(32, 13, 6, 6, Image.rgbaToColor(0, 255, i * 40, 255)), 100))); + await check('opaque animation', animation); + // with transparency for free, the frames are sparse; with an alpha channel that costs as much as RGBA instead of RGB + // (APNG), they're filled in + if (!animation.__optimized__({replace: true, even: false, alpha: () => 1}).slice(1).every(f => f.blend)) panic('opaque animation: expected sparse frames'); + if (animation.__optimized__({replace: true, even: false, alpha: () => 4 / 3}).some(f => f.blend)) panic('opaque animation: frames weren\'t filled in'); + } + + // a GIF frame stays dense when its colors fill the palette and its sparse pixels wouldn't fit (they'd need one more + // color for the unchanged pixels): it stays lossless. The blocks around the center change color, the center doesn't + { + const color = index => Image.rgbaToColor(index, 255 - index, index * 37 & 255, 255); + // 24×24 blocks: the ring around the center has all 256 colors in turn, the center some of them + const ring = (i, j) => i < 4 || i > 19 || j < 4 || j > 19; + const grid = shift => { + const blocks = []; + for (let j = 0, k = 0; j < 24; j++) for (let i = 0; i < 24; i++) blocks.push(ring(i, j) ? (k++ + shift) & 255 : (i * 7 + j * 13) & 255); + return new Image(96, 96).fill((x, y) => color(blocks[(y >> 2) * 24 + (x >> 2)])); + }; + const animation = new Animation([Frame.from(grid(0), 100), Frame.from(grid(1), 100)]); + const decoded = await Animation.decode(await animation.encodeGIF()); + decoded.forEach((frame, i) => { + if (frame.bitmap.some((v, k) => v !== animation[i].bitmap[k])) panic(`palette: GIF frame ${i} lost colors`); + }); + } + + // optimize: false stores the frames as they are: same animation, but no frames merged or cut + { + const dot = Frame.from(new Image(10, 10).setPixelAt(5, 5, 0xffffffff), 100); + const animation = new Animation([new Frame(10, 10, 40), dot, dot.clone(), new Frame(4, 4, {duration: 30, xOffset: 3, yOffset: 1}).fill(0xff0000ff)]); + for (const format of ['GIF', 'APNG', 'WebP']) { + const plain = await Animation.decode(await animation[`encode${format}`]({optimize: false})); + if (plain.length !== 4) panic(`optimize false: ${format} has ${plain.length} frames instead of 4`); + const expected = await composite(animation); + if (format === 'APNG' && plain.some((f, i) => f.bitmap.some((v, k) => v !== expected[i].bitmap[k]))) panic('optimize false: APNG frames differ'); + } + await check('optimize false', animation); + } + + // a mostly static animation (like a screen recording) only stores what changes + { + const background = new Image(160, 120).fill((x, y) => Image.rgbaToColor(x * 7 ^ y * 13, x * y, y * 5, 255)); + const animation = new Animation(Array.from({length: 20}, (_, i) => + Frame.from(background.clone().drawBox(10 + i * 6, 50, 8, 8, 0xffffffff), 50))); + await check('screen recording', animation); + const still = (await background.encode()).length; + for (const format of ['APNG', 'WebP', 'GIF']) { + const size = (await animation[`encode${format}`]()).length; + if (size > 3 * (format === 'GIF' ? (await new Animation([Frame.from(background)]).encodeGIF()).length : still)) + panic(`screen recording: ${format} is ${size} bytes, the changes aren't stored alone`); + } + } +})(); diff --git a/tests/animation.js b/tests/animation.js index c1ca4f5c..f69b4d9b 100644 --- a/tests/animation.js +++ b/tests/animation.js @@ -57,7 +57,8 @@ const square = (color, w, h) => new Image(w, h).fill((x, y) => (x + y) % 5 === 0 // loop counts: -1 forever, 0 once, n repeats for (const loopCount of [-1, 0, 5]) { - const two = new Animation([Frame.from(still), Frame.from(still)], loopCount); + // (different frames: identical ones are merged, and a single frame has no loop count) + const two = new Animation([Frame.from(still), Frame.from(still.clone().invert())], loopCount); for (const format of ['APNG', 'WebP', 'GIF']) if ((await Animation.decode(await two[`encode${format}`]())).loopCount !== loopCount) panic(`${format}: loop count ${loopCount} not kept`); } diff --git a/tests/regressions.js b/tests/regressions.js index 011e25cf..8bcb7226 100644 --- a/tests/regressions.js +++ b/tests/regressions.js @@ -324,7 +324,8 @@ const rgba = (image, x, y) => Image.colorToRGBA(image.getPixelAt(x, y)); // gif frame durations are rounded to centiseconds { - const gif = new GIF([new Frame(2, 2, 16), new Frame(2, 2, 14)]); + // (different frames: identical ones are merged) + const gif = new GIF([new Frame(2, 2, 16), new Frame(2, 2, 14).fill(0xff0000ff)]); const decoded = await GIF.decode(await gif.encode()); assert(decoded[0].duration === 20 && decoded[1].duration === 10, `gif durations should round, got ${decoded[0].duration}, ${decoded[1].duration}`); } @@ -366,7 +367,7 @@ const rgba = (image, x, y) => Image.colorToRGBA(image.getPixelAt(x, y)); assert(frame.disposalMode === 'previous', 'disposalMode should be a name, got ' + frame.disposalMode); frame.disposalMode = 0; assert(frame.disposalMode === Frame.DISPOSAL_ANY, 'disposalMode set by id should read back as name'); - const decoded = await GIF.decode(await new GIF([new Frame(2, 2, 100, 0, 0, 'background'), new Frame(2, 2)]).encode()); + const decoded = await GIF.decode(await new GIF([new Frame(2, 2, 100, 0, 0, 'background'), new Frame(2, 2).fill(0xff0000ff)]).encode()); assert(decoded.length === 2, 'gif with disposal modes should roundtrip'); }