In the field of image processing, the application of convolution matrices to images stands as a cornerstone technique. This post focuses on illustrating this process through an interactive visualization, utilizing the Quadrille filter method alongside custom display functions. The visualization aims to clarify how various masks affect individual pixels in a source image, making the intricate process of image convolution more comprehensible. By offering an interactive experience, it sheds light on the significant impact these matrices have on digital images, providing a deeper insight into a key image manipulation method.

The below interactive visualization demonstrates the process of applying image convolution to a source image, utilizing a specific mask to generate a target image. This visualization offers an interactive display where the pixel under the mouse, along with its neighboring pixels in the source image (displayed on the left), are highlighted to illustrate their role in computing the corresponding pixel in the target image (shown on the right). Beneath this, on the left side, the mask is displayed with its numerical values and a green shading that reflects the relative values of each cell. Correspondingly, on the right side, the visualization showcases the component-wise multiplication of the mask values with the neighboring pixels (from above left), culminating in the final value of the target image’s pixel (highlighted on the right). It is important to note that the dimensions of the neighboring pixels always match the kernel size of the mask.

(move mouse on source or target images and see how a convolution is applied)

code

Quadrille.textColor = 'magenta';
Quadrille.outline = 'lime';
let array;
let source, target;
let sources = {};
let targets = {};
let mask, stringMask;
let masks, maskSelector;
let image;
let resolution;
let alpha;
const size = 345;
const maskLength = 300;

const length = width => width / (2 ** resolution.value());

const toString = cell => cell instanceof p5.Color ? int(red(cell)).toString() : 
                         typeof cell === 'number' ? cell.toString() : '0';

const toNumber = cell => typeof cell === 'string' ? eval(cell) : Number(cell);

const removeRows = (quadrille, dimension) => {
  for (let i = 0; i < dimension; i++) {
    quadrille.delete(0);
    quadrille.delete(quadrille.height - 1);
  }
}

const crop = (quadrille, dimension) => {
  removeRows(quadrille, dimension);
  quadrille.transpose();
  removeRows(quadrille, dimension);
  quadrille.transpose();
}

function createNumberQuadrille(q) {
  const clone = q.clone();
  visitQuadrille(q, (row, col) => clone.fill(row, col, toNumber(q.read(row, col))));
  clone.dimension = (clone.width - 1) / 2;
  clone.min = clone.max = clone.read(0, 0);
  for (const cell of clone) {
    clone.min = (cell.value < clone.min) ? cell.value : clone.min;
    clone.max = (cell.value > clone.max) ? cell.value : clone.max;
  }
  clone.numberDisplay = numberDisplay.bind(clone);
  return clone;
}

function createStringQuadrille(q) {
  const clone = q.clone();
  visitQuadrille(q, (row, col) => clone.fill(row, col, toString(q.read(row, col))));
  return clone;
}

function numberDisplay({graphics, value, outline, outlineWeight, cellLength}) {
  const numberColor = 'lime';
  colorMode(RGB, 255);
  noStroke();
  fill(color(red(numberColor), green(numberColor), blue(numberColor),
             (this.min === this.max) ? 256 / this.size :
             map(value, this.min, this.max, 0, 255)));
  rect(0, 0, cellLength, cellLength);
}

function stringDisplay({graphics, value, cellLength, textColor, textZoom}) {
  textZoom = value.length === 1 ? 0.89 : textZoom;
  graphics.noStroke();
  graphics.fill(textColor);
  graphics.textSize(cellLength * textZoom / value.length);
  graphics.textAlign(CENTER, CENTER);
  graphics.text(value, 0, 0, cellLength, cellLength);
}

function preload() {
  image = loadImage('mandrill.png');
}

function setup() {
  createCanvas(size * 2, size + maskLength);
  image.filter(GRAY);
  resolution = createSelect();
  resolution.position(15, 15);
  resolution.option('low', 3);
  resolution.option('medium', 4);
  resolution.option('high', 5);
  resolution.selected(4);
  resolution.changed(() => {
    source = sources[resolution.value()];
    target = targets[resolution.value()];
  });
  masks = {
    'identity': [
      ['0', '0', '0'],
      ['0', '1', '0'],
      ['0', '0', '0']
    ],
    'ridge': [
      ['-2', '-1', '0'],
      ['-1', '1', '1'],
      ['0', '1', '2']
    ],
    'sharpen': [
      ['0', '-1', '0'],
      ['-1', '5', '-1'],
      ['0', '-1', '0']
    ],
    'box blur': [
      ['1/9', '1/9', '1/9'],
      ['1/9', '1/9', '1/9'],
      ['1/9', '1/9', '1/9']
    ],
    'gaussian blur': [
      ['1/16', '1/8', '1/16'],
      ['1/8', '1/4', '1/8'],
      ['1/16', '1/8', '1/16']
    ],
    'gaussian blur 5x5': [
      ['1/256', '4/256', '6/256', '4/256', '1/256'],
      ['4/256', '16/256', '24/256', '16/256', '4/256'],
      ['6/256', '24/256', '36/256', '24/256', '6/256'],
      ['4/256', '16/256', '24/256', '16/256', '4/256'],
      ['1/256', '4/256', '6/256', '4/256', '1/256']
    ],
    'unsharp 5x5': [
      ['-1/256', '-4/256', '-6/256', '-4/256', '-1/256'],
      ['-4/256', '-16/256', '-24/256', '-16/256', '-4/256'],
      ['-6/256', '-24/256', '476/256', '-24/256', '-6/256'],
      ['-4/256', '-16/256', '-24/256', '-16/256', '-4/256'],
      ['-1/256', '-4/256', '-6/256', '-4/256', '-1/256']
    ]
  }
  maskSelector = createSelect();
  maskSelector.position(15, height - 20);
  for (let mask in masks) {
    maskSelector.option(mask);
  }
  maskSelector.selected('ridge');
  maskSelector.changed(update);
  update();
}

function update() {
  stringMask = createQuadrille(masks[maskSelector.value()]);
  mask = createNumberQuadrille(stringMask);
  for (let pow = 3; pow <= 5; pow++) {
    let quadrille = createQuadrille(2 ** pow + 2 * mask.dimension, image, false);
    let clone = quadrille.clone();
    clone.filter(mask);
    crop(quadrille, mask.dimension);
    crop(clone, mask.dimension);
    sources[pow] = quadrille;
    targets[pow] = clone;
  }
  source = sources[resolution.value()];
  target = targets[resolution.value()];
}

function draw() {
  background(0);
  drawQuadrille(source, {cellLength: length(width / 2), outlineWeight: length(8),
                         outline: 'cyan', tileDisplay: 0});
  drawQuadrille(target, {x: width / 2, cellLength: length(width / 2),
                         outlineWeight: length(8), tileDisplay: 0});
  target.colOffset = width / (2 * length(width / 2));
  displayMask();
  if (mouseX > 0 && mouseX < width && mouseY > 0 && mouseY <= height / 2) {
    const col = mouseX < width / 2 ? source.mouseCol : target.mouseCol;
    const ring = displayRing(source, source.mouseRow, col);
    displayHint(ring);
    displayPixel(target, target.mouseRow, col);
  }
}

function displayMask(x = 0,
                     y = width / 2,
                     cellLength = maskLength / mask.width) {
  drawQuadrille(mask, {x, y, cellLength, numberDisplay: mask.numberDisplay,
                       outlineWeight: length(8)});
  drawQuadrille(stringMask, {x, y, cellLength, numberDisplay: 0, tileDisplay: 0});
}

function displayRing(quadrille = source,
                     row = quadrille.mouseRow,
                     col = quadrille.mouseCol,
                     cellLength = length(width / 2)) {
  const dimension = mask.dimension;
  const ring = quadrille.ring(row, col, dimension);
  let stringRing = createStringQuadrille(ring);
  row += (quadrille.rowOffset ?? 0) - dimension;
  col += (quadrille.colOffset ?? 0) - dimension;
  drawQuadrille(mask, {row, col, numberDisplay: mask.numberDisplay,
                       cellLength, outlineWeight: length(8)});
  drawQuadrille(stringRing, {row, col, tileDisplay: 0, cellLength, textZoom: 1.8});
  return stringRing;
}

function displayHint(stringRing,
                     x = width / 2,
                     y = width / 2,
                     cellLength = maskLength / mask.width) {
  const stringHint = createQuadrille(mask.width, mask.height);
  visitQuadrille(stringRing, (row, col) => 
    stringHint.fill(row, col, stringRing.read(row, col) + '*' +
                              stringMask.read(row, col)));
  const hint = createNumberQuadrille(stringHint);
  drawQuadrille(hint, {x, y, cellLength,
                       numberDisplay: hint.numberDisplay,
                       outlineWeight: length(8)});
  drawQuadrille(stringHint, {x, y, cellLength, numberDisplay: 0,
                             tileDisplay: 0, textZoom: 1.6});
}

function displayPixel(quadrille = target,
                      row = quadrille.mouseRow,
                      col = quadrille.mouseCol,
                      cellLength = length(width / 2)) {
  const pixel = quadrille.ring(row, col, 0);
  const stringPixel = createStringQuadrille(pixel);
  row += quadrille.rowOffset ?? 0;
  col += quadrille.colOffset ?? 0;
  drawQuadrille(pixel, {row, col, numberDisplay: mask.numberDisplay,
                        cellLength, outlineWeight: length(8)});
  drawQuadrille(stringPixel, {row, col, cellLength, tileDisplay: 0,
                              textZoom: 1.8, stringDisplay});
}

Setup

The setup function generates and stores several kernel masks, then invokes the update function which populates dictionaries of source and target quadrilles for the mandrill image at multiple resolutions, using createQuadrille(width, image, coherence) and filter, and initializes the source and target quadrilles according to the chosen resolution.

function update() {
  // create mask from the selector's value
  stringMask = createQuadrille(masks[maskSelector.value()]);
  // convert the string mask to a numerical format
  // (details of this method are covered in a subsequent section)
  mask = createNumberQuadrille(stringMask);
  // loop to create source and target quadrilles at various resolutions
  for (let pow = 3; pow <= 5; pow++) {
    // create the source quadrille at a specific resolution
    let quadrille = createQuadrille(2 ** pow + 2 * mask.dimension, image, false);
    // clone the source and apply the filter to get the target
    let clone = quadrille.clone();
    clone.filter(mask);
    // crop both quadrilles to remove the border
    crop(quadrille, mask.dimension);
    crop(clone, mask.dimension);
    // store the processed quadrilles
    sources[pow] = quadrille;
    targets[pow] = clone;
  }
  // set the current source and target based on the chosen resolution
  source = sources[resolution.value()];
  target = targets[resolution.value()];
}

Draw

The draw function employs the source and target quadrilles to display both the original and the filtered images at a selected resolution. Depending on the mouse position over these quadrilles, the function highlights the respective pixel in the target image and presents several interactive elements. These elements facilitate understanding the convolution process: they include the current mask, the ring of neighboring source pixels (matching the dimension of the mask), the hint showcasing component-wise multiplication between the neighbor and the mask, and the highlighted pixel in the target image.

function draw() {
  background(0);
  // display original image on the left
  drawQuadrille(source, {cellLength: length(width / 2), outlineWeight: length(8),
                         outline: 'cyan', tileDisplay: 0});
  // display filtered image on the right
  drawQuadrille(target, {x: width / 2, cellLength: length(width / 2),
                         outlineWeight: length(8), tileDisplay: 0});
  // set column offset for target image
  target.colOffset = width / (2 * length(width / 2));
  // visualize the current convolution mask
  displayMask();
  // process mouse interactions within the image area
  if (mouseX > 0 && mouseX < width && mouseY > 0 && mouseY <= height / 2) {
    const col = mouseX < width / 2 ? source.mouseCol : target.mouseCol;
    // display the ring of source pixels adjacent to the current mouse position
    const ring = displayRing(source, source.mouseRow, col);
    // show the component-wise multiplication results leading to the target pixel
    displayHint(ring);
    // highlight the resulting pixel in the target image
    displayPixel(target, target.mouseRow, col);
  }
}

Each interactive element in the visualization, including the mask, the ring, the hint, and the highlighted pixel, offers two distinct modes of visualization based on the cell values. One mode presents a shade of green, varying in intensity to indicate the cell’s value relative to the minimum and maximum within the element. The other mode explicitly displays the cell’s numerical value. To achieve these dual representations, the following helper functions are employed:

function createNumberQuadrille(q) {
  const clone = q.clone();
  // convert each cell in the quadrille to a number and store in the clone
  visitQuadrille(q, (row, col) => clone.fill(row, col, toNumber(q.read(row, col))));
  // set the dimension for the convolution process
  clone.dimension = (clone.width - 1) / 2;
  // initialize and find the minimum and maximum values in the quadrille
  clone.min = clone.max = clone.read(0, 0);
  for (const cell of clone) {
    clone.min = (cell.value < clone.min) ? cell.value : clone.min;
    clone.max = (cell.value > clone.max) ? cell.value : clone.max;
  }
  // bind the custom number display function to the clone for visualization
  clone.numberDisplay = numberDisplay.bind(clone);
  return clone;
}

function createStringQuadrille(q) {
  const clone = q.clone();
  // convert each cell in the quadrille to a string for display
  visitQuadrille(q, (row, col) => clone.fill(row, col, toString(q.read(row, col))));
  return clone;
}

Display Mask

The displayMask function visualizes the current convolution mask on the canvas. It draws both the numerical and string representations of the mask, enabling a clear understanding of the mask values in the convolution process.

function displayMask(x = 0,
                     y = width / 2,
                     cellLength = maskLength / mask.width) {
  // draw the numerical mask representation
  drawQuadrille(mask, {x, y, cellLength, numberDisplay: mask.numberDisplay,
                       outlineWeight: length(8)});
  // draw the string representation of the mask
  drawQuadrille(stringMask, {x, y, cellLength, numberDisplay: 0, tileDisplay: 0});
}

Display Ring

The displayRing function showcases the ring of pixels surrounding the mouse position in the source image. This ring is crucial for understanding how each pixel in the target image is computed.

function displayRing(quadrille = source,
                     row = quadrille.mouseRow,
                     col = quadrille.mouseCol,
                     cellLength = length(width / 2)) {
  // calculate the dimension of the mask and create the ring
  const dimension = mask.dimension;
  const ring = quadrille.ring(row, col, dimension);
  let stringRing = createStringQuadrille(ring);
  // adjust row and column positions
  row += (quadrille.rowOffset ?? 0) - dimension;
  col += (quadrille.colOffset ?? 0) - dimension;
  // display the numerical mask overlaying the ring
  drawQuadrille(mask, {row, col, numberDisplay: mask.numberDisplay,
                       cellLength, outlineWeight: length(8)});
  // display the string representation of the ring
  drawQuadrille(stringRing, {row, col, tileDisplay: 0, cellLength, textZoom: 1.8});
  return stringRing;
}

Display Hint

The displayHint function shows the component-wise multiplication between the mask and the ring. This visual cue helps in understanding how the convolution operation is performed to obtain the target pixel.

function displayHint(stringRing,
                     x = width / 2,
                     y = width / 2,
                     cellLength = maskLength / mask.width) {
  // create a quadrille for the hint visualization
  const stringHint = createQuadrille(mask.width, mask.height);
  // fill the hint with the multiplication of ring and mask values
  visitQuadrille(stringRing, (row, col) => 
    stringHint.fill(row, col, stringRing.read(row, col) + '*' +
                              stringMask.read(row, col)));
  // convert the hint to a numerical representation
  const hint = createNumberQuadrille(stringHint);
  // display the numerical hint
  drawQuadrille(hint, {x, y, cellLength,
                       numberDisplay: hint.numberDisplay,
                       outlineWeight: length(8)});
  // display the string representation of the hint
  drawQuadrille(stringHint, {x, y, cellLength, numberDisplay: 0,
                             tileDisplay: 0, textZoom: 1.6});
}

Display Pixel

The displayPixel function highlights the resulting pixel in the target image. It shows both the numerical and string values of the pixel, providing insight into the final output of the convolution process.

function displayPixel(quadrille = target,
                      row = quadrille.mouseRow,
                      col = quadrille.mouseCol,
                      cellLength = length(width / 2)) {
  // extract the pixel at the specified position
  const pixel = quadrille.ring(row, col, 0);
  const stringPixel = createStringQuadrille(pixel);
  // adjust row and column positions
  row += quadrille.rowOffset ?? 0;
  col += quadrille.colOffset ?? 0;
  // display the numerical representation of the pixel
  drawQuadrille(pixel, {row, col, numberDisplay: mask.numberDisplay,
                        cellLength, outlineWeight: length(8)});
  // display the string representation of the pixel
  drawQuadrille(stringPixel, {row, col, cellLength, tileDisplay: 0,
                              textZoom: 1.8, stringDisplay});
}

API references

References

Convolution matrix.

Setup#

Draw#

Display Mask#

Display Ring#

Display Hint#

Display Pixel#

API references#

References#