Patch Sequential

In this tutorial we will show how we can quickly perform patch processing using the features provided by the kornia.augmentation.PatchSequential API.

Install and get data

We install Kornia and some dependencies, and download a simple data sample

!pip install kornia
!pip install kornia-rs

import io

import requests


def download_image(url: str, filename: str = "") -> str:
    filename = url.split("/")[-1] if len(filename) == 0 else filename
    # Download
    bytesio = io.BytesIO(requests.get(url).content)
    # Save file
    with open(filename, "wb") as outfile:
        outfile.write(bytesio.getbuffer())

    return filename


url = "https://raw.githubusercontent.com/kornia/data/main/panda.jpg"
download_image(url)

'panda.jpg'

import kornia as K
import torch
from kornia.augmentation import ImageSequential, PatchSequential
from matplotlib import pyplot as plt

img_tensor = K.io.load_image("panda.jpg", K.io.ImageLoadType.RGB32)[None, ...]  # BxCxHxW
h, w = img_tensor.shape[2:]

plt.imshow(K.tensor_to_image(img_tensor))
plt.axis("off")
plt.show()

Patch Augmentation Sequential with patchwise_apply=True

patchwise_apply is a feature that used to define unique processing pipeline for each patch location. If patchwise_apply=True, the number of pipelines defined must be as same as the number of patches in an image.

pseq = PatchSequential(
    ImageSequential(
        K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.5),
        K.augmentation.RandomPerspective(0.2, p=0.5),
        K.augmentation.RandomSolarize(0.1, 0.1, p=0.5),
    ),
    K.augmentation.RandomAffine(15, [0.1, 0.1], [0.7, 1.2], [0.0, 20.0], p=0.5),
    K.augmentation.RandomPerspective(0.2, p=0.5),
    ImageSequential(
        K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.5),
        K.augmentation.RandomPerspective(0.2, p=0.5),
        K.augmentation.RandomSolarize(0.1, 0.1, p=0.5),
    ),
    K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.5),
    K.augmentation.RandomAffine(15, [0.1, 0.1], [0.7, 1.2], [0.0, 20.0], p=0.5),
    K.augmentation.RandomPerspective(0.2, p=0.5),
    K.augmentation.RandomSolarize(0.1, 0.1, p=0.5),
    K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.5),
    K.augmentation.RandomAffine(15, [0.1, 0.1], [0.7, 1.2], [0.0, 20.0], p=0.5),
    ImageSequential(
        K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.5),
        K.augmentation.RandomPerspective(0.2, p=0.5),
        K.augmentation.RandomSolarize(0.1, 0.1, p=0.5),
    ),
    K.augmentation.RandomSolarize(0.1, 0.1, p=0.5),
    K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.5),
    K.augmentation.RandomAffine(15, [0.1, 0.1], [0.7, 1.2], [0.0, 20.0], p=0.5),
    K.augmentation.RandomPerspective(0.2, p=0.5),
    K.augmentation.RandomSolarize(0.1, 0.1, p=0.5),
    patchwise_apply=True,
    same_on_batch=True,
)
out_tensor = pseq(img_tensor.repeat(2, 1, 1, 1))

plt.figure(figsize=(21, 9))
plt.imshow(K.tensor_to_image(torch.cat([out_tensor[0], out_tensor[1]], dim=2)))
plt.axis("off")
plt.show()

Patch Augmentation Sequential with patchwise_apply=False

If patchwise_apply=False, all the args will be combined and applied as one pipeline for each patch.

pseq = PatchSequential(
    K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.75),
    K.augmentation.RandomAffine(15, [0.1, 0.1], [0.7, 1.2], [0.0, 20.0], p=0.5),
    patchwise_apply=False,
    same_on_batch=False,
)
out_tensor = pseq(img_tensor.repeat(2, 1, 1, 1))

plt.figure(figsize=(21, 9))
plt.imshow(K.tensor_to_image(torch.cat([out_tensor[0], out_tensor[1]], dim=2)))
plt.axis("off")
plt.show()