Random Mosaic Augmentation

In this tutorial we will show how we can quickly perform mosaicing using the features provided by the kornia.augmentation.RandomMosaic API. Mosaicing means taking several input images and combine their random crops into mosaic.

%%capture
!pip install kornia
!pip install kornia-rs

Install and get data

We install Kornia and some dependencies, and download a simple data sample

import io

import requests


def download_image(url: str, filename: str = "") -> str:
    filename = url.split("/")[-1] if len(filename) == 0 else filename
    # Download
    bytesio = io.BytesIO(requests.get(url).content)
    # Save file
    with open(filename, "wb") as outfile:
        outfile.write(bytesio.getbuffer())

    return filename


url = "https://raw.githubusercontent.com/kornia/data/main/panda.jpg"
download_image(url)

import kornia as K
import kornia.utils
import numpy as np
import torch
from matplotlib import pyplot as plt
from PIL import Image


def plot(img, box):
    img_vis = img.clone()
    img_vis = K.utils.draw_rectangle(img_vis, box, color=torch.tensor([255, 0, 0]))
    plt.imshow(K.tensor_to_image(img_vis))
    plt.show()

img1 = K.image_to_tensor(np.array(Image.open("panda.jpg").convert("RGB"))).float() / 255.0
img2 = K.augmentation.RandomEqualize(p=1.0, keepdim=True)(img1)
img3 = K.augmentation.RandomInvert(p=1.0, keepdim=True)(img1)
img4 = K.augmentation.RandomChannelShuffle(p=1.0, keepdim=True)(img1)

plt.figure(figsize=(21, 9))
plt.imshow(K.tensor_to_image(torch.cat([img1, img2, img3, img4], dim=-1)))
plt.show()

import kornia as K
import numpy as np
import torch
from kornia.augmentation import RandomMosaic
from PIL import Image

x = torch.cat(
    [
        K.geometry.resize(img1[None], (224, 224)),
        K.geometry.resize(img2[None], (224, 224)),
        K.geometry.resize(img3[None], (224, 224)),
        K.geometry.resize(img4[None], (224, 224)),
    ]
)

boxes = torch.tensor(
    [
        [
            [70.0, 5, 150, 100],  # head
            [60, 180, 175, 220],  # feet
        ]
    ]
).repeat(4, 1, 1)

aug = RandomMosaic(
    (224, 224),
    mosaic_grid=(2, 2),
    start_ratio_range=(0.3, 0.5),
    p=1.0,
    min_bbox_size=300,
    data_keys=["input", "bbox_xyxy"],
)

y, y1 = aug(x, boxes)

plot(y[:1], y1[:1])