diffsynth/extensions/FastBlend/runners/interpolation.py

from ..patch_match import PyramidPatchMatcher
import os
import numpy as np
from PIL import Image
from tqdm import tqdm


class InterpolationModeRunner:
    def __init__(self):
        pass

    def get_index_dict(self, index_style):
        index_dict = {}
        for i, index in enumerate(index_style):
            index_dict[index] = i
        return index_dict

    def get_weight(self, l, m, r):
        weight_l, weight_r = abs(m - r), abs(m - l)
        if weight_l + weight_r == 0:
            weight_l, weight_r = 0.5, 0.5
        else:
            weight_l, weight_r = weight_l / (weight_l + weight_r), weight_r / (weight_l + weight_r)
        return weight_l, weight_r

    def get_task_group(self, index_style, n):
        task_group = []
        index_style = sorted(index_style)
        # first frame
        if index_style[0]>0:
            tasks = []
            for m in range(index_style[0]):
                tasks.append((index_style[0], m, index_style[0]))
            task_group.append(tasks)
        # middle frames
        for l, r in zip(index_style[:-1], index_style[1:]):
            tasks = []
            for m in range(l, r):
                tasks.append((l, m, r))
            task_group.append(tasks)
        # last frame
        tasks = []
        for m in range(index_style[-1], n):
            tasks.append((index_style[-1], m, index_style[-1]))
        task_group.append(tasks)
        return task_group

    def run(self, frames_guide, frames_style, index_style, batch_size, ebsynth_config, save_path=None):
        patch_match_engine = PyramidPatchMatcher(
            image_height=frames_style[0].shape[0],
            image_width=frames_style[0].shape[1],
            channel=3,
            use_mean_target_style=False,
            use_pairwise_patch_error=True,
            **ebsynth_config
        )
        # task
        index_dict = self.get_index_dict(index_style)
        task_group = self.get_task_group(index_style, len(frames_guide))
        # run
        for tasks in task_group:
            index_start, index_end = min([i[1] for i in tasks]), max([i[1] for i in tasks])
            for batch_id in tqdm(range(0, len(tasks), batch_size), desc=f"Rendering frames {index_start}...{index_end}"):
                tasks_batch = tasks[batch_id: min(batch_id+batch_size, len(tasks))]
                source_guide, target_guide, source_style = [], [], []
                for l, m, r in tasks_batch:
                    # l -> m
                    source_guide.append(frames_guide[l])
                    target_guide.append(frames_guide[m])
                    source_style.append(frames_style[index_dict[l]])
                    # r -> m
                    source_guide.append(frames_guide[r])
                    target_guide.append(frames_guide[m])
                    source_style.append(frames_style[index_dict[r]])
                source_guide = np.stack(source_guide)
                target_guide = np.stack(target_guide)
                source_style = np.stack(source_style)
                _, target_style = patch_match_engine.estimate_nnf(source_guide, target_guide, source_style)
                if save_path is not None:
                    for frame_l, frame_r, (l, m, r) in zip(target_style[0::2], target_style[1::2], tasks_batch):
                        weight_l, weight_r = self.get_weight(l, m, r)
                        frame = frame_l * weight_l + frame_r * weight_r
                        frame = frame.clip(0, 255).astype("uint8")
                        Image.fromarray(frame).save(os.path.join(save_path, "%05d.png" % m))


class InterpolationModeSingleFrameRunner:
    def __init__(self):
        pass

    def run(self, frames_guide, frames_style, index_style, batch_size, ebsynth_config, save_path=None):
        # check input
        tracking_window_size = ebsynth_config["tracking_window_size"]
        if tracking_window_size * 2 >= batch_size:
            raise ValueError("batch_size should be larger than track_window_size * 2")
        frame_style = frames_style[0]
        frame_guide = frames_guide[index_style[0]]
        patch_match_engine = PyramidPatchMatcher(
            image_height=frame_style.shape[0],
            image_width=frame_style.shape[1],
            channel=3,
            **ebsynth_config
        )
        # run
        frame_id, n = 0, len(frames_guide)
        for i in tqdm(range(0, n, batch_size - tracking_window_size * 2), desc=f"Rendering frames 0...{n}"):
            if i + batch_size > n:
                l, r = max(n - batch_size, 0), n
            else:
                l, r = i, i + batch_size
            source_guide = np.stack([frame_guide] * (r-l))
            target_guide = np.stack([frames_guide[i] for i in range(l, r)])
            source_style = np.stack([frame_style] * (r-l))
            _, target_style = patch_match_engine.estimate_nnf(source_guide, target_guide, source_style)
            for i, frame in zip(range(l, r), target_style):
                if i==frame_id:
                    frame = frame.clip(0, 255).astype("uint8")
                    Image.fromarray(frame).save(os.path.join(save_path, "%05d.png" % frame_id))
                    frame_id += 1
                if r < n and r-frame_id <= tracking_window_size:
                    break