Initial snapshot: hand motion pipeline (Dyn-HaMR + dex-retargeting + SPIDER)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

third_party/hamer/third-party/ViTPose/tools/analysis/analyze_logs.py

@@ -0,0 +1,167 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import json
+from collections import defaultdict
+
+import matplotlib.pyplot as plt
+import numpy as np
+import seaborn as sns
+
+
+def cal_train_time(log_dicts, args):
+    for i, log_dict in enumerate(log_dicts):
+        print(f'{"-" * 5}Analyze train time of {args.json_logs[i]}{"-" * 5}')
+        all_times = []
+        for epoch in log_dict.keys():
+            if args.include_outliers:
+                all_times.append(log_dict[epoch]['time'])
+            else:
+                all_times.append(log_dict[epoch]['time'][1:])
+        all_times = np.array(all_times)
+        epoch_ave_time = all_times.mean(-1)
+        slowest_epoch = epoch_ave_time.argmax()
+        fastest_epoch = epoch_ave_time.argmin()
+        std_over_epoch = epoch_ave_time.std()
+        print(f'slowest epoch {slowest_epoch + 1}, '
+              f'average time is {epoch_ave_time[slowest_epoch]:.4f}')
+        print(f'fastest epoch {fastest_epoch + 1}, '
+              f'average time is {epoch_ave_time[fastest_epoch]:.4f}')
+        print(f'time std over epochs is {std_over_epoch:.4f}')
+        print(f'average iter time: {np.mean(all_times):.4f} s/iter')
+        print()
+
+
+def plot_curve(log_dicts, args):
+    if args.backend is not None:
+        plt.switch_backend(args.backend)
+    sns.set_style(args.style)
+    # if legend is None, use {filename}_{key} as legend
+    legend = args.legend
+    if legend is None:
+        legend = []
+        for json_log in args.json_logs:
+            for metric in args.keys:
+                legend.append(f'{json_log}_{metric}')
+    assert len(legend) == (len(args.json_logs) * len(args.keys))
+    metrics = args.keys
+
+    num_metrics = len(metrics)
+    for i, log_dict in enumerate(log_dicts):
+        epochs = list(log_dict.keys())
+        for j, metric in enumerate(metrics):
+            print(f'plot curve of {args.json_logs[i]}, metric is {metric}')
+            if metric not in log_dict[epochs[0]]:
+                raise KeyError(
+                    f'{args.json_logs[i]} does not contain metric {metric}')
+            xs = []
+            ys = []
+            num_iters_per_epoch = log_dict[epochs[0]]['iter'][-1]
+            for epoch in epochs:
+                iters = log_dict[epoch]['iter']
+                if log_dict[epoch]['mode'][-1] == 'val':
+                    iters = iters[:-1]
+                xs.append(np.array(iters) + (epoch - 1) * num_iters_per_epoch)
+                ys.append(np.array(log_dict[epoch][metric][:len(iters)]))
+            xs = np.concatenate(xs)
+            ys = np.concatenate(ys)
+            plt.xlabel('iter')
+            plt.plot(xs, ys, label=legend[i * num_metrics + j], linewidth=0.5)
+            plt.legend()
+        if args.title is not None:
+            plt.title(args.title)
+    if args.out is None:
+        plt.show()
+    else:
+        print(f'save curve to: {args.out}')
+        plt.savefig(args.out)
+        plt.cla()
+
+
+def add_plot_parser(subparsers):
+    parser_plt = subparsers.add_parser(
+        'plot_curve', help='parser for plotting curves')
+    parser_plt.add_argument(
+        'json_logs',
+        type=str,
+        nargs='+',
+        help='path of train log in json format')
+    parser_plt.add_argument(
+        '--keys',
+        type=str,
+        nargs='+',
+        default=['top1_acc'],
+        help='the metric that you want to plot')
+    parser_plt.add_argument('--title', type=str, help='title of figure')
+    parser_plt.add_argument(
+        '--legend',
+        type=str,
+        nargs='+',
+        default=None,
+        help='legend of each plot')
+    parser_plt.add_argument(
+        '--backend', type=str, default=None, help='backend of plt')
+    parser_plt.add_argument(
+        '--style', type=str, default='dark', help='style of plt')
+    parser_plt.add_argument('--out', type=str, default=None)
+
+
+def add_time_parser(subparsers):
+    parser_time = subparsers.add_parser(
+        'cal_train_time',
+        help='parser for computing the average time per training iteration')
+    parser_time.add_argument(
+        'json_logs',
+        type=str,
+        nargs='+',
+        help='path of train log in json format')
+    parser_time.add_argument(
+        '--include-outliers',
+        action='store_true',
+        help='include the first value of every epoch when computing '
+        'the average time')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Analyze Json Log')
+    # currently only support plot curve and calculate average train time
+    subparsers = parser.add_subparsers(dest='task', help='task parser')
+    add_plot_parser(subparsers)
+    add_time_parser(subparsers)
+    args = parser.parse_args()
+    return args
+
+
+def load_json_logs(json_logs):
+    # load and convert json_logs to log_dict, key is epoch, value is a sub dict
+    # keys of sub dict is different metrics, e.g. memory, top1_acc
+    # value of sub dict is a list of corresponding values of all iterations
+    log_dicts = [dict() for _ in json_logs]
+    for json_log, log_dict in zip(json_logs, log_dicts):
+        with open(json_log, 'r') as log_file:
+            for line in log_file:
+                log = json.loads(line.strip())
+                # skip lines without `epoch` field
+                if 'epoch' not in log:
+                    continue
+                epoch = log.pop('epoch')
+                if epoch not in log_dict:
+                    log_dict[epoch] = defaultdict(list)
+                for k, v in log.items():
+                    log_dict[epoch][k].append(v)
+    return log_dicts
+
+
+def main():
+    args = parse_args()
+
+    json_logs = args.json_logs
+    for json_log in json_logs:
+        assert json_log.endswith('.json')
+
+    log_dicts = load_json_logs(json_logs)
+
+    eval(args.task)(log_dicts, args)
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/analysis/benchmark_inference.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import time
+
+import torch
+from mmcv import Config
+from mmcv.cnn import fuse_conv_bn
+from mmcv.parallel import MMDataParallel
+from mmcv.runner.fp16_utils import wrap_fp16_model
+
+from mmpose.datasets import build_dataloader, build_dataset
+from mmpose.models import build_posenet
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='MMPose benchmark a recognizer')
+    parser.add_argument('config', help='test config file path')
+    parser.add_argument(
+        '--log-interval', default=10, help='interval of logging')
+    parser.add_argument(
+        '--fuse-conv-bn',
+        action='store_true',
+        help='Whether to fuse conv and bn, this will slightly increase'
+        'the inference speed')
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+
+    cfg = Config.fromfile(args.config)
+    # set cudnn_benchmark
+    if cfg.get('cudnn_benchmark', False):
+        torch.backends.cudnn.benchmark = True
+
+    # build the dataloader
+    dataset = build_dataset(cfg.data.val)
+    data_loader = build_dataloader(
+        dataset,
+        samples_per_gpu=1,
+        workers_per_gpu=cfg.data.workers_per_gpu,
+        dist=False,
+        shuffle=False)
+
+    # build the model and load checkpoint
+    model = build_posenet(cfg.model)
+    fp16_cfg = cfg.get('fp16', None)
+    if fp16_cfg is not None:
+        wrap_fp16_model(model)
+    if args.fuse_conv_bn:
+        model = fuse_conv_bn(model)
+    model = MMDataParallel(model, device_ids=[0])
+
+    # the first several iterations may be very slow so skip them
+    num_warmup = 5
+    pure_inf_time = 0
+
+    # benchmark with total batch and take the average
+    for i, data in enumerate(data_loader):
+
+        torch.cuda.synchronize()
+        start_time = time.perf_counter()
+        with torch.no_grad():
+            model(return_loss=False, **data)
+
+        torch.cuda.synchronize()
+        elapsed = time.perf_counter() - start_time
+
+        if i >= num_warmup:
+            pure_inf_time += elapsed
+            if (i + 1) % args.log_interval == 0:
+                its = (i + 1 - num_warmup) / pure_inf_time
+                print(f'Done item [{i + 1:<3}],  {its:.2f} items / s')
+    print(f'Overall average: {its:.2f} items / s')
+    print(f'Total time: {pure_inf_time:.2f} s')
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/analysis/benchmark_processing.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+"""This file is for benchmark data loading process. It can also be used to
+refresh the memcached cache. The command line to run this file is:
+
+$ python -m cProfile -o program.prof tools/analysis/benchmark_processing.py
+configs/task/method/[config filename]
+
+Note: When debugging, the `workers_per_gpu` in the config should be set to 0
+during benchmark.
+
+It use cProfile to record cpu running time and output to program.prof
+To visualize cProfile output program.prof, use Snakeviz and run:
+$ snakeviz program.prof
+"""
+import argparse
+
+import mmcv
+from mmcv import Config
+
+from mmpose import __version__
+from mmpose.datasets import build_dataloader, build_dataset
+from mmpose.utils import get_root_logger
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Benchmark data loading')
+    parser.add_argument('config', help='train config file path')
+    args = parser.parse_args()
+    cfg = Config.fromfile(args.config)
+
+    # init logger before other steps
+    logger = get_root_logger()
+    logger.info(f'MMPose Version: {__version__}')
+    logger.info(f'Config: {cfg.text}')
+
+    dataset = build_dataset(cfg.data.train)
+    data_loader = build_dataloader(
+        dataset,
+        samples_per_gpu=1,
+        workers_per_gpu=cfg.data.workers_per_gpu,
+        dist=False,
+        shuffle=False)
+
+    # Start progress bar after first 5 batches
+    prog_bar = mmcv.ProgressBar(
+        len(dataset) - 5 * cfg.data.samples_per_gpu, start=False)
+    for i, data in enumerate(data_loader):
+        if i == 5:
+            prog_bar.start()
+        for _ in data['img']:
+            if i < 5:
+                continue
+            prog_bar.update()
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/analysis/get_flops.py

@@ -0,0 +1,103 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+from functools import partial
+
+import torch
+
+from mmpose.apis.inference import init_pose_model
+
+try:
+    from mmcv.cnn import get_model_complexity_info
+except ImportError:
+    raise ImportError('Please upgrade mmcv to >0.6.2')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Train a recognizer')
+    parser.add_argument('config', help='train config file path')
+    parser.add_argument(
+        '--shape',
+        type=int,
+        nargs='+',
+        default=[256, 192],
+        help='input image size')
+    parser.add_argument(
+        '--input-constructor',
+        '-c',
+        type=str,
+        choices=['none', 'batch'],
+        default='none',
+        help='If specified, it takes a callable method that generates '
+        'input. Otherwise, it will generate a random tensor with '
+        'input shape to calculate FLOPs.')
+    parser.add_argument(
+        '--batch-size', '-b', type=int, default=1, help='input batch size')
+    parser.add_argument(
+        '--not-print-per-layer-stat',
+        '-n',
+        action='store_true',
+        help='Whether to print complexity information'
+        'for each layer in a model')
+    args = parser.parse_args()
+    return args
+
+
+def batch_constructor(flops_model, batch_size, input_shape):
+    """Generate a batch of tensors to the model."""
+    batch = {}
+
+    img = torch.ones(()).new_empty(
+        (batch_size, *input_shape),
+        dtype=next(flops_model.parameters()).dtype,
+        device=next(flops_model.parameters()).device)
+
+    batch['img'] = img
+    return batch
+
+
+def main():
+
+    args = parse_args()
+
+    if len(args.shape) == 1:
+        input_shape = (3, args.shape[0], args.shape[0])
+    elif len(args.shape) == 2:
+        input_shape = (3, ) + tuple(args.shape)
+    else:
+        raise ValueError('invalid input shape')
+
+    model = init_pose_model(args.config)
+
+    if args.input_constructor == 'batch':
+        input_constructor = partial(batch_constructor, model, args.batch_size)
+    else:
+        input_constructor = None
+
+    if args.input_constructor == 'batch':
+        input_constructor = partial(batch_constructor, model, args.batch_size)
+    else:
+        input_constructor = None
+
+    if hasattr(model, 'forward_dummy'):
+        model.forward = model.forward_dummy
+    else:
+        raise NotImplementedError(
+            'FLOPs counter is currently not currently supported with {}'.
+            format(model.__class__.__name__))
+
+    flops, params = get_model_complexity_info(
+        model,
+        input_shape,
+        input_constructor=input_constructor,
+        print_per_layer_stat=(not args.not_print_per_layer_stat))
+    split_line = '=' * 30
+    input_shape = (args.batch_size, ) + input_shape
+    print(f'{split_line}\nInput shape: {input_shape}\n'
+          f'Flops: {flops}\nParams: {params}\n{split_line}')
+    print('!!!Please be cautious if you use the results in papers. '
+          'You may need to check if all ops are supported and verify that the '
+          'flops computation is correct.')
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/analysis/print_config.py

@@ -0,0 +1,27 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+
+from mmcv import Config, DictAction
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Print the whole config')
+    parser.add_argument('config', help='config file path')
+    parser.add_argument(
+        '--options', nargs='+', action=DictAction, help='arguments in dict')
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    args = parse_args()
+
+    cfg = Config.fromfile(args.config)
+    if args.options is not None:
+        cfg.merge_from_dict(args.options)
+    print(f'Config:\n{cfg.pretty_text}')
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/analysis/speed_test.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import time
+
+import torch
+from mmcv import Config
+from mmcv.cnn import fuse_conv_bn
+from mmcv.parallel import MMDataParallel
+from mmcv.runner.fp16_utils import wrap_fp16_model
+
+from mmpose.datasets import build_dataloader, build_dataset
+from mmpose.models import build_posenet
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='MMPose benchmark a recognizer')
+    parser.add_argument('config', help='test config file path')
+    parser.add_argument('--bz', default=32, type=int, help='test config file path')
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+
+    cfg = Config.fromfile(args.config)
+
+    # Since we only care about the forward speed of the network
+    cfg.model.pretrained=None
+    cfg.model.test_cfg.flip_test=False
+    cfg.model.test_cfg.use_udp=False
+    cfg.model.test_cfg.post_process='none'
+
+    # set cudnn_benchmark
+    if cfg.get('cudnn_benchmark', False):
+        torch.backends.cudnn.benchmark = True
+
+    # build the dataloader
+    dataset = build_dataset(cfg.data.val)
+    data_loader = build_dataloader(
+        dataset,
+        samples_per_gpu=args.bz,
+        workers_per_gpu=cfg.data.workers_per_gpu,
+        dist=False,
+        shuffle=False)
+
+    # build the model and load checkpoint
+    model = build_posenet(cfg.model)
+    model = MMDataParallel(model, device_ids=[0])
+    model.eval()
+    
+    # get the example data
+    for i, data in enumerate(data_loader):
+        break
+
+    # the first several iterations may be very slow so skip them
+    num_warmup = 100
+    inference_times = 100
+
+    with torch.no_grad():
+        start_time = time.perf_counter()
+      
+        for i in range(num_warmup):
+            torch.cuda.synchronize()            
+            model(return_loss=False, **data)
+            torch.cuda.synchronize()
+        
+        elapsed = time.perf_counter() - start_time
+        print(f'warmup cost {elapsed} time')
+
+        start_time = time.perf_counter()
+        
+        for i in range(inference_times):
+            torch.cuda.synchronize()
+            model(return_loss=False, **data)
+            torch.cuda.synchronize()
+
+        elapsed = time.perf_counter() - start_time
+        fps = args.bz * inference_times / elapsed
+        print(f'the fps is {fps}')
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/dataset/h36m_to_coco.py

@@ -0,0 +1,165 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import os.path as osp
+from functools import wraps
+
+import mmcv
+import numpy as np
+from PIL import Image
+
+from mmpose.core import SimpleCamera
+
+
+def _keypoint_camera_to_world(keypoints,
+                              camera_params,
+                              image_name=None,
+                              dataset='Body3DH36MDataset'):
+    """Project 3D keypoints from the camera space to the world space.
+
+    Args:
+        keypoints (np.ndarray): 3D keypoints in shape [..., 3]
+        camera_params (dict): Parameters for all cameras.
+        image_name (str): The image name to specify the camera.
+        dataset (str): The dataset type, e.g., Body3DH36MDataset.
+    """
+    cam_key = None
+    if dataset == 'Body3DH36MDataset':
+        subj, rest = osp.basename(image_name).split('_', 1)
+        _, rest = rest.split('.', 1)
+        camera, rest = rest.split('_', 1)
+        cam_key = (subj, camera)
+    else:
+        raise NotImplementedError
+
+    camera = SimpleCamera(camera_params[cam_key])
+    keypoints_world = keypoints.copy()
+    keypoints_world[..., :3] = camera.camera_to_world(keypoints[..., :3])
+
+    return keypoints_world
+
+
+def _get_bbox_xywh(center, scale, w=200, h=200):
+    w = w * scale
+    h = h * scale
+    x = center[0] - w / 2
+    y = center[1] - h / 2
+    return [x, y, w, h]
+
+
+def mmcv_track_func(func):
+
+    @wraps(func)
+    def wrapped_func(args):
+        return func(*args)
+
+    return wrapped_func
+
+
+@mmcv_track_func
+def _get_img_info(img_idx, img_name, img_root):
+    try:
+        im = Image.open(osp.join(img_root, img_name))
+        w, h = im.size
+    except:  # noqa: E722
+        return None
+
+    img = {
+        'file_name': img_name,
+        'height': h,
+        'width': w,
+        'id': img_idx + 1,
+    }
+    return img
+
+
+@mmcv_track_func
+def _get_ann(idx, kpt_2d, kpt_3d, center, scale, imgname, camera_params):
+    bbox = _get_bbox_xywh(center, scale)
+    kpt_3d = _keypoint_camera_to_world(kpt_3d, camera_params, imgname)
+
+    ann = {
+        'id': idx + 1,
+        'category_id': 1,
+        'image_id': idx + 1,
+        'iscrowd': 0,
+        'bbox': bbox,
+        'area': bbox[2] * bbox[3],
+        'num_keypoints': 17,
+        'keypoints': kpt_2d.reshape(-1).tolist(),
+        'keypoints_3d': kpt_3d.reshape(-1).tolist()
+    }
+
+    return ann
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--ann-file', type=str, default='tests/data/h36m/test_h36m_body3d.npz')
+    parser.add_argument(
+        '--camera-param-file', type=str, default='tests/data/h36m/cameras.pkl')
+    parser.add_argument('--img-root', type=str, default='tests/data/h36m')
+    parser.add_argument(
+        '--out-file', type=str, default='tests/data/h36m/h36m_coco.json')
+    parser.add_argument('--full-img-name', action='store_true')
+
+    args = parser.parse_args()
+
+    h36m_data = np.load(args.ann_file)
+    h36m_camera_params = mmcv.load(args.camera_param_file)
+    h36m_coco = {}
+
+    # categories
+    h36m_cats = [{
+        'supercategory':
+        'person',
+        'id':
+        1,
+        'name':
+        'person',
+        'keypoints': [
+            'root (pelvis)', 'left_hip', 'left_knee', 'left_foot', 'right_hip',
+            'right_knee', 'right_foot', 'spine', 'thorax', 'neck_base', 'head',
+            'left_shoulder', 'left_elbow', 'left_wrist', 'right_shoulder',
+            'right_elbow', 'right_wrist'
+        ],
+        'skeleton': [[0, 1], [1, 2], [2, 3], [0, 4], [4, 5], [5, 6], [0, 7],
+                     [7, 8], [8, 9], [9, 10], [8, 11], [11, 12], [12, 13],
+                     [8, 14], [14, 15], [15, 16]],
+    }]
+
+    # images
+    imgnames = h36m_data['imgname']
+    if not args.full_img_name:
+        imgnames = [osp.basename(fn) for fn in imgnames]
+    tasks = [(idx, fn, args.img_root) for idx, fn in enumerate(imgnames)]
+
+    h36m_imgs = mmcv.track_parallel_progress(_get_img_info, tasks, nproc=12)
+
+    # annotations
+    kpts_2d = h36m_data['part']
+    kpts_3d = h36m_data['S']
+    centers = h36m_data['center']
+    scales = h36m_data['scale']
+    tasks = [(idx, ) + args + (h36m_camera_params, )
+             for idx, args in enumerate(
+                 zip(kpts_2d, kpts_3d, centers, scales, imgnames))]
+
+    h36m_anns = mmcv.track_parallel_progress(_get_ann, tasks, nproc=12)
+
+    # remove invalid data
+    h36m_imgs = [img for img in h36m_imgs if img is not None]
+    h36m_img_ids = set([img['id'] for img in h36m_imgs])
+    h36m_anns = [ann for ann in h36m_anns if ann['image_id'] in h36m_img_ids]
+
+    h36m_coco = {
+        'categories': h36m_cats,
+        'images': h36m_imgs,
+        'annotations': h36m_anns,
+    }
+
+    mmcv.dump(h36m_coco, args.out_file)
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/dataset/mat2json.py

@@ -0,0 +1,60 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import json
+import time
+
+from scipy.io import loadmat
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Converting the predicted .mat file to .json file.')
+    parser.add_argument('pred_mat_file', help='input prediction mat file.')
+    parser.add_argument(
+        'gt_json_file',
+        help='input ground-truth json file to get the image name. '
+        'Default: "data/mpii/mpii_val.json" ')
+    parser.add_argument('output_json_file', help='output converted json file.')
+    args = parser.parse_args()
+    return args
+
+
+def save_json(list_file, path):
+    with open(path, 'w') as f:
+        json.dump(list_file, f, indent=4)
+    return 0
+
+
+def convert_mat(pred_mat_file, gt_json_file, output_json_file):
+    res = loadmat(pred_mat_file)
+    preds = res['preds']
+    N = preds.shape[0]
+
+    with open(gt_json_file) as anno_file:
+        anno = json.load(anno_file)
+
+    assert len(anno) == N
+
+    instance = {}
+
+    for pred, ann in zip(preds, anno):
+        ann.pop('joints_vis')
+        ann['joints'] = pred.tolist()
+
+    instance['annotations'] = anno
+    instance['info'] = {}
+    instance['info']['description'] = 'Converted MPII prediction.'
+    instance['info']['year'] = time.strftime('%Y', time.localtime())
+    instance['info']['date_created'] = time.strftime('%Y/%m/%d',
+                                                     time.localtime())
+
+    save_json(instance, output_json_file)
+
+
+def main():
+    args = parse_args()
+    convert_mat(args.pred_mat_file, args.gt_json_file, args.output_json_file)
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/dataset/parse_animalpose_dataset.py

@@ -0,0 +1,436 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import json
+import os
+import re
+import time
+import warnings
+
+import cv2
+import numpy as np
+import xmltodict
+from xtcocotools.coco import COCO
+
+np.random.seed(0)
+
+
+def list_all_files(root_dir, ext='.xml'):
+    """List all files in the root directory and all its sub directories.
+
+    :param root_dir: root directory
+    :param ext: filename extension
+    :return: list of files
+    """
+    files = []
+    file_list = os.listdir(root_dir)
+    for i in range(0, len(file_list)):
+        path = os.path.join(root_dir, file_list[i])
+        if os.path.isdir(path):
+            files.extend(list_all_files(path))
+        if os.path.isfile(path):
+            if path.lower().endswith(ext):
+                files.append(path)
+    return files
+
+
+def get_anno_info():
+    keypoints_info = [
+        'L_Eye',
+        'R_Eye',
+        'L_EarBase',
+        'R_EarBase',
+        'Nose',
+        'Throat',
+        'TailBase',
+        'Withers',
+        'L_F_Elbow',
+        'R_F_Elbow',
+        'L_B_Elbow',
+        'R_B_Elbow',
+        'L_F_Knee',
+        'R_F_Knee',
+        'L_B_Knee',
+        'R_B_Knee',
+        'L_F_Paw',
+        'R_F_Paw',
+        'L_B_Paw',
+        'R_B_Paw',
+    ]
+    skeleton_info = [[1, 2], [1, 3], [2, 4], [1, 5], [2, 5], [5, 6], [6, 8],
+                     [7, 8], [6, 9], [9, 13], [13, 17], [6, 10], [10, 14],
+                     [14, 18], [7, 11], [11, 15], [15, 19], [7, 12], [12, 16],
+                     [16, 20]]
+    category_info = [{
+        'supercategory': 'animal',
+        'id': 1,
+        'name': 'animal',
+        'keypoints': keypoints_info,
+        'skeleton': skeleton_info
+    }]
+
+    return keypoints_info, skeleton_info, category_info
+
+
+def xml2coco_trainval(file_list, img_root, save_path, start_ann_id=0):
+    """Save annotations in coco-format.
+
+    :param file_list: list of data annotation files.
+    :param img_root: the root dir to load images.
+    :param save_path: the path to save transformed annotation file.
+    :param start_ann_id: the starting point to count the annotation id.
+    :param val_num: the number of annotated objects for validation.
+    """
+    images = []
+    annotations = []
+    img_ids = []
+    ann_ids = []
+
+    ann_id = start_ann_id
+
+    name2id = {
+        'L_Eye': 0,
+        'R_Eye': 1,
+        'L_EarBase': 2,
+        'R_EarBase': 3,
+        'Nose': 4,
+        'Throat': 5,
+        'TailBase': 6,
+        'Withers': 7,
+        'L_F_Elbow': 8,
+        'R_F_Elbow': 9,
+        'L_B_Elbow': 10,
+        'R_B_Elbow': 11,
+        'L_F_Knee': 12,
+        'R_F_Knee': 13,
+        'L_B_Knee': 14,
+        'R_B_Knee': 15,
+        'L_F_Paw': 16,
+        'R_F_Paw': 17,
+        'L_B_Paw': 18,
+        'R_B_Paw': 19
+    }
+    for file in file_list:
+        data_anno = xmltodict.parse(open(file).read())['annotation']
+
+        img_id = int(data_anno['image'].split('_')[0] +
+                     data_anno['image'].split('_')[1])
+
+        if img_id not in img_ids:
+            image_name = 'VOC2012/JPEGImages/' + data_anno['image'] + '.jpg'
+            img = cv2.imread(os.path.join(img_root, image_name))
+
+            image = {}
+            image['id'] = img_id
+            image['file_name'] = image_name
+            image['height'] = img.shape[0]
+            image['width'] = img.shape[1]
+
+            images.append(image)
+            img_ids.append(img_id)
+        else:
+            pass
+
+        keypoint_anno = data_anno['keypoints']['keypoint']
+        assert len(keypoint_anno) == 20
+
+        keypoints = np.zeros([20, 3], dtype=np.float32)
+
+        for kpt_anno in keypoint_anno:
+            keypoint_name = kpt_anno['@name']
+            keypoint_id = name2id[keypoint_name]
+
+            visibility = int(kpt_anno['@visible'])
+
+            if visibility == 0:
+                continue
+            else:
+                keypoints[keypoint_id, 0] = float(kpt_anno['@x'])
+                keypoints[keypoint_id, 1] = float(kpt_anno['@y'])
+                keypoints[keypoint_id, 2] = 2
+
+        anno = {}
+        anno['keypoints'] = keypoints.reshape(-1).tolist()
+        anno['image_id'] = img_id
+        anno['id'] = ann_id
+        anno['num_keypoints'] = int(sum(keypoints[:, 2] > 0))
+
+        visible_bounds = data_anno['visible_bounds']
+        anno['bbox'] = [
+            float(visible_bounds['@xmin']),
+            float(visible_bounds['@ymin']),
+            float(visible_bounds['@width']),
+            float(visible_bounds['@height'])
+        ]
+        anno['iscrowd'] = 0
+        anno['area'] = float(anno['bbox'][2] * anno['bbox'][3])
+        anno['category_id'] = 1
+
+        annotations.append(anno)
+        ann_ids.append(ann_id)
+        ann_id += 1
+
+    cocotype = {}
+
+    cocotype['info'] = {}
+    cocotype['info'][
+        'description'] = 'AnimalPose dataset Generated by MMPose Team'
+    cocotype['info']['version'] = '1.0'
+    cocotype['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype['info']['date_created'] = time.strftime('%Y/%m/%d',
+                                                     time.localtime())
+
+    cocotype['images'] = images
+    cocotype['annotations'] = annotations
+
+    keypoints_info, skeleton_info, category_info = get_anno_info()
+
+    cocotype['categories'] = category_info
+
+    os.makedirs(os.path.dirname(save_path), exist_ok=True)
+    json.dump(cocotype, open(save_path, 'w'), indent=4)
+    print('number of images:', len(img_ids))
+    print('number of annotations:', len(ann_ids))
+    print(f'done {save_path}')
+
+
+def xml2coco_test(file_list, img_root, save_path, start_ann_id=0):
+    """Save annotations in coco-format.
+
+    :param file_list: list of data annotation files.
+    :param img_root: the root dir to load images.
+    :param save_path: the path to save transformed annotation file.
+    :param start_ann_id: the starting point to count the annotation id.
+    """
+    images = []
+    annotations = []
+    img_ids = []
+    ann_ids = []
+
+    ann_id = start_ann_id
+
+    name2id = {
+        'L_eye': 0,
+        'R_eye': 1,
+        'L_ear': 2,
+        'R_ear': 3,
+        'Nose': 4,
+        'Throat': 5,
+        'Tail': 6,
+        'withers': 7,
+        'L_F_elbow': 8,
+        'R_F_elbow': 9,
+        'L_B_elbow': 10,
+        'R_B_elbow': 11,
+        'L_F_knee': 12,
+        'R_F_knee': 13,
+        'L_B_knee': 14,
+        'R_B_knee': 15,
+        'L_F_paw': 16,
+        'R_F_paw': 17,
+        'L_B_paw': 18,
+        'R_B_paw': 19
+    }
+
+    cat2id = {'cat': 1, 'cow': 2, 'dog': 3, 'horse': 4, 'sheep': 5}
+
+    for file in file_list:
+        data_anno = xmltodict.parse(open(file).read())['annotation']
+
+        category_id = cat2id[data_anno['category']]
+
+        img_id = category_id * 1000 + int(
+            re.findall(r'\d+', data_anno['image'])[0])
+
+        assert img_id not in img_ids
+
+        # prepare images
+        image_name = os.path.join('animalpose_image_part2',
+                                  data_anno['category'], data_anno['image'])
+        img = cv2.imread(os.path.join(img_root, image_name))
+
+        image = {}
+        image['id'] = img_id
+        image['file_name'] = image_name
+        image['height'] = img.shape[0]
+        image['width'] = img.shape[1]
+
+        images.append(image)
+        img_ids.append(img_id)
+
+        # prepare annotations
+        keypoint_anno = data_anno['keypoints']['keypoint']
+        keypoints = np.zeros([20, 3], dtype=np.float32)
+
+        for kpt_anno in keypoint_anno:
+            keypoint_name = kpt_anno['@name']
+            keypoint_id = name2id[keypoint_name]
+
+            visibility = int(kpt_anno['@visible'])
+
+            if visibility == 0:
+                continue
+            else:
+                keypoints[keypoint_id, 0] = float(kpt_anno['@x'])
+                keypoints[keypoint_id, 1] = float(kpt_anno['@y'])
+                keypoints[keypoint_id, 2] = 2
+
+        anno = {}
+        anno['keypoints'] = keypoints.reshape(-1).tolist()
+        anno['image_id'] = img_id
+        anno['id'] = ann_id
+        anno['num_keypoints'] = int(sum(keypoints[:, 2] > 0))
+
+        visible_bounds = data_anno['visible_bounds']
+        anno['bbox'] = [
+            float(visible_bounds['@xmin']),
+            float(visible_bounds['@xmax']
+                  ),  # typo in original xml: should be 'ymin'
+            float(visible_bounds['@width']),
+            float(visible_bounds['@height'])
+        ]
+        anno['iscrowd'] = 0
+        anno['area'] = float(anno['bbox'][2] * anno['bbox'][3])
+        anno['category_id'] = 1
+
+        annotations.append(anno)
+        ann_ids.append(ann_id)
+        ann_id += 1
+
+    cocotype = {}
+
+    cocotype['info'] = {}
+    cocotype['info'][
+        'description'] = 'AnimalPose dataset Generated by MMPose Team'
+    cocotype['info']['version'] = '1.0'
+    cocotype['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype['info']['date_created'] = time.strftime('%Y/%m/%d',
+                                                     time.localtime())
+
+    cocotype['images'] = images
+    cocotype['annotations'] = annotations
+
+    keypoints_info, skeleton_info, category_info = get_anno_info()
+
+    cocotype['categories'] = category_info
+
+    os.makedirs(os.path.dirname(save_path), exist_ok=True)
+    json.dump(cocotype, open(save_path, 'w'), indent=4)
+    print('=========================================================')
+    print('number of images:', len(img_ids))
+    print('number of annotations:', len(ann_ids))
+    print(f'done {save_path}')
+
+
+def split_train_val(work_dir, trainval_file, train_file, val_file,
+                    val_ann_num):
+    """Split train-val json file into training and validation files.
+
+    :param work_dir: path to load train-val json file, and save split files.
+    :param trainval_file: The input json file combining both train and val.
+    :param trainval_file: The output json file for training.
+    :param trainval_file: The output json file for validation.
+    :param val_ann_num: the number of validation annotations.
+    """
+
+    coco = COCO(os.path.join(work_dir, trainval_file))
+
+    img_list = list(coco.imgs.keys())
+    np.random.shuffle(img_list)
+
+    count = 0
+
+    images_train = []
+    images_val = []
+    annotations_train = []
+    annotations_val = []
+
+    for img_id in img_list:
+        ann_ids = coco.getAnnIds(img_id)
+
+        if count + len(ann_ids) <= val_ann_num:
+            # for validation
+            count += len(ann_ids)
+            images_val.append(coco.imgs[img_id])
+            for ann_id in ann_ids:
+                annotations_val.append(coco.anns[ann_id])
+
+        else:
+            images_train.append(coco.imgs[img_id])
+            for ann_id in ann_ids:
+                annotations_train.append(coco.anns[ann_id])
+
+    if count == val_ann_num:
+        print(f'We have found {count} annotations for validation.')
+    else:
+        warnings.warn(
+            f'We only found {count} annotations, instead of {val_ann_num}.')
+
+    cocotype_train = {}
+    cocotype_val = {}
+
+    keypoints_info, skeleton_info, category_info = get_anno_info()
+
+    cocotype_train['info'] = {}
+    cocotype_train['info'][
+        'description'] = 'AnimalPose dataset Generated by MMPose Team'
+    cocotype_train['info']['version'] = '1.0'
+    cocotype_train['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype_train['info']['date_created'] = time.strftime(
+        '%Y/%m/%d', time.localtime())
+    cocotype_train['images'] = images_train
+    cocotype_train['annotations'] = annotations_train
+    cocotype_train['categories'] = category_info
+
+    json.dump(
+        cocotype_train,
+        open(os.path.join(work_dir, train_file), 'w'),
+        indent=4)
+    print('=========================================================')
+    print('number of images:', len(images_train))
+    print('number of annotations:', len(annotations_train))
+    print(f'done {train_file}')
+
+    cocotype_val['info'] = {}
+    cocotype_val['info'][
+        'description'] = 'AnimalPose dataset Generated by MMPose Team'
+    cocotype_val['info']['version'] = '1.0'
+    cocotype_val['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype_val['info']['date_created'] = time.strftime(
+        '%Y/%m/%d', time.localtime())
+    cocotype_val['images'] = images_val
+    cocotype_val['annotations'] = annotations_val
+    cocotype_val['categories'] = category_info
+
+    json.dump(
+        cocotype_val, open(os.path.join(work_dir, val_file), 'w'), indent=4)
+    print('=========================================================')
+    print('number of images:', len(images_val))
+    print('number of annotations:', len(annotations_val))
+    print(f'done {val_file}')
+
+
+dataset_dir = 'data/animalpose/'
+
+# We choose the images from PascalVOC for train + val
+# In total, train+val: 3608 images, 5117 annotations
+xml2coco_trainval(
+    list_all_files(os.path.join(dataset_dir, 'PASCAL2011_animal_annotation')),
+    dataset_dir,
+    os.path.join(dataset_dir, 'annotations', 'animalpose_trainval.json'),
+    start_ann_id=1000000)
+
+# train: 2798 images, 4000 annotations
+# val: 810 images, 1117 annotations
+split_train_val(
+    os.path.join(dataset_dir, 'annotations'),
+    'animalpose_trainval.json',
+    'animalpose_train.json',
+    'animalpose_val.json',
+    val_ann_num=1117)
+
+# We choose the remaining 1000 images for test
+# 1000 images, 1000 annotations
+xml2coco_test(
+    list_all_files(os.path.join(dataset_dir, 'animalpose_anno2')),
+    dataset_dir,
+    os.path.join(dataset_dir, 'annotations', 'animalpose_test.json'),
+    start_ann_id=0)

third_party/hamer/third-party/ViTPose/tools/dataset/parse_cofw_dataset.py

@@ -0,0 +1,97 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import json
+import os
+import time
+
+import cv2
+import h5py
+import numpy as np
+
+mat_files = ['COFW_train_color.mat', 'COFW_test_color.mat']
+dataset_dir = 'data/cofw/'
+
+image_root = os.path.join(dataset_dir, 'images/')
+annotation_root = os.path.join(dataset_dir, 'annotations/')
+
+os.makedirs(image_root, exist_ok=True)
+os.makedirs(annotation_root, exist_ok=True)
+
+cnt = 0
+for mat_file in mat_files:
+    mat = h5py.File(os.path.join(dataset_dir, mat_file), 'r')
+
+    if 'train' in mat_file:
+        imgs = mat['IsTr']
+        pts = mat['phisTr']
+        bboxes = mat['bboxesTr']
+        is_train = True
+        json_file = 'cofw_train.json'
+    else:
+        imgs = mat['IsT']
+        pts = mat['phisT']
+        bboxes = mat['bboxesT']
+        is_train = False
+        json_file = 'cofw_test.json'
+
+    images = []
+    annotations = []
+
+    num = pts.shape[1]
+    for idx in range(0, num):
+        cnt += 1
+        img = np.array(mat[imgs[0, idx]]).transpose()
+        keypoints = pts[:, idx].reshape(3, -1).transpose()
+        # 2 for valid and 1 for occlusion
+        keypoints[:, 2] = 2 - keypoints[:, 2]
+        # matlab 1-index to python 0-index
+        keypoints[:, :2] -= 1
+        bbox = bboxes[:, idx]
+
+        # check nonnegativity
+        bbox[bbox < 0] = 0
+        keypoints[keypoints < 0] = 0
+
+        image = {}
+        image['id'] = cnt
+        image['file_name'] = f'{str(cnt).zfill(6)}.jpg'
+        image['height'] = img.shape[0]
+        image['width'] = img.shape[1]
+        cv2.imwrite(
+            os.path.join(image_root, image['file_name']),
+            cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
+        images.append(image)
+
+        anno = {}
+        anno['keypoints'] = keypoints.reshape(-1).tolist()
+        anno['image_id'] = cnt
+        anno['id'] = cnt
+        anno['num_keypoints'] = len(keypoints)  # all keypoints are labelled
+        anno['bbox'] = bbox.tolist()
+        anno['iscrowd'] = 0
+        anno['area'] = anno['bbox'][2] * anno['bbox'][3]
+        anno['category_id'] = 1
+
+        annotations.append(anno)
+
+    cocotype = {}
+
+    cocotype['info'] = {}
+    cocotype['info']['description'] = 'COFW Generated by MMPose Team'
+    cocotype['info']['version'] = '1.0'
+    cocotype['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype['info']['date_created'] = time.strftime('%Y/%m/%d',
+                                                     time.localtime())
+
+    cocotype['images'] = images
+    cocotype['annotations'] = annotations
+    cocotype['categories'] = [{
+        'supercategory': 'person',
+        'id': 1,
+        'name': 'face',
+        'keypoints': [],
+        'skeleton': []
+    }]
+
+    ann_path = os.path.join(annotation_root, json_file)
+    json.dump(cocotype, open(ann_path, 'w'))
+    print(f'done {ann_path}')

third_party/hamer/third-party/ViTPose/tools/dataset/parse_deepposekit_dataset.py

@@ -0,0 +1,180 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import json
+import os
+import time
+
+import cv2
+import h5py
+import numpy as np
+
+np.random.seed(0)
+
+
+def save_coco_anno(keypoints_all,
+                   annotated_all,
+                   imgs_all,
+                   keypoints_info,
+                   skeleton_info,
+                   dataset,
+                   img_root,
+                   save_path,
+                   start_img_id=0,
+                   start_ann_id=0):
+    """Save annotations in coco-format.
+
+    :param keypoints_all: keypoint annotations.
+    :param annotated_all: images annotated or not.
+    :param imgs_all: the array of images.
+    :param keypoints_info: information about keypoint name.
+    :param skeleton_info: information about skeleton connection.
+    :param dataset: information about dataset name.
+    :param img_root: the path to save images.
+    :param save_path: the path to save transformed annotation file.
+    :param start_img_id: the starting point to count the image id.
+    :param start_ann_id: the starting point to count the annotation id.
+    """
+    images = []
+    annotations = []
+
+    img_id = start_img_id
+    ann_id = start_ann_id
+
+    num_annotations, keypoints_num, _ = keypoints_all.shape
+
+    for i in range(num_annotations):
+        img = imgs_all[i]
+        keypoints = np.concatenate(
+            [keypoints_all[i], annotated_all[i][:, None] * 2], axis=1)
+
+        min_x, min_y = np.min(keypoints[keypoints[:, 2] > 0, :2], axis=0)
+        max_x, max_y = np.max(keypoints[keypoints[:, 2] > 0, :2], axis=0)
+
+        anno = {}
+        anno['keypoints'] = keypoints.reshape(-1).tolist()
+        anno['image_id'] = img_id
+        anno['id'] = ann_id
+        anno['num_keypoints'] = int(sum(keypoints[:, 2] > 0))
+        anno['bbox'] = [
+            float(min_x),
+            float(min_y),
+            float(max_x - min_x + 1),
+            float(max_y - min_y + 1)
+        ]
+        anno['iscrowd'] = 0
+        anno['area'] = anno['bbox'][2] * anno['bbox'][3]
+        anno['category_id'] = 1
+
+        annotations.append(anno)
+        ann_id += 1
+
+        image = {}
+        image['id'] = img_id
+        image['file_name'] = f'{img_id}.jpg'
+        image['height'] = img.shape[0]
+        image['width'] = img.shape[1]
+
+        images.append(image)
+        img_id += 1
+
+        cv2.imwrite(os.path.join(img_root, image['file_name']), img)
+
+    skeleton = np.concatenate(
+        [np.arange(keypoints_num)[:, None], skeleton_info[:, 0][:, None]],
+        axis=1) + 1
+    skeleton = skeleton[skeleton.min(axis=1) > 0]
+
+    cocotype = {}
+
+    cocotype['info'] = {}
+    cocotype['info'][
+        'description'] = 'DeepPoseKit-Data Generated by MMPose Team'
+    cocotype['info']['version'] = '1.0'
+    cocotype['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype['info']['date_created'] = time.strftime('%Y/%m/%d',
+                                                     time.localtime())
+
+    cocotype['images'] = images
+    cocotype['annotations'] = annotations
+    cocotype['categories'] = [{
+        'supercategory': 'animal',
+        'id': 1,
+        'name': dataset,
+        'keypoints': keypoints_info,
+        'skeleton': skeleton.tolist()
+    }]
+
+    os.makedirs(os.path.dirname(save_path), exist_ok=True)
+    json.dump(cocotype, open(save_path, 'w'), indent=4)
+    print('number of images:', img_id)
+    print('number of annotations:', ann_id)
+    print(f'done {save_path}')
+
+
+for dataset in ['fly', 'locust', 'zebra']:
+    keypoints_info = []
+    if dataset == 'fly':
+        keypoints_info = [
+            'head', 'eyeL', 'eyeR', 'neck', 'thorax', 'abdomen', 'forelegR1',
+            'forelegR2', 'forelegR3', 'forelegR4', 'midlegR1', 'midlegR2',
+            'midlegR3', 'midlegR4', 'hindlegR1', 'hindlegR2', 'hindlegR3',
+            'hindlegR4', 'forelegL1', 'forelegL2', 'forelegL3', 'forelegL4',
+            'midlegL1', 'midlegL2', 'midlegL3', 'midlegL4', 'hindlegL1',
+            'hindlegL2', 'hindlegL3', 'hindlegL4', 'wingL', 'wingR'
+        ]
+    elif dataset == 'locust':
+        keypoints_info = [
+            'head', 'neck', 'thorax', 'abdomen1', 'abdomen2', 'anttipL',
+            'antbaseL', 'eyeL', 'forelegL1', 'forelegL2', 'forelegL3',
+            'forelegL4', 'midlegL1', 'midlegL2', 'midlegL3', 'midlegL4',
+            'hindlegL1', 'hindlegL2', 'hindlegL3', 'hindlegL4', 'anttipR',
+            'antbaseR', 'eyeR', 'forelegR1', 'forelegR2', 'forelegR3',
+            'forelegR4', 'midlegR1', 'midlegR2', 'midlegR3', 'midlegR4',
+            'hindlegR1', 'hindlegR2', 'hindlegR3', 'hindlegR4'
+        ]
+    elif dataset == 'zebra':
+        keypoints_info = [
+            'snout', 'head', 'neck', 'forelegL1', 'forelegR1', 'hindlegL1',
+            'hindlegR1', 'tailbase', 'tailtip'
+        ]
+    else:
+        NotImplementedError()
+
+    dataset_dir = f'data/DeepPoseKit-Data/datasets/{dataset}'
+
+    with h5py.File(
+            os.path.join(dataset_dir, 'annotation_data_release.h5'), 'r') as f:
+        # List all groups
+        annotations = np.array(f['annotations'])
+        annotated = np.array(f['annotated'])
+        images = np.array(f['images'])
+        skeleton_info = np.array(f['skeleton'])
+
+        annotation_num, kpt_num, _ = annotations.shape
+
+        data_list = np.arange(0, annotation_num)
+        np.random.shuffle(data_list)
+
+        val_data_num = annotation_num // 10
+        train_data_num = annotation_num - val_data_num
+
+        train_list = data_list[0:train_data_num]
+        val_list = data_list[train_data_num:]
+
+        img_root = os.path.join(dataset_dir, 'images')
+        os.makedirs(img_root, exist_ok=True)
+
+        save_coco_anno(
+            annotations[train_list], annotated[train_list], images[train_list],
+            keypoints_info, skeleton_info, dataset, img_root,
+            os.path.join(dataset_dir, 'annotations', f'{dataset}_train.json'))
+        save_coco_anno(
+            annotations[val_list],
+            annotated[val_list],
+            images[val_list],
+            keypoints_info,
+            skeleton_info,
+            dataset,
+            img_root,
+            os.path.join(dataset_dir, 'annotations', f'{dataset}_test.json'),
+            start_img_id=train_data_num,
+            start_ann_id=train_data_num)

third_party/hamer/third-party/ViTPose/tools/dataset/parse_macaquepose_dataset.py

@@ -0,0 +1,182 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import csv
+import json
+import os
+import time
+
+import cv2
+import numpy as np
+
+np.random.seed(0)
+
+
+def get_poly_area(x, y):
+    """Calculate area of polygon given (x,y) coordinates (Shoelace formula)
+
+    :param x: np.ndarray(N, )
+    :param y: np.ndarray(N, )
+    :return: area
+    """
+    return float(0.5 *
+                 np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))))
+
+
+def get_seg_area(segmentations):
+    area = 0
+    for segmentation in segmentations:
+        area += get_poly_area(segmentation[:, 0], segmentation[:, 1])
+    return area
+
+
+def save_coco_anno(data_annotation,
+                   img_root,
+                   save_path,
+                   start_img_id=0,
+                   start_ann_id=0,
+                   kpt_num=17):
+    """Save annotations in coco-format.
+
+    :param data_annotation: list of data annotation.
+    :param img_root: the root dir to load images.
+    :param save_path: the path to save transformed annotation file.
+    :param start_img_id: the starting point to count the image id.
+    :param start_ann_id: the starting point to count the annotation id.
+    :param kpt_num: the number of keypoint.
+    """
+    images = []
+    annotations = []
+
+    img_id = start_img_id
+    ann_id = start_ann_id
+
+    for i in range(0, len(data_annotation)):
+        data_anno = data_annotation[i]
+        image_name = data_anno[0]
+
+        img = cv2.imread(os.path.join(img_root, image_name))
+
+        kp_string = data_anno[1]
+        kps = json.loads(kp_string)
+
+        seg_string = data_anno[2]
+        segs = json.loads(seg_string)
+
+        for kp, seg in zip(kps, segs):
+            keypoints = np.zeros([kpt_num, 3])
+            for ind, p in enumerate(kp):
+                if p['position'] is None:
+                    continue
+                else:
+                    keypoints[ind, 0] = p['position'][0]
+                    keypoints[ind, 1] = p['position'][1]
+                    keypoints[ind, 2] = 2
+
+            segmentations = []
+
+            max_x = -1
+            max_y = -1
+            min_x = 999999
+            min_y = 999999
+            for segm in seg:
+                if len(segm['segment']) == 0:
+                    continue
+
+                segmentation = np.array(segm['segment'])
+                segmentations.append(segmentation)
+
+                _max_x, _max_y = segmentation.max(0)
+                _min_x, _min_y = segmentation.min(0)
+
+                max_x = max(max_x, _max_x)
+                max_y = max(max_y, _max_y)
+                min_x = min(min_x, _min_x)
+                min_y = min(min_y, _min_y)
+
+            anno = {}
+            anno['keypoints'] = keypoints.reshape(-1).tolist()
+            anno['image_id'] = img_id
+            anno['id'] = ann_id
+            anno['num_keypoints'] = int(sum(keypoints[:, 2] > 0))
+            anno['bbox'] = [
+                float(min_x),
+                float(min_y),
+                float(max_x - min_x + 1),
+                float(max_y - min_y + 1)
+            ]
+            anno['iscrowd'] = 0
+            anno['area'] = get_seg_area(segmentations)
+            anno['category_id'] = 1
+            anno['segmentation'] = [
+                seg.reshape(-1).tolist() for seg in segmentations
+            ]
+
+            annotations.append(anno)
+            ann_id += 1
+
+        image = {}
+        image['id'] = img_id
+        image['file_name'] = image_name
+        image['height'] = img.shape[0]
+        image['width'] = img.shape[1]
+
+        images.append(image)
+        img_id += 1
+
+    cocotype = {}
+
+    cocotype['info'] = {}
+    cocotype['info']['description'] = 'MacaquePose Generated by MMPose Team'
+    cocotype['info']['version'] = '1.0'
+    cocotype['info']['year'] = time.strftime('%Y', time.localtime())
+    cocotype['info']['date_created'] = time.strftime('%Y/%m/%d',
+                                                     time.localtime())
+
+    cocotype['images'] = images
+    cocotype['annotations'] = annotations
+    cocotype['categories'] = [{
+        'supercategory':
+        'animal',
+        'id':
+        1,
+        'name':
+        'macaque',
+        'keypoints': [
+            'nose', 'left_eye', 'right_eye', 'left_ear', 'right_ear',
+            'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow',
+            'left_wrist', 'right_wrist', 'left_hip', 'right_hip', 'left_knee',
+            'right_knee', 'left_ankle', 'right_ankle'
+        ],
+        'skeleton': [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12],
+                     [7, 13], [6, 7], [6, 8], [7, 9], [8, 10], [9, 11], [2, 3],
+                     [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]]
+    }]
+
+    os.makedirs(os.path.dirname(save_path), exist_ok=True)
+    json.dump(cocotype, open(save_path, 'w'), indent=4)
+    print('number of images:', img_id)
+    print('number of annotations:', ann_id)
+    print(f'done {save_path}')
+
+
+dataset_dir = '/data/macaque/'
+with open(os.path.join(dataset_dir, 'annotations.csv'), 'r') as fp:
+    data_annotation_all = list(csv.reader(fp, delimiter=','))[1:]
+
+np.random.shuffle(data_annotation_all)
+
+data_annotation_train = data_annotation_all[0:12500]
+data_annotation_val = data_annotation_all[12500:]
+
+img_root = os.path.join(dataset_dir, 'images')
+save_coco_anno(
+    data_annotation_train,
+    img_root,
+    os.path.join(dataset_dir, 'annotations', 'macaque_train.json'),
+    kpt_num=17)
+save_coco_anno(
+    data_annotation_val,
+    img_root,
+    os.path.join(dataset_dir, 'annotations', 'macaque_test.json'),
+    start_img_id=12500,
+    start_ann_id=15672,
+    kpt_num=17)

third_party/hamer/third-party/ViTPose/tools/dataset/preprocess_h36m.py

@@ -0,0 +1,417 @@
+# -----------------------------------------------------------------------------
+# Adapted from https://github.com/anibali/h36m-fetch
+# Original license: Copyright (c) Aiden Nibali, under the Apache License.
+# -----------------------------------------------------------------------------
+
+import argparse
+import os
+import pickle
+import tarfile
+import xml.etree.ElementTree as ET
+from os.path import join
+
+import cv2
+import numpy as np
+from spacepy import pycdf
+
+
+class PreprocessH36m:
+    """Preprocess Human3.6M dataset.
+
+    Args:
+        metadata (str): Path to metadata.xml.
+        original_dir (str): Directory of the original dataset with all files
+            compressed. Specifically, .tgz files belonging to subject 1
+            should be placed under the subdirectory 's1'.
+        extracted_dir (str): Directory of the extracted files. If not given, it
+            will be placed under the same parent directory as original_dir.
+        processed_der (str): Directory of the processed files. If not given, it
+            will be placed under the same parent directory as original_dir.
+        sample_rate (int): Downsample FPS to `1 / sample_rate`. Default: 5.
+    """
+
+    def __init__(self,
+                 metadata,
+                 original_dir,
+                 extracted_dir=None,
+                 processed_dir=None,
+                 sample_rate=5):
+        self.metadata = metadata
+        self.original_dir = original_dir
+        self.sample_rate = sample_rate
+
+        if extracted_dir is None:
+            self.extracted_dir = join(
+                os.path.dirname(os.path.abspath(self.original_dir)),
+                'extracted')
+        else:
+            self.extracted_dir = extracted_dir
+
+        if processed_dir is None:
+            self.processed_dir = join(
+                os.path.dirname(os.path.abspath(self.original_dir)),
+                'processed')
+        else:
+            self.processed_dir = processed_dir
+
+        self.subjects = []
+        self.sequence_mappings = {}
+        self.action_names = {}
+        self.camera_ids = []
+        self._load_metadata()
+
+        self.subjects_annot = ['S1', 'S5', 'S6', 'S7', 'S8', 'S9', 'S11']
+        self.subjects_splits = {
+            'train': ['S1', 'S5', 'S6', 'S7', 'S8'],
+            'test': ['S9', 'S11']
+        }
+        self.extract_files = ['Videos', 'D2_Positions', 'D3_Positions_mono']
+        self.movable_joints = [
+            0, 1, 2, 3, 6, 7, 8, 12, 13, 14, 15, 17, 18, 19, 25, 26, 27
+        ]
+        self.scale_factor = 1.2
+        self.image_sizes = {
+            '54138969': {
+                'width': 1000,
+                'height': 1002
+            },
+            '55011271': {
+                'width': 1000,
+                'height': 1000
+            },
+            '58860488': {
+                'width': 1000,
+                'height': 1000
+            },
+            '60457274': {
+                'width': 1000,
+                'height': 1002
+            }
+        }
+
+    def extract_tgz(self):
+        """Extract files from self.extrct_files."""
+        os.makedirs(self.extracted_dir, exist_ok=True)
+        for subject in self.subjects_annot:
+            cur_dir = join(self.original_dir, subject.lower())
+            for file in self.extract_files:
+                filename = join(cur_dir, file + '.tgz')
+                print(f'Extracting {filename} ...')
+                with tarfile.open(filename) as tar:
+                    tar.extractall(self.extracted_dir)
+        print('Extraction done.\n')
+
+    def generate_cameras_file(self):
+        """Generate cameras.pkl which contains camera parameters for 11
+        subjects each with 4 cameras."""
+        cameras = {}
+        for subject in range(1, 12):
+            for camera in range(4):
+                key = (f'S{subject}', self.camera_ids[camera])
+                cameras[key] = self._get_camera_params(camera, subject)
+
+        out_file = join(self.processed_dir, 'annotation_body3d', 'cameras.pkl')
+        with open(out_file, 'wb') as fout:
+            pickle.dump(cameras, fout)
+        print(f'Camera parameters have been written to "{out_file}".\n')
+
+    def generate_annotations(self):
+        """Generate annotations for training and testing data."""
+        output_dir = join(self.processed_dir, 'annotation_body3d',
+                          f'fps{50 // self.sample_rate}')
+        os.makedirs(output_dir, exist_ok=True)
+
+        for data_split in ('train', 'test'):
+            imgnames_all = []
+            centers_all = []
+            scales_all = []
+            kps2d_all = []
+            kps3d_all = []
+            for subject in self.subjects_splits[data_split]:
+                for action, subaction in self.sequence_mappings[subject].keys(
+                ):
+                    if action == '1':
+                        # exclude action "_ALL"
+                        continue
+                    for camera in self.camera_ids:
+                        imgnames, centers, scales, kps2d, kps3d\
+                         = self._load_annotations(
+                            subject, action, subaction, camera)
+                        imgnames_all.append(imgnames)
+                        centers_all.append(centers)
+                        scales_all.append(scales)
+                        kps2d_all.append(kps2d)
+                        kps3d_all.append(kps3d)
+
+            imgnames_all = np.concatenate(imgnames_all)
+            centers_all = np.concatenate(centers_all)
+            scales_all = np.concatenate(scales_all)
+            kps2d_all = np.concatenate(kps2d_all)
+            kps3d_all = np.concatenate(kps3d_all)
+
+            out_file = join(output_dir, f'h36m_{data_split}.npz')
+            np.savez(
+                out_file,
+                imgname=imgnames_all,
+                center=centers_all,
+                scale=scales_all,
+                part=kps2d_all,
+                S=kps3d_all)
+
+            print(
+                f'All annotations of {data_split}ing data have been written to'
+                f' "{out_file}". {len(imgnames_all)} samples in total.\n')
+
+            if data_split == 'train':
+                kps_3d_all = kps3d_all[..., :3]  # remove visibility
+                mean_3d, std_3d = self._get_pose_stats(kps_3d_all)
+
+                kps_2d_all = kps2d_all[..., :2]  # remove visibility
+                mean_2d, std_2d = self._get_pose_stats(kps_2d_all)
+
+                # centered around root
+                # the root keypoint is 0-index
+                kps_3d_rel = kps_3d_all[..., 1:, :] - kps_3d_all[..., :1, :]
+                mean_3d_rel, std_3d_rel = self._get_pose_stats(kps_3d_rel)
+
+                kps_2d_rel = kps_2d_all[..., 1:, :] - kps_2d_all[..., :1, :]
+                mean_2d_rel, std_2d_rel = self._get_pose_stats(kps_2d_rel)
+
+                stats = {
+                    'joint3d_stats': {
+                        'mean': mean_3d,
+                        'std': std_3d
+                    },
+                    'joint2d_stats': {
+                        'mean': mean_2d,
+                        'std': std_2d
+                    },
+                    'joint3d_rel_stats': {
+                        'mean': mean_3d_rel,
+                        'std': std_3d_rel
+                    },
+                    'joint2d_rel_stats': {
+                        'mean': mean_2d_rel,
+                        'std': std_2d_rel
+                    }
+                }
+                for name, stat_dict in stats.items():
+                    out_file = join(output_dir, f'{name}.pkl')
+                    with open(out_file, 'wb') as f:
+                        pickle.dump(stat_dict, f)
+                    print(f'Create statistic data file: {out_file}')
+
+    @staticmethod
+    def _get_pose_stats(kps):
+        """Get statistic information `mean` and `std` of pose data.
+
+        Args:
+            kps (ndarray): keypoints in shape [..., K, C] where K and C is
+                the keypoint category number and dimension.
+        Returns:
+            mean (ndarray): [K, C]
+        """
+        assert kps.ndim > 2
+        K, C = kps.shape[-2:]
+        kps = kps.reshape(-1, K, C)
+        mean = kps.mean(axis=0)
+        std = kps.std(axis=0)
+        return mean, std
+
+    def _load_metadata(self):
+        """Load meta data from metadata.xml."""
+
+        assert os.path.exists(self.metadata)
+
+        tree = ET.parse(self.metadata)
+        root = tree.getroot()
+
+        for i, tr in enumerate(root.find('mapping')):
+            if i == 0:
+                _, _, *self.subjects = [td.text for td in tr]
+                self.sequence_mappings \
+                    = {subject: {} for subject in self.subjects}
+            elif i < 33:
+                action_id, subaction_id, *prefixes = [td.text for td in tr]
+                for subject, prefix in zip(self.subjects, prefixes):
+                    self.sequence_mappings[subject][(action_id, subaction_id)]\
+                        = prefix
+
+        for i, elem in enumerate(root.find('actionnames')):
+            action_id = str(i + 1)
+            self.action_names[action_id] = elem.text
+
+        self.camera_ids \
+            = [elem.text for elem in root.find('dbcameras/index2id')]
+
+        w0 = root.find('w0')
+        self.cameras_raw = [float(num) for num in w0.text[1:-1].split()]
+
+    def _get_base_filename(self, subject, action, subaction, camera):
+        """Get base filename given subject, action, subaction and camera."""
+        return f'{self.sequence_mappings[subject][(action, subaction)]}' + \
+            f'.{camera}'
+
+    def _get_camera_params(self, camera, subject):
+        """Get camera parameters given camera id and subject id."""
+        metadata_slice = np.zeros(15)
+        start = 6 * (camera * 11 + (subject - 1))
+
+        metadata_slice[:6] = self.cameras_raw[start:start + 6]
+        metadata_slice[6:] = self.cameras_raw[265 + camera * 9 - 1:265 +
+                                              (camera + 1) * 9 - 1]
+
+        # extrinsics
+        x, y, z = -metadata_slice[0], metadata_slice[1], -metadata_slice[2]
+
+        R_x = np.array([[1, 0, 0], [0, np.cos(x), np.sin(x)],
+                        [0, -np.sin(x), np.cos(x)]])
+        R_y = np.array([[np.cos(y), 0, np.sin(y)], [0, 1, 0],
+                        [-np.sin(y), 0, np.cos(y)]])
+        R_z = np.array([[np.cos(z), np.sin(z), 0], [-np.sin(z),
+                                                    np.cos(z), 0], [0, 0, 1]])
+        R = (R_x @ R_y @ R_z).T
+        T = metadata_slice[3:6].reshape(-1, 1)
+        # convert unit from millimeter to meter
+        T *= 0.001
+
+        # intrinsics
+        c = metadata_slice[8:10, None]
+        f = metadata_slice[6:8, None]
+
+        # distortion
+        k = metadata_slice[10:13, None]
+        p = metadata_slice[13:15, None]
+
+        return {
+            'R': R,
+            'T': T,
+            'c': c,
+            'f': f,
+            'k': k,
+            'p': p,
+            'w': self.image_sizes[self.camera_ids[camera]]['width'],
+            'h': self.image_sizes[self.camera_ids[camera]]['height'],
+            'name': f'camera{camera + 1}',
+            'id': self.camera_ids[camera]
+        }
+
+    def _load_annotations(self, subject, action, subaction, camera):
+        """Load annotations for a sequence."""
+        subj_dir = join(self.extracted_dir, subject)
+        basename = self._get_base_filename(subject, action, subaction, camera)
+
+        # load 2D keypoints
+        with pycdf.CDF(
+                join(subj_dir, 'MyPoseFeatures', 'D2_Positions',
+                     basename + '.cdf')) as cdf:
+            kps_2d = np.array(cdf['Pose'])
+
+        num_frames = kps_2d.shape[1]
+        kps_2d = kps_2d.reshape((num_frames, 32, 2))[::self.sample_rate,
+                                                     self.movable_joints]
+        kps_2d = np.concatenate([kps_2d, np.ones((len(kps_2d), 17, 1))],
+                                axis=2)
+
+        # load 3D keypoints
+        with pycdf.CDF(
+                join(subj_dir, 'MyPoseFeatures', 'D3_Positions_mono',
+                     basename + '.cdf')) as cdf:
+            kps_3d = np.array(cdf['Pose'])
+
+        kps_3d = kps_3d.reshape(
+            (num_frames, 32, 3))[::self.sample_rate,
+                                 self.movable_joints] / 1000.
+        kps_3d = np.concatenate([kps_3d, np.ones((len(kps_3d), 17, 1))],
+                                axis=2)
+
+        # calculate bounding boxes
+        bboxes = np.stack([
+            np.min(kps_2d[:, :, 0], axis=1),
+            np.min(kps_2d[:, :, 1], axis=1),
+            np.max(kps_2d[:, :, 0], axis=1),
+            np.max(kps_2d[:, :, 1], axis=1)
+        ],
+                          axis=1)
+        centers = np.stack([(bboxes[:, 0] + bboxes[:, 2]) / 2,
+                            (bboxes[:, 1] + bboxes[:, 3]) / 2],
+                           axis=1)
+        scales = self.scale_factor * np.max(
+            bboxes[:, 2:] - bboxes[:, :2], axis=1) / 200
+
+        # extract frames and save imgnames
+        imgnames = []
+        video_path = join(subj_dir, 'Videos', basename + '.mp4')
+        sub_base = subject + '_' + basename.replace(' ', '_')
+        img_dir = join(self.processed_dir, 'images', subject, sub_base)
+        os.makedirs(img_dir, exist_ok=True)
+        prefix = join(subject, sub_base, sub_base)
+
+        cap = cv2.VideoCapture(video_path)
+        i = 0
+        while True:
+            success, img = cap.read()
+            if not success:
+                break
+            if i % self.sample_rate == 0:
+                imgname = f'{prefix}_{i + 1:06d}.jpg'
+                imgnames.append(imgname)
+                dest_path = join(self.processed_dir, 'images', imgname)
+                if not os.path.exists(dest_path):
+                    cv2.imwrite(dest_path, img)
+                if len(imgnames) == len(centers):
+                    break
+            i += 1
+        cap.release()
+        imgnames = np.array(imgnames)
+
+        print(f'Annoatations for sequence "{subject} {basename}" are loaded. '
+              f'{len(imgnames)} samples in total.')
+
+        return imgnames, centers, scales, kps_2d, kps_3d
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--metadata', type=str, required=True, help='Path to metadata.xml')
+    parser.add_argument(
+        '--original',
+        type=str,
+        required=True,
+        help='Directory of the original dataset with all files compressed. '
+        'Specifically, .tgz files belonging to subject 1 should be placed '
+        'under the subdirectory \"s1\".')
+    parser.add_argument(
+        '--extracted',
+        type=str,
+        default=None,
+        help='Directory of the extracted files. If not given, it will be '
+        'placed under the same parent directory as original_dir.')
+    parser.add_argument(
+        '--processed',
+        type=str,
+        default=None,
+        help='Directory of the processed files. If not given, it will be '
+        'placed under the same parent directory as original_dir.')
+    parser.add_argument(
+        '--sample_rate',
+        type=int,
+        default=5,
+        help='Downsample FPS to `1 / sample_rate`. Default: 5.')
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    h36m = PreprocessH36m(
+        metadata=args.metadata,
+        original_dir=args.original,
+        extracted_dir=args.extracted,
+        processed_dir=args.processed,
+        sample_rate=args.sample_rate)
+    h36m.extract_tgz()
+    h36m.generate_cameras_file()
+    h36m.generate_annotations()

third_party/hamer/third-party/ViTPose/tools/dataset/preprocess_mpi_inf_3dhp.py

@@ -0,0 +1,359 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import os
+import pickle
+import shutil
+from os.path import join
+
+import cv2
+import h5py
+import mmcv
+import numpy as np
+from scipy.io import loadmat
+
+train_subjects = [i for i in range(1, 9)]
+test_subjects = [i for i in range(1, 7)]
+train_seqs = [1, 2]
+train_cams = [0, 1, 2, 4, 5, 6, 7, 8]
+train_frame_nums = {
+    (1, 1): 6416,
+    (1, 2): 12430,
+    (2, 1): 6502,
+    (2, 2): 6081,
+    (3, 1): 12488,
+    (3, 2): 12283,
+    (4, 1): 6171,
+    (4, 2): 6675,
+    (5, 1): 12820,
+    (5, 2): 12312,
+    (6, 1): 6188,
+    (6, 2): 6145,
+    (7, 1): 6239,
+    (7, 2): 6320,
+    (8, 1): 6468,
+    (8, 2): 6054
+}
+test_frame_nums = {1: 6151, 2: 6080, 3: 5838, 4: 6007, 5: 320, 6: 492}
+train_img_size = (2048, 2048)
+root_index = 14
+joints_17 = [7, 5, 14, 15, 16, 9, 10, 11, 23, 24, 25, 18, 19, 20, 4, 3, 6]
+
+
+def get_pose_stats(kps):
+    """Get statistic information `mean` and `std` of pose data.
+
+    Args:
+        kps (ndarray): keypoints in shape [..., K, C] where K and C is
+            the keypoint category number and dimension.
+    Returns:
+        mean (ndarray): [K, C]
+    """
+    assert kps.ndim > 2
+    K, C = kps.shape[-2:]
+    kps = kps.reshape(-1, K, C)
+    mean = kps.mean(axis=0)
+    std = kps.std(axis=0)
+    return mean, std
+
+
+def get_annotations(joints_2d, joints_3d, scale_factor=1.2):
+    """Get annotations, including centers, scales, joints_2d and joints_3d.
+
+    Args:
+        joints_2d: 2D joint coordinates in shape [N, K, 2], where N is the
+            frame number, K is the joint number.
+        joints_3d: 3D joint coordinates in shape [N, K, 3], where N is the
+            frame number, K is the joint number.
+        scale_factor: Scale factor of bounding box. Default: 1.2.
+    Returns:
+        centers (ndarray): [N, 2]
+        scales (ndarray): [N,]
+        joints_2d (ndarray): [N, K, 3]
+        joints_3d (ndarray): [N, K, 4]
+    """
+    # calculate joint visibility
+    visibility = (joints_2d[:, :, 0] >= 0) * \
+                 (joints_2d[:, :, 0] < train_img_size[0]) * \
+                 (joints_2d[:, :, 1] >= 0) * \
+                 (joints_2d[:, :, 1] < train_img_size[1])
+    visibility = np.array(visibility, dtype=np.float32)[:, :, None]
+    joints_2d = np.concatenate([joints_2d, visibility], axis=-1)
+    joints_3d = np.concatenate([joints_3d, visibility], axis=-1)
+
+    # calculate bounding boxes
+    bboxes = np.stack([
+        np.min(joints_2d[:, :, 0], axis=1),
+        np.min(joints_2d[:, :, 1], axis=1),
+        np.max(joints_2d[:, :, 0], axis=1),
+        np.max(joints_2d[:, :, 1], axis=1)
+    ],
+                      axis=1)
+    centers = np.stack([(bboxes[:, 0] + bboxes[:, 2]) / 2,
+                        (bboxes[:, 1] + bboxes[:, 3]) / 2],
+                       axis=1)
+    scales = scale_factor * np.max(bboxes[:, 2:] - bboxes[:, :2], axis=1) / 200
+
+    return centers, scales, joints_2d, joints_3d
+
+
+def load_trainset(data_root, out_dir):
+    """Load training data, create annotation file and camera file.
+    Args:
+        data_root: Directory of dataset, which is organized in the following
+            hierarchy:
+                data_root
+                |-- train
+                    |-- S1
+                        |-- Seq1
+                        |-- Seq2
+                    |-- S2
+                    |-- ...
+                |-- test
+                    |-- TS1
+                    |-- TS2
+                    |-- ...
+        out_dir: Directory to save annotation file.
+    """
+    _imgnames = []
+    _centers = []
+    _scales = []
+    _joints_2d = []
+    _joints_3d = []
+    cameras = {}
+
+    img_dir = join(out_dir, 'images')
+    os.makedirs(img_dir, exist_ok=True)
+    annot_dir = join(out_dir, 'annotations')
+    os.makedirs(annot_dir, exist_ok=True)
+
+    for subj in train_subjects:
+        for seq in train_seqs:
+            seq_path = join(data_root, 'train', f'S{subj}', f'Seq{seq}')
+            num_frames = train_frame_nums[(subj, seq)]
+
+            # load camera parametres
+            camera_file = join(seq_path, 'camera.calibration')
+            with open(camera_file, 'r') as fin:
+                lines = fin.readlines()
+                for cam in train_cams:
+                    K = [float(s) for s in lines[cam * 7 + 5][11:-2].split()]
+                    f = np.array([[K[0]], [K[5]]])
+                    c = np.array([[K[2]], [K[6]]])
+                    RT = np.array(
+                        [float(s) for s in lines[cam * 7 + 6][11:-2].split()])
+                    RT = np.reshape(RT, (4, 4))
+                    R = RT[:3, :3]
+                    # convert unit from millimeter to meter
+                    T = RT[:3, 3:] * 0.001
+                    size = [int(s) for s in lines[cam * 7 + 3][14:].split()]
+                    w, h = size
+                    cam_param = dict(
+                        R=R, T=T, c=c, f=f, w=w, h=h, name=f'train_cam_{cam}')
+                    cameras[f'S{subj}_Seq{seq}_Cam{cam}'] = cam_param
+
+            # load annotations
+            annot_file = os.path.join(seq_path, 'annot.mat')
+            annot2 = loadmat(annot_file)['annot2']
+            annot3 = loadmat(annot_file)['annot3']
+            for cam in train_cams:
+                # load 2D and 3D annotations
+                joints_2d = np.reshape(annot2[cam][0][:num_frames],
+                                       (num_frames, 28, 2))[:, joints_17]
+                joints_3d = np.reshape(annot3[cam][0][:num_frames],
+                                       (num_frames, 28, 3))[:, joints_17]
+                joints_3d = joints_3d * 0.001
+                centers, scales, joints_2d, joints_3d = get_annotations(
+                    joints_2d, joints_3d)
+                _centers.append(centers)
+                _scales.append(scales)
+                _joints_2d.append(joints_2d)
+                _joints_3d.append(joints_3d)
+
+                # extract frames from video
+                video_path = join(seq_path, 'imageSequence',
+                                  f'video_{cam}.avi')
+                video = mmcv.VideoReader(video_path)
+                for i in mmcv.track_iter_progress(range(num_frames)):
+                    img = video.read()
+                    if img is None:
+                        break
+                    imgname = f'S{subj}_Seq{seq}_Cam{cam}_{i+1:06d}.jpg'
+                    _imgnames.append(imgname)
+                    cv2.imwrite(join(img_dir, imgname), img)
+
+    _imgnames = np.array(_imgnames)
+    _centers = np.concatenate(_centers)
+    _scales = np.concatenate(_scales)
+    _joints_2d = np.concatenate(_joints_2d)
+    _joints_3d = np.concatenate(_joints_3d)
+
+    out_file = join(annot_dir, 'mpi_inf_3dhp_train.npz')
+    np.savez(
+        out_file,
+        imgname=_imgnames,
+        center=_centers,
+        scale=_scales,
+        part=_joints_2d,
+        S=_joints_3d)
+    print(f'Create annotation file for trainset: {out_file}. '
+          f'{len(_imgnames)} samples in total.')
+
+    out_file = join(annot_dir, 'cameras_train.pkl')
+    with open(out_file, 'wb') as fout:
+        pickle.dump(cameras, fout)
+    print(f'Create camera file for trainset: {out_file}.')
+
+    # get `mean` and `std` of pose data
+    _joints_3d = _joints_3d[..., :3]  # remove visibility
+    mean_3d, std_3d = get_pose_stats(_joints_3d)
+
+    _joints_2d = _joints_2d[..., :2]  # remove visibility
+    mean_2d, std_2d = get_pose_stats(_joints_2d)
+
+    # centered around root
+    _joints_3d_rel = _joints_3d - _joints_3d[..., root_index:root_index + 1, :]
+    mean_3d_rel, std_3d_rel = get_pose_stats(_joints_3d_rel)
+    mean_3d_rel[root_index] = mean_3d[root_index]
+    std_3d_rel[root_index] = std_3d[root_index]
+
+    _joints_2d_rel = _joints_2d - _joints_2d[..., root_index:root_index + 1, :]
+    mean_2d_rel, std_2d_rel = get_pose_stats(_joints_2d_rel)
+    mean_2d_rel[root_index] = mean_2d[root_index]
+    std_2d_rel[root_index] = std_2d[root_index]
+
+    stats = {
+        'joint3d_stats': {
+            'mean': mean_3d,
+            'std': std_3d
+        },
+        'joint2d_stats': {
+            'mean': mean_2d,
+            'std': std_2d
+        },
+        'joint3d_rel_stats': {
+            'mean': mean_3d_rel,
+            'std': std_3d_rel
+        },
+        'joint2d_rel_stats': {
+            'mean': mean_2d_rel,
+            'std': std_2d_rel
+        }
+    }
+    for name, stat_dict in stats.items():
+        out_file = join(annot_dir, f'{name}.pkl')
+        with open(out_file, 'wb') as f:
+            pickle.dump(stat_dict, f)
+        print(f'Create statistic data file: {out_file}')
+
+
+def load_testset(data_root, out_dir, valid_only=True):
+    """Load testing data, create annotation file and camera file.
+
+    Args:
+        data_root: Directory of dataset.
+        out_dir: Directory to save annotation file.
+        valid_only: Only keep frames with valid_label == 1.
+    """
+    _imgnames = []
+    _centers = []
+    _scales = []
+    _joints_2d = []
+    _joints_3d = []
+    cameras = {}
+
+    img_dir = join(out_dir, 'images')
+    os.makedirs(img_dir, exist_ok=True)
+    annot_dir = join(out_dir, 'annotations')
+    os.makedirs(annot_dir, exist_ok=True)
+
+    for subj in test_subjects:
+        subj_path = join(data_root, 'test', f'TS{subj}')
+        num_frames = test_frame_nums[subj]
+
+        # load annotations
+        annot_file = os.path.join(subj_path, 'annot_data.mat')
+        with h5py.File(annot_file, 'r') as fin:
+            annot2 = np.array(fin['annot2']).reshape((-1, 17, 2))
+            annot3 = np.array(fin['annot3']).reshape((-1, 17, 3))
+            valid = np.array(fin['valid_frame']).reshape(-1)
+
+        # manually estimate camera intrinsics
+        fx, cx = np.linalg.lstsq(
+            annot3[:, :, [0, 2]].reshape((-1, 2)),
+            (annot2[:, :, 0] * annot3[:, :, 2]).reshape(-1, 1),
+            rcond=None)[0].flatten()
+        fy, cy = np.linalg.lstsq(
+            annot3[:, :, [1, 2]].reshape((-1, 2)),
+            (annot2[:, :, 1] * annot3[:, :, 2]).reshape(-1, 1),
+            rcond=None)[0].flatten()
+        if subj <= 4:
+            w, h = 2048, 2048
+        else:
+            w, h = 1920, 1080
+        cameras[f'TS{subj}'] = dict(
+            c=np.array([[cx], [cy]]),
+            f=np.array([[fx], [fy]]),
+            w=w,
+            h=h,
+            name=f'test_cam_{subj}')
+
+        # get annotations
+        if valid_only:
+            valid_frames = np.nonzero(valid)[0]
+        else:
+            valid_frames = np.arange(num_frames)
+        joints_2d = annot2[valid_frames, :, :]
+        joints_3d = annot3[valid_frames, :, :] * 0.001
+
+        centers, scales, joints_2d, joints_3d = get_annotations(
+            joints_2d, joints_3d)
+        _centers.append(centers)
+        _scales.append(scales)
+        _joints_2d.append(joints_2d)
+        _joints_3d.append(joints_3d)
+
+        # copy and rename images
+        for i in valid_frames:
+            imgname = f'TS{subj}_{i+1:06d}.jpg'
+            shutil.copyfile(
+                join(subj_path, 'imageSequence', f'img_{i+1:06d}.jpg'),
+                join(img_dir, imgname))
+            _imgnames.append(imgname)
+
+    _imgnames = np.array(_imgnames)
+    _centers = np.concatenate(_centers)
+    _scales = np.concatenate(_scales)
+    _joints_2d = np.concatenate(_joints_2d)
+    _joints_3d = np.concatenate(_joints_3d)
+
+    if valid_only:
+        out_file = join(annot_dir, 'mpi_inf_3dhp_test_valid.npz')
+    else:
+        out_file = join(annot_dir, 'mpi_inf_3dhp_test_all.npz')
+    np.savez(
+        out_file,
+        imgname=_imgnames,
+        center=_centers,
+        scale=_scales,
+        part=_joints_2d,
+        S=_joints_3d)
+    print(f'Create annotation file for testset: {out_file}. '
+          f'{len(_imgnames)} samples in total.')
+
+    out_file = join(annot_dir, 'cameras_test.pkl')
+    with open(out_file, 'wb') as fout:
+        pickle.dump(cameras, fout)
+    print(f'Create camera file for testset: {out_file}.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('data_root', type=str, help='data root')
+    parser.add_argument(
+        'out_dir', type=str, help='directory to save annotation files.')
+    args = parser.parse_args()
+    data_root = args.data_root
+    out_dir = args.out_dir
+
+    load_trainset(data_root, out_dir)
+    load_testset(data_root, out_dir, valid_only=True)

third_party/hamer/third-party/ViTPose/tools/deployment/mmpose2torchserve.py

@@ -0,0 +1,135 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import os.path as osp
+import warnings
+from argparse import ArgumentParser, Namespace
+from tempfile import TemporaryDirectory
+
+import mmcv
+import torch
+from mmcv.runner import CheckpointLoader
+
+try:
+    from model_archiver.model_packaging import package_model
+    from model_archiver.model_packaging_utils import ModelExportUtils
+except ImportError:
+    package_model = None
+
+
+def mmpose2torchserve(config_file: str,
+                      checkpoint_file: str,
+                      output_folder: str,
+                      model_name: str,
+                      model_version: str = '1.0',
+                      force: bool = False):
+    """Converts MMPose model (config + checkpoint) to TorchServe `.mar`.
+
+    Args:
+        config_file:
+            In MMPose config format.
+            The contents vary for each task repository.
+        checkpoint_file:
+            In MMPose checkpoint format.
+            The contents vary for each task repository.
+        output_folder:
+            Folder where `{model_name}.mar` will be created.
+            The file created will be in TorchServe archive format.
+        model_name:
+            If not None, used for naming the `{model_name}.mar` file
+            that will be created under `output_folder`.
+            If None, `{Path(checkpoint_file).stem}` will be used.
+        model_version:
+            Model's version.
+        force:
+            If True, if there is an existing `{model_name}.mar`
+            file under `output_folder` it will be overwritten.
+    """
+
+    mmcv.mkdir_or_exist(output_folder)
+
+    config = mmcv.Config.fromfile(config_file)
+
+    with TemporaryDirectory() as tmpdir:
+        model_file = osp.join(tmpdir, 'config.py')
+        config.dump(model_file)
+        handler_path = osp.join(osp.dirname(__file__), 'mmpose_handler.py')
+        model_name = model_name or osp.splitext(
+            osp.basename(checkpoint_file))[0]
+
+        # use mmcv CheckpointLoader if checkpoint is not from a local file
+        if not osp.isfile(checkpoint_file):
+            ckpt = CheckpointLoader.load_checkpoint(checkpoint_file)
+            checkpoint_file = osp.join(tmpdir, 'checkpoint.pth')
+            with open(checkpoint_file, 'wb') as f:
+                torch.save(ckpt, f)
+
+        args = Namespace(
+            **{
+                'model_file': model_file,
+                'serialized_file': checkpoint_file,
+                'handler': handler_path,
+                'model_name': model_name,
+                'version': model_version,
+                'export_path': output_folder,
+                'force': force,
+                'requirements_file': None,
+                'extra_files': None,
+                'runtime': 'python',
+                'archive_format': 'default'
+            })
+        manifest = ModelExportUtils.generate_manifest_json(args)
+        package_model(args, manifest)
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description='Convert MMPose models to TorchServe `.mar` format.')
+    parser.add_argument('config', type=str, help='config file path')
+    parser.add_argument('checkpoint', type=str, help='checkpoint file path')
+    parser.add_argument(
+        '--output-folder',
+        type=str,
+        required=True,
+        help='Folder where `{model_name}.mar` will be created.')
+    parser.add_argument(
+        '--model-name',
+        type=str,
+        default=None,
+        help='If not None, used for naming the `{model_name}.mar`'
+        'file that will be created under `output_folder`.'
+        'If None, `{Path(checkpoint_file).stem}` will be used.')
+    parser.add_argument(
+        '--model-version',
+        type=str,
+        default='1.0',
+        help='Number used for versioning.')
+    parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        help='overwrite the existing `{model_name}.mar`')
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    # Following strings of text style are from colorama package
+    bright_style, reset_style = '\x1b[1m', '\x1b[0m'
+    red_text, blue_text = '\x1b[31m', '\x1b[34m'
+    white_background = '\x1b[107m'
+
+    msg = white_background + bright_style + red_text
+    msg += 'DeprecationWarning: This tool will be deprecated in future. '
+    msg += blue_text + 'Welcome to use the unified model deployment toolbox '
+    msg += 'MMDeploy: https://github.com/open-mmlab/mmdeploy'
+    msg += reset_style
+    warnings.warn(msg)
+
+    if package_model is None:
+        raise ImportError('`torch-model-archiver` is required.'
+                          'Try: pip install torch-model-archiver')
+
+    mmpose2torchserve(args.config, args.checkpoint, args.output_folder,
+                      args.model_name, args.model_version, args.force)

third_party/hamer/third-party/ViTPose/tools/deployment/mmpose_handler.py

@@ -0,0 +1,80 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import base64
+import os
+
+import mmcv
+import torch
+
+from mmpose.apis import (inference_bottom_up_pose_model,
+                         inference_top_down_pose_model, init_pose_model)
+from mmpose.models.detectors import AssociativeEmbedding, TopDown
+
+try:
+    from ts.torch_handler.base_handler import BaseHandler
+except ImportError:
+    raise ImportError('Please install torchserve.')
+
+
+class MMPoseHandler(BaseHandler):
+
+    def initialize(self, context):
+        properties = context.system_properties
+        self.map_location = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = torch.device(self.map_location + ':' +
+                                   str(properties.get('gpu_id')) if torch.cuda.
+                                   is_available() else self.map_location)
+        self.manifest = context.manifest
+
+        model_dir = properties.get('model_dir')
+        serialized_file = self.manifest['model']['serializedFile']
+        checkpoint = os.path.join(model_dir, serialized_file)
+        self.config_file = os.path.join(model_dir, 'config.py')
+
+        self.model = init_pose_model(self.config_file, checkpoint, self.device)
+        self.initialized = True
+
+    def preprocess(self, data):
+        images = []
+
+        for row in data:
+            image = row.get('data') or row.get('body')
+            if isinstance(image, str):
+                image = base64.b64decode(image)
+            image = mmcv.imfrombytes(image)
+            images.append(image)
+
+        return images
+
+    def inference(self, data, *args, **kwargs):
+        if isinstance(self.model, TopDown):
+            results = self._inference_top_down_pose_model(data)
+        elif isinstance(self.model, (AssociativeEmbedding, )):
+            results = self._inference_bottom_up_pose_model(data)
+        else:
+            raise NotImplementedError(
+                f'Model type {type(self.model)} is not supported.')
+
+        return results
+
+    def _inference_top_down_pose_model(self, data):
+        results = []
+        for image in data:
+            # use dummy person bounding box
+            preds, _ = inference_top_down_pose_model(
+                self.model, image, person_results=None)
+            results.append(preds)
+        return results
+
+    def _inference_bottom_up_pose_model(self, data):
+        results = []
+        for image in data:
+            preds, _ = inference_bottom_up_pose_model(self.model, image)
+            results.append(preds)
+        return results
+
+    def postprocess(self, data):
+        output = [[{
+            'keypoints': pred['keypoints'].tolist()
+        } for pred in preds] for preds in data]
+
+        return output

third_party/hamer/third-party/ViTPose/tools/deployment/pytorch2onnx.py

@@ -0,0 +1,165 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import warnings
+
+import numpy as np
+import torch
+
+from mmpose.apis import init_pose_model
+
+try:
+    import onnx
+    import onnxruntime as rt
+except ImportError as e:
+    raise ImportError(f'Please install onnx and onnxruntime first. {e}')
+
+try:
+    from mmcv.onnx.symbolic import register_extra_symbolics
+except ModuleNotFoundError:
+    raise NotImplementedError('please update mmcv to version>=1.0.4')
+
+
+def _convert_batchnorm(module):
+    """Convert the syncBNs into normal BN3ds."""
+    module_output = module
+    if isinstance(module, torch.nn.SyncBatchNorm):
+        module_output = torch.nn.BatchNorm3d(module.num_features, module.eps,
+                                             module.momentum, module.affine,
+                                             module.track_running_stats)
+        if module.affine:
+            module_output.weight.data = module.weight.data.clone().detach()
+            module_output.bias.data = module.bias.data.clone().detach()
+            # keep requires_grad unchanged
+            module_output.weight.requires_grad = module.weight.requires_grad
+            module_output.bias.requires_grad = module.bias.requires_grad
+        module_output.running_mean = module.running_mean
+        module_output.running_var = module.running_var
+        module_output.num_batches_tracked = module.num_batches_tracked
+    for name, child in module.named_children():
+        module_output.add_module(name, _convert_batchnorm(child))
+    del module
+    return module_output
+
+
+def pytorch2onnx(model,
+                 input_shape,
+                 opset_version=11,
+                 show=False,
+                 output_file='tmp.onnx',
+                 verify=False):
+    """Convert pytorch model to onnx model.
+
+    Args:
+        model (:obj:`nn.Module`): The pytorch model to be exported.
+        input_shape (tuple[int]): The input tensor shape of the model.
+        opset_version (int): Opset version of onnx used. Default: 11.
+        show (bool): Determines whether to print the onnx model architecture.
+            Default: False.
+        output_file (str): Output onnx model name. Default: 'tmp.onnx'.
+        verify (bool): Determines whether to verify the onnx model.
+            Default: False.
+    """
+    model.cpu().eval()
+
+    one_img = torch.randn(input_shape)
+
+    register_extra_symbolics(opset_version)
+    torch.onnx.export(
+        model,
+        one_img,
+        output_file,
+        export_params=True,
+        keep_initializers_as_inputs=True,
+        verbose=show,
+        opset_version=opset_version)
+
+    print(f'Successfully exported ONNX model: {output_file}')
+    if verify:
+        # check by onnx
+        onnx_model = onnx.load(output_file)
+        onnx.checker.check_model(onnx_model)
+
+        # check the numerical value
+        # get pytorch output
+        pytorch_results = model(one_img)
+        if not isinstance(pytorch_results, (list, tuple)):
+            assert isinstance(pytorch_results, torch.Tensor)
+            pytorch_results = [pytorch_results]
+
+        # get onnx output
+        input_all = [node.name for node in onnx_model.graph.input]
+        input_initializer = [
+            node.name for node in onnx_model.graph.initializer
+        ]
+        net_feed_input = list(set(input_all) - set(input_initializer))
+        assert len(net_feed_input) == 1
+        sess = rt.InferenceSession(output_file)
+        onnx_results = sess.run(None,
+                                {net_feed_input[0]: one_img.detach().numpy()})
+
+        # compare results
+        assert len(pytorch_results) == len(onnx_results)
+        for pt_result, onnx_result in zip(pytorch_results, onnx_results):
+            assert np.allclose(
+                pt_result.detach().cpu(), onnx_result, atol=1.e-5
+            ), 'The outputs are different between Pytorch and ONNX'
+        print('The numerical values are same between Pytorch and ONNX')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Convert MMPose models to ONNX')
+    parser.add_argument('config', help='test config file path')
+    parser.add_argument('checkpoint', help='checkpoint file')
+    parser.add_argument('--show', action='store_true', help='show onnx graph')
+    parser.add_argument('--output-file', type=str, default='tmp.onnx')
+    parser.add_argument('--opset-version', type=int, default=11)
+    parser.add_argument(
+        '--verify',
+        action='store_true',
+        help='verify the onnx model output against pytorch output')
+    parser.add_argument(
+        '--shape',
+        type=int,
+        nargs='+',
+        default=[1, 3, 256, 192],
+        help='input size')
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    assert args.opset_version == 11, 'MMPose only supports opset 11 now'
+
+    # Following strings of text style are from colorama package
+    bright_style, reset_style = '\x1b[1m', '\x1b[0m'
+    red_text, blue_text = '\x1b[31m', '\x1b[34m'
+    white_background = '\x1b[107m'
+
+    msg = white_background + bright_style + red_text
+    msg += 'DeprecationWarning: This tool will be deprecated in future. '
+    msg += blue_text + 'Welcome to use the unified model deployment toolbox '
+    msg += 'MMDeploy: https://github.com/open-mmlab/mmdeploy'
+    msg += reset_style
+    warnings.warn(msg)
+
+    model = init_pose_model(args.config, args.checkpoint, device='cpu')
+    model = _convert_batchnorm(model)
+
+    # onnx.export does not support kwargs
+    if hasattr(model, 'forward_dummy'):
+        model.forward = model.forward_dummy
+    else:
+        raise NotImplementedError(
+            'Please implement the forward method for exporting.')
+
+    # convert model to onnx file
+    pytorch2onnx(
+        model,
+        args.shape,
+        opset_version=args.opset_version,
+        show=args.show,
+        output_file=args.output_file,
+        verify=args.verify)

third_party/hamer/third-party/ViTPose/tools/deployment/test_torchserver.py

@@ -0,0 +1,79 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import os
+import os.path as osp
+import warnings
+from argparse import ArgumentParser
+
+import requests
+
+from mmpose.apis import (inference_bottom_up_pose_model,
+                         inference_top_down_pose_model, init_pose_model,
+                         vis_pose_result)
+from mmpose.models import AssociativeEmbedding, TopDown
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument('img', help='Image file')
+    parser.add_argument('config', help='Config file')
+    parser.add_argument('checkpoint', help='Checkpoint file')
+    parser.add_argument('model_name', help='The model name in the server')
+    parser.add_argument(
+        '--inference-addr',
+        default='127.0.0.1:8080',
+        help='Address and port of the inference server')
+    parser.add_argument(
+        '--device', default='cuda:0', help='Device used for inference')
+    parser.add_argument(
+        '--out-dir', default='vis_results', help='Visualization output path')
+    args = parser.parse_args()
+    return args
+
+
+def main(args):
+    os.makedirs(args.out_dir, exist_ok=True)
+
+    # Inference single image by native apis.
+    model = init_pose_model(args.config, args.checkpoint, device=args.device)
+    if isinstance(model, TopDown):
+        pytorch_result, _ = inference_top_down_pose_model(
+            model, args.img, person_results=None)
+    elif isinstance(model, (AssociativeEmbedding, )):
+        pytorch_result, _ = inference_bottom_up_pose_model(model, args.img)
+    else:
+        raise NotImplementedError()
+
+    vis_pose_result(
+        model,
+        args.img,
+        pytorch_result,
+        out_file=osp.join(args.out_dir, 'pytorch_result.png'))
+
+    # Inference single image by torchserve engine.
+    url = 'http://' + args.inference_addr + '/predictions/' + args.model_name
+    with open(args.img, 'rb') as image:
+        response = requests.post(url, image)
+    server_result = response.json()
+
+    vis_pose_result(
+        model,
+        args.img,
+        server_result,
+        out_file=osp.join(args.out_dir, 'torchserve_result.png'))
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    main(args)
+
+    # Following strings of text style are from colorama package
+    bright_style, reset_style = '\x1b[1m', '\x1b[0m'
+    red_text, blue_text = '\x1b[31m', '\x1b[34m'
+    white_background = '\x1b[107m'
+
+    msg = white_background + bright_style + red_text
+    msg += 'DeprecationWarning: This tool will be deprecated in future. '
+    msg += blue_text + 'Welcome to use the unified model deployment toolbox '
+    msg += 'MMDeploy: https://github.com/open-mmlab/mmdeploy'
+    msg += reset_style
+    warnings.warn(msg)

third_party/hamer/third-party/ViTPose/tools/dist_test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+
+CONFIG=$1
+CHECKPOINT=$2
+GPUS=$3
+PORT=${PORT:-29500}
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \
+    $(dirname "$0")/test.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4}

third_party/hamer/third-party/ViTPose/tools/dist_train.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+
+CONFIG=$1
+GPUS=$2
+PORT=${PORT:-29500}
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \
+    $(dirname "$0")/train.py $CONFIG --launcher pytorch ${@:3}

third_party/hamer/third-party/ViTPose/tools/misc/keypoints2coco_without_mmdet.py

@@ -0,0 +1,146 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import json
+import os
+from argparse import ArgumentParser
+
+from mmcv import track_iter_progress
+from PIL import Image
+from xtcocotools.coco import COCO
+
+from mmpose.apis import inference_top_down_pose_model, init_pose_model
+
+
+def main():
+    """Visualize the demo images.
+
+    pose_keypoints require the json_file containing boxes.
+    """
+    parser = ArgumentParser()
+    parser.add_argument('pose_config', help='Config file for detection')
+    parser.add_argument('pose_checkpoint', help='Checkpoint file')
+    parser.add_argument('--img-root', type=str, default='', help='Image root')
+    parser.add_argument(
+        '--json-file',
+        type=str,
+        default='',
+        help='Json file containing image person bboxes in COCO format.')
+    parser.add_argument(
+        '--out-json-file',
+        type=str,
+        default='',
+        help='Output json contains pseudolabeled annotation')
+    parser.add_argument(
+        '--show',
+        action='store_true',
+        default=False,
+        help='whether to show img')
+    parser.add_argument(
+        '--device', default='cuda:0', help='Device used for inference')
+    parser.add_argument(
+        '--kpt-thr', type=float, default=0.3, help='Keypoint score threshold')
+
+    args = parser.parse_args()
+
+    coco = COCO(args.json_file)
+    # build the pose model from a config file and a checkpoint file
+    pose_model = init_pose_model(
+        args.pose_config, args.pose_checkpoint, device=args.device.lower())
+
+    dataset = pose_model.cfg.data['test']['type']
+
+    img_keys = list(coco.imgs.keys())
+
+    # optional
+    return_heatmap = False
+
+    # e.g. use ('backbone', ) to return backbone feature
+    output_layer_names = None
+
+    categories = [{'id': 1, 'name': 'person'}]
+    img_anno_dict = {'images': [], 'annotations': [], 'categories': categories}
+
+    # process each image
+    ann_uniq_id = int(0)
+    for i in track_iter_progress(range(len(img_keys))):
+        # get bounding box annotations
+        image_id = img_keys[i]
+        image = coco.loadImgs(image_id)[0]
+        image_name = os.path.join(args.img_root, image['file_name'])
+
+        width, height = Image.open(image_name).size
+        ann_ids = coco.getAnnIds(image_id)
+
+        # make person bounding boxes
+        person_results = []
+        for ann_id in ann_ids:
+            person = {}
+            ann = coco.anns[ann_id]
+            # bbox format is 'xywh'
+            person['bbox'] = ann['bbox']
+            person_results.append(person)
+
+        pose_results, returned_outputs = inference_top_down_pose_model(
+            pose_model,
+            image_name,
+            person_results,
+            bbox_thr=None,
+            format='xywh',
+            dataset=dataset,
+            return_heatmap=return_heatmap,
+            outputs=output_layer_names)
+
+        # add output of model and bboxes to dict
+        for indx, i in enumerate(pose_results):
+            pose_results[indx]['keypoints'][
+                pose_results[indx]['keypoints'][:, 2] < args.kpt_thr, :3] = 0
+            pose_results[indx]['keypoints'][
+                pose_results[indx]['keypoints'][:, 2] >= args.kpt_thr, 2] = 2
+            x = int(pose_results[indx]['bbox'][0])
+            y = int(pose_results[indx]['bbox'][1])
+            w = int(pose_results[indx]['bbox'][2] -
+                    pose_results[indx]['bbox'][0])
+            h = int(pose_results[indx]['bbox'][3] -
+                    pose_results[indx]['bbox'][1])
+            bbox = [x, y, w, h]
+            area = round((w * h), 0)
+
+            images = {
+                'file_name': image_name.split('/')[-1],
+                'height': height,
+                'width': width,
+                'id': int(image_id)
+            }
+
+            annotations = {
+                'keypoints': [
+                    int(i) for i in pose_results[indx]['keypoints'].reshape(
+                        -1).tolist()
+                ],
+                'num_keypoints':
+                len(pose_results[indx]['keypoints']),
+                'area':
+                area,
+                'iscrowd':
+                0,
+                'image_id':
+                int(image_id),
+                'bbox':
+                bbox,
+                'category_id':
+                1,
+                'id':
+                ann_uniq_id,
+            }
+
+            img_anno_dict['annotations'].append(annotations)
+            ann_uniq_id += 1
+
+        img_anno_dict['images'].append(images)
+
+    # create json
+    with open(args.out_json_file, 'w') as outfile:
+        json.dump(img_anno_dict, outfile, indent=2)
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/misc/publish_model.py

@@ -0,0 +1,43 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import subprocess
+from datetime import date
+
+import torch
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Process a checkpoint to be published')
+    parser.add_argument('in_file', help='input checkpoint filename')
+    parser.add_argument('out_file', help='output checkpoint filename')
+    args = parser.parse_args()
+    return args
+
+
+def process_checkpoint(in_file, out_file):
+    checkpoint = torch.load(in_file, map_location='cpu')
+    # remove optimizer for smaller file size
+    if 'optimizer' in checkpoint:
+        del checkpoint['optimizer']
+    # if it is necessary to remove some sensitive data in checkpoint['meta'],
+    # add the code here.
+    torch.save(checkpoint, out_file)
+    sha = subprocess.check_output(['sha256sum', out_file]).decode()
+    if out_file.endswith('.pth'):
+        out_file_name = out_file[:-4]
+    else:
+        out_file_name = out_file
+
+    date_now = date.today().strftime('%Y%m%d')
+    final_file = out_file_name + f'-{sha[:8]}_{date_now}.pth'
+    subprocess.Popen(['mv', out_file, final_file])
+
+
+def main():
+    args = parse_args()
+    process_checkpoint(args.in_file, args.out_file)
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/model_split.py

@@ -0,0 +1,104 @@
+import torch
+import os
+import argparse
+import copy
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--source', type=str)
+    parser.add_argument('--target', type=str, default=None)
+    args = parser.parse_args()
+    return args
+
+def main():
+
+    args = parse_args()
+
+    if args.target is None:
+        args.target = '/'.join(args.source.split('/')[:-1])
+
+    ckpt = torch.load(args.source, map_location='cpu')
+    
+    experts = dict()
+
+    new_ckpt = copy.deepcopy(ckpt)
+
+    state_dict = new_ckpt['state_dict']
+
+    for key, value in state_dict.items():
+        if 'mlp.experts' in key:
+            experts[key] = value
+
+    keys = ckpt['state_dict'].keys()
+
+    target_expert = 0
+    new_ckpt = copy.deepcopy(ckpt)
+
+    for key in keys:
+        if 'mlp.fc2' in key:
+            value = new_ckpt['state_dict'][key]
+            value = torch.cat([value, experts[key.replace('fc2.', f'experts.{target_expert}.')]], dim=0)
+            new_ckpt['state_dict'][key] = value
+
+    torch.save(new_ckpt, os.path.join(args.target, 'coco.pth'))
+
+    names = ['aic', 'mpii', 'ap10k', 'apt36k','wholebody']
+    num_keypoints = [14, 16, 17, 17, 133]
+    weight_names = ['keypoint_head.deconv_layers.0.weight', 
+                    'keypoint_head.deconv_layers.1.weight', 
+                    'keypoint_head.deconv_layers.1.bias', 
+                    'keypoint_head.deconv_layers.1.running_mean', 
+                    'keypoint_head.deconv_layers.1.running_var', 
+                    'keypoint_head.deconv_layers.1.num_batches_tracked', 
+                    'keypoint_head.deconv_layers.3.weight', 
+                    'keypoint_head.deconv_layers.4.weight', 
+                    'keypoint_head.deconv_layers.4.bias', 
+                    'keypoint_head.deconv_layers.4.running_mean', 
+                    'keypoint_head.deconv_layers.4.running_var', 
+                    'keypoint_head.deconv_layers.4.num_batches_tracked', 
+                    'keypoint_head.final_layer.weight', 
+                    'keypoint_head.final_layer.bias']
+    
+    exist_range = True
+
+    for i in range(5):
+
+        new_ckpt = copy.deepcopy(ckpt)
+
+        target_expert = i + 1
+
+        for key in keys:
+            if 'mlp.fc2' in key:
+                expert_key = key.replace('fc2.', f'experts.{target_expert}.')
+                if expert_key in experts:
+                    value = new_ckpt['state_dict'][key]
+                    value = torch.cat([value, experts[expert_key]], dim=0)
+                else:
+                    exist_range = False
+
+                new_ckpt['state_dict'][key] = value
+
+        if not exist_range:
+            break
+
+        for tensor_name in weight_names:
+            new_ckpt['state_dict'][tensor_name] = new_ckpt['state_dict'][tensor_name.replace('keypoint_head', f'associate_keypoint_heads.{i}')]
+
+        for tensor_name in ['keypoint_head.final_layer.weight', 'keypoint_head.final_layer.bias']:
+            new_ckpt['state_dict'][tensor_name] = new_ckpt['state_dict'][tensor_name][:num_keypoints[i]]
+        
+        # remove unnecessary part in the state dict
+        for j in range(5):
+            # remove associate part
+            for tensor_name in weight_names:
+                new_ckpt['state_dict'].pop(tensor_name.replace('keypoint_head', f'associate_keypoint_heads.{j}'))
+        # remove expert part
+        keys = new_ckpt['state_dict'].keys()
+        for key in list(keys):
+            if 'expert' in keys:
+                new_ckpt['state_dict'].pop(key)
+            
+        torch.save(new_ckpt, os.path.join(args.target, f'{names[i]}.pth'))
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/slurm_test.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+
+set -x
+
+PARTITION=$1
+JOB_NAME=$2
+CONFIG=$3
+CHECKPOINT=$4
+GPUS=${GPUS:-8}
+GPUS_PER_NODE=${GPUS_PER_NODE:-8}
+CPUS_PER_TASK=${CPUS_PER_TASK:-5}
+PY_ARGS=${@:5}
+SRUN_ARGS=${SRUN_ARGS:-""}
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+srun -p ${PARTITION} \
+    --job-name=${JOB_NAME} \
+    --gres=gpu:${GPUS_PER_NODE} \
+    --ntasks=${GPUS} \
+    --ntasks-per-node=${GPUS_PER_NODE} \
+    --cpus-per-task=${CPUS_PER_TASK} \
+    --kill-on-bad-exit=1 \
+    ${SRUN_ARGS} \
+    python -u tools/test.py ${CONFIG} ${CHECKPOINT} --launcher="slurm" ${PY_ARGS}

third_party/hamer/third-party/ViTPose/tools/slurm_train.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Copyright (c) OpenMMLab. All rights reserved.
+
+set -x
+
+PARTITION=$1
+JOB_NAME=$2
+CONFIG=$3
+WORK_DIR=$4
+GPUS=${GPUS:-8}
+GPUS_PER_NODE=${GPUS_PER_NODE:-8}
+CPUS_PER_TASK=${CPUS_PER_TASK:-5}
+SRUN_ARGS=${SRUN_ARGS:-""}
+PY_ARGS=${@:5}
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+srun -p ${PARTITION} \
+    --job-name=${JOB_NAME} \
+    --gres=gpu:${GPUS_PER_NODE} \
+    --ntasks=${GPUS} \
+    --ntasks-per-node=${GPUS_PER_NODE} \
+    --cpus-per-task=${CPUS_PER_TASK} \
+    --kill-on-bad-exit=1 \
+    ${SRUN_ARGS} \
+    python -u tools/train.py ${CONFIG} --work-dir=${WORK_DIR} --launcher="slurm" ${PY_ARGS}

third_party/hamer/third-party/ViTPose/tools/test.py

@@ -0,0 +1,184 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import os
+import os.path as osp
+import warnings
+
+import mmcv
+import torch
+from mmcv import Config, DictAction
+from mmcv.cnn import fuse_conv_bn
+from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
+from mmcv.runner import get_dist_info, init_dist, load_checkpoint
+
+from mmpose.apis import multi_gpu_test, single_gpu_test
+from mmpose.datasets import build_dataloader, build_dataset
+from mmpose.models import build_posenet
+from mmpose.utils import setup_multi_processes
+
+try:
+    from mmcv.runner import wrap_fp16_model
+except ImportError:
+    warnings.warn('auto_fp16 from mmpose will be deprecated from v0.15.0'
+                  'Please install mmcv>=1.1.4')
+    from mmpose.core import wrap_fp16_model
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='mmpose test model')
+    parser.add_argument('config', help='test config file path')
+    parser.add_argument('checkpoint', help='checkpoint file')
+    parser.add_argument('--out', help='output result file')
+    parser.add_argument(
+        '--work-dir', help='the dir to save evaluation results')
+    parser.add_argument(
+        '--fuse-conv-bn',
+        action='store_true',
+        help='Whether to fuse conv and bn, this will slightly increase'
+        'the inference speed')
+    parser.add_argument(
+        '--gpu-id',
+        type=int,
+        default=0,
+        help='id of gpu to use '
+        '(only applicable to non-distributed testing)')
+    parser.add_argument(
+        '--eval',
+        default=None,
+        nargs='+',
+        help='evaluation metric, which depends on the dataset,'
+        ' e.g., "mAP" for MSCOCO')
+    parser.add_argument(
+        '--gpu_collect',
+        action='store_true',
+        help='whether to use gpu to collect results')
+    parser.add_argument('--tmpdir', help='tmp dir for writing some results')
+    parser.add_argument(
+        '--cfg-options',
+        nargs='+',
+        action=DictAction,
+        default={},
+        help='override some settings in the used config, the key-value pair '
+        'in xxx=yyy format will be merged into config file. For example, '
+        "'--cfg-options model.backbone.depth=18 model.backbone.with_cp=True'")
+    parser.add_argument(
+        '--launcher',
+        choices=['none', 'pytorch', 'slurm', 'mpi'],
+        default='none',
+        help='job launcher')
+    parser.add_argument('--local_rank', type=int, default=0)
+    args = parser.parse_args()
+    if 'LOCAL_RANK' not in os.environ:
+        os.environ['LOCAL_RANK'] = str(args.local_rank)
+    return args
+
+
+def merge_configs(cfg1, cfg2):
+    # Merge cfg2 into cfg1
+    # Overwrite cfg1 if repeated, ignore if value is None.
+    cfg1 = {} if cfg1 is None else cfg1.copy()
+    cfg2 = {} if cfg2 is None else cfg2
+    for k, v in cfg2.items():
+        if v:
+            cfg1[k] = v
+    return cfg1
+
+
+def main():
+    args = parse_args()
+
+    cfg = Config.fromfile(args.config)
+
+    if args.cfg_options is not None:
+        cfg.merge_from_dict(args.cfg_options)
+
+    # set multi-process settings
+    setup_multi_processes(cfg)
+
+    # set cudnn_benchmark
+    if cfg.get('cudnn_benchmark', False):
+        torch.backends.cudnn.benchmark = True
+    cfg.model.pretrained = None
+    cfg.data.test.test_mode = True
+
+    # work_dir is determined in this priority: CLI > segment in file > filename
+    if args.work_dir is not None:
+        # update configs according to CLI args if args.work_dir is not None
+        cfg.work_dir = args.work_dir
+    elif cfg.get('work_dir', None) is None:
+        # use config filename as default work_dir if cfg.work_dir is None
+        cfg.work_dir = osp.join('./work_dirs',
+                                osp.splitext(osp.basename(args.config))[0])
+
+    mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
+
+    # init distributed env first, since logger depends on the dist info.
+    if args.launcher == 'none':
+        distributed = False
+    else:
+        distributed = True
+        init_dist(args.launcher, **cfg.dist_params)
+
+    # build the dataloader
+    dataset = build_dataset(cfg.data.test, dict(test_mode=True))
+    # step 1: give default values and override (if exist) from cfg.data
+    loader_cfg = {
+        **dict(seed=cfg.get('seed'), drop_last=False, dist=distributed),
+        **({} if torch.__version__ != 'parrots' else dict(
+               prefetch_num=2,
+               pin_memory=False,
+           )),
+        **dict((k, cfg.data[k]) for k in [
+                   'seed',
+                   'prefetch_num',
+                   'pin_memory',
+                   'persistent_workers',
+               ] if k in cfg.data)
+    }
+    # step2: cfg.data.test_dataloader has higher priority
+    test_loader_cfg = {
+        **loader_cfg,
+        **dict(shuffle=False, drop_last=False),
+        **dict(workers_per_gpu=cfg.data.get('workers_per_gpu', 1)),
+        **dict(samples_per_gpu=cfg.data.get('samples_per_gpu', 1)),
+        **cfg.data.get('test_dataloader', {})
+    }
+    data_loader = build_dataloader(dataset, **test_loader_cfg)
+
+    # build the model and load checkpoint
+    model = build_posenet(cfg.model)
+    fp16_cfg = cfg.get('fp16', None)
+    if fp16_cfg is not None:
+        wrap_fp16_model(model)
+    load_checkpoint(model, args.checkpoint, map_location='cpu')
+
+    if args.fuse_conv_bn:
+        model = fuse_conv_bn(model)
+
+    if not distributed:
+        model = MMDataParallel(model, device_ids=[args.gpu_id])
+        outputs = single_gpu_test(model, data_loader)
+    else:
+        model = MMDistributedDataParallel(
+            model.cuda(),
+            device_ids=[torch.cuda.current_device()],
+            broadcast_buffers=False)
+        outputs = multi_gpu_test(model, data_loader, args.tmpdir,
+                                 args.gpu_collect)
+
+    rank, _ = get_dist_info()
+    eval_config = cfg.get('evaluation', {})
+    eval_config = merge_configs(eval_config, dict(metric=args.eval))
+
+    if rank == 0:
+        if args.out:
+            print(f'\nwriting results to {args.out}')
+            mmcv.dump(outputs, args.out)
+
+        results = dataset.evaluate(outputs, cfg.work_dir, **eval_config)
+        for k, v in sorted(results.items()):
+            print(f'{k}: {v}')
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/train.py

@@ -0,0 +1,195 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import copy
+import os
+import os.path as osp
+import time
+import warnings
+
+import mmcv
+import torch
+from mmcv import Config, DictAction
+from mmcv.runner import get_dist_info, init_dist, set_random_seed
+from mmcv.utils import get_git_hash
+
+from mmpose import __version__
+from mmpose.apis import init_random_seed, train_model
+from mmpose.datasets import build_dataset
+from mmpose.models import build_posenet
+from mmpose.utils import collect_env, get_root_logger, setup_multi_processes
+import mmcv_custom
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Train a pose model')
+    parser.add_argument('config', help='train config file path')
+    parser.add_argument('--work-dir', help='the dir to save logs and models')
+    parser.add_argument(
+        '--resume-from', help='the checkpoint file to resume from')
+    parser.add_argument(
+        '--no-validate',
+        action='store_true',
+        help='whether not to evaluate the checkpoint during training')
+    group_gpus = parser.add_mutually_exclusive_group()
+    group_gpus.add_argument(
+        '--gpus',
+        type=int,
+        help='(Deprecated, please use --gpu-id) number of gpus to use '
+        '(only applicable to non-distributed training)')
+    group_gpus.add_argument(
+        '--gpu-ids',
+        type=int,
+        nargs='+',
+        help='(Deprecated, please use --gpu-id) ids of gpus to use '
+        '(only applicable to non-distributed training)')
+    group_gpus.add_argument(
+        '--gpu-id',
+        type=int,
+        default=0,
+        help='id of gpu to use '
+        '(only applicable to non-distributed training)')
+    parser.add_argument('--seed', type=int, default=None, help='random seed')
+    parser.add_argument(
+        '--deterministic',
+        action='store_true',
+        help='whether to set deterministic options for CUDNN backend.')
+    parser.add_argument(
+        '--cfg-options',
+        nargs='+',
+        action=DictAction,
+        default={},
+        help='override some settings in the used config, the key-value pair '
+        'in xxx=yyy format will be merged into config file. For example, '
+        "'--cfg-options model.backbone.depth=18 model.backbone.with_cp=True'")
+    parser.add_argument(
+        '--launcher',
+        choices=['none', 'pytorch', 'slurm', 'mpi'],
+        default='none',
+        help='job launcher')
+    parser.add_argument('--local_rank', type=int, default=0)
+    parser.add_argument(
+        '--autoscale-lr',
+        action='store_true',
+        help='automatically scale lr with the number of gpus')
+    args = parser.parse_args()
+    if 'LOCAL_RANK' not in os.environ:
+        os.environ['LOCAL_RANK'] = str(args.local_rank)
+
+    return args
+
+
+def main():
+    args = parse_args()
+
+    cfg = Config.fromfile(args.config)
+
+    if args.cfg_options is not None:
+        cfg.merge_from_dict(args.cfg_options)
+
+    # set multi-process settings
+    setup_multi_processes(cfg)
+
+    # set cudnn_benchmark
+    if cfg.get('cudnn_benchmark', False):
+        torch.backends.cudnn.benchmark = True
+
+    # work_dir is determined in this priority: CLI > segment in file > filename
+    if args.work_dir is not None:
+        # update configs according to CLI args if args.work_dir is not None
+        cfg.work_dir = args.work_dir
+    elif cfg.get('work_dir', None) is None:
+        # use config filename as default work_dir if cfg.work_dir is None
+        cfg.work_dir = osp.join('./work_dirs',
+                                osp.splitext(osp.basename(args.config))[0])
+    if args.resume_from is not None:
+        cfg.resume_from = args.resume_from
+    if args.gpus is not None:
+        cfg.gpu_ids = range(1)
+        warnings.warn('`--gpus` is deprecated because we only support '
+                      'single GPU mode in non-distributed training. '
+                      'Use `gpus=1` now.')
+    if args.gpu_ids is not None:
+        cfg.gpu_ids = args.gpu_ids[0:1]
+        warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. '
+                      'Because we only support single GPU mode in '
+                      'non-distributed training. Use the first GPU '
+                      'in `gpu_ids` now.')
+    if args.gpus is None and args.gpu_ids is None:
+        cfg.gpu_ids = [args.gpu_id]
+
+    if args.autoscale_lr:
+        # apply the linear scaling rule (https://arxiv.org/abs/1706.02677)
+        cfg.optimizer['lr'] = cfg.optimizer['lr'] * len(cfg.gpu_ids) / 8
+
+    # init distributed env first, since logger depends on the dist info.
+    if args.launcher == 'none':
+        distributed = False
+        if len(cfg.gpu_ids) > 1:
+            warnings.warn(
+                f'We treat {cfg.gpu_ids} as gpu-ids, and reset to '
+                f'{cfg.gpu_ids[0:1]} as gpu-ids to avoid potential error in '
+                'non-distribute training time.')
+            cfg.gpu_ids = cfg.gpu_ids[0:1]
+    else:
+        distributed = True
+        init_dist(args.launcher, **cfg.dist_params)
+        # re-set gpu_ids with distributed training mode
+        _, world_size = get_dist_info()
+        cfg.gpu_ids = range(world_size)
+
+    # create work_dir
+    mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
+    # init the logger before other steps
+    timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
+    log_file = osp.join(cfg.work_dir, f'{timestamp}.log')
+    logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
+
+    # init the meta dict to record some important information such as
+    # environment info and seed, which will be logged
+    meta = dict()
+    # log env info
+    env_info_dict = collect_env()
+    env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()])
+    dash_line = '-' * 60 + '\n'
+    logger.info('Environment info:\n' + dash_line + env_info + '\n' +
+                dash_line)
+    meta['env_info'] = env_info
+
+    # log some basic info
+    logger.info(f'Distributed training: {distributed}')
+    logger.info(f'Config:\n{cfg.pretty_text}')
+
+    # set random seeds
+    seed = init_random_seed(args.seed)
+    logger.info(f'Set random seed to {seed}, '
+                f'deterministic: {args.deterministic}')
+    set_random_seed(seed, deterministic=args.deterministic)
+    cfg.seed = seed
+    meta['seed'] = seed
+
+    model = build_posenet(cfg.model)
+    datasets = [build_dataset(cfg.data.train)]
+
+    if len(cfg.workflow) == 2:
+        val_dataset = copy.deepcopy(cfg.data.val)
+        val_dataset.pipeline = cfg.data.train.pipeline
+        datasets.append(build_dataset(val_dataset))
+
+    if cfg.checkpoint_config is not None:
+        # save mmpose version, config file content
+        # checkpoints as meta data
+        cfg.checkpoint_config.meta = dict(
+            mmpose_version=__version__ + get_git_hash(digits=7),
+            config=cfg.pretty_text,
+        )
+    train_model(
+        model,
+        datasets,
+        cfg,
+        distributed=distributed,
+        validate=(not args.no_validate),
+        timestamp=timestamp,
+        meta=meta)
+
+
+if __name__ == '__main__':
+    main()

third_party/hamer/third-party/ViTPose/tools/webcam/README.md

@@ -0,0 +1,28 @@
+# MMPose Webcam API
+
+MMPose Webcam API is a handy tool to develop interactive webcam applications with MMPose functions.
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153800450-2522efe8-bc11-457d-9037-d8aee4fc4f36.png">
+<figcaption>MMPose Webcam API Overview</figcaption>
+</div>
+
+## Requirements
+
+* Python >= 3.7.0
+* MMPose >= 0.23.0
+* MMDetection >= 2.21.0
+
+## Tutorials
+
+* [Get started with MMPose Webcam API (Chinese)](/tools/webcam/docs/get_started_cn.md)
+* [Build a Webcam App: A Step-by-step Instruction (Chinese)](/tools/webcam/docs/example_cn.md)
+
+## Examples
+
+* [Pose Estimation](/tools/webcam/configs/examples/): A simple example to estimate and visualize human/animal pose.
+* [Eye Effects](/tools/webcam/configs/eyes/): Apply sunglasses and bug-eye effects.
+* [Face Swap](/tools/webcam/configs/face_swap/): Everybody gets someone else's face.
+* [Meow Dwen Dwen](/tools/webcam/configs/meow_dwen_dwen/): Dress up your cat in Bing Dwen Dwen costume.
+* [Super Saiyan](/tools/webcam/configs/supersaiyan/): Super Saiyan transformation!
+* [New Year](/tools/webcam/configs/newyear/): Set off some firecrackers to celebrate Chinese New Year.

third_party/hamer/third-party/ViTPose/tools/webcam/configs/background/README.md

@@ -0,0 +1,73 @@
+# Matting Effects
+
+We can apply background matting to the videos.
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/background/background.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| b | Toggle the background matting effect on/off. |
+| h | Show help information. |
+| m | Show the monitoring information. |
+| q | Exit. |
+
+Note that the demo will automatically save the output video into a file `record.mp4`.
+
+### Configuration
+
+- **Choose a detection model**
+
+Users can choose detection models from the [MMDetection Model Zoo](https://mmdetection.readthedocs.io/en/v2.20.0/model_zoo.html). Just set the `model_config` and `model_checkpoint` in the detector node accordingly, and the model will be automatically downloaded and loaded.
+Note that in order to perform background matting, the model should be able to produce segmentation masks.
+
+```python
+# 'DetectorNode':
+# This node performs object detection from the frame image using an
+# MMDetection model.
+dict(
+    type='DetectorNode',
+    name='Detector',
+    model_config='demo/mmdetection_cfg/mask_rcnn_r50_fpn_2x_coco.py',
+    model_checkpoint='https://download.openmmlab.com/'
+    'mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_2x_coco/'
+    'mask_rcnn_r50_fpn_2x_coco_bbox_mAP-0.392'
+    '__segm_mAP-0.354_20200505_003907-3e542a40.pth',
+    input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+    output_buffer='det_result'),
+```
+
+- **Run the demo without GPU**
+
+If you don't have GPU and CUDA in your device, the demo can run with only CPU by setting `device='cpu'` in all model nodes. For example:
+
+```python
+dict(
+    type='DetectorNode',
+    name='Detector',
+    model_config='demo/mmdetection_cfg/mask_rcnn_r50_fpn_2x_coco.py',
+    model_checkpoint='https://download.openmmlab.com/'
+    'mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_2x_coco/'
+    'mask_rcnn_r50_fpn_2x_coco_bbox_mAP-0.392'
+    '__segm_mAP-0.354_20200505_003907-3e542a40.pth',
+    device='cpu',
+    input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+    output_buffer='det_result'),
+```
+
+- **Debug webcam and display**
+
+You can launch the webcam runner with a debug config:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/examples/test_camera.py
+```

third_party/hamer/third-party/ViTPose/tools/webcam/configs/background/background.py

@@ -0,0 +1,93 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Matting Effects',
+    camera_id=0,
+    camera_fps=10,
+    synchronous=False,
+    # Define nodes.
+    # The configuration of a node usually includes:
+    #   1. 'type': Node class name
+    #   2. 'name': Node name
+    #   3. I/O buffers (e.g. 'input_buffer', 'output_buffer'): specify the
+    #       input and output buffer names. This may depend on the node class.
+    #   4. 'enable_key': assign a hot-key to toggle enable/disable this node.
+    #       This may depend on the node class.
+    #   5. Other class-specific arguments
+    nodes=[
+        # 'DetectorNode':
+        # This node performs object detection from the frame image using an
+        # MMDetection model.
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/mask_rcnn_r50_fpn_2x_coco.py',
+            model_checkpoint='https://download.openmmlab.com/'
+            'mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_2x_coco/'
+            'mask_rcnn_r50_fpn_2x_coco_bbox_mAP-0.392'
+            '__segm_mAP-0.354_20200505_003907-3e542a40.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        # 'TopDownPoseEstimatorNode':
+        # This node performs keypoint detection from the frame image using an
+        # MMPose top-down model. Detection results is needed.
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Human Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://openmmlab-share.oss-cn-hangz'
+            'hou.aliyuncs.com/mmpose/top_down/vipnas/vipnas_mbv3_co'
+            'co_wholebody_256x192_dark-e2158108_20211205.pth',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='human_pose'),
+        # 'ModelResultBindingNode':
+        # This node binds the latest model inference result with the current
+        # frame. (This means the frame image and inference result may be
+        # asynchronous).
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='human_pose',
+            output_buffer='frame'),
+        # 'MattingNode':
+        # This node draw the matting visualization result in the frame image.
+        # mask results is needed.
+        dict(
+            type='BackgroundNode',
+            name='Visualizer',
+            enable_key='b',
+            enable=True,
+            frame_buffer='frame',
+            output_buffer='vis_bg',
+            cls_names=['person']),
+        # 'NoticeBoardNode':
+        # This node show a notice board with given content, e.g. help
+        # information.
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            frame_buffer='vis_bg',
+            output_buffer='vis',
+            content_lines=[
+                'This is a demo for background changing effects. Have fun!',
+                '', 'Hot-keys:', '"b": Change background',
+                '"h": Show help information',
+                '"m": Show diagnostic information', '"q": Exit'
+            ],
+        ),
+        # 'MonitorNode':
+        # This node show diagnostic information in the frame image. It can
+        # be used for debugging or monitoring system resource status.
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis',
+            output_buffer='_display_')  # `_frame_` is a runner-reserved buffer
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/examples/README.md

@@ -0,0 +1,110 @@
+# Pose Estimation Demo
+
+This demo performs human bounding box and keypoint detection, and visualizes results.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/15977946/148911749-57b263c0-1075-4a65-af53-b51fc815da68.gif" width="600px" alt><br>
+</div>
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/examples/pose_estimation.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| v | Toggle the pose visualization on/off. |
+| h | Show help information. |
+| m | Show the monitoring information. |
+| q | Exit. |
+
+Note that the demo will automatically save the output video into a file `record.mp4`.
+
+### Configuration
+
+- **Choose a detection model**
+
+Users can choose detection models from the [MMDetection Model Zoo](https://mmdetection.readthedocs.io/en/v2.20.0/model_zoo.html). Just set the `model_config` and `model_checkpoint` in the detector node accordingly, and the model will be automatically downloaded and loaded.
+
+```python
+# 'DetectorNode':
+    # This node performs object detection from the frame image using an
+    # MMDetection model.
+dict(
+    type='DetectorNode',
+    name='Detector',
+    model_config='demo/mmdetection_cfg/'
+    'ssdlite_mobilenetv2_scratch_600e_coco.py',
+    model_checkpoint='https://download.openmmlab.com'
+    '/mmdetection/v2.0/ssd/'
+    'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+    'scratch_600e_coco_20210629_110627-974d9307.pth',
+    input_buffer='_input_',
+    output_buffer='det_result')
+```
+
+- **Choose a or more pose models**
+
+In this demo we use two [top-down](https://github.com/open-mmlab/mmpose/tree/master/configs/body/2d_kpt_sview_rgb_img/topdown_heatmap) pose estimation models for humans and animals respectively. Users can choose models from the [MMPose Model Zoo](https://mmpose.readthedocs.io/en/latest/modelzoo.html). To apply different pose models on different instance types, you can add multiple pose estimator nodes with `cls_names` set accordingly.
+
+```python
+# 'TopDownPoseEstimatorNode':
+# This node performs keypoint detection from the frame image using an
+# MMPose top-down model. Detection results is needed.
+dict(
+    type='TopDownPoseEstimatorNode',
+    name='Human Pose Estimator',
+    model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+    'topdown_heatmap/coco-wholebody/'
+    'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+    model_checkpoint='https://openmmlab-share.oss-cn-hangz'
+    'hou.aliyuncs.com/mmpose/top_down/vipnas/vipnas_mbv3_co'
+    'co_wholebody_256x192_dark-e2158108_20211205.pth',
+    cls_names=['person'],
+    input_buffer='det_result',
+    output_buffer='human_pose'),
+dict(
+    type='TopDownPoseEstimatorNode',
+    name='Animal Pose Estimator',
+    model_config='configs/animal/2d_kpt_sview_rgb_img/topdown_heatmap'
+    '/animalpose/hrnet_w32_animalpose_256x256.py',
+    model_checkpoint='https://download.openmmlab.com/mmpose/animal/'
+    'hrnet/hrnet_w32_animalpose_256x256-1aa7f075_20210426.pth',
+    cls_names=['cat', 'dog', 'horse', 'sheep', 'cow'],
+    input_buffer='human_pose',
+    output_buffer='animal_pose')
+```
+
+- **Run the demo without GPU**
+
+If you don't have GPU and CUDA in your device, the demo can run with only CPU by setting `device='cpu'` in all model nodes. For example:
+
+```python
+dict(
+    type='DetectorNode',
+    name='Detector',
+    model_config='demo/mmdetection_cfg/'
+    'ssdlite_mobilenetv2_scratch_600e_coco.py',
+    model_checkpoint='https://download.openmmlab.com'
+    '/mmdetection/v2.0/ssd/'
+    'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+    'scratch_600e_coco_20210629_110627-974d9307.pth',
+    device='cpu',
+    input_buffer='_input_',
+    output_buffer='det_result')
+```
+
+- **Debug webcam and display**
+
+You can lanch the webcam runner with a debug config:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/examples/test_camera.py
+```

third_party/hamer/third-party/ViTPose/tools/webcam/configs/examples/pose_estimation.py

@@ -0,0 +1,115 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Pose Estimation',
+    camera_id=0,
+    camera_fps=20,
+    synchronous=False,
+    # Define nodes.
+    # The configuration of a node usually includes:
+    #   1. 'type': Node class name
+    #   2. 'name': Node name
+    #   3. I/O buffers (e.g. 'input_buffer', 'output_buffer'): specify the
+    #       input and output buffer names. This may depend on the node class.
+    #   4. 'enable_key': assign a hot-key to toggle enable/disable this node.
+    #       This may depend on the node class.
+    #   5. Other class-specific arguments
+    nodes=[
+        # 'DetectorNode':
+        # This node performs object detection from the frame image using an
+        # MMDetection model.
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/'
+            'ssdlite_mobilenetv2_scratch_600e_coco.py',
+            model_checkpoint='https://download.openmmlab.com'
+            '/mmdetection/v2.0/ssd/'
+            'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+            'scratch_600e_coco_20210629_110627-974d9307.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        # 'TopDownPoseEstimatorNode':
+        # This node performs keypoint detection from the frame image using an
+        # MMPose top-down model. Detection results is needed.
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Human Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://download.openmmlab.com/mmpose/top_down/'
+            'vipnas/vipnas_mbv3_coco_wholebody_256x192_dark'
+            '-e2158108_20211205.pth',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='human_pose'),
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Animal Pose Estimator',
+            model_config='configs/animal/2d_kpt_sview_rgb_img/topdown_heatmap'
+            '/animalpose/hrnet_w32_animalpose_256x256.py',
+            model_checkpoint='https://download.openmmlab.com/mmpose/animal/'
+            'hrnet/hrnet_w32_animalpose_256x256-1aa7f075_20210426.pth',
+            cls_names=['cat', 'dog', 'horse', 'sheep', 'cow'],
+            input_buffer='human_pose',
+            output_buffer='animal_pose'),
+        # 'ModelResultBindingNode':
+        # This node binds the latest model inference result with the current
+        # frame. (This means the frame image and inference result may be
+        # asynchronous).
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='animal_pose',
+            output_buffer='frame'),
+        # 'PoseVisualizerNode':
+        # This node draw the pose visualization result in the frame image.
+        # Pose results is needed.
+        dict(
+            type='PoseVisualizerNode',
+            name='Visualizer',
+            enable_key='v',
+            frame_buffer='frame',
+            output_buffer='vis'),
+        # 'NoticeBoardNode':
+        # This node show a notice board with given content, e.g. help
+        # information.
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            enable=True,
+            frame_buffer='vis',
+            output_buffer='vis_notice',
+            content_lines=[
+                'This is a demo for pose visualization and simple image '
+                'effects. Have fun!', '', 'Hot-keys:',
+                '"v": Pose estimation result visualization',
+                '"s": Sunglasses effect B-)', '"b": Bug-eye effect 0_0',
+                '"h": Show help information',
+                '"m": Show diagnostic information', '"q": Exit'
+            ],
+        ),
+        # 'MonitorNode':
+        # This node show diagnostic information in the frame image. It can
+        # be used for debugging or monitoring system resource status.
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis_notice',
+            output_buffer='display'),
+        # 'RecorderNode':
+        # This node save the output video into a file.
+        dict(
+            type='RecorderNode',
+            name='Recorder',
+            out_video_file='record.mp4',
+            frame_buffer='display',
+            output_buffer='_display_'
+            # `_display_` is a runner-reserved buffer
+        )
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/examples/test_camera.py

@@ -0,0 +1,19 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    name='Debug CamRunner',
+    camera_id=0,
+    camera_fps=20,
+    nodes=[
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            frame_buffer='_frame_',
+            output_buffer='display'),
+        dict(
+            type='RecorderNode',
+            name='Recorder',
+            out_video_file='webcam_output.mp4',
+            frame_buffer='display',
+            output_buffer='_display_')
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/eyes/README.md

@@ -0,0 +1,31 @@
+# Sunglasses and Bug-eye Effects
+
+We can apply fun effects on videos with pose estimation results, like adding sunglasses on the face, or make the eyes look bigger.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/15977946/124059525-ce20c580-da5d-11eb-8e4a-2d96cd31fe9f.gif" width="600px" alt><br>
+</div>
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/examples/pose_estimation.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| s | Toggle the sunglasses effect on/off. |
+| b | Toggle the bug-eye effect on/off. |
+| h | Show help information. |
+| m | Show the monitoring information. |
+| q | Exit. |
+
+### Configuration
+
+See the [README](/tools/webcam/configs/examples/README.md#configuration) of pose estimation demo for model configurations.

third_party/hamer/third-party/ViTPose/tools/webcam/configs/eyes/eyes.py

@@ -0,0 +1,114 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Eye Effects',
+    camera_id=0,
+    camera_fps=20,
+    synchronous=False,
+    # Define nodes.
+    # The configuration of a node usually includes:
+    #   1. 'type': Node class name
+    #   2. 'name': Node name
+    #   3. I/O buffers (e.g. 'input_buffer', 'output_buffer'): specify the
+    #       input and output buffer names. This may depend on the node class.
+    #   4. 'enable_key': assign a hot-key to toggle enable/disable this node.
+    #       This may depend on the node class.
+    #   5. Other class-specific arguments
+    nodes=[
+        # 'DetectorNode':
+        # This node performs object detection from the frame image using an
+        # MMDetection model.
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/'
+            'ssdlite_mobilenetv2_scratch_600e_coco.py',
+            model_checkpoint='https://download.openmmlab.com'
+            '/mmdetection/v2.0/ssd/'
+            'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+            'scratch_600e_coco_20210629_110627-974d9307.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        # 'TopDownPoseEstimatorNode':
+        # This node performs keypoint detection from the frame image using an
+        # MMPose top-down model. Detection results is needed.
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Human Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://openmmlab-share.oss-cn-hangz'
+            'hou.aliyuncs.com/mmpose/top_down/vipnas/vipnas_mbv3_co'
+            'co_wholebody_256x192_dark-e2158108_20211205.pth',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='human_pose'),
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Animal Pose Estimator',
+            model_config='configs/animal/2d_kpt_sview_rgb_img/topdown_heatmap'
+            '/animalpose/hrnet_w32_animalpose_256x256.py',
+            model_checkpoint='https://download.openmmlab.com/mmpose/animal/'
+            'hrnet/hrnet_w32_animalpose_256x256-1aa7f075_20210426.pth',
+            cls_names=['cat', 'dog', 'horse', 'sheep', 'cow'],
+            input_buffer='human_pose',
+            output_buffer='animal_pose'),
+        # 'ModelResultBindingNode':
+        # This node binds the latest model inference result with the current
+        # frame. (This means the frame image and inference result may be
+        # asynchronous).
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='animal_pose',
+            output_buffer='frame'),
+        # 'SunglassesNode':
+        # This node draw the sunglasses effect in the frame image.
+        # Pose results is needed.
+        dict(
+            type='SunglassesNode',
+            name='Visualizer',
+            enable_key='s',
+            enable=True,
+            frame_buffer='frame',
+            output_buffer='vis_sunglasses'),
+        # 'BugEyeNode':
+        # This node draw the bug-eye effetc in the frame image.
+        # Pose results is needed.
+        dict(
+            type='BugEyeNode',
+            name='Visualizer',
+            enable_key='b',
+            enable=False,
+            frame_buffer='vis_sunglasses',
+            output_buffer='vis_bugeye'),
+        # 'NoticeBoardNode':
+        # This node show a notice board with given content, e.g. help
+        # information.
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            frame_buffer='vis_bugeye',
+            output_buffer='vis',
+            content_lines=[
+                'This is a demo for pose visualization and simple image '
+                'effects. Have fun!', '', 'Hot-keys:',
+                '"s": Sunglasses effect B-)', '"b": Bug-eye effect 0_0',
+                '"h": Show help information',
+                '"m": Show diagnostic information', '"q": Exit'
+            ],
+        ),
+        # 'MonitorNode':
+        # This node show diagnostic information in the frame image. It can
+        # be used for debugging or monitoring system resource status.
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis',
+            output_buffer='_display_')  # `_frame_` is a runner-reserved buffer
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/face_swap/README.md

@@ -0,0 +1,31 @@
+# Sunglasses and Bug-eye Effects
+
+Look! Where is my face?:eyes: And whose face is it?:laughing:
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/15977946/149705000-8818d55c-9264-409f-b995-77718cc6c822.gif" width="600px" alt><br>
+</div>
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/face_swap/face_swap.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| s | Switch between modes <ul><li>Shuffle: Randomly shuffle all faces</li><li>Clone: Choose one face and clone it for everyone</li><li>None: Nothing happens and everyone is safe :)</li></ul>|
+| v | Toggle the pose visualization on/off. |
+| h | Show help information. |
+| m | Show diagnostic information. |
+| q | Exit. |
+
+### Configuration
+
+See the [README](/tools/webcam/configs/examples/README.md#configuration) of pose estimation demo for model configurations.

third_party/hamer/third-party/ViTPose/tools/webcam/configs/face_swap/face_swap.py

@@ -0,0 +1,79 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    name='FaceSwap',
+    camera_id=0,
+    camera_fps=20,
+    synchronous=False,
+    nodes=[
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/'
+            'ssdlite_mobilenetv2_scratch_600e_coco.py',
+            model_checkpoint='https://download.openmmlab.com'
+            '/mmdetection/v2.0/ssd/'
+            'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+            'scratch_600e_coco_20210629_110627-974d9307.pth',
+            device='cpu',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='TopDown Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_res50_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://openmmlab-share.oss-cn-hangzhou'
+            '.aliyuncs.com/mmpose/top_down/vipnas/'
+            'vipnas_res50_wholebody_256x192_dark-67c0ce35_20211112.pth',
+            device='cpu',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='pose_result'),
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='pose_result',
+            output_buffer='frame'),
+        dict(
+            type='FaceSwapNode',
+            name='FaceSwapper',
+            mode_key='s',
+            frame_buffer='frame',
+            output_buffer='face_swap'),
+        dict(
+            type='PoseVisualizerNode',
+            name='Visualizer',
+            enable_key='v',
+            frame_buffer='face_swap',
+            output_buffer='vis_pose'),
+        dict(
+            type='NoticeBoardNode',
+            name='Help Information',
+            enable_key='h',
+            content_lines=[
+                'Swap your faces! ',
+                'Hot-keys:',
+                '"v": Toggle the pose visualization on/off.',
+                '"s": Switch between modes: Shuffle, Clone and None',
+                '"h": Show help information',
+                '"m": Show diagnostic information',
+                '"q": Exit',
+            ],
+            frame_buffer='vis_pose',
+            output_buffer='vis_notice'),
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis_notice',
+            output_buffer='display'),
+        dict(
+            type='RecorderNode',
+            name='Recorder',
+            out_video_file='faceswap_output.mp4',
+            frame_buffer='display',
+            output_buffer='_display_')
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/meow_dwen_dwen/README.md

@@ -0,0 +1,44 @@
+# Meow Dwen Dwen
+
+Do you know [Bing DwenDwen (冰墩墩)](https://en.wikipedia.org/wiki/Bing_Dwen_Dwen_and_Shuey_Rhon_Rhon), the mascot of 2022 Beijing Olympic Games?
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/15977946/152742707-c0c51844-e1d0-42d0-9a12-e369002e082f.jpg" width="224px" alt><br>
+</div>
+
+Now you can dress your cat up in this costume and TA-DA! Be prepared for super cute **Meow Dwen Dwen**.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/15977946/152942243-a17194a2-0fd1-4467-993c-634f6d7966d8.gif" width="300px" alt><br>
+</div>
+
+You are a dog fan? Hold on, here comes Woof Dwen Dwen.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/15977946/152942747-84240692-3944-48a5-b60b-e60bd0a4339c.gif" width="300px" alt><br>
+</div>
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/meow_dwen_dwen/meow_dwen_dwen.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| s | Change the background. |
+| h | Show help information. |
+| m | Show diagnostic information. |
+| q | Exit. |
+
+### Configuration
+
+- **Use video input**
+
+As you can see in the config, we set `camera_id` as the path of the input image. You can also set it as a video file path (or url), or a webcam ID number (e.g. `camera_id=0`), to capture the dynamic face from the video input.

third_party/hamer/third-party/ViTPose/tools/webcam/configs/meow_dwen_dwen/meow_dwen_dwen.py

@@ -0,0 +1,92 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Little fans of 2022 Beijing Winter Olympics',
+    # Cat image
+    camera_id='https://user-images.githubusercontent.com/'
+    '15977946/152932036-b5554cf8-24cf-40d6-a358-35a106013f11.jpeg',
+    # Dog image
+    # camera_id='https://user-images.githubusercontent.com/'
+    # '15977946/152932051-cd280b35-8066-45a0-8f52-657c8631aaba.jpg',
+    camera_fps=20,
+    nodes=[
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/'
+            'ssdlite_mobilenetv2_scratch_600e_coco.py',
+            model_checkpoint='https://download.openmmlab.com'
+            '/mmdetection/v2.0/ssd/'
+            'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+            'scratch_600e_coco_20210629_110627-974d9307.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Animal Pose Estimator',
+            model_config='configs/animal/2d_kpt_sview_rgb_img/topdown_heatmap'
+            '/ap10k/hrnet_w32_ap10k_256x256.py',
+            model_checkpoint='https://download.openmmlab.com/mmpose/animal/'
+            'hrnet/hrnet_w32_ap10k_256x256-18aac840_20211029.pth',
+            cls_names=['cat', 'dog'],
+            input_buffer='det_result',
+            output_buffer='animal_pose'),
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='TopDown Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_res50_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://openmmlab-share.oss-cn-hangzhou'
+            '.aliyuncs.com/mmpose/top_down/vipnas/'
+            'vipnas_res50_wholebody_256x192_dark-67c0ce35_20211112.pth',
+            device='cpu',
+            cls_names=['person'],
+            input_buffer='animal_pose',
+            output_buffer='human_pose'),
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='human_pose',
+            output_buffer='frame'),
+        dict(
+            type='XDwenDwenNode',
+            name='XDwenDwen',
+            mode_key='s',
+            resource_file='tools/webcam/configs/meow_dwen_dwen/'
+            'resource-info.json',
+            out_shape=(480, 480),
+            frame_buffer='frame',
+            output_buffer='vis'),
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            enable=False,
+            frame_buffer='vis',
+            output_buffer='vis_notice',
+            content_lines=[
+                'Let your pet put on a costume of Bing-Dwen-Dwen, '
+                'the mascot of 2022 Beijing Winter Olympics. Have fun!', '',
+                'Hot-keys:', '"s": Change the background',
+                '"h": Show help information',
+                '"m": Show diagnostic information', '"q": Exit'
+            ],
+        ),
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis_notice',
+            output_buffer='display'),
+        dict(
+            type='RecorderNode',
+            name='Recorder',
+            out_video_file='record.mp4',
+            frame_buffer='display',
+            output_buffer='_display_'
+            # `_display_` is a runner-reserved buffer
+        )
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/meow_dwen_dwen/resource-info.json

@@ -0,0 +1,26 @@
+[
+    {
+        "id": 1,
+        "result": "{\"width\":690,\"height\":713,\"valid\":true,\"rotate\":0,\"step_1\":{\"toolName\":\"pointTool\",\"result\":[{\"x\":374.86387434554973,\"y\":262.8020942408377,\"attribute\":\"\",\"valid\":true,\"id\":\"8SK9cVyu\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":2},{\"x\":492.8261780104712,\"y\":285.2,\"attribute\":\"\",\"valid\":true,\"id\":\"qDk54WsI\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":1},{\"x\":430.11204188481673,\"y\":318.0502617801047,\"attribute\":\"\",\"valid\":true,\"id\":\"4H80L7lL\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":3}]},\"step_2\":{\"dataSourceStep\":0,\"toolName\":\"polygonTool\",\"result\":[{\"id\":\"pwUsrf9u\",\"sourceID\":\"\",\"valid\":true,\"textAttribute\":\"\",\"pointList\":[{\"x\":423.3926701570681,\"y\":191.87539267015708},{\"x\":488.3465968586388,\"y\":209.04712041884818},{\"x\":535.3821989528797,\"y\":248.6167539267016},{\"x\":549.5675392670157,\"y\":306.8513089005236},{\"x\":537.6219895287959,\"y\":349.407329842932},{\"x\":510.74450261780106,\"y\":381.51099476439794},{\"x\":480.1340314136126,\"y\":394.9497382198953},{\"x\":411.4471204188482,\"y\":390.47015706806286},{\"x\":355.45235602094243,\"y\":373.29842931937173},{\"x\":306.17696335078534,\"y\":327.00942408376966},{\"x\":294.97801047120424,\"y\":284.45340314136126},{\"x\":306.9235602094241,\"y\":245.6303664921466},{\"x\":333.8010471204189,\"y\":217.25968586387435},{\"x\":370.3842931937173,\"y\":196.35497382198955}],\"attribute\":\"\",\"order\":1}]}}",
+        "url": "https://user-images.githubusercontent.com/15977946/152742677-35fe8a01-bd06-4a12-a02e-949e7d71f28a.jpg",
+        "fileName": "bing_dwen_dwen1.jpg"
+    },
+    {
+        "id": 2,
+        "result": "{\"width\":690,\"height\":659,\"valid\":true,\"rotate\":0,\"step_1\":{\"dataSourceStep\":0,\"toolName\":\"pointTool\",\"result\":[{\"x\":293.2460732984293,\"y\":242.89842931937173,\"attribute\":\"\",\"valid\":true,\"id\":\"KgPs39bY\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":1},{\"x\":170.41675392670155,\"y\":270.50052356020944,\"attribute\":\"\",\"valid\":true,\"id\":\"XwHyoBFU\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":2},{\"x\":224.24083769633506,\"y\":308.45340314136126,\"attribute\":\"\",\"valid\":true,\"id\":\"Qfs4YfuB\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":3}]},\"step_2\":{\"dataSourceStep\":0,\"toolName\":\"polygonTool\",\"result\":[{\"id\":\"ts5jlJxb\",\"sourceID\":\"\",\"valid\":true,\"textAttribute\":\"\",\"pointList\":[{\"x\":178.69738219895285,\"y\":184.93403141361256},{\"x\":204.91937172774865,\"y\":172.5130890052356},{\"x\":252.5329842931937,\"y\":169.0628272251309},{\"x\":295.3162303664921,\"y\":175.27329842931937},{\"x\":333.95916230366487,\"y\":195.2848167539267},{\"x\":360.18115183246067,\"y\":220.1267015706806},{\"x\":376.0523560209424,\"y\":262.909947643979},{\"x\":373.98219895287957,\"y\":296.0324607329843},{\"x\":344.99999999999994,\"y\":335.365445026178},{\"x\":322.22827225130885,\"y\":355.37696335078533},{\"x\":272.544502617801,\"y\":378.1486910994764},{\"x\":221.48062827225127,\"y\":386.42931937172773},{\"x\":187.6680628272251,\"y\":385.7392670157068},{\"x\":158.68586387434553,\"y\":369.1780104712042},{\"x\":137.98429319371724,\"y\":337.43560209424083},{\"x\":127.63350785340312,\"y\":295.34240837696336},{\"x\":131.0837696335078,\"y\":242.89842931937173},{\"x\":147.64502617801045,\"y\":208.3958115183246}],\"attribute\":\"\",\"order\":1}]}}",
+        "url": "https://user-images.githubusercontent.com/15977946/152742707-c0c51844-e1d0-42d0-9a12-e369002e082f.jpg",
+        "fileName": "bing_dwen_dwen2.jpg"
+    },
+    {
+        "id": 3,
+        "result": "{\"width\":690,\"height\":811,\"valid\":true,\"rotate\":0,\"step_1\":{\"dataSourceStep\":0,\"toolName\":\"pointTool\",\"result\":[{\"x\":361.13507853403144,\"y\":300.62198952879584,\"attribute\":\"\",\"valid\":true,\"id\":\"uAtbXtf2\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":1},{\"x\":242.24502617801048,\"y\":317.60628272251313,\"attribute\":\"\",\"valid\":true,\"id\":\"iLtceHMA\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":2},{\"x\":302.5392670157068,\"y\":356.67015706806285,\"attribute\":\"\",\"valid\":true,\"id\":\"n9MTlJ6A\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":3}]},\"step_2\":{\"dataSourceStep\":0,\"toolName\":\"polygonTool\",\"result\":[{\"id\":\"5sTLU5wF\",\"sourceID\":\"\",\"valid\":true,\"textAttribute\":\"\",\"pointList\":[{\"x\":227.80837696335078,\"y\":247.12146596858642},{\"x\":248.18952879581153,\"y\":235.23246073298432},{\"x\":291.4994764397906,\"y\":225.04188481675394},{\"x\":351.7937172774869,\"y\":229.28795811518327},{\"x\":393.40523560209425,\"y\":245.42303664921468},{\"x\":424.8261780104712,\"y\":272.59790575916236},{\"x\":443.5089005235602,\"y\":298.07434554973827},{\"x\":436.7151832460733,\"y\":345.6303664921466},{\"x\":406.1434554973822,\"y\":382.9958115183247},{\"x\":355.1905759162304,\"y\":408.4722513089006},{\"x\":313.57905759162304,\"y\":419.5120418848168},{\"x\":262.6261780104712,\"y\":417.81361256544506},{\"x\":224.41151832460733,\"y\":399.9801047120419},{\"x\":201.48272251308902,\"y\":364.3130890052356},{\"x\":194.68900523560208,\"y\":315.0586387434555},{\"x\":202.33193717277487,\"y\":272.59790575916236}],\"attribute\":\"\",\"order\":1}]}}",
+        "url": "https://user-images.githubusercontent.com/15977946/152742728-99392ecf-8f5c-46cf-b5c4-fe7fb6b39976.jpg",
+        "fileName": "bing_dwen_dwen3.jpg"
+    },
+    {
+        "id": 4,
+        "result": "{\"width\":690,\"height\":690,\"valid\":true,\"rotate\":0,\"step_1\":{\"dataSourceStep\":0,\"toolName\":\"pointTool\",\"result\":[{\"x\":365.9528795811519,\"y\":464.5759162303665,\"attribute\":\"\",\"valid\":true,\"id\":\"IKprTuHS\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":1},{\"x\":470.71727748691103,\"y\":445.06806282722516,\"attribute\":\"\",\"valid\":true,\"id\":\"Z90CWkEI\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":2},{\"x\":410.74869109947645,\"y\":395.2146596858639,\"attribute\":\"\",\"valid\":true,\"id\":\"UWRstKZk\",\"sourceID\":\"\",\"textAttribute\":\"\",\"order\":3}]},\"step_2\":{\"dataSourceStep\":0,\"toolName\":\"polygonTool\",\"result\":[{\"id\":\"C30Pc9Ww\",\"sourceID\":\"\",\"valid\":true,\"textAttribute\":\"\",\"pointList\":[{\"x\":412.91623036649213,\"y\":325.85340314136124},{\"x\":468.5497382198953,\"y\":335.9685863874345},{\"x\":501.78534031413614,\"y\":369.2041884816754},{\"x\":514.0680628272252,\"y\":415.44502617801044},{\"x\":504.67539267015707,\"y\":472.5235602094241},{\"x\":484.44502617801044,\"y\":497.0890052356021},{\"x\":443.26178010471205,\"y\":512.9842931937172},{\"x\":389.7958115183246,\"y\":518.7643979057591},{\"x\":336.32984293193715,\"y\":504.31413612565444},{\"x\":302.3717277486911,\"y\":462.40837696335075},{\"x\":298.0366492146597,\"y\":416.89005235602093},{\"x\":318.26701570680626,\"y\":372.0942408376963},{\"x\":363.0628272251309,\"y\":341.0261780104712}],\"attribute\":\"\",\"order\":1}]}}",
+        "url": "https://user-images.githubusercontent.com/15977946/152742755-9dc75f89-4156-4103-9c6d-f35f1f409d11.jpg",
+        "fileName": "bing_dwen_dwen4.jpg"
+    }
+]

third_party/hamer/third-party/ViTPose/tools/webcam/configs/newyear/README.md

@@ -0,0 +1,31 @@
+# New Year Hat and Firecracker Effects
+
+This demo provides new year effects with pose estimation results, like adding hat on the head and firecracker in the hands.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/28900607/149774252-8bff7738-2a54-4480-a31c-0a574e7159cb.gif" width="600px" alt><br>
+</div>
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/newyear/new_year.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| t | Toggle the hat effect on/off. |
+| f | Toggle the firecracker effect on/off. |
+| h | Show help information. |
+| m | Show the monitoring information. |
+| q | Exit. |
+
+### Configuration
+
+See the [README](/tools/webcam/configs/examples/README.md#configuration) of pose estimation demo for model configurations.

third_party/hamer/third-party/ViTPose/tools/webcam/configs/newyear/new_year.py

@@ -0,0 +1,122 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Pose Estimation',
+    camera_id=0,
+    camera_fps=20,
+    synchronous=False,
+    # Define nodes.
+    # The configuration of a node usually includes:
+    #   1. 'type': Node class name
+    #   2. 'name': Node name
+    #   3. I/O buffers (e.g. 'input_buffer', 'output_buffer'): specify the
+    #       input and output buffer names. This may depend on the node class.
+    #   4. 'enable_key': assign a hot-key to toggle enable/disable this node.
+    #       This may depend on the node class.
+    #   5. Other class-specific arguments
+    nodes=[
+        # 'DetectorNode':
+        # This node performs object detection from the frame image using an
+        # MMDetection model.
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/'
+            'ssdlite_mobilenetv2_scratch_600e_coco.py',
+            model_checkpoint='https://download.openmmlab.com'
+            '/mmdetection/v2.0/ssd/'
+            'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+            'scratch_600e_coco_20210629_110627-974d9307.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        # 'TopDownPoseEstimatorNode':
+        # This node performs keypoint detection from the frame image using an
+        # MMPose top-down model. Detection results is needed.
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Human Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://openmmlab-share.oss-cn-hangz'
+            'hou.aliyuncs.com/mmpose/top_down/vipnas/vipnas_mbv3_co'
+            'co_wholebody_256x192_dark-e2158108_20211205.pth',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='human_pose'),
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Animal Pose Estimator',
+            model_config='configs/animal/2d_kpt_sview_rgb_img/topdown_heatmap'
+            '/animalpose/hrnet_w32_animalpose_256x256.py',
+            model_checkpoint='https://download.openmmlab.com/mmpose/animal/'
+            'hrnet/hrnet_w32_animalpose_256x256-1aa7f075_20210426.pth',
+            cls_names=['cat', 'dog', 'horse', 'sheep', 'cow'],
+            input_buffer='human_pose',
+            output_buffer='animal_pose'),
+        # 'ModelResultBindingNode':
+        # This node binds the latest model inference result with the current
+        # frame. (This means the frame image and inference result may be
+        # asynchronous).
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='animal_pose',
+            output_buffer='frame'),
+        # 'HatNode':
+        # This node draw the hat effect in the frame image.
+        # Pose results is needed.
+        dict(
+            type='HatNode',
+            name='Visualizer',
+            enable_key='t',
+            frame_buffer='frame',
+            output_buffer='vis_hat'),
+        # 'FirecrackerNode':
+        # This node draw the firecracker effect in the frame image.
+        # Pose results is needed.
+        dict(
+            type='FirecrackerNode',
+            name='Visualizer',
+            enable_key='f',
+            frame_buffer='vis_hat',
+            output_buffer='vis_firecracker'),
+        # 'NoticeBoardNode':
+        # This node show a notice board with given content, e.g. help
+        # information.
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            enable=True,
+            frame_buffer='vis_firecracker',
+            output_buffer='vis_notice',
+            content_lines=[
+                'This is a demo for pose visualization and simple image '
+                'effects. Have fun!', '', 'Hot-keys:', '"t": Hat effect',
+                '"f": Firecracker effect', '"h": Show help information',
+                '"m": Show diagnostic information', '"q": Exit'
+            ],
+        ),
+        # 'MonitorNode':
+        # This node show diagnostic information in the frame image. It can
+        # be used for debugging or monitoring system resource status.
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis_notice',
+            output_buffer='display'),
+        # 'RecorderNode':
+        # This node save the output video into a file.
+        dict(
+            type='RecorderNode',
+            name='Recorder',
+            out_video_file='record.mp4',
+            frame_buffer='display',
+            output_buffer='_display_'
+            # `_display_` is a runner-reserved buffer
+        )
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/supersaiyan/README.md

@@ -0,0 +1,96 @@
+# Super Saiyan Effects
+
+We can apply fun effects on videos with pose estimation results, like Super Saiyan transformation.
+
+https://user-images.githubusercontent.com/11788150/150138076-2192079f-068a-4d43-bf27-2f1fd708cabc.mp4
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/supersaiyan/saiyan.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| s | Toggle the Super Saiyan effect on/off. |
+| h | Show help information. |
+| m | Show the monitoring information. |
+| q | Exit. |
+
+Note that the demo will automatically save the output video into a file `record.mp4`.
+
+### Configuration
+
+- **Choose a detection model**
+
+Users can choose detection models from the [MMDetection Model Zoo](https://mmdetection.readthedocs.io/en/v2.20.0/model_zoo.html). Just set the `model_config` and `model_checkpoint` in the detector node accordingly, and the model will be automatically downloaded and loaded.
+
+```python
+# 'DetectorNode':
+# This node performs object detection from the frame image using an
+# MMDetection model.
+dict(
+    type='DetectorNode',
+    name='Detector',
+    model_config='demo/mmdetection_cfg/mask_rcnn_r50_fpn_2x_coco.py',
+    model_checkpoint='https://download.openmmlab.com/'
+    'mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_2x_coco/'
+    'mask_rcnn_r50_fpn_2x_coco_bbox_mAP-0.392'
+    '__segm_mAP-0.354_20200505_003907-3e542a40.pth',
+    input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+    output_buffer='det_result'),
+```
+
+- **Choose a or more pose models**
+
+In this demo we use two [top-down](https://github.com/open-mmlab/mmpose/tree/master/configs/body/2d_kpt_sview_rgb_img/topdown_heatmap) pose estimation models for humans and animals respectively. Users can choose models from the [MMPose Model Zoo](https://mmpose.readthedocs.io/en/latest/modelzoo.html). To apply different pose models on different instance types, you can add multiple pose estimator nodes with `cls_names` set accordingly.
+
+```python
+# 'TopDownPoseEstimatorNode':
+# This node performs keypoint detection from the frame image using an
+# MMPose top-down model. Detection results is needed.
+dict(
+    type='TopDownPoseEstimatorNode',
+    name='Human Pose Estimator',
+    model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+    'topdown_heatmap/coco-wholebody/'
+    'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+    model_checkpoint='https://openmmlab-share.oss-cn-hangz'
+    'hou.aliyuncs.com/mmpose/top_down/vipnas/vipnas_mbv3_co'
+    'co_wholebody_256x192_dark-e2158108_20211205.pth',
+    cls_names=['person'],
+    input_buffer='det_result',
+    output_buffer='human_pose')
+```
+
+- **Run the demo without GPU**
+
+If you don't have GPU and CUDA in your device, the demo can run with only CPU by setting `device='cpu'` in all model nodes. For example:
+
+```python
+dict(
+    type='DetectorNode',
+    name='Detector',
+    model_config='demo/mmdetection_cfg/mask_rcnn_r50_fpn_2x_coco.py',
+    model_checkpoint='https://download.openmmlab.com/'
+    'mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_2x_coco/'
+    'mask_rcnn_r50_fpn_2x_coco_bbox_mAP-0.392'
+    '__segm_mAP-0.354_20200505_003907-3e542a40.pth',
+    device='cpu',
+    input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+    output_buffer='det_result'),
+```
+
+- **Debug webcam and display**
+
+You can launch the webcam runner with a debug config:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/examples/test_camera.py
+```

third_party/hamer/third-party/ViTPose/tools/webcam/configs/supersaiyan/saiyan.py

@@ -0,0 +1,93 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Super Saiyan Effects',
+    camera_id=0,
+    camera_fps=30,
+    synchronous=False,
+    # Define nodes.
+    # The configuration of a node usually includes:
+    #   1. 'type': Node class name
+    #   2. 'name': Node name
+    #   3. I/O buffers (e.g. 'input_buffer', 'output_buffer'): specify the
+    #       input and output buffer names. This may depend on the node class.
+    #   4. 'enable_key': assign a hot-key to toggle enable/disable this node.
+    #       This may depend on the node class.
+    #   5. Other class-specific arguments
+    nodes=[
+        # 'DetectorNode':
+        # This node performs object detection from the frame image using an
+        # MMDetection model.
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/mask_rcnn_r50_fpn_2x_coco.py',
+            model_checkpoint='https://download.openmmlab.com/'
+            'mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_2x_coco/'
+            'mask_rcnn_r50_fpn_2x_coco_bbox_mAP-0.392'
+            '__segm_mAP-0.354_20200505_003907-3e542a40.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        # 'TopDownPoseEstimatorNode':
+        # This node performs keypoint detection from the frame image using an
+        # MMPose top-down model. Detection results is needed.
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Human Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://openmmlab-share.oss-cn-hangz'
+            'hou.aliyuncs.com/mmpose/top_down/vipnas/vipnas_mbv3_co'
+            'co_wholebody_256x192_dark-e2158108_20211205.pth',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='human_pose'),
+        # 'ModelResultBindingNode':
+        # This node binds the latest model inference result with the current
+        # frame. (This means the frame image and inference result may be
+        # asynchronous).
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='human_pose',
+            output_buffer='frame'),
+        # 'SaiyanNode':
+        # This node draw the Super Saiyan effect in the frame image.
+        # Pose results is needed.
+        dict(
+            type='SaiyanNode',
+            name='Visualizer',
+            enable_key='s',
+            cls_names=['person'],
+            enable=True,
+            frame_buffer='frame',
+            output_buffer='vis_saiyan'),
+        # 'NoticeBoardNode':
+        # This node show a notice board with given content, e.g. help
+        # information.
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            frame_buffer='vis_saiyan',
+            output_buffer='vis',
+            content_lines=[
+                'This is a demo for super saiyan effects. Have fun!', '',
+                'Hot-keys:', '"s": Saiyan effect',
+                '"h": Show help information',
+                '"m": Show diagnostic information', '"q": Exit'
+            ],
+        ),
+        # 'MonitorNode':
+        # This node show diagnostic information in the frame image. It can
+        # be used for debugging or monitoring system resource status.
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis',
+            output_buffer='_display_')  # `_frame_` is a runner-reserved buffer
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/configs/valentinemagic/README.md

@@ -0,0 +1,35 @@
+# Valentine Magic
+
+Do you want to show your **love** to your beloved one, especially on Valentine's Day? Express it with your pose using MMPose right away and see the Valentine Magic!
+
+Try to pose a hand heart gesture, and see what will happen?
+
+Prefer a blow kiss? Here comes your flying heart~
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/87690686/153837208-8975eda1-17d5-4ee7-8c83-9b535e5ee9d9.gif" width="300px" alt><br>
+</div>
+
+## Instruction
+
+### Get started
+
+Launch the demo from the mmpose root directory:
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/valentinemagic/valentinemagic.py
+```
+
+### Hotkeys
+
+| Hotkey | Function |
+| -- | -- |
+| l | Toggle the Valentine Magic effect on/off. |
+| v | Toggle the pose visualization on/off. |
+| h | Show help information. |
+| m | Show diagnostic information. |
+| q | Exit. |
+
+### Configuration
+
+See the [README](/tools/webcam/configs/examples/README.md#configuration) of pose estimation demo for model configurations.

third_party/hamer/third-party/ViTPose/tools/webcam/configs/valentinemagic/valentinemagic.py

@@ -0,0 +1,118 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+runner = dict(
+    # Basic configurations of the runner
+    name='Human Pose and Effects',
+    camera_id=0,
+    camera_fps=30,
+
+    # Define nodes.
+    #
+    # The configuration of a node usually includes:
+    #   1. 'type': Node class name
+    #   2. 'name': Node name
+    #   3. I/O buffers (e.g. 'input_buffer', 'output_buffer'): specify the
+    #       input and output buffer names. This may depend on the node class.
+    #   4. 'enable_key': assign a hot-key to toggle enable/disable this node.
+    #       This may depend on the node class.
+    #   5. Other class-specific arguments
+    nodes=[
+        # 'DetectorNode':
+        # This node performs object detection from the frame image using an
+        # MMDetection model.
+        dict(
+            type='DetectorNode',
+            name='Detector',
+            model_config='demo/mmdetection_cfg/'
+            'ssdlite_mobilenetv2_scratch_600e_coco.py',
+            model_checkpoint='https://download.openmmlab.com'
+            '/mmdetection/v2.0/ssd/'
+            'ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_'
+            'scratch_600e_coco_20210629_110627-974d9307.pth',
+            input_buffer='_input_',  # `_input_` is a runner-reserved buffer
+            output_buffer='det_result'),
+        # 'TopDownPoseEstimatorNode':
+        # This node performs keypoint detection from the frame image using an
+        # MMPose top-down model. Detection results is needed.
+        dict(
+            type='TopDownPoseEstimatorNode',
+            name='Human Pose Estimator',
+            model_config='configs/wholebody/2d_kpt_sview_rgb_img/'
+            'topdown_heatmap/coco-wholebody/'
+            'vipnas_mbv3_coco_wholebody_256x192_dark.py',
+            model_checkpoint='https://download.openmmlab.com/mmpose/top_down/'
+            'vipnas/vipnas_mbv3_coco_wholebody_256x192_dark'
+            '-e2158108_20211205.pth',
+            cls_names=['person'],
+            input_buffer='det_result',
+            output_buffer='pose_result'),
+        # 'ModelResultBindingNode':
+        # This node binds the latest model inference result with the current
+        # frame. (This means the frame image and inference result may be
+        # asynchronous).
+        dict(
+            type='ModelResultBindingNode',
+            name='ResultBinder',
+            frame_buffer='_frame_',  # `_frame_` is a runner-reserved buffer
+            result_buffer='pose_result',
+            output_buffer='frame'),
+        # 'PoseVisualizerNode':
+        # This node draw the pose visualization result in the frame image.
+        # Pose results is needed.
+        dict(
+            type='PoseVisualizerNode',
+            name='Visualizer',
+            enable_key='v',
+            enable=False,
+            frame_buffer='frame',
+            output_buffer='vis'),
+        # 'ValentineMagicNode':
+        # This node draw heart in the image.
+        # It can launch dynamically expanding heart from the middle of
+        # hands if the persons pose a "hand heart" gesture or blow a kiss.
+        # Only there are two persons in the image can trigger this effect.
+        # Pose results is needed.
+        dict(
+            type='ValentineMagicNode',
+            name='Visualizer',
+            enable_key='l',
+            frame_buffer='vis',
+            output_buffer='vis_heart',
+        ),
+        # 'NoticeBoardNode':
+        # This node show a notice board with given content, e.g. help
+        # information.
+        dict(
+            type='NoticeBoardNode',
+            name='Helper',
+            enable_key='h',
+            enable=False,
+            frame_buffer='vis_heart',
+            output_buffer='vis_notice',
+            content_lines=[
+                'This is a demo for pose visualization and simple image '
+                'effects. Have fun!', '', 'Hot-keys:',
+                '"h": Show help information', '"l": LoveHeart Effect',
+                '"v": PoseVisualizer', '"m": Show diagnostic information',
+                '"q": Exit'
+            ],
+        ),
+        # 'MonitorNode':
+        # This node show diagnostic information in the frame image. It can
+        # be used for debugging or monitoring system resource status.
+        dict(
+            type='MonitorNode',
+            name='Monitor',
+            enable_key='m',
+            enable=False,
+            frame_buffer='vis_notice',
+            output_buffer='display'),  # `_frame_` is a runner-reserved buffer
+        # 'RecorderNode':
+        # This node record the frames into a local file. It can save the
+        # visualiztion results. Uncommit the following lines to turn it on.
+        dict(
+            type='RecorderNode',
+            name='Recorder',
+            out_video_file='record.mp4',
+            frame_buffer='display',
+            output_buffer='_display_')
+    ])

third_party/hamer/third-party/ViTPose/tools/webcam/docs/example_cn.md

@@ -0,0 +1,171 @@
+# 开发示例：给猫咪戴上太阳镜
+
+## 设计思路
+
+在动手之前，我们先考虑如何实现这个功能：
+
+- 首先，要做目标检测，找到图像中的猫咪
+- 接着，要估计猫咪的关键点位置，比如左右眼的位置
+- 最后，把太阳镜素材图片贴在合适的位置，TA-DA！
+
+按照这个思路，下面我们来看如何一步一步实现它。
+
+## Step 1：从一个现成的 Config 开始
+
+在 WebcamAPI 中，已经添加了一些实现常用功能的 Node，并提供了对应的 config 示例。利用这些可以减少用户的开发量。例如，我们可以以上面的姿态估计 demo 为基础。它的 config 位于 `tools/webcam/configs/example/pose_estimation.py`。为了更直观，我们把这个 config 中的功能节点表示成以下流程图：
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153801397-640f2b45-64e7-41b3-8b00-670c16c57df5.png">
+<figcaption>Pose Estimation Config 示意</figcaption>
+</div>
+
+可以看到，这个 config 已经实现了我们设计思路中“1-目标检测”和“2-关键点检测”的功能。我们还需要实现“3-贴素材图”功能，这就需要定义一个新的 Node了。
+
+## Step 2：实现一个新 Node
+
+在 WebcamAPI 我们定义了以下 2 个 Node 基类：
+
+1. Node：所有 node 的基类，实现了初始化，绑定 runner，启动运行，数据输入输出等基本功能。子类通过重写抽象方法`process()`方法定义具体的 node 功能。
+2. FrameDrawingNode：用来绘制图像的 node 基类。FrameDrawingNode继承自 Node 并进一步封装了`process()`方法，提供了抽象方法`draw()`供子类实现具体的图像绘制功能。
+
+显然，“贴素材图”这个功能属于图像绘制，因此我们只需要继承 BaseFrameEffectNode 类即可。具体实现如下：
+
+```python
+# 假设该文件路径为
+# <MMPose Root>/tools/webcam/webcam_apis/nodes/sunglasses_node.py
+from mmpose.core import apply_sunglasses_effect
+from ..utils import (load_image_from_disk_or_url,
+    get_eye_keypoint_ids)
+from .frame_drawing_node import FrameDrawingNode
+from .builder import NODES
+
+@NODES.register_module()  # 将 SunglassesNode 注册到 NODES（Registry）
+class SunglassesNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 src_img_path: Optional[str] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        # 加载素材图片
+        if src_img_path is None:
+            # The image attributes to:
+            # https://www.vecteezy.com/free-vector/glass
+            # Glass Vectors by Vecteezy
+            src_img_path = ('https://raw.githubusercontent.com/open-mmlab/'
+                            'mmpose/master/demo/resources/sunglasses.jpg')
+        self.src_img = load_image_from_disk_or_url(src_img_path)
+
+    def draw(self, frame_msg):
+        # 获取当前帧图像
+        canvas = frame_msg.get_image()
+        # 获取姿态估计结果
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+
+        # 给每个目标添加太阳镜效果
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            # 获取目标左、右眼关键点位置
+            left_eye_idx, right_eye_idx = get_eye_keypoint_ids(model_cfg)
+            # 根据双眼位置，绘制太阳镜
+            canvas = apply_sunglasses_effect(canvas, preds, self.src_img,
+                                             left_eye_idx, right_eye_idx)
+        return canvas
+```
+
+这里对代码实现中用到的一些函数和类稍作说明：
+
+1. `NODES`：是一个 mmcv.Registry 实例。相信用过 OpenMMLab 系列的同学都对 Registry 不陌生。这里用 NODES来注册和管理所有的 node 类，从而让用户可以在 config 中通过类的名称（如 "DetectorNode"，"SunglassesNode" 等）来指定使用对应的 node。
+2. `load_image_from_disk_or_url`：用来从本地路径或 url 读取图片
+3. `get_eye_keypoint_ids`：根据模型配置文件（model_cfg）中记录的数据集信息，返回双眼关键点的索引。如 COCO 格式对应的左右眼索引为 $(1,2)$
+4. `apply_sunglasses_effect`：将太阳镜绘制到原图中的合适位置，具体步骤为：
+    - 在素材图片上定义一组源锚点 $(s_1, s_2, s_3, s_4)$
+    - 根据目标左右眼关键点位置 $(k_1, k_2)$，计算目标锚点 $(t_1, t_2, t_3, t_4)$
+    - 通过源锚点和目标锚点，计算几何变换矩阵（平移，缩放，旋转），将素材图片做变换后贴入原图片。即可将太阳镜绘制在合适的位置。
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153773612-bcf86b91-31a3-47b5-886d-e33577016f85.png">
+<figcaption>太阳镜特效原理示意</figcaption>
+</div>
+
+### Get Advanced：关于 Node 和 FrameEffectNode
+
+[Node 类](/tools/webcam/webcam_apis/nodes/node.py) ：继承自 Thread 类。正如我们在前面 数据流 部分提到的，所有节点都在各自的线程中彼此异步运行。在`Node.run()` 方法中定义了节点的基本运行逻辑：
+
+1. 当 buffer 中有数据时，会触发一次运行
+2. 调用`process()`来执行具体的功能。`process()`是一个抽象接口，由子类具体实现
+    - 特别地，如果节点需要实现“开/关”功能，则还需要实现`bypass()`方法，以定义节点“关”时的行为。`bypass()`与`process()`的输入输出接口完全相同。在run()中会根据`Node.enable`的状态，调用`process()`或`bypass()`
+3. 将运行结果发送到输出 buffer
+
+在继承 Node 类实现具体的节点类时，通常需要完成以下工作：
+
+1. 在__init__()中注册输入、输出 buffer，并调用基类的__init__()方法
+2. 实现process()和bypass()（如需要）方法
+
+[FrameDrawingNode 类](tools/webcam/webcam_apis/nodes/frame_drawing_node.py) ：继承自 Node 类，对`process()`和`bypass()`方法做了进一步封装：
+
+- process()：从接到输入中提取帧图像，传入draw()方法中绘图。draw()是一个抽象接口，有子类实现
+- bypass()：直接将节点输入返回
+
+### Get Advanced: 关于节点的输入、输出格式
+
+我们定义了[FrameMessage 类](tools/webcam/webcam_apis/utils/message.py)作为节点间通信的数据结构。也就是说，通常情况下节点的输入、输出和 buffer 中存储的元素，都是 FrameMessage 类的实例。FrameMessage 通常用来存储视频中1帧的信息，它提供了简单的接口，用来提取和存入数据：
+
+- `get_image()`：返回图像
+- `set_image()`：设置图像
+- `add_detection_result()`：添加一个目标检测模型的结果
+- `get_detection_results()`：返回所有目标检测结果
+- `add_pose_result()`：添加一个姿态估计模型的结果
+- `get_pose_results()`：返回所有姿态估计结果
+
+## Step 3：调整 Config
+
+有了 Step 2 中实现的 SunglassesNode，我们只要把它加入 config 里就可以使用了。比如，我们可以把它放在“Visualizer” node 之后：
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153801499-590a7810-b231-4a38-8053-c7d33af1535a.png">
+<figcaption>修改后的 Config，添加了 SunglassesNode 节点</figcaption>
+</div>
+
+具体的写法如下：
+
+```python
+runner = dict(
+    # runner的基本参数
+    name='Everybody Wears Sunglasses',
+    camera_id=0,
+    camera_fps=20,
+    # 定义了若干节点(node)
+    nodes=[
+        ...,
+        dict(
+            type='SunglassesNode',  # 节点类名称
+            name='Sunglasses',  # 节点名，由用户自己定义
+            frame_buffer='vis',  # 输入
+            output_buffer='sunglasses',  # 输出
+            enable_key='s',  # 定义开关快捷键
+            enable=True,)  # 启动时默认的开关状态
+        ...]  # 更多节点
+)
+```
+
+此外，用户还可以根据需求调整 config 中的参数。一些常用的设置包括：
+
+1. 选择摄像头：可以通过设置camera_id参数指定使用的摄像头。通常电脑上的默认摄像头 id 为 0，如果有多个则 id 数字依次增大。此外，也可以给camera_id设置一个本地视频文件的路径，从而使用该视频文件作为应用程序的输入
+2. 选择模型：可以通过模型推理节点（如 DetectorNode，TopDownPoseEstimationNode）的model_config和model_checkpoint参数来配置。用户可以根据自己的需求（如目标物体类别，关键点类别等）和硬件情况选用合适的模型
+3. 设置快捷键：一些 node 支持使用快捷键开关，用户可以设置对应的enable_key（快捷键）和enable（默认开关状态）参数
+4. 提示信息：通过设置 NoticeBoardNode 的 content_lines参数，可以在程序运行时在画面上显示提示信息，帮助使用者快速了解这个应用程序的功能和操作方法
+
+最后，将修改过的 config 存到文件`tools/webcam/configs/sunglasses.py`中，就可以运行了：
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/sunglasses.py
+```

third_party/hamer/third-party/ViTPose/tools/webcam/docs/get_started_cn.md

@@ -0,0 +1,123 @@
+# MMPose Webcam API 快速上手
+
+## 什么是 MMPose Webcam API
+
+MMPose WebcamAPI 是一套简单的应用开发接口，可以帮助用户方便的调用 MMPose 以及其他 OpenMMLab 算法库中的算法，实现基于摄像头输入视频的交互式应用。
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153800450-2522efe8-bc11-457d-9037-d8aee4fc4f36.png">
+<figcaption>MMPose Webcam API 框架概览</figcaption>
+</div>
+
+## 运行一个 Demo
+
+我们将从一个简单的 Demo 开始，向您介绍 MMPose WebcamAPI 的功能和特性，并详细展示如何基于这个 API 搭建自己的应用。为了使用 MMPose WebcamAPI，您只需要做简单的准备：
+
+1. 一台计算机（最好有 GPU 和 CUDA 环境，但这并不是必须的）
+1. 一个摄像头。计算机自带摄像头或者外接 USB 摄像头均可
+1. 安装 MMPose
+    - 在 OpenMMLab [官方仓库](https://github.com/open-mmlab/mmpose) fork MMPose 到自己的 github，并 clone 到本地
+    - 安装 MMPose，只需要按照我们的 [安装文档](https://mmpose.readthedocs.io/zh_CN/latest/install.html) 中的步骤操作即可
+
+完成准备工作后，请在命令行进入 MMPose 根目录，执行以下指令，即可运行 demo：
+
+```shell
+python tools/webcam/run_webcam.py --config tools/webcam/configs/examples/pose_estimation.py
+```
+
+这个 demo 实现了目标检测，姿态估计和可视化功能，效果如下：
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153772158-a5702193-3d3f-40c8-bd6b-ab186979d1b4.png">
+<figcaption>Pose Estimation Demo 效果</figcaption>
+</div>
+
+## Demo 里面有什么？
+
+### 从 Config 说起
+
+成功运行 demo 后，我们来看一下它是怎样工作的。在启动脚本 `tools/webcam/run_webcam.py` 中可以看到，这里的操作很简单：首先读取了一个 config 文件，接着使用 config 构建了一个 runner ，最后调用了 runner 的 `run()` 方法，这样 demo 就开始运行了。
+
+```python
+# tools/webcam/run_webcam.py
+
+def launch():
+    # 读取 config 文件
+    args = parse_args()
+    cfg = mmcv.Config.fromfile(args.config)
+    # 构建 runner（WebcamRunner类的实例）
+    runner = WebcamRunner(**cfg.runner)
+    # 调用 run()方法，启动程序
+    runner.run()
+
+
+if __name__ == '__main__':
+    launch()
+```
+
+我们先不深究 runner 为何物，而是接着看一下这个 config 文件的内容。省略掉细节和注释，可以发现 config 的结构大致包含两部分（如下图所示）：
+
+1. Runner 的基本参数，如 camera_id，camera_fps 等。这部分比較好理解，是一些在读取视频时的必要设置
+2. 一系列＂节点＂（Node），每个节点属于特定的类型（type），并有对应的一些参数
+
+```python
+runner = dict(
+    # runner的基本参数
+    name='Pose Estimation',
+    camera_id=0,
+    camera_fps=20,
+    # 定义了若干节点(Node)
+    Nodes=[
+        dict(
+            type='DetectorNode',  # 节点１类型
+            name='Detector',  # 节点１名字
+            input_buffer='_input_',  # 节点１数据输入
+            output_buffer='det_result',  # 节点１数据输出
+            ...), # 节点１其他参数
+        dict(
+            type='TopDownPoseEstimatorNode',  # 节点２类型
+            name='Human Pose Estimator',  # 节点２名字
+            input_buffer='det_result',  # 节点2数据输入
+            output_buffer='pose_result',  # 节点2数据输出
+            ...),  # 节点２参数
+        ...]  # 更多节点
+)
+```
+
+### 核心概念：Runner 和 Node
+
+到这里，我们已经引出了 MMPose WebcamAPI 的２个最重要的概念：runner 和 Node，下面做正式介绍：
+
+- Runner：Runner 类是程序的主体，提供了程序启动的入口runner.run()方法，并负责视频读入，输出显示等功能。此外，runner 中会包含若干个 Node，分别负责在视频帧的处理中执行不同的功能。
+- Node：Node 类用来定义功能模块，例如模型推理，可视化，特效绘制等都可以通过定义一个对应的 Node 来实现。如上面的 config 例子中，2 个节点的功能分别是做目标检测（Detector）和姿态估计（TopDownPoseEstimator）
+
+Runner 和 Node 的关系简单来说如下图所示：
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153772839-104430bd-de0e-4ee5-bd67-dff4e52d784c.png">
+<figcaption>Runner 和 Node 逻辑关系示意</figcaption>
+</div>
+
+### 数据流
+
+一个重要的问题是：当一帧视频数据被 runner 读取后，会按照怎样的顺序通过所有的 Node 并最终被输出（显示）呢？
+答案就是 config 中每个 Node 的输入输出配置。如示例 config 中，可以看到每个 Node 都有`input_buffer`，`output_buffer`等参数，用来定义该节点的输入输出。通过这种连接关系，所有的 Node 构成了一个有向无环图结构，如下图所示：
+
+<div align=center>
+<img src="https://user-images.githubusercontent.com/15977946/153772900-9619ae80-3e64-4b40-bc1e-b184405e3d5b.png">
+<figcaption>数据流示意</figcaption>
+</div>
+
+图中的每个 Data Buffer 就是一个用来存放数据的容器。用户不需要关注 buffer 的具体细节，只需要将其简单理解成 Node 输入输出的名字即可。用户在 config 中可以任意定义这些名字，不过要注意有以下几个特殊的名字：
+
+- _input_：存放 runner 读入的视频帧，用于模型推理
+- _frame_ ：存放 runner 读入的视频帧，用于可视化
+- _display_：存放经过所以 Node 处理后的结果，用于在屏幕上显示
+
+当一帧视频数据被 runner 读入后，会被放进 _input_ 和 _frame_ 两个 buffer 中，然后按照 config 中定义的 Node 连接关系依次通过各个 Node ，最终到达 _display_ ，并被 runner 读出显示在屏幕上。
+
+#### Get Advanced: 关于 buffer
+
+- Buffer 本质是一个有限长度的队列，在 runner 中会包含一个 BufferManager 实例（见`mmpose/tools/webcam/webcam_apis/buffer.py'）来生成和管理所有 buffer。Node 会按照 config 从对应的 buffer 中读出或写入数据。
+- 当一个 buffer 已满（达到最大长度）时，写入数据的操作通常不会被 block，而是会将 buffer 中已有的最早一条数据“挤出去”。
+- 为什么有_input_和_frame_两个输入呢？因为有些 Node 的操作较为耗时（如目标检测，姿态估计等需要模型推理的 Node）。为了保证显示的流畅，我们通常用_input_来作为这类耗时较大的操作的输入，而用_frame_来实时绘制可视化的结果。因为各个节点是异步运行的，这样就可以保证可视化的实时和流畅。

third_party/hamer/third-party/ViTPose/tools/webcam/run_webcam.py

@@ -0,0 +1,38 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+
+from argparse import ArgumentParser
+
+from mmcv import Config, DictAction
+from webcam_apis import WebcamRunner
+
+
+def parse_args():
+    parser = ArgumentParser('Lauch webcam runner')
+    parser.add_argument(
+        '--config',
+        type=str,
+        default='tools/webcam/configs/meow_dwen_dwen/meow_dwen_dwen.py')
+
+    parser.add_argument(
+        '--cfg-options',
+        nargs='+',
+        action=DictAction,
+        default={},
+        help='override some settings in the used config, the key-value pair '
+        'in xxx=yyy format will be merged into config file. For example, '
+        "'--cfg-options runner.camera_id=1 runner.synchronous=True'")
+
+    return parser.parse_args()
+
+
+def launch():
+    args = parse_args()
+    cfg = Config.fromfile(args.config)
+    cfg.merge_from_dict(args.cfg_options)
+
+    runner = WebcamRunner(**cfg.runner)
+    runner.run()
+
+
+if __name__ == '__main__':
+    launch()

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .webcam_runner import WebcamRunner
+
+__all__ = ['WebcamRunner']

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .builder import NODES
+from .faceswap_node import FaceSwapNode
+from .frame_effect_node import (BackgroundNode, BugEyeNode, MoustacheNode,
+                                NoticeBoardNode, PoseVisualizerNode,
+                                SaiyanNode, SunglassesNode)
+from .helper_node import ModelResultBindingNode, MonitorNode, RecorderNode
+from .mmdet_node import DetectorNode
+from .mmpose_node import TopDownPoseEstimatorNode
+from .valentinemagic_node import ValentineMagicNode
+from .xdwendwen_node import XDwenDwenNode
+
+__all__ = [
+    'NODES', 'PoseVisualizerNode', 'DetectorNode', 'TopDownPoseEstimatorNode',
+    'MonitorNode', 'BugEyeNode', 'SunglassesNode', 'ModelResultBindingNode',
+    'NoticeBoardNode', 'RecorderNode', 'FaceSwapNode', 'MoustacheNode',
+    'SaiyanNode', 'BackgroundNode', 'XDwenDwenNode', 'ValentineMagicNode'
+]

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/builder.py

@@ -0,0 +1,4 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from mmcv.utils import Registry
+
+NODES = Registry('node')

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/faceswap_node.py

@@ -0,0 +1,254 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from enum import IntEnum
+from typing import List, Union
+
+import cv2
+import numpy as np
+
+from mmpose.datasets import DatasetInfo
+from .builder import NODES
+from .frame_drawing_node import FrameDrawingNode
+
+
+class Mode(IntEnum):
+    NONE = 0,
+    SHUFFLE = 1,
+    CLONE = 2
+
+
+@NODES.register_module()
+class FaceSwapNode(FrameDrawingNode):
+
+    def __init__(
+        self,
+        name: str,
+        frame_buffer: str,
+        output_buffer: Union[str, List[str]],
+        mode_key: Union[str, int],
+    ):
+        super().__init__(name, frame_buffer, output_buffer, enable=True)
+
+        self.mode_key = mode_key
+        self.mode_index = 0
+        self.register_event(
+            self.mode_key, is_keyboard=True, handler_func=self.switch_mode)
+        self.history = dict(mode=None)
+        self._mode = Mode.SHUFFLE
+
+    @property
+    def mode(self):
+        return self._mode
+
+    def switch_mode(self):
+        """Switch modes by updating mode index."""
+        self._mode = Mode((self._mode + 1) % len(Mode))
+
+    def draw(self, frame_msg):
+
+        if self.mode == Mode.NONE:
+            self.history = {'mode': Mode.NONE}
+            return frame_msg.get_image()
+
+        # Init history
+        if self.history['mode'] != self.mode:
+            self.history = {'mode': self.mode, 'target_map': {}}
+
+        # Merge pose results
+        pose_preds = self._merge_pose_results(frame_msg.get_pose_results())
+        num_target = len(pose_preds)
+
+        # Show mode
+        img = frame_msg.get_image()
+        canvas = img.copy()
+        if self.mode == Mode.SHUFFLE:
+            mode_txt = 'Shuffle'
+        else:
+            mode_txt = 'Clone'
+
+        cv2.putText(canvas, mode_txt, (10, 50), cv2.FONT_HERSHEY_DUPLEX, 0.8,
+                    (255, 126, 0), 1)
+
+        # Skip if target number is less than 2
+        if num_target >= 2:
+            # Generate new mapping if target number changes
+            if num_target != len(self.history['target_map']):
+                if self.mode == Mode.SHUFFLE:
+                    self.history['target_map'] = self._get_swap_map(num_target)
+                else:
+                    self.history['target_map'] = np.repeat(
+                        np.random.choice(num_target), num_target)
+
+            # # Draw on canvas
+            for tar_idx, src_idx in enumerate(self.history['target_map']):
+                face_src = self._get_face_info(pose_preds[src_idx])
+                face_tar = self._get_face_info(pose_preds[tar_idx])
+                canvas = self._swap_face(img, canvas, face_src, face_tar)
+
+        return canvas
+
+    def _crop_face_by_contour(self, img, contour):
+        mask = np.zeros(img.shape[:2], dtype=np.uint8)
+        cv2.fillPoly(mask, [contour.astype(np.int32)], 1)
+        mask = cv2.dilate(
+            mask, kernel=np.ones((9, 9), dtype=np.uint8), anchor=(4, 0))
+        x1, y1, w, h = cv2.boundingRect(mask)
+        x2 = x1 + w
+        y2 = y1 + h
+        bbox = np.array([x1, y1, x2, y2], dtype=np.int64)
+        patch = img[y1:y2, x1:x2]
+        mask = mask[y1:y2, x1:x2]
+
+        return bbox, patch, mask
+
+    def _swap_face(self, img_src, img_tar, face_src, face_tar):
+
+        if face_src['dataset'] == face_tar['dataset']:
+            # Use full keypoints for face alignment
+            kpts_src = face_src['contour']
+            kpts_tar = face_tar['contour']
+        else:
+            # Use only common landmarks (eyes and nose) for face alignment if
+            # source and target have differenet data type
+            # (e.g. human vs animal)
+            kpts_src = face_src['landmarks']
+            kpts_tar = face_tar['landmarks']
+
+        # Get everything local
+        bbox_src, patch_src, mask_src = self._crop_face_by_contour(
+            img_src, face_src['contour'])
+
+        bbox_tar, _, mask_tar = self._crop_face_by_contour(
+            img_tar, face_tar['contour'])
+
+        kpts_src = kpts_src - bbox_src[:2]
+        kpts_tar = kpts_tar - bbox_tar[:2]
+
+        # Compute affine transformation matrix
+        trans_mat, _ = cv2.estimateAffine2D(
+            kpts_src.astype(np.float32), kpts_tar.astype(np.float32))
+        patch_warp = cv2.warpAffine(
+            patch_src,
+            trans_mat,
+            dsize=tuple(bbox_tar[2:] - bbox_tar[:2]),
+            borderValue=(0, 0, 0))
+        mask_warp = cv2.warpAffine(
+            mask_src,
+            trans_mat,
+            dsize=tuple(bbox_tar[2:] - bbox_tar[:2]),
+            borderValue=(0, 0, 0))
+
+        # Target mask
+        mask_tar = mask_tar & mask_warp
+        mask_tar_soft = cv2.GaussianBlur(mask_tar * 255, (3, 3), 3)
+
+        # Blending
+        center = tuple((0.5 * (bbox_tar[:2] + bbox_tar[2:])).astype(np.int64))
+        img_tar = cv2.seamlessClone(patch_warp, img_tar, mask_tar_soft, center,
+                                    cv2.NORMAL_CLONE)
+        return img_tar
+
+    @staticmethod
+    def _get_face_info(pose_pred):
+        keypoints = pose_pred['keypoints'][:, :2]
+        model_cfg = pose_pred['model_cfg']
+        dataset_info = DatasetInfo(model_cfg.data.test.dataset_info)
+
+        face_info = {
+            'dataset': dataset_info.dataset_name,
+            'landmarks': None,  # For alignment
+            'contour': None,  # For mask generation
+            'bbox': None  # For image warping
+        }
+
+        # Fall back to hard coded keypoint id
+
+        if face_info['dataset'] == 'coco':
+            face_info['landmarks'] = np.stack([
+                keypoints[1],  # left eye
+                keypoints[2],  # right eye
+                keypoints[0],  # nose
+                0.5 * (keypoints[5] + keypoints[6]),  # neck (shoulder center)
+            ])
+        elif face_info['dataset'] == 'coco_wholebody':
+            face_info['landmarks'] = np.stack([
+                keypoints[1],  # left eye
+                keypoints[2],  # right eye
+                keypoints[0],  # nose
+                keypoints[32],  # chin
+            ])
+            contour_ids = list(range(23, 40)) + list(range(40, 50))[::-1]
+            face_info['contour'] = keypoints[contour_ids]
+        elif face_info['dataset'] == 'ap10k':
+            face_info['landmarks'] = np.stack([
+                keypoints[0],  # left eye
+                keypoints[1],  # right eye
+                keypoints[2],  # nose
+                keypoints[3],  # neck
+            ])
+        elif face_info['dataset'] == 'animalpose':
+            face_info['landmarks'] = np.stack([
+                keypoints[0],  # left eye
+                keypoints[1],  # right eye
+                keypoints[4],  # nose
+                keypoints[5],  # throat
+            ])
+        elif face_info['dataset'] == 'wflw':
+            face_info['landmarks'] = np.stack([
+                keypoints[97],  # left eye
+                keypoints[96],  # right eye
+                keypoints[54],  # nose
+                keypoints[16],  # chine
+            ])
+            contour_ids = list(range(33))[::-1] + list(range(33, 38)) + list(
+                range(42, 47))
+            face_info['contour'] = keypoints[contour_ids]
+        else:
+            raise ValueError('Can not obtain face landmark information'
+                             f'from dataset: {face_info["type"]}')
+
+        # Face region
+        if face_info['contour'] is None:
+            # Manually defined counter of face region
+            left_eye, right_eye, nose = face_info['landmarks'][:3]
+            eye_center = 0.5 * (left_eye + right_eye)
+            w_vec = right_eye - left_eye
+            eye_dist = np.linalg.norm(w_vec) + 1e-6
+            w_vec = w_vec / eye_dist
+            h_vec = np.array([w_vec[1], -w_vec[0]], dtype=w_vec.dtype)
+            w = max(0.5 * eye_dist, np.abs(np.dot(nose - eye_center, w_vec)))
+            h = np.abs(np.dot(nose - eye_center, h_vec))
+
+            left_top = eye_center + 1.5 * w * w_vec - 0.5 * h * h_vec
+            right_top = eye_center - 1.5 * w * w_vec - 0.5 * h * h_vec
+            left_bottom = eye_center + 1.5 * w * w_vec + 4 * h * h_vec
+            right_bottom = eye_center - 1.5 * w * w_vec + 4 * h * h_vec
+
+            face_info['contour'] = np.stack(
+                [left_top, right_top, right_bottom, left_bottom])
+
+        # Get tight bbox of face region
+        face_info['bbox'] = np.array([
+            face_info['contour'][:, 0].min(), face_info['contour'][:, 1].min(),
+            face_info['contour'][:, 0].max(), face_info['contour'][:, 1].max()
+        ]).astype(np.int64)
+
+        return face_info
+
+    @staticmethod
+    def _merge_pose_results(pose_results):
+        preds = []
+        if pose_results is not None:
+            for prefix, pose_result in enumerate(pose_results):
+                model_cfg = pose_result['model_cfg']
+                for idx, _pred in enumerate(pose_result['preds']):
+                    pred = _pred.copy()
+                    pred['id'] = f'{prefix}.{_pred.get("track_id", str(idx))}'
+                    pred['model_cfg'] = model_cfg
+                    preds.append(pred)
+        return preds
+
+    @staticmethod
+    def _get_swap_map(num_target):
+        ids = np.random.choice(num_target, num_target, replace=False)
+        target_map = ids[(ids + 1) % num_target]
+        return target_map

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/frame_drawing_node.py

@@ -0,0 +1,65 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from abc import abstractmethod
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+
+from ..utils import FrameMessage, Message
+from .node import Node
+
+
+class FrameDrawingNode(Node):
+    """Base class for Node that draw on single frame images.
+
+    Args:
+        name (str, optional): The node name (also thread name).
+        frame_buffer (str): The name of the input buffer.
+        output_buffer (str | list): The name(s) of the output buffer(s).
+        enable_key (str | int, optional): Set a hot-key to toggle
+            enable/disable of the node. If an int value is given, it will be
+            treated as an ascii code of a key. Please note:
+                1. If enable_key is set, the bypass method need to be
+                    overridden to define the node behavior when disabled
+                2. Some hot-key has been use for particular use. For example:
+                    'q', 'Q' and 27 are used for quit
+            Default: None
+        enable (bool): Default enable/disable status. Default: True.
+    """
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True):
+
+        super().__init__(name=name, enable_key=enable_key)
+
+        # Register buffers
+        self.register_input_buffer(frame_buffer, 'frame', essential=True)
+        self.register_output_buffer(output_buffer)
+
+        self._enabled = enable
+
+    def process(self, input_msgs: Dict[str, Message]) -> Union[Message, None]:
+        frame_msg = input_msgs['frame']
+
+        img = self.draw(frame_msg)
+        frame_msg.set_image(img)
+
+        return frame_msg
+
+    def bypass(self, input_msgs: Dict[str, Message]) -> Union[Message, None]:
+        return input_msgs['frame']
+
+    @abstractmethod
+    def draw(self, frame_msg: FrameMessage) -> np.ndarray:
+        """Draw on the frame image with information from the single frame.
+
+        Args:
+            frame_meg (FrameMessage): The frame to get information from and
+                draw on.
+
+        Returns:
+            array: The output image
+        """

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/frame_effect_node.py

@@ -0,0 +1,917 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from typing import Dict, List, Optional, Tuple, Union
+
+import cv2
+import numpy as np
+from mmcv import color_val
+
+from mmpose.core import (apply_bugeye_effect, apply_sunglasses_effect,
+                         imshow_bboxes, imshow_keypoints)
+from mmpose.datasets import DatasetInfo
+from ..utils import (FrameMessage, copy_and_paste, expand_and_clamp,
+                     get_cached_file_path, get_eye_keypoint_ids,
+                     get_face_keypoint_ids, get_wrist_keypoint_ids,
+                     load_image_from_disk_or_url, screen_matting)
+from .builder import NODES
+from .frame_drawing_node import FrameDrawingNode
+
+try:
+    import psutil
+    psutil_proc = psutil.Process()
+except (ImportError, ModuleNotFoundError):
+    psutil_proc = None
+
+
+@NODES.register_module()
+class PoseVisualizerNode(FrameDrawingNode):
+    """Draw the bbox and keypoint detection results.
+
+    Args:
+        name (str, optional): The node name (also thread name).
+        frame_buffer (str): The name of the input buffer.
+        output_buffer (str|list): The name(s) of the output buffer(s).
+        enable_key (str|int, optional): Set a hot-key to toggle enable/disable
+            of the node. If an int value is given, it will be treated as an
+            ascii code of a key. Please note:
+                1. If enable_key is set, the bypass method need to be
+                    overridden to define the node behavior when disabled
+                2. Some hot-key has been use for particular use. For example:
+                    'q', 'Q' and 27 are used for quit
+            Default: None
+        enable (bool): Default enable/disable status. Default: True.
+        kpt_thr (float): The threshold of keypoint score. Default: 0.3.
+        radius (int): The radius of keypoint. Default: 4.
+        thickness (int): The thickness of skeleton. Default: 2.
+        bbox_color (str|tuple|dict): If a single color (a str like 'green' or
+            a tuple like (0, 255, 0)), it will used to draw the bbox.
+            Optionally, a dict can be given as a map from class labels to
+            colors.
+    """
+
+    default_bbox_color = {
+        'person': (148, 139, 255),
+        'cat': (255, 255, 0),
+        'dog': (255, 255, 0),
+    }
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 kpt_thr: float = 0.3,
+                 radius: int = 4,
+                 thickness: int = 2,
+                 bbox_color: Optional[Union[str, Tuple, Dict]] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        self.kpt_thr = kpt_thr
+        self.radius = radius
+        self.thickness = thickness
+        if bbox_color is None:
+            self.bbox_color = self.default_bbox_color
+        elif isinstance(bbox_color, dict):
+            self.bbox_color = {k: color_val(v) for k, v in bbox_color.items()}
+        else:
+            self.bbox_color = color_val(bbox_color)
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+
+        if not pose_results:
+            return canvas
+
+        for pose_result in frame_msg.get_pose_results():
+            model_cfg = pose_result['model_cfg']
+            dataset_info = DatasetInfo(model_cfg.dataset_info)
+
+            # Extract bboxes and poses
+            bbox_preds = []
+            bbox_labels = []
+            pose_preds = []
+            for pred in pose_result['preds']:
+                if 'bbox' in pred:
+                    bbox_preds.append(pred['bbox'])
+                    bbox_labels.append(pred.get('label', None))
+                pose_preds.append(pred['keypoints'])
+
+            # Get bbox colors
+            if isinstance(self.bbox_color, dict):
+                bbox_colors = [
+                    self.bbox_color.get(label, (0, 255, 0))
+                    for label in bbox_labels
+                ]
+            else:
+                bbox_labels = self.bbox_color
+
+            # Draw bboxes
+            if bbox_preds:
+                bboxes = np.vstack(bbox_preds)
+
+                imshow_bboxes(
+                    canvas,
+                    bboxes,
+                    labels=bbox_labels,
+                    colors=bbox_colors,
+                    text_color='white',
+                    font_scale=0.5,
+                    show=False)
+
+            # Draw poses
+            if pose_preds:
+                imshow_keypoints(
+                    canvas,
+                    pose_preds,
+                    skeleton=dataset_info.skeleton,
+                    kpt_score_thr=0.3,
+                    pose_kpt_color=dataset_info.pose_kpt_color,
+                    pose_link_color=dataset_info.pose_link_color,
+                    radius=self.radius,
+                    thickness=self.thickness)
+
+        return canvas
+
+
+@NODES.register_module()
+class SunglassesNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 src_img_path: Optional[str] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        if src_img_path is None:
+            # The image attributes to:
+            # https://www.vecteezy.com/free-vector/glass
+            # Glass Vectors by Vecteezy
+            src_img_path = 'demo/resources/sunglasses.jpg'
+        self.src_img = load_image_from_disk_or_url(src_img_path)
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            left_eye_idx, right_eye_idx = get_eye_keypoint_ids(model_cfg)
+
+            canvas = apply_sunglasses_effect(canvas, preds, self.src_img,
+                                             left_eye_idx, right_eye_idx)
+        return canvas
+
+
+@NODES.register_module()
+class SpriteNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 src_img_path: Optional[str] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        if src_img_path is None:
+            # Sprites of Touhou characters :)
+            # Come from https://www.deviantart.com/shadowbendy/art/Touhou-rpg-maker-vx-Sprite-1-812746920  # noqa: E501
+            src_img_path = (
+                'https://user-images.githubusercontent.com/'
+                '26739999/151532276-33f968d9-917f-45e3-8a99-ebde60be83bb.png')
+        self.src_img = load_image_from_disk_or_url(
+            src_img_path, cv2.IMREAD_UNCHANGED)[:144, :108]
+        tmp = np.array(np.split(self.src_img, range(36, 144, 36), axis=0))
+        tmp = np.array(np.split(tmp, range(36, 108, 36), axis=2))
+        self.sprites = tmp
+        self.pos = None
+        self.anime_frame = 0
+
+    def apply_sprite_effect(self,
+                            img,
+                            pose_results,
+                            left_hand_index,
+                            right_hand_index,
+                            kpt_thr=0.5):
+        """Apply sprite effect.
+
+        Args:
+            img (np.ndarray): Image data.
+            pose_results (list[dict]): The pose estimation results containing:
+                - "keypoints" ([K,3]): detection result in [x, y, score]
+            left_hand_index (int): Keypoint index of left hand
+            right_hand_index (int): Keypoint index of right hand
+            kpt_thr (float): The score threshold of required keypoints.
+        """
+
+        hm, wm = self.sprites.shape[2:4]
+        # anchor points in the sunglasses mask
+        if self.pos is None:
+            self.pos = [img.shape[0] // 2, img.shape[1] // 2]
+
+        if len(pose_results) == 0:
+            return img
+
+        kpts = pose_results[0]['keypoints']
+
+        if kpts[left_hand_index, 2] < kpt_thr and kpts[right_hand_index,
+                                                       2] < kpt_thr:
+            aim = self.pos
+        else:
+            kpt_lhand = kpts[left_hand_index, :2][::-1]
+            kpt_rhand = kpts[right_hand_index, :2][::-1]
+
+            def distance(a, b):
+                return (a[0] - b[0])**2 + (a[1] - b[1])**2
+
+            # Go to the nearest hand
+            if distance(kpt_lhand, self.pos) < distance(kpt_rhand, self.pos):
+                aim = kpt_lhand
+            else:
+                aim = kpt_rhand
+
+        pos_thr = 15
+        if aim[0] < self.pos[0] - pos_thr:
+            # Go down
+            sprite = self.sprites[self.anime_frame][3]
+            self.pos[0] -= 1
+        elif aim[0] > self.pos[0] + pos_thr:
+            # Go up
+            sprite = self.sprites[self.anime_frame][0]
+            self.pos[0] += 1
+        elif aim[1] < self.pos[1] - pos_thr:
+            # Go right
+            sprite = self.sprites[self.anime_frame][1]
+            self.pos[1] -= 1
+        elif aim[1] > self.pos[1] + pos_thr:
+            # Go left
+            sprite = self.sprites[self.anime_frame][2]
+            self.pos[1] += 1
+        else:
+            # Stay
+            self.anime_frame = 0
+            sprite = self.sprites[self.anime_frame][0]
+
+        if self.anime_frame < 2:
+            self.anime_frame += 1
+        else:
+            self.anime_frame = 0
+
+        x = self.pos[0] - hm // 2
+        y = self.pos[1] - wm // 2
+        x = max(0, min(x, img.shape[0] - hm))
+        y = max(0, min(y, img.shape[0] - wm))
+
+        # Overlay image with transparent
+        img[x:x + hm, y:y +
+            wm] = (img[x:x + hm, y:y + wm] * (1 - sprite[:, :, 3:] / 255) +
+                   sprite[:, :, :3] * (sprite[:, :, 3:] / 255)).astype('uint8')
+
+        return img
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            # left_hand_idx, right_hand_idx = get_wrist_keypoint_ids(model_cfg)  # noqa: E501
+            left_hand_idx, right_hand_idx = get_eye_keypoint_ids(model_cfg)
+
+            canvas = self.apply_sprite_effect(canvas, preds, left_hand_idx,
+                                              right_hand_idx)
+        return canvas
+
+
+@NODES.register_module()
+class BackgroundNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 src_img_path: Optional[str] = None,
+                 cls_ids: Optional[List] = None,
+                 cls_names: Optional[List] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        self.cls_ids = cls_ids
+        self.cls_names = cls_names
+
+        if src_img_path is None:
+            src_img_path = 'https://user-images.githubusercontent.com/'\
+                           '11788150/149731957-abd5c908-9c7f-45b2-b7bf-'\
+                           '821ab30c6a3e.jpg'
+        self.src_img = load_image_from_disk_or_url(src_img_path)
+
+    def apply_background_effect(self,
+                                img,
+                                det_results,
+                                background_img,
+                                effect_region=(0.2, 0.2, 0.8, 0.8)):
+        """Change background.
+
+        Args:
+            img (np.ndarray): Image data.
+            det_results (list[dict]): The detection results containing:
+
+                - "cls_id" (int): Class index.
+                - "label" (str): Class label (e.g. 'person').
+                - "bbox" (ndarray:(5, )): bounding box result
+                    [x, y, w, h, score].
+                - "mask" (ndarray:(w, h)): instance segmentation result.
+            background_img (np.ndarray): Background image.
+            effect_region (tuple(4, )): The region to apply mask,
+                the coordinates are normalized (x1, y1, x2, y2).
+        """
+        if len(det_results) > 0:
+            # Choose the one with the highest score.
+            det_result = det_results[0]
+            bbox = det_result['bbox']
+            mask = det_result['mask'].astype(np.uint8)
+            img = copy_and_paste(img, background_img, mask, bbox,
+                                 effect_region)
+            return img
+        else:
+            return background_img
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        if canvas.shape != self.src_img.shape:
+            self.src_img = cv2.resize(self.src_img, canvas.shape[:2])
+        det_results = frame_msg.get_detection_results()
+        if not det_results:
+            return canvas
+
+        full_preds = []
+        for det_result in det_results:
+            preds = det_result['preds']
+            if self.cls_ids:
+                # Filter results by class ID
+                filtered_preds = [
+                    p for p in preds if p['cls_id'] in self.cls_ids
+                ]
+            elif self.cls_names:
+                # Filter results by class name
+                filtered_preds = [
+                    p for p in preds if p['label'] in self.cls_names
+                ]
+            else:
+                filtered_preds = preds
+            full_preds.extend(filtered_preds)
+
+        canvas = self.apply_background_effect(canvas, full_preds, self.src_img)
+
+        return canvas
+
+
+@NODES.register_module()
+class SaiyanNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 hair_img_path: Optional[str] = None,
+                 light_video_path: Optional[str] = None,
+                 cls_ids: Optional[List] = None,
+                 cls_names: Optional[List] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        self.cls_ids = cls_ids
+        self.cls_names = cls_names
+
+        if hair_img_path is None:
+            hair_img_path = 'https://user-images.githubusercontent.com/'\
+                            '11788150/149732117-fcd2d804-dc2c-426c-bee7-'\
+                            '94be6146e05c.png'
+        self.hair_img = load_image_from_disk_or_url(hair_img_path)
+
+        if light_video_path is None:
+            light_video_path = get_cached_file_path(
+                'https://'
+                'user-images.githubusercontent.com/11788150/149732080'
+                '-ea6cfeda-0dc5-4bbb-892a-3831e5580520.mp4')
+        self.light_video_path = light_video_path
+        self.light_video = cv2.VideoCapture(self.light_video_path)
+
+    def apply_saiyan_effect(self,
+                            img,
+                            pose_results,
+                            saiyan_img,
+                            light_frame,
+                            face_indices,
+                            bbox_thr=0.3,
+                            kpt_thr=0.5):
+        """Apply saiyan hair effect.
+
+        Args:
+            img (np.ndarray): Image data.
+            pose_results (list[dict]): The pose estimation results containing:
+                - "keypoints" ([K,3]): keypoint detection result
+                    in [x, y, score]
+            saiyan_img (np.ndarray): Saiyan image with transparent background.
+            light_frame (np.ndarray): Light image with green screen.
+            face_indices (int): Keypoint index of the face
+            kpt_thr (float): The score threshold of required keypoints.
+        """
+        img = img.copy()
+        im_shape = img.shape
+        # Apply lightning effects.
+        light_mask = screen_matting(light_frame, color='green')
+
+        # anchor points in the mask
+        pts_src = np.array(
+            [
+                [84, 398],  # face kpt 0
+                [331, 393],  # face kpt 16
+                [84, 145],
+                [331, 140]
+            ],
+            dtype=np.float32)
+
+        for pose in pose_results:
+            bbox = pose['bbox']
+
+            if bbox[-1] < bbox_thr:
+                continue
+
+            mask_inst = pose['mask']
+            # cache
+            fg = img[np.where(mask_inst)]
+
+            bbox = expand_and_clamp(bbox[:4], im_shape, s=3.0)
+            # Apply light effects between fg and bg
+            img = copy_and_paste(
+                light_frame,
+                img,
+                light_mask,
+                effect_region=(bbox[0] / im_shape[1], bbox[1] / im_shape[0],
+                               bbox[2] / im_shape[1], bbox[3] / im_shape[0]))
+            # pop
+            img[np.where(mask_inst)] = fg
+
+            # Apply Saiyan hair effects
+            kpts = pose['keypoints']
+            if kpts[face_indices[0], 2] < kpt_thr or kpts[face_indices[16],
+                                                          2] < kpt_thr:
+                continue
+
+            kpt_0 = kpts[face_indices[0], :2]
+            kpt_16 = kpts[face_indices[16], :2]
+            # orthogonal vector
+            vo = (kpt_0 - kpt_16)[::-1] * [-1, 1]
+
+            # anchor points in the image by eye positions
+            pts_tar = np.vstack([kpt_0, kpt_16, kpt_0 + vo, kpt_16 + vo])
+
+            h_mat, _ = cv2.findHomography(pts_src, pts_tar)
+            patch = cv2.warpPerspective(
+                saiyan_img,
+                h_mat,
+                dsize=(img.shape[1], img.shape[0]),
+                borderValue=(0, 0, 0))
+            mask_patch = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY)
+            mask_patch = (mask_patch > 1).astype(np.uint8)
+            img = cv2.copyTo(patch, mask_patch, img)
+
+        return img
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+
+        det_results = frame_msg.get_detection_results()
+        if not det_results:
+            return canvas
+
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            face_indices = get_face_keypoint_ids(model_cfg)
+
+            ret, frame = self.light_video.read()
+            if not ret:
+                self.light_video = cv2.VideoCapture(self.light_video_path)
+                ret, frame = self.light_video.read()
+
+            canvas = self.apply_saiyan_effect(canvas, preds, self.hair_img,
+                                              frame, face_indices)
+
+        return canvas
+
+
+@NODES.register_module()
+class MoustacheNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 src_img_path: Optional[str] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        if src_img_path is None:
+            src_img_path = 'https://user-images.githubusercontent.com/'\
+                           '11788150/149732141-3afbab55-252a-428c-b6d8'\
+                           '-0e352f432651.jpeg'
+        self.src_img = load_image_from_disk_or_url(src_img_path)
+
+    def apply_moustache_effect(self,
+                               img,
+                               pose_results,
+                               moustache_img,
+                               face_indices,
+                               kpt_thr=0.5):
+        """Apply moustache effect.
+
+        Args:
+            img (np.ndarray): Image data.
+            pose_results (list[dict]): The pose estimation results containing:
+                - "keypoints" ([K,3]): keypoint detection result
+                    in [x, y, score]
+            moustache_img (np.ndarray): Moustache image with white background.
+            left_eye_index (int): Keypoint index of left eye
+            right_eye_index (int): Keypoint index of right eye
+            kpt_thr (float): The score threshold of required keypoints.
+        """
+
+        hm, wm = moustache_img.shape[:2]
+        # anchor points in the moustache mask
+        pts_src = np.array([[1164, 741], [1729, 741], [1164, 1244],
+                            [1729, 1244]],
+                           dtype=np.float32)
+
+        for pose in pose_results:
+            kpts = pose['keypoints']
+            if kpts[face_indices[32], 2] < kpt_thr \
+                    or kpts[face_indices[34], 2] < kpt_thr \
+                    or kpts[face_indices[61], 2] < kpt_thr \
+                    or kpts[face_indices[63], 2] < kpt_thr:
+                continue
+
+            kpt_32 = kpts[face_indices[32], :2]
+            kpt_34 = kpts[face_indices[34], :2]
+            kpt_61 = kpts[face_indices[61], :2]
+            kpt_63 = kpts[face_indices[63], :2]
+            # anchor points in the image by eye positions
+            pts_tar = np.vstack([kpt_32, kpt_34, kpt_61, kpt_63])
+
+            h_mat, _ = cv2.findHomography(pts_src, pts_tar)
+            patch = cv2.warpPerspective(
+                moustache_img,
+                h_mat,
+                dsize=(img.shape[1], img.shape[0]),
+                borderValue=(255, 255, 255))
+            #  mask the white background area in the patch with a threshold 200
+            mask = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY)
+            mask = (mask < 200).astype(np.uint8)
+            img = cv2.copyTo(patch, mask, img)
+
+        return img
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            face_indices = get_face_keypoint_ids(model_cfg)
+            canvas = self.apply_moustache_effect(canvas, preds, self.src_img,
+                                                 face_indices)
+        return canvas
+
+
+@NODES.register_module()
+class BugEyeNode(FrameDrawingNode):
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            left_eye_idx, right_eye_idx = get_eye_keypoint_ids(model_cfg)
+
+            canvas = apply_bugeye_effect(canvas, preds, left_eye_idx,
+                                         right_eye_idx)
+        return canvas
+
+
+@NODES.register_module()
+class NoticeBoardNode(FrameDrawingNode):
+
+    default_content_lines = ['This is a notice board!']
+
+    def __init__(
+        self,
+        name: str,
+        frame_buffer: str,
+        output_buffer: Union[str, List[str]],
+        enable_key: Optional[Union[str, int]] = None,
+        enable: bool = True,
+        content_lines: Optional[List[str]] = None,
+        x_offset: int = 20,
+        y_offset: int = 20,
+        y_delta: int = 15,
+        text_color: Union[str, Tuple[int, int, int]] = 'black',
+        background_color: Union[str, Tuple[int, int, int]] = (255, 183, 0),
+        text_scale: float = 0.4,
+    ):
+        super().__init__(name, frame_buffer, output_buffer, enable_key, enable)
+
+        self.x_offset = x_offset
+        self.y_offset = y_offset
+        self.y_delta = y_delta
+        self.text_color = color_val(text_color)
+        self.background_color = color_val(background_color)
+        self.text_scale = text_scale
+
+        if content_lines:
+            self.content_lines = content_lines
+        else:
+            self.content_lines = self.default_content_lines
+
+    def draw(self, frame_msg: FrameMessage) -> np.ndarray:
+        img = frame_msg.get_image()
+        canvas = np.full(img.shape, self.background_color, dtype=img.dtype)
+
+        x = self.x_offset
+        y = self.y_offset
+
+        max_len = max([len(line) for line in self.content_lines])
+
+        def _put_line(line=''):
+            nonlocal y
+            cv2.putText(canvas, line, (x, y), cv2.FONT_HERSHEY_DUPLEX,
+                        self.text_scale, self.text_color, 1)
+            y += self.y_delta
+
+        for line in self.content_lines:
+            _put_line(line)
+
+        x1 = max(0, self.x_offset)
+        x2 = min(img.shape[1], int(x + max_len * self.text_scale * 20))
+        y1 = max(0, self.y_offset - self.y_delta)
+        y2 = min(img.shape[0], y)
+
+        src1 = canvas[y1:y2, x1:x2]
+        src2 = img[y1:y2, x1:x2]
+        img[y1:y2, x1:x2] = cv2.addWeighted(src1, 0.5, src2, 0.5, 0)
+
+        return img
+
+
+@NODES.register_module()
+class HatNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 src_img_path: Optional[str] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key)
+
+        if src_img_path is None:
+            # The image attributes to:
+            # http://616pic.com/sucai/1m9i70p52.html
+            src_img_path = 'https://user-images.githubusercontent.' \
+                           'com/28900607/149766271-2f591c19-9b67-4' \
+                           'd92-8f94-c272396ca141.png'
+        self.src_img = load_image_from_disk_or_url(src_img_path,
+                                                   cv2.IMREAD_UNCHANGED)
+
+    @staticmethod
+    def apply_hat_effect(img,
+                         pose_results,
+                         hat_img,
+                         left_eye_index,
+                         right_eye_index,
+                         kpt_thr=0.5):
+        """Apply hat effect.
+        Args:
+            img (np.ndarray): Image data.
+            pose_results (list[dict]): The pose estimation results containing:
+                - "keypoints" ([K,3]): keypoint detection result in
+                    [x, y, score]
+            hat_img (np.ndarray): Hat image with white alpha channel.
+            left_eye_index (int): Keypoint index of left eye
+            right_eye_index (int): Keypoint index of right eye
+            kpt_thr (float): The score threshold of required keypoints.
+        """
+        img_orig = img.copy()
+
+        img = img_orig.copy()
+        hm, wm = hat_img.shape[:2]
+        # anchor points in the sunglasses mask
+        a = 0.3
+        b = 0.7
+        pts_src = np.array([[a * wm, a * hm], [a * wm, b * hm],
+                            [b * wm, a * hm], [b * wm, b * hm]],
+                           dtype=np.float32)
+
+        for pose in pose_results:
+            kpts = pose['keypoints']
+
+            if kpts[left_eye_index, 2] < kpt_thr or \
+                    kpts[right_eye_index, 2] < kpt_thr:
+                continue
+
+            kpt_leye = kpts[left_eye_index, :2]
+            kpt_reye = kpts[right_eye_index, :2]
+            # orthogonal vector to the left-to-right eyes
+            vo = 0.5 * (kpt_reye - kpt_leye)[::-1] * [-1, 1]
+            veye = 0.5 * (kpt_reye - kpt_leye)
+
+            # anchor points in the image by eye positions
+            pts_tar = np.vstack([
+                kpt_reye + 1 * veye + 5 * vo, kpt_reye + 1 * veye + 1 * vo,
+                kpt_leye - 1 * veye + 5 * vo, kpt_leye - 1 * veye + 1 * vo
+            ])
+
+            h_mat, _ = cv2.findHomography(pts_src, pts_tar)
+            patch = cv2.warpPerspective(
+                hat_img,
+                h_mat,
+                dsize=(img.shape[1], img.shape[0]),
+                borderValue=(255, 255, 255))
+            #  mask the white background area in the patch with a threshold 200
+            mask = (patch[:, :, -1] > 128)
+            patch = patch[:, :, :-1]
+            mask = mask * (cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY) > 30)
+            mask = mask.astype(np.uint8)
+
+            img = cv2.copyTo(patch, mask, img)
+        return img
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            left_eye_idx, right_eye_idx = get_eye_keypoint_ids(model_cfg)
+
+            canvas = self.apply_hat_effect(canvas, preds, self.src_img,
+                                           left_eye_idx, right_eye_idx)
+        return canvas
+
+
+@NODES.register_module()
+class FirecrackerNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 src_img_path: Optional[str] = None):
+
+        super().__init__(name, frame_buffer, output_buffer, enable_key)
+
+        if src_img_path is None:
+            self.src_img_path = 'https://user-images.githubusercontent' \
+                                '.com/28900607/149766281-6376055c-ed8b' \
+                                '-472b-991f-60e6ae6ee1da.gif'
+        src_img = cv2.VideoCapture(self.src_img_path)
+
+        self.frame_list = []
+        ret, frame = src_img.read()
+        while frame is not None:
+            self.frame_list.append(frame)
+            ret, frame = src_img.read()
+        self.num_frames = len(self.frame_list)
+        self.frame_idx = 0
+        self.frame_period = 4  # each frame in gif lasts for 4 frames in video
+
+    @staticmethod
+    def apply_firecracker_effect(img,
+                                 pose_results,
+                                 firecracker_img,
+                                 left_wrist_idx,
+                                 right_wrist_idx,
+                                 kpt_thr=0.5):
+        """Apply firecracker effect.
+        Args:
+            img (np.ndarray): Image data.
+            pose_results (list[dict]): The pose estimation results containing:
+                - "keypoints" ([K,3]): keypoint detection result in
+                    [x, y, score]
+            firecracker_img (np.ndarray): Firecracker image with white
+                background.
+            left_wrist_idx (int): Keypoint index of left wrist
+            right_wrist_idx (int): Keypoint index of right wrist
+            kpt_thr (float): The score threshold of required keypoints.
+        """
+
+        hm, wm = firecracker_img.shape[:2]
+        # anchor points in the firecracker mask
+        pts_src = np.array([[0. * wm, 0. * hm], [0. * wm, 1. * hm],
+                            [1. * wm, 0. * hm], [1. * wm, 1. * hm]],
+                           dtype=np.float32)
+
+        h, w = img.shape[:2]
+        h_tar = h / 3
+        w_tar = h_tar / hm * wm
+
+        for pose in pose_results:
+            kpts = pose['keypoints']
+
+            if kpts[left_wrist_idx, 2] > kpt_thr:
+                kpt_lwrist = kpts[left_wrist_idx, :2]
+                # anchor points in the image by eye positions
+                pts_tar = np.vstack([
+                    kpt_lwrist - [w_tar / 2, 0],
+                    kpt_lwrist - [w_tar / 2, -h_tar],
+                    kpt_lwrist + [w_tar / 2, 0],
+                    kpt_lwrist + [w_tar / 2, h_tar]
+                ])
+
+                h_mat, _ = cv2.findHomography(pts_src, pts_tar)
+                patch = cv2.warpPerspective(
+                    firecracker_img,
+                    h_mat,
+                    dsize=(img.shape[1], img.shape[0]),
+                    borderValue=(255, 255, 255))
+                #  mask the white background area in the patch with
+                # a threshold 200
+                mask = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY)
+                mask = (mask < 240).astype(np.uint8)
+                img = cv2.copyTo(patch, mask, img)
+
+            if kpts[right_wrist_idx, 2] > kpt_thr:
+                kpt_rwrist = kpts[right_wrist_idx, :2]
+
+                # anchor points in the image by eye positions
+                pts_tar = np.vstack([
+                    kpt_rwrist - [w_tar / 2, 0],
+                    kpt_rwrist - [w_tar / 2, -h_tar],
+                    kpt_rwrist + [w_tar / 2, 0],
+                    kpt_rwrist + [w_tar / 2, h_tar]
+                ])
+
+                h_mat, _ = cv2.findHomography(pts_src, pts_tar)
+                patch = cv2.warpPerspective(
+                    firecracker_img,
+                    h_mat,
+                    dsize=(img.shape[1], img.shape[0]),
+                    borderValue=(255, 255, 255))
+                #  mask the white background area in the patch with
+                # a threshold 200
+                mask = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY)
+                mask = (mask < 240).astype(np.uint8)
+                img = cv2.copyTo(patch, mask, img)
+
+        return img
+
+    def draw(self, frame_msg):
+        canvas = frame_msg.get_image()
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+
+        frame = self.frame_list[self.frame_idx // self.frame_period]
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+            preds = pose_result['preds']
+            left_wrist_idx, right_wrist_idx = get_wrist_keypoint_ids(model_cfg)
+
+            canvas = self.apply_firecracker_effect(canvas, preds, frame,
+                                                   left_wrist_idx,
+                                                   right_wrist_idx)
+        self.frame_idx = (self.frame_idx + 1) % (
+            self.num_frames * self.frame_period)
+
+        return canvas

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/helper_node.py

@@ -0,0 +1,296 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import logging
+import time
+from queue import Full, Queue
+from threading import Thread
+from typing import List, Optional, Union
+
+import cv2
+import numpy as np
+from mmcv import color_val
+
+from mmpose.utils.timer import RunningAverage
+from .builder import NODES
+from .node import Node
+
+try:
+    import psutil
+    psutil_proc = psutil.Process()
+except (ImportError, ModuleNotFoundError):
+    psutil_proc = None
+
+
+@NODES.register_module()
+class ModelResultBindingNode(Node):
+
+    def __init__(self, name: str, frame_buffer: str, result_buffer: str,
+                 output_buffer: Union[str, List[str]]):
+        super().__init__(name=name, enable=True)
+        self.synchronous = None
+
+        # Cache the latest model result
+        self.last_result_msg = None
+        self.last_output_msg = None
+
+        # Inference speed analysis
+        self.frame_fps = RunningAverage(window=10)
+        self.frame_lag = RunningAverage(window=10)
+        self.result_fps = RunningAverage(window=10)
+        self.result_lag = RunningAverage(window=10)
+
+        # Register buffers
+        # Note that essential buffers will be set in set_runner() because
+        # it depends on the runner.synchronous attribute.
+        self.register_input_buffer(result_buffer, 'result', essential=False)
+        self.register_input_buffer(frame_buffer, 'frame', essential=False)
+        self.register_output_buffer(output_buffer)
+
+    def set_runner(self, runner):
+        super().set_runner(runner)
+
+        # Set synchronous according to the runner
+        if runner.synchronous:
+            self.synchronous = True
+            essential_input = 'result'
+        else:
+            self.synchronous = False
+            essential_input = 'frame'
+
+        # Set essential input buffer according to the synchronous setting
+        for buffer_info in self._input_buffers:
+            if buffer_info.input_name == essential_input:
+                buffer_info.essential = True
+
+    def process(self, input_msgs):
+        result_msg = input_msgs['result']
+
+        # Update last result
+        if result_msg is not None:
+            # Update result FPS
+            if self.last_result_msg is not None:
+                self.result_fps.update(
+                    1.0 /
+                    (result_msg.timestamp - self.last_result_msg.timestamp))
+            # Update inference latency
+            self.result_lag.update(time.time() - result_msg.timestamp)
+            # Update last inference result
+            self.last_result_msg = result_msg
+
+        if not self.synchronous:
+            # Asynchronous mode: Bind the latest result with the current frame.
+            frame_msg = input_msgs['frame']
+
+            self.frame_lag.update(time.time() - frame_msg.timestamp)
+
+            # Bind result to frame
+            if self.last_result_msg is not None:
+                frame_msg.set_full_results(
+                    self.last_result_msg.get_full_results())
+                frame_msg.merge_route_info(
+                    self.last_result_msg.get_route_info())
+
+            output_msg = frame_msg
+
+        else:
+            # Synchronous mode: Directly output the frame that the model result
+            # was obtained from.
+            self.frame_lag.update(time.time() - result_msg.timestamp)
+            output_msg = result_msg
+
+        # Update frame fps and lag
+        if self.last_output_msg is not None:
+            self.frame_lag.update(time.time() - output_msg.timestamp)
+            self.frame_fps.update(
+                1.0 / (output_msg.timestamp - self.last_output_msg.timestamp))
+        self.last_output_msg = output_msg
+
+        return output_msg
+
+    def _get_node_info(self):
+        info = super()._get_node_info()
+        info['result_fps'] = self.result_fps.average()
+        info['result_lag (ms)'] = self.result_lag.average() * 1000
+        info['frame_fps'] = self.frame_fps.average()
+        info['frame_lag (ms)'] = self.frame_lag.average() * 1000
+        return info
+
+
+@NODES.register_module()
+class MonitorNode(Node):
+
+    _default_ignore_items = ['timestamp']
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = False,
+                 x_offset=20,
+                 y_offset=20,
+                 y_delta=15,
+                 text_color='black',
+                 background_color=(255, 183, 0),
+                 text_scale=0.4,
+                 ignore_items: Optional[List[str]] = None):
+        super().__init__(name=name, enable_key=enable_key, enable=enable)
+
+        self.x_offset = x_offset
+        self.y_offset = y_offset
+        self.y_delta = y_delta
+        self.text_color = color_val(text_color)
+        self.background_color = color_val(background_color)
+        self.text_scale = text_scale
+        if ignore_items is None:
+            self.ignore_items = self._default_ignore_items
+        else:
+            self.ignore_items = ignore_items
+
+        self.register_input_buffer(frame_buffer, 'frame', essential=True)
+        self.register_output_buffer(output_buffer)
+
+    def process(self, input_msgs):
+        frame_msg = input_msgs['frame']
+
+        frame_msg.update_route_info(
+            node_name='System Info',
+            node_type='dummy',
+            info=self._get_system_info())
+
+        img = frame_msg.get_image()
+        route_info = frame_msg.get_route_info()
+        img = self._show_route_info(img, route_info)
+
+        frame_msg.set_image(img)
+        return frame_msg
+
+    def _get_system_info(self):
+        sys_info = {}
+        if psutil_proc is not None:
+            sys_info['CPU(%)'] = psutil_proc.cpu_percent()
+            sys_info['Memory(%)'] = psutil_proc.memory_percent()
+        return sys_info
+
+    def _show_route_info(self, img, route_info):
+        canvas = np.full(img.shape, self.background_color, dtype=img.dtype)
+
+        x = self.x_offset
+        y = self.y_offset
+
+        max_len = 0
+
+        def _put_line(line=''):
+            nonlocal y, max_len
+            cv2.putText(canvas, line, (x, y), cv2.FONT_HERSHEY_DUPLEX,
+                        self.text_scale, self.text_color, 1)
+            y += self.y_delta
+            max_len = max(max_len, len(line))
+
+        for node_info in route_info:
+            title = f'{node_info["node"]}({node_info["node_type"]})'
+            _put_line(title)
+            for k, v in node_info['info'].items():
+                if k in self.ignore_items:
+                    continue
+                if isinstance(v, float):
+                    v = f'{v:.1f}'
+                _put_line(f'    {k}: {v}')
+
+        x1 = max(0, self.x_offset)
+        x2 = min(img.shape[1], int(x + max_len * self.text_scale * 20))
+        y1 = max(0, self.y_offset - self.y_delta)
+        y2 = min(img.shape[0], y)
+
+        src1 = canvas[y1:y2, x1:x2]
+        src2 = img[y1:y2, x1:x2]
+        img[y1:y2, x1:x2] = cv2.addWeighted(src1, 0.5, src2, 0.5, 0)
+
+        return img
+
+    def bypass(self, input_msgs):
+        return input_msgs['frame']
+
+
+@NODES.register_module()
+class RecorderNode(Node):
+    """Record the frames into a local file."""
+
+    def __init__(
+        self,
+        name: str,
+        frame_buffer: str,
+        output_buffer: Union[str, List[str]],
+        out_video_file: str,
+        out_video_fps: int = 30,
+        out_video_codec: str = 'mp4v',
+        buffer_size: int = 30,
+    ):
+        super().__init__(name=name, enable_key=None, enable=True)
+
+        self.queue = Queue(maxsize=buffer_size)
+        self.out_video_file = out_video_file
+        self.out_video_fps = out_video_fps
+        self.out_video_codec = out_video_codec
+        self.vwriter = None
+
+        # Register buffers
+        self.register_input_buffer(frame_buffer, 'frame', essential=True)
+        self.register_output_buffer(output_buffer)
+
+        # Start a new thread to write frame
+        self.t_record = Thread(target=self._record, args=(), daemon=True)
+        self.t_record.start()
+
+    def process(self, input_msgs):
+
+        frame_msg = input_msgs['frame']
+        img = frame_msg.get_image() if frame_msg is not None else None
+        img_queued = False
+
+        while not img_queued:
+            try:
+                self.queue.put(img, timeout=1)
+                img_queued = True
+                logging.info(f'{self.name}: recorder received one frame!')
+            except Full:
+                logging.info(f'{self.name}: recorder jamed!')
+
+        return frame_msg
+
+    def _record(self):
+
+        while True:
+
+            img = self.queue.get()
+
+            if img is None:
+                break
+
+            if self.vwriter is None:
+                fourcc = cv2.VideoWriter_fourcc(*self.out_video_codec)
+                fps = self.out_video_fps
+                frame_size = (img.shape[1], img.shape[0])
+                self.vwriter = cv2.VideoWriter(self.out_video_file, fourcc,
+                                               fps, frame_size)
+                assert self.vwriter.isOpened()
+
+            self.vwriter.write(img)
+
+        logging.info('Video recorder released!')
+        if self.vwriter is not None:
+            self.vwriter.release()
+
+    def on_exit(self):
+        try:
+            # Try putting a None into the output queue so the self.vwriter will
+            # be released after all queue frames have been written to file.
+            self.queue.put(None, timeout=1)
+            self.t_record.join(timeout=1)
+        except Full:
+            pass
+
+        if self.t_record.is_alive():
+            # Force to release self.vwriter
+            logging.info('Video recorder forced release!')
+            if self.vwriter is not None:
+                self.vwriter.release()

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/mmdet_node.py

@@ -0,0 +1,84 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from typing import List, Optional, Union
+
+from .builder import NODES
+from .node import Node
+
+try:
+    from mmdet.apis import inference_detector, init_detector
+    has_mmdet = True
+except (ImportError, ModuleNotFoundError):
+    has_mmdet = False
+
+
+@NODES.register_module()
+class DetectorNode(Node):
+
+    def __init__(self,
+                 name: str,
+                 model_config: str,
+                 model_checkpoint: str,
+                 input_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 device: str = 'cuda:0'):
+        # Check mmdetection is installed
+        assert has_mmdet, 'Please install mmdet to run the demo.'
+        super().__init__(name=name, enable_key=enable_key, enable=True)
+
+        self.model_config = model_config
+        self.model_checkpoint = model_checkpoint
+        self.device = device.lower()
+
+        # Init model
+        self.model = init_detector(
+            self.model_config,
+            self.model_checkpoint,
+            device=self.device.lower())
+
+        # Register buffers
+        self.register_input_buffer(input_buffer, 'input', essential=True)
+        self.register_output_buffer(output_buffer)
+
+    def bypass(self, input_msgs):
+        return input_msgs['input']
+
+    def process(self, input_msgs):
+        input_msg = input_msgs['input']
+
+        img = input_msg.get_image()
+
+        preds = inference_detector(self.model, img)
+        det_result = self._post_process(preds)
+
+        input_msg.add_detection_result(det_result, tag=self.name)
+        return input_msg
+
+    def _post_process(self, preds):
+        if isinstance(preds, tuple):
+            dets = preds[0]
+            segms = preds[1]
+        else:
+            dets = preds
+            segms = [None] * len(dets)
+
+        assert len(dets) == len(self.model.CLASSES)
+        assert len(segms) == len(self.model.CLASSES)
+        result = {'preds': [], 'model_cfg': self.model.cfg.copy()}
+
+        for i, (cls_name, bboxes,
+                masks) in enumerate(zip(self.model.CLASSES, dets, segms)):
+            if masks is None:
+                masks = [None] * len(bboxes)
+            else:
+                assert len(masks) == len(bboxes)
+
+            preds_i = [{
+                'cls_id': i,
+                'label': cls_name,
+                'bbox': bbox,
+                'mask': mask,
+            } for (bbox, mask) in zip(bboxes, masks)]
+            result['preds'].extend(preds_i)
+
+        return result

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/mmpose_node.py

@@ -0,0 +1,122 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import time
+from typing import Dict, List, Optional, Union
+
+from mmpose.apis import (get_track_id, inference_top_down_pose_model,
+                         init_pose_model)
+from ..utils import Message
+from .builder import NODES
+from .node import Node
+
+
+@NODES.register_module()
+class TopDownPoseEstimatorNode(Node):
+
+    def __init__(self,
+                 name: str,
+                 model_config: str,
+                 model_checkpoint: str,
+                 input_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 enable: bool = True,
+                 device: str = 'cuda:0',
+                 cls_ids: Optional[List] = None,
+                 cls_names: Optional[List] = None,
+                 bbox_thr: float = 0.5):
+        super().__init__(name=name, enable_key=enable_key, enable=enable)
+
+        # Init model
+        self.model_config = model_config
+        self.model_checkpoint = model_checkpoint
+        self.device = device.lower()
+
+        self.cls_ids = cls_ids
+        self.cls_names = cls_names
+        self.bbox_thr = bbox_thr
+
+        # Init model
+        self.model = init_pose_model(
+            self.model_config,
+            self.model_checkpoint,
+            device=self.device.lower())
+
+        # Store history for pose tracking
+        self.track_info = {
+            'next_id': 0,
+            'last_pose_preds': [],
+            'last_time': None
+        }
+
+        # Register buffers
+        self.register_input_buffer(input_buffer, 'input', essential=True)
+        self.register_output_buffer(output_buffer)
+
+    def bypass(self, input_msgs):
+        return input_msgs['input']
+
+    def process(self, input_msgs: Dict[str, Message]) -> Message:
+
+        input_msg = input_msgs['input']
+        img = input_msg.get_image()
+        det_results = input_msg.get_detection_results()
+
+        if det_results is None:
+            raise ValueError(
+                'No detection results are found in the frame message.'
+                f'{self.__class__.__name__} should be used after a '
+                'detector node.')
+
+        full_det_preds = []
+        for det_result in det_results:
+            det_preds = det_result['preds']
+            if self.cls_ids:
+                # Filter detection results by class ID
+                det_preds = [
+                    p for p in det_preds if p['cls_id'] in self.cls_ids
+                ]
+            elif self.cls_names:
+                # Filter detection results by class name
+                det_preds = [
+                    p for p in det_preds if p['label'] in self.cls_names
+                ]
+            full_det_preds.extend(det_preds)
+
+        # Inference pose
+        pose_preds, _ = inference_top_down_pose_model(
+            self.model,
+            img,
+            full_det_preds,
+            bbox_thr=self.bbox_thr,
+            format='xyxy')
+
+        # Pose tracking
+        current_time = time.time()
+        if self.track_info['last_time'] is None:
+            fps = None
+        elif self.track_info['last_time'] >= current_time:
+            fps = None
+        else:
+            fps = 1.0 / (current_time - self.track_info['last_time'])
+
+        pose_preds, next_id = get_track_id(
+            pose_preds,
+            self.track_info['last_pose_preds'],
+            self.track_info['next_id'],
+            use_oks=False,
+            tracking_thr=0.3,
+            use_one_euro=True,
+            fps=fps)
+
+        self.track_info['next_id'] = next_id
+        self.track_info['last_pose_preds'] = pose_preds.copy()
+        self.track_info['last_time'] = current_time
+
+        pose_result = {
+            'preds': pose_preds,
+            'model_cfg': self.model.cfg.copy(),
+        }
+
+        input_msg.add_pose_result(pose_result, tag=self.name)
+
+        return input_msg

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/node.py

@@ -0,0 +1,372 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import logging
+import time
+from abc import ABCMeta, abstractmethod
+from dataclasses import dataclass
+from queue import Empty
+from threading import Thread
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+from mmcv.utils.misc import is_method_overridden
+
+from mmpose.utils import StopWatch
+from ..utils import Message, VideoEndingMessage, limit_max_fps
+
+
+@dataclass
+class BufferInfo():
+    """Dataclass for buffer information."""
+    buffer_name: str
+    input_name: Optional[str] = None
+    essential: bool = False
+
+
+@dataclass
+class EventInfo():
+    """Dataclass for event handler information."""
+    event_name: str
+    is_keyboard: bool = False
+    handler_func: Optional[Callable] = None
+
+
+class Node(Thread, metaclass=ABCMeta):
+    """Base interface of functional module.
+
+    Parameters:
+        name (str, optional): The node name (also thread name).
+        enable_key (str|int, optional): Set a hot-key to toggle enable/disable
+            of the node. If an int value is given, it will be treated as an
+            ascii code of a key. Please note:
+                1. If enable_key is set, the bypass method need to be
+                    overridden to define the node behavior when disabled
+                2. Some hot-key has been use for particular use. For example:
+                    'q', 'Q' and 27 are used for quit
+            Default: None
+        max_fps (int): Maximum FPS of the node. This is to avoid the node
+            running unrestrictedly and causing large resource consuming.
+            Default: 30
+        input_check_interval (float): Minimum interval (in millisecond) between
+            checking if input is ready. Default: 0.001
+        enable (bool): Default enable/disable status. Default: True.
+        daemon (bool): Whether node is a daemon. Default: True.
+    """
+
+    def __init__(self,
+                 name: Optional[str] = None,
+                 enable_key: Optional[Union[str, int]] = None,
+                 max_fps: int = 30,
+                 input_check_interval: float = 0.01,
+                 enable: bool = True,
+                 daemon=False):
+        super().__init__(name=name, daemon=daemon)
+        self._runner = None
+        self._enabled = enable
+        self.enable_key = enable_key
+        self.max_fps = max_fps
+        self.input_check_interval = input_check_interval
+
+        # A partitioned buffer manager the runner's buffer manager that
+        # only accesses the buffers related to the node
+        self._buffer_manager = None
+
+        # Input/output buffers are a list of registered buffers' information
+        self._input_buffers = []
+        self._output_buffers = []
+
+        # Event manager is a copy of assigned runner's event manager
+        self._event_manager = None
+
+        # A list of registered event information
+        # See register_event() for more information
+        # Note that we recommend to handle events in nodes by registering
+        # handlers, but one can still access the raw event by _event_manager
+        self._registered_events = []
+
+        # A list of (listener_threads, event_info)
+        # See set_runner() for more information
+        self._event_listener_threads = []
+
+        # A timer to calculate node FPS
+        self._timer = StopWatch(window=10)
+
+        # Register enable toggle key
+        if self.enable_key:
+            # If the node allows toggling enable, it should override the
+            # `bypass` method to define the node behavior when disabled.
+            if not is_method_overridden('bypass', Node, self.__class__):
+                raise NotImplementedError(
+                    f'The node {self.__class__} does not support toggling'
+                    'enable but got argument `enable_key`. To support toggling'
+                    'enable, please override the `bypass` method of the node.')
+
+            self.register_event(
+                event_name=self.enable_key,
+                is_keyboard=True,
+                handler_func=self._toggle_enable,
+            )
+
+    @property
+    def registered_buffers(self):
+        return self._input_buffers + self._output_buffers
+
+    @property
+    def registered_events(self):
+        return self._registered_events.copy()
+
+    def _toggle_enable(self):
+        self._enabled = not self._enabled
+
+    def register_input_buffer(self,
+                              buffer_name: str,
+                              input_name: str,
+                              essential: bool = False):
+        """Register an input buffer, so that Node can automatically check if
+        data is ready, fetch data from the buffers and format the inputs to
+        feed into `process` method.
+
+        This method can be invoked multiple times to register multiple input
+        buffers.
+
+        The subclass of Node should invoke `register_input_buffer` in its
+        `__init__` method.
+
+        Args:
+            buffer_name (str): The name of the buffer
+            input_name (str): The name of the fetched message from the
+                corresponding buffer
+            essential (bool): An essential input means the node will wait
+                until the input is ready before processing. Otherwise, an
+                inessential input will not block the processing, instead
+                a None will be fetched if the buffer is not ready.
+        """
+        buffer_info = BufferInfo(buffer_name, input_name, essential)
+        self._input_buffers.append(buffer_info)
+
+    def register_output_buffer(self, buffer_name: Union[str, List[str]]):
+        """Register one or multiple output buffers, so that the Node can
+        automatically send the output of the `process` method to these buffers.
+
+        The subclass of Node should invoke `register_output_buffer` in its
+        `__init__` method.
+
+        Args:
+            buffer_name (str|list): The name(s) of the output buffer(s).
+        """
+
+        if not isinstance(buffer_name, list):
+            buffer_name = [buffer_name]
+
+        for name in buffer_name:
+            buffer_info = BufferInfo(name)
+            self._output_buffers.append(buffer_info)
+
+    def register_event(self,
+                       event_name: str,
+                       is_keyboard: bool = False,
+                       handler_func: Optional[Callable] = None):
+        """Register an event. All events used in the node need to be registered
+        in __init__(). If a callable handler is given, a thread will be create
+        to listen and handle the event when the node starts.
+
+        Args:
+            Args:
+            event_name (str|int): The event name. If is_keyboard==True,
+                event_name should be a str (as char) or an int (as ascii)
+            is_keyboard (bool): Indicate whether it is an keyboard
+                event. If True, the argument event_name will be regarded as a
+                key indicator.
+            handler_func (callable, optional): The event handler function,
+                which should be a collable object with no arguments or
+                return values. Default: None.
+        """
+        event_info = EventInfo(event_name, is_keyboard, handler_func)
+        self._registered_events.append(event_info)
+
+    def set_runner(self, runner):
+        # Get partitioned buffer manager
+        buffer_names = [
+            buffer.buffer_name
+            for buffer in self._input_buffers + self._output_buffers
+        ]
+        self._buffer_manager = runner.buffer_manager.get_sub_manager(
+            buffer_names)
+
+        # Get event manager
+        self._event_manager = runner.event_manager
+
+    def _get_input_from_buffer(self) -> Tuple[bool, Optional[Dict]]:
+        """Get and pack input data if it's ready. The function returns a tuple
+        of a status flag and a packed data dictionary. If input_buffer is
+        ready, the status flag will be True, and the packed data is a dict
+        whose items are buffer names and corresponding messages (unready
+        additional buffers will give a `None`). Otherwise, the status flag is
+        False and the packed data is None.
+
+        Returns:
+            bool: status flag
+            dict[str, Message]: the packed inputs where the key is the buffer
+                name and the value is the Message got from the corresponding
+                buffer.
+        """
+        buffer_manager = self._buffer_manager
+
+        if buffer_manager is None:
+            raise ValueError(f'{self.name}: Runner not set!')
+
+        # Check that essential buffers are ready
+        for buffer_info in self._input_buffers:
+            if buffer_info.essential and buffer_manager.is_empty(
+                    buffer_info.buffer_name):
+                return False, None
+
+        # Default input
+        result = {
+            buffer_info.input_name: None
+            for buffer_info in self._input_buffers
+        }
+
+        for buffer_info in self._input_buffers:
+            try:
+                result[buffer_info.input_name] = buffer_manager.get(
+                    buffer_info.buffer_name, block=False)
+            except Empty:
+                if buffer_info.essential:
+                    # Return unsuccessful flag if any
+                    # essential input is unready
+                    return False, None
+
+        return True, result
+
+    def _send_output_to_buffers(self, output_msg):
+        """Send output of the process method to registered output buffers.
+
+        Args:
+            output_msg (Message): output message
+            force (bool, optional): If True, block until the output message
+                has been put into all output buffers. Default: False
+        """
+        for buffer_info in self._output_buffers:
+            buffer_name = buffer_info.buffer_name
+            self._buffer_manager.put_force(buffer_name, output_msg)
+
+    @abstractmethod
+    def process(self, input_msgs: Dict[str, Message]) -> Union[Message, None]:
+        """The core method that implement the function of the node. This method
+        will be invoked when the node is enabled and the input data is ready.
+
+        All subclasses of Node should override this method.
+
+        Args:
+            input_msgs (dict): The input data collected from the buffers. For
+                each item, the key is the `input_name` of the registered input
+                buffer, while the value is a Message instance fetched from the
+                buffer (or None if the buffer is unessential and not ready).
+
+        Returns:
+            Message: The output message of the node. It will be send to all
+                registered output buffers.
+        """
+
+    def bypass(self, input_msgs: Dict[str, Message]) -> Union[Message, None]:
+        """The method that defines the node behavior when disabled. Note that
+        if the node has an `enable_key`, this method should be override.
+
+        The method input/output is same as it of `process` method.
+
+        Args:
+            input_msgs (dict): The input data collected from the buffers. For
+                each item, the key is the `input_name` of the registered input
+                buffer, while the value is a Message instance fetched from the
+                buffer (or None if the buffer is unessential and not ready).
+
+        Returns:
+            Message: The output message of the node. It will be send to all
+                registered output buffers.
+        """
+        raise NotImplementedError
+
+    def _get_node_info(self):
+        """Get route information of the node."""
+        info = {'fps': self._timer.report('_FPS_'), 'timestamp': time.time()}
+        return info
+
+    def on_exit(self):
+        """This method will be invoked on event `_exit_`.
+
+        Subclasses should override this method to specifying the exiting
+        behavior.
+        """
+
+    def run(self):
+        """Method representing the Node's activity.
+
+        This method override the standard run() method of Thread. Users should
+        not override this method in subclasses.
+        """
+
+        logging.info(f'Node {self.name} starts')
+
+        # Create event listener threads
+        for event_info in self._registered_events:
+
+            if event_info.handler_func is None:
+                continue
+
+            def event_listener():
+                while True:
+                    with self._event_manager.wait_and_handle(
+                            event_info.event_name, event_info.is_keyboard):
+                        event_info.handler_func()
+
+            t_listener = Thread(target=event_listener, args=(), daemon=True)
+            t_listener.start()
+            self._event_listener_threads.append(t_listener)
+
+        # Loop
+        while True:
+            # Exit
+            if self._event_manager.is_set('_exit_'):
+                self.on_exit()
+                break
+
+            # Check if input is ready
+            input_status, input_msgs = self._get_input_from_buffer()
+
+            # Input is not ready
+            if not input_status:
+                time.sleep(self.input_check_interval)
+                continue
+
+            # If a VideoEndingMessage is received, broadcast the signal
+            # without invoking process() or bypass()
+            video_ending = False
+            for _, msg in input_msgs.items():
+                if isinstance(msg, VideoEndingMessage):
+                    self._send_output_to_buffers(msg)
+                    video_ending = True
+                    break
+
+            if video_ending:
+                self.on_exit()
+                break
+
+            # Check if enabled
+            if not self._enabled:
+                # Override bypass method to define node behavior when disabled
+                output_msg = self.bypass(input_msgs)
+            else:
+                with self._timer.timeit():
+                    with limit_max_fps(self.max_fps):
+                        # Process
+                        output_msg = self.process(input_msgs)
+
+                if output_msg:
+                    # Update route information
+                    node_info = self._get_node_info()
+                    output_msg.update_route_info(node=self, info=node_info)
+
+            # Send output message
+            if output_msg is not None:
+                self._send_output_to_buffers(output_msg)
+
+        logging.info(f'{self.name}: process ending.')

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/valentinemagic_node.py

@@ -0,0 +1,340 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import time
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Union
+
+import cv2
+import numpy as np
+
+from ..utils import (FrameMessage, get_eye_keypoint_ids, get_hand_keypoint_ids,
+                     get_mouth_keypoint_ids, load_image_from_disk_or_url)
+from .builder import NODES
+from .frame_drawing_node import FrameDrawingNode
+
+
+@dataclass
+class HeartInfo():
+    """Dataclass for heart information."""
+    heart_type: int
+    start_time: float
+    start_pos: Tuple[int, int]
+    end_pos: Tuple[int, int]
+
+
+@NODES.register_module()
+class ValentineMagicNode(FrameDrawingNode):
+
+    def __init__(self,
+                 name: str,
+                 frame_buffer: str,
+                 output_buffer: Union[str, List[str]],
+                 enable_key: Optional[Union[str, int]] = None,
+                 kpt_vis_thr: float = 0.3,
+                 hand_heart_angle_thr: float = 90.0,
+                 longest_duration: float = 2.0,
+                 largest_ratio: float = 0.25,
+                 hand_heart_img_path: Optional[str] = None,
+                 flying_heart_img_path: Optional[str] = None,
+                 hand_heart_dis_ratio_thr: float = 1.0,
+                 flying_heart_dis_ratio_thr: float = 3.5,
+                 num_persons: int = 2):
+
+        super().__init__(
+            name, frame_buffer, output_buffer, enable_key=enable_key)
+
+        if hand_heart_img_path is None:
+            hand_heart_img_path = 'https://user-images.githubusercontent.com/'\
+                           '87690686/149731850-ea946766-a4e8-4efa-82f5'\
+                           '-e2f0515db8ae.png'
+        if flying_heart_img_path is None:
+            flying_heart_img_path = 'https://user-images.githubusercontent.'\
+                                    'com/87690686/153554948-937ce496-33dd-4'\
+                                    '9ab-9829-0433fd7c13c4.png'
+
+        self.hand_heart = load_image_from_disk_or_url(hand_heart_img_path)
+        self.flying_heart = load_image_from_disk_or_url(flying_heart_img_path)
+
+        self.kpt_vis_thr = kpt_vis_thr
+        self.hand_heart_angle_thr = hand_heart_angle_thr
+        self.hand_heart_dis_ratio_thr = hand_heart_dis_ratio_thr
+        self.flying_heart_dis_ratio_thr = flying_heart_dis_ratio_thr
+        self.longest_duration = longest_duration
+        self.largest_ratio = largest_ratio
+        self.num_persons = num_persons
+
+        # record the heart infos for each person
+        self.heart_infos = {}
+
+    def _cal_distance(self, p1: np.ndarray, p2: np.ndarray) -> np.float64:
+        """calculate the distance of points p1 and p2."""
+        return np.sqrt((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2)
+
+    def _cal_angle(self, p1: np.ndarray, p2: np.ndarray, p3: np.ndarray,
+                   p4: np.ndarray) -> np.float64:
+        """calculate the angle of vectors v1(constructed by points p2 and p1)
+        and v2(constructed by points p4 and p3)"""
+        v1 = p2 - p1
+        v2 = p4 - p3
+
+        vector_prod = v1[0] * v2[0] + v1[1] * v2[1]
+        length_prod = np.sqrt(pow(v1[0], 2) + pow(v1[1], 2)) * np.sqrt(
+            pow(v2[0], 2) + pow(v2[1], 2))
+        cos = vector_prod * 1.0 / (length_prod * 1.0 + 1e-6)
+
+        return (np.arccos(cos) / np.pi) * 180
+
+    def _check_heart(self, pred: Dict[str,
+                                      np.ndarray], hand_indices: List[int],
+                     mouth_index: int, eye_indices: List[int]) -> int:
+        """Check the type of Valentine Magic based on the pose results and
+        keypoint indices of hand, mouth. and eye.
+
+        Args:
+            pred(dict): The pose estimation results containing:
+                - "keypoints" (np.ndarray[K,3]): keypoint detection result
+                                                 in [x, y, score]
+            hand_indices(list[int]): keypoint indices of hand
+            mouth_index(int): keypoint index of mouth
+            eye_indices(list[int]): keypoint indices of eyes
+
+        Returns:
+            int: a number representing the type of heart pose,
+                 0: None, 1: hand heart, 2: left hand blow kiss,
+                 3: right hand blow kiss
+        """
+        kpts = pred['keypoints']
+
+        left_eye_idx, right_eye_idx = eye_indices
+        left_eye_pos = kpts[left_eye_idx][:2]
+        right_eye_pos = kpts[right_eye_idx][:2]
+        eye_dis = self._cal_distance(left_eye_pos, right_eye_pos)
+
+        # these indices are corresoponding to the following keypoints:
+        # left_hand_root, left_pinky_finger1,
+        # left_pinky_finger3, left_pinky_finger4,
+        # right_hand_root, right_pinky_finger1
+        # right_pinky_finger3, right_pinky_finger4
+
+        both_hands_vis = True
+        for i in [0, 17, 19, 20, 21, 38, 40, 41]:
+            if kpts[hand_indices[i]][2] < self.kpt_vis_thr:
+                both_hands_vis = False
+
+        if both_hands_vis:
+            p1 = kpts[hand_indices[20]][:2]
+            p2 = kpts[hand_indices[19]][:2]
+            p3 = kpts[hand_indices[17]][:2]
+            p4 = kpts[hand_indices[0]][:2]
+            left_angle = self._cal_angle(p1, p2, p3, p4)
+
+            p1 = kpts[hand_indices[41]][:2]
+            p2 = kpts[hand_indices[40]][:2]
+            p3 = kpts[hand_indices[38]][:2]
+            p4 = kpts[hand_indices[21]][:2]
+            right_angle = self._cal_angle(p1, p2, p3, p4)
+
+            hand_dis = self._cal_distance(kpts[hand_indices[20]][:2],
+                                          kpts[hand_indices[41]][:2])
+
+            if (left_angle < self.hand_heart_angle_thr
+                    and right_angle < self.hand_heart_angle_thr
+                    and hand_dis / eye_dis < self.hand_heart_dis_ratio_thr):
+                return 1
+
+        # these indices are corresoponding to the following keypoints:
+        # left_middle_finger1, left_middle_finger4,
+        left_hand_vis = True
+        for i in [9, 12]:
+            if kpts[hand_indices[i]][2] < self.kpt_vis_thr:
+                left_hand_vis = False
+                break
+        # right_middle_finger1, right_middle_finger4
+
+        right_hand_vis = True
+        for i in [30, 33]:
+            if kpts[hand_indices[i]][2] < self.kpt_vis_thr:
+                right_hand_vis = False
+                break
+
+        mouth_vis = True
+        if kpts[mouth_index][2] < self.kpt_vis_thr:
+            mouth_vis = False
+
+        if (not left_hand_vis and not right_hand_vis) or not mouth_vis:
+            return 0
+
+        mouth_pos = kpts[mouth_index]
+
+        left_mid_hand_pos = (kpts[hand_indices[9]][:2] +
+                             kpts[hand_indices[12]][:2]) / 2
+        lefthand_mouth_dis = self._cal_distance(left_mid_hand_pos, mouth_pos)
+
+        if lefthand_mouth_dis / eye_dis < self.flying_heart_dis_ratio_thr:
+            return 2
+
+        right_mid_hand_pos = (kpts[hand_indices[30]][:2] +
+                              kpts[hand_indices[33]][:2]) / 2
+        righthand_mouth_dis = self._cal_distance(right_mid_hand_pos, mouth_pos)
+
+        if righthand_mouth_dis / eye_dis < self.flying_heart_dis_ratio_thr:
+            return 3
+
+        return 0
+
+    def _get_heart_route(self, heart_type: int, cur_pred: Dict[str,
+                                                               np.ndarray],
+                         tar_pred: Dict[str,
+                                        np.ndarray], hand_indices: List[int],
+                         mouth_index: int) -> Tuple[int, int]:
+        """get the start and end position of the heart, based on two keypoint
+        results and keypoint indices of hand and mouth.
+
+        Args:
+            cur_pred(dict): The pose estimation results of current person,
+                containing: the following keys:
+                - "keypoints" (np.ndarray[K,3]): keypoint detection result
+                                                 in [x, y, score]
+            tar_pred(dict): The pose estimation results of target person,
+                containing: the following keys:
+                - "keypoints" (np.ndarray[K,3]): keypoint detection result
+                                                 in [x, y, score]
+            hand_indices(list[int]): keypoint indices of hand
+            mouth_index(int): keypoint index of mouth
+
+        Returns:
+            tuple(int): the start position of heart
+            tuple(int): the end position of heart
+        """
+        cur_kpts = cur_pred['keypoints']
+
+        assert heart_type in [1, 2,
+                              3], 'Can not determine the type of heart effect'
+
+        if heart_type == 1:
+            p1 = cur_kpts[hand_indices[20]][:2]
+            p2 = cur_kpts[hand_indices[41]][:2]
+        elif heart_type == 2:
+            p1 = cur_kpts[hand_indices[9]][:2]
+            p2 = cur_kpts[hand_indices[12]][:2]
+        elif heart_type == 3:
+            p1 = cur_kpts[hand_indices[30]][:2]
+            p2 = cur_kpts[hand_indices[33]][:2]
+
+        cur_x, cur_y = (p1 + p2) / 2
+        # the mid point of two fingers
+        start_pos = (int(cur_x), int(cur_y))
+
+        tar_kpts = tar_pred['keypoints']
+        end_pos = tar_kpts[mouth_index][:2]
+
+        return start_pos, end_pos
+
+    def _draw_heart(self, canvas: np.ndarray, heart_info: HeartInfo,
+                    t_pass: float) -> np.ndarray:
+        """draw the heart according to heart info and time."""
+        start_x, start_y = heart_info.start_pos
+        end_x, end_y = heart_info.end_pos
+
+        scale = t_pass / self.longest_duration
+
+        max_h, max_w = canvas.shape[:2]
+        hm, wm = self.largest_ratio * max_h, self.largest_ratio * max_h
+        new_h, new_w = int(hm * scale), int(wm * scale)
+
+        x = int(start_x + scale * (end_x - start_x))
+        y = int(start_y + scale * (end_y - start_y))
+
+        y1 = max(0, y - int(new_h / 2))
+        y2 = min(max_h - 1, y + int(new_h / 2))
+
+        x1 = max(0, x - int(new_w / 2))
+        x2 = min(max_w - 1, x + int(new_w / 2))
+
+        target = canvas[y1:y2 + 1, x1:x2 + 1].copy()
+        new_h, new_w = target.shape[:2]
+
+        if new_h == 0 or new_w == 0:
+            return canvas
+
+        assert heart_info.heart_type in [
+            1, 2, 3
+        ], 'Can not determine the type of heart effect'
+        if heart_info.heart_type == 1:  # hand heart
+            patch = self.hand_heart.copy()
+        elif heart_info.heart_type >= 2:  # hand blow kiss
+            patch = self.flying_heart.copy()
+            if heart_info.start_pos[0] > heart_info.end_pos[0]:
+                patch = patch[:, ::-1]
+
+        patch = cv2.resize(patch, (new_w, new_h))
+        mask = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY)
+        mask = (mask < 100)[..., None].astype(np.float32) * 0.8
+
+        canvas[y1:y2 + 1, x1:x2 + 1] = patch * mask + target * (1 - mask)
+
+        return canvas
+
+    def draw(self, frame_msg: FrameMessage) -> np.ndarray:
+        canvas = frame_msg.get_image()
+
+        pose_results = frame_msg.get_pose_results()
+        if not pose_results:
+            return canvas
+
+        for pose_result in pose_results:
+            model_cfg = pose_result['model_cfg']
+
+            preds = [pred.copy() for pred in pose_result['preds']]
+            # if number of persons in the image is less than 2,
+            # no heart effect will be triggered
+            if len(preds) < self.num_persons:
+                continue
+
+            # if number of persons in the image is more than 2,
+            # only use the first two pose results
+            preds = preds[:self.num_persons]
+            ids = [preds[i]['track_id'] for i in range(self.num_persons)]
+
+            for id in self.heart_infos.copy():
+                if id not in ids:
+                    # if the id of a person not in previous heart_infos,
+                    # delete the corresponding field
+                    del self.heart_infos[id]
+
+            for i in range(self.num_persons):
+                id = preds[i]['track_id']
+
+                # if the predicted person in previous heart_infos,
+                # draw the heart
+                if id in self.heart_infos.copy():
+                    t_pass = time.time() - self.heart_infos[id].start_time
+
+                    # the time passed since last heart pose less than
+                    # longest_duration, continue to draw the heart
+                    if t_pass < self.longest_duration:
+                        canvas = self._draw_heart(canvas, self.heart_infos[id],
+                                                  t_pass)
+                    # reset corresponding heart info
+                    else:
+                        del self.heart_infos[id]
+                else:
+                    hand_indices = get_hand_keypoint_ids(model_cfg)
+                    mouth_index = get_mouth_keypoint_ids(model_cfg)
+                    eye_indices = get_eye_keypoint_ids(model_cfg)
+
+                    # check the type of Valentine Magic based on pose results
+                    # and keypoint indices of hand and mouth
+                    heart_type = self._check_heart(preds[i], hand_indices,
+                                                   mouth_index, eye_indices)
+                    # trigger a Valentine Magic effect
+                    if heart_type:
+                        # get the route of heart
+                        start_pos, end_pos = self._get_heart_route(
+                            heart_type, preds[i],
+                            preds[self.num_persons - 1 - i], hand_indices,
+                            mouth_index)
+                        start_time = time.time()
+                        self.heart_infos[id] = HeartInfo(
+                            heart_type, start_time, start_pos, end_pos)
+
+        return canvas

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/nodes/xdwendwen_node.py

@@ -0,0 +1,240 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import json
+from dataclasses import dataclass
+from typing import List, Tuple, Union
+
+import cv2
+import numpy as np
+
+from mmpose.datasets.dataset_info import DatasetInfo
+from ..utils import load_image_from_disk_or_url
+from .builder import NODES
+from .frame_drawing_node import FrameDrawingNode
+
+
+@dataclass
+class DynamicInfo:
+    pos_curr: Tuple[int, int] = (0, 0)
+    pos_step: Tuple[int, int] = (0, 0)
+    step_curr: int = 0
+
+
+@NODES.register_module()
+class XDwenDwenNode(FrameDrawingNode):
+    """An effect drawing node that captures the face of a cat or dog and blend
+    it into a Bing-Dwen-Dwen (the mascot of 2022 Beijing Winter Olympics).
+
+    Parameters:
+        name (str, optional): The node name (also thread name).
+        frame_buffer (str): The name of the input buffer.
+        output_buffer (str | list): The name(s) of the output buffer(s).
+        mode_key (str | int): A hot key to switch the background image.
+        resource_file (str): The annotation file of resource images, which
+            should be in Labelbee format and contain both facial keypoint and
+            region annotations.
+        out_shape (tuple): The shape of output frame in (width, height).
+    """
+
+    dynamic_scale = 0.15
+    dynamic_max_step = 15
+
+    def __init__(
+        self,
+        name: str,
+        frame_buffer: str,
+        output_buffer: Union[str, List[str]],
+        mode_key: Union[str, int],
+        resource_file: str,
+        out_shape: Tuple[int, int] = (480, 480),
+        rigid_transform: bool = True,
+    ):
+        super().__init__(name, frame_buffer, output_buffer, enable=True)
+
+        self.mode_key = mode_key
+        self.mode_index = 0
+        self.out_shape = out_shape
+        self.rigid = rigid_transform
+
+        self.latest_pred = None
+
+        self.dynamic_info = DynamicInfo()
+
+        self.register_event(
+            self.mode_key, is_keyboard=True, handler_func=self.switch_mode)
+
+        self._init_resource(resource_file)
+
+    def _init_resource(self, resource_file):
+
+        # The resource_file is a JSON file that contains the facial
+        # keypoint and mask annotation information of the resource files.
+        # The annotations should follow the label-bee standard format.
+        # See https://github.com/open-mmlab/labelbee-client for details.
+        with open(resource_file) as f:
+            anns = json.load(f)
+        resource_infos = []
+
+        for ann in anns:
+            # Load image
+            img = load_image_from_disk_or_url(ann['url'])
+            # Load result
+            rst = json.loads(ann['result'])
+
+            # Check facial keypoint information
+            assert rst['step_1']['toolName'] == 'pointTool'
+            assert len(rst['step_1']['result']) == 3
+
+            keypoints = sorted(
+                rst['step_1']['result'], key=lambda x: x['order'])
+            keypoints = np.array([[pt['x'], pt['y']] for pt in keypoints])
+
+            # Check facial mask
+            assert rst['step_2']['toolName'] == 'polygonTool'
+            assert len(rst['step_2']['result']) == 1
+            assert len(rst['step_2']['result'][0]['pointList']) > 2
+
+            mask_pts = np.array(
+                [[pt['x'], pt['y']]
+                 for pt in rst['step_2']['result'][0]['pointList']])
+
+            mul = 1.0 + self.dynamic_scale
+
+            w_scale = self.out_shape[0] / img.shape[1] * mul
+            h_scale = self.out_shape[1] / img.shape[0] * mul
+
+            img = cv2.resize(
+                img,
+                dsize=None,
+                fx=w_scale,
+                fy=h_scale,
+                interpolation=cv2.INTER_CUBIC)
+
+            keypoints *= [w_scale, h_scale]
+            mask_pts *= [w_scale, h_scale]
+
+            mask = cv2.fillPoly(
+                np.zeros(img.shape[:2], dtype=np.uint8),
+                [mask_pts.astype(np.int32)],
+                color=1)
+
+            res = {
+                'img': img,
+                'keypoints': keypoints,
+                'mask': mask,
+            }
+            resource_infos.append(res)
+
+        self.resource_infos = resource_infos
+
+        self._reset_dynamic()
+
+    def switch_mode(self):
+        self.mode_index = (self.mode_index + 1) % len(self.resource_infos)
+
+    def _reset_dynamic(self):
+        x_tar = np.random.randint(int(self.out_shape[0] * self.dynamic_scale))
+        y_tar = np.random.randint(int(self.out_shape[1] * self.dynamic_scale))
+
+        x_step = (x_tar -
+                  self.dynamic_info.pos_curr[0]) / self.dynamic_max_step
+        y_step = (y_tar -
+                  self.dynamic_info.pos_curr[1]) / self.dynamic_max_step
+
+        self.dynamic_info.pos_step = (x_step, y_step)
+        self.dynamic_info.step_curr = 0
+
+    def draw(self, frame_msg):
+
+        full_pose_results = frame_msg.get_pose_results()
+
+        pred = None
+        if full_pose_results:
+            for pose_results in full_pose_results:
+                if not pose_results['preds']:
+                    continue
+
+                pred = pose_results['preds'][0].copy()
+                pred['dataset'] = DatasetInfo(pose_results['model_cfg'].data.
+                                              test.dataset_info).dataset_name
+
+                self.latest_pred = pred
+                break
+
+        # Use the latest pose result if there is none available in
+        # the current frame.
+        if pred is None:
+            pred = self.latest_pred
+
+        # Get the background image and facial annotations
+        res = self.resource_infos[self.mode_index]
+        img = frame_msg.get_image()
+        canvas = res['img'].copy()
+        mask = res['mask']
+        kpts_tar = res['keypoints']
+
+        if pred is not None:
+            if pred['dataset'] == 'ap10k':
+                # left eye: 0, right eye: 1, nose: 2
+                kpts_src = pred['keypoints'][[0, 1, 2], :2]
+            elif pred['dataset'] == 'coco_wholebody':
+                # left eye: 1, right eye 2, nose: 0
+                kpts_src = pred['keypoints'][[1, 2, 0], :2]
+            else:
+                raise ValueError('Can not obtain face landmark information'
+                                 f'from dataset: {pred["type"]}')
+
+            trans_mat = self._get_transform(kpts_src, kpts_tar)
+
+            warp = cv2.warpAffine(img, trans_mat, dsize=canvas.shape[:2])
+            cv2.copyTo(warp, mask, canvas)
+
+        # Add random movement to the background
+        xc, yc = self.dynamic_info.pos_curr
+        xs, ys = self.dynamic_info.pos_step
+        w, h = self.out_shape
+
+        x = min(max(int(xc), 0), canvas.shape[1] - w + 1)
+        y = min(max(int(yc), 0), canvas.shape[0] - h + 1)
+
+        canvas = canvas[y:y + h, x:x + w]
+
+        self.dynamic_info.pos_curr = (xc + xs, yc + ys)
+        self.dynamic_info.step_curr += 1
+
+        if self.dynamic_info.step_curr == self.dynamic_max_step:
+            self._reset_dynamic()
+
+        return canvas
+
+    def _get_transform(self, kpts_src, kpts_tar):
+        if self.rigid:
+            # rigid transform
+            n = kpts_src.shape[0]
+            X = np.zeros((n * 2, 4), dtype=np.float32)
+            U = np.zeros((n * 2, 1), dtype=np.float32)
+            X[:n, :2] = kpts_src
+            X[:n, 2] = 1
+            X[n:, 0] = kpts_src[:, 1]
+            X[n:, 1] = -kpts_src[:, 0]
+            X[n:, 3] = 1
+
+            U[:n, 0] = kpts_tar[:, 0]
+            U[n:, 0] = kpts_tar[:, 1]
+
+            M = np.linalg.pinv(X).dot(U).flatten()
+
+            trans_mat = np.array([[M[0], M[1], M[2]], [-M[1], M[0], M[3]]],
+                                 dtype=np.float32)
+
+        else:
+            # normal affine transform
+            # adaptive horizontal flipping
+            if (np.linalg.norm(kpts_tar[0] - kpts_tar[2]) -
+                    np.linalg.norm(kpts_tar[1] - kpts_tar[2])) * (
+                        np.linalg.norm(kpts_src[0] - kpts_src[2]) -
+                        np.linalg.norm(kpts_src[1] - kpts_src[2])) < 0:
+                kpts_src = kpts_src[[1, 0, 2], :]
+            trans_mat, _ = cv2.estimateAffine2D(
+                kpts_src.astype(np.float32), kpts_tar.astype(np.float32))
+
+        return trans_mat

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/utils/__init__.py

@@ -0,0 +1,31 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .buffer import BufferManager
+from .event import EventManager
+from .message import FrameMessage, Message, VideoEndingMessage
+from .misc import (ImageCapture, copy_and_paste, expand_and_clamp,
+                   get_cached_file_path, is_image_file, limit_max_fps,
+                   load_image_from_disk_or_url, screen_matting)
+from .pose import (get_eye_keypoint_ids, get_face_keypoint_ids,
+                   get_hand_keypoint_ids, get_mouth_keypoint_ids,
+                   get_wrist_keypoint_ids)
+
+__all__ = [
+    'BufferManager',
+    'EventManager',
+    'FrameMessage',
+    'Message',
+    'limit_max_fps',
+    'VideoEndingMessage',
+    'load_image_from_disk_or_url',
+    'get_cached_file_path',
+    'screen_matting',
+    'expand_and_clamp',
+    'copy_and_paste',
+    'is_image_file',
+    'ImageCapture',
+    'get_eye_keypoint_ids',
+    'get_face_keypoint_ids',
+    'get_wrist_keypoint_ids',
+    'get_mouth_keypoint_ids',
+    'get_hand_keypoint_ids',
+]

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/utils/buffer.py

@@ -0,0 +1,106 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from functools import wraps
+from queue import Queue
+from typing import Dict, List, Optional
+
+from mmcv import is_seq_of
+
+__all__ = ['BufferManager']
+
+
+def check_buffer_registered(exist=True):
+
+    def wrapper(func):
+
+        @wraps(func)
+        def wrapped(manager, name, *args, **kwargs):
+            if exist:
+                # Assert buffer exist
+                if name not in manager:
+                    raise ValueError(f'Fail to call {func.__name__}: '
+                                     f'buffer "{name}" is not registered.')
+            else:
+                # Assert buffer not exist
+                if name in manager:
+                    raise ValueError(f'Fail to call {func.__name__}: '
+                                     f'buffer "{name}" is already registered.')
+            return func(manager, name, *args, **kwargs)
+
+        return wrapped
+
+    return wrapper
+
+
+class Buffer(Queue):
+
+    def put_force(self, item):
+        """Force to put an item into the buffer.
+
+        If the buffer is already full, the earliest item in the buffer will be
+        remove to make room for the incoming item.
+        """
+        with self.mutex:
+            if self.maxsize > 0:
+                while self._qsize() >= self.maxsize:
+                    _ = self._get()
+                    self.unfinished_tasks -= 1
+
+            self._put(item)
+            self.unfinished_tasks += 1
+            self.not_empty.notify()
+
+
+class BufferManager():
+
+    def __init__(self,
+                 buffer_type: type = Buffer,
+                 buffers: Optional[Dict] = None):
+        self.buffer_type = buffer_type
+        if buffers is None:
+            self._buffers = {}
+        else:
+            if is_seq_of(list(buffers.values()), buffer_type):
+                self._buffers = buffers.copy()
+            else:
+                raise ValueError('The values of buffers should be instance '
+                                 f'of {buffer_type}')
+
+    def __contains__(self, name):
+        return name in self._buffers
+
+    @check_buffer_registered(False)
+    def register_buffer(self, name, maxsize=0):
+        self._buffers[name] = self.buffer_type(maxsize)
+
+    @check_buffer_registered()
+    def put(self, name, item, block=True, timeout=None):
+        self._buffers[name].put(item, block, timeout)
+
+    @check_buffer_registered()
+    def put_force(self, name, item):
+        self._buffers[name].put_force(item)
+
+    @check_buffer_registered()
+    def get(self, name, block=True, timeout=None):
+        return self._buffers[name].get(block, timeout)
+
+    @check_buffer_registered()
+    def is_empty(self, name):
+        return self._buffers[name].empty()
+
+    @check_buffer_registered()
+    def is_full(self, name):
+        return self._buffers[name].full()
+
+    def get_sub_manager(self, buffer_names: List[str]):
+        buffers = {name: self._buffers[name] for name in buffer_names}
+        return BufferManager(self.buffer_type, buffers)
+
+    def get_info(self):
+        buffer_info = {}
+        for name, buffer in self._buffers.items():
+            buffer_info[name] = {
+                'size': buffer.size,
+                'maxsize': buffer.maxsize
+            }
+        return buffer_info

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/utils/event.py

@@ -0,0 +1,59 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from collections import defaultdict
+from contextlib import contextmanager
+from threading import Event
+from typing import Optional
+
+
+class EventManager():
+
+    def __init__(self):
+        self._events = defaultdict(Event)
+
+    def register_event(self,
+                       event_name: str = None,
+                       is_keyboard: bool = False):
+        if is_keyboard:
+            event_name = self._get_keyboard_event_name(event_name)
+        self._events[event_name] = Event()
+
+    def set(self, event_name: str = None, is_keyboard: bool = False):
+        if is_keyboard:
+            event_name = self._get_keyboard_event_name(event_name)
+        return self._events[event_name].set()
+
+    def wait(self,
+             event_name: str = None,
+             is_keyboard: Optional[bool] = False,
+             timeout: Optional[float] = None):
+        if is_keyboard:
+            event_name = self._get_keyboard_event_name(event_name)
+        return self._events[event_name].wait(timeout)
+
+    def is_set(self,
+               event_name: str = None,
+               is_keyboard: Optional[bool] = False):
+        if is_keyboard:
+            event_name = self._get_keyboard_event_name(event_name)
+        return self._events[event_name].is_set()
+
+    def clear(self,
+              event_name: str = None,
+              is_keyboard: Optional[bool] = False):
+        if is_keyboard:
+            event_name = self._get_keyboard_event_name(event_name)
+        return self._events[event_name].clear()
+
+    @staticmethod
+    def _get_keyboard_event_name(key):
+        return f'_keyboard_{chr(key) if isinstance(key,int) else key}'
+
+    @contextmanager
+    def wait_and_handle(self,
+                        event_name: str = None,
+                        is_keyboard: Optional[bool] = False):
+        self.wait(event_name, is_keyboard)
+        try:
+            yield
+        finally:
+            self.clear(event_name, is_keyboard)

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/utils/message.py

@@ -0,0 +1,204 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import time
+import uuid
+import warnings
+from typing import Dict, List, Optional
+
+import numpy as np
+
+
+class Message():
+    """Message base class.
+
+    All message class should inherit this class. The basic use of a Message
+    instance is to carray a piece of text message (self.msg) and a dict that
+    stores structured data (self.data), e.g. frame image, model prediction,
+    et al.
+
+    A message may also hold route information, which is composed of
+    information of all nodes the message has passed through.
+
+    Parameters:
+        msg (str): The text message.
+        data (dict, optional): The structured data.
+    """
+
+    def __init__(self, msg: str = '', data: Optional[Dict] = None):
+        self.msg = msg
+        self.data = data if data else {}
+        self.route_info = []
+        self.timestamp = time.time()
+        self.id = uuid.uuid4()
+
+    def update_route_info(self,
+                          node=None,
+                          node_name: Optional[str] = None,
+                          node_type: Optional[str] = None,
+                          info: Optional[Dict] = None):
+        """Append new node information to the route information.
+
+        Args:
+            node (Node, optional): An instance of Node that provides basic
+                information like the node name and type. Default: None.
+            node_name (str, optional): The node name. If node is given,
+                node_name will be ignored. Default: None.
+            node_type (str, optional): The class name of the node. If node
+                is given, node_type will be ignored. Default: None.
+            info (dict, optional): The node information, which is usually
+                given by node.get_node_info(). Default: None.
+        """
+        if node is not None:
+            if node_name is not None or node_type is not None:
+                warnings.warn(
+                    '`node_name` and `node_type` will be overridden if node'
+                    'is provided.')
+            node_name = node.name
+            node_type = node.__class__.__name__
+
+        node_info = {'node': node_name, 'node_type': node_type, 'info': info}
+        self.route_info.append(node_info)
+
+    def set_route_info(self, route_info: List):
+        """Directly set the entire route information.
+
+        Args:
+            route_info (list): route information to set to the message.
+        """
+        self.route_info = route_info
+
+    def merge_route_info(self, route_info: List):
+        """Merge the given route information into the original one of the
+        message. This is used for combining route information from multiple
+        messages. The node information in the route will be reordered according
+        to their timestamps.
+
+        Args:
+            route_info (list): route information to merge.
+        """
+        self.route_info += route_info
+        self.route_info.sort(key=lambda x: x.get('timestamp', np.inf))
+
+    def get_route_info(self) -> List:
+        return self.route_info.copy()
+
+
+class VideoEndingMessage(Message):
+    """A special message to indicate the input video is ending."""
+
+
+class FrameMessage(Message):
+    """The message to store information of a video frame.
+
+    A FrameMessage instance usually holds following data in self.data:
+        - image (array): The frame image
+        - detection_results (list): A list to hold detection results of
+            multiple detectors. Each element is a tuple (tag, result)
+        - pose_results (list): A list to hold pose estimation results of
+            multiple pose estimator. Each element is a tuple (tag, result)
+    """
+
+    def __init__(self, img):
+        super().__init__(data=dict(image=img))
+
+    def get_image(self):
+        """Get the frame image.
+
+        Returns:
+            array: The frame image.
+        """
+        return self.data.get('image', None)
+
+    def set_image(self, img):
+        """Set the frame image to the message."""
+        self.data['image'] = img
+
+    def add_detection_result(self, result, tag: str = None):
+        """Add the detection result from one model into the message's
+        detection_results.
+
+        Args:
+            tag (str, optional): Give a tag to the result, which can be used
+                to retrieve specific results.
+        """
+        if 'detection_results' not in self.data:
+            self.data['detection_results'] = []
+        self.data['detection_results'].append((tag, result))
+
+    def get_detection_results(self, tag: str = None):
+        """Get detection results of the message.
+
+        Args:
+            tag (str, optional): If given, only the results with the tag
+                will be retrieved. Otherwise all results will be retrieved.
+                Default: None.
+
+        Returns:
+            list[dict]: The retrieved detection results
+        """
+        if 'detection_results' not in self.data:
+            return None
+        if tag is None:
+            results = [res for _, res in self.data['detection_results']]
+        else:
+            results = [
+                res for _tag, res in self.data['detection_results']
+                if _tag == tag
+            ]
+        return results
+
+    def add_pose_result(self, result, tag=None):
+        """Add the pose estimation result from one model into the message's
+        pose_results.
+
+        Args:
+            tag (str, optional): Give a tag to the result, which can be used
+                to retrieve specific results.
+        """
+        if 'pose_results' not in self.data:
+            self.data['pose_results'] = []
+        self.data['pose_results'].append((tag, result))
+
+    def get_pose_results(self, tag=None):
+        """Get pose estimation results of the message.
+
+        Args:
+            tag (str, optional): If given, only the results with the tag
+                will be retrieved. Otherwise all results will be retrieved.
+                Default: None.
+
+        Returns:
+            list[dict]: The retrieved pose results
+        """
+        if 'pose_results' not in self.data:
+            return None
+        if tag is None:
+            results = [res for _, res in self.data['pose_results']]
+        else:
+            results = [
+                res for _tag, res in self.data['pose_results'] if _tag == tag
+            ]
+        return results
+
+    def get_full_results(self):
+        """Get all model predictions of the message.
+
+        See set_full_results() for inference.
+
+        Returns:
+            dict: All model predictions, including:
+                - detection_results
+                - pose_results
+        """
+        result_keys = ['detection_results', 'pose_results']
+        results = {k: self.data[k] for k in result_keys}
+        return results
+
+    def set_full_results(self, results):
+        """Set full model results directly.
+
+        Args:
+            results (dict): All model predictions including:
+                - detection_results (list): see also add_detection_results()
+                - pose_results (list): see also add_pose_results()
+        """
+        self.data.update(results)

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/utils/misc.py

@@ -0,0 +1,343 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import os
+import os.path as osp
+import sys
+import time
+from contextlib import contextmanager
+from typing import Optional
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+import cv2
+import numpy as np
+from torch.hub import HASH_REGEX, download_url_to_file
+
+
+@contextmanager
+def limit_max_fps(fps: Optional[float]):
+    t_start = time.time()
+    try:
+        yield
+    finally:
+        t_end = time.time()
+        if fps is not None:
+            t_sleep = 1.0 / fps - t_end + t_start
+            if t_sleep > 0:
+                time.sleep(t_sleep)
+
+
+def _is_url(filename):
+    """Check if the file is a url link.
+
+    Args:
+        filename (str): the file name or url link.
+
+    Returns:
+        bool: is url or not.
+    """
+    prefixes = ['http://', 'https://']
+    for p in prefixes:
+        if filename.startswith(p):
+            return True
+    return False
+
+
+def load_image_from_disk_or_url(filename, readFlag=cv2.IMREAD_COLOR):
+    """Load an image file, from disk or url.
+
+    Args:
+        filename (str): file name on the disk or url link.
+        readFlag (int): readFlag for imdecode.
+
+    Returns:
+        np.ndarray: A loaded image
+    """
+    if _is_url(filename):
+        # download the image, convert it to a NumPy array, and then read
+        # it into OpenCV format
+        resp = urlopen(filename)
+        image = np.asarray(bytearray(resp.read()), dtype='uint8')
+        image = cv2.imdecode(image, readFlag)
+        return image
+    else:
+        image = cv2.imread(filename, readFlag)
+        return image
+
+
+def mkdir_or_exist(dir_name, mode=0o777):
+    if dir_name == '':
+        return
+    dir_name = osp.expanduser(dir_name)
+    os.makedirs(dir_name, mode=mode, exist_ok=True)
+
+
+def get_cached_file_path(url,
+                         save_dir=None,
+                         progress=True,
+                         check_hash=False,
+                         file_name=None):
+    r"""Loads the Torch serialized object at the given URL.
+
+    If downloaded file is a zip file, it will be automatically decompressed
+
+    If the object is already present in `model_dir`, it's deserialized and
+    returned.
+    The default value of ``model_dir`` is ``<hub_dir>/checkpoints`` where
+    ``hub_dir`` is the directory returned by :func:`~torch.hub.get_dir`.
+
+    Args:
+        url (str): URL of the object to download
+        save_dir (str, optional): directory in which to save the object
+        progress (bool, optional): whether or not to display a progress bar
+            to stderr. Default: True
+        check_hash(bool, optional): If True, the filename part of the URL
+            should follow the naming convention ``filename-<sha256>.ext``
+            where ``<sha256>`` is the first eight or more digits of the
+            SHA256 hash of the contents of the file. The hash is used to
+            ensure unique names and to verify the contents of the file.
+            Default: False
+        file_name (str, optional): name for the downloaded file. Filename
+            from ``url`` will be used if not set. Default: None.
+    """
+    if save_dir is None:
+        save_dir = os.path.join('webcam_resources')
+
+    mkdir_or_exist(save_dir)
+
+    parts = urlparse(url)
+    filename = os.path.basename(parts.path)
+    if file_name is not None:
+        filename = file_name
+    cached_file = os.path.join(save_dir, filename)
+    if not os.path.exists(cached_file):
+        sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
+        hash_prefix = None
+        if check_hash:
+            r = HASH_REGEX.search(filename)  # r is Optional[Match[str]]
+            hash_prefix = r.group(1) if r else None
+        download_url_to_file(url, cached_file, hash_prefix, progress=progress)
+    return cached_file
+
+
+def screen_matting(img, color_low=None, color_high=None, color=None):
+    """Screen Matting.
+
+    Args:
+        img (np.ndarray): Image data.
+        color_low (tuple): Lower limit (b, g, r).
+        color_high (tuple): Higher limit (b, g, r).
+        color (str): Support colors include:
+
+            - 'green' or 'g'
+            - 'blue' or 'b'
+            - 'black' or 'k'
+            - 'white' or 'w'
+    """
+
+    if color_high is None or color_low is None:
+        if color is not None:
+            if color.lower() == 'g' or color.lower() == 'green':
+                color_low = (0, 200, 0)
+                color_high = (60, 255, 60)
+            elif color.lower() == 'b' or color.lower() == 'blue':
+                color_low = (230, 0, 0)
+                color_high = (255, 40, 40)
+            elif color.lower() == 'k' or color.lower() == 'black':
+                color_low = (0, 0, 0)
+                color_high = (40, 40, 40)
+            elif color.lower() == 'w' or color.lower() == 'white':
+                color_low = (230, 230, 230)
+                color_high = (255, 255, 255)
+            else:
+                NotImplementedError(f'Not supported color: {color}.')
+        else:
+            ValueError('color or color_high | color_low should be given.')
+
+    mask = cv2.inRange(img, np.array(color_low), np.array(color_high)) == 0
+
+    return mask.astype(np.uint8)
+
+
+def expand_and_clamp(box, im_shape, s=1.25):
+    """Expand the bbox and clip it to fit the image shape.
+
+    Args:
+        box (list): x1, y1, x2, y2
+        im_shape (ndarray): image shape (h, w, c)
+        s (float): expand ratio
+
+    Returns:
+        list: x1, y1, x2, y2
+    """
+
+    x1, y1, x2, y2 = box[:4]
+    w = x2 - x1
+    h = y2 - y1
+    deta_w = w * (s - 1) / 2
+    deta_h = h * (s - 1) / 2
+
+    x1, y1, x2, y2 = x1 - deta_w, y1 - deta_h, x2 + deta_w, y2 + deta_h
+
+    img_h, img_w = im_shape[:2]
+
+    x1 = min(max(0, int(x1)), img_w - 1)
+    y1 = min(max(0, int(y1)), img_h - 1)
+    x2 = min(max(0, int(x2)), img_w - 1)
+    y2 = min(max(0, int(y2)), img_h - 1)
+
+    return [x1, y1, x2, y2]
+
+
+def _find_connected_components(mask):
+    """Find connected components and sort with areas.
+
+    Args:
+        mask (ndarray): instance segmentation result.
+
+    Returns:
+        ndarray (N, 5): Each item contains (x, y, w, h, area).
+    """
+    num, labels, stats, centroids = cv2.connectedComponentsWithStats(mask)
+    stats = stats[stats[:, 4].argsort()]
+    return stats
+
+
+def _find_bbox(mask):
+    """Find the bounding box for the mask.
+
+    Args:
+        mask (ndarray): Mask.
+
+    Returns:
+        list(4, ): Returned box (x1, y1, x2, y2).
+    """
+    mask_shape = mask.shape
+    if len(mask_shape) == 3:
+        assert mask_shape[-1] == 1, 'the channel of the mask should be 1.'
+    elif len(mask_shape) == 2:
+        pass
+    else:
+        NotImplementedError()
+
+    h, w = mask_shape[:2]
+    mask_w = mask.sum(0)
+    mask_h = mask.sum(1)
+
+    left = 0
+    right = w - 1
+    up = 0
+    down = h - 1
+
+    for i in range(w):
+        if mask_w[i] > 0:
+            break
+        left += 1
+
+    for i in range(w - 1, left, -1):
+        if mask_w[i] > 0:
+            break
+        right -= 1
+
+    for i in range(h):
+        if mask_h[i] > 0:
+            break
+        up += 1
+
+    for i in range(h - 1, up, -1):
+        if mask_h[i] > 0:
+            break
+        down -= 1
+
+    return [left, up, right, down]
+
+
+def copy_and_paste(img,
+                   background_img,
+                   mask,
+                   bbox=None,
+                   effect_region=(0.2, 0.2, 0.8, 0.8),
+                   min_size=(20, 20)):
+    """Copy the image region and paste to the background.
+
+    Args:
+        img (np.ndarray): Image data.
+        background_img (np.ndarray): Background image data.
+        mask (ndarray): instance segmentation result.
+        bbox (ndarray): instance bbox, (x1, y1, x2, y2).
+        effect_region (tuple(4, )): The region to apply mask, the coordinates
+            are normalized (x1, y1, x2, y2).
+    """
+    background_img = background_img.copy()
+    background_h, background_w = background_img.shape[:2]
+    region_h = (effect_region[3] - effect_region[1]) * background_h
+    region_w = (effect_region[2] - effect_region[0]) * background_w
+    region_aspect_ratio = region_w / region_h
+
+    if bbox is None:
+        bbox = _find_bbox(mask)
+    instance_w = bbox[2] - bbox[0]
+    instance_h = bbox[3] - bbox[1]
+
+    if instance_w > min_size[0] and instance_h > min_size[1]:
+        aspect_ratio = instance_w / instance_h
+        if region_aspect_ratio > aspect_ratio:
+            resize_rate = region_h / instance_h
+        else:
+            resize_rate = region_w / instance_w
+
+        mask_inst = mask[int(bbox[1]):int(bbox[3]), int(bbox[0]):int(bbox[2])]
+        img_inst = img[int(bbox[1]):int(bbox[3]), int(bbox[0]):int(bbox[2])]
+        img_inst = cv2.resize(img_inst, (int(
+            resize_rate * instance_w), int(resize_rate * instance_h)))
+        mask_inst = cv2.resize(
+            mask_inst,
+            (int(resize_rate * instance_w), int(resize_rate * instance_h)),
+            interpolation=cv2.INTER_NEAREST)
+
+        mask_ids = list(np.where(mask_inst == 1))
+        mask_ids[1] += int(effect_region[0] * background_w)
+        mask_ids[0] += int(effect_region[1] * background_h)
+
+        background_img[tuple(mask_ids)] = img_inst[np.where(mask_inst == 1)]
+
+    return background_img
+
+
+def is_image_file(path):
+    if isinstance(path, str):
+        if path.lower().endswith(('.png', '.jpg', '.jpeg', '.tiff', '.bmp')):
+            return True
+    return False
+
+
+class ImageCapture:
+    """A mock-up version of cv2.VideoCapture that always return a const image.
+
+    Args:
+        image (str | ndarray): The image or image path
+    """
+
+    def __init__(self, image):
+        if isinstance(image, str):
+            self.image = load_image_from_disk_or_url(image)
+        else:
+            self.image = image
+
+    def isOpened(self):
+        return (self.image is not None)
+
+    def read(self):
+        return True, self.image.copy()
+
+    def release(self):
+        pass
+
+    def get(self, propId):
+        if propId == cv2.CAP_PROP_FRAME_WIDTH:
+            return self.image.shape[1]
+        elif propId == cv2.CAP_PROP_FRAME_HEIGHT:
+            return self.image.shape[0]
+        elif propId == cv2.CAP_PROP_FPS:
+            return np.nan
+        else:
+            raise NotImplementedError()

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/utils/pose.py

@@ -0,0 +1,226 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from typing import List, Tuple
+
+from mmcv import Config
+
+from mmpose.datasets.dataset_info import DatasetInfo
+
+
+def get_eye_keypoint_ids(model_cfg: Config) -> Tuple[int, int]:
+    """A helpfer function to get the keypoint indices of left and right eyes
+    from the model config.
+
+    Args:
+        model_cfg (Config): pose model config.
+
+    Returns:
+        int: left eye keypoint index.
+        int: right eye keypoint index.
+    """
+    left_eye_idx = None
+    right_eye_idx = None
+
+    # try obtaining eye point ids from dataset_info
+    try:
+        dataset_info = DatasetInfo(model_cfg.data.test.dataset_info)
+        left_eye_idx = dataset_info.keypoint_name2id.get('left_eye', None)
+        right_eye_idx = dataset_info.keypoint_name2id.get('right_eye', None)
+    except AttributeError:
+        left_eye_idx = None
+        right_eye_idx = None
+
+    if left_eye_idx is None or right_eye_idx is None:
+        # Fall back to hard coded keypoint id
+        dataset_name = model_cfg.data.test.type
+        if dataset_name in {
+                'TopDownCocoDataset', 'TopDownCocoWholeBodyDataset'
+        }:
+            left_eye_idx = 1
+            right_eye_idx = 2
+        elif dataset_name in {'AnimalPoseDataset', 'AnimalAP10KDataset'}:
+            left_eye_idx = 0
+            right_eye_idx = 1
+        else:
+            raise ValueError('Can not determine the eye keypoint id of '
+                             f'{dataset_name}')
+
+    return left_eye_idx, right_eye_idx
+
+
+def get_face_keypoint_ids(model_cfg: Config) -> Tuple[int, int]:
+    """A helpfer function to get the keypoint indices of the face from the
+    model config.
+
+    Args:
+        model_cfg (Config): pose model config.
+
+    Returns:
+        list[int]: face keypoint index.
+    """
+    face_indices = None
+
+    # try obtaining nose point ids from dataset_info
+    try:
+        dataset_info = DatasetInfo(model_cfg.data.test.dataset_info)
+        for id in range(68):
+            face_indices.append(
+                dataset_info.keypoint_name2id.get(f'face_{id}', None))
+    except AttributeError:
+        face_indices = None
+
+    if face_indices is None:
+        # Fall back to hard coded keypoint id
+        dataset_name = model_cfg.data.test.type
+        if dataset_name in {'TopDownCocoWholeBodyDataset'}:
+            face_indices = list(range(23, 91))
+        else:
+            raise ValueError('Can not determine the face id of '
+                             f'{dataset_name}')
+
+    return face_indices
+
+
+def get_wrist_keypoint_ids(model_cfg: Config) -> Tuple[int, int]:
+    """A helpfer function to get the keypoint indices of left and right wrist
+    from the model config.
+
+    Args:
+        model_cfg (Config): pose model config.
+    Returns:
+        int: left wrist keypoint index.
+        int: right wrist keypoint index.
+    """
+
+    # try obtaining eye point ids from dataset_info
+    try:
+        dataset_info = DatasetInfo(model_cfg.data.test.dataset_info)
+        left_wrist_idx = dataset_info.keypoint_name2id.get('left_wrist', None)
+        right_wrist_idx = dataset_info.keypoint_name2id.get(
+            'right_wrist', None)
+    except AttributeError:
+        left_wrist_idx = None
+        right_wrist_idx = None
+
+    if left_wrist_idx is None or right_wrist_idx is None:
+        # Fall back to hard coded keypoint id
+        dataset_name = model_cfg.data.test.type
+        if dataset_name in {
+                'TopDownCocoDataset', 'TopDownCocoWholeBodyDataset'
+        }:
+            left_wrist_idx = 9
+            right_wrist_idx = 10
+        elif dataset_name == 'AnimalPoseDataset':
+            left_wrist_idx = 16
+            right_wrist_idx = 17
+        elif dataset_name == 'AnimalAP10KDataset':
+            left_wrist_idx = 7
+            right_wrist_idx = 10
+        else:
+            raise ValueError('Can not determine the eye keypoint id of '
+                             f'{dataset_name}')
+
+    return left_wrist_idx, right_wrist_idx
+
+
+def get_mouth_keypoint_ids(model_cfg: Config) -> Tuple[int, int]:
+    """A helpfer function to get the keypoint indices of the left and right
+    part of mouth from the model config.
+
+    Args:
+        model_cfg (Config): pose model config.
+    Returns:
+        int: left-part mouth keypoint index.
+        int: right-part mouth keypoint index.
+    """
+    # try obtaining mouth point ids from dataset_info
+    try:
+        dataset_info = DatasetInfo(model_cfg.data.test.dataset_info)
+        mouth_index = dataset_info.keypoint_name2id.get('face-62', None)
+    except AttributeError:
+        mouth_index = None
+
+    if mouth_index is None:
+        # Fall back to hard coded keypoint id
+        dataset_name = model_cfg.data.test.type
+        if dataset_name == 'TopDownCocoWholeBodyDataset':
+            mouth_index = 85
+        else:
+            raise ValueError('Can not determine the eye keypoint id of '
+                             f'{dataset_name}')
+
+    return mouth_index
+
+
+def get_hand_keypoint_ids(model_cfg: Config) -> List[int]:
+    """A helpfer function to get the keypoint indices of left and right hand
+    from the model config.
+
+    Args:
+        model_cfg (Config): pose model config.
+    Returns:
+        list[int]: hand keypoint indices.
+    """
+    # try obtaining hand keypoint ids from dataset_info
+    try:
+        hand_indices = []
+        dataset_info = DatasetInfo(model_cfg.data.test.dataset_info)
+
+        hand_indices.append(
+            dataset_info.keypoint_name2id.get('left_hand_root', None))
+
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'left_thumb{id}', None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'left_forefinger{id}',
+                                                  None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'left_middle_finger{id}',
+                                                  None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'left_ring_finger{id}',
+                                                  None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'left_pinky_finger{id}',
+                                                  None))
+
+        hand_indices.append(
+            dataset_info.keypoint_name2id.get('right_hand_root', None))
+
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'right_thumb{id}', None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'right_forefinger{id}',
+                                                  None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'right_middle_finger{id}',
+                                                  None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'right_ring_finger{id}',
+                                                  None))
+        for id in range(1, 5):
+            hand_indices.append(
+                dataset_info.keypoint_name2id.get(f'right_pinky_finger{id}',
+                                                  None))
+
+    except AttributeError:
+        hand_indices = None
+
+    if hand_indices is None:
+        # Fall back to hard coded keypoint id
+        dataset_name = model_cfg.data.test.type
+        if dataset_name in {'TopDownCocoWholeBodyDataset'}:
+            hand_indices = list(range(91, 133))
+        else:
+            raise ValueError('Can not determine the hand id of '
+                             f'{dataset_name}')
+
+    return hand_indices

third_party/hamer/third-party/ViTPose/tools/webcam/webcam_apis/webcam_runner.py

@@ -0,0 +1,272 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import logging
+import sys
+import time
+import warnings
+from contextlib import nullcontext
+from threading import Thread
+from typing import Dict, List, Optional, Tuple, Union
+
+import cv2
+
+from .nodes import NODES
+from .utils import (BufferManager, EventManager, FrameMessage, ImageCapture,
+                    VideoEndingMessage, is_image_file, limit_max_fps)
+
+DEFAULT_FRAME_BUFFER_SIZE = 1
+DEFAULT_INPUT_BUFFER_SIZE = 1
+DEFAULT_DISPLAY_BUFFER_SIZE = 0
+DEFAULT_USER_BUFFER_SIZE = 1
+
+
+class WebcamRunner():
+    """An interface for building webcam application from config.
+
+    Parameters:
+        name (str): Runner name.
+        camera_id (int | str): The camera ID (usually the ID of the default
+            camera is 0). Alternatively a file path or a URL can be given
+            to load from a video or image file.
+        camera_frame_shape (tuple, optional): Set the frame shape of the
+            camera in (width, height). If not given, the default frame shape
+            will be used. This argument is only valid when using a camera
+            as the input source. Default: None
+        camera_fps (int): Video reading maximum FPS. Default: 30
+        buffer_sizes (dict, optional): A dict to specify buffer sizes. The
+            key is the buffer name and the value is the buffer size.
+            Default: None
+        nodes (list): Node configs.
+    """
+
+    def __init__(self,
+                 name: str = 'Default Webcam Runner',
+                 camera_id: Union[int, str] = 0,
+                 camera_fps: int = 30,
+                 camera_frame_shape: Optional[Tuple[int, int]] = None,
+                 synchronous: bool = False,
+                 buffer_sizes: Optional[Dict[str, int]] = None,
+                 nodes: Optional[List[Dict]] = None):
+
+        # Basic parameters
+        self.name = name
+        self.camera_id = camera_id
+        self.camera_fps = camera_fps
+        self.camera_frame_shape = camera_frame_shape
+        self.synchronous = synchronous
+
+        # self.buffer_manager manages data flow between runner and nodes
+        self.buffer_manager = BufferManager()
+        # self.event_manager manages event-based asynchronous communication
+        self.event_manager = EventManager()
+        # self.node_list holds all node instance
+        self.node_list = []
+        # self.vcap is used to read camera frames. It will be built when the
+        # runner starts running
+        self.vcap = None
+
+        # Register runner events
+        self.event_manager.register_event('_exit_', is_keyboard=False)
+        if self.synchronous:
+            self.event_manager.register_event('_idle_', is_keyboard=False)
+
+        # Register nodes
+        if not nodes:
+            raise ValueError('No node is registered to the runner.')
+
+        # Register default buffers
+        if buffer_sizes is None:
+            buffer_sizes = {}
+        # _frame_ buffer
+        frame_buffer_size = buffer_sizes.get('_frame_',
+                                             DEFAULT_FRAME_BUFFER_SIZE)
+        self.buffer_manager.register_buffer('_frame_', frame_buffer_size)
+        # _input_ buffer
+        input_buffer_size = buffer_sizes.get('_input_',
+                                             DEFAULT_INPUT_BUFFER_SIZE)
+        self.buffer_manager.register_buffer('_input_', input_buffer_size)
+        # _display_ buffer
+        display_buffer_size = buffer_sizes.get('_display_',
+                                               DEFAULT_DISPLAY_BUFFER_SIZE)
+        self.buffer_manager.register_buffer('_display_', display_buffer_size)
+
+        # Build all nodes:
+        for node_cfg in nodes:
+            logging.info(f'Create node: {node_cfg.name}({node_cfg.type})')
+            node = NODES.build(node_cfg)
+
+            # Register node
+            self.node_list.append(node)
+
+            # Register buffers
+            for buffer_info in node.registered_buffers:
+                buffer_name = buffer_info.buffer_name
+                if buffer_name in self.buffer_manager:
+                    continue
+                buffer_size = buffer_sizes.get(buffer_name,
+                                               DEFAULT_USER_BUFFER_SIZE)
+                self.buffer_manager.register_buffer(buffer_name, buffer_size)
+                logging.info(
+                    f'Register user buffer: {buffer_name}({buffer_size})')
+
+            # Register events
+            for event_info in node.registered_events:
+                self.event_manager.register_event(
+                    event_name=event_info.event_name,
+                    is_keyboard=event_info.is_keyboard)
+                logging.info(f'Register event: {event_info.event_name}')
+
+        # Set runner for nodes
+        # This step is performed after node building when the runner has
+        # create full buffer/event managers and can
+        for node in self.node_list:
+            logging.info(f'Set runner for node: {node.name})')
+            node.set_runner(self)
+
+    def _read_camera(self):
+        """Continually read video frames and put them into buffers."""
+
+        camera_id = self.camera_id
+        fps = self.camera_fps
+
+        # Build video capture
+        if is_image_file(camera_id):
+            self.vcap = ImageCapture(camera_id)
+        else:
+            self.vcap = cv2.VideoCapture(camera_id)
+            if self.camera_frame_shape is not None:
+                width, height = self.camera_frame_shape
+                self.vcap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
+                self.vcap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
+
+        if not self.vcap.isOpened():
+            warnings.warn(f'Cannot open camera (ID={camera_id})')
+            sys.exit()
+
+        # Read video frames in a loop
+        first_frame = True
+        while not self.event_manager.is_set('_exit_'):
+            if self.synchronous:
+                if first_frame:
+                    cm = nullcontext()
+                else:
+                    # Read a new frame until the last frame has been processed
+                    cm = self.event_manager.wait_and_handle('_idle_')
+            else:
+                # Read frames with a maximum FPS
+                cm = limit_max_fps(fps)
+
+            first_frame = False
+
+            with cm:
+                # Read a frame
+                ret_val, frame = self.vcap.read()
+                if ret_val:
+                    # Put frame message (for display) into buffer `_frame_`
+                    frame_msg = FrameMessage(frame)
+                    self.buffer_manager.put('_frame_', frame_msg)
+
+                    # Put input message (for model inference or other use)
+                    # into buffer `_input_`
+                    input_msg = FrameMessage(frame.copy())
+                    input_msg.update_route_info(
+                        node_name='Camera Info',
+                        node_type='dummy',
+                        info=self._get_camera_info())
+                    self.buffer_manager.put_force('_input_', input_msg)
+
+                else:
+                    # Put a video ending signal
+                    self.buffer_manager.put('_frame_', VideoEndingMessage())
+
+        self.vcap.release()
+
+    def _display(self):
+        """Continually obtain and display output frames."""
+
+        output_msg = None
+
+        while not self.event_manager.is_set('_exit_'):
+            while self.buffer_manager.is_empty('_display_'):
+                time.sleep(0.001)
+
+            # Set _idle_ to allow reading next frame
+            if self.synchronous:
+                self.event_manager.set('_idle_')
+
+            # acquire output from buffer
+            output_msg = self.buffer_manager.get('_display_')
+
+            # None indicates input stream ends
+            if isinstance(output_msg, VideoEndingMessage):
+                self.event_manager.set('_exit_')
+                break
+
+            img = output_msg.get_image()
+
+            # show in a window
+            cv2.imshow(self.name, img)
+
+            # handle keyboard input
+            key = cv2.waitKey(1)
+            if key != -1:
+                self._on_keyboard_input(key)
+
+        cv2.destroyAllWindows()
+
+    def _on_keyboard_input(self, key):
+        """Handle the keyboard input."""
+
+        if key in (27, ord('q'), ord('Q')):
+            logging.info(f'Exit event captured: {key}')
+            self.event_manager.set('_exit_')
+        else:
+            logging.info(f'Keyboard event captured: {key}')
+            self.event_manager.set(key, is_keyboard=True)
+
+    def _get_camera_info(self):
+        """Return the camera information in a dict."""
+
+        frame_width = self.vcap.get(cv2.CAP_PROP_FRAME_WIDTH)
+        frame_height = self.vcap.get(cv2.CAP_PROP_FRAME_HEIGHT)
+        frame_rate = self.vcap.get(cv2.CAP_PROP_FPS)
+
+        cam_info = {
+            'Camera ID': self.camera_id,
+            'Source resolution': f'{frame_width}x{frame_height}',
+            'Source FPS': frame_rate,
+        }
+
+        return cam_info
+
+    def run(self):
+        """Program entry.
+
+        This method starts all nodes as well as video I/O in separate threads.
+        """
+
+        try:
+            # Start node threads
+            non_daemon_nodes = []
+            for node in self.node_list:
+                node.start()
+                if not node.daemon:
+                    non_daemon_nodes.append(node)
+
+            # Create a thread to read video frames
+            t_read = Thread(target=self._read_camera, args=())
+            t_read.start()
+
+            # Run display in the main thread
+            self._display()
+            logging.info('Display shut down')
+
+            # joint non-daemon nodes and runner threads
+            logging.info('Camera reading about to join')
+            t_read.join()
+
+            for node in non_daemon_nodes:
+                logging.info(f'Node {node.name} about to join')
+                node.join()
+
+        except KeyboardInterrupt:
+            pass