Files
2026-11-03 19:46:08 +08:00

333 lines
15 KiB
Python

# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
from __future__ import annotations
import datetime
import git
import os
import pathlib
import statistics
import torch
import warnings
from collections import deque
import rsl_rl
from rsl_rl.utils.log_writer import LogWriter
from rsl_rl.utils.utils import resolve_class
from rsl_rl.utils.wandb_log_writer import WandbLogWriter
class Logger:
"""Logger to save the learning metrics to different logging services."""
def __init__(
self,
log_dir: str | None,
cfg: dict,
env_cfg: dict | object,
num_envs: int,
is_distributed: bool,
gpu_world_size: int,
gpu_global_rank: int,
device: str,
) -> None:
"""Initialize buffers and logging state for a training run."""
self.log_dir = log_dir
self.cfg = cfg
self.env_cfg = env_cfg
self.num_envs = num_envs
self.gpu_world_size = gpu_world_size
self.device = device
self.git_status_repos = [rsl_rl.__file__]
self.tot_timesteps = 0
self.tot_time = 0
self.writer: LogWriter | None = None
# Create buffers
self.ep_extras = []
self.rewbuffer = deque(maxlen=100)
self.lenbuffer = deque(maxlen=100)
self.cur_reward_sum = torch.zeros(self.num_envs, dtype=torch.float, device=self.device)
self.cur_episode_length = torch.zeros(self.num_envs, dtype=torch.float, device=self.device)
# Create RND buffers
if self.cfg["algorithm"].get("rnd_cfg"):
self.erewbuffer = deque(maxlen=100)
self.irewbuffer = deque(maxlen=100)
self.cur_ereward_sum = torch.zeros(self.num_envs, dtype=torch.float, device=self.device)
self.cur_ireward_sum = torch.zeros(self.num_envs, dtype=torch.float, device=self.device)
# Decide whether to disable logging
# Note: We only log from the process with rank 0 (main process)
self.disable_logs = is_distributed and gpu_global_rank != 0
def init_logging_writer(self) -> None:
"""Initialize the logging writer and save the code state.
.. note::
The writer is constructed from ``cfg["logger"]``, which should be a dict with a ``"class_name"`` key plus
any additional constructor kwargs (see :class:`~rsl_rl.utils.LogWriter`). The plain string aliases
``"wandb"`` and ``"neptune"`` are deprecated; use ``"WandbLogWriter"`` and ``"NeptuneLogWriter"`` in the
dict form instead. ``"tensorboard"`` (the default) is still accepted as a plain string.
"""
if self.log_dir is not None and not self.disable_logs:
logger_cfg = self.cfg.get("logger", "tensorboard")
writer_name = logger_cfg if isinstance(logger_cfg, str) else logger_cfg["class_name"]
# Handle deprecated plain string logger types for W&B and Neptune
if writer_name == "wandb" and isinstance(logger_cfg, str):
warnings.warn(
"cfg['logger'] = 'wandb' is deprecated. "
"Use cfg['logger'] = {'class_name': 'WandbLogWriter', 'project_name': ...} instead.",
DeprecationWarning,
stacklevel=2,
)
writer_name = "WandbLogWriter"
logger_cfg = {"class_name": writer_name, "project_name": self.cfg.get("wandb_project")}
elif writer_name == "neptune" and isinstance(logger_cfg, str):
warnings.warn(
"cfg['logger'] = 'neptune' is deprecated. "
"Use cfg['logger'] = {'class_name': 'NeptuneLogWriter', 'project_name': ...} instead.",
DeprecationWarning,
stacklevel=2,
)
writer_name = "NeptuneLogWriter"
logger_cfg = {"class_name": writer_name, "project_name": self.cfg.get("neptune_project")}
if writer_name == "tensorboard":
from torch.utils.tensorboard import SummaryWriter
self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) # type: ignore
else:
writer_class, writer_cfg = resolve_class(logger_cfg) # type: ignore
self.writer = writer_class(log_dir=self.log_dir, **writer_cfg) # type: ignore
else:
self.writer = None
# Save code state
files_to_upload = self._store_code_state()
# Upload configuration and code state to external logging service if supported
if isinstance(self.writer, LogWriter):
self.writer.store_config(self.env_cfg, self.cfg)
for path in files_to_upload:
self.writer.save_file(path)
def process_env_step(
self,
rewards: torch.Tensor,
dones: torch.Tensor,
extras: dict,
intrinsic_rewards: torch.Tensor | None = None,
) -> None:
"""Add metrics from the environment step to the buffers."""
if self.writer is not None:
if "episode" in extras:
self.ep_extras.append(extras["episode"])
elif "log" in extras:
self.ep_extras.append(extras["log"])
# Update rewards and episode length
if intrinsic_rewards is not None:
self.cur_ereward_sum += rewards
self.cur_ireward_sum += intrinsic_rewards
self.cur_reward_sum += rewards + intrinsic_rewards
else:
self.cur_reward_sum += rewards
self.cur_episode_length += 1
# Clear data for completed episodes
new_ids = (dones > 0).nonzero(as_tuple=False)
self.rewbuffer.extend(self.cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist())
self.lenbuffer.extend(self.cur_episode_length[new_ids][:, 0].cpu().numpy().tolist())
self.cur_reward_sum[new_ids] = 0
self.cur_episode_length[new_ids] = 0
if intrinsic_rewards is not None:
self.erewbuffer.extend(self.cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist())
self.irewbuffer.extend(self.cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist())
self.cur_ereward_sum[new_ids] = 0
self.cur_ireward_sum[new_ids] = 0
def log(
self,
it: int,
start_it: int,
total_it: int,
collect_time: float,
learn_time: float,
loss_dict: dict,
learning_rate: float,
action_std: torch.Tensor,
rnd_weight: float | None,
print_minimal: bool = False,
width: int = 80,
pad: int = 40,
) -> None:
"""Log the training metrics to the logging service and print them to the console.
If videos are available, they are uploaded to the logging service (W&B) as well.
"""
if self.writer is not None:
collection_size = self.cfg["num_steps_per_env"] * self.num_envs * self.gpu_world_size
iteration_time = collect_time + learn_time
self.tot_timesteps += collection_size
self.tot_time += iteration_time
# Log episode extras
extras_string = ""
if self.ep_extras:
# Iterate over all keys in the episode info dictionary
for key in dict.fromkeys(k for ep_info in self.ep_extras for k in ep_info):
infotensor = torch.tensor([], device=self.device)
# Iterate over all steps
for ep_info in self.ep_extras:
# Handle missing, scalar, and zero dimensional tensors
if key not in ep_info:
continue
if not isinstance(ep_info[key], torch.Tensor):
ep_info[key] = torch.Tensor([ep_info[key]])
if len(ep_info[key].shape) == 0:
ep_info[key] = ep_info[key].unsqueeze(0)
infotensor = torch.cat((infotensor, ep_info[key].to(self.device)))
value = torch.mean(infotensor)
if "/" in key:
self.writer.add_scalar(key, value, it) # type: ignore
extras_string += f"""{f"{key}:":>{pad}} {value:.4f}\n"""
else:
self.writer.add_scalar("Episode/" + key, value, it) # type: ignore
extras_string += f"""{f"Mean episode {key}:":>{pad}} {value:.4f}\n"""
# Log losses
for key, value in loss_dict.items():
self.writer.add_scalar(f"Loss/{key}", value, it)
self.writer.add_scalar("Loss/learning_rate", learning_rate, it)
# Log std
self.writer.add_scalar("Policy/mean_std", action_std.mean().item(), it)
# Log performance
fps = int(collection_size / (collect_time + learn_time))
self.writer.add_scalar("Perf/total_fps", fps, it)
self.writer.add_scalar("Perf/collection_time", collect_time, it)
self.writer.add_scalar("Perf/learning_time", learn_time, it)
# Log rewards and episode length
if len(self.rewbuffer) > 0:
if self.cfg["algorithm"].get("rnd_cfg"):
self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(self.erewbuffer), it)
self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(self.irewbuffer), it)
self.writer.add_scalar("Rnd/weight", rnd_weight, it) # type: ignore
self.writer.add_scalar("Train/mean_reward", statistics.mean(self.rewbuffer), it)
self.writer.add_scalar("Train/mean_episode_length", statistics.mean(self.lenbuffer), it)
if not isinstance(self.writer, WandbLogWriter):
self.writer.add_scalar(
"Train/mean_reward/time", statistics.mean(self.rewbuffer), int(self.tot_time)
)
self.writer.add_scalar(
"Train/mean_episode_length/time", statistics.mean(self.lenbuffer), int(self.tot_time)
)
# Print to console
log_string = f"""{"#" * width}\n"""
log_string += f"""\033[1m{f" Learning iteration {it}/{total_it} ".center(width)}\033[0m \n\n"""
# Print run name if provided
run_name = self.cfg.get("run_name")
log_string += f"""{"Run name:":>{pad}} {run_name}\n""" if run_name else ""
# Print performance
log_string += (
f"""{"Total steps:":>{pad}} {self.tot_timesteps} \n"""
f"""{"Steps per second:":>{pad}} {fps:.0f} \n"""
f"""{"Collection time:":>{pad}} {collect_time:.3f}s \n"""
f"""{"Learning time:":>{pad}} {learn_time:.3f}s \n"""
)
# Print losses
for key, value in loss_dict.items():
log_string += f"""{f"Mean {key} loss:":>{pad}} {value:.4f}\n"""
# Print rewards and episode length
if len(self.rewbuffer) > 0:
if self.cfg["algorithm"].get("rnd_cfg"):
log_string += f"""{"Mean extrinsic reward:":>{pad}} {statistics.mean(self.erewbuffer):.2f}\n"""
log_string += f"""{"Mean intrinsic reward:":>{pad}} {statistics.mean(self.irewbuffer):.2f}\n"""
log_string += f"""{"Mean reward:":>{pad}} {statistics.mean(self.rewbuffer):.2f}\n"""
log_string += f"""{"Mean episode length:":>{pad}} {statistics.mean(self.lenbuffer):.2f}\n"""
# Print std
log_string += f"""{"Mean action std:":>{pad}} {action_std.mean().item():.2f}\n"""
# Print episode extras
if not print_minimal:
log_string += extras_string
# Print footer
done_it = it + 1 - start_it
remaining_it = total_it - start_it - done_it
eta = self.tot_time / done_it * remaining_it
log_string += (
f"""{"-" * width}\n"""
f"""{"Iteration time:":>{pad}} {iteration_time:.2f}s\n"""
f"""{"Time elapsed:":>{pad}} {datetime.timedelta(seconds=int(self.tot_time))}\n"""
f"""{"ETA:":>{pad}} {datetime.timedelta(seconds=int(eta))}\n"""
)
print(log_string)
# Upload available videos to external logging service if supported
if isinstance(self.writer, LogWriter):
for video in pathlib.Path(self.log_dir).rglob("*.mp4"): # type: ignore
self.writer.save_video(video, it)
# Clear extras buffer
self.ep_extras.clear()
def save_model(self, path: str, it: int) -> None:
"""Save the model to external logging service if specified."""
if isinstance(self.writer, LogWriter):
self.writer.save_model(path, it)
def stop_logging_writer(self) -> None:
"""Stop the logging writer."""
if isinstance(self.writer, LogWriter):
self.writer.stop()
def _store_code_state(self) -> list[str]:
"""Store the current git diff of the code repositories involved in the experiment."""
files_to_upload = []
if self.log_dir is not None and not self.disable_logs:
git_log_dir = os.path.join(self.log_dir, "git")
os.makedirs(git_log_dir, exist_ok=True)
# Iterate over all repositories to log
for repository_file_path in self.git_status_repos:
try:
repo = git.Repo(repository_file_path, search_parent_directories=True)
t = repo.head.commit.tree
commit_hash = repo.head.commit.hexsha
except Exception:
print(f"Could not find git repository in {repository_file_path}. Skipping.")
continue
# Get the name of the repository
repo_name = pathlib.Path(repo.working_dir).name
diff_file_name = os.path.join(git_log_dir, f"{repo_name}.diff")
# Check if the diff file already exists
if os.path.isfile(diff_file_name):
continue
# Write the diff file
print(f"Storing git diff for '{repo_name}' in: {diff_file_name}")
with open(diff_file_name, "x", encoding="utf-8") as f:
content = (
f"--- git commit ---\n{commit_hash}\n\n\n"
f"--- git status ---\n{repo.git.status()} \n\n\n"
f"--- git diff ---\n{repo.git.diff(t)}"
)
f.write(content)
# Add the file path to the list of files to be uploaded
files_to_upload.append(diff_file_name)
return files_to_upload