Add an introspect module for structs.

Also missing struct fields to the Python bindings that were discovered as a result of the new `introspect` metadata.

PiperOrigin-RevId: 523947396
Change-Id: I367fd2dc8d5fd7e5712a45b70a5c50da51d152e4
This commit is contained in:
Saran Tunyasuvunakool
2023-04-13 02:49:47 -07:00
committed by Copybara-Service
parent bcb19fd85c
commit 2e23594fe8
13 changed files with 6721 additions and 14 deletions
+2
View File
@@ -19,6 +19,7 @@ General
- Added :ref:`visual-global<visual-global>` flag :ref:`ellipsoidinertia<visual-global-ellipsoidinertia>` to visualize
equivalent body inertias with ellipsoids instead of the default boxes.
- Added documentation for :ref:`engine plugins<exPlugin>`.
- Added struct information to the ``introspect`` module.
Python bindings
^^^^^^^^^^^^^^^
@@ -31,6 +32,7 @@ Python bindings
state concurrently with the internal ``mj_forward``, resulting in e.g.
`MuJoCo stack overflow error<https://github.com/deepmind/mujoco/issues/783>`_
or `segmentation fault<https://github.com/deepmind/mujoco/issues/790>`_.
- Added a small number of missing struct fields discovered through the new ``introspect`` metadata.
Bug fixes
^^^^^^^^^
+4
View File
@@ -593,8 +593,11 @@
#define MJDATA_SCALAR \
X( int, nstack ) \
X( int, nbuffer ) \
X( int, nplugin ) \
X( int, pstack ) \
X( int, parena ) \
X( int, maxuse_stack ) \
X( int, maxuse_arena ) \
X( int, maxuse_con ) \
X( int, maxuse_efc ) \
X( int, solver_iter ) \
@@ -606,6 +609,7 @@
X( int, ne ) \
X( int, nf ) \
X( int, nefc ) \
X( int, nnzJ ) \
X( int, ncon ) \
X( mjtNum, time )
+79 -3
View File
@@ -22,7 +22,7 @@ from typing import Dict, Optional, Sequence, Tuple, Union
# We are relying on Clang to do the actual source parsing and are only doing
# a little bit of extra parsing of function parameter type declarations here.
# These patterns are here for sanity checking rather than actual parsing.
VALID_TYPE_NAME_PATTERN = re.compile('[A-Za-z_][A-Za-z0-9_]*')
VALID_TYPE_NAME_PATTERN = re.compile('(struct )?[A-Za-z_][A-Za-z0-9_]*')
C_INVALID_TYPE_NAMES = frozenset([
'auto', 'break', 'case', 'const', 'continue', 'default', 'do', 'else',
'enum', 'extern', 'for', 'goto', 'if', 'inline', 'register', 'restrict',
@@ -94,7 +94,7 @@ class ArrayType:
"""Represents a C array type."""
inner_type: Union[ValueType, 'PointerType']
extents: Tuple[int]
extents: Tuple[int, ...]
def __init__(self, inner_type: Union[ValueType, 'PointerType'],
extents: Sequence[int]):
@@ -169,7 +169,7 @@ class FunctionDecl:
name: str
return_type: Union[ValueType, ArrayType, PointerType]
parameters: Tuple[FunctionParameterDecl]
parameters: Tuple[FunctionParameterDecl, ...]
doc: str
def __init__(self, name: str,
@@ -218,3 +218,79 @@ class EnumDecl:
self.name = name
self.declname = declname
self.values = _EnumDeclValues(values)
@dataclasses.dataclass
class StructFieldDecl:
"""Represents a field in a struct or union declaration."""
name: str
type: Union[
ValueType,
ArrayType,
PointerType,
'AnonymousStructDecl',
'AnonymousUnionDecl',
]
doc: str
def __str__(self):
return self.type.decl(self.name)
@property
def decltype(self) -> str:
return self.type.decl()
@dataclasses.dataclass
class AnonymousStructDecl:
"""Represents an anonymous struct declaration."""
fields: Tuple[Union[StructFieldDecl, 'AnonymousUnionDecl'], ...]
def __init__(self, fields: Sequence[StructFieldDecl]):
self.fields = tuple(fields)
def __str__(self):
return self.decl()
def _inner_decl(self):
return '; '.join(str(field) for field in self.fields) + ';'
def decl(self, name_or_decl: Optional[str] = None):
parts = ['struct', f'{{{self._inner_decl()}}}']
if name_or_decl:
parts.append(name_or_decl)
return ' '.join(parts)
class AnonymousUnionDecl(AnonymousStructDecl):
"""Represents an anonymous union declaration."""
def decl(self, name_or_decl: Optional[str] = None):
parts = ['union', f'{{{self._inner_decl()}}}']
if name_or_decl:
parts.append(name_or_decl)
return ' '.join(parts)
@dataclasses.dataclass
class StructDecl:
"""Represents a struct declaration."""
name: str
declname: str
fields: Tuple[Union[StructFieldDecl, AnonymousUnionDecl], ...]
def __init__(self, name: str,
declname: str,
fields: Sequence[Union[StructFieldDecl, AnonymousUnionDecl]]):
self.name = name
self.declname = declname
self.fields = tuple(fields)
def decl(self, name_or_decl: Optional[str] = None) -> str:
parts = [self.name]
if name_or_decl:
parts.append(name_or_decl)
return ' '.join(parts)
+56
View File
@@ -118,6 +118,62 @@ class AstNodesTest(absltest.TestCase):
complex_type.decl('var'),
'const unsigned int (* * const (* const * var[9])[7])[3][4]')
def test_struct_decl(self):
struct = ast_nodes.StructDecl(
name='mystruct',
declname='struct mystruct_',
fields=[
ast_nodes.StructFieldDecl(
name='foo',
type=ast_nodes.ValueType('int'),
doc='',
)
],
)
self.assertEqual(struct.decl('var'), 'mystruct var')
def test_anonymous_struct_decl(self):
struct = ast_nodes.AnonymousStructDecl(
fields=[
ast_nodes.StructFieldDecl(
name='foo',
type=ast_nodes.ValueType('int'),
doc='',
),
ast_nodes.StructFieldDecl(
name='bar',
type=ast_nodes.ArrayType(
inner_type=ast_nodes.ValueType('float'), extents=(3,)
),
doc='',
),
],
)
self.assertEqual(str(struct), 'struct {int foo; float bar[3];}')
self.assertEqual(struct.decl('var'), 'struct {int foo; float bar[3];} var')
self.assertEqual(struct.fields[0].decltype, 'int')
self.assertEqual(struct.fields[1].decltype, 'float [3]')
def test_anonymous_union_decl(self):
union = ast_nodes.AnonymousUnionDecl(
fields=[
ast_nodes.StructFieldDecl(
name='foo',
type=ast_nodes.ValueType('int'),
doc='',
),
ast_nodes.StructFieldDecl(
name='bar',
type=ast_nodes.ArrayType(
inner_type=ast_nodes.ValueType('float'), extents=(3,)
),
doc='',
),
],
)
self.assertEqual(str(union), 'union {int foo; float bar[3];}')
self.assertEqual(union.decl('var'), 'union {int foo; float bar[3];} var')
if __name__ == '__main__':
absltest.main()
+2 -3
View File
@@ -27,8 +27,7 @@ from absl import flags
from introspect import ast_nodes
from . import formatter
FLAGS = flags.FLAGS
flags.DEFINE_string(
_JSON_PATH = flags.DEFINE_string(
'json_path', None,
'Path to the JSON file representing the Clang AST for mujoco.h')
@@ -85,7 +84,7 @@ def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
with open(FLAGS.json_path, 'r', encoding='utf-8') as f:
with open(_JSON_PATH.value, 'r', encoding='utf-8') as f:
root = json.load(f)
visitor = MjEnumVisitor()
+5 -5
View File
@@ -28,9 +28,9 @@ from introspect import ast_nodes
from introspect import type_parsing
from . import formatter
FLAGS = flags.FLAGS
flags.DEFINE_string('header_path', None, 'Path to the original mujoco.h')
flags.DEFINE_string(
_HEADER_PATH = flags.DEFINE_string(
'header_path', None, 'Path to the original mujoco.h')
_JSON_PATH = flags.DEFINE_string(
'json_path', None,
'Path to the JSON file representing the Clang AST for mujoco.h')
@@ -113,10 +113,10 @@ def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
with open(FLAGS.json_path, 'r', encoding='utf-8') as f:
with open(_JSON_PATH.value, 'r', encoding='utf-8') as f:
root = json.load(f)
with open(FLAGS.header_path, 'r') as f:
with open(_HEADER_PATH.value, 'r') as f:
visitor = MjFunctionVisitor(f.read())
traverse(root, visitor)
+248
View File
@@ -0,0 +1,248 @@
# Copyright 2023 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generates structs.py.
The JSON input can be generated via:
clang -Xclang -ast-dump=json -fsyntax-only -fparse-all-comments -x c mujoco.h
"""
import itertools
import json
import os
import re
from typing import Any, Mapping, Sequence, Union
from absl import app
from absl import flags
from introspect import ast_nodes
from introspect import type_parsing
from . import formatter
_JSON_PATH = flags.DEFINE_string(
'json_path', None,
'Path to the JSON file representing the Clang AST for mujoco.h')
ClangJsonNode = Mapping[str, Any]
_ANONYMOUS_KEY_PATTERN = re.compile(r'\d+:\d+(?=\))')
_EXCLUDED = (
'mjpPlugin',
'mjpPlugin_',
)
def traverse(node, visitor):
visitor.visit(node)
children = node.get('inner', [])
for child in children:
traverse(child, visitor)
class _AnonymousTypePlaceholder(ast_nodes.ValueType):
def __init__(self, anonymous_key: str):
self.name = anonymous_key
self.is_const = False
self.is_volatile = False
class MjStructVisitor:
"""A Clang AST JSON node visitor for MuJoCo API struct declarations."""
def __init__(self):
self._structs = {}
self._anonymous = {}
self._typedefs = {}
def _normalize_type(
self, declname: str
) -> Union[
ast_nodes.ValueType, ast_nodes.PointerType, ast_nodes.ArrayType]:
"""Resolves anonymous structs/unions and looks up existing typedefs."""
# Check for anonymous struct/union.
if '(unnamed ' in declname:
m = _ANONYMOUS_KEY_PATTERN.search(declname)
if not m:
raise RuntimeError('cannot parse anonymous key from {m!r}')
return _AnonymousTypePlaceholder(m.group(0))
# Lookup typedef name and use it instead if one exists.
for k, v in self._typedefs.items():
if declname == v.declname:
return type_parsing.parse_type(k)
# No valid normalization, just parse the declname.
return type_parsing.parse_type(declname)
def _make_comment(self, node: ClangJsonNode) -> str:
"""Makes a comment string from a Clang AST FullComment node."""
kind = node.get('kind')
if kind == 'TextComment':
return node['text'].replace('\N{NO-BREAK SPACE}', '&nbsp;').strip()
else:
strings = []
for child in node['inner']:
strings.append(self._make_comment(child))
return ''.join(strings).strip()
def _make_field(
self, node: ClangJsonNode
) -> Union[ast_nodes.StructFieldDecl, _AnonymousTypePlaceholder]:
"""Makes a StructFieldDecl object from a Clang AST FieldDecl node."""
doc = ''
for child in node.get('inner', ()):
if child['kind'] == 'FullComment':
doc = self._make_comment(child)
if 'name' in node:
field_type = self._normalize_type(node['type']['qualType'])
return ast_nodes.StructFieldDecl(
name=node['name'], type=field_type, doc=doc)
else:
return _AnonymousTypePlaceholder(self._make_anonymous_key(node))
def _make_struct(
self, node: ClangJsonNode
) -> Union[ast_nodes.AnonymousStructDecl, ast_nodes.StructDecl]:
"""Makes a Decl object from a Clang AST RecordDecl node."""
name = f"{node['tagUsed']} {node['name']}" if 'name' in node else ''
fields = []
for child in node['inner']:
child_kind = child.get('kind')
if child_kind == 'FieldDecl':
fields.append(self._make_field(child))
if name:
return ast_nodes.StructDecl(name=name, declname=name, fields=fields)
elif node['tagUsed'] == 'union':
return ast_nodes.AnonymousUnionDecl(fields=fields)
else:
return ast_nodes.AnonymousStructDecl(fields=fields)
def _is_mujoco_type(self, node: ClangJsonNode) -> bool:
node_name = node.get('name', '')
included_from = os.path.basename(
node['loc'].get('includedFrom', {}).get('file', '')
)
return node_name not in _EXCLUDED and (
node_name.startswith('mj')
or included_from == 'mujoco.h'
or included_from.startswith('mj')
)
def _make_anonymous_key(self, node: ClangJsonNode) -> str:
line = node['loc']['line']
col = node['loc']['col']
return f'{line}:{col}'
def visit(self, node: ClangJsonNode) -> None:
"""Visits a JSON node."""
if node.get('kind') == 'RecordDecl' and self._is_mujoco_type(node):
struct_decl = self._make_struct(node)
if hasattr(struct_decl, 'name'):
self._structs[struct_decl.name] = struct_decl
else:
anonymous_key = self._make_anonymous_key(node)
if anonymous_key in self._anonymous:
raise RuntimeError(
f'duplicate key for anonymous struct: {anonymous_key}')
self._anonymous[anonymous_key] = struct_decl
elif (node.get('kind') == 'TypedefDecl' and
node['type']['qualType'].startswith('struct mj') and
node['name'] not in _EXCLUDED):
struct = self._structs[node['type']['qualType']]
self._typedefs[node['name']] = ast_nodes.StructDecl(
name=node['name'], declname=struct.declname, fields=struct.fields)
def resolve_all_anonymous(self) -> None:
"""Replaces anonymous struct placeholders with corresponding decl."""
for struct in itertools.chain(
self._structs.values(), self._typedefs.values()
):
fields = []
for field in struct.fields:
if isinstance(field, _AnonymousTypePlaceholder):
fields.append(self._anonymous[field.name])
elif isinstance(field.type, _AnonymousTypePlaceholder):
fields.append(
ast_nodes.StructFieldDecl(
name=field.name,
type=self._anonymous[field.type.name],
doc=field.doc,
)
)
else:
fields.append(field)
struct.fields = tuple(fields)
@property
def structs(self) -> Mapping[str, ast_nodes.StructDecl]:
return self._structs
@property
def typedefs(self) -> Mapping[str, ast_nodes.StructDecl]:
return self._typedefs
def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
with open(_JSON_PATH.value, 'r', encoding='utf-8') as f:
root = json.load(f)
visitor = MjStructVisitor()
traverse(root, visitor)
visitor.resolve_all_anonymous()
structs_str = formatter.format_as_python_code(visitor.typedefs)
print(f'''
# Copyright 2023 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides information about MuJoCo API structs.
DO NOT EDIT. THIS FILE IS AUTOMATICALLY GENERATED.
"""
from typing import Mapping
from .ast_nodes import AnonymousStructDecl
from .ast_nodes import AnonymousUnionDecl
from .ast_nodes import ArrayType
from .ast_nodes import PointerType
from .ast_nodes import StructDecl
from .ast_nodes import StructFieldDecl
from .ast_nodes import ValueType
STRUCTS: Mapping[str, StructDecl] = {structs_str}
'''.strip()) # `print` adds a trailing newline
if __name__ == '__main__':
app.run(main)
Executable → Regular
View File
File diff suppressed because it is too large Load Diff
+130
View File
@@ -0,0 +1,130 @@
# Copyright 2023 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for structs.py."""
import re
from absl.testing import absltest
from . import ast_nodes
from . import structs
from . import type_parsing
class StructsTest(absltest.TestCase):
def test_mjData(self): # pylint: disable=invalid-name
struct_decl = structs.STRUCTS['mjData']
self.assertEqual(struct_decl.name, 'mjData')
self.assertEqual(struct_decl.declname, 'struct mjData_')
field_names = set()
for field in struct_decl.fields:
self.assertNotIn(field.name, field_names)
field_names.add(field.name)
if field.name == 'warning':
self.assertEqual(field.type,
type_parsing.parse_type('mjWarningStat[8]'))
self.assertEqual(field.doc, 'warning statistics')
elif field.name == 'qpos':
self.assertEqual(field.type, type_parsing.parse_type('mjtNum*'))
self.assertEqual(re.sub(r'\s+', ' ', field.doc), 'position (nq x 1)')
self.assertIn('warning', field_names)
self.assertIn('qpos', field_names)
def test_mjVisual(self): # pylint: disable=invalid-name
struct_decl = structs.STRUCTS['mjVisual']
self.assertEqual(struct_decl.name, 'mjVisual')
self.assertEqual(struct_decl.declname, 'struct mjVisual_')
outer_fields = set()
for outer_field in struct_decl.fields:
self.assertNotIn(outer_field.name, outer_fields)
outer_fields.add(outer_field.name)
self.assertIsInstance(outer_field.type, ast_nodes.AnonymousStructDecl)
inner_fields = set()
if outer_field.name == 'global':
for inner_field in outer_field.type.fields:
self.assertNotIn(inner_field.name, inner_fields)
inner_fields.add(inner_field.name)
if inner_field.name == 'ipd':
self.assertEqual(inner_field.type, type_parsing.parse_type('float'))
self.assertEqual(
inner_field.doc, 'inter-pupilary distance for free camera'
)
elif inner_field.name == 'offwidth':
self.assertEqual(inner_field.type, type_parsing.parse_type('int'))
self.assertEqual(inner_field.doc, 'width of offscreen buffer')
self.assertIn('ipd', inner_fields)
self.assertIn('offwidth', inner_fields)
elif outer_field.name == 'headlight':
for inner_field in outer_field.type.fields:
self.assertNotIn(inner_field.name, inner_fields)
inner_fields.add(inner_field.name)
if inner_field.name in {'ambient', 'diffuse', 'specular'}:
self.assertEqual(inner_field.type,
type_parsing.parse_type('float[3]'))
self.assertEqual(inner_field.doc,
f'{inner_field.name} rgb (alpha=1)')
elif inner_field.name == 'active':
self.assertEqual(inner_field.type, type_parsing.parse_type('int'))
self.assertEqual(inner_field.doc, 'is headlight active')
self.assertIn('ambient', inner_fields)
self.assertIn('diffuse', inner_fields)
self.assertIn('specular', inner_fields)
self.assertIn('active', inner_fields)
self.assertIn('global', outer_fields)
self.assertIn('headlight', outer_fields)
def test_mjuiItem(self): # pylint: disable=invalid-name
struct_decl = structs.STRUCTS['mjuiItem']
self.assertEqual(struct_decl.name, 'mjuiItem')
self.assertEqual(struct_decl.declname, 'struct mjuiItem_')
found_anonymous_union = False
outer_fields = set()
for outer_field in struct_decl.fields:
if isinstance(outer_field, ast_nodes.AnonymousUnionDecl):
self.assertFalse(found_anonymous_union)
found_anonymous_union = True
inner_fields = set()
for inner_field in outer_field.fields:
self.assertNotIn(inner_field.name, inner_fields)
inner_fields.add(inner_field.name)
if inner_field.name == 'single':
self.assertEqual(inner_field.type,
type_parsing.parse_type('struct mjuiItemSingle_'))
self.assertEqual(inner_field.doc, 'check and button')
elif inner_field.name == 'multi':
self.assertEqual(inner_field.type,
type_parsing.parse_type('struct mjuiItemMulti_'))
self.assertEqual(inner_field.doc, 'static, radio and select')
self.assertIn('single', inner_fields)
self.assertIn('multi', inner_fields)
else:
self.assertNotIn(outer_field.name, outer_fields)
outer_fields.add(outer_field.name)
if outer_field.name == 'pdata':
self.assertEqual(outer_field.type, type_parsing.parse_type('void*'))
self.assertEqual(outer_field.doc, 'data pointer (type-specific)')
self.assertTrue(found_anonymous_union)
self.assertIn('pdata', outer_fields)
if __name__ == '__main__':
absltest.main()
+1 -1
View File
@@ -37,7 +37,7 @@ def _parse_qualifiers(
if part in qualifiers:
counter[part] += 1
if counter[part] > 1:
ValueError('duplicate qualifier: {part!r}')
raise ValueError('duplicate qualifier: {part!r}')
else:
non_qualifiers.append(part)
is_qualifier = dict()
+1
View File
@@ -214,6 +214,7 @@ PYBIND11_MODULE(_render, pymodule) {
X(windowStereo);
X(windowDoublebuffer);
X(currentBuffer);
X(readPixelFormat);
#undef X
#define X(var) \
+2 -2
View File
@@ -23,10 +23,8 @@
#include <ios>
#include <iostream>
#include <memory>
#include <numeric>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
@@ -2167,6 +2165,7 @@ This is useful for example when the MJB is not available as a file on disk.)"));
})
X(maxgeom);
X(ngeom);
X(nskin);
X(nlight);
X(enabletransform);
X(scale);
@@ -2228,6 +2227,7 @@ This is useful for example when the MJB is not available as a file on disk.)"));
X(textrgb);
X(linergb);
X(range);
X(highlight);
X(linepnt);
X(linedata);
X(xaxispixel);