"""
.. Mdl API
"""
import os
from valvemdl.constants import MDL_VERSIONS, STUDIO_ANIM_MASK, VTX_EXTENSIONS
from valvemdl.exceptions import MdlSidecarMissingError
from valvemdl.loader import (load_animations, load_attachments,
load_bodyparts, load_bonecontrollers, load_bones,
load_boneflexdrivers, load_cdtextures,
load_flexcontrollers, load_flexcontrollerui,
load_flexdescs, load_flexrules, load_hitboxsets,
load_ikchains, load_includemodels,
load_linearbones, load_poseparameters,
load_sequences, load_skins,
load_srcbonetransforms, load_textures,
read_string)
from valvemdl.structs.studiohdr import studiohdr_t, studiohdr2_t
from valvemdl.structs.vvd import mstudiovertex_t
from valvemdl.unloader import rebuild
from valvemdl.vtx import Vtx
from valvemdl.vvd import Vvd
[docs]
class Mdl(object):
"""Contains all the data from an Mdl file.
An mdl is small (a couple of kilobytes is typical) and its contents form one
interlinked graph rather than a set of independent blocks, so the file is
parsed eagerly and in full.
Its sidecars are not. A model is really a set of files -- ``.mdl``,
``.vvd``, and one ``.vtx`` per graphics backend -- and most callers only
ever want the mdl. Opening an mdl therefore only *detects* the sidecars
sitting next to it; they are parsed on first access to :py:attr:`vvd<vvd>`
or :py:attr:`vtx<vtx>`, and cached from then on.
"""
[docs]
def __init__(self, path=None):
"""Creates an empty instance of Mdl.
:param path: A path to an existing mdl file.
:type path: str, optional
"""
self.source_path = path #: :type: (str) - The location of the parsed file.
self.header = None #: :type: (construct.Container) - The parsed studiohdr_t.
#: :type: (construct.Container) - The parsed studiohdr2_t, None when absent.
self.header2 = None
# the file, verbatim. kept because every offset in the format is
# relative to its own structure, and because the writer will need the
# original bytes to compare against.
self._data = None
#: :type: (list[construct.Container]) - The skeleton, in file order.
self.bones = []
#: :type: (list[construct.Container]) - The materials the model names.
self.textures = []
#: :type: (list[str]) - The material search paths, in engine order.
self.cdtextures = []
#: :type: (list[list[int]]) - The skin table, one row per skin family.
self.skins = []
# each bodypart carries .models, and each model carries .meshes
#: :type: (list[construct.Container]) - The geometry tree.
self.bodyparts = []
# each set carries .hitboxes
#: :type: (list[construct.Container]) - The hitbox sets.
self.hitboxsets = []
#: :type: (list[construct.Container]) - The attachment points.
self.attachments = []
#: :type: (list[construct.Container]) - The pose parameters.
self.poseparameters = []
#: :type: (list[construct.Container]) - The $includemodel groups.
self.includemodels = []
# each carries .movements, .sections and .ikrules
#: :type: (list[construct.Container]) - The animations, local to this file.
self.animations = []
# each carries .events, .autolayers, .iklocks and .animindices
#: :type: (list[construct.Container]) - The sequences, local to this file.
self.sequences = []
# each carries .links
#: :type: (list[construct.Container]) - The ik chains.
self.ikchains = []
#: :type: (list[construct.Container]) - The bone controllers.
self.bonecontrollers = []
#: :type: (list[construct.Container]) - The flex targets.
self.flexdescs = []
#: :type: (list[construct.Container]) - The flex controllers.
self.flexcontrollers = []
# each carries .ops
#: :type: (list[construct.Container]) - The flex rules.
self.flexrules = []
#: :type: (list[construct.Container]) - The flex controller ui groups.
self.flexcontrollerui = []
#: :type: (construct.Container) - studiohdr2's linear bone table, or None.
self.linearbones = None
#: :type: (list[construct.Container]) - The source bone transforms.
self.srcbonetransforms = []
# each carries .controls
#: :type: (list[construct.Container]) - The bone flex drivers.
self.boneflexdrivers = []
self._vvd = None
self._vtx = {}
self._header2_truncated = False
#: :type: (dict[str, str]) - Sidecars found next to the mdl, by extension.
self.sidecar_paths = {}
if self.source_path:
with open(self.source_path, 'rb') as f:
self._data = f.read()
self.header = studiohdr_t.parse(self._data)
# a truncated file can point studiohdr2index past its own end.
# don't crash on it: a broken model is exactly the one somebody is
# trying to look at, and validate() has more useful things to say
# about it than a StreamError from 250 bytes deep would.
index = self.header.studiohdr2index
if index and index + studiohdr2_t.sizeof() <= len(self._data):
self.header2 = studiohdr2_t.parse(self._data[index:])
elif index:
self._header2_truncated = True
self.bones = load_bones(self, self._data)
self.textures = load_textures(self, self._data)
self.cdtextures = load_cdtextures(self, self._data)
self.skins = load_skins(self, self._data)
self.bodyparts = load_bodyparts(self, self._data)
self.hitboxsets = load_hitboxsets(self, self._data)
self.attachments = load_attachments(self, self._data)
self.poseparameters = load_poseparameters(self, self._data)
self.includemodels = load_includemodels(self, self._data)
self.bonecontrollers = load_bonecontrollers(self, self._data)
self.flexdescs = load_flexdescs(self, self._data)
self.flexcontrollers = load_flexcontrollers(self, self._data)
self.flexrules = load_flexrules(self, self._data)
self.flexcontrollerui = load_flexcontrollerui(self, self._data)
self.linearbones = load_linearbones(self, self._data)
self.srcbonetransforms = load_srcbonetransforms(self, self._data)
self.boneflexdrivers = load_boneflexdrivers(self, self._data)
self.animations = load_animations(self, self._data)
self.sequences = load_sequences(self, self._data)
self.ikchains = load_ikchains(self, self._data)
self._find_sidecars()
@property
def version(self):
"""Team Fortress 2 ships 44 through 48. :ref:`See the constants
page<constants>`.
:type: (int) - The mdl version.
"""
return self.header.version
@property
def checksum(self):
"""The engine refuses to load a set whose checksums disagree.
:type: (int) - Ties the mdl to its vvd and vtx files.
"""
return self.header.checksum
@property
def name(self):
"""The model's own idea of its path, as baked in at compile time.
``studiohdr_t.name`` is a fixed 64 byte buffer, so studiomdl spills
longer paths into ``studiohdr2_t.sznameindex`` instead. This resolves
whichever one is live, the same way the engine's ``pszName()`` does.
:type: (str) - The model's path.
"""
if self.header2 is not None and self.header2.sznameindex:
# sznameindex is relative to the studiohdr2_t, not to the file
return read_string(self._data, self.header.studiohdr2index +
self.header2.sznameindex)
return self.header.name
@property
def surfaceprop(self):
"""The model-wide surface property, named in the qc by
``$surfaceprop``. Individual bones carry their own, which is what the
engine actually uses when it needs to know what a given bone is made
of.
:type: (str) - A surfaceprop name, e.g. ``'metal'``.
"""
return read_string(self._data, self.header.surfacepropindex)
@property
def models(self):
"""Every model in the file, flattened out of the bodypart tree.
Note that these are *alternatives* -- a model here is one option inside
one bodypart, not one model in the everyday sense. Only one per
bodypart is ever drawn at a time.
:type: (list[construct.Container]) - All the models.
"""
return [model for bodypart in self.bodyparts
for model in bodypart.models]
@property
def meshes(self):
"""Every mesh in the file, flattened out of the bodypart tree.
:type: (list[construct.Container]) - All the meshes.
"""
return [mesh for model in self.models for mesh in model.meshes]
[docs]
def bodygroup(self, bodynum):
"""Decodes a packed bodygroup int into one model per bodypart.
studiomdl packs the whole choice of alternatives into a single integer,
which entities carry as ``m_nBody``. Each bodypart pulls its own choice
back out with its ``base`` as the stride.
:param bodynum: The packed value.
:type bodynum: int
:returns: The chosen model for each bodypart, in bodypart order.
:rtype: list[construct.Container]
"""
chosen = []
for bodypart in self.bodyparts:
if not bodypart.nummodels or not bodypart.base:
continue
index = (bodynum // bodypart.base) % bodypart.nummodels
chosen.append(bodypart.models[index])
return chosen
[docs]
def save(self, destination=None):
"""Writes the model back out.
The bytes are rebuilt from the object graph rather than copied from the
file, so what lands on disk reflects the model as it now is.
:param destination: Where to write. Overwrites the original when not
given.
:type destination: str, optional
"""
with open(destination or self.source_path, 'wb') as f:
f.write(bytes(rebuild(self).data))
[docs]
def coverage(self):
"""Reports how much of the file |proj_name| can account for.
Rebuilding into an empty buffer means every byte has to be put there by
something; anything left is something we have not understood. This
reports what happened, which is more honest than a bare "it worked":
- ``struct`` -- bytes regenerated from parsed structures.
- ``opaque`` -- bytes copied at an extent we worked out ourselves, the
compressed animation streams above all.
- ``unclaimed`` -- bytes nothing wrote. Harmless where the original is
zero (alignment padding, which the rebuild gets for free) and a real
gap where it is not.
:returns: Byte counts, plus ``differ``: how many bytes the rebuild got
wrong. Zero means the round trip is byte-exact.
:rtype: dict
"""
out = rebuild(self)
report = out.report()
report['differ'] = sum(1 for a, b in zip(out.data, self._data)
if a != b)
return report
[docs]
def skin_texture(self, skin, material):
"""Resolves a mesh's material through the skin table.
A mesh does not name its texture directly. ``mstudiomesh_t.material``
is a column in the skin table, and the row is whichever skin the entity
happens to be using -- which is how one model ships red and blu variants
without duplicating any geometry.
:param skin: The skin family, 0 being the default.
:type skin: int
:param material: A mesh's material field.
:type material: int
:returns: The texture, or None when the model has no skin table to
resolve through.
:rtype: construct.Container
"""
if not self.skins or skin >= len(self.skins):
return None
row = self.skins[skin]
if material >= len(row):
return None
index = row[material]
if index < 0 or index >= len(self.textures):
return None
return self.textures[index]
def _find_sidecars(self):
# a model is a set of files; note which of them are actually here
base = os.path.splitext(self.source_path)[0]
for ext in ['.vvd'] + VTX_EXTENSIONS:
candidate = base + ext
if os.path.isfile(candidate):
self.sidecar_paths[ext] = candidate
@property
def vvd(self):
"""The model's vertex file, parsed on first access.
:type: (Vvd) - The vertex file.
:raises MdlSidecarMissingError: when there is no vvd next to the mdl.
"""
if self._vvd is None:
if '.vvd' not in self.sidecar_paths:
raise MdlSidecarMissingError(
os.path.splitext(self.source_path)[0] + '.vvd')
self._vvd = Vvd(self.sidecar_paths['.vvd'])
return self._vvd
@property
def vtx(self):
"""Defaults to the ``dx90`` flavour; use :py:meth:`get_vtx` for the
others.
:type: (Vtx) - The model's optimized mesh file, parsed on first access.
"""
return self.get_vtx()
[docs]
def get_vtx(self, extension='.dx90.vtx'):
"""Provides one of the model's vtx files, parsing it on first access.
studiomdl emits a vtx per graphics backend. They describe the same
meshes optimized differently, so they are not interchangeable.
:param extension: Which flavour to load. :ref:`See the constants
page<constants>` for the available values.
:type extension: str, optional
:returns: The parsed vtx.
:rtype: Vtx
:raises MdlSidecarMissingError: when that flavour is not next to the
mdl.
"""
if extension not in self._vtx:
if extension not in self.sidecar_paths:
raise MdlSidecarMissingError(
os.path.splitext(self.source_path)[0] + extension)
self._vtx[extension] = Vtx(self.sidecar_paths[extension])
return self._vtx[extension]
[docs]
def validate(self):
"""Checks the invariants the format promises, and returns what is
broken.
This is deliberately a report rather than an exception: a model that
fails one of these is still worth looking at, and the failures
themselves are interesting.
:returns: One human readable string per broken invariant. Empty when
the model is sound.
:rtype: list[str]
"""
problems = []
if self.version not in MDL_VERSIONS:
problems.append('unknown version {0}'.format(self.version))
actual = os.path.getsize(self.source_path)
if self.header.length != actual:
problems.append(
'header length ({0}) != file size ({1})'.format(
self.header.length, actual))
if self._header2_truncated:
problems.append(
'studiohdr2index ({0}) + 256 runs past the end of the file '
'({1})'.format(self.header.studiohdr2index, actual))
if len(self.bones) != self.header.numbones:
problems.append(
'read {0} of {1} bones; the array runs past the end of the '
'file'.format(len(self.bones), self.header.numbones))
if len(self.textures) != self.header.numtextures:
problems.append(
'read {0} of {1} textures; the array runs past the end of the '
'file'.format(len(self.textures), self.header.numtextures))
for family, row in enumerate(self.skins):
if len(row) != self.header.numskinref:
problems.append(
'skin family {0} is {1} wide, expected {2}'.format(
family, len(row), self.header.numskinref))
for i, index in enumerate(row):
if index < 0 or index >= len(self.textures):
problems.append(
'skin family {0} slot {1} names texture {2}, of '
'{3}'.format(family, i, index, len(self.textures)))
if len(self.bodyparts) != self.header.numbodyparts:
problems.append(
'read {0} of {1} bodyparts; the array runs past the end of '
'the file'.format(len(self.bodyparts),
self.header.numbodyparts))
for bodypart in self.bodyparts:
if len(bodypart.models) != bodypart.nummodels:
problems.append(
'bodypart {0} declares {1} models but only {2} fit in the '
'file'.format(bodypart.name, bodypart.nummodels,
len(bodypart.models)))
for model in bodypart.models:
# vertexindex is a byte offset into the vvd, so it has to be a
# whole number of vertices. a non-multiple means we, or the
# file, are wrong about what the field means.
if model.vertexindex % mstudiovertex_t.sizeof():
problems.append(
'model {0} has a vertexindex ({1}) that is not a '
'multiple of sizeof(mstudiovertex_t)'.format(
model.name, model.vertexindex))
if len(model.meshes) != model.nummeshes:
problems.append(
'model {0} declares {1} meshes but only {2} fit in '
'the file'.format(model.name, model.nummeshes,
len(model.meshes)))
for mesh in model.meshes:
if mesh.modelindex > 0:
problems.append(
'mesh {0} of model {1} has a modelindex ({2}) '
'that does not point backwards'.format(
mesh.meshid, model.name, mesh.modelindex))
if mesh.vertexoffset + mesh.numvertices > model.numvertices:
problems.append(
'mesh {0} of model {1} runs past the end of its '
'model vertices'.format(mesh.meshid, model.name))
for i, bone in enumerate(self.bones):
if bone.parent < -1 or bone.parent >= len(self.bones):
problems.append(
'bone {0} ({1}) has an out of range parent ({2})'.format(
i, bone.name, bone.parent))
elif bone.parent >= i:
# the engine builds world transforms in a single forward pass,
# which only works because studiomdl emits parents first
problems.append(
'bone {0} ({1}) has a parent ({2}) that does not precede '
'it'.format(i, bone.name, bone.parent))
for ext in sorted(self.sidecar_paths):
sidecar = self.vvd if ext == '.vvd' else self.get_vtx(ext)
if sidecar.checksum != self.checksum:
problems.append(
'{0} checksum ({1}) != mdl checksum ({2})'.format(
ext, sidecar.checksum, self.checksum))
problems.extend(self._validate_bone_references())
problems.extend(self._validate_flexes())
problems.extend(self._validate_animations())
problems.extend(self._validate_against_vvd())
return problems
def _validate_flexes(self):
problems = []
for i, rule in enumerate(self.flexrules):
if rule.flex < 0 or rule.flex >= len(self.flexdescs):
problems.append(
'flex rule {0} targets flexdesc {1}, of {2}'.format(
i, rule.flex, len(self.flexdescs)))
if len(rule.ops) != rule.numops:
problems.append(
'flex rule {0} declares {1} ops but only {2} fit in the '
'file'.format(i, rule.numops, len(rule.ops)))
for mesh in self.meshes:
for flex in mesh.flexes:
if flex.flexdesc < 0 or flex.flexdesc >= len(self.flexdescs):
problems.append(
'a flex targets flexdesc {0}, of {1}'.format(
flex.flexdesc, len(self.flexdescs)))
if len(flex.vertanims) != flex.numverts:
problems.append(
'a flex declares {0} vertanims but only {1} fit in '
'the file'.format(flex.numverts, len(flex.vertanims)))
for model in self.models:
for eyeball in model.eyeballs:
if eyeball.bone < 0 or eyeball.bone >= len(self.bones):
problems.append(
'eyeball {0} names bone {1}, of {2}'.format(
eyeball.name, eyeball.bone, len(self.bones)))
if self.linearbones is not None:
if self.linearbones.numbones != len(self.bones):
problems.append(
'the linear bone table has {0} bones but the model has '
'{1}'.format(self.linearbones.numbones, len(self.bones)))
else:
# the table is a transposed copy of the bones, so it has to
# agree with them. a writer that updates one and not the other
# gets a model that animates from one and draws from the other.
for i, parent in enumerate(self.linearbones.parent):
if parent != self.bones[i].parent:
problems.append(
'the linear bone table gives bone {0} parent {1}, '
'but the bone says {2}'.format(
i, parent, self.bones[i].parent))
break
for driver in self.boneflexdrivers:
if driver.m_nBoneIndex < 0 or \
driver.m_nBoneIndex >= len(self.bones):
problems.append(
'a bone flex driver names bone {0}, of {1}'.format(
driver.m_nBoneIndex, len(self.bones)))
for control in driver.controls:
if control.m_nFlexControllerIndex < 0 or \
control.m_nFlexControllerIndex >= \
len(self.flexcontrollers):
problems.append(
'a bone flex driver control names flex controller '
'{0}, of {1}'.format(control.m_nFlexControllerIndex,
len(self.flexcontrollers)))
return problems
def _validate_anim_nodes(self, anim):
"""Byte accounting over one animation's node stream.
This is the check that matters for :any:`mstudioanim_t`, because it is
the one thing that cannot be fooled. The nodes are variable length, so
the only way to know we have understood a node is to measure what it
occupies and see whether the next one starts exactly there. If our idea
of a node's extent is wrong by a single byte, ``nextoffset`` disagrees
and we hear about it -- unlike a decoder, which would happily produce
plausible nonsense.
Note that ``studiobyteswap.cpp`` describes ``mstudioanim_t`` as 4 bytes
and stops, so there is no datadesc to check this against. This stands
in for it.
"""
problems = []
for i, node in enumerate(anim.nodes):
if node.extent is None:
problems.append(
'animation {0}: node {1} (bone {2}, flags 0x{3:02x}) could '
'not be walked'.format(anim.name, i, node.bone,
node.flags))
continue
if node.terminator:
# bone 255 ends the list; it names no bone and carries no
# payload, so there is nothing else to check about it
continue
if node.flags & ~STUDIO_ANIM_MASK:
problems.append(
'animation {0}: node {1} has unknown flag bits '
'0x{2:02x}'.format(anim.name, i,
node.flags & ~STUDIO_ANIM_MASK))
if node.bone >= len(self.bones):
problems.append(
'animation {0}: node {1} names bone {2}, of {3}'.format(
anim.name, i, node.bone, len(self.bones)))
# the accounting itself: a node followed by another must occupy
# exactly the gap between them
if node.nextoffset and node.nextoffset != node.extent:
problems.append(
'animation {0}: node {1} (bone {2}, flags 0x{3:02x}) spans '
'{4} bytes but the next node is {5} away'.format(
anim.name, i, node.bone, node.flags, node.extent,
node.nextoffset))
return problems
def _validate_animations(self):
problems = []
if len(self.animations) != self.header.numlocalanim:
problems.append(
'read {0} of {1} animations; the array runs past the end of '
'the file'.format(len(self.animations),
self.header.numlocalanim))
if len(self.sequences) != self.header.numlocalseq:
problems.append(
'read {0} of {1} sequences; the array runs past the end of '
'the file'.format(len(self.sequences),
self.header.numlocalseq))
for anim in self.animations:
# baseptr walks back to the studiohdr_t, which is at 0, so it is
# exactly the negation of the animdesc's own position
if anim._offset + anim.baseptr != 0:
problems.append(
'animation {0} has a baseptr ({1}) that does not resolve '
'to the studiohdr_t'.format(anim.name, anim.baseptr))
if anim.animblock < 0:
problems.append(
'animation {0} has animblock {1}; the model needs '
'recompiling'.format(anim.name, anim.animblock))
for rule in anim.ikrules:
if rule.chain < 0 or rule.chain >= len(self.ikchains):
problems.append(
'animation {0} has an ik rule naming chain {1}, of '
'{2}'.format(anim.name, rule.chain,
len(self.ikchains)))
problems.extend(self._validate_anim_nodes(anim))
for seq in self.sequences:
if seq._offset + seq.baseptr != 0:
problems.append(
'sequence {0} has a baseptr ({1}) that does not resolve '
'to the studiohdr_t'.format(seq.label, seq.baseptr))
for row in seq.animindices:
for index in row:
if index < 0 or index >= len(self.animations):
problems.append(
'sequence {0} blends animation {1}, of {2}'.format(
seq.label, index, len(self.animations)))
# paramindex is only ours to check when two things hold. -1 means
# explicitly unused, and a groupsize of 0 means the axis does not
# exist, in which case studiomdl leaves paramindex as 0 rather than
# -1 and the value means nothing. And a model with $includemodel
# indexes the *merged* pose parameter list, which we do not build:
# the nine c_*_arms viewmodels are exactly this case.
if self.includemodels:
continue
for i in range(2):
param = seq.paramindex[i]
if param == -1 or not seq.groupsize[i]:
continue
if param < 0 or param >= len(self.poseparameters):
problems.append(
'sequence {0} names pose parameter {1}, of '
'{2}'.format(seq.label, param,
len(self.poseparameters)))
for chain in self.ikchains:
for link in chain.links:
if link.bone < 0 or link.bone >= len(self.bones):
problems.append(
'ik chain {0} has a link naming bone {1}, of '
'{2}'.format(chain.name, link.bone, len(self.bones)))
return problems
def _validate_bone_references(self):
# hitboxes, attachments and mouths all name a bone by index, and a
# stale one is the kind of damage a decompile-recompile leaves behind
problems = []
count = len(self.bones)
for hitboxset in self.hitboxsets:
for i, hitbox in enumerate(hitboxset.hitboxes):
if hitbox.bone < 0 or hitbox.bone >= count:
problems.append(
'hitbox {0} of set {1} names bone {2}, of {3}'.format(
i, hitboxset.name, hitbox.bone, count))
for attachment in self.attachments:
if attachment.localbone < 0 or attachment.localbone >= count:
problems.append(
'attachment {0} names bone {1}, of {2}'.format(
attachment.name, attachment.localbone, count))
return problems
def _validate_against_vvd(self):
# the mdl and the vvd describe the same vertices from two sides, and
# they have to agree: every mesh says how many vertices it keeps at
# each lod, and the vvd says the totals. this is the check that would
# catch us misreading vertexindex as a count rather than a byte offset.
if '.vvd' not in self.sidecar_paths or not self.bodyparts:
return []
problems = []
header = self.vvd.header
meshes = self.meshes
for lod in range(min(header.numLODs, len(header.numLODVertexes))):
ours = sum(mesh.vertexdata.numLODVertexes[lod] for mesh in meshes)
theirs = header.numLODVertexes[lod]
if ours != theirs:
problems.append(
'meshes hold {0} vertices at lod {1} but the vvd says '
'{2}'.format(ours, lod, theirs))
for model in self.models:
ours = sum(mesh.numvertices for mesh in model.meshes)
if ours != model.numvertices:
problems.append(
'model {0} says {1} vertices but its meshes hold '
'{2}'.format(model.name, model.numvertices, ours))
return problems
[docs]
def __repr__(self):
return "<{0}.{1} v{2} '{3}' at {4}>".format(type(self).__module__,
type(self).__name__,
self.version,
self.name,
hex(id(self)))
