dump_pbr.py

import argparse, sys, os, math, io
from typing import *
import bpy
import bmesh
from mathutils import Vector, Matrix
import numpy as np
from PIL import Image
import pickle


"""=============== BLENDER ==============="""

IMPORT_FUNCTIONS: Dict[str, Callable] = {
    "obj": bpy.ops.import_scene.obj if bpy.app.version[0] < 4 else bpy.ops.wm.obj_import,
    "glb": bpy.ops.import_scene.gltf,
    "gltf": bpy.ops.import_scene.gltf,
    "usd": bpy.ops.import_scene.usd,
    "fbx": bpy.ops.import_scene.fbx,
    "stl": bpy.ops.import_mesh.stl if bpy.app.version[0] < 4 else bpy.ops.wm.stl_import,
    "usda": bpy.ops.import_scene.usda,
    "dae": bpy.ops.wm.collada_import,
    "ply": bpy.ops.import_mesh.ply if bpy.app.version[0] < 4 else bpy.ops.wm.ply_import,
    "abc": bpy.ops.wm.alembic_import,
    "blend": bpy.ops.wm.append,
}


def init_scene() -> None:
    """Resets the scene to a clean state.

    Returns:
        None
    """
    # delete everything
    for obj in bpy.data.objects:
        bpy.data.objects.remove(obj, do_unlink=True)

    # delete all the materials
    for material in bpy.data.materials:
        bpy.data.materials.remove(material, do_unlink=True)

    # delete all the textures
    for texture in bpy.data.textures:
        bpy.data.textures.remove(texture, do_unlink=True)

    # delete all the images
    for image in bpy.data.images:
        bpy.data.images.remove(image, do_unlink=True)


def load_object(object_path: str) -> None:
    """Loads a model with a supported file extension into the scene.

    Args:
        object_path (str): Path to the model file.

    Raises:
        ValueError: If the file extension is not supported.

    Returns:
        None
    """
    file_extension = object_path.split(".")[-1].lower()
    if file_extension is None:
        raise ValueError(f"Unsupported file type: {object_path}")

    if file_extension == "usdz":
        # install usdz io package
        dirname = os.path.dirname(os.path.realpath(__file__))
        usdz_package = os.path.join(dirname, "io_scene_usdz.zip")
        bpy.ops.preferences.addon_install(filepath=usdz_package)
        # enable it
        addon_name = "io_scene_usdz"
        bpy.ops.preferences.addon_enable(module=addon_name)
        # import the usdz
        from io_scene_usdz.import_usdz import import_usdz

        import_usdz(context, filepath=object_path, materials=True, animations=True)
        return None

    # load from existing import functions
    import_function = IMPORT_FUNCTIONS[file_extension]

    print(f"Loading object from {object_path}")
    if file_extension == "blend":
        import_function(directory=object_path, link=False)
    elif file_extension in {"glb", "gltf"}:
        import_function(filepath=object_path, merge_vertices=True, import_shading='NORMALS', bone_heuristic='TEMPERANCE')
    else:
        import_function(filepath=object_path)
        
        
def delete_invisible_objects() -> None:
    """Deletes all invisible objects in the scene.

    Returns:
        None
    """
    to_remove = []
    for obj in bpy.context.scene.objects:
        if obj.hide_viewport or obj.hide_render:
            obj.hide_viewport = False
            obj.hide_render = False
            obj.hide_select = False
            to_remove.append(obj)
    for obj in to_remove:
        bpy.data.objects.remove(obj, do_unlink=True)

    # Delete invisible collections
    invisible_collections = [col for col in bpy.data.collections if col.hide_viewport]
    for col in invisible_collections:
        bpy.data.collections.remove(col)
      

def scene_bbox() -> Tuple[Vector, Vector]:
    """Returns the bounding box of the scene.

    Taken from Shap-E rendering script
    (https://github.com/openai/shap-e/blob/main/shap_e/rendering/blender/blender_script.py#L68-L82)

    Returns:
        Tuple[Vector, Vector]: The minimum and maximum coordinates of the bounding box.
    """
    bbox_min = (math.inf,) * 3
    bbox_max = (-math.inf,) * 3
    found = False
    scene_meshes = [obj for obj in bpy.context.scene.objects.values() if isinstance(obj.data, bpy.types.Mesh)]
    for obj in scene_meshes:
        found = True
        for coord in obj.bound_box:
            coord = Vector(coord)
            coord = obj.matrix_world @ coord
            bbox_min = tuple(min(x, y) for x, y in zip(bbox_min, coord))
            bbox_max = tuple(max(x, y) for x, y in zip(bbox_max, coord))
    if not found:
        raise RuntimeError("no objects in scene to compute bounding box for")
    return Vector(bbox_min), Vector(bbox_max)


def normalize_scene() -> Tuple[float, Vector]:
    """Normalizes the scene by scaling and translating it to fit in a unit cube centered
    at the origin.

    Mostly taken from the Point-E / Shap-E rendering script
    (https://github.com/openai/point-e/blob/main/point_e/evals/scripts/blender_script.py#L97-L112),
    but fix for multiple root objects: (see bug report here:
    https://github.com/openai/shap-e/pull/60).

    Returns:
        Tuple[float, Vector]: The scale factor and the offset applied to the scene.
    """
    scene_root_objects = [obj for obj in bpy.context.scene.objects.values() if not obj.parent]
    if len(scene_root_objects) > 1:
        # create an empty object to be used as a parent for all root objects
        scene = bpy.data.objects.new("ParentEmpty", None)
        bpy.context.scene.collection.objects.link(scene)

        # parent all root objects to the empty object
        for obj in scene_root_objects:
            obj.parent = scene
    else:
        scene = scene_root_objects[0]

    bbox_min, bbox_max = scene_bbox()
    scale = 1 / max(bbox_max - bbox_min)
    scene.scale = scene.scale * scale

    # Apply scale to matrix_world.
    bpy.context.view_layer.update()
    bbox_min, bbox_max = scene_bbox()
    offset = -(bbox_min + bbox_max) / 2
    scene.matrix_world.translation += offset
    
    return scale, offset


# =============== NODE TREE PARSING ===============

def extract_image(tex_node, channels):
        image = tex_node.image
        pixels = np.array(image.pixels[:])
        data = pixels.reshape(image.size[1], image.size[0], -1)
        data = data[..., channels]

        if data.dtype != np.uint8:
            data = np.clip(data, 0.0, 1.0)
            data = (data * 255).astype(np.uint8)

        if len(data.shape) == 2:  # Single channel
            pil_image = Image.fromarray(data, mode='L')
        elif data.shape[2] == 3:
            pil_image = Image.fromarray(data, mode='RGB')
        elif data.shape[2] == 4:
            pil_image = Image.fromarray(data, mode='RGBA')
        else:
            raise ValueError("Unsupported channel shape for image")

        buffer = io.BytesIO()
        pil_image.save(buffer, format='PNG')
        png_bytes = buffer.getvalue()

        return {
            'image': png_bytes,
            'interpolation': tex_node.interpolation,
            'extension': tex_node.extension,
        }


def try_extract_image(link, expected_channel='RGB'):
    """
    Tries to extract an image from a texture node link.
    Supported sub tree modes:
      - RGB:
        TEX_IMAGE ->
      - R, G, B:
        TEX_IMAGE -> SEPARATE_COLOR ->
      - A:
        TEX_IMAGE ->
    """
    assert expected_channel in ['RGB', 'R', 'G', 'B', 'A'], "Unsupported channel"

    if expected_channel == 'RGB':
        assert link.from_node.type == 'TEX_IMAGE', "Material is not supported"
        assert link.from_socket.name == 'Color', "Material is not supported"
        tex_node = link.from_node
        return extract_image(tex_node, [0, 1, 2])

    if expected_channel in ['R', 'G', 'B']:
        socket_name = {
            'R': 'Red',
            'G': 'Green',
            'B': 'Blue',
        }[expected_channel]
        assert link.from_node.type == 'SEPARATE_COLOR' and link.from_node.mode == 'RGB', \
            f"Material is not supported, {link.from_node.type}, {link.from_node.mode}"
        assert link.from_socket.name == socket_name, "Material is not supported"
        sep_node = link.from_node
        assert sep_node.inputs[0].is_linked and sep_node.inputs[0].links[0].from_node.type == 'TEX_IMAGE', \
            "Material is not supported"
        assert sep_node.inputs[0].links[0].from_socket.name == 'Color', "Material is not supported"
        tex_node = sep_node.inputs[0].links[0].from_node
        channel_index = {
            'R': 0,
            'G': 1,
            'B': 2,
        }[expected_channel]
        return extract_image(tex_node, channel_index)

    if expected_channel == 'A':
        assert link.from_node.type == 'TEX_IMAGE', "Material is not supported"
        assert link.from_socket.name == 'Alpha', "Material is not supported"
        tex_node = link.from_node
        return extract_image(tex_node, 3)


def try_extract_factor(link, mode='color'):
    """
    Tries to extract a factor from a math node link.
    Supported sub tree modes:
      - color:
       ANY -> MIX(MULTIPLY) ->
      - scalar:
       ANY -> MATH(MULTIPLY) ->
    """
    assert mode in ['color','scalar'], "Unsupported mode"

    if mode == 'color':
        if link.from_node.type == 'MIX':
            mix_node = link.from_node
            assert mix_node.data_type == 'RGBA' and mix_node.blend_type == 'MULTIPLY', f"Material is not supported, {mix_node.data_type}, {mix_node.blend_type}"
            assert not mix_node.inputs['Factor'].is_linked and mix_node.inputs['Factor'].default_value == 1.0, \
                "Material is not supported"
            if mix_node.inputs['A'].is_linked:
                assert not mix_node.inputs['B'].is_linked, "Material is not supported"
                return (list(mix_node.inputs['B'].default_value)[:3], mix_node.inputs['A'].links[0])
            else:
                assert not mix_node.inputs['A'].is_linked, "Material is not supported"
                assert mix_node.inputs['B'].is_linked, "Material is not supported"
                return (list(mix_node.inputs['A'].default_value)[:3], mix_node.inputs['B'].links[0])
        return ([1.0, 1.0, 1.0], link)

    if mode =='scalar':
        if link.from_node.type == 'MATH':
            math_node = link.from_node
            assert math_node.operation == 'MULTIPLY', "Material is not supported"
            assert math_node.inputs[0].is_linked, "Material is not supported"
            assert not math_node.inputs[1].is_linked, "Material is not supported"
            return (math_node.inputs[1].default_value, math_node.inputs[0].links[0])
        return (1.0, link)


def try_extract_image_with_factor(link, expected_channel='RGB'):
    """
    Tries to extract an image and a factor from a texture node link.
    """
    factor, link = try_extract_factor(link, 'color' if expected_channel in ['RGB'] else 'scalar')
    image = try_extract_image(link, expected_channel)
    return (factor, image)


def main(arg):    
    # Initialize context
    if arg.object.endswith(".blend"):
        delete_invisible_objects()
    else:
        init_scene()
        load_object(arg.object)
    print('[INFO] Scene initialized.')
    
    # Normalize scene
    scale, offset = normalize_scene()
    print('[INFO] Scene normalized.')
    
    # Start dumping
    depsgraph = bpy.context.evaluated_depsgraph_get()
    scene = bpy.context.scene
    output = {
        'materials': [],
        'objects': [],
    }

    # Dumping materials
    for mat in bpy.data.materials:
        assert mat.use_nodes == True, "Material is not supported"

        pack = {
            "baseColorFactor": [1.0, 1.0, 1.0],
            "alphaFactor": 1.0,
            "metallicFactor": 1.0,
            "roughnessFactor": 1.0,
            "alphaMode": "OPAQUE",
            "alphaCutoff": 0.5,
            "baseColorTexture": None,
            "alphaTexture": None,
            "metallicTexture": None,
            "roughnessTexture": None,
        }

        try:
            principled_node = mat.node_tree.nodes.get('Principled BSDF')
            assert principled_node is not None, "Material is not supported"

            # Handle base color
            if not principled_node.inputs['Base Color'].is_linked:
                pack["baseColorFactor"] = list(principled_node.inputs['Base Color'].default_value)
            else:
                link = principled_node.inputs['Base Color'].links[0]
                if link.from_node.type == 'RGB':
                    pack["baseColorFactor"] = list(link.from_node.outputs[0].default_value)
                else:
                    factor, image = try_extract_image_with_factor(link, 'RGB')
                    pack["baseColorFactor"] = factor
                    pack["baseColorTexture"] = image

            # Handle alpha
            if not principled_node.inputs['Alpha'].is_linked:
                pack["alphaFactor"] = principled_node.inputs['Alpha'].default_value
                if pack["alphaFactor"] < 1.0:
                    pack["alphaMode"] = "BLEND"
            else:
                link = principled_node.inputs['Alpha'].links[0]
                node = link.from_node
                if node.type == 'VALUE':
                    pack["alphaFactor"] = node.outputs[0].default_value
                    if pack["alphaFactor"] < 1.0:
                        pack["alphaMode"] = "BLEND"
                else:
                    pack["alphaMode"] = "BLEND"
                    if node.type == 'MATH':
                        if node.operation == 'ROUND':
                            assert node.inputs[0].is_linked, "Material is not supported"
                            pack["alphaMode"] = "MASK"
                            link = node.inputs[0].links[0]
                        elif node.operation == 'SUBTRACT':
                            assert node.inputs[0].default_value == 1.0 and \
                                node.inputs[1].is_linked and \
                                node.inputs[1].links[0].from_node.type == 'MATH' and \
                                node.inputs[1].links[0].from_node.operation == 'LESS_THAN', \
                                "Material is not supported"
                            assert node.inputs[1].links[0].from_node.inputs[0].is_linked, "Material is not supported"
                            pack["alphaMode"] = "MASK"
                            pack["alphaCutoff"] = node.inputs[1].links[0].from_node.inputs[1].default_value
                            link = node.inputs[1].links[0].from_node.inputs[0].links[0]
                    factor, image = try_extract_image_with_factor(link, 'A')
                    pack["alphaFactor"] = factor
                    pack["alphaTexture"] = image

            # Handle metallic
            if not principled_node.inputs['Metallic'].is_linked:
                pack["metallicFactor"] = principled_node.inputs['Metallic'].default_value
            else:
                link = principled_node.inputs['Metallic'].links[0]
                node = link.from_node
                if node.type == 'VALUE':
                    pack["metallicFactor"] = node.outputs[0].default_value
                else:
                    factor, image = try_extract_image_with_factor(link, 'B')
                    pack["metallicFactor"] = factor
                    pack["metallicTexture"] = image

            # Handle roughness
            if not principled_node.inputs['Roughness'].is_linked:
                pack["roughnessFactor"] = principled_node.inputs['Roughness'].default_value
            else:
                link = principled_node.inputs['Roughness'].links[0]
                node = link.from_node
                if node.type == 'VALUE':
                    pack["roughnessFactor"] = node.outputs[0].default_value
                else:
                    factor, image = try_extract_image_with_factor(link, 'G')
                    pack["roughnessFactor"] = factor
                    pack["roughnessTexture"] = image

            output['materials'].append(pack)
        except:
            with open(arg.output_path + '_error.txt', 'w') as f:
                f.write(str([[n.name] for n in mat.node_tree.nodes]))
            raise RuntimeError("Material is not supported")

    # Dumping meshes
    for obj in scene.objects:
        if obj.type != 'MESH':
            continue
        
        pack = {
            "vertices": None,
            "faces": None,
            "uvs": None,
            "matIDs": None,
        }
        
        eval_obj = obj.evaluated_get(depsgraph)
        eval_mesh = eval_obj.to_mesh()
        
        bm = bmesh.new()
        bm.from_mesh(eval_mesh)
        bm.transform(obj.matrix_world)
        bmesh.ops.triangulate(bm, faces=bm.faces)
        bm.to_mesh(eval_mesh)
        bm.free()
                
        pack["vertices"] = np.array([
            v.co[:] for v in eval_mesh.vertices
        ], dtype=np.float32)   # (N, 3)
        
        pack["faces"] = np.array([
            [eval_mesh.loops[i].vertex_index for i in poly.loop_indices]
            for poly in eval_mesh.polygons
        ], dtype=np.int32)   # (F, 3)
        
        pack["normals"] = np.array([
            [eval_mesh.loops[i].normal for i in poly.loop_indices]
            for poly in eval_mesh.polygons
        ], dtype=np.float32)  # (F, 3, 3)
        
        if eval_mesh.uv_layers.active is not None:
            pack["uvs"] = np.array([
                [eval_mesh.uv_layers.active.data[i].uv for i in poly.loop_indices]
                for poly in eval_mesh.polygons
            ], dtype=np.float32)  # (F, 3, 2)

        pack["mat_ids"] = np.array([
            bpy.data.materials.find(obj.material_slots[poly.material_index].name)
            if len(obj.material_slots) > 0 and obj.material_slots[poly.material_index].material is not None else -1
            for poly in eval_mesh.polygons
        ], dtype=np.int32)

        output['objects'].append(pack)

    # Save output
    os.makedirs(os.path.dirname(arg.output_path), exist_ok=True)
    with open(arg.output_path, 'wb') as f:
        pickle.dump(output, f)
    print('[INFO] Output saved to {}.'.format(arg.output_path))

        
if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Renders given obj file by rotation a camera around it.')
    parser.add_argument('--object', type=str, help='Path to the 3D model file to be rendered.')
    parser.add_argument('--output_path', type=str, default='/tmp', help='The path the output will be dumped to.')
    argv = sys.argv[sys.argv.index("--") + 1:]
    args = parser.parse_args(argv)

    main(args)