v1.0

direct3d/utils/image.py

+import rembg
+import numpy as np
+from PIL import Image
+from torchvision import transforms as T
+
+
+def crop_recenter(image_no_bg, thereshold=100):
+    image_no_bg_np = np.array(image_no_bg)
+    if image_no_bg_np.shape[2] == 3:
+        return image_no_bg
+    mask = (image_no_bg_np[..., -1]).astype(np.uint8)
+    mask_bin = mask > thereshold
+    
+    H, W = image_no_bg_np.shape[:2]
+    
+    valid_pixels = mask_bin.astype(np.float32).nonzero()
+    if np.sum(mask_bin) < (H*W) * 0.001:
+        min_h = 0
+        max_h = H - 1
+        min_w = 0
+        max_w = W -1
+    else:
+        min_h, max_h = valid_pixels[0].min(), valid_pixels[0].max()
+        min_w, max_w = valid_pixels[1].min(), valid_pixels[1].max()
+    
+    if min_h < 0:
+        min_h = 0
+    if min_w < 0:
+        min_w = 0
+    if max_h > H:
+        max_h = H - 1
+    if max_w > W:
+        max_w = W - 1
+
+    image_no_bg_np = image_no_bg_np[min_h:max_h+1, min_w:max_w+1]
+    image_no_bg = Image.fromarray(image_no_bg_np)
+    return image_no_bg
+
+
+def pad_to_same_size(image, pad_value=1):
+    image = np.array(image)
+    h, w, _ = image.shape
+    image_temp = image.copy()
+    if h != w:
+        # find the max one and pad the other side with white
+        max_size = max(h, w)
+        
+        pad_h = max_size - h
+        pad_w = max_size - w
+        pad_h_top = max(pad_h // 2, 0)
+        pad_h_bottom = max(pad_h - pad_h_top, 0)
+        pad_w_left = max(pad_w // 2, 0)
+        pad_w_right = max(pad_w - pad_w_left, 0)
+        
+        image_temp = np.pad(
+            image[..., :3], 
+            ((pad_h_top, pad_h_bottom), (pad_w_left, pad_w_right), (0, 0)),
+            constant_values=pad_value
+        )
+        if image.shape[2] == 4:
+            image_bg = np.pad(
+                image[..., 3:], 
+                ((pad_h_top, pad_h_bottom), (pad_w_left, pad_w_right), (0, 0)),
+                constant_values=0
+            )
+            image = np.concatenate([image_temp, image_bg], axis=2)
+        else:
+            image = image_temp
+        
+    return Image.fromarray(image)
+
+
+def remove_bg(image):
+    image = rembg.remove(image)
+    return image
+
+
+def preprocess(image, rmbg=True):
+
+    if rmbg:
+        image = remove_bg(image)
+
+    image = crop_recenter(image)
+    image = pad_to_same_size(image, pad_value=255)
+    image = np.array(image)
+    image = image / 255.
+    if image.shape[2] == 4:
+        image = image[..., :3] * image[..., 3:] + (1 - image[..., 3:])
+    image = Image.fromarray((image * 255).astype('uint8'), "RGB")
+
+    W, H = image.size[:2]
+    pad_margin = int(W * 0.04)
+    image_transforms = T.Compose([
+        T.Pad((pad_margin, pad_margin, pad_margin, pad_margin), fill=255),
+        T.ToTensor(),
+    ])
+
+    image = image_transforms(image)
+
+    return image
+    
\ No newline at end of file

direct3d/utils/triplane.py

+import torch
+import torch.nn.functional as F
+from einops import rearrange
+
+
+def generate_planes():
+    """
+    Defines planes by the three vectors that form the "axes" of the
+    plane. Should work with arbitrary number of planes and planes of
+    arbitrary orientation.
+
+    Bugfix reference: https://github.com/NVlabs/eg3d/issues/67
+    """
+    return torch.tensor([[[1, 0, 0],
+                            [0, 1, 0],
+                            [0, 0, 1]],
+                            [[1, 0, 0],
+                            [0, 0, 1],
+                            [0, 1, 0]],
+                            [[0, 0, 1],
+                            [0, 1, 0],
+                            [1, 0, 0]]])
+
+
+def project_onto_planes(planes, coordinates):
+    """
+    Does a projection of a 3D point onto a batch of 2D planes,
+    returning 2D plane coordinates.
+
+    Takes plane axes of shape n_planes, 3, 3
+    # Takes coordinates of shape N, M, 3
+    # returns projections of shape N*n_planes, M, 2
+    """
+    N, M, C = coordinates.shape
+    n_planes, _, _ = planes.shape
+    coordinates = coordinates.unsqueeze(1).expand(-1, n_planes, -1, -1).reshape(N*n_planes, M, 3)
+    inv_planes = torch.linalg.inv(planes.to(coordinates.dtype)).unsqueeze(0).expand(N, -1, -1, -1).reshape(N*n_planes, 3, 3)
+    projections = torch.bmm(coordinates, inv_planes)
+    return projections[..., :2]
+
+
+def sample_from_planes(plane_axes, plane_features, coordinates, mode='bilinear', padding_mode='zeros', box_warp=None):
+    assert padding_mode == 'zeros'
+    N, n_planes, C, H, W = plane_features.shape
+    _, M, _ = coordinates.shape
+    plane_features = rearrange(plane_features, "N_b N_t C H_t W_t -> (N_b N_t) C H_t W_t")
+
+    coordinates = (2/box_warp) * coordinates 
+
+    projected_coordinates = project_onto_planes(plane_axes, coordinates).unsqueeze(1)
+    output_features = F.grid_sample(plane_features.float(), projected_coordinates.float(), mode=mode, padding_mode=padding_mode, align_corners=False).permute(0, 3, 2, 1).reshape(N, n_planes, M, C)
+    return output_features
\ No newline at end of file

direct3d/utils/util.py

+import importlib
+
+
+def instantiate_from_config(config):
+    if not "target" in config:
+        if config == '__is_first_stage__':
+            return None
+        elif config == "__is_unconditional__":
+            return None
+        raise KeyError("Expected key `target` to instantiate.")
+    return get_obj_from_str(config["target"])(**config.get("params", dict()))
+
+
+def get_obj_from_str(string, reload=False):
+    module, cls = string.rsplit(".", 1)
+    if reload:
+        module_imp = importlib.import_module(module)
+        importlib.reload(module_imp)
+    return getattr(importlib.import_module(module, package=None), cls)

docker/Dockerfile

+FROM image.sourcefind.cn:5000/dcu/admin/base/pytorch:2.3.0-py3.10-dtk24.04.3-ubuntu20.04
+ENV DEBIAN_FRONTEND=noninteractive
+# RUN yum update && yum install -y git cmake wget build-essential
+# RUN source /opt/dtk-24.04.3/env.sh
+# # 安装pip相关依赖
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com
+

docker/requirements.txt

+torch>=2.1.0
+scikit-image
+trimesh
+tqdm
+einops
+numpy
+transformers==4.40.2
+diffusers
+rembg
+cython
+omegaconf
+onnxruntime

docker_start.sh

+docker run -it --shm-size=64G -v $PWD/Direct3D:/home/Direct3D -v /public/DL_DATA/AI:/home/AI -v /opt/hyhal:/opt/hyhal:ro --privileged=true --device=/dev/kfd --device=/dev/dri/ --group-add video --name 3d b272aae8ec72 bash
+
+# python -m torch.utils.collect_env
+

facebook/dinov2-large/README.md

+---
+license: apache-2.0
+tags:
+- dino
+- vision
+---
+
+# Vision Transformer (large-sized model) trained using DINOv2
+
+Vision Transformer (ViT) model trained using the DINOv2 method. It was introduced in the paper [DINOv2: Learning Robust Visual Features without Supervision](https://arxiv.org/abs/2304.07193) by Oquab et al. and first released in [this repository](https://github.com/facebookresearch/dinov2). 
+
+Disclaimer: The team releasing DINOv2 did not write a model card for this model so this model card has been written by the Hugging Face team.
+
+## Model description
+
+The Vision Transformer (ViT) is a transformer encoder model (BERT-like) pretrained on a large collection of images in a self-supervised fashion. 
+
+Images are presented to the model as a sequence of fixed-size patches, which are linearly embedded. One also adds a [CLS] token to the beginning of a sequence to use it for classification tasks. One also adds absolute position embeddings before feeding the sequence to the layers of the Transformer encoder.
+
+Note that this model does not include any fine-tuned heads. 
+
+By pre-training the model, it learns an inner representation of images that can then be used to extract features useful for downstream tasks: if you have a dataset of labeled images for instance, you can train a standard classifier by placing a linear layer on top of the pre-trained encoder. One typically places a linear layer on top of the [CLS] token, as the last hidden state of this token can be seen as a representation of an entire image.
+
+## Intended uses & limitations
+
+You can use the raw model for feature extraction. See the [model hub](https://huggingface.co/models?search=facebook/dinov2) to look for
+fine-tuned versions on a task that interests you.
+
+### How to use
+
+Here is how to use this model:
+
+```python
+from transformers import AutoImageProcessor, AutoModel
+from PIL import Image
+import requests
+
+url = 'http://images.cocodataset.org/val2017/000000039769.jpg'
+image = Image.open(requests.get(url, stream=True).raw)
+
+processor = AutoImageProcessor.from_pretrained('facebook/dinov2-large')
+model = AutoModel.from_pretrained('facebook/dinov2-large')
+
+inputs = processor(images=image, return_tensors="pt")
+outputs = model(**inputs)
+last_hidden_states = outputs.last_hidden_state
+```
+
+### BibTeX entry and citation info
+
+```bibtex
+misc{oquab2023dinov2,
+      title={DINOv2: Learning Robust Visual Features without Supervision}, 
+      author={Maxime Oquab and Timothée Darcet and Théo Moutakanni and Huy Vo and Marc Szafraniec and Vasil Khalidov and Pierre Fernandez and Daniel Haziza and Francisco Massa and Alaaeldin El-Nouby and Mahmoud Assran and Nicolas Ballas and Wojciech Galuba and Russell Howes and Po-Yao Huang and Shang-Wen Li and Ishan Misra and Michael Rabbat and Vasu Sharma and Gabriel Synnaeve and Hu Xu and Hervé Jegou and Julien Mairal and Patrick Labatut and Armand Joulin and Piotr Bojanowski},
+      year={2023},
+      eprint={2304.07193},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}
+```
\ No newline at end of file

model.properties

+# 模型编码
+modelCode=1422
+# 模型名称
+modelName=Direct3D_pytorch
+# 模型描述
+modelDescription=Neural4D 2.0将3D AI的生成效果提升至“人工级水平”，算力需求暴降80%，DreamTech已开源其前期研究成果Neural4D 1.0（本项目）。
+# 应用场景
+appScenario=推理,3D生成,动漫,广媒,影视,制造,医疗,家居,教育
+# 框架类型
+frameType=pytorch

openai/clip-vit-large-patch14/README.md

+---
+tags:
+- vision
+widget:
+- src: https://huggingface.co/datasets/mishig/sample_images/resolve/main/cat-dog-music.png
+  candidate_labels: playing music, playing sports
+  example_title: Cat & Dog
+---
+
+# Model Card: CLIP
+
+Disclaimer: The model card is taken and modified from the official CLIP repository, it can be found [here](https://github.com/openai/CLIP/blob/main/model-card.md).
+
+## Model Details
+
+The CLIP model was developed by researchers at OpenAI to learn about what contributes to robustness in computer vision tasks. The model was also developed to test the ability of models to generalize to arbitrary image classification tasks in a zero-shot manner. It was not developed for general model deployment - to deploy models like CLIP, researchers will first need to carefully study their capabilities in relation to the specific context they’re being deployed within.
+
+### Model Date
+
+January 2021
+
+### Model Type
+
+The base model uses a ViT-L/14 Transformer architecture as an image encoder and uses a masked self-attention Transformer as a text encoder. These encoders are trained to maximize the similarity of (image, text) pairs via a contrastive loss.
+
+The original implementation had two variants: one using a ResNet image encoder and the other using a Vision Transformer. This repository has the variant with the Vision Transformer.
+
+
+### Documents
+
+- [Blog Post](https://openai.com/blog/clip/)
+- [CLIP Paper](https://arxiv.org/abs/2103.00020)
+
+
+### Use with Transformers
+
+```python
+from PIL import Image
+import requests
+
+from transformers import CLIPProcessor, CLIPModel
+
+model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")
+processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")
+
+url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
+
+inputs = processor(text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True)
+
+outputs = model(**inputs)
+logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+```
+
+
+## Model Use
+
+### Intended Use
+
+The model is intended as a research output for research communities. We hope that this model will enable researchers to better understand and explore zero-shot, arbitrary image classification. We also hope it can be used for interdisciplinary studies of the potential impact of such models - the CLIP paper includes a discussion of potential downstream impacts to provide an example for this sort of analysis.
+
+#### Primary intended uses
+
+The primary intended users of these models are AI researchers.
+
+We primarily imagine the model will be used by researchers to better understand robustness, generalization, and other capabilities, biases, and constraints of computer vision models.
+
+### Out-of-Scope Use Cases
+
+**Any** deployed use case of the model - whether commercial or not - is currently out of scope. Non-deployed use cases such as image search in a constrained environment, are also not recommended unless there is thorough in-domain testing of the model with a specific, fixed class taxonomy. This is because our safety assessment demonstrated a high need for task specific testing especially given the variability of CLIP’s performance with different class taxonomies. This makes untested and unconstrained deployment of the model in any use case currently potentially harmful. 
+
+Certain use cases which would fall under the domain of surveillance and facial recognition are always out-of-scope regardless of performance of the model. This is because the use of artificial intelligence for tasks such as these can be premature currently given the lack of testing norms and checks to ensure its fair use.
+
+Since the model has not been purposefully trained in or evaluated on any languages other than English, its use should be limited to English language use cases.
+
+
+
+## Data
+
+The model was trained on publicly available image-caption data. This was done through a combination of crawling a handful of websites and using commonly-used pre-existing image datasets such as [YFCC100M](http://projects.dfki.uni-kl.de/yfcc100m/). A large portion of the data comes from our crawling of the internet. This means that the data is more representative of people and societies most connected to the internet which tend to skew towards more developed nations, and younger, male users.
+
+### Data Mission Statement
+
+Our goal with building this dataset was to test out robustness and generalizability in computer vision tasks. As a result, the focus was on gathering large quantities of data from different publicly-available internet data sources. The data was gathered in a mostly non-interventionist manner. However, we only crawled websites that had policies against excessively violent and adult images and allowed us to filter out such content. We do not intend for this dataset to be used as the basis for any commercial or deployed model and will not be releasing the dataset.
+
+
+
+## Performance and Limitations
+
+### Performance
+
+We have evaluated the performance of CLIP on a wide range of benchmarks across a variety of computer vision datasets such as OCR to texture recognition to fine-grained classification. The paper describes model performance on the following datasets:
+
+- Food101
+- CIFAR10   
+- CIFAR100   
+- Birdsnap
+- SUN397
+- Stanford Cars
+- FGVC Aircraft
+- VOC2007
+- DTD
+- Oxford-IIIT Pet dataset
+- Caltech101
+- Flowers102
+- MNIST   
+- SVHN 
+- IIIT5K   
+- Hateful Memes   
+- SST-2
+- UCF101
+- Kinetics700
+- Country211
+- CLEVR Counting
+- KITTI Distance
+- STL-10
+- RareAct
+- Flickr30
+- MSCOCO
+- ImageNet
+- ImageNet-A
+- ImageNet-R
+- ImageNet Sketch
+- ObjectNet (ImageNet Overlap)
+- Youtube-BB
+- ImageNet-Vid
+
+## Limitations
+
+CLIP and our analysis of it have a number of limitations. CLIP currently struggles with respect to certain tasks such as fine grained classification and counting objects. CLIP also poses issues with regards to fairness and bias which we discuss in the paper and briefly in the next section. Additionally, our approach to testing CLIP also has an important limitation- in many cases we have used linear probes to evaluate the performance of CLIP and there is evidence suggesting that linear probes can underestimate model performance.
+
+### Bias and Fairness
+
+We find that the performance of CLIP - and the specific biases it exhibits - can depend significantly on class design and the choices one makes for categories to include and exclude. We tested the risk of certain kinds of denigration with CLIP by classifying images of people from [Fairface](https://arxiv.org/abs/1908.04913) into crime-related and non-human animal categories. We found significant disparities with respect to race and gender. Additionally, we found that these disparities could shift based on how the classes were constructed. (Details captured in the Broader Impacts Section in the paper).
+
+We also tested the performance of CLIP on gender, race and age classification using the Fairface dataset (We default to using race categories as they are constructed in the Fairface dataset.) in order to assess quality of performance across different demographics. We found accuracy >96% across all races for gender classification with ‘Middle Eastern’ having the highest accuracy (98.4%) and ‘White’ having the lowest (96.5%). Additionally, CLIP averaged ~93% for racial classification and ~63% for age classification. Our use of evaluations to test for gender, race and age classification as well as denigration harms is simply to evaluate performance of the model across people and surface potential risks and not to demonstrate an endorsement/enthusiasm for such tasks.
+
+
+
+## Feedback
+
+### Where to send questions or comments about the model
+
+Please use [this Google Form](https://forms.gle/Uv7afRH5dvY34ZEs9)
\ No newline at end of file

requirements.txt

+torch>=2.1.0
+scikit-image
+trimesh
+tqdm
+einops
+numpy
+transformers==4.40.2
+diffusers
+rembg
+cython
+omegaconf
+onnxruntime

setup.py

+from setuptools import setup, find_packages
+
+
+setup(
+    name="direct3d",
+    version="1.0.0",
+    description="Direct3D: Scalable Image-to-3D Generation via 3D Latent Diffusion Transformer",
+    packages=find_packages(),
+    python_requires=">=3.10",
+    install_requires=[
+        "torch",
+        "numpy",
+        "cython",
+        "trimesh",
+        "diffusers",
+    ],
+)
\ No newline at end of file

test.py

+from direct3d.pipeline import Direct3dPipeline
+pipeline = Direct3dPipeline.from_pretrained("DreamTechAI/Direct3D")
+pipeline.to("cuda")
+mesh = pipeline(
+    "assets/devil.png",
+    remove_background=False, # set to True if the background of the image needs to be removed
+    mc_threshold=-1.0,
+    guidance_scale=4.0,
+    num_inference_steps=50,
+)["meshes"][0]
+mesh.export("output.obj")