feat: 初始提交

.gitignore

+.idea
+chenyh

Dockerfile

+FROM image.sourcefind.cn:5000/dcu/admin/base/jupyterlab-pytorch:2.3.0-ubuntu22.04-dtk24.04.2-py3.10-devel as base
+ARG IMAGE=janusflow-1.3b
+ARG IMAGE_UPPER=JanusFlow-1.3B
+ARG BRANCH=dcu
+RUN cd /root && git clone -b $BRANCH http://developer.hpccube.com/codes/chenpangpang/$IMAGE.git
+WORKDIR /root/$IMAGE/$IMAGE_UPPER
+RUN pip install -r requirements.txt
+
+#########
+# Prod  #
+#########
+FROM image.sourcefind.cn:5000/dcu/admin/base/jupyterlab-pytorch:2.3.0-ubuntu22.04-dtk24.04.2-py3.10-devel
+ARG IMAGE=janusflow-1.3b
+ARG IMAGE_UPPER=JanusFlow-1.3B
+COPY chenyh/$IMAGE/frpc_linux_amd64_v* /opt/conda/lib/python3.10/site-packages/gradio/
+RUN chmod +x /opt/conda/lib/python3.10/site-packages/gradio/frpc_linux_amd64_v*
+COPY chenyh/$IMAGE/ /root/$IMAGE_UPPER/
+COPY --from=base /opt/conda/lib/python3.10/site-packages /opt/conda/lib/python3.10/site-packages
+COPY --from=base /root/$IMAGE/$IMAGE_UPPER /root/$IMAGE_UPPER
+COPY --from=base /root/$IMAGE/启动器.ipynb /root/$IMAGE/start.sh  /root/
+COPY --from=base /root/$IMAGE/assets/  /root/assets/

JanusFlow-1.3B/README.md

+---
+title: JanusFlow 1.3B
+emoji: 🏃
+colorFrom: yellow
+colorTo: red
+sdk: gradio
+sdk_version: 5.5.0
+app_file: app.py
+pinned: false
+license: mit
+short_description: Huggingface space for JanusFlow-1.3B
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

JanusFlow-1.3B/app.py

+import gradio as gr
+import torch
+from janus.janusflow.models import MultiModalityCausalLM, VLChatProcessor
+from PIL import Image
+from diffusers.models import AutoencoderKL
+import numpy as np
+
+cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+# Load model and processor
+model_path = "deepseek-ai/JanusFlow-1.3B"
+vl_chat_processor = VLChatProcessor.from_pretrained(model_path)
+tokenizer = vl_chat_processor.tokenizer
+
+vl_gpt = MultiModalityCausalLM.from_pretrained(model_path)
+vl_gpt = vl_gpt.to(torch.bfloat16).to(cuda_device).eval()
+
+# remember to use bfloat16 dtype, this vae doesn't work with fp16
+vae = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae")
+vae = vae.to(torch.bfloat16).to(cuda_device).eval()
+
+
+# Multimodal Understanding function
+@torch.inference_mode()
+# Multimodal Understanding function
+def multimodal_understanding(image, question, seed, top_p, temperature):
+    # Clear CUDA cache before generating
+    torch.cuda.empty_cache()
+
+    # set seed
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+    torch.cuda.manual_seed(seed)
+
+    conversation = [
+        {
+            "role": "User",
+            "content": f"<image_placeholder>\n{question}",
+            "images": [image],
+        },
+        {"role": "Assistant", "content": ""},
+    ]
+
+    pil_images = [Image.fromarray(image)]
+    prepare_inputs = vl_chat_processor(
+        conversations=conversation, images=pil_images, force_batchify=True
+    ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16)
+
+    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
+
+    outputs = vl_gpt.language_model.generate(
+        inputs_embeds=inputs_embeds,
+        attention_mask=prepare_inputs.attention_mask,
+        pad_token_id=tokenizer.eos_token_id,
+        bos_token_id=tokenizer.bos_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        max_new_tokens=512,
+        do_sample=False if temperature == 0 else True,
+        use_cache=True,
+        temperature=temperature,
+        top_p=top_p,
+    )
+
+    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
+
+    return answer
+
+
+@torch.inference_mode()
+def generate(
+        input_ids,
+        cfg_weight: float = 2.0,
+        num_inference_steps: int = 30
+):
+    # we generate 5 images at a time, *2 for CFG
+    tokens = torch.stack([input_ids] * 10).cuda()
+    tokens[5:, 1:] = vl_chat_processor.pad_id
+    inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens)
+    print(inputs_embeds.shape)
+
+    # we remove the last <bog> token and replace it with t_emb later
+    inputs_embeds = inputs_embeds[:, :-1, :]
+
+    # generate with rectified flow ode
+    # step 1: encode with vision_gen_enc
+    z = torch.randn((5, 4, 48, 48), dtype=torch.bfloat16).cuda()
+
+    dt = 1.0 / num_inference_steps
+    dt = torch.zeros_like(z).cuda().to(torch.bfloat16) + dt
+
+    # step 2: run ode
+    attention_mask = torch.ones((10, inputs_embeds.shape[1] + 577)).to(vl_gpt.device)
+    attention_mask[5:, 1:inputs_embeds.shape[1]] = 0
+    attention_mask = attention_mask.int()
+    for step in range(num_inference_steps):
+        # prepare inputs for the llm
+        z_input = torch.cat([z, z], dim=0)  # for cfg
+        t = step / num_inference_steps * 1000.
+        t = torch.tensor([t] * z_input.shape[0]).to(dt)
+        z_enc = vl_gpt.vision_gen_enc_model(z_input, t)
+        z_emb, t_emb, hs = z_enc[0], z_enc[1], z_enc[2]
+        z_emb = z_emb.view(z_emb.shape[0], z_emb.shape[1], -1).permute(0, 2, 1)
+        z_emb = vl_gpt.vision_gen_enc_aligner(z_emb)
+        llm_emb = torch.cat([inputs_embeds, t_emb.unsqueeze(1), z_emb], dim=1)
+
+        # input to the llm
+        # we apply attention mask for CFG: 1 for tokens that are not masked, 0 for tokens that are masked.
+        if step == 0:
+            outputs = vl_gpt.language_model.model(inputs_embeds=llm_emb,
+                                                  use_cache=True,
+                                                  attention_mask=attention_mask,
+                                                  past_key_values=None)
+            past_key_values = []
+            for kv_cache in past_key_values:
+                k, v = kv_cache[0], kv_cache[1]
+                past_key_values.append((k[:, :, :inputs_embeds.shape[1], :], v[:, :, :inputs_embeds.shape[1], :]))
+            past_key_values = tuple(past_key_values)
+        else:
+            outputs = vl_gpt.language_model.model(inputs_embeds=llm_emb,
+                                                  use_cache=True,
+                                                  attention_mask=attention_mask,
+                                                  past_key_values=past_key_values)
+        hidden_states = outputs.last_hidden_state
+
+        # transform hidden_states back to v
+        hidden_states = vl_gpt.vision_gen_dec_aligner(vl_gpt.vision_gen_dec_aligner_norm(hidden_states[:, -576:, :]))
+        hidden_states = hidden_states.reshape(z_emb.shape[0], 24, 24, 768).permute(0, 3, 1, 2)
+        v = vl_gpt.vision_gen_dec_model(hidden_states, hs, t_emb)
+        v_cond, v_uncond = torch.chunk(v, 2)
+        v = cfg_weight * v_cond - (cfg_weight - 1.) * v_uncond
+        z = z + dt * v
+
+    # step 3: decode with vision_gen_dec and sdxl vae
+    decoded_image = vae.decode(z / vae.config.scaling_factor).sample
+
+    images = decoded_image.float().clip_(-1., 1.).permute(0, 2, 3, 1).cpu().numpy()
+    images = ((images + 1) / 2. * 255).astype(np.uint8)
+
+    return images
+
+
+def unpack(dec, width, height, parallel_size=5):
+    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
+    dec = np.clip((dec + 1) / 2 * 255, 0, 255)
+
+    visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
+    visual_img[:, :, :] = dec
+
+    return visual_img
+
+
+@torch.inference_mode()
+def generate_image(prompt,
+                   seed=None,
+                   guidance=5,
+                   num_inference_steps=30):
+    # Clear CUDA cache and avoid tracking gradients
+    torch.cuda.empty_cache()
+    # Set the seed for reproducible results
+    if seed is not None:
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed(seed)
+        np.random.seed(seed)
+
+    with torch.no_grad():
+        messages = [{'role': 'User', 'content': prompt},
+                    {'role': 'Assistant', 'content': ''}]
+        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages,
+                                                                           sft_format=vl_chat_processor.sft_format,
+                                                                           system_prompt='')
+        text = text + vl_chat_processor.image_start_tag
+        input_ids = torch.LongTensor(tokenizer.encode(text))
+        images = generate(input_ids,
+                          cfg_weight=guidance,
+                          num_inference_steps=num_inference_steps)
+        return [Image.fromarray(images[i]).resize((1024, 1024), Image.LANCZOS) for i in range(images.shape[0])]
+
+
+# Gradio interface
+with gr.Blocks() as demo:
+    gr.Markdown(value="# Multimodal Understanding")
+    # with gr.Row():
+    with gr.Row():
+        image_input = gr.Image()
+        with gr.Column():
+            question_input = gr.Textbox(label="Question")
+            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
+            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
+            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
+
+    understanding_button = gr.Button("Chat")
+    understanding_output = gr.Textbox(label="Response")
+
+    examples_inpainting = gr.Examples(
+        label="Multimodal Understanding examples",
+        examples=[
+            [
+                "explain this meme",
+                "./doge.png",
+            ],
+            [
+                "Convert the formula into latex code.",
+                "./equation.png",
+            ],
+        ],
+        inputs=[question_input, image_input],
+    )
+
+    gr.Markdown(value="# Text-to-Image Generation")
+
+    with gr.Row():
+        cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=2, step=0.5, label="CFG Weight")
+        step_input = gr.Slider(minimum=1, maximum=50, value=30, step=1, label="Number of Inference Steps")
+
+    prompt_input = gr.Textbox(label="Prompt")
+    seed_input = gr.Number(label="Seed (Optional)", precision=0, value=12345)
+
+    generation_button = gr.Button("Generate Images")
+
+    image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
+
+    examples_t2i = gr.Examples(
+        label="Text to image generation examples.",
+        examples=[
+            "Master shifu racoon wearing drip attire as a street gangster.",
+            "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.",
+            "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.",
+        ],
+        inputs=prompt_input,
+    )
+
+    understanding_button.click(
+        multimodal_understanding,
+        inputs=[image_input, question_input, und_seed_input, top_p, temperature],
+        outputs=understanding_output
+    )
+
+    generation_button.click(
+        fn=generate_image,
+        inputs=[prompt_input, seed_input, cfg_weight_input, step_input],
+        outputs=image_output
+    )
+
+demo.launch(share=True, ssr_mode=False, server_name="0.0.0.0")

JanusFlow-1.3B/requirements.txt

+accelerate
+diffusers
+gradio
+git+https://github.com/deepseek-ai/Janus
\ No newline at end of file

hf_down.py

+# pip install huggingface-cli
+import os
+import requests
+import json
+
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+model_list = [
+        "deepseek-ai/JanusFlow-1.3B",
+        "stabilityai/sdxl-vae"
+]
+
+for model_path in model_list:
+    os.system(f"huggingface-cli download --resume-download  {model_path} --local-dir ./{model_path} --local-dir-use-symlinks False")

start.sh

+#!/bin/bash
+
+cd /root/PhotoMaker-V2
+python app.py

启动器.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e5c5a211-2ccd-4341-af10-ac546484b91f",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## 项目介绍\n",
+    "- 原项目地址：https://huggingface.co/spaces/deepseek-ai/JanusFlow-1.3B\n",
+    "- JanusFlow-1.3B：一款聊天机器人，支持多模态理解和文生图\n",
+    "## 运行资源\n",
+    "- 项目在异构加速卡AI，dtk24.04.2上适配。\n",
+    "## 使用说明\n",
+    "- 启动和重启 Notebook 点上方工具栏中的「重启并运行所有单元格」。出现如下内容就算成功了：\n",
+    "    - `Running on local URL:  http://0.0.0.0:7860`\n",
+    "    - `Running on public URL: https://xxxxxxxxxxxxxxx.gradio.live`\n",
+    "- 通过以下方式开启页面：\n",
+    "    - 控制台打开「自定义服务」了，访问自定义服务端口号设置为7860\n",
+    "    - 直接打开显示的公开链接`public URL`\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53a96614-e2d2-4710-a82b-0d5ca9cb9872",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# 启动\n",
+    "!sh start.sh"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "---\n",
+    "**扫码关注公众号，获取更多资讯**<br>\n",
+    "<div align=center>\n",
+    "<img src=\"assets/二维码.jpeg\" width = 20% />\n",
+    "</div>\n"
+   ],
+   "metadata": {
+    "collapsed": false
+   },
+   "id": "2f54158c2967bc25"
+  },
+  {
+   "cell_type": "code",
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   },
+   "id": "6dc59fbbcf222b6b",
+   "execution_count": null
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}