##!/usr/bin/env bash
#
#CONFIG=$1
#CHECKPOINT=$2
#GPUS=$3
#PORT=${PORT:-29503}
#
#PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
#python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \
#    $(dirname "$0")/test.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} --eval bbox


#!/usr/bin/env bash

CONFIG=$1
CHECKPOINT=$2
GPUS=$3
NNODES=${NNODES:-1}
NODE_RANK=${NODE_RANK:-0}
PORT=${PORT:-29500}
MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"}

PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
torchrun \
    --nnodes=$NNODES \
    --nproc_per_node=$GPUS \
    --node_rank=$NODE_RANK \
    --master_addr=$MASTER_ADDR \
    --master_port=$PORT \
    --no-python \
    bash -c '
        # 获取 NUMA 拓扑映射
        numa_map=( $(hy-smi --showtopo | grep "Numa Node" | awk "{print \$6}") )

        LOCAL_RANK=${LOCAL_RANK:-0}
        NUMA_ID=${numa_map[$LOCAL_RANK]}

        echo "[Rank $LOCAL_RANK] Bind to NUMA node $NUMA_ID"

        numactl --cpunodebind=${NUMA_ID} --membind=${NUMA_ID} \
            python '"$(dirname "$0")"'/test.py \
            "$@"
    ' _ \
    $CONFIG \
    $CHECKPOINT \
    --launcher pytorch \
    ${@:4}
