update workflow (#189)

* update workflow

* tiny fix

* format

* format

.github/workflows/cuda/cu113-Windows-env.sh

-#!/bin/bash
-
-CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
-PATH=${CUDA_HOME}/bin:$PATH
-PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="6.0+PTX"

.github/workflows/cuda/cu113-Windows.sh

-#!/bin/bash
-
-# Install NVIDIA drivers, see:
-# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
-curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
-7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
-
-export CUDA_SHORT=11.3
-export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
-export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
-
-# Install CUDA:
-curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
-echo ""
-echo "Installing from ${CUDA_FILE}..."
-PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
-echo "Done!"
-rm -f "${CUDA_FILE}"

.github/workflows/cuda/cu115-Linux-env.sh

-#!/bin/bash
-
-CUDA_HOME=/usr/local/cuda-11.5
-LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
-PATH=${CUDA_HOME}/bin:${PATH}
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"

.github/workflows/cuda/cu115-Linux.sh

-#!/bin/bash
-
-OS=ubuntu1804
-
-wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
-sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
-wget -nv https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda-repo-${OS}-11-5-local_11.5.2-495.29.05-1_amd64.deb
-sudo dpkg -i cuda-repo-${OS}-11-5-local_11.5.2-495.29.05-1_amd64.deb
-sudo apt-key add /var/cuda-repo-${OS}-11-5-local/7fa2af80.pub
-
-sudo apt-get -qq update
-sudo apt install -y cuda-nvcc-11-5 cuda-libraries-dev-11-5
-sudo apt clean
-
-rm -f https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda-repo-${OS}-11-5-local_11.5.2-495.29.05-1_amd64.deb

.github/workflows/cuda/cu115-Windows-env.sh

-#!/bin/bash
-
-CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
-PATH=${CUDA_HOME}/bin:$PATH
-PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="6.0+PTX"

.github/workflows/cuda/cu116-Linux-env.sh

-#!/bin/bash
-
-CUDA_HOME=/usr/local/cuda-11.6
-LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
-PATH=${CUDA_HOME}/bin:${PATH}
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"

.github/workflows/cuda/cu116-Linux.sh

-#!/bin/bash
-
-OS=ubuntu1804
-
-wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
-sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
-wget -nv https://developer.download.nvidia.com/compute/cuda/11.6.2/local_installers/cuda-repo-${OS}-11-6-local_11.6.2-510.47.03-1_amd64.deb
-sudo dpkg -i cuda-repo-${OS}-11-6-local_11.6.2-510.47.03-1_amd64.deb
-sudo apt-key add /var/cuda-repo-${OS}-11-6-local/7fa2af80.pub
-
-sudo apt-get -qq update
-sudo apt install -y cuda-nvcc-11-6 cuda-libraries-dev-11-6
-sudo apt clean
-
-rm -f https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda-repo-${OS}-11-6-local_11.6.2-510.47.03-1_amd64.deb

.github/workflows/cuda/cu116-Windows-env.sh

-#!/bin/bash
-
-CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
-PATH=${CUDA_HOME}/bin:$PATH
-PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="6.0+PTX"

.github/workflows/cuda/cu116-Windows.sh

-#!/bin/bash
-
-# TODO We currently use CUDA 11.3 to build CUDA 11.6 Windows wheels
-
-# Install NVIDIA drivers, see:
-# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
-curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
-7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
-
-export CUDA_SHORT=11.3
-export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
-export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
-
-# Install CUDA:
-curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
-echo ""
-echo "Installing from ${CUDA_FILE}..."
-PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
-echo "Done!"
-rm -f "${CUDA_FILE}"

.github/workflows/cuda/cu117-Linux-env.sh

-#!/bin/bash
-
-CUDA_HOME=/usr/local/cuda-11.7
-LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
-PATH=${CUDA_HOME}/bin:${PATH}
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"

.github/workflows/cuda/cu117-Linux.sh

-#!/bin/bash
-
-OS=ubuntu1804
-
-wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
-sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
-wget -nv https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda-repo-${OS}-11-7-local_11.7.1-515.65.01-1_amd64.deb
-sudo dpkg -i cuda-repo-${OS}-11-7-local_11.7.1-515.65.01-1_amd64.deb
-sudo cp /var/cuda-repo-${OS}-11-7-local/cuda-*-keyring.gpg /usr/share/keyrings/
-
-sudo apt-get -qq update
-sudo apt install -y cuda-nvcc-11-7 cuda-libraries-dev-11-7
-sudo apt clean
-
-rm -f https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda-repo-${OS}-11-7-local_11.7.1-515.65.01-1_amd64.deb

.github/workflows/cuda/cu117-Windows-env.sh

-#!/bin/bash
-
-CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
-PATH=${CUDA_HOME}/bin:$PATH
-PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
-
-export FORCE_CUDA=1
-export TORCH_CUDA_ARCH_LIST="6.0+PTX"

.github/workflows/cuda/cu117-Windows.sh

-#!/bin/bash
-
-# TODO We currently use CUDA 11.3 to build CUDA 11.7 Windows wheels
-
-# Install NVIDIA drivers, see:
-# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
-curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
-7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
-
-export CUDA_SHORT=11.3
-export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
-export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
-
-# Install CUDA:
-curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
-echo ""
-echo "Installing from ${CUDA_FILE}..."
-PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
-echo "Done!"
-rm -f "${CUDA_FILE}"

scripts/run_aws_listing.py

 import argparse
-import os
+from collections import defaultdict
 
 from boto3 import client
 
@@ -10,7 +10,14 @@ parser.add_argument("--bucket", type=str, required=True)
 parser.add_argument("--region", type=str, required=True)
 args = parser.parse_args()
 
-URL = f"https://{args.bucket}.s3.{args.region}.amazonaws.com/"
+ROOT_URL = f"https://{args.bucket}.s3.{args.region}.amazonaws.com/whl"
+html = "<!DOCTYPE html>\n<html>\n<body>\n{}\n</body>\n</html>"
+href = '  <a href="{}">{}</a><br/>'
+args = {
+    "ContentType": "text/html",
+    "CacheControl": "max-age=300",
+    "ACL": "public-read",
+}
 
 s3 = client(
     "s3",
@@ -18,37 +25,43 @@ s3 = client(
     aws_secret_access_key=args.secret_access_key,
 )
 
-responses = s3.list_objects_v2(Bucket=args.bucket, Prefix="whl/")["Contents"]
+bucket = s3.Bucket(name="nerfacc-bucket")
 
-subdirectories = {}
-for data in responses:
-    splits = data["Key"].split("/")
-    if len(splits) == 3:
-        subdirectories[splits[1]] = []
+wheels_dict = defaultdict(list)
+for obj in bucket.objects.filter(Prefix="whl"):
+    if obj.key[-3:] != "whl":
+        continue
+    torch_version, wheel = obj.key.split("/")[-2:]
+    wheel = f"{torch_version}/{wheel}"
+    wheels_dict[torch_version].append(wheel)
 
-for dir in subdirectories.keys():
-    responses = s3.list_objects_v2(Bucket=args.bucket, Prefix=f"whl/{dir}")[
-        "Contents"
+index_html = html.format(
+    "\n".join(
+        [
+            href.format(f"{torch_version}.html".replace("+", "%2B"), version)
+            for version in wheels_dict
         ]
-    for data in responses:
-        splits = data["Key"].split("/")
-        if len(splits) == 3:
-            subdirectories[dir].append(splits[2])
-
-for dir, files in subdirectories.items():
-    lines = ""
-    for file in files:
-        href = os.path.join(URL, "whl", dir, file)
-        lines += f"<a href='{href}'>{file}</a>\n<br>\n"
-
-    html = f"<html>\n<head></head>\n<body>\n{lines}\n</body>\n</html>\n"
-    html_file = f"/tmp/{dir}.html"
-    with open(html_file, "w") as f:
-        f.write(html)
-
-    s3.upload_file(
-        html_file,
-        args.bucket,
-        f"whl/{dir}.html",
-        ExtraArgs={"ContentType": "text/html"},
+    )
+)
+
+with open("index.html", "w") as f:
+    f.write(index_html)
+bucket.Object("whl/index.html").upload_file("index.html", args)
+
+for torch_version, wheel_names in wheels_dict.items():
+    torch_version_html = html.format(
+        "\n".join(
+            [
+                href.format(
+                    f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"), wheel_name
+                )
+                for wheel_name in wheel_names
+            ]
+        )
+    )
+
+    with open(f"{torch_version}.html", "w") as f:
+        f.write(torch_version_html)
+    bucket.Object(f"whl/{torch_version}.html").upload_file(
+        f"{torch_version}.html", args
     )