Move requirements into their own directory (#12547)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

requirements-neuron.txt → requirements/neuron.txt

 # Common dependencies
--r requirements-common.txt
+-r common.txt
 
 # Dependencies for Neuron devices
 torch-neuronx >= 2.5.0

requirements-openvino.txt → requirements/openvino.txt

 # Common dependencies
--r requirements-common.txt
+-r common.txt
 
 torch == 2.5.1 #  should be aligned with "common" vLLM torch version
 openvino >= 2024.4.0 # since 2024.4.0 both CPU and GPU support Paged Attention

requirements-rocm-build.txt → requirements/rocm-build.txt

 # Common dependencies
--r requirements-common.txt
+-r common.txt
 
 --extra-index-url https://download.pytorch.org/whl/rocm6.2
 torch==2.5.1

requirements-rocm.txt → requirements/rocm.txt

 # Common dependencies
--r requirements-common.txt
+-r common.txt
 
 numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
 

requirements-test.txt → requirements/test.txt

 # This file was autogenerated by uv via the following command:
-#    uv pip compile requirements-test.in -o requirements-test.txt
+#    uv pip compile requirements/test.in -o requirements/test.txt
 absl-py==2.1.0
     # via rouge-score
 accelerate==1.0.1
@@ -32,9 +32,9 @@ attrs==24.2.0
 audioread==3.0.1
     # via librosa
 awscli==1.35.23
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 bitsandbytes==0.45.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 black==24.10.0
     # via datamodel-code-generator
 boto3==1.35.57
@@ -47,7 +47,7 @@ botocore==1.35.57
 bounded-pool-executor==0.0.3
     # via pqdm
 buildkite-test-collector==0.1.9
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 certifi==2024.8.30
     # via
     #   httpcore
@@ -79,7 +79,7 @@ cupy-cuda12x==13.3.0
 cycler==0.12.1
     # via matplotlib
 datamodel-code-generator==0.26.3
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 dataproperty==1.0.1
     # via
     #   pytablewriter
@@ -91,7 +91,7 @@ datasets==3.0.2
 decorator==5.1.1
     # via librosa
 decord==0.6.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 dill==0.3.8
     # via
     #   datasets
@@ -104,7 +104,7 @@ docutils==0.16
     # via awscli
 einops==0.8.0
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   encodec
     #   vector-quantize-pytorch
     #   vocos
@@ -145,7 +145,7 @@ fsspec==2024.9.0
     #   huggingface-hub
     #   torch
 genai-perf==0.0.8
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 genson==1.3.0
     # via datamodel-code-generator
 h11==0.14.0
@@ -155,7 +155,7 @@ hiredis==3.0.0
 httpcore==1.0.6
     # via httpx
 httpx==0.27.2
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 huggingface-hub==0.26.2
     # via
     #   accelerate
@@ -187,7 +187,7 @@ jinja2==3.1.4
     #   datamodel-code-generator
     #   torch
 jiwer==3.0.5
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 jmespath==1.0.1
     # via
     #   boto3
@@ -214,11 +214,11 @@ lazy-loader==0.4
 libnacl==2.1.0
     # via tensorizer
 librosa==0.10.2.post1
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 llvmlite==0.43.0
     # via numba
 lm-eval==0.4.4
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 lxml==5.3.0
     # via sacrebleu
 markdown-it-py==3.0.0
@@ -226,7 +226,7 @@ markdown-it-py==3.0.0
 markupsafe==3.0.2
     # via jinja2
 matplotlib==3.9.2
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 mbstrdecoder==1.1.3
     # via
     #   dataproperty
@@ -235,7 +235,7 @@ mbstrdecoder==1.1.3
 mdurl==0.1.2
     # via markdown-it-py
 mistral-common==1.5.1
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 more-itertools==10.5.0
     # via lm-eval
 mpmath==1.3.0
@@ -264,7 +264,7 @@ numexpr==2.10.1
     # via lm-eval
 numpy==1.26.4
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   accelerate
     #   bitsandbytes
     #   contourpy
@@ -366,7 +366,7 @@ patsy==1.0.1
     # via statsmodels
 peft==0.13.2
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   lm-eval
 pillow==10.4.0
     # via
@@ -388,7 +388,7 @@ pooch==1.8.2
 portalocker==2.10.1
     # via sacrebleu
 pqdm==0.2.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 propcache==0.2.0
     # via yarl
 protobuf==5.28.3
@@ -426,7 +426,7 @@ pytablewriter==1.2.0
     # via lm-eval
 pytest==8.3.3
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   buildkite-test-collector
     #   genai-perf
     #   pytest-asyncio
@@ -435,15 +435,15 @@ pytest==8.3.3
     #   pytest-rerunfailures
     #   pytest-shard
 pytest-asyncio==0.24.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 pytest-forked==1.6.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 pytest-mock==3.14.0
     # via genai-perf
 pytest-rerunfailures==14.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 pytest-shard==0.1.2
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 python-dateutil==2.9.0.post0
     # via
     #   botocore
@@ -473,7 +473,7 @@ pyyaml==6.0.2
 rapidfuzz==3.12.1
     # via jiwer
 ray==2.43.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 redis==5.2.0
     # via tensorizer
 referencing==0.35.1
@@ -512,9 +512,9 @@ rpds-py==0.20.1
 rsa==4.7.2
     # via awscli
 runai-model-streamer==0.11.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 runai-model-streamer-s3==0.11.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 s3transfer==0.10.3
     # via
     #   awscli
@@ -540,7 +540,7 @@ scipy==1.13.1
     #   statsmodels
     #   vocos
 sentence-transformers==3.2.1
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 sentencepiece==0.2.0
     # via mistral-common
 setuptools==75.8.0
@@ -557,7 +557,7 @@ sniffio==1.3.1
     #   httpx
 soundfile==0.12.1
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   librosa
 soxr==0.5.0.post1
     # via librosa
@@ -580,7 +580,7 @@ tenacity==9.0.0
     #   lm-eval
     #   plotly
 tensorizer==2.9.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 threadpoolctl==3.5.0
     # via scikit-learn
 tiktoken==0.7.0
@@ -588,12 +588,12 @@ tiktoken==0.7.0
     #   lm-eval
     #   mistral-common
 timm==1.0.11
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 tokenizers==0.21.0
     # via transformers
 torch==2.5.1
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   accelerate
     #   bitsandbytes
     #   encodec
@@ -609,7 +609,7 @@ torch==2.5.1
     #   vocos
 torchaudio==2.5.1
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   encodec
     #   vocos
 torchvision==0.20.1
@@ -630,19 +630,19 @@ tqdm-multiprocess==0.0.11
     # via lm-eval
 transformers==4.48.2
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   genai-perf
     #   lm-eval
     #   peft
     #   sentence-transformers
     #   transformers-stream-generator
 transformers-stream-generator==0.0.5
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 triton==3.1.0
     # via torch
 tritonclient==2.51.0
     # via
-    #   -r requirements-test.in
+    #   -r requirements/test.in
     #   genai-perf
 typepy==1.3.2
     # via
@@ -668,9 +668,9 @@ urllib3==2.2.3
     #   responses
     #   tritonclient
 vector-quantize-pytorch==1.21.2
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 vocos==0.1.0
-    # via -r requirements-test.in
+    # via -r requirements/test.in
 word2number==1.1
     # via lm-eval
 xxhash==3.5.0

requirements-tpu.txt → requirements/tpu.txt

 # Common dependencies
--r requirements-common.txt
+-r common.txt
 
 # Dependencies for TPU
 cmake>=3.26

requirements-xpu.txt → requirements/xpu.txt

 # Common dependencies
--r requirements-common.txt
+-r common.txt
 
 ray >= 2.9
 cmake>=3.26

setup.py

@@ -27,7 +27,7 @@ def load_module_from_path(module_name, path):
     return module
 
 
-ROOT_DIR = os.path.dirname(__file__)
+ROOT_DIR = Path(__file__).parent
 logger = logging.getLogger(__name__)
 
 # cannot import envs directly because it depends on vllm,
@@ -520,10 +520,6 @@ def get_nvcc_cuda_version() -> Version:
     return nvcc_cuda_version
 
 
-def get_path(*filepath) -> str:
-    return os.path.join(ROOT_DIR, *filepath)
-
-
 def get_gaudi_sw_version():
     """
     Returns the driver version.
@@ -591,9 +587,10 @@ def get_vllm_version() -> str:
 
 def get_requirements() -> list[str]:
     """Get Python package dependencies from requirements.txt."""
+    requirements_dir = ROOT_DIR / "requirements"
 
     def _read_requirements(filename: str) -> list[str]:
-        with open(get_path(filename)) as f:
+        with open(requirements_dir / filename) as f:
             requirements = f.read().strip().split("\n")
         resolved_requirements = []
         for line in requirements:
@@ -606,9 +603,9 @@ def get_requirements() -> list[str]:
         return resolved_requirements
 
     if _no_device():
-        requirements = _read_requirements("requirements-common.txt")
+        requirements = _read_requirements("common.txt")
     elif _is_cuda():
-        requirements = _read_requirements("requirements-cuda.txt")
+        requirements = _read_requirements("cuda.txt")
         cuda_major, cuda_minor = torch.version.cuda.split(".")
         modified_requirements = []
         for req in requirements:
@@ -619,19 +616,19 @@ def get_requirements() -> list[str]:
             modified_requirements.append(req)
         requirements = modified_requirements
     elif _is_hip():
-        requirements = _read_requirements("requirements-rocm.txt")
+        requirements = _read_requirements("rocm.txt")
     elif _is_neuron():
-        requirements = _read_requirements("requirements-neuron.txt")
+        requirements = _read_requirements("neuron.txt")
     elif _is_hpu():
-        requirements = _read_requirements("requirements-hpu.txt")
+        requirements = _read_requirements("hpu.txt")
     elif _is_openvino():
-        requirements = _read_requirements("requirements-openvino.txt")
+        requirements = _read_requirements("openvino.txt")
     elif _is_tpu():
-        requirements = _read_requirements("requirements-tpu.txt")
+        requirements = _read_requirements("tpu.txt")
     elif _is_cpu():
-        requirements = _read_requirements("requirements-cpu.txt")
+        requirements = _read_requirements("cpu.txt")
     elif _is_xpu():
-        requirements = _read_requirements("requirements-xpu.txt")
+        requirements = _read_requirements("xpu.txt")
     else:
         raise ValueError(
             "Unsupported platform, please use CUDA, ROCm, Neuron, HPU, "

use_existing_torch.py

@@ -2,7 +2,7 @@
 
 import glob
 
-requires_files = glob.glob('requirements*.txt')
+requires_files = glob.glob('requirements/*.txt')
 requires_files += ["pyproject.toml"]
 for file in requires_files:
     print(f">>> cleaning {file}")