Skip to content

GitLab

Unverified Commit aaddc9c8 authored by Shengqi Chen's avatar Shengqi Chen Committed by GitHub
Browse files

[CI] fix silent error in nightly wheel index generation script, add generation... 


[CI] fix silent error in nightly wheel index generation script, add generation time to HTML index (#30060)
Signed-off-by: default avatarShengqi Chen <harry-chen@outlook.com>
parent 263c38d7
Hide whitespace changes
Inline Side-by-side
Showing with 30 additions and 8 deletions
.buildkite/scripts/generate-nightly-index.py
View file @ aaddc9c8
...@@ -9,6 +9,7 @@ import argparse ...@@ -9,6 +9,7 @@ import argparse
import json import json
import sys import sys
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from urllib.parse import quote from urllib.parse import quote
...@@ -20,6 +21,7 @@ if not sys.version_info >= (3, 12): ...@@ -20,6 +21,7 @@ if not sys.version_info >= (3, 12):
INDEX_HTML_TEMPLATE = """<!DOCTYPE html> INDEX_HTML_TEMPLATE = """<!DOCTYPE html>
<html> <html>
<!-- {comment} -->
<meta name="pypi:repository-version" content="1.0"> <meta name="pypi:repository-version" content="1.0">
<body> <body>
{items} {items}
...@@ -90,7 +92,7 @@ def parse_from_filename(file: str) -> WheelFileInfo: ...@@ -90,7 +92,7 @@ def parse_from_filename(file: str) -> WheelFileInfo:
) )
def generate_project_list(subdir_names: list[str]) -> str: def generate_project_list(subdir_names: list[str], comment: str = "") -> str:
""" """
Generate project list HTML content linking to each project & variant sub-directory. Generate project list HTML content linking to each project & variant sub-directory.
""" """
...@@ -98,11 +100,14 @@ def generate_project_list(subdir_names: list[str]) -> str: ...@@ -98,11 +100,14 @@ def generate_project_list(subdir_names: list[str]) -> str:
for name in sorted(subdir_names): for name in sorted(subdir_names):
name = name.strip("/").strip(".") name = name.strip("/").strip(".")
href_tags.append(f' <a href="{name}/">{name}/</a><br/>') href_tags.append(f' <a href="{name}/">{name}/</a><br/>')
return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags)) return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
def generate_package_index_and_metadata( def generate_package_index_and_metadata(
wheel_files: list[WheelFileInfo], wheel_base_dir: Path, index_base_dir: Path wheel_files: list[WheelFileInfo],
wheel_base_dir: Path,
index_base_dir: Path,
comment: str = "",
) -> tuple[str, str]: ) -> tuple[str, str]:
""" """
Generate package index HTML content for a specific package, linking to actual wheel files. Generate package index HTML content for a specific package, linking to actual wheel files.
...@@ -120,7 +125,7 @@ def generate_package_index_and_metadata( ...@@ -120,7 +125,7 @@ def generate_package_index_and_metadata(
file_meta = asdict(file) file_meta = asdict(file)
file_meta["path"] = file_path_quoted file_meta["path"] = file_path_quoted
metadata.append(file_meta) metadata.append(file_meta)
index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags)) index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
metadata_str = json.dumps(metadata, indent=2) metadata_str = json.dumps(metadata, indent=2)
return index_str, metadata_str return index_str, metadata_str
...@@ -131,6 +136,7 @@ def generate_index_and_metadata( ...@@ -131,6 +136,7 @@ def generate_index_and_metadata(
index_base_dir: Path, index_base_dir: Path,
default_variant: str | None = None, default_variant: str | None = None,
alias_to_default: str | None = None, alias_to_default: str | None = None,
comment: str = "",
): ):
""" """
Generate index for all wheel files. Generate index for all wheel files.
...@@ -141,6 +147,7 @@ def generate_index_and_metadata( ...@@ -141,6 +147,7 @@ def generate_index_and_metadata(
index_base_dir (Path): Base directory to store index files. index_base_dir (Path): Base directory to store index files.
default_variant (str | None): The default variant name, if any. default_variant (str | None): The default variant name, if any.
alias_to_default (str | None): Alias variant name for the default variant, if any. alias_to_default (str | None): Alias variant name for the default variant, if any.
comment (str | None): Optional comment to include in the generated HTML files.
First, parse all wheel files to extract metadata. First, parse all wheel files to extract metadata.
We need to collect all wheel files for each variant, and generate an index for it (in a sub-directory). We need to collect all wheel files for each variant, and generate an index for it (in a sub-directory).
...@@ -234,6 +241,10 @@ def generate_index_and_metadata( ...@@ -234,6 +241,10 @@ def generate_index_and_metadata(
variant_to_files[alias_to_default] = variant_to_files["default"].copy() variant_to_files[alias_to_default] = variant_to_files["default"].copy()
print(f"Alias variant '{alias_to_default}' created for default variant.") print(f"Alias variant '{alias_to_default}' created for default variant.")
# Generate comment in HTML header
comment_str = f" ({comment})" if comment else ""
comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}"
# Generate index for each variant # Generate index for each variant
subdir_names = set() subdir_names = set()
for variant, files in variant_to_files.items(): for variant, files in variant_to_files.items():
...@@ -253,7 +264,7 @@ def generate_index_and_metadata( ...@@ -253,7 +264,7 @@ def generate_index_and_metadata(
subdir_names = subdir_names.union(packages) subdir_names = subdir_names.union(packages)
else: else:
# generate project list for this variant directly # generate project list for this variant directly
project_list_str = generate_project_list(sorted(packages)) project_list_str = generate_project_list(sorted(packages), comment_tmpl)
with open(variant_dir / "index.html", "w") as f: with open(variant_dir / "index.html", "w") as f:
f.write(project_list_str) f.write(project_list_str)
...@@ -263,7 +274,7 @@ def generate_index_and_metadata( ...@@ -263,7 +274,7 @@ def generate_index_and_metadata(
package_dir = variant_dir / package package_dir = variant_dir / package
package_dir.mkdir(parents=True, exist_ok=True) package_dir.mkdir(parents=True, exist_ok=True)
index_str, metadata_str = generate_package_index_and_metadata( index_str, metadata_str = generate_package_index_and_metadata(
package_files, wheel_base_dir, package_dir package_files, wheel_base_dir, package_dir, comment
) )
with open(package_dir / "index.html", "w") as f: with open(package_dir / "index.html", "w") as f:
f.write(index_str) f.write(index_str)
...@@ -271,7 +282,7 @@ def generate_index_and_metadata( ...@@ -271,7 +282,7 @@ def generate_index_and_metadata(
f.write(metadata_str) f.write(metadata_str)
# Generate top-level project list index # Generate top-level project list index
project_list_str = generate_project_list(sorted(subdir_names)) project_list_str = generate_project_list(sorted(subdir_names), comment_tmpl)
with open(index_base_dir / "index.html", "w") as f: with open(index_base_dir / "index.html", "w") as f:
f.write(project_list_str) f.write(project_list_str)
...@@ -283,6 +294,7 @@ if __name__ == "__main__": ...@@ -283,6 +294,7 @@ if __name__ == "__main__":
--current-objects <path_to_json> : path to JSON file containing current S3 objects listing in this version directory --current-objects <path_to_json> : path to JSON file containing current S3 objects listing in this version directory
--output-dir <output_directory> : directory to store generated index files --output-dir <output_directory> : directory to store generated index files
--alias-to-default <alias_variant_name> : (optional) alias variant name for the default variant --alias-to-default <alias_variant_name> : (optional) alias variant name for the default variant
--comment <comment_string> : (optional) comment string to include in generated HTML files
""" """
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
...@@ -312,6 +324,12 @@ if __name__ == "__main__": ...@@ -312,6 +324,12 @@ if __name__ == "__main__":
default=None, default=None,
help="Alias variant name for the default variant", help="Alias variant name for the default variant",
) )
parser.add_argument(
"--comment",
type=str,
default="",
help="Optional comment string to include in generated HTML files",
)
args = parser.parse_args() args = parser.parse_args()
...@@ -366,5 +384,6 @@ if __name__ == "__main__": ...@@ -366,5 +384,6 @@ if __name__ == "__main__":
index_base_dir=index_base_dir, index_base_dir=index_base_dir,
default_variant=None, default_variant=None,
alias_to_default=args.alias_to_default, alias_to_default=args.alias_to_default,
comment=args.comment.strip(),
) )
print(f"Successfully generated index and metadata in {output_dir}") print(f"Successfully generated index and metadata in {output_dir}")
.buildkite/scripts/upload-wheels.sh
View file @ aaddc9c8
...@@ -81,7 +81,10 @@ else ...@@ -81,7 +81,10 @@ else
alias_arg="" alias_arg=""
fi fi
$PYTHON pip install regex && .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" $alias_arg # HACK: we do not need regex module here, but it is required by pre-commit hook
# To avoid any external dependency, we simply replace it back to the stdlib re module
sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" $alias_arg
# copy indices to /<commit>/ unconditionally # copy indices to /<commit>/ unconditionally
echo "Uploading indices to $S3_COMMIT_PREFIX" echo "Uploading indices to $S3_COMMIT_PREFIX"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment