Skip to content

GitLab

"docs/reference/support-matrix.md" did not exist on "2befa388d2126d5a91d8e124b8167d36eeb7dfe1"
Unverified Commit 5d11f75e authored by Nate Mailhot's avatar Nate Mailhot Committed by GitHub
Browse files

feat: add container layer metrics (#4628) 


Signed-off-by: default avatarNate Mailhot <nmailhot@nvidia.com>
parent c9d7d95f
Hide whitespace changes
Inline Side-by-side
Showing with 482 additions and 20 deletions
.github/actions/docker-build/action.yml
View file @ 5d11f75e
......@@ -62,6 +62,8 @@ runs:
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #v3.11.1
with:
driver: docker
# Enable BuildKit for enhanced metadata
buildkitd-flags: --debug
- name: Login to ECR
shell: bash
env:
......@@ -88,6 +90,8 @@ runs:
AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
PLATFORM: ${{ inputs.platform }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_JOB: ${{ github.job }}
run: |
# Determine image tag
if [ -n "${{ inputs.image_tag }}" ]; then
......@@ -97,10 +101,16 @@ runs:
fi
BUILD_START_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
echo "🕐 Build started at: ${BUILD_START_TIME}"
echo "BUILD_START_TIME=${BUILD_START_TIME}" >> $GITHUB_ENV
echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
# Create build logs directory
mkdir -p build-logs
BUILD_LOG_FILE="build-logs/build-${{ inputs.framework }}-$(echo '${{ inputs.platform }}' | sed 's/linux\///').log"
echo "BUILD_LOG_FILE=${BUILD_LOG_FILE}" >> $GITHUB_ENV
echo "📝 Build log will be saved to: ${BUILD_LOG_FILE}"
# Collect optional overrides provided by the workflow
EXTRA_ARGS=""
if [ -n "${{ inputs.base_image_tag }}" ]; then
......@@ -116,6 +126,7 @@ runs:
EXTRA_ARGS+=" --build-arg TORCH_BACKEND=${{ inputs.torch_backend }}"
fi
# Execute build and capture output (show on console AND save to file)
./container/build.sh --tag "$IMAGE_TAG" \
--target ${{ inputs.target }} \
--vllm-max-jobs 10 \
......@@ -123,17 +134,20 @@ runs:
--platform ${{ inputs.platform }} \
--use-sccache \
--sccache-bucket "$SCCACHE_S3_BUCKET" \
--sccache-region "$AWS_DEFAULT_REGION" $EXTRA_ARGS
--sccache-region "$AWS_DEFAULT_REGION" $EXTRA_ARGS 2>&1 | tee "${BUILD_LOG_FILE}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
BUILD_END_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
echo "🕐 Build ended at: ${BUILD_END_TIME}"
echo "BUILD_END_TIME=${BUILD_END_TIME}" >> $GITHUB_ENV
# Exit with the build's exit code
exit ${BUILD_EXIT_CODE}
- name: Capture Build Metrics
id: metrics
shell: bash
run: |
echo "📊 Capturing build metrics for ${{ inputs.framework }}..."
# Create metrics directory
mkdir -p build-metrics
......@@ -162,21 +176,17 @@ runs:
echo "⚠️ No image tag available"
fi
echo "📊 Final metrics captured"
# Create consolidated metrics JSON file
echo "🔍 Debug: inputs.platform = '${{ inputs.platform }}'"
PLATFORM_ARCH=$(echo "${{ inputs.platform }}" | sed 's/linux\///')
echo "🔍 Debug: PLATFORM_ARCH = '${PLATFORM_ARCH}'"
echo " Architecture: ${PLATFORM_ARCH}"
echo "PLATFORM_ARCH=${PLATFORM_ARCH}" >> $GITHUB_ENV
JOB_KEY="${{ inputs.framework }}-${PLATFORM_ARCH}"
echo "🔍 Debug: JOB_KEY = '${JOB_KEY}'"
echo " Job Key: ${JOB_KEY}"
# Create job-specific metrics file
mkdir -p build-metrics
METRICS_FILE="build-metrics/metrics-${{ inputs.framework }}-${PLATFORM_ARCH}-${{ github.run_id }}-${{ job.check_run_id }}.json"
# Create the job metrics file directly
# Create the job metrics file
cat > "$METRICS_FILE" << EOF
{
"framework": "${{ inputs.framework }}",
......@@ -190,15 +200,103 @@ runs:
}
EOF
echo "📁 Created build metrics file for ${JOB_KEY}:"
cat "$METRICS_FILE"
# Metrics captured and saved to JSON file
- name: Generate Comprehensive Build Metrics
id: comprehensive-metrics
if: always()
shell: bash
run: |
echo "=========================================="
echo "📊 GENERATING COMPREHENSIVE BUILD METRICS"
echo "=========================================="
# Create metrics directory
mkdir -p build-metrics
PLATFORM_ARCH="${{ env.PLATFORM_ARCH }}"
WORKFLOW_ID="${{ github.run_id }}"
JOB_ID="${{ job.check_run_id }}"
FRAMEWORK_LOWER=$(echo "${{ inputs.framework }}" | tr '[:upper:]' '[:lower:]')
# Make parser executable
chmod +x .github/scripts/parse_buildkit_output.py
# Check for build logs and build stage arguments dynamically
BASE_BUILD_LOG="build-logs/base-image-build.log"
FRAMEWORK_BUILD_LOG="build-logs/framework-${FRAMEWORK_LOWER}-build.log"
# Path to container metadata created in previous step
CONTAINER_METADATA="build-metrics/metrics-${{ inputs.framework }}-${PLATFORM_ARCH}-${WORKFLOW_ID}-${JOB_ID}.json"
# Output single comprehensive JSON with all build stages
COMPREHENSIVE_JSON="build-metrics/build-${{ inputs.framework }}-${PLATFORM_ARCH}-${WORKFLOW_ID}-${JOB_ID}.json"
echo "🚀 Parsing BuildKit outputs and merging with container metrics..."
# Upload job-specific build metrics as artifact
- name: Upload Build Metrics
# Build stage arguments dynamically based on which logs exist
STAGE_ARGS=()
if [ -f "$BASE_BUILD_LOG" ]; then
echo " ✓ Found base image log: ${BASE_BUILD_LOG}"
STAGE_ARGS+=("base:${BASE_BUILD_LOG}")
else
echo " ℹ️ No base image log found"
fi
if [ -f "$FRAMEWORK_BUILD_LOG" ]; then
echo " ✓ Found framework log: ${FRAMEWORK_BUILD_LOG}"
STAGE_ARGS+=("runtime:${FRAMEWORK_BUILD_LOG}")
else
echo " ℹ️ No framework log found"
fi
# Check for any additional stage logs (e.g., build-logs/stage3-*.log)
for extra_log in build-logs/stage*.log; do
if [ -f "$extra_log" ]; then
stage_name=$(basename "$extra_log" .log)
echo " ✓ Found additional stage log: ${extra_log} (${stage_name})"
STAGE_ARGS+=("${stage_name}:${extra_log}")
fi
done
echo "Container Metadata: ${CONTAINER_METADATA}"
echo "Output: ${COMPREHENSIVE_JSON}"
echo ""
# Run parser with all discovered stages
# Usage: parse_buildkit_output.py <output_json> <stage1_name:log_file> [stage2_name:log_file] ... [--metadata=<file>]
set +e
python3 .github/scripts/parse_buildkit_output.py \
"$COMPREHENSIVE_JSON" \
"${STAGE_ARGS[@]}" \
"--metadata=${CONTAINER_METADATA}"
PARSER_EXIT_CODE=$?
set -e
echo ""
echo "📊 Parser exit code: ${PARSER_EXIT_CODE}"
if [ ${PARSER_EXIT_CODE} -eq 0 ] && [ -f "$COMPREHENSIVE_JSON" ]; then
echo "✅ Comprehensive build metrics generated successfully"
echo "📄 Output file: ${COMPREHENSIVE_JSON}"
echo ""
echo "=========================================="
echo "📋 FULL JSON OUTPUT (for debugging)"
echo "=========================================="
cat "$COMPREHENSIVE_JSON"
echo ""
echo "=========================================="
else
echo "⚠️ Metrics generation had issues but continuing..."
fi
# Upload comprehensive build metrics as artifact
- name: Upload Comprehensive Build Metrics
uses: actions/upload-artifact@v4
if: always()
with:
name: build-metrics-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}-${{ github.run_id }}-${{ job.check_run_id }}
path: build-metrics/metrics-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}-${{ github.run_id }}-${{ job.check_run_id }}.json
path: build-metrics/build-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}-${{ github.run_id }}-${{ job.check_run_id }}.json
retention-days: 7
.github/scripts/parse_buildkit_output.py 0 → 100755
View file @ 5d11f75e
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""
Parse BuildKit output to extract detailed step-by-step metadata.
BuildKit provides rich information about each build step including timing,
cache status, sizes, and layer IDs.
"""
import json
import re
import sys
from datetime import datetime, timezone
from typing import Any, Dict, List
class BuildKitParser:
"""Parser for BuildKit output logs"""
def __init__(self):
self.steps: List[Dict[str, Any]] = []
self.current_step = None
self.step_counter = 0
def parse_log(self, log_content: str) -> Dict[str, Any]:
"""
Parse BuildKit log output and extract step metadata.
BuildKit output format (with --progress=plain):
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 2.34kB done
#1 DONE 0.1s
#2 [internal] load metadata for nvcr.io/nvidia/cuda:12.8...
#2 DONE 2.3s
#3 [1/5] FROM nvcr.io/nvidia/cuda:12.8...
#3 resolve nvcr.io/nvidia/cuda:12.8... done
#3 CACHED
#4 [2/5] RUN apt-get update && apt-get install...
#4 0.234 Reading package lists...
#4 DONE 45.2s
"""
lines = log_content.split("\n")
step_data = {}
current_step_num = None
for line in lines:
line = line.strip()
if not line:
continue
# Match step headers: #N [...]
step_match = re.match(r"^#(\d+)\s+\[(.*?)\](.*)$", line)
if step_match:
step_num = step_match.group(1)
step_name = step_match.group(2).strip()
step_command = step_match.group(3).strip()
if step_num not in step_data:
step_data[step_num] = {
"step_number": int(step_num),
"step_name": step_name,
"command": step_command,
"status": "unknown",
"cached": False,
"duration_sec": 0.0,
"size_transferred": 0,
}
current_step_num = step_num
continue
# Match step status lines: #N DONE 1.2s, #N CACHED, #N ERROR
if current_step_num:
# DONE with timing
done_match = re.match(
rf"^#{current_step_num}\s+DONE\s+([\d.]+)s?", line
)
if done_match:
step_data[current_step_num]["status"] = "done"
step_data[current_step_num]["duration_sec"] = float(
done_match.group(1)
)
continue
# CACHED
if re.match(rf"^#{current_step_num}\s+CACHED", line):
step_data[current_step_num]["status"] = "cached"
step_data[current_step_num]["cached"] = True
continue
# ERROR
if re.match(rf"^#{current_step_num}\s+ERROR", line):
step_data[current_step_num]["status"] = "error"
continue
# Substep information (timing and progress)
substep_match = re.match(
rf"^#{current_step_num}\s+([\d.]+)\s+(.*)", line
)
if substep_match:
message = substep_match.group(2)
# Extract size information
size_match = re.search(r"([\d.]+)\s*([KMGT]?i?B)", message)
if size_match:
size_bytes = self._parse_size(
size_match.group(1), size_match.group(2)
)
step_data[current_step_num]["size_transferred"] += size_bytes
continue
# Convert to sorted list
steps = [step_data[num] for num in sorted(step_data.keys(), key=int)]
# Calculate aggregate statistics
cached_steps = sum(1 for s in steps if s["cached"])
total_steps = len(steps)
cache_hit_rate = (cached_steps / total_steps * 100) if total_steps > 0 else 0.0
total_size = sum(s["size_transferred"] for s in steps)
# Create single stage for this Docker build (stage name will be updated from metadata)
build_duration_sec = sum(s["duration_sec"] for s in steps if not s["cached"])
stage_metrics = [
{
"stage_name": "unknown", # Will be set from container metadata
"total_steps": total_steps,
"cached_steps": cached_steps,
"built_steps": total_steps - cached_steps,
"build_duration_sec": round(build_duration_sec, 2),
"cache_hit_rate": round(cache_hit_rate, 2),
}
]
return {
"container": {
"total_steps": total_steps,
"cached_steps": cached_steps,
"built_steps": total_steps - cached_steps,
"overall_cache_hit_rate": round(cache_hit_rate, 2),
"total_size_transferred_bytes": total_size,
},
"stages": stage_metrics,
"layers": steps,
"metadata": {
"parsed_at": datetime.now(timezone.utc).isoformat(),
"parser_version": "1.0",
},
}
def _parse_size(self, value: str, unit: str) -> int:
"""Convert size string to bytes"""
try:
val = float(value)
except ValueError:
return 0
# Normalize unit
unit = unit.upper().replace("I", "") # Remove 'i' from KiB, MiB, etc.
multipliers = {
"B": 1,
"KB": 1024,
"MB": 1024**2,
"GB": 1024**3,
"TB": 1024**4,
}
return int(val * multipliers.get(unit, 1))
def main():
"""Main entry point"""
if len(sys.argv) < 3:
print(
"Usage: parse_buildkit_output.py <output_json> <stage1_name:log_file> [stage2_name:log_file] ... [--metadata=<container_metadata_json>]",
file=sys.stderr,
)
print(
"Example: parse_buildkit_output.py output.json base:base.log runtime:framework.log --metadata=meta.json",
file=sys.stderr,
)
sys.exit(1)
output_json = sys.argv[1]
# Parse arguments to find stage logs and metadata
stage_logs = [] # List of (stage_name, log_file) tuples
container_metadata_file = None
for arg in sys.argv[2:]:
if arg.startswith("--metadata="):
container_metadata_file = arg.split("=", 1)[1]
elif ":" in arg:
stage_name, log_file = arg.split(":", 1)
stage_logs.append((stage_name, log_file))
else:
# Backwards compatibility: assume unnamed logs are base, runtime, etc.
if not stage_logs:
stage_logs.append(("base", arg))
elif len(stage_logs) == 1:
stage_logs.append(("runtime", arg))
else:
stage_logs.append((f"stage{len(stage_logs)}", arg))
# Initialize combined structure
combined_data = {"container": {}, "stages": [], "layers": []}
total_steps = 0
total_cached = 0
total_size = 0
# Parse each stage log
for stage_name, log_file in stage_logs:
try:
with open(log_file, "r") as f:
log_content = f.read()
parser = BuildKitParser()
stage_data = parser.parse_log(log_content)
# Add stage with custom name
if stage_data.get("stages"):
stage_info = stage_data["stages"][0].copy()
stage_info["stage_name"] = stage_name
combined_data["stages"].append(stage_info)
# Add layers with stage identifier
for layer in stage_data.get("layers", []):
layer["stage"] = stage_name
combined_data["layers"].append(layer)
# Accumulate metrics
total_steps += stage_data["container"]["total_steps"]
total_cached += stage_data["container"]["cached_steps"]
total_size += stage_data["container"]["total_size_transferred_bytes"]
print(
f"✅ Parsed {stage_name} stage: {stage_data['container']['total_steps']} steps",
file=sys.stderr,
)
except FileNotFoundError:
print(
f"⚠️ Log file not found for {stage_name} stage: {log_file}",
file=sys.stderr,
)
except Exception as e:
print(f"Warning: Could not parse {stage_name} log: {e}", file=sys.stderr)
# Calculate rolled-up container metrics
total_built = total_steps - total_cached
overall_cache_hit_rate = (
(total_cached / total_steps * 100) if total_steps > 0 else 0.0
)
combined_data["container"] = {
"total_steps": total_steps,
"cached_steps": total_cached,
"built_steps": total_built,
"overall_cache_hit_rate": round(overall_cache_hit_rate, 2),
"total_size_transferred_bytes": total_size,
}
build_data = combined_data
# Merge container metadata if provided
if container_metadata_file:
try:
with open(container_metadata_file, "r") as f:
container_metadata = json.load(f)
# Merge into container section (overwrites BuildKit fields with action.yml values)
# Note: We don't overwrite stage names since they're explicitly set from log file names
build_data["container"].update(container_metadata)
except Exception as e:
print(f"Warning: Could not read container metadata: {e}", file=sys.stderr)
# Output JSON
try:
with open(output_json, "w") as f:
json.dump(build_data, f, indent=2)
print(f"✅ Build data written to: {output_json}", file=sys.stderr)
except Exception as e:
print(f"Error writing JSON file: {e}", file=sys.stderr)
sys.exit(1)
# Print summary to stderr for immediate feedback
container = build_data["container"]
print("", file=sys.stderr)
print("📊 Build Summary:", file=sys.stderr)
print(
f" Steps: {container['total_steps']} total, "
f"{container['cached_steps']} cached, "
f"{container['built_steps']} built",
file=sys.stderr,
)
print(
f" Cache Hit Rate: {container['overall_cache_hit_rate']:.1f}%",
file=sys.stderr,
)
if __name__ == "__main__":
main()
container/build.sh
View file @ 5d11f75e
......@@ -873,15 +873,76 @@ if [[ -z "${DEV_IMAGE_INPUT:-}" ]]; then
echo "======================================"
echo "Starting Build 1: Base Image"
echo "======================================"
$RUN_PREFIX docker build -f "${SOURCE_DIR}/Dockerfile" --target dev $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO --tag $DYNAMO_BASE_IMAGE $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE
# Create build log directory for BuildKit reports
BUILD_LOG_DIR="${BUILD_CONTEXT}/build-logs"
mkdir -p "${BUILD_LOG_DIR}"
BASE_BUILD_LOG="${BUILD_LOG_DIR}/base-image-build.log"
# Use BuildKit for enhanced metadata
if [ -z "$RUN_PREFIX" ]; then
if docker buildx version &>/dev/null; then
docker buildx build --progress=plain --load -f "${SOURCE_DIR}/Dockerfile" --target dev $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO --tag $DYNAMO_BASE_IMAGE $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE 2>&1 | tee "${BASE_BUILD_LOG}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
else
DOCKER_BUILDKIT=1 docker build --progress=plain -f "${SOURCE_DIR}/Dockerfile" --target dev $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO --tag $DYNAMO_BASE_IMAGE $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE 2>&1 | tee "${BASE_BUILD_LOG}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
fi
if [ ${BUILD_EXIT_CODE} -ne 0 ]; then
exit ${BUILD_EXIT_CODE}
fi
else
$RUN_PREFIX docker build -f "${SOURCE_DIR}/Dockerfile" --target dev $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO --tag $DYNAMO_BASE_IMAGE $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE
fi
# Start framework build
echo "======================================"
echo "Starting Build 2: Framework Image"
echo "======================================"
FRAMEWORK_BUILD_LOG="${BUILD_LOG_DIR}/framework-${FRAMEWORK,,}-build.log"
BUILD_ARGS+=" --build-arg DYNAMO_BASE_IMAGE=${DYNAMO_BASE_IMAGE}"
$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE
# Use BuildKit for enhanced metadata
if [ -z "$RUN_PREFIX" ]; then
if docker buildx version &>/dev/null; then
docker buildx build --progress=plain --load -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE 2>&1 | tee "${FRAMEWORK_BUILD_LOG}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
else
DOCKER_BUILDKIT=1 docker build --progress=plain -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE 2>&1 | tee "${FRAMEWORK_BUILD_LOG}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
fi
if [ ${BUILD_EXIT_CODE} -ne 0 ]; then
exit ${BUILD_EXIT_CODE}
fi
else
$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE
fi
else
$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE
# Create build log directory for BuildKit reports
BUILD_LOG_DIR="${BUILD_CONTEXT}/build-logs"
mkdir -p "${BUILD_LOG_DIR}"
SINGLE_BUILD_LOG="${BUILD_LOG_DIR}/single-stage-build.log"
# Use BuildKit for enhanced metadata
if [ -z "$RUN_PREFIX" ]; then
if docker buildx version &>/dev/null; then
docker buildx build --progress=plain --load -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE 2>&1 | tee "${SINGLE_BUILD_LOG}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
else
DOCKER_BUILDKIT=1 docker build --progress=plain -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE 2>&1 | tee "${SINGLE_BUILD_LOG}"
BUILD_EXIT_CODE=${PIPESTATUS[0]}
fi
if [ ${BUILD_EXIT_CODE} -ne 0 ]; then
exit ${BUILD_EXIT_CODE}
fi
else
$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $CACHE_TO $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE
fi
fi
fi
......@@ -928,4 +989,4 @@ elif [[ "${LOCAL_DEV_BUILD:-}" == "true" ]]; then
fi
{ set +x; } 2>/dev/null
{ set +x; } 2>/dev/null
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment