ci: OPS-1930: Increase timeout and add max matrix concurrency (#4130)

Signed-off-by: Dillon Cullinan <dcullinan@nvidia.com>
Signed-off-by: Dillon Cullinan <dcullinan92@gmail.com>
Co-authored-by: Ran Rubin <ranrubin@gmail.com>

.github/workflows/container-validation-backends.yml

@@ -431,6 +431,7 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         profile:
           - agg
@@ -485,7 +486,7 @@ jobs:
         export GRAPH_NAME=$(yq e '.metadata.name' $DEPLOYMENT_FILE)
         echo "Waiting for all pods with label nvidia.com/dynamo-graph-deployment-name: $GRAPH_NAME"
         # Wait for all pods with the deployment label to be ready
-        kubectl wait --for=condition=ready pod -l "nvidia.com/dynamo-graph-deployment-name=$GRAPH_NAME" -n ${KUBE_NS} --timeout=1000s
+        kubectl wait --for=condition=ready pod -l "nvidia.com/dynamo-graph-deployment-name=$GRAPH_NAME" -n ${KUBE_NS} --timeout=1300s
 
         # Debug: Show final pod statuses for the deployment
         echo "=== FINAL POD STATUSES ==="
@@ -580,6 +581,7 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         profile:
           - agg
@@ -600,6 +602,7 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         profile:
           - agg