Fix a `BridgeTower` test (#23694)

fix
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/bridgetower/test_modeling_bridgetower.py

@@ -627,7 +627,8 @@ class BridgeTowerModelTrainingTest(unittest.TestCase):
         non_used_layer_names = ["text_model.pooler"]
         if model_class == BridgeTowerForMaskedLM:
             non_used_layer_names = non_used_layer_names + [
-                "cross_modal_image_layers.5",
+                # This number `1` actually depends on the number of layers in `cross_modal_image_layers` (by minus 1)
+                "cross_modal_image_layers.1",
                 "cross_modal_image_pooler",
                 "cross_modal_text_pooler",
             ]