diff --git a/scripts/generate_config_list.sh b/scripts/generate_config_list.sh
new file mode 100755
index 0000000..22d3fca
--- /dev/null
+++ b/scripts/generate_config_list.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -eu
+
+json_files=$(cd configurations; find . -iname "*.json" | sort | sed 's|^\./||')
+
+MESON_FILE=configurations/meson.build
+
+SELF=$(basename "$0")
+
+# Generate the Meson file
+{
+    echo "# This file is auto-generated. Do not edit manually."
+    echo "# File content generated with ${SELF}."
+    echo "configs = ["
+    for file in $json_files; do
+        echo "    '${file}',"
+    done
+    echo "]"
+} > "$MESON_FILE"
diff --git a/scripts/run-ci.sh b/scripts/run-ci.sh
index ca63842..9364a97 100755
--- a/scripts/run-ci.sh
+++ b/scripts/run-ci.sh
@@ -1,2 +1,9 @@
 #!/bin/sh
+
+set -e
+
 scripts/validate_configs.py -v -k -e test/expected-schema-errors.txt
+
+# fail the script if someone forgot to update the list of configurations
+scripts/generate_config_list.sh
+git --no-pager diff --exit-code -- .