diff --git a/QUICK_START_TESTING.md b/QUICK_START_TESTING.md index 9430799..a910a6c 100644 --- a/QUICK_START_TESTING.md +++ b/QUICK_START_TESTING.md @@ -3,14 +3,14 @@ ## TL;DR ```bash -# Test all configs with human-readable output +# Test all configs with human-readable output (default) ./run-config-tests.sh # Generate HTML report -./run-config-tests.sh html +./run-config-tests.sh --format html -# Test single config -./run-config-tests.sh human ECS4150-TM.json +# Test single config with HTML output +./run-config-tests.sh --format html cfg0.json # Results are in: output/ ``` @@ -20,17 +20,27 @@ ### Test All Configurations ```bash -./run-config-tests.sh human # Console output with colors -./run-config-tests.sh html # Interactive HTML report -./run-config-tests.sh json # Machine-readable JSON +# Stub mode (default - fast, proto.c parsing only) +./run-config-tests.sh # Console output with colors (default) +./run-config-tests.sh --format html # Interactive HTML report +./run-config-tests.sh --format json # Machine-readable JSON + +# Platform mode (integration testing with platform code) +./run-config-tests.sh --mode platform # Console output +./run-config-tests.sh --mode platform --format html # HTML report ``` ### Test Single Configuration ```bash -./run-config-tests.sh human cfg0.json -./run-config-tests.sh html ECS4150-ACL.json -./run-config-tests.sh json ECS4150-TM.json +# Stub mode (default) +./run-config-tests.sh cfg0.json # Human output (default) +./run-config-tests.sh --format html cfg0.json # HTML report +./run-config-tests.sh --format json cfg0.json # JSON output + +# Platform mode +./run-config-tests.sh --mode platform cfg0.json # Human output +./run-config-tests.sh --mode platform --format html cfg0.json # HTML report ``` ### View Results @@ -106,7 +116,7 @@ ls config-samples/*.json ### Example Pipeline ```yaml - name: Run tests - run: ./run-config-tests.sh json + run: ./run-config-tests.sh --format json - name: Check results run: | if [ $? -eq 0 ]; then @@ -141,14 +151,34 @@ ECS4150_VLAN.json # VLAN configuration ✅ Feature coverage (implemented vs documented features) ✅ Error handling (invalid configs, missing fields) +## Test Modes + +### Stub Mode (Default - Fast) +- Tests proto.c parsing only +- Uses simple platform stubs +- Shows base properties only +- Execution time: ~30 seconds +- Use for: Quick validation, CI/CD + +### Platform Mode (Integration) +- Tests proto.c + platform code (plat-gnma.c) +- Uses platform implementation with mocks +- Shows base AND platform properties +- Tracks hardware application functions +- Execution time: ~45 seconds +- Use for: Platform-specific validation + ## Quick Reference | Task | Command | |------|---------| -| Test everything | `./run-config-tests.sh` | -| HTML report | `./run-config-tests.sh html` | -| JSON output | `./run-config-tests.sh json` | -| Single config | `./run-config-tests.sh human cfg0.json` | +| Test everything (stub) | `./run-config-tests.sh` | +| Test everything (platform) | `./run-config-tests.sh --mode platform` | +| HTML report | `./run-config-tests.sh --format html` | +| JSON output | `./run-config-tests.sh --format json` | +| Single config | `./run-config-tests.sh cfg0.json` | +| Single config HTML | `./run-config-tests.sh -f html cfg0.json` | +| Platform mode single config | `./run-config-tests.sh -m platform cfg0.json` | | View HTML | `open output/test-report.html` | | View results | `cat output/test-results.txt` | | Parse JSON | `cat output/test-report.json \| jq` | diff --git a/README.md b/README.md index c0acbc1..942eb76 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ The repository includes a comprehensive testing framework for configuration vali ./run-config-tests.sh # Generate HTML report -./run-config-tests.sh html +./run-config-tests.sh --format html # Or run tests directly in the tests directory cd tests/config-parser diff --git a/TESTING_FRAMEWORK.md b/TESTING_FRAMEWORK.md index b189326..7795935 100644 --- a/TESTING_FRAMEWORK.md +++ b/TESTING_FRAMEWORK.md @@ -42,7 +42,7 @@ This testing framework includes multiple documentation files, each serving a spe 4. **[TEST_CONFIG_PARSER_DESIGN.md](TEST_CONFIG_PARSER_DESIGN.md)** - Test framework architecture - Multi-layer validation design - - Property metadata system (628 properties) + - Property metadata system (398 schema properties) - Property inspection engine - Test execution flow diagrams - Data structures and algorithms @@ -64,17 +64,18 @@ This testing framework includes multiple documentation files, each serving a spe **RECOMMENDED: Use the test runner script** (handles Docker automatically): ```bash -# Test all configurations (human-readable output) -./run-config-tests.sh +# Test all configurations in STUB mode (default - fast, proto.c only) +./run-config-tests.sh # Human-readable output +./run-config-tests.sh --format html # HTML report +./run-config-tests.sh --format json # JSON report -# Generate HTML report -./run-config-tests.sh html - -# Generate JSON report -./run-config-tests.sh json +# Test all configurations in PLATFORM mode (integration testing) +./run-config-tests.sh --mode platform # Human-readable output +./run-config-tests.sh --mode platform --format html # HTML report # Test single configuration -./run-config-tests.sh human cfg0.json +./run-config-tests.sh cfg0.json # Stub mode +./run-config-tests.sh --mode platform cfg0.json # Platform mode ``` **Alternative: Run tests directly in Docker** (manual Docker management): @@ -83,21 +84,29 @@ This testing framework includes multiple documentation files, each serving a spe # Build the Docker environment first (if not already built) make build-host-env -# Run all tests (schema + parser) - RECOMMENDED +# Run all tests in STUB mode (default - fast) docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make test-config-full" -# Run individual test suites +# Run all tests in PLATFORM mode (integration) +docker exec ucentral_client_build_env bash -c \ + "cd /root/ols-nos/tests/config-parser && make test-config-full USE_PLATFORM=brcm-sonic" + +# Run individual test suites (stub mode) docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make validate-schema" docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make test-config" -# Generate test reports +# Generate test reports (stub mode) docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make test-config-html" +# Generate test reports (platform mode) +docker exec ucentral_client_build_env bash -c \ + "cd /root/ols-nos/tests/config-parser && make test-config-html USE_PLATFORM=brcm-sonic" + # Copy report files out of container to view docker cp ucentral_client_build_env:/root/ols-nos/tests/config-parser/test-report.html output/ ``` @@ -108,17 +117,24 @@ docker cp ucentral_client_build_env:/root/ols-nos/tests/config-parser/test-repor # Navigate to test directory cd tests/config-parser -# Run all tests (schema + parser) +# Run all tests in STUB mode (default) make test-config-full +# Run all tests in PLATFORM mode +make test-config-full USE_PLATFORM=brcm-sonic + # Run individual test suites make validate-schema # Schema validation only make test-config # Parser tests only -# Generate test reports +# Generate test reports (stub mode) make test-config-html # HTML report (browser-viewable) make test-config-json # JSON report (machine-readable) make test-config-junit # JUnit XML (CI/CD integration) + +# Generate test reports (platform mode) +make test-config-html USE_PLATFORM=brcm-sonic +make test-config-json USE_PLATFORM=brcm-sonic ``` **Note:** Running tests in Docker is the preferred method as it provides a consistent, reproducible environment regardless of your host OS (macOS, Linux, Windows). @@ -164,12 +180,12 @@ make test-config-junit # JUnit XML (CI/CD integration) - Hardware constraint validation ### Property Tracking System -- Database of 628 properties and their processing status +- Database of all schema properties and their implementation status (398 canonical properties) - Tracks which properties are parsed by which functions - Identifies unimplemented features - Status classification: CONFIGURED, IGNORED, SYSTEM, INVALID, Unknown - Property usage reports across all test configurations -- 199 properties (32%) with line number references +- Properties with line numbers are implemented, line_number=0 means not yet implemented ### Two-Layer Validation Strategy @@ -297,7 +313,7 @@ vi config-samples/test-new-feature.json ./run-config-tests.sh # Generate full HTML report for review -./run-config-tests.sh html +./run-config-tests.sh --format html open output/test-report.html # Check for property database accuracy @@ -310,7 +326,7 @@ open output/test-report.html test-configurations: stage: test script: - - ./run-config-tests.sh json + - ./run-config-tests.sh --format json artifacts: paths: - output/test-report.json @@ -333,7 +349,7 @@ static struct property_metadata properties[] = { .source_line = 1119, .notes = "Enable/disable ethernet interface" }, - // ... 628 total entries (199 with line numbers) ... + // ... entries for all 398 schema properties (with line numbers for implemented properties) ... }; ``` diff --git a/TEST_CONFIG_PARSER_DESIGN.md b/TEST_CONFIG_PARSER_DESIGN.md index a9e0ae6..4154465 100644 --- a/TEST_CONFIG_PARSER_DESIGN.md +++ b/TEST_CONFIG_PARSER_DESIGN.md @@ -18,9 +18,9 @@ The framework validates configurations through three complementary layers: ### Layer 3: Property Tracking - Deep recursive inspection of JSON tree to classify every property -- Maps properties to property metadata database (628 properties total) +- Maps properties to property metadata database (398 schema properties) - Tracks which properties are CONFIGURED, IGNORED, INVALID, UNKNOWN, etc. -- 199 properties (32%) include line number references to proto.c +- Properties with line numbers are implemented in proto.c; line_number=0 means not yet implemented ## 2. **Property Metadata System** @@ -36,12 +36,12 @@ struct property_metadata { }; ``` -**Database contains 628 entries** documenting: -- Which properties are actively parsed (PROP_CONFIGURED) +**Database contains entries for all 398 schema properties** documenting: +- Which properties are actively parsed (PROP_CONFIGURED with line numbers) +- Which are not yet implemented (line_number=0) - Which are intentionally ignored (PROP_IGNORED) - Which need platform implementation (PROP_UNKNOWN) - Which are structural containers (PROP_SYSTEM) -- Line numbers for 199 properties (32%) showing exact parsing locations ### Property Status Classification - **PROP_CONFIGURED**: Successfully processed by parser @@ -293,7 +293,7 @@ The design elegantly separates concerns: 4. **Validation layer** verifies specific features (optional validators) 5. **Reporting layer** generates multiple output formats -The property metadata database is the **crown jewel** - it documents the implementation status of 560+ configuration properties, enabling automated detection of unimplemented features and validation of parser coverage. +The property metadata database is the **crown jewel** - it documents the implementation status of all 398 schema properties, enabling automated detection of unimplemented features and validation of parser coverage. ## Related Documentation diff --git a/TEST_RUNNER_README.md b/TEST_RUNNER_README.md index ddce55d..668fc37 100644 --- a/TEST_RUNNER_README.md +++ b/TEST_RUNNER_README.md @@ -32,11 +32,16 @@ ### Basic Syntax ```bash -./run-config-tests.sh [format] [config-file] +./run-config-tests.sh [OPTIONS] [config-file] ``` -**Parameters:** -- `format` (optional): Output format - `html`, `json`, or `human` (default: `human`) +**Options:** +- `-f, --format FORMAT`: Output format - `html`, `json`, or `human` (default: `human`) +- `-m, --mode MODE`: Test mode - `stub` or `platform` (default: `stub`) +- `-p, --platform NAME`: Platform name for platform mode (default: `brcm-sonic`) +- `-h, --help`: Show help message + +**Arguments:** - `config-file` (optional): Specific config file to test (default: test all configs) ### Examples @@ -47,27 +52,32 @@ # Human-readable output (default) ./run-config-tests.sh -# Human-readable output (explicit) -./run-config-tests.sh human - # HTML report -./run-config-tests.sh html +./run-config-tests.sh --format html +# OR short form: +./run-config-tests.sh -f html # JSON output -./run-config-tests.sh json +./run-config-tests.sh --format json +# OR short form: +./run-config-tests.sh -f json ``` #### Test Single Configuration ```bash -# Test single config with human output -./run-config-tests.sh human cfg0.json +# Test single config with human output (default) +./run-config-tests.sh cfg0.json # Test single config with HTML report -./run-config-tests.sh html ECS4150-TM.json +./run-config-tests.sh --format html cfg0.json +# OR short form: +./run-config-tests.sh -f html cfg0.json # Test single config with JSON output -./run-config-tests.sh json ECS4150-ACL.json +./run-config-tests.sh --format json cfg0.json +# OR short form: +./run-config-tests.sh -f json cfg0.json ``` ## Output Files @@ -210,7 +220,7 @@ The script uses exit codes for CI/CD integration: **CI/CD Example:** ```bash -./run-config-tests.sh json +./run-config-tests.sh --format json if [ $? -eq 0 ]; then echo "All tests passed!" else @@ -321,7 +331,7 @@ make build-host-env # OR let script build it automatically # Run tests (script provides better output management) -./run-config-tests.sh html +./run-config-tests.sh --format html ``` ### With CI/CD @@ -338,7 +348,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Run config tests - run: ./run-config-tests.sh json + run: ./run-config-tests.sh --format json - name: Upload test results uses: actions/upload-artifact@v3 with: @@ -352,7 +362,7 @@ jobs: test-configs: stage: test script: - - ./run-config-tests.sh json + - ./run-config-tests.sh --format json artifacts: paths: - output/test-report.json @@ -364,7 +374,7 @@ test-configs: ```groovy stage('Test Configurations') { steps { - sh './run-config-tests.sh html' + sh './run-config-tests.sh --format html' publishHTML([ reportDir: 'output', reportFiles: 'test-report.html', @@ -382,7 +392,7 @@ stage('Test Configurations') { # .git/hooks/pre-commit echo "Running configuration tests..." -./run-config-tests.sh human +./run-config-tests.sh if [ $? -ne 0 ]; then echo "Tests failed. Commit aborted." @@ -406,7 +416,7 @@ OUTPUT_DIR="$SCRIPT_DIR/my-custom-output" ```bash # Test all ACL configs for config in config-samples/*ACL*.json; do - ./run-config-tests.sh json "$(basename $config)" + ./run-config-tests.sh --format json "$(basename $config)" done ``` @@ -424,7 +434,7 @@ wait ```bash # Generate all format reports for format in human html json; do - ./run-config-tests.sh $format + ./run-config-tests.sh --format $format done # Timestamp reports diff --git a/run-config-tests.sh b/run-config-tests.sh index 29db7a5..9f6b8f3 100755 --- a/run-config-tests.sh +++ b/run-config-tests.sh @@ -240,8 +240,8 @@ run_tests() { exit 1 fi - # Build test binary with appropriate mode - build_cmd="cd $BUILD_DIR && make test-config-parser $use_platform_flag" + # Build test binary with appropriate mode (clean first to ensure correct flags) + build_cmd="cd $BUILD_DIR && make clean && make test-config-parser $use_platform_flag" case "$FORMAT" in html) @@ -266,17 +266,17 @@ run_tests() { case "$FORMAT" in html) output_file="test-report.html" - test_cmd="cd $BUILD_DIR && make test-config-html $use_platform_flag" + test_cmd="cd $BUILD_DIR && make clean && make test-config-html $use_platform_flag" copy_files=("$output_file") ;; json) output_file="test-report.json" - test_cmd="cd $BUILD_DIR && make test-config-json $use_platform_flag" + test_cmd="cd $BUILD_DIR && make clean && make test-config-json $use_platform_flag" copy_files=("$output_file") ;; human) output_file="test-results.txt" - test_cmd="cd $BUILD_DIR && make test-config-full $use_platform_flag 2>&1 | tee $BUILD_DIR/$output_file" + test_cmd="cd $BUILD_DIR && make clean && make test-config-full $use_platform_flag 2>&1 | tee $BUILD_DIR/$output_file" copy_files=("$output_file") ;; esac diff --git a/tests/MAINTENANCE.md b/tests/MAINTENANCE.md index 41574ac..30036ef 100644 --- a/tests/MAINTENANCE.md +++ b/tests/MAINTENANCE.md @@ -62,15 +62,13 @@ SCHEMA_SOURCE="../../config-samples/ucentral.schema.pretty.json" # SCHEMA_SOURCE="../../ols-ucentral-schema/schema" # 2. Extract all properties from schema -# For JSON schema file: -python3 -c "import json; print('\n'.join(sorted(set( - k for d in json.load(open('$SCHEMA_SOURCE'))['properties'].values() - for k in d.get('properties', {}).keys() -))))" | sed 's/\[\]$//' > /tmp/all-schema-properties.txt +# For JSON schema file (recommended): +python3 extract-schema-properties.py ../../config-samples/ucentral.schema.pretty.json \ + 2>/dev/null > /tmp/all-schema-properties.txt # For YAML schema directory (if using ols-ucentral-schema repo): # python3 extract-schema-properties.py ../../ols-ucentral-schema/schema ucentral.yml \ -# 2>/dev/null | sed 's/\[\]$//' > /tmp/all-schema-properties.txt +# 2>/dev/null > /tmp/all-schema-properties.txt # 3. Generate base database (proto.c) python3 generate-database-from-schema.py \ @@ -78,21 +76,17 @@ python3 generate-database-from-schema.py \ /tmp/all-schema-properties.txt \ /tmp/base-database-new.c -# 4. Fix array name -sed -i '' 's/property_database\[\]/base_property_database[]/' \ - /tmp/base-database-new.c - -# 5. Generate platform database (plat-gnma.c) +# 4. Generate platform database (plat-gnma.c) python3 generate-platform-database-from-schema.py \ ../../src/ucentral-client/platform/brcm-sonic/plat-gnma.c \ /tmp/all-schema-properties.txt \ /tmp/platform-database-new.c -# 6. Install new databases +# 5. Install new databases cp /tmp/base-database-new.c ../config-parser/property-database-base.c cp /tmp/platform-database-new.c ../config-parser/property-database-platform-brcm-sonic.c -# 7. Test in Docker +# 6. Test in Docker docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make clean && make test-config-full" ``` @@ -127,16 +121,13 @@ Generates `property-database-base.c` from proto.c: ```bash cd tests/tools -# Extract schema properties from included JSON file (strip trailing []) -python3 -c "import json; print('\n'.join(sorted(set( - k for d in json.load(open('../../config-samples/ucentral.schema.pretty.json'))['properties'].values() - for k in d.get('properties', {}).keys() -))))" | sed 's/\[\]$//' > /tmp/schema-props.txt +# Extract schema properties from included JSON file +python3 extract-schema-properties.py ../../config-samples/ucentral.schema.pretty.json \ + 2>/dev/null > /tmp/schema-props.txt # OR if using YAML from ols-ucentral-schema repository: -# python3 extract-schema-properties.py \ -# ../../ols-ucentral-schema/schema ucentral.yml 2>/dev/null | \ -# sed 's/\[\]$//' > /tmp/schema-props.txt +# python3 extract-schema-properties.py ../../ols-ucentral-schema/schema ucentral.yml \ +# 2>/dev/null > /tmp/schema-props.txt # Generate database python3 generate-database-from-schema.py \ @@ -144,8 +135,7 @@ python3 generate-database-from-schema.py \ /tmp/schema-props.txt \ /tmp/base-db.c -# Fix array name and install -sed -i '' 's/property_database\[\]/base_property_database[]/' /tmp/base-db.c +# Install database cp /tmp/base-db.c ../config-parser/property-database-base.c ``` diff --git a/tests/README.md b/tests/README.md index 43a8daf..e308806 100644 --- a/tests/README.md +++ b/tests/README.md @@ -53,6 +53,25 @@ This testing framework includes multiple documentation files, each serving a spe ## Quick Reference +### Test Modes + +The testing framework supports two modes: + +**Stub Mode (Default - Fast)** +- Tests proto.c parsing only +- Uses simple platform stubs (test-stubs.c) +- Shows base properties only (proto.c) +- Fast execution (~30 seconds) +- Use for: Quick validation, CI/CD pipelines + +**Platform Mode (Integration)** +- Tests proto.c + platform implementation (plat-gnma.c) +- Uses platform code with hardware mocks +- Shows base AND platform properties (proto.c → plat-gnma.c) +- Tracks hardware application functions called +- Slower execution (~45 seconds) +- Use for: Platform-specific validation, integration testing + ### Running Tests **RECOMMENDED: Run tests inside Docker build environment** to eliminate OS-specific issues (works on macOS, Linux, Windows): @@ -61,10 +80,14 @@ This testing framework includes multiple documentation files, each serving a spe # Build the Docker environment first (if not already built) make build-host-env -# Run all tests (schema + parser) - RECOMMENDED +# Run all tests in STUB mode (default - fast) docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make test-config-full" +# Run all tests in PLATFORM mode (integration) +docker exec ucentral_client_build_env bash -c \ + "cd /root/ols-nos/tests/config-parser && make test-config-full USE_PLATFORM=brcm-sonic" + # Run individual test suites docker exec ucentral_client_build_env bash -c \ "cd /root/ols-nos/tests/config-parser && make validate-schema" @@ -93,18 +116,25 @@ docker cp ucentral_client_build_env:/root/ols-nos/tests/config-parser/test-resul # Navigate to test directory cd tests/config-parser -# Run all tests (schema + parser) +# Run all tests in STUB mode (default) make test-config-full +# Run all tests in PLATFORM mode +make test-config-full USE_PLATFORM=brcm-sonic + # Run individual test suites make validate-schema # Schema validation only make test-config # Parser tests only make test # Unit tests -# Generate test reports +# Generate test reports (stub mode) make test-config-html # HTML report (browser-viewable) make test-config-json # JSON report (machine-readable) make test-config-junit # JUnit XML (CI/CD integration) + +# Generate test reports (platform mode) +make test-config-html USE_PLATFORM=brcm-sonic +make test-config-json USE_PLATFORM=brcm-sonic ``` **Note:** Running tests in Docker is the preferred method as it provides a consistent, reproducible environment regardless of your host OS (macOS, Linux, Windows). @@ -114,7 +144,7 @@ make test-config-junit # JUnit XML (CI/CD integration) **Test Implementation:** - `tests/config-parser/test-config-parser.c` (3445 lines) - Parser test framework with property tracking - `tests/config-parser/test-stubs.c` (214 lines) - Platform function stubs for testing -- `tests/schema/validate-schema.py` (305 lines) - Standalone schema validator +- `tests/schema/validate-schema.py` (649 lines) - Standalone schema validator with undefined property detection - `tests/config-parser/config-parser.h` - Test header exposing cfg_parse() **Configuration Files:** @@ -150,7 +180,7 @@ make test-config-junit # JUnit XML (CI/CD integration) - Hardware constraint validation ### Property Tracking System -- Database of 450+ properties and their processing status +- Database of all schema properties and their implementation status (398 canonical properties) - Tracks which properties are parsed by which functions - Identifies unimplemented features - Status classification: CONFIGURED, IGNORED, SYSTEM, INVALID, Unknown @@ -206,8 +236,8 @@ The testing framework was added with minimal impact to production code: ### New Files Added 1. `tests/config-parser/test-config-parser.c` - Complete test framework (3445 lines) 2. `tests/config-parser/test-stubs.c` - Platform stubs (214 lines) -3. `tests/schema/validate-schema.py` - Schema validator (305 lines) -4. `src/ucentral-client/include/config-parser.h` - Test header +3. `tests/schema/validate-schema.py` - Schema validator (649 lines) +4. `tests/config-parser/config-parser.h` - Test header 5. `tests/config-parser/TEST_CONFIG_README.md` - Framework documentation 6. `tests/schema/SCHEMA_VALIDATOR_README.md` - Validator documentation 7. `tests/MAINTENANCE.md` - Maintenance procedures @@ -318,7 +348,7 @@ static struct property_info properties[] = { .status = PROP_CONFIGURED, .notes = "Enable/disable ethernet interface" }, - // ... 450+ more entries ... + // ... entries for all 398 schema properties ... }; ``` diff --git a/tests/config-parser/Makefile b/tests/config-parser/Makefile index eda3e27..bf70b8e 100644 --- a/tests/config-parser/Makefile +++ b/tests/config-parser/Makefile @@ -156,7 +156,7 @@ endif # Run schema validation validate-schema: @echo "========= Schema Validation =========" - @python3 $(SCHEMA_VALIDATOR) $(CONFIG_SAMPLES) || true + @python3 $(SCHEMA_VALIDATOR) $(CONFIG_SAMPLES) --check-undefined || true @echo "========= Schema validation completed ========" # Combined validation: schema + parser diff --git a/tests/schema/SCHEMA_VALIDATOR_README.md b/tests/schema/SCHEMA_VALIDATOR_README.md index 26d62f4..dff7d02 100644 --- a/tests/schema/SCHEMA_VALIDATOR_README.md +++ b/tests/schema/SCHEMA_VALIDATOR_README.md @@ -1,15 +1,21 @@ # uCentral Schema Validator -A modular, portable tool for validating JSON configuration files against the uCentral schema. +A modular, portable tool for validating JSON configuration files against the uCentral schema with advanced undefined property detection and typo suggestions. ## Features -- **Standalone Operation**: Works independently without external dependencies beyond Python 3 + jsonschema -- **Modular Design**: Easy to port to other repositories (platform-specific implementations, etc.) +- **Schema Validation**: Full JSON Schema Draft-7 validation (types, enums, constraints, etc.) +- **Undefined Property Detection**: Identifies properties in config not defined in schema +- **Smart Typo Detection**: Suggests corrections for likely misspellings + - Separator mismatches: `lldp_admin_status` → `lldp-admin-status` (underscore vs dash) + - Case mismatches: `lacpEnable` → `lacp-enable` (camelCase vs dash-case) + - Similar spelling: Edit distance analysis with confidence scoring - **Multiple Output Formats**: Human-readable and machine-readable JSON - **Directory Validation**: Validate entire directories of configs at once -- **CI/CD Ready**: Exit codes suitable for automated testing +- **CI/CD Ready**: Exit codes and strict mode for pipeline integration - **Schema Auto-Detection**: Automatically finds schema in common locations +- **Modular Design**: Easy to port to other repositories (platform-specific implementations, etc.) +- **Standalone Operation**: Works independently without external dependencies beyond Python 3 + jsonschema ## Installation @@ -28,24 +34,50 @@ pip3 install jsonschema ### Basic Usage ```bash -# Validate a single file +# Schema validation only (default behavior) python3 validate-schema.py config.json +# Check for undefined properties (informational warnings) +python3 validate-schema.py config.json --check-undefined + +# Strict mode: treat undefined properties as errors (for CI/CD) +python3 validate-schema.py config.json --strict-schema + # Validate all configs in a directory -python3 validate-schema.py ../../config-samples/ +python3 validate-schema.py ../../config-samples/ --check-undefined # Specify custom schema python3 validate-schema.py config.json --schema path/to/schema.json ``` +### Understanding Undefined Properties + +**Undefined properties are NOT validation errors** - they are informational warnings that help identify: + +1. **Typos/Misspellings**: Properties that won't be applied even though config is valid + - Example: `lldp_admin_status` instead of `lldp-admin-status` + - Example: `lacpEnable` instead of `lacp-enable` + +2. **Vendor-Specific Extensions**: ODM/vendor proprietary properties not in schema + - Risk: May change without notice, not portable across platforms + - Recommendation: Document and coordinate with schema maintainers + +3. **Deprecated Properties**: Properties removed from newer schema versions + - Check schema version compatibility + +**When to use each mode:** +- **Default mode** (no flags): Standard validation, undefined properties ignored +- **`--check-undefined`**: Development mode, see warnings but don't fail builds +- **`--strict-schema`**: CI/CD enforcement mode, fail on any undefined properties + ### Output Formats ```bash # Human-readable output (default) -python3 validate-schema.py config.json +python3 validate-schema.py config.json --check-undefined # Machine-readable JSON output -python3 validate-schema.py config.json --format json > report.json +python3 validate-schema.py config.json --check-undefined --format json > report.json ``` ### Via Makefile @@ -64,21 +96,58 @@ make test-config-full ## Exit Codes - `0`: All configurations are valid -- `1`: One or more configurations failed validation + - Undefined properties don't affect exit code unless `--strict-schema` is used +- `1`: Validation errors OR (strict mode AND undefined properties found) - `2`: File/schema errors (file not found, invalid schema, etc.) ## Output Examples -### Valid Configuration +### Valid Configuration (Schema Only) ``` -✓ Valid: cfg0.json +✓ Schema Valid: cfg0.json ``` -### Invalid Configuration +### Valid Configuration with Undefined Properties Check ``` -✗ Invalid: bad-config.json +✓ Schema Valid: cfg0.json +✓ All properties defined in schema +``` + +### Configuration with Undefined Properties + +``` +✓ Schema Valid: test_config.json + +ℹ️ Undefined Properties (informational): + Found 3 property/properties not in schema + These may be: + • Typos/misspellings (won't be applied even though config is valid) + • Vendor-specific extensions (not portable, may change) + • Deprecated properties (check schema version) + + 1. ethernet[].lldp_admin_status + → Not defined in schema + → Possible matches: + ✓ ethernet[].lldp-interface-config.lldp-admin-status (use '-' not '_') + + 2. ethernet[].lacpEnable + → Not defined in schema + → Possible matches: + ✓ ethernet[].lacp-config.lacp-enable (use dash-case not camelCase) + ? interfaces[].ipv4.ip-arp-inspect-vlan.vlan-enable (similar spelling) + + 3. ethernet[].custom-property + → Not defined in schema +``` + +**Note**: This config is schema-valid (exit code 0), but has informational warnings about undefined properties. + +### Invalid Configuration (Schema Errors) + +``` +✗ Schema Invalid: bad-config.json Found 2 validation error(s): Error 1: @@ -92,10 +161,13 @@ make test-config-full Validator: maximum ``` +**Note**: This config fails schema validation (exit code 1). + ### Directory Summary ``` Summary: 37 file(s) checked, 34 valid, 3 invalid + 5 file(s) with undefined properties ``` ## Integration with test-config-parser.c @@ -143,43 +215,133 @@ is_valid, errors = validator.validate_config(config) results = validator.validate_directory("/path/to/configs") ``` +## Typo Detection Features + +The validator includes intelligent typo detection that identifies naming mistakes and suggests corrections: + +### Separator Mismatches + +**Underscore vs Dash:** +``` +ethernet[].lldp_admin_status ❌ Underscore +ethernet[].lldp-admin-status ✓ Correct (dash-case) +``` + +**camelCase vs dash-case:** +``` +ethernet[].lacpEnable ❌ camelCase +ethernet[].lacp-enable ✓ Correct (dash-case) +``` + +### Similar Spelling + +Uses Levenshtein distance algorithm to find properties with similar spelling: +``` +services.logSettings.enabled ❌ Not in schema +services.log.enabled ✓ Possible match (edit distance: 1) +``` + +### Confidence Levels + +- **✓ High confidence**: Exact match after normalization (separator/case fix) +- **? Medium confidence**: Similar spelling (edit distance 2-3) +- **No suggestion**: No similar properties found (likely vendor-specific) + ## Common Validation Errors -### Type Errors +### Schema Validation Errors (Exit Code 1) +These are actual schema violations that must be fixed: + +**Type Errors:** ``` $.ethernet is not of type 'array' ``` - **Fix**: Ensure `ethernet` is an array: `"ethernet": [...]` -### Out of Range - +**Out of Range:** ``` $.interfaces[0].vlan.id is greater than the maximum of 4094 ``` - **Fix**: VLAN IDs must be between 1-4094 -### Required Property Missing - +**Required Property Missing:** ``` 'uuid' is a required property ``` - **Fix**: Add the required field: `"uuid": 1234567890` -### Additional Properties Not Allowed - +**Additional Properties Not Allowed:** ``` Additional properties are not allowed ('unknown_field' was unexpected) ``` +**Fix**: Remove the field or check spelling (only if schema has `additionalProperties: false`) -**Fix**: Remove the field or check spelling +### Undefined Properties (Exit Code 0 by default) + +These are informational warnings that don't cause validation failure: + +**Typo/Misspelling:** +``` +ℹ️ ethernet[].lldp_admin_status not in schema + Suggestion: ethernet[].lldp-interface-config.lldp-admin-status +``` +**Impact**: Property won't be applied even though config is valid + +**Vendor Extension:** +``` +ℹ️ ethernet[].edgecore-specific-property not in schema + No suggestions found +``` +**Impact**: Not portable, may change without notice + +## CI/CD Integration + +### Basic Pipeline + +```yaml +validate-configs: + stage: test + script: + # Standard validation (undefined properties are warnings) + - python3 validate-schema.py config-samples/ + artifacts: + when: on_failure + paths: + - validation-report.json +``` + +### Strict Enforcement Pipeline + +```yaml +validate-configs-strict: + stage: test + script: + # Strict mode: fail on undefined properties + - python3 validate-schema.py config-samples/ --strict-schema + # Generate JSON report for analysis + - python3 validate-schema.py config-samples/ --check-undefined --format json > report.json + artifacts: + always: + paths: + - report.json +``` + +### Development Workflow + +```bash +# Before committing: check for typos +python3 validate-schema.py my-config.json --check-undefined + +# Review suggestions and fix obvious typos +# Document any intentional vendor-specific properties + +# CI/CD will enforce with --strict-schema +``` ## Files -- **validate-schema.py**: Standalone schema validator script (305 lines) +- **validate-schema.py**: Standalone schema validator script (649 lines) - **Makefile**: Build targets for schema validation - **test-config-parser.c**: Enhanced with schema validation integration - **SCHEMA_VALIDATOR_README.md**: This documentation diff --git a/tests/schema/validate-schema.py b/tests/schema/validate-schema.py index 8c2cb3b..6ad3b42 100755 --- a/tests/schema/validate-schema.py +++ b/tests/schema/validate-schema.py @@ -6,9 +6,15 @@ A modular, standalone tool for validating JSON configuration files against the uCentral schema. Can be used independently or integrated into test suites. Usage: - # Validate a single file + # Validate a single file (schema validation only) ./validate-schema.py config.json + # Check for undefined properties (informational, doesn't affect exit code) + ./validate-schema.py config.json --check-undefined + + # Strict mode: treat undefined properties as errors (for CI/CD) + ./validate-schema.py config.json --strict-schema + # Validate with specific schema ./validate-schema.py config.json --schema path/to/schema.json @@ -18,7 +24,19 @@ Usage: # Machine-readable JSON output ./validate-schema.py config.json --format json - # Exit code: 0 = all valid, 1 = validation errors, 2 = file/schema errors +Exit codes: + 0 = all valid (undefined properties don't affect this unless --strict-schema) + 1 = validation errors OR (strict mode AND undefined properties found) + 2 = file/schema errors (file not found, invalid schema, etc.) + +Undefined Properties: + Properties in config but not defined in schema are INFORMATIONAL warnings, + not validation errors. They may indicate: + • Typos/misspellings (property won't be applied even though config is valid) + • Vendor-specific extensions (not portable across platforms) + • Deprecated properties (check schema version) + + Use --strict-schema in CI/CD pipelines to enforce schema compliance. Author: Generated for OLS uCentral Client License: BSD-3-Clause @@ -29,7 +47,8 @@ import json import argparse import os from pathlib import Path -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple, Optional, Set +from collections import defaultdict try: import jsonschema @@ -39,6 +58,84 @@ except ImportError: sys.exit(2) +def levenshtein_distance(s1: str, s2: str) -> int: + """ + Calculate Levenshtein distance between two strings. + Returns minimum number of single-character edits needed to change s1 into s2. + """ + if len(s1) < len(s2): + return levenshtein_distance(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + # Cost of insertions, deletions, or substitutions + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + +def normalize_separator(s: str, target: str = '-') -> str: + """Convert string separators. Default converts underscores and camelCase to dashes.""" + # Handle camelCase: insert dash before uppercase letters + result = [] + for i, char in enumerate(s): + if i > 0 and char.isupper() and s[i-1].islower(): + result.append('-') + result.append(char.lower()) + + normalized = ''.join(result) + # Replace underscores with target separator + normalized = normalized.replace('_', target) + return normalized + + +def detect_naming_issue(config_prop: str, schema_prop: str) -> Optional[Dict[str, str]]: + """ + Detect if config_prop is a likely typo/variation of schema_prop. + Returns dict with issue type and confidence, or None if no clear match. + """ + # Extract just the property name (last component after last dot or bracket) + def get_prop_name(path: str) -> str: + # Handle array notation: ethernet[].lldp-config -> lldp-config + if '.' in path: + return path.split('.')[-1] + return path + + config_name = get_prop_name(config_prop) + schema_name = get_prop_name(schema_prop) + + # Check for exact match after normalization + config_normalized = normalize_separator(config_name, '-') + schema_normalized = normalize_separator(schema_name, '-') + + if config_normalized == schema_normalized and config_name != schema_name: + # Separator mismatch (dash vs underscore vs camelCase) + if '_' in config_name and '-' in schema_name: + return {'type': 'separator_mismatch', 'detail': 'underscore_vs_dash', 'confidence': 'high'} + elif any(c.isupper() for c in config_name) and '-' in schema_name: + return {'type': 'separator_mismatch', 'detail': 'camelCase_vs_dash', 'confidence': 'high'} + else: + return {'type': 'separator_mismatch', 'detail': 'format_difference', 'confidence': 'high'} + + # Check Levenshtein distance + distance = levenshtein_distance(config_name.lower(), schema_name.lower()) + if distance <= 2: + return {'type': 'similar_spelling', 'detail': f'edit_distance_{distance}', 'confidence': 'high' if distance == 1 else 'medium'} + elif distance <= 3: + return {'type': 'similar_spelling', 'detail': f'edit_distance_{distance}', 'confidence': 'medium'} + + return None + + class SchemaValidator: """ Modular schema validator for uCentral configurations. @@ -47,17 +144,23 @@ class SchemaValidator: It has no dependencies on specific file paths or repository structure. """ - def __init__(self, schema_path: Optional[str] = None): + def __init__(self, schema_path: Optional[str] = None, check_undefined: bool = False, + similarity_threshold: int = 3): """ Initialize validator with schema. Args: schema_path: Path to JSON schema file. If None, attempts to find schema in common locations relative to this script. + check_undefined: If True, check for properties in config not defined in schema + similarity_threshold: Maximum Levenshtein distance for suggesting similar properties """ self.schema_path = schema_path self.schema = None self.validator = None + self.check_undefined = check_undefined + self.similarity_threshold = similarity_threshold + self._schema_properties = None # Cache of all valid schema property paths self._load_schema() def _find_default_schema(self) -> Optional[str]: @@ -98,7 +201,185 @@ class SchemaValidator: # Create validator self.validator = Draft7Validator(self.schema) - def validate_file(self, config_path: str) -> Tuple[bool, List[Dict]]: + def _extract_schema_properties(self, schema: Dict = None, path: str = "", visited: Set[str] = None) -> Set[str]: + """ + Recursively extract all valid property paths from the schema. + + Args: + schema: Schema object (or sub-schema) to process + path: Current path prefix + visited: Set of visited $ref paths to prevent infinite recursion + + Returns: + Set of all valid property paths in the schema + """ + if schema is None: + schema = self.schema + + if visited is None: + visited = set() + + properties = set() + + # Handle $ref references + if '$ref' in schema: + ref_path = schema['$ref'] + + # Prevent infinite recursion + if ref_path in visited: + return properties + visited.add(ref_path) + + # Resolve $ref (handle #/$defs/name references) + if ref_path.startswith('#/'): + ref_parts = ref_path[2:].split('/') + ref_schema = self.schema + for part in ref_parts: + ref_schema = ref_schema.get(part, {}) + + # Recursively extract from referenced schema + return self._extract_schema_properties(ref_schema, path, visited) + + # Handle object properties + if 'properties' in schema: + for prop_name, prop_schema in schema['properties'].items(): + prop_path = f"{path}.{prop_name}" if path else prop_name + properties.add(prop_path) + + # Recursively process nested properties + nested = self._extract_schema_properties(prop_schema, prop_path, visited.copy()) + properties.update(nested) + + # Handle arrays with items schema + if 'items' in schema: + items_schema = schema['items'] + # Use [] notation for arrays + array_path = f"{path}[]" if path else "[]" + + # Extract properties from array items + nested = self._extract_schema_properties(items_schema, array_path, visited.copy()) + properties.update(nested) + + # Handle additional properties (if true, allows any property) + # We don't add these to valid properties as they're wildcards + + return properties + + def _extract_config_properties(self, config: Dict, path: str = "") -> Set[str]: + """ + Recursively extract all property paths from a configuration object. + + Args: + config: Configuration object to analyze + path: Current path prefix + + Returns: + Set of all property paths in the configuration + """ + properties = set() + + if isinstance(config, dict): + for key, value in config.items(): + prop_path = f"{path}.{key}" if path else key + properties.add(prop_path) + + # Recursively process nested values + nested = self._extract_config_properties(value, prop_path) + properties.update(nested) + + elif isinstance(config, list): + # For arrays, use [] notation and process all items + array_path = f"{path}[]" if path else "[]" + + for item in config: + nested = self._extract_config_properties(item, array_path) + properties.update(nested) + + return properties + + def _find_similar_properties(self, config_prop: str, schema_props: Set[str]) -> List[Dict]: + """ + Find schema properties similar to a config property. + + Args: + config_prop: Property path from configuration + schema_props: Set of all valid schema property paths + + Returns: + List of suggestions with similarity information + """ + suggestions = [] + + for schema_prop in schema_props: + issue = detect_naming_issue(config_prop, schema_prop) + if issue: + suggestions.append({ + 'schema_property': schema_prop, + 'issue_type': issue['type'], + 'detail': issue['detail'], + 'confidence': issue['confidence'] + }) + + # Sort by confidence (high first) and then alphabetically + confidence_order = {'high': 0, 'medium': 1, 'low': 2} + suggestions.sort(key=lambda x: (confidence_order.get(x['confidence'], 3), x['schema_property'])) + + return suggestions + + def _check_undefined_properties(self, config: Dict) -> Dict[str, any]: + """ + Check for properties in config that are not defined in schema. + + Args: + config: Configuration object to check + + Returns: + Dict with undefined property analysis results + """ + # Extract all valid schema properties (with caching) + if self._schema_properties is None: + self._schema_properties = self._extract_schema_properties() + + # Extract all config properties + config_props = self._extract_config_properties(config) + + # Find undefined properties + undefined = [] + + for config_prop in sorted(config_props): + # Check if property or its array form exists in schema + is_defined = False + + # Direct match + if config_prop in self._schema_properties: + is_defined = True + else: + # Check with array index normalization: ethernet[0].speed -> ethernet[].speed + normalized_prop = config_prop + import re + # Replace array indices with [] + normalized_prop = re.sub(r'\[\d+\]', '[]', normalized_prop) + + if normalized_prop in self._schema_properties: + is_defined = True + + if not is_defined: + # Find similar properties + suggestions = self._find_similar_properties(config_prop, self._schema_properties) + + undefined.append({ + 'path': config_prop, + 'suggestions': suggestions[:3] # Top 3 suggestions + }) + + return { + 'total_config_properties': len(config_props), + 'total_schema_properties': len(self._schema_properties), + 'undefined_count': len(undefined), + 'undefined_properties': undefined + } + + def validate_file(self, config_path: str) -> Tuple[bool, List[Dict], Optional[Dict]]: """Validate a single configuration file against the schema.""" try: with open(config_path, 'r') as f: @@ -108,18 +389,26 @@ class SchemaValidator: 'path': '$', 'message': f'Invalid JSON: {e}', 'validator': 'json_parse' - }] + }], None except FileNotFoundError: return False, [{ 'path': '$', 'message': f'File not found: {config_path}', 'validator': 'file_access' - }] + }], None return self.validate_config(config) - def validate_config(self, config: Dict) -> Tuple[bool, List[Dict]]: - """Validate a configuration object against the schema.""" + def validate_config(self, config: Dict) -> Tuple[bool, List[Dict], Optional[Dict]]: + """ + Validate a configuration object against the schema. + + Returns: + Tuple of (is_valid, errors, undefined_analysis) + - is_valid: True if no schema validation errors + - errors: List of schema validation errors + - undefined_analysis: Dict with undefined property analysis (if check_undefined=True) + """ errors = [] for error in sorted(self.validator.iter_errors(config), key=str): @@ -133,9 +422,14 @@ class SchemaValidator: 'schema_path': '.'.join(str(p) for p in error.absolute_schema_path) if error.absolute_schema_path else '$' }) - return len(errors) == 0, errors + # Check for undefined properties if enabled + undefined_analysis = None + if self.check_undefined: + undefined_analysis = self._check_undefined_properties(config) - def validate_directory(self, dir_path: str, pattern: str = "*.json") -> Dict[str, Tuple[bool, List[Dict]]]: + return len(errors) == 0, errors, undefined_analysis + + def validate_directory(self, dir_path: str, pattern: str = "*.json") -> Dict[str, Tuple[bool, List[Dict], Optional[Dict]]]: """Validate all JSON files in a directory.""" results = {} dir_path_obj = Path(dir_path) @@ -154,14 +448,16 @@ class SchemaValidator: return results -def format_human_output(filename: str, is_valid: bool, errors: List[Dict]) -> str: +def format_human_output(filename: str, is_valid: bool, errors: List[Dict], + undefined_analysis: Optional[Dict] = None) -> str: """Format validation results in human-readable format.""" output = [] + # Schema validation results if is_valid: - output.append(f"✓ Valid: {filename}") + output.append(f"✓ Schema Valid: {filename}") else: - output.append(f"✗ Invalid: {filename}") + output.append(f"✗ Schema Invalid: {filename}") output.append(f" Found {len(errors)} validation error(s):") for i, error in enumerate(errors, 1): @@ -171,26 +467,69 @@ def format_human_output(filename: str, is_valid: bool, errors: List[Dict]) -> st if error.get('validator'): output.append(f" Validator: {error['validator']}") + # Undefined properties analysis (informational warnings, not errors) + if undefined_analysis and undefined_analysis['undefined_count'] > 0: + output.append(f"\nℹ️ Undefined Properties (informational):") + output.append(f" Found {undefined_analysis['undefined_count']} property/properties not in schema") + output.append(f" These may be:") + output.append(f" • Typos/misspellings (won't be applied even though config is valid)") + output.append(f" • Vendor-specific extensions (not portable, may change)") + output.append(f" • Deprecated properties (check schema version)\n") + + for i, item in enumerate(undefined_analysis['undefined_properties'], 1): + output.append(f" {i}. {item['path']}") + output.append(f" → Not defined in schema") + + if item['suggestions']: + output.append(f" → Possible matches:") + for suggestion in item['suggestions']: + confidence_icon = "✓" if suggestion['confidence'] == 'high' else "?" + detail_msg = "" + if suggestion['issue_type'] == 'separator_mismatch': + if 'underscore_vs_dash' in suggestion['detail']: + detail_msg = " (use '-' not '_')" + elif 'camelCase_vs_dash' in suggestion['detail']: + detail_msg = " (use dash-case not camelCase)" + elif suggestion['issue_type'] == 'similar_spelling': + detail_msg = f" (similar spelling)" + + output.append(f" {confidence_icon} {suggestion['schema_property']}{detail_msg}") + output.append("") # Blank line between items + + elif undefined_analysis and undefined_analysis['undefined_count'] == 0: + output.append(f"✓ All properties defined in schema") + return '\n'.join(output) -def format_json_output(results: Dict[str, Tuple[bool, List[Dict]]]) -> str: +def format_json_output(results: Dict[str, Tuple[bool, List[Dict], Optional[Dict]]]) -> str: """Format validation results as JSON.""" output = { 'summary': { 'total': len(results), - 'valid': sum(1 for is_valid, _ in results.values() if is_valid), - 'invalid': sum(1 for is_valid, _ in results.values() if not is_valid) + 'valid': sum(1 for is_valid, _, _ in results.values() if is_valid), + 'invalid': sum(1 for is_valid, _, _ in results.values() if not is_valid), + 'with_undefined_properties': sum(1 for _, _, undefined in results.values() + if undefined and undefined['undefined_count'] > 0) }, 'results': {} } - for filename, (is_valid, errors) in results.items(): - output['results'][filename] = { - 'valid': is_valid, + for filename, (is_valid, errors, undefined_analysis) in results.items(): + result_data = { + 'schema_valid': is_valid, 'errors': errors } + if undefined_analysis: + result_data['schema_compliance'] = { + 'total_config_properties': undefined_analysis['total_config_properties'], + 'undefined_count': undefined_analysis['undefined_count'], + 'undefined_properties': undefined_analysis['undefined_properties'] + } + + output['results'][filename] = result_data + return json.dumps(output, indent=2) @@ -219,12 +558,26 @@ Examples: parser.add_argument('--pattern', '-p', default='*.json', help='File pattern for directory validation (default: *.json)') + parser.add_argument('--check-undefined', '-u', + action='store_true', + help='Check for properties not defined in schema (informational, does not affect exit code)') + parser.add_argument('--strict-schema', + action='store_true', + help='Treat undefined properties as errors (exit code 1). Use for CI/CD enforcement. (implies --check-undefined)') + parser.add_argument('--similarity-threshold', '-t', + type=int, + default=3, + help='Maximum edit distance for suggesting similar properties (default: 3)') args = parser.parse_args() + # --strict-schema implies --check-undefined + check_undefined = args.check_undefined or args.strict_schema + # Initialize validator try: - validator = SchemaValidator(args.schema) + validator = SchemaValidator(args.schema, check_undefined=check_undefined, + similarity_threshold=args.similarity_threshold) except (FileNotFoundError, ValueError) as e: print(f"ERROR: {e}", file=sys.stderr) return 2 @@ -240,8 +593,8 @@ Examples: results = {} if path_obj.is_file(): - is_valid, errors = validator.validate_file(args.path) - results[path_obj.name] = (is_valid, errors) + is_valid, errors, undefined_analysis = validator.validate_file(args.path) + results[path_obj.name] = (is_valid, errors, undefined_analysis) elif path_obj.is_dir(): try: results = validator.validate_directory(args.path, args.pattern) @@ -256,19 +609,40 @@ Examples: if args.format == 'json': print(format_json_output(results)) else: - for filename, (is_valid, errors) in results.items(): - print(format_human_output(filename, is_valid, errors)) + for filename, (is_valid, errors, undefined_analysis) in results.items(): + print(format_human_output(filename, is_valid, errors, undefined_analysis)) print() # Blank line between files # Summary for multiple files if len(results) > 1: - valid_count = sum(1 for is_valid, _ in results.values() if is_valid) + valid_count = sum(1 for is_valid, _, _ in results.values() if is_valid) invalid_count = len(results) - valid_count - print(f"Summary: {len(results)} file(s) checked, {valid_count} valid, {invalid_count} invalid") + undefined_count = sum(1 for _, _, undefined in results.values() + if undefined and undefined['undefined_count'] > 0) - # Exit code: 0 if all valid, 1 if any invalid - all_valid = all(is_valid for is_valid, _ in results.values()) - return 0 if all_valid else 1 + print(f"Summary: {len(results)} file(s) checked, {valid_count} valid, {invalid_count} invalid") + if check_undefined: + print(f" {undefined_count} file(s) with undefined properties") + + # Exit code logic + # Only schema validation errors cause failure by default + all_valid = all(is_valid for is_valid, _, _ in results.values()) + + # In strict mode, undefined properties also cause failure + has_undefined = False + if args.strict_schema: + has_undefined = any(undefined and undefined['undefined_count'] > 0 + for _, _, undefined in results.values()) + + # Exit codes: + # 0 = all valid (undefined properties don't affect this unless --strict-schema) + # 1 = validation errors OR (strict mode AND undefined properties found) + if not all_valid: + return 1 + elif args.strict_schema and has_undefined: + return 1 + + return 0 if __name__ == '__main__':