From 33ae4d92ef4f30767ade6b595736012ed12f971f Mon Sep 17 00:00:00 2001 From: Elizabeth Maher Date: Fri, 5 Jun 2026 16:06:01 -0700 Subject: [PATCH 1/7] fix: strip schemaId/typeName from operation PATCH during reconciliation After spec import, APIM assigns its own auto-generated schema IDs to operation request/response representations. The reconciliation PATCH was sending the source instance's schema IDs, which don't exist on the target. APIM silently drops schemaId/typeName when the referenced schema doesn't exist, causing the compare phase to report missing fields. By stripping these fields from the PATCH body, APIM retains the schema references it assigned during its own spec import. The compare script already normalizes auto-generated hex IDs, so both sides match. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/services/api-publisher.ts | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index e7a23c3..01a663e 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -434,6 +434,13 @@ async function reconcileOperationsAfterSpecImport( } } + // Strip schemaId/typeName from representations in request/responses. + // These reference source-specific auto-generated schema IDs that won't exist + // on the target after a fresh spec import. APIM re-links representations to + // its own schema IDs during import, so sending stale source IDs causes APIM + // to silently drop the fields. + stripRepresentationSchemaRefs(patchProps); + if (Object.keys(patchProps).length === 0) return; const patchBody: Record = { properties: patchProps }; @@ -456,6 +463,47 @@ async function reconcileOperationsAfterSpecImport( } } +/** + * Strip schemaId and typeName from all representations in request/responses. + * After a spec import, APIM assigns its own auto-generated schema IDs to + * operation representations. The persisted JSON references the *source* instance's + * schema IDs, which don't exist on the target. Sending them in a PATCH causes APIM + * to silently drop the fields. By stripping them, we let APIM keep its own + * freshly-assigned schema references intact while still reconciling other metadata + * (displayName, description, templateParameters, etc.). + */ +function stripRepresentationSchemaRefs(patchProps: Record): void { + const SCHEMA_REF_FIELDS = ['schemaId', 'typeName']; + + function stripFromRepresentations(representations: unknown): void { + if (!Array.isArray(representations)) return; + for (const rep of representations) { + if (rep && typeof rep === 'object') { + for (const field of SCHEMA_REF_FIELDS) { + delete (rep as Record)[field]; + } + } + } + } + + // Strip from request.representations + const request = patchProps.request; + if (request && typeof request === 'object') { + const req = request as Record; + stripFromRepresentations(req.representations); + } + + // Strip from responses[].representations + const responses = patchProps.responses; + if (Array.isArray(responses)) { + for (const response of responses) { + if (response && typeof response === 'object') { + stripFromRepresentations((response as Record).representations); + } + } + } +} + /** * Extract revision number from API name (e.g., "my-api;rev=2" -> 2) */ From 40f213136f2d37e2a345f9b65d298f8531ae97fc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 23:13:40 +0000 Subject: [PATCH 2/7] chore: make reconciliation comments terse --- src/services/api-publisher.ts | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index 01a663e..bab8de3 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -434,11 +434,7 @@ async function reconcileOperationsAfterSpecImport( } } - // Strip schemaId/typeName from representations in request/responses. - // These reference source-specific auto-generated schema IDs that won't exist - // on the target after a fresh spec import. APIM re-links representations to - // its own schema IDs during import, so sending stale source IDs causes APIM - // to silently drop the fields. + // Drop source schema refs; APIM rebinds these on spec import. stripRepresentationSchemaRefs(patchProps); if (Object.keys(patchProps).length === 0) return; @@ -463,15 +459,7 @@ async function reconcileOperationsAfterSpecImport( } } -/** - * Strip schemaId and typeName from all representations in request/responses. - * After a spec import, APIM assigns its own auto-generated schema IDs to - * operation representations. The persisted JSON references the *source* instance's - * schema IDs, which don't exist on the target. Sending them in a PATCH causes APIM - * to silently drop the fields. By stripping them, we let APIM keep its own - * freshly-assigned schema references intact while still reconciling other metadata - * (displayName, description, templateParameters, etc.). - */ +/** Remove schema refs from request/response representations before PATCH. */ function stripRepresentationSchemaRefs(patchProps: Record): void { const SCHEMA_REF_FIELDS = ['schemaId', 'typeName']; @@ -486,14 +474,14 @@ function stripRepresentationSchemaRefs(patchProps: Record): voi } } - // Strip from request.representations + // request.representations const request = patchProps.request; if (request && typeof request === 'object') { const req = request as Record; stripFromRepresentations(req.representations); } - // Strip from responses[].representations + // responses[].representations const responses = patchProps.responses; if (Array.isArray(responses)) { for (const response of responses) { From 37babf0375bda10fffe1f4fa07cc8751d389e336 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 23:14:31 +0000 Subject: [PATCH 3/7] docs: tighten reconciliation comments --- src/services/api-publisher.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index bab8de3..72bb736 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -434,7 +434,7 @@ async function reconcileOperationsAfterSpecImport( } } - // Drop source schema refs; APIM rebinds these on spec import. + // Strip source schema refs; APIM rebinds on import and drops stale IDs. stripRepresentationSchemaRefs(patchProps); if (Object.keys(patchProps).length === 0) return; @@ -459,7 +459,7 @@ async function reconcileOperationsAfterSpecImport( } } -/** Remove schema refs from request/response representations before PATCH. */ +/** Strip source schema refs from request/response representations before PATCH. */ function stripRepresentationSchemaRefs(patchProps: Record): void { const SCHEMA_REF_FIELDS = ['schemaId', 'typeName']; @@ -474,14 +474,14 @@ function stripRepresentationSchemaRefs(patchProps: Record): voi } } - // request.representations + // Strip schema refs from request.representations. const request = patchProps.request; if (request && typeof request === 'object') { const req = request as Record; stripFromRepresentations(req.representations); } - // responses[].representations + // Strip schema refs from responses[].representations. const responses = patchProps.responses; if (Array.isArray(responses)) { for (const response of responses) { From 20c8eb92f2d901af51be5b7829672402734e1b9b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 23:37:14 +0000 Subject: [PATCH 4/7] docs(squad): record schema-ref reconciliation decision --- .squad/decisions.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.squad/decisions.md b/.squad/decisions.md index 6e575ee..bb6c42b 100644 --- a/.squad/decisions.md +++ b/.squad/decisions.md @@ -2,6 +2,12 @@ ## Active Decisions +### 2026-06-05T23:40:00Z: Operation reconciliation PATCH strips schema refs +**By:** ApimExpert + TypeScriptDev +**Status:** Implemented +**What:** `reconcileOperationsAfterSpecImport` removes `schemaId` and `typeName` from operation `request.representations` and `responses[].representations` before PATCH. +**Why:** Those IDs are source-instance specific. After target spec import, APIM assigns its own schema IDs; sending stale source IDs causes APIM to drop schema refs. + ### 2026-05-28T23:06:01Z: Team-Wide Evidence Standard **By:** User directive (anonymized) **Status:** Active directive From f48ea54fc1df8cbee2253631ac57e1364cf4792b Mon Sep 17 00:00:00 2001 From: Elizabeth Maher Date: Mon, 8 Jun 2026 18:46:03 +0000 Subject: [PATCH 5/7] updating cmd to create log file --- tests/integration/all-resource-types/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/all-resource-types/README.md b/tests/integration/all-resource-types/README.md index 2e08954..c5acfd0 100644 --- a/tests/integration/all-resource-types/README.md +++ b/tests/integration/all-resource-types/README.md @@ -23,12 +23,13 @@ cd tests/integration/all-resource-types #### Bash ```bash -set -o pipefail && log_file="tests/integration/all-resource-types/phases/logs/roundtrip-premium-$(date +%Y%m%d-%H%M%S).log" && echo "Logging to $log_file" && pwsh -NoLogo -NoProfile -File ./tests/integration/all-resource-types/run-roundtrip-test.ps1 -SkuName Premium 2>&1 | tee "$log_file" +set -o pipefail && log_file="tests/integration/all-resource-types/phases/logs/roundtrip-premium-$(date +%Y%m%d-%H%M%S).log" && mkdir -p "$(dirname "$log_file")" && echo "Logging to $log_file" && pwsh -NoLogo -NoProfile -File ./tests/integration/all-resource-types/run-roundtrip-test.ps1 -SkuName Premium 2>&1 | tee "$log_file" ``` #### Powershell ```powershell $logFile = "tests/integration/all-resource-types/phases/logs/roundtrip-premium-$((Get-Date).ToString('yyyyMMdd-HHmmss')).log" +New-Item -ItemType Directory -Path (Split-Path -Parent $logFile) -Force | Out-Null Write-Host "Logging to $logFile" .\tests\integration\all-resource-types\run-roundtrip-test.ps1 -SkuName Premium 2>&1 | Tee-Object -FilePath $logFile if ($LASTEXITCODE -ne 0) { throw "Round-trip failed with exit code $LASTEXITCODE. See $logFile" } From 38145fee9906f44f69389adbf59eed785298a102 Mon Sep 17 00:00:00 2001 From: Elizabeth Maher Date: Mon, 8 Jun 2026 18:58:07 +0000 Subject: [PATCH 6/7] updating squad files --- .squad/decisions.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.squad/decisions.md b/.squad/decisions.md index bb6c42b..15a5c6f 100644 --- a/.squad/decisions.md +++ b/.squad/decisions.md @@ -2,6 +2,18 @@ ## Active Decisions +### 2026-06-08T18:52:23Z: Long-Term Maintainability Over Quick Fixes +**By:** User directive (anonymized) +**Status:** Active directive +**What:** When fixing code in tests or product logic, avoid quick/easy fixes and prefer solutions optimized for long-term maintainability. +**Why:** Establishes a durable engineering quality standard and reduces short-lived patchwork changes. + +### 2026-06-08T18:50:20Z: Round-Trip Test Default Execution Standard +**By:** User directive (anonymized) +**Status:** Active directive +**What:** When the user asks to run the round-trip test, follow the workflow documented in `tests/integration/all-resource-types/README.md` Quick Commands, default to StandardV2 SKU, and always save the log. Use Premium SKU only when branch-specific changes require Premium-only coverage. +**Why:** Enforces consistent execution defaults and reliable log capture for repeatable integration test runs. + ### 2026-06-05T23:40:00Z: Operation reconciliation PATCH strips schema refs **By:** ApimExpert + TypeScriptDev **Status:** Implemented From 257dc6db7b534bfb7123148df0ae89822652b7e1 Mon Sep 17 00:00:00 2001 From: Elizabeth Maher Date: Mon, 8 Jun 2026 18:58:31 +0000 Subject: [PATCH 7/7] skipping schema ids, doing semantic compare --- .../Compare-ApimInstance.ps1 | 327 +++++--------- .../modules/CompareSemantics.psm1 | 410 ++++++++++++++++++ 2 files changed, 512 insertions(+), 225 deletions(-) create mode 100644 tests/integration/all-resource-types/modules/CompareSemantics.psm1 diff --git a/tests/integration/all-resource-types/Compare-ApimInstance.ps1 b/tests/integration/all-resource-types/Compare-ApimInstance.ps1 index 1be1b28..8c4ca37 100644 --- a/tests/integration/all-resource-types/Compare-ApimInstance.ps1 +++ b/tests/integration/all-resource-types/Compare-ApimInstance.ps1 @@ -28,6 +28,8 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'modules/CompareSemantics.psm1') -Force + # ── Constants ─────────────────────────────────────────────────────────────── # Use the newest APIM ARM API version so resource types introduced in newer @@ -57,12 +59,26 @@ $RequestResponseIgnoredProperties = @('description') # Properties ignored on representation objects (have 'contentType' or 'schemaId'): # - description: SOAP/WSDL import generates descriptions that vary -# - schemaId/typeName: For SOAP APIs whose operations have auto-generated IDs, -# PATCH reconciliation is skipped and these fields may differ. For REST APIs, -# PATCH reconciliation restores them after spec import, so they should match. -# Ignored conditionally only for SOAP-like auto-generated operation IDs. +# - schemaId/typeName: Operation reconciliation strips these before PATCH because +# APIM rebinds representation schema refs during import. Values are therefore +# not stable for round-trip comparison and are ignored for operation resources. $RepresentationIgnoredProperties = @('description') -$SoapAutoGeneratedRepresentationIgnoredProperties = @('schemaId', 'typeName') +$RepresentationSchemaRefIgnoredProperties = @('schemaId', 'typeName') + +# Cache of normalized API schema semantics per instance/api, keyed as: +# "{instance}|{apiName}" => @{ schemaId => normalizedSchemaJson } +$ApiSchemaSemanticCache = @{} + +$NormalizationContext = New-CompareNormalizationContext ` + -SourceName $SourceApimName -TargetName $TargetApimName ` + -SourceSub $SourceSubscriptionId -TargetSub $TargetSubscriptionId ` + -SourceRg $SourceResourceGroup -TargetRg $TargetResourceGroup ` + -StripTopLevelFields $StripTopLevelFields ` + -StripReadOnlyProperties $StripReadOnlyProperties ` + -StripTimestampProperties $StripTimestampProperties ` + -RequestResponseIgnoredProperties $RequestResponseIgnoredProperties ` + -RepresentationIgnoredProperties $RepresentationIgnoredProperties ` + -RepresentationSchemaRefIgnoredProperties $RepresentationSchemaRefIgnoredProperties # ── Helpers ───────────────────────────────────────────────────────────────── @@ -120,6 +136,49 @@ function Get-ResourceName { return ($ResourceId -split '/')[-1] } +function Copy-JsonObject { + <# Deep-clones a PSCustomObject/hashtable via JSON round-trip. #> + param([Parameter(Mandatory)] $Value) + return ($Value | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100) +} + +function Get-ApiSchemaSemanticMap { + <# + .SYNOPSIS + Returns a map of { schemaId => normalized schema JSON } for one API. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $InstanceKey, + [Parameter(Mandatory)] [string] $BaseUrl, + [Parameter(Mandatory)] [string] $ApiName + ) + + $cacheKey = "$InstanceKey|$ApiName" + if ($ApiSchemaSemanticCache.Contains($cacheKey)) { + return $ApiSchemaSemanticCache[$cacheKey] + } + + $schemaMap = @{} + try { + $schemas = Get-ArmResourceList -Url "$BaseUrl/apis/$ApiName/schemas" + foreach ($schema in $schemas) { + $schemaId = Get-ResourceName -ResourceId $schema.id + if (-not $schemaId) { continue } + + $schemaNorm = ConvertTo-NormalizedResource -Resource $schema + $schemaSemantics = $schemaNorm | ConvertTo-Json -Depth 50 -Compress + $schemaMap[$schemaId] = $schemaSemantics + } + } + catch { + Write-Verbose "Could not load schemas for API $ApiName on $InstanceKey — $_" + } + + $ApiSchemaSemanticCache[$cacheKey] = $schemaMap + return $schemaMap +} + function Build-ResourceMap { <# .SYNOPSIS @@ -145,43 +204,10 @@ function Build-ResourceMap { [Parameter(Mandatory)] [string] $TargetRg ) - $map = [ordered]@{} - - # Handle empty or null input - if ($null -eq $Items -or $Items.Count -eq 0) { - return $map - } - - $autoIdItems = [System.Collections.Generic.List[object]]::new() - - foreach ($item in $Items) { - $rName = Get-ResourceName -ResourceId $item.id - if ($rName -in $ExcludeNames) { continue } - - # Auto-generated IDs: 24-char lowercase hex OR UUID format (8-4-4-4-12) - # These get regenerated on publish, so key by normalised content instead - if ($rName -match '^[0-9a-f]{24}$' -or $rName -match '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') { - $autoIdItems.Add($item) - } else { - $map[$rName] = $item - } - } - - # Sort auto-ID items by canonical normalized resource JSON (same normalization - # pipeline used for diffing) so positional keys remain stable across instances. - if ($autoIdItems.Count -gt 0) { - $sorted = $autoIdItems | Sort-Object { - $normResource = ConvertTo-NormalizedResource -Resource $_ - $normResource | ConvertTo-Json -Depth 50 -Compress - } - $i = 0 - foreach ($item in $sorted) { - $map["{{auto-id-$i}}"] = $item - $i++ - } + return CompareSemantics\Build-ResourceMap -Items $Items -ExcludeNames $ExcludeNames -NormalizeResource { + param($resource) + ConvertTo-NormalizedResource -Resource $resource } - - return $map } function ConvertTo-NormalizedPropertyValue { @@ -203,95 +229,7 @@ function ConvertTo-NormalizedPropertyValue { [switch] $IgnoreRepresentationSchemaRefs ) - if ($null -eq $Value) { return $null } - - # --- String --- - if ($Value -is [string]) { - $s = $Value - # Normalize ARM resource-ID paths (subscription, RG, service name) - $s = $s -replace [regex]::Escape("/subscriptions/$SourceSub/resourceGroups/$SourceRg/providers/Microsoft.ApiManagement/service/$SourceName"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}/providers/Microsoft.ApiManagement/service/{{apim-name}}' - $s = $s -replace [regex]::Escape("/subscriptions/$TargetSub/resourceGroups/$TargetRg/providers/Microsoft.ApiManagement/service/$TargetName"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}/providers/Microsoft.ApiManagement/service/{{apim-name}}' - # Broader subscription/RG normalization for other resource types - $s = $s -replace [regex]::Escape("/subscriptions/$SourceSub/resourceGroups/$SourceRg"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}' - $s = $s -replace [regex]::Escape("/subscriptions/$TargetSub/resourceGroups/$TargetRg"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}' - $s = $s -replace [regex]::Escape("/subscriptions/$SourceSub"), '/subscriptions/{{sub}}' - $s = $s -replace [regex]::Escape("/subscriptions/$TargetSub"), '/subscriptions/{{sub}}' - # Neutralize service name in any remaining positions - $s = $s -replace [regex]::Escape($SourceName), '{{apim-name}}' - $s = $s -replace [regex]::Escape($TargetName), '{{apim-name}}' - # Normalize Key Vault URIs — different vault names per RG - $s = $s -replace 'https://[a-zA-Z0-9-]+\.vault\.azure\.net', 'https://{{keyvault}}.vault.azure.net' - # Normalize Key Vault secret names (src-* vs tgt-*) - $s = $s -replace '/secrets/(src|tgt)-', '/secrets/{{prefix}}-' - # Normalize App Insights resource IDs (different AI instance names per RG) - $s = $s -replace '/providers/Microsoft\.Insights/components/[a-zA-Z0-9-]+', '/providers/Microsoft.Insights/components/{{appinsights}}' - # Normalize Event Hub namespace names in resource IDs - $s = $s -replace '/providers/Microsoft\.EventHub/namespaces/[a-zA-Z0-9-]+', '/providers/Microsoft.EventHub/namespaces/{{eventhub}}' - # Normalize auto-generated APIM IDs (24-char hex strings like schema IDs, named value IDs) - $s = $s -replace '\b[0-9a-f]{24}\b', '{{auto-id}}' - # Normalize GUIDs - $s = $s -replace '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '{{guid}}' - return $s - } - - # --- Array --- - if ($Value -is [System.Collections.IEnumerable] -and $Value -isnot [string] -and $Value -isnot [System.Collections.IDictionary]) { - $normalized = @(foreach ($item in $Value) { - ConvertTo-NormalizedPropertyValue -Value $item ` - -SourceName $SourceName -TargetName $TargetName ` - -SourceSub $SourceSub -TargetSub $TargetSub ` - -SourceRg $SourceRg -TargetRg $TargetRg ` - -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs - }) - # Sort for order-independent comparison - $sorted = $normalized | Sort-Object { ($_ | ConvertTo-Json -Depth 50 -Compress) } - return @($sorted) - } - - # --- Object / Hashtable --- - if ($Value -is [System.Collections.IDictionary]) { - $out = [ordered]@{} - # Detect request/response objects (have 'representations' array) and representation items - $isRequestResponse = $Value.Contains('representations') - $isRepresentation = $Value.Contains('contentType') -or $Value.Contains('schemaId') - foreach ($key in ($Value.Keys | Sort-Object)) { - if ($IsRoot -and $key -in $StripReadOnlyProperties) { continue } - if ($key -in $StripTimestampProperties) { continue } # Strip timestamps at any level - if ($isRequestResponse -and $key -in $RequestResponseIgnoredProperties) { continue } - if ($isRepresentation -and $key -in $RepresentationIgnoredProperties) { continue } - if ($IgnoreRepresentationSchemaRefs -and $isRepresentation -and $key -in $SoapAutoGeneratedRepresentationIgnoredProperties) { continue } - $out[$key] = ConvertTo-NormalizedPropertyValue -Value $Value[$key] ` - -SourceName $SourceName -TargetName $TargetName ` - -SourceSub $SourceSub -TargetSub $TargetSub ` - -SourceRg $SourceRg -TargetRg $TargetRg ` - -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs - } - return $out - } - - # PSCustomObject (from ConvertFrom-Json) — NOT primitives which also have PSObject - if ($Value -is [PSCustomObject]) { - $out = [ordered]@{} - # Detect request/response objects (have 'representations' array) and representation items - $isRequestResponse = $null -ne ($Value.PSObject.Properties | Where-Object { $_.Name -eq 'representations' }) - $isRepresentation = $null -ne ($Value.PSObject.Properties | Where-Object { $_.Name -eq 'contentType' -or $_.Name -eq 'schemaId' }) - foreach ($prop in ($Value.PSObject.Properties | Sort-Object Name)) { - if ($IsRoot -and $prop.Name -in $StripReadOnlyProperties) { continue } - if ($prop.Name -in $StripTimestampProperties) { continue } # Strip timestamps at any level - if ($isRequestResponse -and $prop.Name -in $RequestResponseIgnoredProperties) { continue } - if ($isRepresentation -and $prop.Name -in $RepresentationIgnoredProperties) { continue } - if ($IgnoreRepresentationSchemaRefs -and $isRepresentation -and $prop.Name -in $SoapAutoGeneratedRepresentationIgnoredProperties) { continue } - $out[$prop.Name] = ConvertTo-NormalizedPropertyValue -Value $prop.Value ` - -SourceName $SourceName -TargetName $TargetName ` - -SourceSub $SourceSub -TargetSub $TargetSub ` - -SourceRg $SourceRg -TargetRg $TargetRg ` - -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs - } - return $out - } - - # Primitive (int, bool, etc.) - return $Value + return CompareSemantics\ConvertTo-NormalizedPropertyValue -Value $Value -Context $NormalizationContext -IsRoot:$IsRoot -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs } function ConvertTo-NormalizedResource { @@ -304,54 +242,19 @@ function ConvertTo-NormalizedResource { [Parameter(Mandatory)] $Resource ) - $resourceId = if ($Resource.PSObject.Properties['id']) { [string] $Resource.id } else { '' } + $resourceId = if ($Resource.PSObject.Properties['id']) { [string]$Resource.id } else { '' } $ignoreRepresentationSchemaRefs = Test-ShouldIgnoreRepresentationSchemaRefs -ResourceId $resourceId - - $clone = [ordered]@{} - foreach ($prop in $Resource.PSObject.Properties) { - if ($prop.Name -in $StripTopLevelFields) { continue } - $clone[$prop.Name] = $prop.Value - } - - # Normalize the properties bag (read-only fields, instance names, etc.) - if ($clone.Contains('properties')) { - $clone['properties'] = ConvertTo-NormalizedPropertyValue -Value $clone['properties'] ` - -SourceName $SourceApimName -TargetName $TargetApimName ` - -SourceSub $SourceSubscriptionId -TargetSub $TargetSubscriptionId ` - -SourceRg $SourceResourceGroup -TargetRg $TargetResourceGroup ` - -IsRoot ` - -IgnoreRepresentationSchemaRefs:$ignoreRepresentationSchemaRefs - } - - # Normalize any other top-level bags (e.g., location, sku) - foreach ($key in @($clone.Keys)) { - if ($key -eq 'properties') { continue } - $clone[$key] = ConvertTo-NormalizedPropertyValue -Value $clone[$key] ` - -SourceName $SourceApimName -TargetName $TargetApimName ` - -SourceSub $SourceSubscriptionId -TargetSub $TargetSubscriptionId ` - -SourceRg $SourceResourceGroup -TargetRg $TargetResourceGroup ` - -IgnoreRepresentationSchemaRefs:$ignoreRepresentationSchemaRefs - } - - return $clone + return CompareSemantics\ConvertTo-NormalizedResource -Resource $Resource -Context $NormalizationContext -IgnoreRepresentationSchemaRefs:$ignoreRepresentationSchemaRefs } function Test-ShouldIgnoreRepresentationSchemaRefs { <# .SYNOPSIS - Returns $true for operation resources with auto-generated IDs, where - schemaId/typeName can legitimately differ after SOAP/WSDL import. + Returns $true for operation resources where representation schema refs + are intentionally stripped during reconciliation PATCH. #> param([string] $ResourceId) - - if (-not $ResourceId) { return $false } - if ($ResourceId -notmatch '/operations/([^/]+)$') { return $false } - - $operationName = $Matches[1] - return ( - $operationName -match '^[0-9a-f]{24}$' -or - $operationName -match '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' - ) + return Test-IsApiOperationResource -ResourceId $ResourceId } function Compare-NormalizedResources { @@ -366,59 +269,7 @@ function Compare-NormalizedResources { [string] $Path = '' ) - $diffs = [System.Collections.Generic.List[string]]::new() - - $sourceJson = $Source | ConvertTo-Json -Depth 50 -Compress - $targetJson = $Target | ConvertTo-Json -Depth 50 -Compress - - if ($sourceJson -eq $targetJson) { return ,$diffs } - - # Walk keys for a readable diff - $allKeys = @() - if ($Source -is [System.Collections.IDictionary]) { $allKeys += $Source.Keys } - if ($Target -is [System.Collections.IDictionary]) { $allKeys += $Target.Keys } - $allKeys = $allKeys | Select-Object -Unique | Sort-Object - - foreach ($key in $allKeys) { - $currentPath = if ($Path) { "$Path.$key" } else { $key } - $hasSource = $Source -is [System.Collections.IDictionary] -and $Source.Contains($key) - $hasTarget = $Target -is [System.Collections.IDictionary] -and $Target.Contains($key) - - if ($hasSource -and -not $hasTarget) { - $diffs.Add(" MISSING in target: $currentPath") - continue - } - if (-not $hasSource -and $hasTarget) { - $diffs.Add(" EXTRA in target: $currentPath") - continue - } - - $sv = $Source[$key] - $tv = $Target[$key] - $svJson = $sv | ConvertTo-Json -Depth 50 -Compress - $tvJson = $tv | ConvertTo-Json -Depth 50 -Compress - - if ($svJson -ne $tvJson) { - # If both are dicts, recurse for finer detail - if ($sv -is [System.Collections.IDictionary] -and $tv -is [System.Collections.IDictionary]) { - $sub = Compare-NormalizedResources -Source $sv -Target $tv -Path $currentPath - if ($sub -is [System.Collections.IEnumerable] -and $sub -isnot [string]) { - foreach ($d in $sub) { $diffs.Add($d) } - } - } - else { - $diffs.Add(" DIFF at $currentPath`n source: $svJson`n target: $tvJson") - } - } - } - - # Fallback: if JSON differs but no key-level diffs found, report the full diff - if ($diffs.Count -eq 0) { - $pathPrefix = if ($Path) { "${Path}: " } else { '' } - $diffs.Add(" ${pathPrefix}JSON differs`n source: $sourceJson`n target: $targetJson") - } - - return ,$diffs + return CompareSemantics\Compare-NormalizedResources -Source $Source -Target $Target -Path $Path } function Test-SkipSecretValue { @@ -535,8 +386,34 @@ function Compare-ResourceType { $srcResource = $sourceMap[$name] $tgtResource = $targetMap[$name] - $srcNorm = ConvertTo-NormalizedResource -Resource $srcResource - $tgtNorm = ConvertTo-NormalizedResource -Resource $tgtResource + # For API operations, compare representation schema semantics (schema payload) + # rather than unstable schemaId/typeName values. + $srcId = if ($srcResource.PSObject.Properties['id']) { [string]$srcResource.id } else { '' } + $tgtId = if ($tgtResource.PSObject.Properties['id']) { [string]$tgtResource.id } else { '' } + + if ((Test-IsApiOperationResource -ResourceId $srcId) -or (Test-IsApiOperationResource -ResourceId $tgtId)) { + $srcWork = Copy-JsonObject -Value $srcResource + $tgtWork = Copy-JsonObject -Value $tgtResource + + $srcApiName = Get-ApiNameFromOperationResourceId -ResourceId $srcId + $tgtApiName = Get-ApiNameFromOperationResourceId -ResourceId $tgtId + $apiName = if ($srcApiName) { $srcApiName } else { $tgtApiName } + + if ($apiName) { + $srcSchemaMap = Get-ApiSchemaSemanticMap -InstanceKey 'source' -BaseUrl $SourceBase -ApiName $apiName + $tgtSchemaMap = Get-ApiSchemaSemanticMap -InstanceKey 'target' -BaseUrl $TargetBase -ApiName $apiName + + Add-RepresentationSchemaSemantics -Resource $srcWork -SchemaSemanticMap $srcSchemaMap + Add-RepresentationSchemaSemantics -Resource $tgtWork -SchemaSemanticMap $tgtSchemaMap + } + + $srcNorm = ConvertTo-NormalizedResource -Resource $srcWork + $tgtNorm = ConvertTo-NormalizedResource -Resource $tgtWork + } + else { + $srcNorm = ConvertTo-NormalizedResource -Resource $srcResource + $tgtNorm = ConvertTo-NormalizedResource -Resource $tgtResource + } # Skip secret named-value .value if ($SkipSecretValues -and (Test-SkipSecretValue $srcResource)) { diff --git a/tests/integration/all-resource-types/modules/CompareSemantics.psm1 b/tests/integration/all-resource-types/modules/CompareSemantics.psm1 new file mode 100644 index 0000000..5475ba8 --- /dev/null +++ b/tests/integration/all-resource-types/modules/CompareSemantics.psm1 @@ -0,0 +1,410 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS +Deep-clones a PSCustomObject/hashtable via JSON round-trip. + +.PARAMETER Value +Object to clone. + +.OUTPUTS +System.Object +#> +function Copy-JsonObject { + param([Parameter(Mandatory)] $Value) + return ($Value | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100) +} + +<# +.SYNOPSIS +Gets a property value from hashtable or PSCustomObject. + +.PARAMETER Object +Source object. + +.PARAMETER Name +Property name. + +.OUTPUTS +System.Object +#> +function Get-ObjectPropertyValue { + param( + [Parameter(Mandatory)] $Object, + [Parameter(Mandatory)] [string] $Name + ) + + if ($Object -is [System.Collections.IDictionary]) { + if ($Object.Contains($Name)) { return $Object[$Name] } + return $null + } + + if ($Object -is [PSCustomObject]) { + $p = $Object.PSObject.Properties[$Name] + if ($p) { return $p.Value } + return $null + } + + return $null +} + +<# +.SYNOPSIS +Sets a property value on hashtable or PSCustomObject. + +.PARAMETER Object +Target object. + +.PARAMETER Name +Property name. + +.PARAMETER Value +Property value. + +.OUTPUTS +None +#> +function Set-ObjectPropertyValue { + param( + [Parameter(Mandatory)] $Object, + [Parameter(Mandatory)] [string] $Name, + [AllowNull()] $Value + ) + + if ($Object -is [System.Collections.IDictionary]) { + $Object[$Name] = $Value + return + } + + if ($Object -is [PSCustomObject]) { + $Object | Add-Member -NotePropertyName $Name -NotePropertyValue $Value -Force + } +} + +<# +.SYNOPSIS +Returns true when a resource ID points at an API operation resource. + +.PARAMETER ResourceId +ARM resource ID. + +.OUTPUTS +System.Boolean +#> +function Test-IsApiOperationResource { + param([string] $ResourceId) + if (-not $ResourceId) { return $false } + return $ResourceId -match '/apis/[^/]+/operations/[^/]+$' +} + +<# +.SYNOPSIS +Extracts API name from an API operation ARM resource ID. + +.PARAMETER ResourceId +ARM resource ID. + +.OUTPUTS +System.String +#> +function Get-ApiNameFromOperationResourceId { + param([string] $ResourceId) + if (-not $ResourceId) { return $null } + if ($ResourceId -notmatch '/apis/([^/]+)/operations/[^/]+$') { return $null } + return $Matches[1] +} + +<# +.SYNOPSIS +Adds representation schema semantic tokens using schemaId->semantic map. + +.DESCRIPTION +Annotates request/response representations with __schemaSemantic to enable +comparison by schema content rather than unstable schemaId/typeName values. + +.PARAMETER Resource +Operation resource object (PSCustomObject/hashtable). + +.PARAMETER SchemaSemanticMap +Hashtable mapping schemaId to normalized schema semantic string. + +.OUTPUTS +None +#> +function Add-RepresentationSchemaSemantics { + [CmdletBinding()] + param( + [Parameter(Mandatory)] $Resource, + [Parameter(Mandatory)] [hashtable] $SchemaSemanticMap + ) + + $properties = Get-ObjectPropertyValue -Object $Resource -Name 'properties' + if (-not $properties) { return } + + function Set-RepresentationSemanticToken { + param([AllowNull()] $Representation) + + if (-not $Representation) { return } + if ($Representation -isnot [PSCustomObject] -and $Representation -isnot [System.Collections.IDictionary]) { return } + + $schemaId = Get-ObjectPropertyValue -Object $Representation -Name 'schemaId' + if (-not $schemaId) { return } + + $semantic = if ($SchemaSemanticMap.Contains($schemaId)) { + $SchemaSemanticMap[$schemaId] + } + else { + "{{missing-schema:$schemaId}}" + } + + Set-ObjectPropertyValue -Object $Representation -Name '__schemaSemantic' -Value $semantic + } + + $request = Get-ObjectPropertyValue -Object $properties -Name 'request' + if ($request) { + $reqReps = Get-ObjectPropertyValue -Object $request -Name 'representations' + if ($reqReps -is [System.Collections.IEnumerable] -and $reqReps -isnot [string]) { + foreach ($rep in $reqReps) { Set-RepresentationSemanticToken -Representation $rep } + } + } + + $responses = Get-ObjectPropertyValue -Object $properties -Name 'responses' + if ($responses -is [System.Collections.IEnumerable] -and $responses -isnot [string]) { + foreach ($response in $responses) { + $respReps = Get-ObjectPropertyValue -Object $response -Name 'representations' + if ($respReps -is [System.Collections.IEnumerable] -and $respReps -isnot [string]) { + foreach ($rep in $respReps) { Set-RepresentationSemanticToken -Representation $rep } + } + } + } +} + +function New-CompareNormalizationContext { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $SourceName, + [Parameter(Mandatory)] [string] $TargetName, + [Parameter(Mandatory)] [string] $SourceSub, + [Parameter(Mandatory)] [string] $TargetSub, + [Parameter(Mandatory)] [string] $SourceRg, + [Parameter(Mandatory)] [string] $TargetRg, + [string[]] $StripTopLevelFields = @(), + [string[]] $StripReadOnlyProperties = @(), + [string[]] $StripTimestampProperties = @(), + [string[]] $RequestResponseIgnoredProperties = @(), + [string[]] $RepresentationIgnoredProperties = @(), + [string[]] $RepresentationSchemaRefIgnoredProperties = @() + ) + + return [ordered]@{ + SourceName = $SourceName + TargetName = $TargetName + SourceSub = $SourceSub + TargetSub = $TargetSub + SourceRg = $SourceRg + TargetRg = $TargetRg + StripTopLevelFields = $StripTopLevelFields + StripReadOnlyProperties = $StripReadOnlyProperties + StripTimestampProperties = $StripTimestampProperties + RequestResponseIgnoredProperties = $RequestResponseIgnoredProperties + RepresentationIgnoredProperties = $RepresentationIgnoredProperties + RepresentationSchemaRefIgnoredProperties = $RepresentationSchemaRefIgnoredProperties + } +} + +function Get-ResourceNameFromId { + param([string] $ResourceId) + if (-not $ResourceId) { return $null } + return ($ResourceId -split '/')[-1] +} + +function ConvertTo-NormalizedPropertyValue { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [AllowNull()] $Value, + [Parameter(Mandatory)] [hashtable] $Context, + [switch] $IsRoot, + [switch] $IgnoreRepresentationSchemaRefs + ) + + if ($null -eq $Value) { return $null } + + if ($Value -is [string]) { + $s = $Value + $s = $s -replace [regex]::Escape("/subscriptions/$($Context.SourceSub)/resourceGroups/$($Context.SourceRg)/providers/Microsoft.ApiManagement/service/$($Context.SourceName)"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}/providers/Microsoft.ApiManagement/service/{{apim-name}}' + $s = $s -replace [regex]::Escape("/subscriptions/$($Context.TargetSub)/resourceGroups/$($Context.TargetRg)/providers/Microsoft.ApiManagement/service/$($Context.TargetName)"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}/providers/Microsoft.ApiManagement/service/{{apim-name}}' + $s = $s -replace [regex]::Escape("/subscriptions/$($Context.SourceSub)/resourceGroups/$($Context.SourceRg)"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}' + $s = $s -replace [regex]::Escape("/subscriptions/$($Context.TargetSub)/resourceGroups/$($Context.TargetRg)"), '/subscriptions/{{sub}}/resourceGroups/{{rg}}' + $s = $s -replace [regex]::Escape("/subscriptions/$($Context.SourceSub)"), '/subscriptions/{{sub}}' + $s = $s -replace [regex]::Escape("/subscriptions/$($Context.TargetSub)"), '/subscriptions/{{sub}}' + $s = $s -replace [regex]::Escape($Context.SourceName), '{{apim-name}}' + $s = $s -replace [regex]::Escape($Context.TargetName), '{{apim-name}}' + $s = $s -replace 'https://[a-zA-Z0-9-]+\.vault\.azure\.net', 'https://{{keyvault}}.vault.azure.net' + $s = $s -replace '/secrets/(src|tgt)-', '/secrets/{{prefix}}-' + $s = $s -replace '/providers/Microsoft\.Insights/components/[a-zA-Z0-9-]+', '/providers/Microsoft.Insights/components/{{appinsights}}' + $s = $s -replace '/providers/Microsoft\.EventHub/namespaces/[a-zA-Z0-9-]+', '/providers/Microsoft.EventHub/namespaces/{{eventhub}}' + $s = $s -replace '\b[0-9a-f]{24}\b', '{{auto-id}}' + $s = $s -replace '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '{{guid}}' + return $s + } + + if ($Value -is [System.Collections.IEnumerable] -and $Value -isnot [string] -and $Value -isnot [System.Collections.IDictionary]) { + $normalized = @(foreach ($item in $Value) { + ConvertTo-NormalizedPropertyValue -Value $item -Context $Context -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs + }) + $sorted = $normalized | Sort-Object { ($_ | ConvertTo-Json -Depth 50 -Compress) } + return @($sorted) + } + + if ($Value -is [System.Collections.IDictionary]) { + $out = [ordered]@{} + $isRequestResponse = $Value.Contains('representations') + $isRepresentation = $Value.Contains('contentType') -or $Value.Contains('schemaId') + foreach ($key in ($Value.Keys | Sort-Object)) { + if ($IsRoot -and $key -in $Context.StripReadOnlyProperties) { continue } + if ($key -in $Context.StripTimestampProperties) { continue } + if ($isRequestResponse -and $key -in $Context.RequestResponseIgnoredProperties) { continue } + if ($isRepresentation -and $key -in $Context.RepresentationIgnoredProperties) { continue } + if ($IgnoreRepresentationSchemaRefs -and $isRepresentation -and $key -in $Context.RepresentationSchemaRefIgnoredProperties) { continue } + $out[$key] = ConvertTo-NormalizedPropertyValue -Value $Value[$key] -Context $Context -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs + } + return $out + } + + if ($Value -is [PSCustomObject]) { + $out = [ordered]@{} + $isRequestResponse = $null -ne ($Value.PSObject.Properties | Where-Object { $_.Name -eq 'representations' }) + $isRepresentation = $null -ne ($Value.PSObject.Properties | Where-Object { $_.Name -eq 'contentType' -or $_.Name -eq 'schemaId' }) + foreach ($prop in ($Value.PSObject.Properties | Sort-Object Name)) { + if ($IsRoot -and $prop.Name -in $Context.StripReadOnlyProperties) { continue } + if ($prop.Name -in $Context.StripTimestampProperties) { continue } + if ($isRequestResponse -and $prop.Name -in $Context.RequestResponseIgnoredProperties) { continue } + if ($isRepresentation -and $prop.Name -in $Context.RepresentationIgnoredProperties) { continue } + if ($IgnoreRepresentationSchemaRefs -and $isRepresentation -and $prop.Name -in $Context.RepresentationSchemaRefIgnoredProperties) { continue } + $out[$prop.Name] = ConvertTo-NormalizedPropertyValue -Value $prop.Value -Context $Context -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs + } + return $out + } + + return $Value +} + +function ConvertTo-NormalizedResource { + [CmdletBinding()] + param( + [Parameter(Mandatory)] $Resource, + [Parameter(Mandatory)] [hashtable] $Context, + [switch] $IgnoreRepresentationSchemaRefs + ) + + $clone = [ordered]@{} + foreach ($prop in $Resource.PSObject.Properties) { + if ($prop.Name -in $Context.StripTopLevelFields) { continue } + $clone[$prop.Name] = $prop.Value + } + + if ($clone.Contains('properties')) { + $clone['properties'] = ConvertTo-NormalizedPropertyValue -Value $clone['properties'] -Context $Context -IsRoot -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs + } + + foreach ($key in @($clone.Keys)) { + if ($key -eq 'properties') { continue } + $clone[$key] = ConvertTo-NormalizedPropertyValue -Value $clone[$key] -Context $Context -IgnoreRepresentationSchemaRefs:$IgnoreRepresentationSchemaRefs + } + + return $clone +} + +function Build-ResourceMap { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]] $Items, + [string[]] $ExcludeNames = @(), + [Parameter(Mandatory)] [scriptblock] $NormalizeResource + ) + + $map = [ordered]@{} + if ($null -eq $Items -or $Items.Count -eq 0) { return $map } + + $autoIdItems = [System.Collections.Generic.List[object]]::new() + foreach ($item in $Items) { + $rName = Get-ResourceNameFromId -ResourceId $item.id + if ($rName -in $ExcludeNames) { continue } + + if ($rName -match '^[0-9a-f]{24}$' -or $rName -match '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') { + $autoIdItems.Add($item) + } + else { + $map[$rName] = $item + } + } + + if ($autoIdItems.Count -gt 0) { + $sorted = $autoIdItems | Sort-Object { + $normResource = & $NormalizeResource $_ + $normResource | ConvertTo-Json -Depth 50 -Compress + } + $i = 0 + foreach ($item in $sorted) { + $map["{{auto-id-$i}}"] = $item + $i++ + } + } + + return $map +} + +function Compare-NormalizedResources { + [CmdletBinding()] + param( + [Parameter(Mandatory)] $Source, + [Parameter(Mandatory)] $Target, + [string] $Path = '' + ) + + $diffs = [System.Collections.Generic.List[string]]::new() + $sourceJson = $Source | ConvertTo-Json -Depth 50 -Compress + $targetJson = $Target | ConvertTo-Json -Depth 50 -Compress + if ($sourceJson -eq $targetJson) { return ,$diffs } + + $allKeys = @() + if ($Source -is [System.Collections.IDictionary]) { $allKeys += $Source.Keys } + if ($Target -is [System.Collections.IDictionary]) { $allKeys += $Target.Keys } + $allKeys = $allKeys | Select-Object -Unique | Sort-Object + + foreach ($key in $allKeys) { + $currentPath = if ($Path) { "$Path.$key" } else { $key } + $hasSource = $Source -is [System.Collections.IDictionary] -and $Source.Contains($key) + $hasTarget = $Target -is [System.Collections.IDictionary] -and $Target.Contains($key) + + if ($hasSource -and -not $hasTarget) { $diffs.Add(" MISSING in target: $currentPath"); continue } + if (-not $hasSource -and $hasTarget) { $diffs.Add(" EXTRA in target: $currentPath"); continue } + + $sv = $Source[$key] + $tv = $Target[$key] + $svJson = $sv | ConvertTo-Json -Depth 50 -Compress + $tvJson = $tv | ConvertTo-Json -Depth 50 -Compress + + if ($svJson -ne $tvJson) { + if ($sv -is [System.Collections.IDictionary] -and $tv -is [System.Collections.IDictionary]) { + $sub = Compare-NormalizedResources -Source $sv -Target $tv -Path $currentPath + if ($sub -is [System.Collections.IEnumerable] -and $sub -isnot [string]) { + foreach ($d in $sub) { $diffs.Add($d) } + } + } + else { + $diffs.Add(" DIFF at $currentPath`n source: $svJson`n target: $tvJson") + } + } + } + + if ($diffs.Count -eq 0) { + $pathPrefix = if ($Path) { "${Path}: " } else { '' } + $diffs.Add(" ${pathPrefix}JSON differs`n source: $sourceJson`n target: $targetJson") + } + + return ,$diffs +} + +Export-ModuleMember -Function Test-IsApiOperationResource, Get-ApiNameFromOperationResourceId, Add-RepresentationSchemaSemantics, New-CompareNormalizationContext, ConvertTo-NormalizedPropertyValue, ConvertTo-NormalizedResource, Build-ResourceMap, Compare-NormalizedResources \ No newline at end of file