diff --git a/DevProxy.Abstractions.Tests/Extensions/ConfigurationPlaceholderTests.cs b/DevProxy.Abstractions.Tests/Extensions/ConfigurationPlaceholderTests.cs new file mode 100644 index 000000000..4e5d7ec78 --- /dev/null +++ b/DevProxy.Abstractions.Tests/Extensions/ConfigurationPlaceholderTests.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Extensions.Configuration; +using Xunit; + +namespace DevProxy.Abstractions.Tests.Extensions; + +public sealed class ConfigurationPlaceholderTests : IDisposable +{ + private const string OsVariable = "DEVPROXY_TEST_PLACEHOLDER_OS"; + private readonly string _folder = Directory.CreateTempSubdirectory("devproxy-placeholders").FullName; + + public ConfigurationPlaceholderTests() + { + Environment.SetEnvironmentVariable(OsVariable, "from-os"); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable(OsVariable, null); + Directory.Delete(_folder, true); + } + + [Fact] + public void AddConfigFile_Json_ResolvesPlaceholders() + { + var path = Write("devproxyrc.json", $$""" + { + "env": { "FROM_SECTION": "section-${{{OsVariable}}}" }, + "plugin": { + "os": "${{{OsVariable}}}", + "alias": "${env:{{OsVariable}}}", + "section": "${FROM_SECTION}", + "cli": "${FROM_CLI}", + "escaped": "$${{{OsVariable}}}", + "missing": "${DEVPROXY_TEST_PLACEHOLDER_MISSING}", + "port": "${PORT}" + } + } + """); + + var (config, unresolved) = Build(path, new Dictionary { ["FROM_CLI"] = "cli", ["PORT"] = "8000" }); + + Assert.Equal("from-os", config["plugin:os"]); + Assert.Equal("from-os", config["plugin:alias"]); + Assert.Equal("section-from-os", config["env:FROM_SECTION"]); + Assert.Equal("section-from-os", config["plugin:section"]); + Assert.Equal("cli", config["plugin:cli"]); + Assert.Equal($"${{{OsVariable}}}", config["plugin:escaped"]); + Assert.Equal("${DEVPROXY_TEST_PLACEHOLDER_MISSING}", config["plugin:missing"]); + Assert.Equal(8000, config.GetValue("plugin:port")); + Assert.Equal(["DEVPROXY_TEST_PLACEHOLDER_MISSING"], unresolved); + } + + [Fact] + public void AddConfigFile_CliVariablesOverrideEnvSection() + { + var path = Write("devproxyrc.json", """ + { "env": { "NAME": "section" }, "value": "${NAME}" } + """); + + var (config, _) = Build(path, new Dictionary { ["NAME"] = "cli" }); + + Assert.Equal("cli", config["value"]); + } + + [Fact] + public void AddConfigFile_CliVariablesDifferingInCasing_LastValueWins() + { + var path = Write("devproxyrc.json", """ + { "value": "${NAME}" } + """); + + var (config, _) = Build(path, new Dictionary { ["NAME"] = "first", ["name"] = "second" }); + + Assert.Equal(OperatingSystem.IsWindows() ? "second" : "first", config["value"]); + } + + [Fact] + public void AddConfigFile_SkipsInlineRewriteRules() + { + var path = Write("devproxyrc.json", $$""" + { + "plugins": [ + { "name": "RewritePlugin", "enabled": true, "pluginPath": "plugins/DevProxy.Plugins.dll", "configSection": "myRewrites" } + ], + "myRewrites": { + "rewritesFile": "${{{OsVariable}}}.json", + "rewrites": [ { "out": { "url": "/users/${{{OsVariable}}}" } } ] + }, + "other": { "rewrites": [ "${{{OsVariable}}}" ] } + } + """); + + var (config, _) = Build(path, null); + + Assert.Equal("from-os.json", config["myRewrites:rewritesFile"]); + Assert.Equal($"/users/${{{OsVariable}}}", config["myRewrites:rewrites:0:out:url"]); + Assert.Equal("from-os", config["other:rewrites:0"]); + } + + [Fact] + public void AddConfigFile_SkipsInlineRewriteRules_WithPlaceholdersInPluginMetadata() + { + var path = Write("devproxyrc.json", $$""" + { + "env": { "PLUGIN": "RewritePlugin", "SECTION": "myRewrites" }, + "plugins": [ { "name": "${PLUGIN}", "configSection": "${SECTION}" } ], + "myRewrites": { "rewrites": [ { "out": { "url": "/users/${{{OsVariable}}}" } } ] } + } + """); + + var (config, _) = Build(path, null); + + Assert.Equal("RewritePlugin", config["plugins:0:name"]); + Assert.Equal("myRewrites", config["plugins:0:configSection"]); + Assert.Equal($"/users/${{{OsVariable}}}", config["myRewrites:rewrites:0:out:url"]); + } + + [Fact] + public void AddConfigFile_Yaml_ResolvesPlaceholders() + { + var path = Write("devproxyrc.yaml", $$""" + env: + FROM_SECTION: section + plugin: + os: ${{{OsVariable}}} + section: "${FROM_SECTION}" + list: + - ${env:{{OsVariable}}} + """); + + var (config, unresolved) = Build(path, null); + + Assert.Equal("from-os", config["plugin:os"]); + Assert.Equal("section", config["plugin:section"]); + Assert.Equal("from-os", config["plugin:list:0"]); + Assert.Empty(unresolved); + } + + private string Write(string name, string contents) + { + var path = Path.Combine(_folder, name); + File.WriteAllText(path, contents); + return path; + } + + private static (IConfigurationRoot Config, IReadOnlyCollection Unresolved) Build(string path, IReadOnlyDictionary? variables) + { + var config = new ConfigurationBuilder().AddConfigFile(path, environmentVariables: variables).Build(); + var provider = Assert.Single(config.Providers.OfType()); + return (config, provider.UnresolvedEnvironmentVariables); + } +} diff --git a/DevProxy.Abstractions.Tests/Utils/EnvironmentVariablePlaceholdersTests.cs b/DevProxy.Abstractions.Tests/Utils/EnvironmentVariablePlaceholdersTests.cs new file mode 100644 index 000000000..c82ef6d99 --- /dev/null +++ b/DevProxy.Abstractions.Tests/Utils/EnvironmentVariablePlaceholdersTests.cs @@ -0,0 +1,211 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.Utils; +using System.Text.Json; +using Xunit; + +namespace DevProxy.Abstractions.Tests.Utils; + +public class EnvironmentVariablePlaceholdersTests +{ + private static readonly Dictionary Variables = new() + { + ["TENANT_ID"] = "contoso", + ["CLIENT_ID"] = "1234", + ["QUOTED"] = "a \"quoted\" \\ value" + }; + + private static string? Lookup(string name) => Variables.TryGetValue(name, out var value) ? value : null; + + [Theory] + [InlineData("${TENANT_ID}", "contoso")] + [InlineData("${env:TENANT_ID}", "contoso")] + [InlineData("https://login/${TENANT_ID}/v2.0", "https://login/contoso/v2.0")] + [InlineData("${TENANT_ID}${CLIENT_ID}", "contoso1234")] + [InlineData("$${TENANT_ID}", "${TENANT_ID}")] + [InlineData("$${env:TENANT_ID}", "${env:TENANT_ID}")] + [InlineData("no placeholders", "no placeholders")] + [InlineData("{TENANT_ID} @TENANT_ID", "{TENANT_ID} @TENANT_ID")] + [InlineData("${not a var}", "${not a var}")] + [InlineData("$TENANT_ID", "$TENANT_ID")] + public void Resolve_ReplacesPlaceholders(string input, string expected) => + Assert.Equal(expected, EnvironmentVariablePlaceholders.Resolve(input, Lookup)); + + [Fact] + public void Resolve_LeavesUnsetVariablesAndReportsThemOnce() + { + var unresolved = new List(); + + var result = EnvironmentVariablePlaceholders.Resolve("${MISSING}/${TENANT_ID}/${env:MISSING}", Lookup, unresolved); + + Assert.Equal("${MISSING}/contoso/${env:MISSING}", result); + Assert.Equal(["MISSING"], unresolved); + } + + [Fact] + public void Resolve_DoesNotReportEscapedPlaceholders() + { + var unresolved = new List(); + + _ = EnvironmentVariablePlaceholders.Resolve("$${MISSING}", Lookup, unresolved); + + Assert.Empty(unresolved); + } + + [Fact] + public void ResolveInJson_ResolvesStringValuesOnly() + { + var json = """ + { + // comment + "${TENANT_ID}": "${TENANT_ID}", + "nested": { "items": ["${CLIENT_ID}", 1, true, null] }, + "quoted": "${QUOTED}", + } + """; + + var result = EnvironmentVariablePlaceholders.ResolveInJson(json, Lookup); + + using var doc = JsonDocument.Parse(result); + var root = doc.RootElement; + Assert.Equal("contoso", root.GetProperty("${TENANT_ID}").GetString()); + Assert.Equal("1234", root.GetProperty("nested").GetProperty("items")[0].GetString()); + Assert.Equal(1, root.GetProperty("nested").GetProperty("items")[1].GetInt32()); + Assert.Equal("a \"quoted\" \\ value", root.GetProperty("quoted").GetString()); + } + + [Fact] + public void ResolveInJson_ReturnsOriginalWhenNoPlaceholders() + { + var json = "{ /* keep */ \"a\": \"b\" }"; + + Assert.Same(json, EnvironmentVariablePlaceholders.ResolveInJson(json, Lookup)); + } + + [Fact] + public void ResolveInJson_ResolvesUnicodeEscapedPlaceholders() + { + var json = """{ "audience": "\u0024{CLIENT_ID}" }"""; + + var result = EnvironmentVariablePlaceholders.ResolveInJson(json, Lookup); + + using var doc = JsonDocument.Parse(result); + Assert.Equal("1234", doc.RootElement.GetProperty("audience").GetString()); + } + + [Fact] + public void ResolveInJson_ReturnsOriginalWhenEscapesWithoutPlaceholders() + { + var json = """{ "a": "\u00e9" }"""; + + Assert.Same(json, EnvironmentVariablePlaceholders.ResolveInJson(json, Lookup)); + } + + [Fact] + public void ResolveInYaml_ResolvesScalarValuesOnly() + { + var yaml = """ + ${TENANT_ID}: "${TENANT_ID}" + quoted: "${QUOTED}" + base: &base + id: ${CLIENT_ID} + copy: *base + """; + + var result = EnvironmentVariablePlaceholders.ResolveInYaml(yaml, Lookup); + + var stream = new YamlDotNet.RepresentationModel.YamlStream(); + stream.Load(new StringReader(result)); + var root = (YamlDotNet.RepresentationModel.YamlMappingNode)stream.Documents[0].RootNode; + Assert.Equal("contoso", root["${TENANT_ID}"].ToString()); + Assert.Equal("a \"quoted\" \\ value", root["quoted"].ToString()); + Assert.Equal("1234", ((YamlDotNet.RepresentationModel.YamlMappingNode)root["copy"])["id"].ToString()); + } + + [Fact] + public void ResolveInYaml_ReturnsOriginalWhenNoPlaceholders() + { + var yaml = "a: b # keep"; + + Assert.Same(yaml, EnvironmentVariablePlaceholders.ResolveInYaml(yaml, Lookup)); + } + + [Fact] + public void Resolve_TransformsOnlyTextOutsidePlaceholders() + { + var result = EnvironmentVariablePlaceholders.Resolve( + "{TENANT_ID} $${TENANT_ID} ${CLIENT_ID} ${MISSING}", + name => name == "CLIENT_ID" ? "{TENANT_ID}" : Lookup(name), + null, + text => text.Replace("{TENANT_ID}", "legacy", StringComparison.Ordinal)); + + Assert.Equal("legacy ${TENANT_ID} {TENANT_ID} ${MISSING}", result); + } + + [Theory] + [InlineData("${env:ProgramFiles(x86)}", @"C:\Program Files (x86)")] + [InlineData("${env:FOO-BAR}", "dash")] + [InlineData("${env:FOO.BAR}", "dot")] + [InlineData("$${env:ProgramFiles(x86)}", "${env:ProgramFiles(x86)}")] + [InlineData("${ProgramFiles(x86)}", "${ProgramFiles(x86)}")] + [InlineData("${FOO-BAR}", "${FOO-BAR}")] + public void Resolve_EnvPrefixAcceptsAnyNameCharactersExceptBrace(string input, string expected) + { + var variables = new Dictionary + { + ["ProgramFiles(x86)"] = @"C:\Program Files (x86)", + ["FOO-BAR"] = "dash", + ["FOO.BAR"] = "dot" + }; + var unresolved = new List(); + + var result = EnvironmentVariablePlaceholders.Resolve(input, n => variables.TryGetValue(n, out var v) ? v : null, unresolved); + + Assert.Equal(expected, result); + Assert.Empty(unresolved); + } + + [Fact] + public void Resolve_ReportsUnsetEnvPrefixedNameWithSpecialCharacters() + { + var unresolved = new List(); + + var result = EnvironmentVariablePlaceholders.Resolve("${env:MISSING(x86)}", Lookup, unresolved); + + Assert.Equal("${env:MISSING(x86)}", result); + Assert.Equal(["MISSING(x86)"], unresolved); + } + + [Fact] + public void Resolve_TransformsTextAroundEnvPrefixedNamesWithSpecialCharacters() + { + var result = EnvironmentVariablePlaceholders.Resolve( + "{TENANT_ID} ${env:FOO-BAR} $${env:FOO-BAR}", + name => name == "FOO-BAR" ? "{TENANT_ID}" : null, + null, + text => text.Replace("{TENANT_ID}", "legacy", StringComparison.Ordinal)); + + Assert.Equal("legacy {TENANT_ID} ${env:FOO-BAR}", result); + } + + [Fact] + public void ResolveInYaml_AliasUsedAsKey_IsResolvedWithItsAnchor() + { + // intentional: aliases share the anchored node, so a key that aliases a + // resolved value changes with it + var yaml = """ + base: &id ${TENANT_ID} + *id : value + """; + + var result = EnvironmentVariablePlaceholders.ResolveInYaml(yaml, Lookup); + + var stream = new YamlDotNet.RepresentationModel.YamlStream(); + stream.Load(new StringReader(result)); + var root = (YamlDotNet.RepresentationModel.YamlMappingNode)stream.Documents[0].RootNode; + Assert.Equal("contoso", root["base"].ToString()); + Assert.Equal("value", root["contoso"].ToString()); + } +} \ No newline at end of file diff --git a/DevProxy.Abstractions/Extensions/EnvironmentVariableConfigurationExtensions.cs b/DevProxy.Abstractions/Extensions/EnvironmentVariableConfigurationExtensions.cs new file mode 100644 index 000000000..5f0327f20 --- /dev/null +++ b/DevProxy.Abstractions/Extensions/EnvironmentVariableConfigurationExtensions.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.Utils; +using Microsoft.Extensions.Configuration.Json; + +#pragma warning disable IDE0130 +namespace Microsoft.Extensions.Configuration; +#pragma warning restore IDE0130 + +/// +/// A configuration provider that resolves environment variable placeholders +/// (${NAME}, ${env:NAME}) in configuration values. +/// +public interface IEnvironmentVariablePlaceholderProvider +{ + /// + /// Path of the configuration file. + /// + string? Path { get; } + + /// + /// Names of environment variables referenced in the configuration file that couldn't be resolved during the last load. + /// + IReadOnlyCollection UnresolvedEnvironmentVariables { get; } +} + +/// +/// JSON configuration source that resolves environment variable placeholders. +/// +public sealed class EnvironmentVariableJsonConfigurationSource : JsonConfigurationSource +{ + /// + /// Variables that take precedence over the env configuration section and process environment variables. + /// + public IReadOnlyDictionary? EnvironmentVariables { get; set; } + + /// + public override IConfigurationProvider Build(IConfigurationBuilder builder) + { + EnsureDefaults(builder); + return new EnvironmentVariableJsonConfigurationProvider(this); + } +} + +/// +/// JSON configuration provider that resolves environment variable placeholders. +/// +public sealed class EnvironmentVariableJsonConfigurationProvider(EnvironmentVariableJsonConfigurationSource source) : + JsonConfigurationProvider(source), IEnvironmentVariablePlaceholderProvider +{ + /// + public string? Path => source.Path; + + /// + public IReadOnlyCollection UnresolvedEnvironmentVariables { get; private set; } = []; + + /// + public override void Load(Stream stream) + { + base.Load(stream); + UnresolvedEnvironmentVariables = ConfigurationPlaceholderResolver.Resolve(Data, source.EnvironmentVariables); + } +} + +/// +/// Resolves environment variable placeholders in flattened configuration data. +/// +public static class ConfigurationPlaceholderResolver +{ + private const string EnvSectionPrefix = "env:"; + + private static StringComparer VariableNameComparer => + OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + + /// + /// Resolves placeholders in configuration values in place. Variables are looked up in + /// first, then in the env configuration + /// section, and finally in process environment variables. Values in the env + /// section itself are resolved against and + /// process environment variables only. + /// + /// Names of variables that couldn't be resolved. + public static IReadOnlyCollection Resolve(IDictionary data, IReadOnlyDictionary? environmentVariables) + { + ArgumentNullException.ThrowIfNull(data); + + var unresolved = new List(); + Dictionary? overrides = null; + if (environmentVariables is not null) + { + overrides = new(VariableNameComparer); + // use the indexer so that names differing only in casing on Windows don't throw; last value wins + foreach (var (name, value) in environmentVariables) + { + overrides[name] = value; + } + } + + string? baseLookup(string name) + { + return overrides is not null && overrides.TryGetValue(name, out var value) + ? value + : Environment.GetEnvironmentVariable(name); + } + + var envSection = new Dictionary(VariableNameComparer); + foreach (var key in data.Keys.ToArray()) + { + if (!IsEnvSectionVariable(key)) + { + continue; + } + + var value = data[key]; + if (value is not null) + { + value = EnvironmentVariablePlaceholders.Resolve(value, baseLookup, unresolved); + data[key] = value; + envSection[key[EnvSectionPrefix.Length..]] = value; + } + } + + string? lookup(string name) + { + if (overrides is not null && overrides.TryGetValue(name, out var value)) + { + return value; + } + return envSection.TryGetValue(name, out value) ? value : Environment.GetEnvironmentVariable(name); + } + + var rewriteRulePrefixes = GetRewriteRulePrefixes(data, lookup); + foreach (var key in data.Keys.ToArray()) + { + if (IsEnvSectionVariable(key) || + rewriteRulePrefixes.Any(p => key.StartsWith(p, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + var value = data[key]; + if (value is not null) + { + data[key] = EnvironmentVariablePlaceholders.Resolve(value, lookup, unresolved); + } + } + + return unresolved; + } + + // RewritePlugin rules use ${name} for regex named groups and resolve ${env:NAME} + // per request, so inline rules are excluded from load-time resolution + private static string[] GetRewriteRulePrefixes(IDictionary data, Func lookup) + { + // plugin names and sections may use placeholders too; resolve them here without + // changing data, which is resolved afterwards + string? resolve(string? value) + { + return value is null ? null : EnvironmentVariablePlaceholders.Resolve(value, lookup); + } + + const string pluginsPrefix = "plugins:"; + const string nameSuffix = ":name"; + const string rewritePluginName = "RewritePlugin"; + + return [.. data + .Where(e => + e.Key.StartsWith(pluginsPrefix, StringComparison.OrdinalIgnoreCase) && + e.Key.EndsWith(nameSuffix, StringComparison.OrdinalIgnoreCase) && + e.Key.Count(c => c == ':') == 2 && + string.Equals(resolve(e.Value), rewritePluginName, StringComparison.Ordinal)) + .Select(e => data.TryGetValue($"{e.Key[..^nameSuffix.Length]}:configSection", out var section) ? resolve(section) : null) + .Where(section => !string.IsNullOrEmpty(section)) + .Select(section => $"{section}:rewrites:")]; + } + + private static bool IsEnvSectionVariable(string key) => + key.StartsWith(EnvSectionPrefix, StringComparison.OrdinalIgnoreCase) && + key.IndexOf(ConfigurationPath.KeyDelimiter, EnvSectionPrefix.Length, StringComparison.Ordinal) < 0; +} diff --git a/DevProxy.Abstractions/Extensions/YamlConfigurationExtensions.cs b/DevProxy.Abstractions/Extensions/YamlConfigurationExtensions.cs index feb5ab0c1..59d743c0c 100644 --- a/DevProxy.Abstractions/Extensions/YamlConfigurationExtensions.cs +++ b/DevProxy.Abstractions/Extensions/YamlConfigurationExtensions.cs @@ -14,6 +14,12 @@ namespace Microsoft.Extensions.Configuration; /// public sealed class YamlConfigurationSource : FileConfigurationSource { + /// + /// Variables used to resolve environment variable placeholders. They take precedence over + /// the env configuration section and process environment variables. + /// + public IReadOnlyDictionary? EnvironmentVariables { get; set; } + /// public override IConfigurationProvider Build(IConfigurationBuilder builder) { @@ -25,8 +31,14 @@ public override IConfigurationProvider Build(IConfigurationBuilder builder) /// /// A YAML file configuration provider that supports anchors and merge keys. /// -public sealed class YamlConfigurationProvider : FileConfigurationProvider +public sealed class YamlConfigurationProvider : FileConfigurationProvider, IEnvironmentVariablePlaceholderProvider { + /// + public string? Path => Source.Path; + + /// + public IReadOnlyCollection UnresolvedEnvironmentVariables { get; private set; } = []; + /// /// Initializes a new instance of the class. /// @@ -57,6 +69,8 @@ public override void Load(Stream stream) { FlattenYamlNode(mappingNode, string.Empty); } + + UnresolvedEnvironmentVariables = ConfigurationPlaceholderResolver.Resolve(Data, ((YamlConfigurationSource)Source).EnvironmentVariables); } private void FlattenYamlNode(YamlNode node, string prefix) @@ -250,12 +264,14 @@ public static class YamlConfigurationExtensions /// The path to the YAML file. /// Whether the file is optional. /// Whether to reload on change. + /// Variables used to resolve ${NAME} placeholders before the env section and process environment variables. /// The configuration builder. public static IConfigurationBuilder AddYamlFile( this IConfigurationBuilder builder, string path, bool optional = false, - bool reloadOnChange = false) + bool reloadOnChange = false, + IReadOnlyDictionary? environmentVariables = null) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrEmpty(path); @@ -266,6 +282,7 @@ public static IConfigurationBuilder AddYamlFile( s.Path = path; s.Optional = optional; s.ReloadOnChange = reloadOnChange; + s.EnvironmentVariables = environmentVariables; s.ResolveFileProvider(); }); } @@ -277,21 +294,32 @@ public static IConfigurationBuilder AddYamlFile( /// The path to the configuration file. /// Whether the file is optional. /// Whether to reload on change. + /// Variables used to resolve ${NAME} placeholders before the env section and process environment variables. /// The configuration builder. + /// Environment variable placeholders (${NAME}, ${env:NAME}) in configuration values are resolved when the file is loaded. public static IConfigurationBuilder AddConfigFile( this IConfigurationBuilder builder, string path, bool optional = false, - bool reloadOnChange = false) + bool reloadOnChange = false, + IReadOnlyDictionary? environmentVariables = null) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrEmpty(path); if (ProxyYaml.IsYamlFile(path)) { - return builder.AddYamlFile(path, optional, reloadOnChange); + return builder.AddYamlFile(path, optional, reloadOnChange, environmentVariables); } - return builder.AddJsonFile(path, optional, reloadOnChange); + return builder.Add(s => + { + s.FileProvider = null; + s.Path = path; + s.Optional = optional; + s.ReloadOnChange = reloadOnChange; + s.EnvironmentVariables = environmentVariables; + s.ResolveFileProvider(); + }); } } diff --git a/DevProxy.Abstractions/Plugins/BaseLoader.cs b/DevProxy.Abstractions/Plugins/BaseLoader.cs index b06cc255c..8bd505f99 100644 --- a/DevProxy.Abstractions/Plugins/BaseLoader.cs +++ b/DevProxy.Abstractions/Plugins/BaseLoader.cs @@ -24,6 +24,11 @@ public abstract class BaseLoader(HttpClient httpClient, ILogger logger, IProxyCo private volatile bool _isDisposed; protected abstract string FilePath { get; } + /// + /// Whether to resolve environment variable placeholders (${NAME}, ${env:NAME}) + /// in string values of the loaded file. Defaults to true. + /// + protected virtual bool ResolveEnvironmentVariables => true; protected ILogger Logger { get; } = logger; public async Task InitFileWatcherAsync(CancellationToken cancellationToken) @@ -138,6 +143,13 @@ private async Task LoadFileContentsAsync(CancellationToken cancellationToken) jsonContents = fileContents; } + if (ResolveEnvironmentVariables) + { + var unresolvedVariables = new List(); + jsonContents = EnvironmentVariablePlaceholders.ResolveInJson(jsonContents, unresolvedVariables); + EnvironmentVariablePlaceholders.LogUnresolvedVariables(Logger, unresolvedVariables, FilePath); + } + if (!_validateSchemas || await ValidateFileContentsAsync(jsonContents, cancellationToken)) { LoadData(jsonContents); diff --git a/DevProxy.Abstractions/Utils/EnvironmentVariablePlaceholders.cs b/DevProxy.Abstractions/Utils/EnvironmentVariablePlaceholders.cs new file mode 100644 index 000000000..a6bbe76d4 --- /dev/null +++ b/DevProxy.Abstractions/Utils/EnvironmentVariablePlaceholders.cs @@ -0,0 +1,314 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Extensions.Logging; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using YamlDotNet.RepresentationModel; + +namespace DevProxy.Abstractions.Utils; + +/// +/// Resolves environment variable placeholders in user-authored files. +/// Supports ${NAME} and ${env:NAME}. Use $${NAME} to +/// output a literal ${NAME}. Placeholders that reference variables +/// that aren't set are left as-is. +/// +public static partial class EnvironmentVariablePlaceholders +{ + private static readonly JsonSerializerOptions _writeOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + /// + /// Returns true if the value contains something that looks like a placeholder or an escaped placeholder. + /// + public static bool ContainsPlaceholder(string? value) => + !string.IsNullOrEmpty(value) && PlaceholderRegex().IsMatch(value); + + /// + /// Resolves placeholders in a string value using process environment variables. + /// + public static string Resolve(string value, ICollection? unresolvedVariables = null) => + Resolve(value, Environment.GetEnvironmentVariable, unresolvedVariables); + + /// + /// Resolves placeholders in a string value using the specified lookup. + /// + /// Value to resolve. + /// Function that returns the value of a variable or null if it isn't set. + /// Collects names of variables that couldn't be resolved. + public static string Resolve(string value, Func lookup, ICollection? unresolvedVariables = null) => + Resolve(value, lookup, unresolvedVariables, null); + + /// + /// Resolves placeholders in a string value using the specified lookup. + /// + /// Value to resolve. + /// Function that returns the value of a variable or null if it isn't set. + /// Collects names of variables that couldn't be resolved. + /// Optional transformation applied only to the text outside placeholders. + public static string Resolve(string value, Func lookup, ICollection? unresolvedVariables, Func? transformText) + { + ArgumentNullException.ThrowIfNull(value); + ArgumentNullException.ThrowIfNull(lookup); + + if (transformText is not null) + { + return ResolveWithTextTransform(value, lookup, unresolvedVariables, transformText); + } + + if (!value.Contains("${", StringComparison.Ordinal)) + { + return value; + } + + return PlaceholderRegex().Replace(value, match => ResolveMatch(match, lookup, unresolvedVariables)); + } + + /// + /// Applies a transformation to the text outside placeholders, leaving placeholders + /// (including escaped ones) unchanged. + /// + public static string TransformOutsidePlaceholders(string value, Func transformText) + { + ArgumentNullException.ThrowIfNull(value); + ArgumentNullException.ThrowIfNull(transformText); + + var result = new StringBuilder(); + var position = 0; + foreach (Match match in PlaceholderRegex().Matches(value)) + { + _ = result + .Append(transformText(value[position..match.Index])) + .Append(match.Value); + position = match.Index + match.Length; + } + _ = result.Append(transformText(value[position..])); + return result.ToString(); + } + + private static string ResolveWithTextTransform(string value, Func lookup, ICollection? unresolvedVariables, Func transformText) + { + var result = new StringBuilder(); + var position = 0; + foreach (Match match in PlaceholderRegex().Matches(value)) + { + _ = result + .Append(transformText(value[position..match.Index])) + .Append(ResolveMatch(match, lookup, unresolvedVariables)); + position = match.Index + match.Length; + } + _ = result.Append(transformText(value[position..])); + return result.ToString(); + } + + private static string ResolveMatch(Match match, Func lookup, ICollection? unresolvedVariables) + { + var name = match.Groups["name"].Value; + if (match.Groups["escape"].Success) + { + // $${NAME} -> literal ${NAME} + return match.Value[1..]; + } + + var variableValue = lookup(name); + if (variableValue is null) + { + if (unresolvedVariables is not null && !unresolvedVariables.Contains(name)) + { + unresolvedVariables.Add(name); + } + return match.Value; + } + + return variableValue; + } + + /// + /// Resolves placeholders in all string values of a JSON document. Property names aren't changed. + /// Returns the original string if it doesn't contain any placeholders. + /// + public static string ResolveInJson(string json, Func lookup, ICollection? unresolvedVariables = null) => + ResolveInJson(json, lookup, unresolvedVariables, null); + + /// + /// Resolves placeholders in all string values of a JSON document. Property names aren't changed. + /// Returns the original string if no values changed. + /// + /// JSON document. + /// Function that returns the value of a variable or null if it isn't set. + /// Collects names of variables that couldn't be resolved. + /// Optional transformation applied only to the text outside placeholders. + public static string ResolveInJson(string json, Func lookup, ICollection? unresolvedVariables, Func? transformText) + { + ArgumentNullException.ThrowIfNull(json); + ArgumentNullException.ThrowIfNull(lookup); + + // string values may contain JSON escapes (eg. \u0024{NAME}) so only skip parsing when there are none + if (transformText is null && + !ContainsPlaceholder(json) && + !json.Contains("\\u", StringComparison.Ordinal)) + { + return json; + } + + var node = JsonNode.Parse(json, documentOptions: ProxyUtils.JsonDocumentOptions); + if (node is null) + { + return json; + } + + var changed = false; + node = ResolveInJsonNode(node, value => + { + var resolved = Resolve(value, lookup, unresolvedVariables, transformText); + changed |= !string.Equals(value, resolved, StringComparison.Ordinal); + return resolved; + }); + return changed ? node.ToJsonString(_writeOptions) : json; + } + + /// + /// Resolves placeholders in all string values of a JSON document using process environment variables. + /// + public static string ResolveInJson(string json, ICollection? unresolvedVariables = null) => + ResolveInJson(json, Environment.GetEnvironmentVariable, unresolvedVariables); + + /// + /// Resolves placeholders in all scalar values of a YAML document. Mapping keys aren't changed. + /// Returns the original string if no values changed. + /// + /// YAML document. + /// Function that returns the value of a variable or null if it isn't set. + /// Collects names of variables that couldn't be resolved. + /// Optional transformation applied only to the text outside placeholders. + public static string ResolveInYaml(string yaml, Func lookup, ICollection? unresolvedVariables = null, Func? transformText = null) + { + ArgumentNullException.ThrowIfNull(yaml); + ArgumentNullException.ThrowIfNull(lookup); + + var stream = new YamlStream(); + using (var reader = new StringReader(yaml)) + { + stream.Load(reader); + } + + var changed = false; + var visited = new HashSet(ReferenceEqualityComparer.Instance); + void resolveNode(YamlNode node) + { + // aliases point to the same node instance so each node is resolved once + if (!visited.Add(node)) + { + return; + } + + switch (node) + { + case YamlScalarNode scalar when scalar.Value is not null: + var resolved = Resolve(scalar.Value, lookup, unresolvedVariables, transformText); + if (!string.Equals(scalar.Value, resolved, StringComparison.Ordinal)) + { + scalar.Value = resolved; + changed = true; + } + break; + case YamlMappingNode mapping: + foreach (var child in mapping.Children) + { + resolveNode(child.Value); + } + break; + case YamlSequenceNode sequence: + foreach (var child in sequence.Children) + { + resolveNode(child); + } + break; + default: + break; + } + } + + foreach (var document in stream.Documents) + { + resolveNode(document.RootNode); + } + + if (!changed) + { + return yaml; + } + + using var writer = new StringWriter(); + stream.Save(writer, assignAnchors: false); + return writer.ToString(); + } + + /// + /// Logs a warning for each environment variable that couldn't be resolved. + /// + public static void LogUnresolvedVariables(ILogger logger, IEnumerable unresolvedVariables, string source) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(unresolvedVariables); + + foreach (var name in unresolvedVariables) + { + logger.LogWarning( + "Environment variable {Name} referenced in {Source} isn't set. Leaving the placeholder as-is", + name, source); + } + } + + private static JsonNode ResolveInJsonNode(JsonNode node, Func resolve) + { + switch (node) + { + case JsonObject obj: + foreach (var property in obj.ToArray()) + { + if (property.Value is not null) + { + var resolved = ResolveInJsonNode(property.Value, resolve); + if (!ReferenceEquals(resolved, property.Value)) + { + obj[property.Key] = resolved; + } + } + } + return obj; + case JsonArray array: + for (var i = 0; i < array.Count; i++) + { + var item = array[i]; + if (item is not null) + { + var resolved = ResolveInJsonNode(item, resolve); + if (!ReferenceEquals(resolved, item)) + { + array[i] = resolved; + } + } + } + return array; + case JsonValue value when value.GetValueKind() == JsonValueKind.String: + var s = value.GetValue(); + var r = resolve(s); + return string.Equals(s, r, StringComparison.Ordinal) ? value : JsonValue.Create(r); + default: + return node; + } + } + + // ${env:NAME} accepts any characters except } so that names like ProgramFiles(x86) work; + // bare ${NAME} is limited to identifiers to avoid matching unrelated ${...} text + [GeneratedRegex(@"(?\$)?\$\{(?:env:(?[^}]+)|(?[A-Za-z_][A-Za-z0-9_]*))\}")] + private static partial Regex PlaceholderRegex(); +} diff --git a/DevProxy.Integration.Tests/MinimalPermissionsUtilsTests.cs b/DevProxy.Integration.Tests/MinimalPermissionsUtilsTests.cs new file mode 100644 index 000000000..1086c4cad --- /dev/null +++ b/DevProxy.Integration.Tests/MinimalPermissionsUtilsTests.cs @@ -0,0 +1,141 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Plugins.Utils; +using Microsoft.Extensions.Logging; +using System.Text.Json; +using Xunit; +using YamlDotNet.RepresentationModel; + +namespace DevProxy.Integration.Tests; + +public sealed class MinimalPermissionsUtilsTests : IDisposable +{ + private const string Variable = "DEVPROXY_TEST_SPEC_TENANT"; + + public MinimalPermissionsUtilsTests() + { + Environment.SetEnvironmentVariable(Variable, "a \"quoted\" value"); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable(Variable, null); + } + + [Fact] + public void ResolveApiSpecVariables_Yaml_ResolvesValuesOnly() + { + var yaml = $$""" + info: + title: "${{{Variable}}}" + ${{{Variable}}}: key + """; + var logger = new WarningLogger(); + + var result = MinimalPermissionsUtils.ResolveApiSpecVariables(yaml, "spec.yaml", [], logger); + + var stream = new YamlStream(); + stream.Load(new StringReader(result)); + var root = (YamlMappingNode)stream.Documents[0].RootNode; + Assert.Equal("a \"quoted\" value", ((YamlMappingNode)root["info"])["title"].ToString()); + Assert.Equal("key", root["${" + Variable + "}"].ToString()); + Assert.Empty(logger.Warnings); + } + + [Theory] + [InlineData("spec.json", """{ "a": "$${TENANT_ID}", "b": "{TENANT_ID}" }""")] + [InlineData("spec.yaml", "a: $${TENANT_ID}\nb: '{TENANT_ID}'")] + public void ResolveApiSpecVariables_LegacyVariablesDontChangeEscapedPlaceholders(string fileName, string contents) + { + var logger = new WarningLogger(); + + var result = MinimalPermissionsUtils.ResolveApiSpecVariables(contents, fileName, new() { ["TENANT_ID"] = "contoso" }, logger); + + var (a, b) = fileName.EndsWith(".json", StringComparison.Ordinal) ? ReadJson(result) : ReadYaml(result); + Assert.Equal("${TENANT_ID}", a); + Assert.Equal("contoso", b); + _ = Assert.Single(logger.Warnings, w => w.Contains("deprecated", StringComparison.Ordinal)); + } + + [Fact] + public void ResolveApiSpecVariables_Yaml_ResolvesUnquotedLegacyVariables() + { + var yaml = """ + servers: + - url: {BASE_URL} + escaped: $${BASE_URL} + """; + var logger = new WarningLogger(); + + var result = MinimalPermissionsUtils.ResolveApiSpecVariables(yaml, "spec.yaml", new() { ["BASE_URL"] = "https://api.contoso.com" }, logger); + + var stream = new YamlStream(); + stream.Load(new StringReader(result)); + var root = (YamlMappingNode)stream.Documents[0].RootNode; + var server = (YamlMappingNode)((YamlSequenceNode)root["servers"])[0]; + Assert.Equal("https://api.contoso.com", Assert.IsType(server["url"]).Value); + Assert.Equal("${BASE_URL}", root["escaped"].ToString()); + _ = Assert.Single(logger.Warnings, w => w.Contains("deprecated", StringComparison.Ordinal)); + } + + [Fact] + public void ResolveApiSpecVariables_Json_ResolvesLegacyVariablesInKeys() + { + var json = """{ "paths": { "/{TENANT_ID}/users/{id}": {}, "/$${TENANT_ID}": {} } }"""; + var logger = new WarningLogger(); + + var result = MinimalPermissionsUtils.ResolveApiSpecVariables(json, "spec.json", new() { ["TENANT_ID"] = "contoso" }, logger); + + using var doc = JsonDocument.Parse(result); + var paths = doc.RootElement.GetProperty("paths"); + Assert.True(paths.TryGetProperty("/contoso/users/{id}", out _)); + // keys aren't resolved, but the legacy pass must not change placeholders + Assert.True(paths.TryGetProperty("/$${TENANT_ID}", out _)); + _ = Assert.Single(logger.Warnings, w => w.Contains("deprecated", StringComparison.Ordinal)); + } + + [Fact] + public void ResolveApiSpecVariables_EscapedPlaceholderOnly_NoDeprecationWarning() + { + var logger = new WarningLogger(); + + var result = MinimalPermissionsUtils.ResolveApiSpecVariables("""{ "a": "$${TENANT_ID}" }""", "spec.json", new() { ["TENANT_ID"] = "contoso" }, logger); + + using var doc = JsonDocument.Parse(result); + Assert.Equal("${TENANT_ID}", doc.RootElement.GetProperty("a").GetString()); + Assert.Empty(logger.Warnings); + } + + private static (string? A, string? B) ReadJson(string json) + { + using var doc = JsonDocument.Parse(json); + return (doc.RootElement.GetProperty("a").GetString(), doc.RootElement.GetProperty("b").GetString()); + } + + private static (string? A, string? B) ReadYaml(string yaml) + { + var stream = new YamlStream(); + stream.Load(new StringReader(yaml)); + var root = (YamlMappingNode)stream.Documents[0].RootNode; + return (root["a"].ToString(), root["b"].ToString()); + } + + private sealed class WarningLogger : ILogger + { + public List Warnings { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (logLevel == LogLevel.Warning) + { + Warnings.Add(formatter(state, exception)); + } + } + } +} \ No newline at end of file diff --git a/DevProxy.Plugins/ApiCenter/ApiCenterClient.cs b/DevProxy.Plugins/ApiCenter/ApiCenterClient.cs index 7cb5fd027..fa7ddf5c1 100644 --- a/DevProxy.Plugins/ApiCenter/ApiCenterClient.cs +++ b/DevProxy.Plugins/ApiCenter/ApiCenterClient.cs @@ -55,6 +55,8 @@ internal ApiCenterClient(ApiCenterClientConfiguration configuration, ILogger _configuration.RewritesFile; + // ${name} in out.url is a regex named group reference, so leave placeholders + // as-is. RewritePlugin resolves ${env:NAME} per request. + protected override bool ResolveEnvironmentVariables => false; protected override void LoadData(string fileContents) { diff --git a/DevProxy.Plugins/Reporting/MinimalPermissionsGuidancePlugin.cs b/DevProxy.Plugins/Reporting/MinimalPermissionsGuidancePlugin.cs index f83fa5ffa..8e6c3faa7 100644 --- a/DevProxy.Plugins/Reporting/MinimalPermissionsGuidancePlugin.cs +++ b/DevProxy.Plugins/Reporting/MinimalPermissionsGuidancePlugin.cs @@ -6,6 +6,7 @@ using DevProxy.Abstractions.Proxy; using DevProxy.Abstractions.Utils; using DevProxy.Plugins.Models; +using DevProxy.Plugins.Utils; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.OpenApi; @@ -254,7 +255,7 @@ private async Task> LoadApiSpecsAsync(string try { var fileContents = await File.ReadAllTextAsync(file, cancellationToken); - fileContents = ProxyUtils.ReplaceVariables(fileContents, ProxyConfiguration.Env, v => $"{{{v}}}"); + fileContents = MinimalPermissionsUtils.ResolveApiSpecVariables(fileContents, file, ProxyConfiguration.Env, Logger); var settings = new OpenApiReaderSettings(); settings.AddYamlReader(); diff --git a/DevProxy.Plugins/Reporting/MinimalPermissionsPlugin.cs b/DevProxy.Plugins/Reporting/MinimalPermissionsPlugin.cs index 943b4b2ec..89ab020a1 100644 --- a/DevProxy.Plugins/Reporting/MinimalPermissionsPlugin.cs +++ b/DevProxy.Plugins/Reporting/MinimalPermissionsPlugin.cs @@ -6,6 +6,7 @@ using DevProxy.Abstractions.Proxy; using DevProxy.Abstractions.Utils; using DevProxy.Plugins.Models; +using DevProxy.Plugins.Utils; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.OpenApi; @@ -189,7 +190,7 @@ private async Task> LoadApiSpecsAsync(string try { var fileContents = await File.ReadAllTextAsync(file, cancellationToken); - fileContents = ProxyUtils.ReplaceVariables(fileContents, ProxyConfiguration.Env, v => $"{{{v}}}"); + fileContents = MinimalPermissionsUtils.ResolveApiSpecVariables(fileContents, file, ProxyConfiguration.Env, Logger); var settings = new OpenApiReaderSettings(); settings.AddYamlReader(); diff --git a/DevProxy.Plugins/Utils/MinimalPermissionsUtils.cs b/DevProxy.Plugins/Utils/MinimalPermissionsUtils.cs index e8e690678..4b204251a 100644 --- a/DevProxy.Plugins/Utils/MinimalPermissionsUtils.cs +++ b/DevProxy.Plugins/Utils/MinimalPermissionsUtils.cs @@ -2,13 +2,67 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using DevProxy.Abstractions.Utils; using Microsoft.Extensions.Logging; using System.IdentityModel.Tokens.Jwt; +using System.Text.Json; +using YamlDotNet.Core; namespace DevProxy.Plugins.Utils; static class MinimalPermissionsUtils { + /// + /// Resolves environment variable placeholders (${NAME}) in an API + /// specification and the deprecated {NAME} variables passed with --env. + /// + public static string ResolveApiSpecVariables(string fileContents, string filePath, Dictionary legacyVariables, ILogger logger) + { + var unresolvedVariables = new List(); + var usesLegacyVariables = false; + + // legacy {NAME} variables are replaced only in text outside of ${NAME} placeholders + // so that escaped placeholders and resolved values aren't changed + string replaceLegacyVariables(string text) + { + var replaced = ProxyUtils.ReplaceVariables(text, legacyVariables, v => $"{{{v}}}"); + usesLegacyVariables |= !string.Equals(text, replaced, StringComparison.Ordinal); + return replaced; + } + Func? transformText = legacyVariables.Count > 0 ? replaceLegacyVariables : null; + + string resolved; + try + { + // resolve values in the parsed document so that values with quotes don't break it + // and keys aren't changed + // legacy variables were always replaced in the raw text, including keys like + // OpenAPI paths and unquoted YAML values like `url: {NAME}`, so keep doing that + var withLegacyVariables = transformText is null + ? fileContents + : EnvironmentVariablePlaceholders.TransformOutsidePlaceholders(fileContents, transformText); + resolved = Path.GetExtension(filePath).Equals(".json", StringComparison.OrdinalIgnoreCase) + ? EnvironmentVariablePlaceholders.ResolveInJson(withLegacyVariables, Environment.GetEnvironmentVariable, unresolvedVariables) + : EnvironmentVariablePlaceholders.ResolveInYaml(withLegacyVariables, Environment.GetEnvironmentVariable, unresolvedVariables); + } + catch (Exception ex) when (ex is JsonException or YamlException) + { + unresolvedVariables.Clear(); + usesLegacyVariables = false; + resolved = EnvironmentVariablePlaceholders.Resolve(fileContents, Environment.GetEnvironmentVariable, unresolvedVariables, transformText); + } + EnvironmentVariablePlaceholders.LogUnresolvedVariables(logger, unresolvedVariables, filePath); + + if (usesLegacyVariables) + { + logger.LogWarning( + "Referencing variables using {{NAME}} in {File} is deprecated and will be removed in a future version. Use ${{NAME}} instead", + filePath); + } + + return resolved; + } + /// /// Gets the scopes from the JWT token. /// diff --git a/DevProxy.Tests/BaseLoaderPlaceholderTests.cs b/DevProxy.Tests/BaseLoaderPlaceholderTests.cs new file mode 100644 index 000000000..48c5482a5 --- /dev/null +++ b/DevProxy.Tests/BaseLoaderPlaceholderTests.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.Plugins; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using System.Text.Json; +using Xunit; + +namespace DevProxy.Tests; + +public sealed class BaseLoaderPlaceholderTests : IDisposable +{ + private const string Variable = "DEVPROXY_TEST_LOADER_PLACEHOLDER"; + private readonly string _folder = Directory.CreateTempSubdirectory("devproxy-loader").FullName; + + public BaseLoaderPlaceholderTests() + { + Environment.SetEnvironmentVariable(Variable, "resolved \"value\""); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable(Variable, null); + Directory.Delete(_folder, true); + } + + [Theory] + [InlineData("mocks.json", $$"""{ "body": "${{{Variable}}}", "missing": "${DEVPROXY_TEST_LOADER_MISSING}" }""")] + [InlineData("mocks.yaml", $$""" + body: ${{{Variable}}} + missing: ${DEVPROXY_TEST_LOADER_MISSING} + """)] + public async Task LoadsFileWithResolvedPlaceholders(string fileName, string contents) + { + var loader = await LoadAsync(fileName, contents, resolve: true); + + using var doc = JsonDocument.Parse(loader.Contents!); + Assert.Equal("resolved \"value\"", doc.RootElement.GetProperty("body").GetString()); + Assert.Equal("${DEVPROXY_TEST_LOADER_MISSING}", doc.RootElement.GetProperty("missing").GetString()); + } + + [Fact] + public async Task LeavesPlaceholdersWhenResolutionIsDisabled() + { + var loader = await LoadAsync("rewrites.json", $$"""{ "url": "${{{Variable}}}" }""", resolve: false); + + using var doc = JsonDocument.Parse(loader.Contents!); + Assert.Equal($"${{{Variable}}}", doc.RootElement.GetProperty("url").GetString()); + } + + private async Task LoadAsync(string fileName, string contents, bool resolve) + { + var path = Path.Combine(_folder, fileName); + await File.WriteAllTextAsync(path, contents); + using var httpClient = new HttpClient(); + var loader = new TestLoader(httpClient, NullLogger.Instance, path, resolve); + await loader.InitFileWatcherAsync(CancellationToken.None); + loader.Dispose(); + return loader; + } + + private sealed class TestLoader(HttpClient httpClient, ILogger logger, string filePath, bool resolve) : + BaseLoader(httpClient, logger, new FakeProxyConfiguration()) + { + public string? Contents { get; private set; } + protected override string FilePath => filePath; + protected override bool ResolveEnvironmentVariables => resolve; + protected override void LoadData(string fileContents) => Contents = fileContents; + } +} diff --git a/DevProxy/Extensions/ConfigurationManagerExtensions.cs b/DevProxy/Extensions/ConfigurationManagerExtensions.cs index d37e514c8..d9a9392cc 100644 --- a/DevProxy/Extensions/ConfigurationManagerExtensions.cs +++ b/DevProxy/Extensions/ConfigurationManagerExtensions.cs @@ -20,7 +20,7 @@ public static ConfigurationManager ConfigureDevProxyConfig(this ConfigurationMan { if (!string.IsNullOrEmpty(configFile) && File.Exists(configFile)) { - _ = configuration.AddConfigFile(configFile, optional: false, reloadOnChange: true); + _ = configuration.AddConfigFile(configFile, optional: false, reloadOnChange: true, environmentVariables: options.Env); return configuration; } } diff --git a/DevProxy/Program.cs b/DevProxy/Program.cs index ec32588f4..a573ed74f 100644 --- a/DevProxy/Program.cs +++ b/DevProxy/Program.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using DevProxy; +using DevProxy.Abstractions.Utils; using DevProxy.Commands; using DevProxy.Proxy; using DevProxy.State; @@ -359,6 +360,19 @@ static string EscapeArgument(string arg) return arg; } +static void WriteUnresolvedEnvironmentVariables(IConfiguration configuration, DevProxyConfigOptions options) +{ + var isJson = options.Output == DevProxy.Abstractions.Proxy.OutputFormat.Json || IsJsonOutputRequested(Environment.GetCommandLineArgs()); + foreach (var provider in ((IConfigurationRoot)configuration).Providers.OfType()) + { + foreach (var name in provider.UnresolvedEnvironmentVariables) + { + var message = $"Environment variable {name} referenced in {provider.Path ?? "configuration file"} isn't set. Leaving the placeholder as-is"; + Console.Error.WriteLine(isJson ? FormatJsonLogEntry("warning", message) : message); + } + } +} + static WebApplication BuildApplication( DevProxyConfigOptions options, ProcessEnvironment environment, HttpProxyEnvironment proxyEnvironment) { @@ -368,6 +382,22 @@ static WebApplication BuildApplication( var builder = WebApplication.CreateBuilder(); _ = builder.Configuration.ConfigureDevProxyConfig(options); + try + { + return BuildApplicationCore(builder, options, environment, proxyEnvironment); + } + catch + { + // config values with unresolved placeholders can fail startup before + // they're logged, so report them to help troubleshoot + WriteUnresolvedEnvironmentVariables(builder.Configuration, options); + throw; + } +} + +static WebApplication BuildApplicationCore( + WebApplicationBuilder builder, DevProxyConfigOptions options, ProcessEnvironment environment, HttpProxyEnvironment proxyEnvironment) +{ environment.Apply(builder.Configuration, options.Env); var changedProxyVariables = proxyEnvironment.GetChangedVariables(); _ = Announcement.ShowOnceAsync(); @@ -390,6 +420,12 @@ static WebApplication BuildApplication( var app = builder.Build(); + foreach (var provider in ((IConfigurationRoot)builder.Configuration).Providers.OfType()) + { + EnvironmentVariablePlaceholders.LogUnresolvedVariables( + app.Logger, provider.UnresolvedEnvironmentVariables, provider.Path ?? "configuration file"); + } + if (changedProxyVariables.Length > 0) { app.Logger.LogWarning( diff --git a/schemas/v4.0.0/rc.schema.json b/schemas/v4.0.0/rc.schema.json index 4db88db85..796ce5227 100644 --- a/schemas/v4.0.0/rc.schema.json +++ b/schemas/v4.0.0/rc.schema.json @@ -37,7 +37,7 @@ }, "env": { "type": "object", - "description": "Environment variables to set for the Dev Proxy process. Values are also used to replace {name} placeholders in API spec files used by the MinimalPermissions plugins. Values passed with --env replace these. Changes to HTTP_PROXY, HTTPS_PROXY, ALL_PROXY, or NO_PROXY (including lowercase variants) require a full stop/start; hot reload cannot refresh cached HTTP proxy settings.", + "description": "Environment variables to set for the Dev Proxy process. Values can be referenced as ${NAME} placeholders in this config file and in plugin data files. Values are also used to replace {name} placeholders in API spec files used by the MinimalPermissions plugins (deprecated; use ${NAME} instead). Values passed with --env replace these. Changes to HTTP_PROXY, HTTPS_PROXY, ALL_PROXY, or NO_PROXY (including lowercase variants) require a full stop/start; hot reload cannot refresh cached HTTP proxy settings.", "propertyNames": { "minLength": 1, "pattern": "^[^=]+$" diff --git a/skills/dev-proxy/references/analyze-api-usage.md b/skills/dev-proxy/references/analyze-api-usage.md index 95bf4368f..231950558 100644 --- a/skills/dev-proxy/references/analyze-api-usage.md +++ b/skills/dev-proxy/references/analyze-api-usage.md @@ -282,15 +282,15 @@ Use `ApiCenterOnboardingPlugin` to check whether the APIs an app uses are regist } ``` -For CI/CD, use environment variables with `@` prefix: +For CI/CD, reference environment variables with `${VAR_NAME}` (the older `@VAR_NAME` syntax is deprecated): ```json { "apiCenterOnboardingPlugin": { - "subscriptionId": "@AZURE_SUBSCRIPTION_ID", - "resourceGroupName": "@AZURE_RESOURCE_GROUP_NAME", - "serviceName": "@AZURE_APIC_INSTANCE_NAME", - "workspace": "@AZURE_APIC_WORKSPACE_NAME", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "resourceGroupName": "${AZURE_RESOURCE_GROUP_NAME}", + "serviceName": "${AZURE_APIC_INSTANCE_NAME}", + "workspace": "${AZURE_APIC_WORKSPACE_NAME}", "createApicEntryForNewApis": false } } diff --git a/skills/dev-proxy/references/ci-cd-integration.md b/skills/dev-proxy/references/ci-cd-integration.md index f3a98d222..48a0c2fe0 100644 --- a/skills/dev-proxy/references/ci-cd-integration.md +++ b/skills/dev-proxy/references/ci-cd-integration.md @@ -380,14 +380,14 @@ Run `devproxy api show` for endpoint information. All endpoints require `Authori ## Environment Variables for Azure API Center -Azure API Center plugins support `@ENV_VAR` syntax for CI/CD: +Reference environment variables in config files with `${ENV_VAR}` (the older `@ENV_VAR` syntax is deprecated): ```json { "apiCenterOnboardingPlugin": { - "subscriptionId": "@AZURE_SUBSCRIPTION_ID", - "resourceGroupName": "@AZURE_RESOURCE_GROUP_NAME", - "serviceName": "@AZURE_APIC_INSTANCE_NAME" + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "resourceGroupName": "${AZURE_RESOURCE_GROUP_NAME}", + "serviceName": "${AZURE_APIC_INSTANCE_NAME}" } } ``` diff --git a/skills/dev-proxy/references/configuration.md b/skills/dev-proxy/references/configuration.md index 7cbd22228..bc4b76d94 100644 --- a/skills/dev-proxy/references/configuration.md +++ b/skills/dev-proxy/references/configuration.md @@ -46,6 +46,28 @@ A configuration file follows a specific property order: `$schema`, then `plugins - File paths in Dev Proxy configuration files are relative to the file where they are defined, unless they start with `~/` (or `~\` on Windows), which expands to the user's home directory. - Configuration hot-reloads on file save (v2.1.0+) — no restart needed. Works for both the main config and plugin-specific files. +## Environment Variable Placeholders + +Reference environment variables in config files and plugin data files (mocks, CRUD API definitions and data, error responses, OpenAPI specs used by MinimalPermissions plugins, and so on) with `${VAR_NAME}`. `${env:VAR_NAME}` works too and also accepts names with characters like `(`, `-`, or `.`, for example `${env:ProgramFiles(x86)}`. Dev Proxy resolves placeholders when it loads the file. + +```json +{ + "entraAuthConfig": { + "audience": "${ENTRA_APP_CLIENT_ID}", + "issuer": "https://login.microsoftonline.com/${ENTRA_APP_TENANT_ID}/v2.0" + } +} +``` + +- In the config file, variables resolve from `--env` values first, then the config `env` section, then the OS environment. Values in the `env` section can themselves reference OS environment variables or `--env` values. +- Plugin data files resolve from the process environment, which includes the `env` section and `--env` values. +- If a variable isn't set, Dev Proxy leaves the placeholder as-is and logs a warning. +- Use `$${VAR_NAME}` for a literal `${VAR_NAME}`. +- Placeholders resolve in string values only (not property names). Config values like ports can use a string placeholder (`"port": "${PORT}"`); data files need the value's expected type. +- `${...}` placeholders resolve once at load; `@`-tokens such as `@dynamic` and `@request.body.*` resolve per request. +- Exception: RewritePlugin rewrite rules (in rewrites files or inline in the plugin config section) don't resolve placeholders at load because `${name}` in `out.url` refers to regex capture groups. Use `${env:VAR_NAME}`, which RewritePlugin resolves per request. +- Deprecated: `@VAR_NAME` in Azure API Center plugin settings and `{VAR_NAME}` in OpenAPI specs used by MinimalPermissions plugins still work but log a warning. Use `${VAR_NAME}` instead. + ## Plugin Architecture Dev Proxy has four plugin types: