Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 113 additions & 2 deletions crates/openshell-server/src/certgen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

use clap::Args;
use k8s_openapi::ByteString;
use k8s_openapi::api::core::v1::Secret;
use k8s_openapi::api::core::v1::{ConfigMap, Secret};
use kube::Client;
use kube::api::{Api, ObjectMeta, PostParams};
use miette::{IntoDiagnostic, Result, WrapErr};
Expand Down Expand Up @@ -78,6 +78,20 @@ pub struct CertgenArgs {
/// For local debugging.
#[arg(long)]
dry_run: bool,

/// Name of a ConfigMap to create containing the CA certificate (key: ca.crt)
/// for BackendTLSPolicy backend validation. In full PKI mode, the CA comes
/// from the generated bundle. In --jwt-only mode, the CA is read from
/// --backend-ca-source-secret.
#[arg(long, value_name = "NAME")]
backend_ca_configmap_name: Option<String>,

/// Name of an existing Secret containing a ca.crt key to populate the
/// backend CA ConfigMap from. Required with --jwt-only when
/// --backend-ca-configmap-name is set (typically the server TLS Secret
/// created by cert-manager).
#[arg(long, value_name = "NAME", requires = "backend_ca_configmap_name")]
backend_ca_source_secret: Option<String>,
}

pub async fn run(args: CertgenArgs) -> Result<()> {
Expand All @@ -97,7 +111,13 @@ pub async fn run(args: CertgenArgs) -> Result<()> {
run_local(dir, &args.server_sans)
} else {
let bundle = generate_pki(&args.server_sans)?;
run_kubernetes(&args, &bundle).await
run_kubernetes(&args, &bundle).await?;

if let Some(ref cm_name) = args.backend_ca_configmap_name {
create_backend_ca_configmap_if_needed(&args, &bundle, cm_name).await?;
}

Ok(())
}
}

Expand Down Expand Up @@ -293,6 +313,97 @@ async fn create_tls_secrets(
Ok(())
}

async fn create_backend_ca_configmap_if_needed(
args: &CertgenArgs,
bundle: &PkiBundle,
configmap_name: &str,
) -> Result<()> {
let namespace = args
.namespace
.as_deref()
.ok_or_else(|| miette::miette!("--namespace is required (or set POD_NAMESPACE)"))?;

let client = Client::try_default()
.await
.into_diagnostic()
.wrap_err("failed to construct Kubernetes client for backend CA ConfigMap")?;
let api: Api<ConfigMap> = Api::namespaced(client.clone(), namespace);

if api
.get_opt(configmap_name)
.await
.into_diagnostic()
.wrap_err_with(|| format!("failed to read configmap {configmap_name}"))?
.is_some()
{
info!(
namespace = %namespace,
configmap = %configmap_name,
"Backend CA ConfigMap already exists, skipping."
);
return Ok(());
}

let ca_pem = if !args.jwt_only {
bundle.ca_cert_pem.clone()
} else if let Some(source_secret) = &args.backend_ca_source_secret {
let secret_api: Api<Secret> = Api::namespaced(client, namespace);
match secret_api
.get_opt(source_secret)
.await
.into_diagnostic()
.wrap_err_with(|| format!("failed to read secret {source_secret}"))?
{
Some(secret) => {
let data = secret.data.ok_or_else(|| {
miette::miette!("secret {source_secret} has no data")
})?;
let ca = data.get("ca.crt").ok_or_else(|| {
miette::miette!("secret {source_secret} has no ca.crt key")
})?;
String::from_utf8(ca.0.clone())
.into_diagnostic()
.wrap_err("ca.crt is not valid UTF-8")?
}
None => {
warn!(
secret = %source_secret,
configmap = %configmap_name,
"Backend CA source secret not found; ConfigMap not created. \
Create it manually or run helm upgrade after the TLS secret exists."
);
return Ok(());
}
}
} else {
return Err(miette::miette!(
"--backend-ca-source-secret is required with --jwt-only \
and --backend-ca-configmap-name"
));
};

let configmap = ConfigMap {
metadata: ObjectMeta {
name: Some(configmap_name.to_string()),
..Default::default()
},
data: Some(BTreeMap::from([("ca.crt".to_string(), ca_pem)])),
..Default::default()
};

api.create(&PostParams::default(), &configmap)
.await
.into_diagnostic()
.wrap_err_with(|| format!("failed to create configmap {configmap_name}"))?;

info!(
namespace = %namespace,
configmap = %configmap_name,
"Backend CA ConfigMap created."
);
Ok(())
}

fn tls_secret(name: &str, crt_pem: &str, key_pem: &str, ca_pem: &str) -> Secret {
let mut data = BTreeMap::new();
data.insert(
Expand Down
53 changes: 53 additions & 0 deletions crates/openshell-server/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,59 @@ mod tests {
));
}

#[test]
fn generate_certs_backend_ca_configmap_flags_parse() {
let _lock = ENV_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let _g1 = EnvVarGuard::remove("OPENSHELL_DB_URL");
let _g2 = EnvVarGuard::remove("POD_NAMESPACE");

let cli = Cli::try_parse_from([
"openshell-gateway",
"generate-certs",
"--namespace",
"openshell",
"--jwt-only",
"--jwt-secret-name",
"openshell-jwt-keys",
"--backend-ca-configmap-name",
"openshell-backend-ca",
"--backend-ca-source-secret",
"openshell-server-tls",
])
.expect("backend CA ConfigMap flags should parse with --jwt-only");

assert!(matches!(
cli.command,
Some(super::Commands::GenerateCerts(_))
));
}

#[test]
fn generate_certs_backend_ca_source_secret_requires_configmap_name() {
let _lock = ENV_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let _g1 = EnvVarGuard::remove("OPENSHELL_DB_URL");
let _g2 = EnvVarGuard::remove("POD_NAMESPACE");

let err = Cli::try_parse_from([
"openshell-gateway",
"generate-certs",
"--namespace",
"openshell",
"--jwt-only",
"--jwt-secret-name",
"openshell-jwt-keys",
"--backend-ca-source-secret",
"openshell-server-tls",
])
.expect_err("--backend-ca-source-secret should require --backend-ca-configmap-name");

assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument);
}

#[test]
fn bare_invocation_with_no_db_url_parses_for_runtime_defaults() {
// db_url is Option<String> at the clap level so subcommand parsing
Expand Down
6 changes: 6 additions & 0 deletions deploy/helm/openshell/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version <vers
--set securityContext.runAsUser=null
```

On OpenShift 4.22+, end-to-end TLS is supported via `BackendTLSPolicy`. See the
[OpenShift install guide](https://docs.nvidia.com/openshell/latest/kubernetes/openshift#end-to-end-tls-openshift-422) for details.

## Available versions

| Tag | Source | Notes |
Expand Down Expand Up @@ -161,6 +164,9 @@ add `ci/values-spire.yaml` to the OpenShell release values files.
| certManager.serverDnsNames | list | `["openshell","openshell.openshell.svc","openshell.openshell.svc.cluster.local","localhost","openshell.localhost","*.openshell.localhost","host.docker.internal"]` | DNS SANs on the cert-manager-issued server certificate. |
| certManager.serverIpAddresses | list | `["127.0.0.1"]` | IP SANs on the cert-manager-issued server certificate. |
| fullnameOverride | string | `""` | Override the full generated resource name. |
| grpcRoute.backendTLSPolicy.caCertificateConfigMapName | string | `""` | Name of the ConfigMap containing the CA certificate (key: ca.crt) used to validate the gateway pod's TLS certificate. Defaults to <fullname>-backend-ca when empty. When pkiInitJob is enabled (default), the certgen hook creates this ConfigMap automatically. With cert-manager, the ConfigMap is created on the first helm upgrade after cert-manager has issued the server certificate. |
| grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and a ConfigMap containing the CA certificate for backend validation. |
| grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (<fullname>.<namespace>.svc.cluster.local) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. |
| grpcRoute.enabled | bool | `false` | Create a Gateway API GRPCRoute for the gateway service. |
| grpcRoute.gateway.className | string | `"eg"` | GatewayClass to reference. Envoy Gateway installs one named "eg". |
| grpcRoute.gateway.create | bool | `false` | When true, a Gateway resource is created in the release namespace. Set to false and provide name/namespace to attach to a pre-existing Gateway. |
Expand Down
3 changes: 3 additions & 0 deletions deploy/helm/openshell/README.md.gotmpl
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version <vers
--set securityContext.runAsUser=null
```

On OpenShift 4.22+, end-to-end TLS is supported via `BackendTLSPolicy`. See the
[OpenShift install guide](https://docs.nvidia.com/openshell/latest/kubernetes/openshift#end-to-end-tls-openshift-422) for details.

## Available versions

| Tag | Source | Notes |
Expand Down
4 changes: 2 additions & 2 deletions deploy/helm/openshell/templates/_gateway-workload.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ spec:
- name: tls-cert
mountPath: /etc/openshell-tls/server
readOnly: true
{{- if or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }}
{{- if and (not .Values.grpcRoute.backendTLSPolicy.enabled) (or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret)) }}
- name: tls-client-ca
mountPath: /etc/openshell-tls/client-ca
readOnly: true
Expand Down Expand Up @@ -152,7 +152,7 @@ spec:
- name: tls-cert
secret:
secretName: {{ .Values.server.tls.certSecretName }}
{{- if or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }}
{{- if and (not .Values.grpcRoute.backendTLSPolicy.enabled) (or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret)) }}
- name: tls-client-ca
secret:
{{- if or (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }}
Expand Down
7 changes: 7 additions & 0 deletions deploy/helm/openshell/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,13 @@ Returns a YAML list. Append extra SANs from values with range loops.
| toYaml }}
{{- end }}

{{/*
Name of the ConfigMap holding the backend CA for BackendTLSPolicy validation.
*/}}
{{- define "openshell.backendCaConfigMapName" -}}
{{- .Values.grpcRoute.backendTLSPolicy.caCertificateConfigMapName | default (printf "%s-backend-ca" (include "openshell.fullname" .)) -}}
{{- end }}

{{/*
Gateway workload kind. StatefulSet is the default because the default SQLite
database requires persistent per-pod storage.
Expand Down
26 changes: 26 additions & 0 deletions deploy/helm/openshell/templates/backend-tls-policy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

{{- if .Values.grpcRoute.backendTLSPolicy.enabled }}
{{- if .Values.server.disableTls }}
{{- fail "grpcRoute.backendTLSPolicy requires the gateway pod to serve TLS; set server.disableTls=false" }}
{{- end }}
apiVersion: gateway.networking.k8s.io/v1
kind: BackendTLSPolicy
metadata:
name: {{ include "openshell.fullname" . }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "openshell.labels" . | nindent 4 }}
spec:
targetRefs:
- group: ""
kind: Service
name: {{ include "openshell.fullname" . }}
validation:
caCertificateRefs:
- group: ""
kind: ConfigMap
name: {{ include "openshell.backendCaConfigMapName" . }}
hostname: {{ default (printf "%s.%s.svc.cluster.local" (include "openshell.fullname" .) .Release.Namespace) .Values.grpcRoute.backendTLSPolicy.hostname }}
{{- end }}
11 changes: 11 additions & 0 deletions deploy/helm/openshell/templates/certgen.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "create"]
{{- if .Values.grpcRoute.backendTLSPolicy.enabled }}
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create"]
{{- end }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
Expand Down Expand Up @@ -114,4 +119,10 @@ spec:
- --server-san={{ . }}
{{- end }}
{{- end }}
{{- if .Values.grpcRoute.backendTLSPolicy.enabled }}
- --backend-ca-configmap-name={{ include "openshell.backendCaConfigMapName" . }}
{{- if .Values.certManager.enabled }}
- --backend-ca-source-secret={{ .Values.server.tls.certSecretName }}
{{- end }}
{{- end }}
{{- end }}
2 changes: 2 additions & 0 deletions deploy/helm/openshell/templates/gateway-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,10 @@ data:
[openshell.gateway.tls]
cert_path = "/etc/openshell-tls/server/tls.crt"
key_path = "/etc/openshell-tls/server/tls.key"
{{- if not .Values.grpcRoute.backendTLSPolicy.enabled }}
client_ca_path = "/etc/openshell-tls/client-ca/ca.crt"
{{- end }}
{{- end }}

{{- if .Values.server.auth.allowUnauthenticatedUsers }}

Expand Down
4 changes: 2 additions & 2 deletions deploy/helm/openshell/templates/gateway.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ spec:
{{- if not .Values.grpcRoute.gateway.listener.tls.certificateRefs }}
{{- fail "grpcRoute.gateway.listener.tls.certificateRefs is required when grpcRoute.gateway.listener.protocol is HTTPS" }}
{{- end }}
{{- if not .Values.server.disableTls }}
{{- fail "grpcRoute.gateway.listener.protocol=HTTPS terminates TLS at Envoy Gateway, which forwards plaintext gRPC to the gateway pod; set server.disableTls=true so the pod listens plaintext (this chart does not render a BackendTLSPolicy for re-encryption to a TLS backend)" }}
{{- if and (not .Values.server.disableTls) (not .Values.grpcRoute.backendTLSPolicy.enabled) }}
{{- fail "grpcRoute.gateway.listener.protocol=HTTPS terminates TLS at the Gateway listener. Either set server.disableTls=true so the pod listens plaintext, or enable grpcRoute.backendTLSPolicy for end-to-end TLS re-encryption to the gateway pod." }}
{{- end }}
tls:
mode: Terminate
Expand Down
27 changes: 27 additions & 0 deletions deploy/helm/openshell/tests/gateway_config_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,33 @@ tests:
path: data["gateway.toml"]
pattern: '\[openshell\.gateway\.tls\]'

- it: omits client_ca_path from the TLS section when backendTLSPolicy is enabled
template: templates/gateway-config.yaml
set:
grpcRoute.backendTLSPolicy.enabled: true
asserts:
- matchRegex:
path: data["gateway.toml"]
pattern: '\[openshell\.gateway\.tls\]'
- matchRegex:
path: data["gateway.toml"]
pattern: 'cert_path\s*='
- matchRegex:
path: data["gateway.toml"]
pattern: 'key_path\s*='
- notMatchRegex:
path: data["gateway.toml"]
pattern: 'client_ca_path\s*='

- it: renders client_ca_path when backendTLSPolicy is disabled
template: templates/gateway-config.yaml
set:
grpcRoute.backendTLSPolicy.enabled: false
asserts:
- matchRegex:
path: data["gateway.toml"]
pattern: 'client_ca_path\s*=\s*"/etc/openshell-tls/client-ca/ca\.crt"'

- it: renders server_sans from certManager.serverDnsNames
set:
certManager.enabled: true
Expand Down
19 changes: 19 additions & 0 deletions deploy/helm/openshell/tests/statefulset_client_ca_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,25 @@ tests:
- notExists:
path: spec.template.spec.volumes[3].secret.items

- it: omits client CA volume and mount when backendTLSPolicy is enabled
template: templates/statefulset.yaml
set:
pkiInitJob.enabled: true
certManager.enabled: false
grpcRoute.backendTLSPolicy.enabled: true
asserts:
- notContains:
path: spec.template.spec.volumes
content:
name: tls-client-ca
any: true
- notContains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: tls-client-ca
mountPath: /etc/openshell-tls/client-ca
readOnly: true

# When cert-manager owns TLS, does not share its CA, and no separate client CA
# secret is configured, there is no client CA to mount: the volume must not
# render rather than mounting an empty secret name.
Expand Down
Loading
Loading