From 86cf2b6bb383ec1e38e031e6a2aab8bed76d044b Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 24 Oct 2026 10:09:52 -0500 Subject: [PATCH 01/28] Add AWS inventory with region and organization discovery Include optional SDK integration, cross-account role assumption, regression tests, and anonymized setup examples. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/test.yml | 4 +- Dockerfile | 3 + README.md | 5 + docs/source_aws.md | 484 ++++++++++++++ module/config/file_output.py | 4 +- module/netbox/object_classes.py | 1 + module/sources/__init__.py | 3 +- module/sources/aws/client.py | 156 +++++ module/sources/aws/config.py | 147 +++++ module/sources/aws/connection.py | 314 +++++++++ pyproject.toml | 3 + requirements-aws.txt | 2 + settings-example.ini | 52 ++ tests/test_aws_source.py | 1015 ++++++++++++++++++++++++++++++ uv.lock | 111 +++- 15 files changed, 2296 insertions(+), 8 deletions(-) create mode 100644 docs/source_aws.md create mode 100644 module/sources/aws/client.py create mode 100644 module/sources/aws/config.py create mode 100644 module/sources/aws/connection.py create mode 100644 requirements-aws.txt create mode 100644 tests/test_aws_source.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7c870977..661e28c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Test on: push: - branches: [ "main", "development" ] + branches: [ "main", "development", "AWS" ] pull_request: branches: [ "main", "development" ] workflow_dispatch: @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements-aws.txt pip install pytest - name: Install vcsim diff --git a/Dockerfile b/Dockerfile index 0852fa81..be3a2093 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,16 @@ FROM python:3.14-slim AS builder COPY requirements.txt . +COPY requirements-aws.txt . ARG VENV=/opt/netbox-sync/venv +ARG INSTALL_AWS=false # Install dependencies RUN python3 -m venv $VENV && \ $VENV/bin/python3 -m pip install --upgrade pip && \ $VENV/bin/pip install -r requirements.txt && \ + if [ "$INSTALL_AWS" = "true" ]; then $VENV/bin/pip install -r requirements-aws.txt; fi && \ $VENV/bin/pip install vmware-vcenter==9.1.1.0 && \ $VENV/bin/python3 -m pip uninstall -y pip && \ find $VENV -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf diff --git a/README.md b/README.md index f395b1e7..a6efed89 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ This is a tool to sync data from different sources to a NetBox instance. Available source types: * VMware vCenter Server +* AWS EC2, EBS, and network interfaces (see [AWS source](docs/source_aws.md)) * [bb-ricardo/check_redfish](https://github.com/bb-Ricardo/check_redfish) inventory files **IMPORTANT: READ INSTRUCTIONS CAREFULLY BEFORE RUNNING THIS PROGRAM** @@ -38,6 +39,10 @@ This ensures stale objects are removed from NetBox keeping an accurate current s * VMWare vCenter >= 6.0 #### Source: check_redfish (if used) * check_redfish >= 1.2.0 +#### Source: AWS (if used) +* Optional boto3 SDK, AWS read-only credentials, and automatic or explicit region discovery +* Optional AWS Organizations discovery with collector permissions and a trusted inventory role in each member account +* Existing VPC-specific NetBox VRFs for IP import; see [AWS setup](docs/source_aws.md) # Installing * here we assume we install in ```/opt``` diff --git a/docs/source_aws.md b/docs/source_aws.md new file mode 100644 index 00000000..d31b9939 --- /dev/null +++ b/docs/source_aws.md @@ -0,0 +1,484 @@ +# AWS EC2 source + +The `aws` source imports EC2 instances, attached EBS volumes, attached network +interfaces (ENIs), MAC addresses, and permitted private IPv4 / IPv6 addresses. +AWS is queried read-only; writes are only made to NetBox. This branch is based +on current main and does not depend on the oVirt or OpenStack branches. + +## Installation and authentication + +```sh +pip install -r requirements-aws.txt +# Alternatively: +uv sync --extra aws +``` + +Other sources do not require boto3. An enabled AWS source without its SDK or +credentials fails explicitly rather than continuing with empty inventory. +For Docker, enable the SDK explicitly: + +```sh +docker build --build-arg INSTALL_AWS=true -t netbox-sync:aws . +``` + +Use the standard boto3 credential chain: + +* EC2 instance roles, ECS task roles, or EKS workload identity. +* A shared AWS profile (`profile` below), including an assume-role profile for + cross-account access. +* `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`; temporary credentials also need + `AWS_SESSION_TOKEN`. Prefer roles over stored access keys. + +By default each configured source handles one AWS account. Regions are discovered +automatically by default (`regions = auto`); an explicit list is also supported. +For multiple accounts, use organization mode below or configure separate named +sources/profiles. The optional +`account_id` guard verifies the account returned by STS before importing anything. +Profile-based assume-role credential refresh is handled by boto3. Credentials +must allow access to every configured region. + +No SSH access, guest agents, inbound connections, or AWS write permissions are +needed. Outbound HTTPS to the regional EC2 and STS endpoints is required. + +Example IAM policy (attach to a dedicated role or non-root identity): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeVolumes", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeRegions", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +An assume-role profile additionally needs `sts:AssumeRole` permission on the +target role and a matching target-account trust policy (and external ID where +required). Single-account mode does not use Organizations permissions. + +## Configuration + +```ini +[source/production-aws] +type = aws +regions = auto +discovery_region = us-east-1 +profile = inventory +account_id = 123456789012 +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +vpc_vrf_relation = us-east-1/vpc-0123456789abcdef0 = Production VPC +tag_keys = Environment, Owner +tenant = Operations +``` + +All options also support the application's usual `NBS_SOURCE__*` +environment variables. `regions` and `tag_keys` accept YAML lists as well as +comma-separated strings. Quote the account ID in YAML so it remains a string. +`python netbox-sync.py -g` generates the full configuration. + +### Automatic or explicit regions + +`regions = auto` (also the default when omitted) calls EC2 `DescribeRegions` and +queries regions whose opt-in status is `opt-in-not-required` or `opted-in`. +Disabled regions (`not-opted-in`) are excluded. Each resource is mapped to the +region that returned it, never to the bootstrap region. + +Auto discovery requires a bootstrap endpoint: set `discovery_region`, or provide +a region through the AWS profile or `AWS_DEFAULT_REGION`. There is no hardcoded +commercial-region fallback; choose an endpoint in the appropriate AWS partition +(for example GovCloud or China when applicable). The account's enabled regions +in that partition are discovered. A missing bootstrap region fails explicitly. + +To restrict scope, use `regions = us-east-1, us-west-2` or a YAML list. Explicit +mode does not call `DescribeRegions`, does not need that permission, and ignores +`discovery_region`. Do not combine `auto` and named regions in the same list. + +Auto mode requires `ec2:DescribeRegions` in addition to the inventory permissions. +Failure to discover regions, an empty enabled-region result, or failure to query +any enabled region aborts synchronization before NetBox writes or pruning. +There is no silent fallback to a partial region list. Use explicit mode when +credentials are intentionally restricted to a subset of enabled regions. +Switching between modes changes discovery scope; inspect a dry run before +allowing writes, particularly if narrowing to an explicit list. + +### Organization / multi-account discovery + +For an organization with many member accounts, enable organization mode: + +```ini +[source/aws-organization] +type = aws +profile = inventory-collector +organization = True +organization_role_name = NetBoxInventoryReadOnly +discovery_region = us-east-1 +regions = auto +sync_private_ips = False +; Optional safety check: the account hosting the collector credentials +account_id = 123456789012 +; Start with one or two member accounts before enabling the full organization: +;organization_account_ids = 111122223333, 444455556666 +;organization_external_id = your-administrator-provided-external-id +;organization_include_management = False +``` + +The collector enumerates all account pages (including empty pages with a +continuation token), selects accounts with `State = ACTIVE`, and excludes the +organization's management account by default. It assumes the named role in each +selected account, verifies the resulting account identity, then discovers that +account's regions and resources. No EC2 permissions are needed on the collector +identity when management-account scanning is disabled. + +`organization_account_ids` is an optional allowlist, not an account-ID guard. +Quote IDs in YAML lists. An explicitly empty list is invalid; omit the option to +select all active members. Unknown, inactive, or excluded IDs in that list cause +an error rather than silently reducing scope. To scan the management account, set +`organization_include_management = True` and deploy the same inventory role +there. A role path such as `service-accounts/NetBoxInventoryReadOnly` is supported. +Role ARNs use the collector's AWS partition; cross-partition discovery is not +supported. + +All selected accounts are collected and validated before reconciliation. An +Organizations API error, role-assumption failure, account mismatch, or regional +discovery failure aborts the entire run before NetBox writes or pruning. No +inaccessible account is treated as empty. A selection with no eligible accounts +also fails. Account discovery and scanning are sequential. Fresh one-hour STS +credentials are obtained immediately before each account scan, so time spent +scanning earlier accounts does not consume later accounts' credential lifetime. +These member credentials are held only in memory and are not automatically +renewed within an individual account scan; expiration fails the run safely. +Collector profile credential refresh remains managed by boto3. + +#### AWS profile + +The profile supplies **collector credentials**, not a separate profile for every +member. No custom organization settings are needed in the AWS profile; the +organization options belong in the application's source section. + +For an existing named credentials entry `inventory-collector`, the AWS shared +config can contain: + +```ini +[profile inventory-collector] +region = us-east-1 +``` + +Store its credentials in the standard AWS shared credentials file under +`[inventory-collector]`, or use an existing assume-role/SSO profile supported by +boto3. On Windows the standard files are `%USERPROFILE%\.aws\config` and +`%USERPROFILE%\.aws\credentials`; on Linux they are `~/.aws/config` and +`~/.aws/credentials`. Authenticate SSO profiles before an interactive test. +For unattended jobs prefer a workload IAM role or an automatically refreshable +credential provider, not an expiring interactive SSO login. + +For example, if the collector itself uses an assume-role profile: + +```ini +[profile inventory-collector] +role_arn = arn:aws:iam::123456789012:role/NetBoxOrganizationCollector +source_profile = existing-base-profile +region = us-east-1 +``` + +The base profile must be authorized to assume that collector role, and the role +must trust it. The collector role then needs the permissions below. If the +collector runs with an attached IAM role or environment credentials, omit the +application `profile` setting entirely. Never store access keys in the source +configuration or commit credential files. + +#### Complete example: reporting roles in management and member accounts + +All account IDs, profile names, role names, and user names below are fictitious +examples. Replace them with your deployment's values. + +In this example, an IAM user's keys assume the management account's +`ExampleInventoryRole`, which then assumes a role also named +`ExampleInventoryRole` in each member account: + +```text +IAM user credentials (example-inventory-client profile) + -> arn:aws:iam::123456789012:role/ExampleInventoryRole + -> arn:aws:iam:::role/ExampleInventoryRole + -> member account EC2 discovery +``` + +Create these files for the OS user running the importer, on the machine or in +the container where it runs. Files on a workstation are not automatically +available to a remote host, service account, or container. Adjust account IDs +and role names for your deployment. + +**1. AWS shared credentials** (`%USERPROFILE%\.aws\credentials` on Windows, +`~/.aws/credentials` on Linux): + +```ini +[example-inventory-client] +aws_access_key_id = REPLACE_WITH_EXISTING_IAM_USER_ACCESS_KEY +aws_secret_access_key = REPLACE_WITH_EXISTING_IAM_USER_SECRET_KEY +; For temporary base credentials, also provide: +;aws_session_token = REPLACE_WITH_SESSION_TOKEN +``` + +Keep this file private and out of version control. The importer does not load +the standalone test script's `.env` file or its custom +`NETBOX_AWS_ACCESS_KEY_ID` / `NETBOX_AWS_SECRET_ACCESS_KEY` variables. +Alternatively, standard `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and +`AWS_SESSION_TOKEN` variables can supply base credentials; to use those for +the collector-role profile below, replace `source_profile` with +`credential_source = Environment`. Do not set both options. + +**2. AWS shared config** (`%USERPROFILE%\.aws\config` on Windows, +`~/.aws/config` on Linux): + +```ini +[profile example-inventory] +role_arn = arn:aws:iam::123456789012:role/ExampleInventoryRole +source_profile = example-inventory-client +region = us-east-1 +``` + +The `source_profile` must match the credentials section from step 1. The +`profile` prefix belongs in the shared config section, not the credentials +section. This profile performs the first role assumption; the importer performs +the member-account role assumptions. + +**3. Application settings** (add to your existing NetBox configuration): + +```ini +[source/aws] +type = aws +profile = example-inventory +organization = True +organization_role_name = ExampleInventoryRole +account_id = 123456789012 +regions = auto +discovery_region = us-east-1 +sync_private_ips = False + +; Optional: initially select one actual member account. +;organization_account_ids = 111122223333 +``` + +`profile` identifies the management-account AWS profile. +`organization_role_name` identifies the role inside each member account, even +though the profile and role names are related here. Setting `account_id` is an +optional guard for the collector account; it does not select member accounts. +The management account is excluded from EC2 scanning by default. +`sync_private_ips = False` allows the initial VM/disk/interface test without +pre-created VPC VRFs; configure VRFs and permitted subnets before enabling IP +import. + +Omit `organization_external_id` unless the member roles' trust policies require +an `sts:ExternalId` value. An external ID is not automatically the management +account ID. If the collector role itself requires an external ID, set +`external_id` in the AWS shared config profile; that is separate from the +application's member-role `organization_external_id`. + +Before running the importer, verify the first role assumption with the AWS CLI +under the same OS user and environment: + +```sh +aws sts get-caller-identity --profile example-inventory +``` + +The ARN should resemble +`arn:aws:sts::123456789012:assumed-role/ExampleInventoryRole/`, +not the original IAM user's ARN. This checks only the collector identity, not +member-role trust or all inventory permissions. Then run a dry run: + +```sh +python netbox-sync.py -n -c settings.ini +``` + +For this layout, adapt the policies below to use `ExampleInventoryRole` rather than +`NetBoxInventoryReadOnly`. The IAM user must be allowed to assume the management +role, and that role must trust the IAM user. The management role needs +Organizations access and permission to assume the member roles. Each member +role must trust +`arn:aws:iam::123456789012:role/ExampleInventoryRole` +and have the inventory read permissions (for example AWS-managed +`ReadOnlyAccess`). The importer does not create these roles or grants. + +#### Collector permissions + +Run from the management account or an appropriately delegated administrator +allowed to list organization accounts. Attach this policy to the collector user +or role, adjusting the role path and partition if needed: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "organizations:DescribeOrganization", + "organizations:ListAccounts" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::*:role/NetBoxInventoryReadOnly" + } + ] +} +``` + +Restrict the role ARN account list to your member accounts where feasible. +AWS-managed `ReadOnlyAccess` covers the EC2 inventory permissions but does not +replace the required cross-account `sts:AssumeRole` grant. + +#### Member-account inventory role + +Deploy `NetBoxInventoryReadOnly` in every selected member account, for example +with CloudFormation StackSets. Attach the EC2 read policy above or AWS-managed +`ReadOnlyAccess`. Set its trust policy to allow the **exact collector principal**. +For an example IAM user acting directly as the collector, that policy is: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::123456789012:user/example-inventory-client" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +If using the collector-role profile above, replace the principal with +`arn:aws:iam::123456789012:role/NetBoxOrganizationCollector`. +If your administrator requires an external ID, add a +`Condition: {"StringEquals": {"sts:ExternalId": ""}}` to the trust +statement and set the same `organization_external_id` in the application. +MFA-protected member-role assumption is not interactive; use an administrator- +approved unattended trust policy. SCPs and permissions boundaries must also +allow the calls. No roles or IAM policies are created by the importer. + +Adding an allowlist, excluding management, or removing/inactivating an account +changes the source's authoritative scope and can orphan previously imported +objects under the existing pruning rules. Review a dry run before changing +scope. Avoid overlapping single-account and organization source definitions. + +### VPC address-space isolation + +The current NetBox integration reads VRFs and prefixes but does not create them. +Before enabling IP import, create a **distinct VRF for every VPC** in NetBox. +The default VRF name is: + +```text +AWS // +``` + +For example: `AWS 123456789012/us-east-1/vpc-0123456789abcdef0`. +Use `vpc_vrf_relation` to select an existing differently named VRF. A mapping +key may be `account-id/region/vpc-id` in organization mode; this takes precedence +over a legacy `region/vpc-id` mapping. For example: + +```ini +vpc_vrf_relation = 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC +``` + +A mapping must not share a VRF between discovered VPCs. VRFs must remain distinct across +accounts and separately configured sources too. Organization mode validates +VRF isolation across all selected accounts before reconciliation. + +Missing VRFs for permitted addresses abort the run before inventory updates. +There is no fallback to global address space. Subnet mask lengths come from AWS +subnet CIDRs; NetBox prefixes are neither required nor created. Identical IPs in +different VPCs remain separate NetBox IPs. + +`permitted_subnets` is required to populate IP addresses; leaving it undefined +imports no IPs, matching other source behavior. `sync_private_ips = False` +imports VMs, disks, and interfaces without requiring VRFs; previously managed +IPs on discovered interfaces are retained. DNS lookup defaults to false because +private AWS DNS is often inaccessible from the sync host. + +## Object mapping and lifecycle + +| AWS | NetBox | +| --- | --- | +| Account ID | Cluster group `AWS ` | +| Account and region | Cluster `AWS /`, type `AWS EC2` | +| Region | Logical site `AWS ` | +| Instance ID | VM name; serial on NetBox 4.1+ | +| Instance type | CPU and memory, plus custom field | +| Name tag / Availability Zone | Custom fields, not identity or physical sites | +| ENI ID | VM interface name and custom field | +| ENI MAC | MAC object on NetBox 4.2+; legacy interface field on older versions | +| Private IPv4 / IPv6 | IP in the VPC's existing VRF, assigned to the ENI | +| EBS volume | Virtual disk named by volume ID | + +Region sites and clusters are logical containers, not claims about physical +AWS facilities or hypervisor placement. Instance IDs within account/region +clusters avoid duplicate or changing Name tags merging unrelated VMs. Memory +uses the instance type's MiB capacity; CPU uses instance CPU options when +reported, otherwise the type's default vCPU count. This is provisioned capacity, +not utilization. + +Running instances are `active`; stopped, pending, stopping, and shutting-down +instances are `offline`. Terminated instances are excluded and subsequently +follow the existing NetBox orphaning/pruning configuration. An empty, successfully +queried account/region is authoritative. All configured regions and paginated +responses are collected before reconciliation. AWS access failures, credential +failures, missing attached resources, and missing VRF mappings abort the run +before the application's NetBox write/prune phase. Transient attachment changes +may therefore require retrying the run rather than importing partial data. + +NetBox 3.7+ gets separate virtual disks; older versions get aggregate VM disk +size. Disk sizes use GiB before NetBox 4.1 and MiB from 4.1 onward, consistent +with the existing VMware source's binary-size convention. Only attached EBS +volumes are imported; instance-store disks and unattached volumes are excluded. + +The primary IPv4 is the primary private address on device-index-zero ENI; a +reported primary IPv6 on that ENI is handled similarly. Public IPv4 mappings +are stored on the ENI as `aws_public_ipv4`, not assigned as guest IPs or made +primary. IPv6 addresses are actual interface addresses even when globally +routable. Delegated address prefixes are not imported. + +Only explicitly selected `tag_keys` become tags (`AWS:=`). Their +combined names must fit NetBox's 100-character limit. Existing unrelated NetBox +tags are preserved. Metadata custom fields use the `aws_` prefix. Their names +should not conflict with user-defined fields of incompatible types. + +Avoid overlapping source definitions for the same account/region. Narrowing +regions, disabling IP import, or changing permitted networks affects the +source's managed scope; review a dry run before making such changes. The +existing pruning settings determine what happens to objects no longer reported. + +## Validation + +Start with: + +```sh +python netbox-sync.py -n -c settings.ini +``` + +Tests use botocore `Stubber` and the real in-memory NetBox inventory, without +AWS credentials or network calls: + +```sh +pip install -r requirements-aws.txt pytest +python -m pytest tests/test_aws_source.py +``` + +Live AWS/NetBox validation is still required before production use. This source +does not inventory RDS, load balancers, ECS/EKS workloads, security groups, +software, or guest operating-system configuration. diff --git a/module/config/file_output.py b/module/config/file_output.py index 6a002f15..9acf4e5a 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -17,6 +17,7 @@ from module.sources.vmware.config import VMWareConfig from module.sources.check_redfish.config import CheckRedfishConfig from module.sources.hetzner.config import HetznerConfig +from module.sources.aws.config import AWSConfig from module.common.logging import get_logger from module.config import default_config_file_path, source_config_section_name from module.config.files import ConfigFile, ConfigFileINI, ConfigFileYAML @@ -35,7 +36,8 @@ class ConfigFileOutput(DescriptionFormatterMixin): source_config_list = [ VMWareConfig, CheckRedfishConfig, - HetznerConfig + HetznerConfig, + AWSConfig ] header = f"Welcome to the {__description__} configuration file." diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index e6e8c813..dacc36fe 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2032,6 +2032,7 @@ def __init__(self, *args, **kwargs): "untagged_vlan": NBVLAN, "tagged_vlans": NBVLANList, "description": 200, + "custom_fields": NBCustomField, "tags": NBTagList } super().__init__(*args, **kwargs) diff --git a/module/sources/__init__.py b/module/sources/__init__.py index 0666da30..5cdc0f1b 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -10,6 +10,7 @@ # define all available sources here from module.sources.vmware.connection import VMWareHandler from module.sources.hetzner.connection import HetznerHandler +from module.sources.aws.connection import AWSHandler from module.sources.check_redfish.import_inventory import CheckRedfish from module.common.logging import get_logger @@ -19,7 +20,7 @@ from module.config import source_config_section_name # list of valid sources -valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler] +valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler, AWSHandler] def validate_source(source_class_object=None, state="pre"): diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py new file mode 100644 index 00000000..8fc2f308 --- /dev/null +++ b/module/sources/aws/client.py @@ -0,0 +1,156 @@ +from module.common.logging import get_logger + +log = get_logger() + +try: + import boto3 + from botocore.config import Config + from botocore.exceptions import BotoCoreError, ClientError +except ModuleNotFoundError as exc: + if exc.name != "boto3": + raise + boto3 = None + + +class AWSDiscoveryError(RuntimeError): + """Discovery failed; do not reconcile this run with NetBox.""" + + +class AWSClient: + def __init__(self, profile=None, *, session=None): + if boto3 is None: + raise AWSDiscoveryError("AWS SDK missing. Install dependencies from requirements-aws.txt.") + try: + self.session = session if session is not None else boto3.Session(profile_name=profile) + except BotoCoreError as exc: + raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc + + def discover_organization(self, regions, role_name, expected_account=None, discovery_region=None, + include_management=False, external_id=None, account_ids=None): + """Collect all selected accounts; never return a partial organization.""" + clients = [] + current_account = None + try: + bootstrap = (discovery_region or self.session.region_name) if regions == ["auto"] else regions[0] + if not bootstrap: + raise AWSDiscoveryError( + "Organization discovery requires discovery_region or an AWS profile/environment region " + "(AWS_DEFAULT_REGION).") + config = Config(retries={"mode": "standard", "max_attempts": 5}, + connect_timeout=10, read_timeout=60) + sts = self.session.client("sts", region_name=bootstrap, config=config) + clients.append(sts) + identity = sts.get_caller_identity() + if expected_account is not None and identity["Account"] != expected_account: + raise AWSDiscoveryError( + f"AWS account mismatch: expected {expected_account}, authenticated as {identity['Account']}.") + partition = identity["Arn"].split(":")[1] + organizations = self.session.client("organizations", region_name=bootstrap, config=config) + clients.append(organizations) + management_account = organizations.describe_organization()["Organization"]["MasterAccountId"] + accounts = set() + for page in organizations.get_paginator("list_accounts").paginate(): + for account in page["Accounts"]: + account_id = account["Id"] + if account["State"] != "ACTIVE": + log.info(f"AWS organization: excluding account '{account_id}' in state '{account['State']}'.") + continue + if account_id == management_account and not include_management: + continue + accounts.add(account_id) + if account_ids: + missing = set(account_ids) - accounts + if missing: + raise AWSDiscoveryError( + f"Requested organization accounts are not active/eligible: {', '.join(sorted(missing))}.") + accounts.intersection_update(account_ids) + if not accounts: + raise AWSDiscoveryError("Organization discovery returned no eligible accounts.") + snapshots = {} + for current_account in sorted(accounts): + role_arn = f"arn:{partition}:iam::{current_account}:role/{role_name}" + log.info(f"AWS organization: discovering account '{current_account}' using '{role_arn}'.") + request = {"RoleArn": role_arn, "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} + if external_id: + request["ExternalId"] = external_id + credentials = sts.assume_role(**request)["Credentials"] + member = AWSClient(session=boto3.Session( + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], region_name=bootstrap)) + _, snapshot = member.discover(regions, current_account, discovery_region) + snapshots[current_account] = snapshot + return snapshots + except (BotoCoreError, ClientError, KeyError, AWSDiscoveryError) as exc: + context = f" for account '{current_account}'" if current_account else "" + raise AWSDiscoveryError(f"Incomplete AWS organization discovery{context}: {exc}") from exc + finally: + for client in clients: + client.close() + + def discover(self, regions, expected_account=None, discovery_region=None): + """Fully collect all configured regions before any inventory changes.""" + clients = [] + try: + config = Config(retries={"mode": "standard", "max_attempts": 5}, + connect_timeout=10, read_timeout=60) + automatic = regions == ["auto"] + bootstrap = (discovery_region or self.session.region_name) if automatic else regions[0] + if not bootstrap: + raise AWSDiscoveryError( + "Automatic region discovery requires discovery_region or an AWS profile/environment region " + "(AWS_DEFAULT_REGION).") + sts = self.session.client("sts", region_name=bootstrap, config=config) + clients.append(sts) + account = sts.get_caller_identity()["Account"] + if expected_account is not None and account != expected_account: + raise AWSDiscoveryError( + f"AWS account mismatch: expected {expected_account}, authenticated as {account}.") + if automatic: + discovery_client = self.session.client("ec2", region_name=bootstrap, config=config) + clients.append(discovery_client) + response = discovery_client.describe_regions(AllRegions=True) + regions = sorted({ + entry["RegionName"] for entry in response["Regions"] + if entry["OptInStatus"] in ("opt-in-not-required", "opted-in") + }) + if not regions: + raise AWSDiscoveryError("Automatic region discovery returned no enabled regions.") + log.info(f"AWS account '{account}' enabled regions: {', '.join(regions)}") + snapshot = {} + for region in regions: + ec2 = self.session.client("ec2", region_name=region, config=config) + clients.append(ec2) + instances = [ + instance + for page in ec2.get_paginator("describe_instances").paginate() + for reservation in page.get("Reservations", []) + for instance in reservation.get("Instances", []) + if instance["State"]["Name"] != "terminated" + ] + data = {"instances": instances} + for operation, result_key, key in ( + ("describe_network_interfaces", "NetworkInterfaces", "NetworkInterfaceId"), + ("describe_volumes", "Volumes", "VolumeId"), + ("describe_subnets", "Subnets", "SubnetId"), + ("describe_vpcs", "Vpcs", "VpcId"), + ): + data[result_key] = { + item[key]: item + for page in ec2.get_paginator(operation).paginate() + for item in page.get(result_key, []) + } + instance_types = {} + names = sorted({item["InstanceType"] for item in instances}) + for start in range(0, len(names), 100): + for page in ec2.get_paginator("describe_instance_types").paginate( + InstanceTypes=names[start:start + 100]): + instance_types.update({item["InstanceType"]: item for item in page["InstanceTypes"]}) + data["InstanceTypes"] = instance_types + snapshot[region] = data + return account, snapshot + except (BotoCoreError, ClientError, KeyError) as exc: + raise AWSDiscoveryError(f"Incomplete AWS discovery; synchronization aborted: {exc}") from exc + finally: + for client in clients: + client.close() diff --git a/module/sources/aws/config.py b/module/sources/aws/config.py new file mode 100644 index 00000000..75bfc774 --- /dev/null +++ b/module/sources/aws/config.py @@ -0,0 +1,147 @@ +import re + +from module.common.logging import get_logger +from module.common.misc import quoted_split +from module.config import source_config_section_name +from module.config.base import ConfigBase +from module.config.option import ConfigOption +from module.sources.common.config import ( + config_option_enabled_definition, + config_option_permitted_subnets_definition, + config_option_type_definition, +) +from module.sources.common.permitted_subnets import PermittedSubnets + +log = get_logger() + + +class AWSConfig(ConfigBase): + section_name = source_config_section_name + source_name_example = "my-aws-example" + source_name = None + + def __init__(self): + self.options = [ + ConfigOption(**config_option_enabled_definition), + ConfigOption(**{**config_option_type_definition, "config_example": "aws"}), + ConfigOption("regions", str, default_value="auto", + description="""Use 'auto' to discover all account-enabled regions, or specify + regions to import (comma-separated or a YAML list). Do not mix auto with region names."""), + ConfigOption("discovery_region", str, + description="""Bootstrap region for automatic region discovery. Defaults to the + AWS profile/environment region. This selects the API endpoint, not resource placement."""), + ConfigOption("profile", str, + description="AWS SDK profile. Otherwise use the standard credential chain."), + ConfigOption("account_id", str, + description="""Optional expected 12-digit credential account ID. + In organization mode this guards the collector account, not member accounts."""), + ConfigOption("organization", bool, default_value=False, + description="Discover active AWS Organizations accounts and assume an inventory role in each."), + ConfigOption("organization_role_name", str, default_value="NetBoxInventoryReadOnly", + description="Inventory role name (including any path) deployed in each selected account."), + ConfigOption("organization_external_id", str, sensitive=True, + description="Optional external ID required by the member inventory roles' trust policies."), + ConfigOption("organization_include_management", bool, default_value=False, + description="Also scan the organization management account using its inventory role."), + ConfigOption("organization_account_ids", str, + description="""Optional account allowlist (comma-separated or YAML list). + Unset selects all active member accounts. Management inclusion is controlled separately."""), + ConfigOption(**config_option_permitted_subnets_definition), + ConfigOption("sync_private_ips", bool, default_value=True, + description="Import private IPv4 and IPv6 into pre-existing VPC-specific VRFs."), + ConfigOption("vpc_vrf_relation", str, + description="""Comma-separated region/vpc-id = NetBox VRF name mappings. + Use account-id/region/vpc-id keys for account-specific mappings in organization mode. + Unmapped VPCs require a VRF named AWS //. + Each VPC must have a distinct VRF; global address space is never used.""", + config_example="us-east-1/vpc-0123456789abcdef0 = Production VPC"), + ConfigOption("tag_keys", str, + description="AWS tag keys to copy as namespaced NetBox tags; unset imports none."), + ConfigOption("tenant", str, description="Optional NetBox tenant for AWS VMs and IPs."), + ConfigOption("dns_name_lookup", bool, default_value=False, + description="Perform reverse DNS lookups for imported private IPs."), + ] + super().__init__() + + def validate_options(self): + for key in ("profile", "tenant", "discovery_region", + "organization_role_name", "organization_external_id"): + value = self.get_option_by_name(key).value + if value is not None and not isinstance(value, str): + log.error(f"AWS '{self.source_name}' option '{key}' must be a string.") + self.set_validation_failed() + + role = self.get_option_by_name("organization_role_name").value + if isinstance(role, str) and ( + len(role) > 512 or not re.fullmatch(r"(?:[A-Za-z0-9_+=,.@-]+/)*[A-Za-z0-9_+=,.@-]{1,64}", role)): + log.error(f"AWS '{self.source_name}' organization_role_name must be a role name with an optional path.") + self.set_validation_failed() + external_id = self.get_option_by_name("organization_external_id").value + if isinstance(external_id, str) and not re.fullmatch(r"[A-Za-z0-9_+=,.@:/-]{2,1224}", external_id): + log.error(f"AWS '{self.source_name}' organization_external_id has an invalid format.") + self.set_validation_failed() + + for key in ("regions", "tag_keys", "organization_account_ids"): + option = self.get_option_by_name(key) + value = option.value + if value is not None and not isinstance(value, (str, list)): + log.error(f"AWS '{self.source_name}' option '{key}' must be a string or list.") + self.set_validation_failed() + continue + entries = value if isinstance(value, list) else quoted_split(value) if value else [] + if any(not isinstance(entry, str) or not entry.strip() for entry in entries): + log.error(f"AWS '{self.source_name}' option '{key}' contains an invalid entry.") + self.set_validation_failed() + continue + entries = list(dict.fromkeys(entry.strip() for entry in entries)) + if key == "regions" and not entries: + log.error(f"AWS '{self.source_name}' requires 'auto' or at least one explicit region.") + self.set_validation_failed() + if key == "regions" and "auto" in entries and entries != ["auto"]: + log.error(f"AWS '{self.source_name}' cannot mix 'auto' with explicit regions.") + self.set_validation_failed() + if key == "organization_account_ids" and any( + not re.fullmatch(r"[0-9]{12}", entry) for entry in entries): + log.error(f"AWS '{self.source_name}' organization_account_ids must contain 12-digit strings.") + self.set_validation_failed() + if key == "organization_account_ids" and value is not None and not entries: + log.error(f"AWS '{self.source_name}' organization_account_ids must be unset or nonempty.") + self.set_validation_failed() + option.set_value(entries) + + if not self.get_option_by_name("organization").value and ( + self.get_option_by_name("organization_account_ids").value or + self.get_option_by_name("organization_external_id").value or + self.get_option_by_name("organization_include_management").value): + log.error(f"AWS '{self.source_name}' organization options require organization = True.") + self.set_validation_failed() + + account = self.get_option_by_name("account_id").value + if account is not None and (not isinstance(account, str) or len(account) != 12 or not account.isdigit()): + log.error(f"AWS '{self.source_name}' account_id must be a 12-digit string.") + self.set_validation_failed() + + relation = self.get_option_by_name("vpc_vrf_relation") + mappings = {} + relation_value = relation.value + if relation_value is not None and not isinstance(relation_value, str): + log.error(f"AWS '{self.source_name}' vpc_vrf_relation must be a string.") + self.set_validation_failed() + relation_value = None + for entry in quoted_split(relation_value) if relation_value else []: + key, separator, value = entry.partition("=") + key, value = key.strip(' "\''), value.strip(' "\'') + parts = key.split("/") + if not separator or len(parts) not in (2, 3) or not all(parts) or not value or key in mappings or ( + len(parts) == 3 and not re.fullmatch(r"[0-9]{12}", parts[0])): + log.error(f"Malformed or duplicate AWS VPC/VRF mapping '{entry}'.") + self.set_validation_failed() + continue + mappings[key] = value + relation.set_value(mappings) + + permitted = self.get_option_by_name("permitted_subnets") + subnets = PermittedSubnets(permitted.value) + if subnets.validation_failed: + self.set_validation_failed() + permitted.set_value(subnets) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py new file mode 100644 index 00000000..913df14f --- /dev/null +++ b/module/sources/aws/connection.py @@ -0,0 +1,314 @@ +from ipaddress import ip_address, ip_interface, ip_network + +from packaging.version import Version + +from module.common.logging import get_logger +from module.common.misc import do_error_exit +from module.common.support import normalize_mac_address +from module.netbox import ( + NBCustomField, NBCluster, NBClusterGroup, NBClusterType, NBIPAddress, + NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) +from module.netbox.inventory import NetBoxInventory +from module.sources.aws.client import AWSClient, AWSDiscoveryError +from module.sources.aws.config import AWSConfig +from module.sources.common.source_base import SourceBase + +log = get_logger() + + +class AWSHandler(SourceBase): + source_type = "aws" + dependent_netbox_objects = [ + NBTag, NBCustomField, NBTenant, NBSite, NBClusterGroup, NBClusterType, + NBCluster, NBVM, NBVMInterface, NBMACAddress, NBIPAddress, NBVRF, NBVirtualDisk, + ] + + def __init__(self, name=None): + if not name: + raise ValueError(f"Invalid source name '{name}'.") + self.name = name + self.inventory = NetBoxInventory() + config = AWSConfig() + config.source_name = name + self.settings = config.parse() + self.set_source_tag() + self.init_successful = False + if not self.settings.enabled: + log.info(f"Source '{name}' is disabled. Skipping.") + return + try: + self.client = AWSClient(profile=self.settings.profile) + except AWSDiscoveryError as exc: + self.abort(exc) + self.init_successful = True + + def abort(self, error): + message = f"AWS source '{self.name}': {error}. No NetBox writes or pruning will be performed." + log.error(message) + do_error_exit(message) + + def apply(self): + try: + if self.settings.organization: + snapshots = self.client.discover_organization( + self.settings.regions, self.settings.organization_role_name, + expected_account=self.settings.account_id, + discovery_region=self.settings.discovery_region, + include_management=self.settings.organization_include_management, + external_id=self.settings.organization_external_id, + account_ids=self.settings.organization_account_ids) + else: + account, snapshot = self.client.discover( + self.settings.regions, self.settings.account_id, self.settings.discovery_region) + snapshots = {account: snapshot} + vrfs = {} + for account, snapshot in snapshots.items(): + self.validate_snapshot(account, snapshot, vrfs) + for account, snapshot in snapshots.items(): + self.sync_snapshot(account, snapshot) + except (AWSDiscoveryError, ValueError, KeyError) as exc: + self.abort(exc) + + def vrf_name(self, account, region, vpc): + return self.settings.vpc_vrf_relation.get( + f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( + f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) + + def get_vrf(self, account, region, vpc): + name = self.vrf_name(account, region, vpc) + vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) + if vrf is None: + raise ValueError(f"Create NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") + return vrf + + def addresses(self, eni, subnet): + networks = [ip_network(subnet["CidrBlock"])] + networks.extend( + ip_network(entry["Ipv6CidrBlock"]) + for entry in subnet.get("Ipv6CidrBlockAssociationSet", []) + if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" + ) + addresses = [ + (entry["PrivateIpAddress"], entry.get("Primary", False)) + for entry in eni.get("PrivateIpAddresses", []) + ] + addresses.extend( + (entry["Ipv6Address"], entry.get("IsPrimaryIpv6", False)) + for entry in eni.get("Ipv6Addresses", []) + ) + result = [] + for value, primary in addresses: + address = ip_address(value) + network = next((net for net in networks + if net.version == address.version and address in net), None) + if network is None: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' IP '{value}' has no matching AWS subnet") + value = f"{address}/{network.prefixlen}" + if self.settings.permitted_subnets.permitted(value, interface_name=eni["NetworkInterfaceId"]): + result.append((value, primary)) + return result + + def validate_snapshot(self, account, snapshot, vrfs=None): + if vrfs is None: + vrfs = {} + for region, data in snapshot.items(): + for instance in data["instances"]: + spec = data["InstanceTypes"].get(instance["InstanceType"]) + if spec is None: + raise ValueError(f"Missing AWS instance type '{instance['InstanceType']}'") + if "DefaultVCpus" not in spec["VCpuInfo"] or "SizeInMiB" not in spec["MemoryInfo"]: + raise ValueError(f"Incomplete AWS specifications for '{instance['InstanceType']}'") + for tag in instance.get("Tags", []): + if tag["Key"] in self.settings.tag_keys and len(f"AWS:{tag['Key']}={tag['Value']}") > 100: + raise ValueError(f"AWS tag '{tag['Key']}' exceeds the NetBox tag name limit (100 characters)") + for attachment in instance.get("BlockDeviceMappings", []): + volume_id = attachment.get("Ebs", {}).get("VolumeId") + if volume_id and volume_id not in data["Volumes"]: + raise ValueError(f"Missing attached EBS volume '{volume_id}'") + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + raise ValueError(f"Missing attached ENI '{reference['NetworkInterfaceId']}'") + if eni.get("Attachment", {}).get("InstanceId") != instance["InstanceId"]: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' changed attachment during discovery") + if not eni.get("MacAddress"): + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' has no MAC address") + if not self.settings.sync_private_ips: + continue + vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) + if vpc not in data["Vpcs"] or subnet is None or subnet["VpcId"] != vpc: + raise ValueError(f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}'") + if self.addresses(eni, subnet): + vrf = self.get_vrf(account, region, vpc) + identity = (account, region, vpc) + if vrf in vrfs and vrfs[vrf] != identity: + raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") + vrfs[vrf] = identity + + def custom_fields(self): + fields = ( + ("aws_account_id", "AWS account ID", [NBVM.object_type]), + ("aws_region", "AWS region", [NBVM.object_type]), + ("aws_instance_id", "AWS instance ID", [NBVM.object_type]), + ("aws_name", "AWS Name tag", [NBVM.object_type]), + ("aws_instance_type", "AWS instance type", [NBVM.object_type]), + ("aws_availability_zone", "AWS Availability Zone", [NBVM.object_type]), + ("aws_vpc_id", "AWS VPC ID", [NBVM.object_type, NBVMInterface.object_type]), + ("aws_eni_id", "AWS ENI ID", [NBVMInterface.object_type]), + ("aws_subnet_id", "AWS subnet ID", [NBVMInterface.object_type]), + ("aws_public_ipv4", "AWS public IPv4", [NBVMInterface.object_type]), + ) + for name, label, types in fields: + self.add_update_custom_field({ + "name": name, "label": label, "object_types": types, "type": "text", + "description": f"Reported by AWS ({self.name})", + }) + + def sync_snapshot(self, account, snapshot): + self.custom_fields() + group = self.inventory.add_update_object( + NBClusterGroup, data={"name": f"AWS {account}"}, source=self) + api_version = Version(self.inventory.netbox_api_version) + for region, data in snapshot.items(): + site = self.inventory.add_update_object( + NBSite, data={"name": f"AWS {region}"}, source=self) + cluster_data = { + "name": f"AWS {account}/{region}", "group": group, + "type": {"name": "AWS EC2"}, + } + if api_version >= Version("4.2"): + cluster_data.update(scope_type="dcim.site", scope_id=site) + else: + cluster_data["site"] = site + cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) + for instance in data["instances"]: + self.sync_instance(account, region, instance, data, site, cluster, api_version) + + def sync_instance(self, account, region, instance, data, site, cluster, api_version): + instance_id = instance["InstanceId"] + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + spec = data["InstanceTypes"][instance["InstanceType"]] + vm_data = { + "name": instance_id, "cluster": cluster, + "status": "active" if instance["State"]["Name"] == "running" else "offline", + "vcpus": spec["VCpuInfo"]["DefaultVCpus"], + "memory": spec["MemoryInfo"]["SizeInMiB"], + "custom_fields": { + "aws_account_id": account, "aws_region": region, + "aws_instance_id": instance_id, "aws_name": tags.get("Name", ""), + "aws_instance_type": instance["InstanceType"], + "aws_availability_zone": instance["Placement"]["AvailabilityZone"], + "aws_vpc_id": instance.get("VpcId", ""), + }, + "tags": [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags], + } + cpu = instance.get("CpuOptions", {}) + if cpu.get("CoreCount") is not None and cpu.get("ThreadsPerCore") is not None: + vm_data["vcpus"] = cpu["CoreCount"] * cpu["ThreadsPerCore"] + if api_version >= Version("3.3"): + vm_data["site"] = site + if api_version >= Version("4.1"): + vm_data["serial"] = instance_id + if self.settings.tenant: + vm_data["tenant"] = {"name": self.settings.tenant} + disks = [ + data["Volumes"][entry["Ebs"]["VolumeId"]] + for entry in instance.get("BlockDeviceMappings", []) if "Ebs" in entry + ] + if api_version < Version("3.7"): + vm_data["disk"] = sum(volume["Size"] for volume in disks) + vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) + for tag in vm.get_tags(): + if any(tag.startswith(f"AWS:{key}=") for key in self.settings.tag_keys) and tag not in vm_data["tags"]: + vm.remove_tags(tag) + if api_version >= Version("3.7"): + for volume in disks: + self.add_update_vm_child(NBVirtualDisk, vm, { + "name": volume["VolumeId"], "virtual_machine": vm, + "size": volume["Size"] * (1024 if api_version >= Version("4.1") else 1), + "description": f"AWS EBS {volume['VolumeType']}", + }) + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"][reference["NetworkInterfaceId"]] + self.sync_interface(account, region, vm, eni, data, api_version) + + def add_update_vm_child(self, object_type, vm, data): + # Display-name matching loses the VM's account/cluster when names repeat. + existing = next((item for item in self.inventory.get_all_items(object_type) + if item.data.get("virtual_machine") is vm and + item.data.get("name") == data["name"]), None) + if existing is None: + return self.inventory.add_object(object_type, data=data, source=self) + existing.update(data=data, source=self) + return existing + + def sync_interface(self, account, region, vm, eni, data, api_version): + interface_data = { + "name": eni["NetworkInterfaceId"], "virtual_machine": vm, "enabled": True, + "description": eni.get("Description", ""), + "custom_fields": { + "aws_eni_id": eni["NetworkInterfaceId"], "aws_vpc_id": eni.get("VpcId", ""), + "aws_subnet_id": eni.get("SubnetId", ""), + "aws_public_ipv4": ", ".join(sorted({ + address["Association"]["PublicIp"] + for address in eni.get("PrivateIpAddresses", []) + if address.get("Association", {}).get("PublicIp") + })), + }, + } + mac = normalize_mac_address(eni.get("MacAddress")) + if api_version < Version("4.2"): + interface_data["mac_address"] = mac + interface = self.add_update_vm_child(NBVMInterface, vm, interface_data) + if api_version >= Version("4.2") and mac is not None: + mac_object = next((item for item in self.inventory.get_all_items(NBMACAddress) + if item.data.get("mac_address") == mac and + item.data.get("assigned_object_id") in (None, interface)), None) + mac_data = {"mac_address": mac, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if mac_object is None: + mac_object = self.inventory.add_object(NBMACAddress, data=mac_data, source=self) + else: + mac_object.update(data=mac_data, source=self) + for old in interface.get_mac_addresses(): + if old is not mac_object: + old.remove_interface_association() + interface.update(data={"primary_mac_address": mac_object}, source=self) + if not self.settings.sync_private_ips: + for address in interface.get_ip_addresses(): + if self.source_tag in address.get_tags(): + address.set_source(self) + return + subnet = data["Subnets"][eni["SubnetId"]] + addresses = self.addresses(eni, subnet) + discovered = [] + for address, primary in addresses: + vrf = self.get_vrf(account, region, eni["VpcId"]) + parsed = ip_interface(address) + candidates = [ + item for item in self.inventory.get_all_items(NBIPAddress) + if ip_interface(item.data["address"]).ip == parsed.ip and item.data.get("vrf") is vrf + ] + if len(candidates) > 1: + raise ValueError(f"Multiple NetBox IPs match '{address}' in VRF '{vrf.data['name']}'") + existing = candidates[0] if candidates else None + if existing is not None and existing.data.get("assigned_object_id") is not None and \ + existing.get_interface() is not interface: + owner = existing.get_device_vm() + if owner is None or (owner.source is not self and self.source_tag not in owner.get_tags()): + raise ValueError(f"IP '{address}' in VRF '{vrf.data['name']}' belongs to another source") + ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if self.settings.tenant: + ip_data["tenant"] = {"name": self.settings.tenant} + if existing is None: + existing = self.inventory.add_object(NBIPAddress, data=ip_data, source=self) + else: + existing.update(data=ip_data, source=self) + discovered.append(existing) + if eni.get("Attachment", {}).get("DeviceIndex") == 0 and primary: + vm.update(data={f"primary_ip{parsed.version}": existing}, source=self) + for old in interface.get_ip_addresses(): + if old not in discovered and (old.source is self or self.source_tag in old.get_tags()): + old.remove_interface_association() diff --git a/pyproject.toml b/pyproject.toml index 9c284b23..0cf28b08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ dependencies = [ ] [project.optional-dependencies] +aws = [ + "boto3>=1.40,<2", +] test = [ "pytest>=9.1.1", ] diff --git a/requirements-aws.txt b/requirements-aws.txt new file mode 100644 index 00000000..368a9493 --- /dev/null +++ b/requirements-aws.txt @@ -0,0 +1,2 @@ +-r requirements.txt +boto3>=1.40,<2 diff --git a/settings-example.ini b/settings-example.ini index 75b725ff..6140ea7c 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -611,4 +611,56 @@ inventory_file_path = /full/path/to/inventory/files ; Hetzner Cloud API token used to authenticate against the Hetzner Cloud API. api_token = +[source/my-aws-example] + +; Install optional SDK: pip install -r requirements-aws.txt +; See docs/source_aws.md for read-only IAM permissions and VPC VRF setup. +;enabled = True +;type = aws +regions = auto +; Bootstrap endpoint for auto; defaults to the AWS profile/environment region. +;discovery_region = us-east-1 +; Alternatively restrict discovery: regions = us-east-1, us-west-2 +;profile = inventory +;account_id = 123456789012 +; Enable organization scanning using the profile's collector credentials. +;organization = False +;organization_role_name = NetBoxInventoryReadOnly +;organization_include_management = False +; Optional allowlist for an initial member-account test; unset scans all active members. +;organization_account_ids = 111122223333, 444455556666 +;organization_external_id = +; See docs/source_aws.md for collector permissions and member-role trust policies. +;sync_private_ips = True +;permitted_subnets = 10.0.0.0/8, fd00::/8 +;vpc_vrf_relation = us-east-1/vpc-0123456789abcdef0 = Production VPC +; Organization account-specific key: 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC +;tag_keys = Environment, Owner +;tenant = Operations +;dns_name_lookup = False + +; Alternative organization setup: uncomment this source instead of the AWS +; source above, not alongside an overlapping source. +; All identifiers below are fictitious examples; replace with your deployment values. +; AWS shared credentials: [example-inventory-client] with the existing IAM user keys. +; AWS shared config: +; [profile example-inventory] +; role_arn = arn:aws:iam::123456789012:role/ExampleInventoryRole +; source_profile = example-inventory-client +; region = us-east-1 +; These files must be available to the OS user/environment running netbox-sync. +; See docs/source_aws.md for complete examples and member-role trust requirements. +;[source/aws-organization] +;type = aws +;profile = example-inventory +;organization = True +;organization_role_name = ExampleInventoryRole +;account_id = 123456789012 +;regions = auto +;discovery_region = us-east-1 +;sync_private_ips = False +; Optional initial scope: replace with an actual member account ID. +;organization_account_ids = 111122223333 +; Leave organization_external_id unset unless the member trust policy requires it. + ;EOF diff --git a/tests/test_aws_source.py b/tests/test_aws_source.py new file mode 100644 index 00000000..d93b537b --- /dev/null +++ b/tests/test_aws_source.py @@ -0,0 +1,1015 @@ +import copy +import os +from contextlib import ExitStack +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +boto3 = pytest.importorskip("boto3") +from botocore.stub import Stubber + +from module.config.file_output import ConfigFileOutput +from module.config.parser import ConfigParser +from module.netbox import ( + NBCluster, NBIPAddress, NBMACAddress, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) +from module.netbox.connection import NetBoxHandler +from module.sources import instantiate_sources, valid_sources +from module.sources.aws import client as aws_client +from module.sources.aws.client import AWSClient +from module.sources.aws.config import AWSConfig +from module.sources.aws.connection import AWSHandler + +ACCOUNT = "123456789012" +REGION = "us-east-1" +INSTANCE = "i-0123456789abcdef0" +VPC = "vpc-0123456789abcdef0" +SUBNET = "subnet-0123456789abcdef0" +ENI = "eni-0123456789abcdef0" +VOLUME = "vol-0123456789abcdef0" + + +def inventory_data(): + return { + "instances": [{ + "InstanceId": INSTANCE, "InstanceType": "t3.small", + "State": {"Name": "running"}, + "Placement": {"AvailabilityZone": "us-east-1a"}, + "VpcId": VPC, "SubnetId": SUBNET, + "Tags": [{"Key": "Name", "Value": "web"}, {"Key": "Environment", "Value": "test"}], + "NetworkInterfaces": [{"NetworkInterfaceId": ENI}], + "BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"VolumeId": VOLUME}}], + }], + "InstanceTypes": {"t3.small": { + "InstanceType": "t3.small", "VCpuInfo": {"DefaultVCpus": 2}, + "MemoryInfo": {"SizeInMiB": 2048}, + }}, + "NetworkInterfaces": {ENI: { + "NetworkInterfaceId": ENI, "VpcId": VPC, "SubnetId": SUBNET, + "Attachment": {"InstanceId": INSTANCE, "DeviceIndex": 0}, + "MacAddress": "02:aa:bb:cc:dd:ee", + "PrivateIpAddresses": [ + {"PrivateIpAddress": "10.0.0.10", "Primary": True, + "Association": {"PublicIp": "198.51.100.10"}}, + {"PrivateIpAddress": "10.0.0.11", "Primary": False}, + ], + "Ipv6Addresses": [{"Ipv6Address": "2001:db8::10", "IsPrimaryIpv6": True}], + }}, + "Volumes": {VOLUME: {"VolumeId": VOLUME, "Size": 20, "VolumeType": "gp3"}}, + "Subnets": {SUBNET: { + "SubnetId": SUBNET, "VpcId": VPC, "CidrBlock": "10.0.0.0/24", + "Ipv6CidrBlockAssociationSet": [{ + "Ipv6CidrBlock": "2001:db8::/64", "Ipv6CidrBlockState": {"State": "associated"}, + }], + }}, + "Vpcs": {VPC: {"VpcId": VPC}}, + } + + +@pytest.fixture(autouse=True) +def windows_config_paths(monkeypatch): + if os.name == "nt": + monkeypatch.setattr(ConfigParser, "get_config_file_path", staticmethod(os.path.abspath)) + + +@pytest.fixture +def make_source(inventory, load_config, monkeypatch): + def make(extra="", data=None, regions=REGION): + load_config(f""" +[source/test-aws] +type = aws +regions = {regions} +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +tag_keys = Environment +{extra} +""") + discovery = Mock(return_value=(ACCOUNT, {REGION: data if data is not None else inventory_data()})) + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + monkeypatch.setattr(AWSClient, "discover", discovery) + sources = instantiate_sources() + assert len(sources) == 1 and sources[0].init_successful + return sources[0] + return make + + +@pytest.fixture +def vpc_vrf(inventory): + return inventory.add_object(NBVRF, data={ + "id": 1, "name": f"AWS {ACCOUNT}/{REGION}/{VPC}", + }, read_from_netbox=True) + + +def test_registration(): + assert AWSHandler in valid_sources + assert AWSConfig in ConfigFileOutput.source_config_list + assert NBVRF in AWSHandler.dependent_netbox_objects + assert NBMACAddress in AWSHandler.dependent_netbox_objects + + +@pytest.mark.parametrize("api_version", ["3.6.0", "3.7.0", "4.1.0", "4.3.0"]) +def test_ec2_ebs_eni_import(make_source, inventory, vpc_vrf, api_version): + inventory.netbox_api_version = api_version + source = make_source("tenant = Operations") + source.apply() + vm, = inventory.get_all_items(NBVM) + cluster, = inventory.get_all_items(NBCluster) + interface, = inventory.get_all_items(NBVMInterface) + assert vm.data["name"] == INSTANCE + assert vm.data["status"] == "active" + assert vm.data["vcpus"] == 2 + assert vm.data["memory"] == 2048 + assert vm.data["custom_fields"]["aws_name"] == "web" + assert vm.data["tenant"].data["name"] == "Operations" + assert "AWS:Environment=test" in vm.get_tags() + assert interface.data["name"] == ENI + assert interface.data["custom_fields"]["aws_public_ipv4"] == "198.51.100.10" + ips = inventory.get_all_items(NBIPAddress) + assert {item.data["address"] for item in ips} == { + "10.0.0.10/24", "10.0.0.11/24", "2001:db8::10/64", + } + assert all(item.data["vrf"] is vpc_vrf for item in ips) + assert vm.data["primary_ip4"].data["address"] == "10.0.0.10/24" + assert vm.data["primary_ip6"].data["address"] == "2001:db8::10/64" + if api_version >= "4.2": + assert cluster.data["scope_type"] == "dcim.site" + assert cluster.data["scope_id"].data["name"] == f"AWS {REGION}" + mac, = inventory.get_all_items(NBMACAddress) + assert mac is interface.data["primary_mac_address"] + if api_version < "3.7": + assert vm.data["disk"] == 20 + else: + disk, = inventory.get_all_items(NBVirtualDisk) + assert disk.data["name"] == VOLUME + assert disk.data["size"] == (20 if api_version < "4.1" else 20480) + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert len(inventory.get_all_items(NBVMInterface)) == 1 + assert len(inventory.get_all_items(NBIPAddress)) == 3 + + +def test_overlapping_vpcs_have_distinct_ips(make_source, inventory, vpc_vrf): + data = inventory_data() + second = copy.deepcopy(data) + other_instance, other_eni = "i-11111111111111111", "eni-11111111111111111" + other_vpc, other_subnet = "vpc-11111111111111111", "subnet-11111111111111111" + vm = second["instances"][0] + vm.update(InstanceId=other_instance, VpcId=other_vpc, SubnetId=other_subnet, + NetworkInterfaces=[{"NetworkInterfaceId": other_eni}], BlockDeviceMappings=[]) + eni = second["NetworkInterfaces"].pop(ENI) + eni.update(NetworkInterfaceId=other_eni, VpcId=other_vpc, SubnetId=other_subnet, + MacAddress="02:aa:bb:cc:dd:ff") + eni["Attachment"]["InstanceId"] = other_instance + subnet = second["Subnets"].pop(SUBNET) + subnet.update(SubnetId=other_subnet, VpcId=other_vpc) + data["instances"].append(vm) + data["NetworkInterfaces"][other_eni] = eni + data["Subnets"][other_subnet] = subnet + data["Vpcs"][other_vpc] = {"VpcId": other_vpc} + inventory.add_object(NBVRF, data={ + "id": 2, "name": f"AWS {ACCOUNT}/{REGION}/{other_vpc}", + }, read_from_netbox=True) + source = make_source(data=data) + source.apply() + assert len(inventory.get_all_items(NBVM)) == 2 + assert len(inventory.get_all_items(NBIPAddress)) == 6 + + +def test_missing_vrf_aborts_before_inventory_changes(make_source, inventory, caplog): + source = make_source() + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert "Create NetBox VRF" in caplog.text + + +def test_missing_volume_aborts_before_inventory_changes(make_source, inventory, vpc_vrf): + data = inventory_data() + data["Volumes"].clear() + source = make_source(data=data) + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + + +def test_stopped_instance_without_public_ip_is_retained(make_source, inventory, vpc_vrf): + data = inventory_data() + data["instances"][0]["State"]["Name"] = "stopped" + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"][0].pop("Association") + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["status"] == "offline" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + assert inventory.get_all_items(NBVMInterface)[0].data["custom_fields"]["aws_public_ipv4"] == "" + + +def test_name_tag_rename_does_not_duplicate_vm(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + data["instances"][0]["Tags"][0]["Value"] = "renamed" + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["name"] == INSTANCE + assert vm.data["custom_fields"]["aws_name"] == "renamed" + + +def test_removed_aws_tag_is_reconciled_but_user_tag_is_kept(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + vm.add_tags("user-managed") + data["instances"][0]["Tags"] = [{"Key": "Environment", "Value": "production"}] + source.apply() + assert "AWS:Environment=test" not in vm.get_tags() + assert "AWS:Environment=production" in vm.get_tags() + assert "user-managed" in vm.get_tags() + + +def test_removed_secondary_ip_and_mac_are_detached(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + secondary = next(ip for ip in inventory.get_all_items(NBIPAddress) + if ip.data["address"] == "10.0.0.11/24") + old_mac, = inventory.get_all_items(NBMACAddress) + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"].pop() + data["NetworkInterfaces"][ENI]["MacAddress"] = "02:00:00:00:00:01" + source.apply() + assert "assigned_object_id" in secondary.unset_items + assert "assigned_object_id" in old_mac.unset_items + interface, = inventory.get_all_items(NBVMInterface) + assert interface.data["primary_mac_address"].data["mac_address"] == "02:00:00:00:00:01" + + +def test_primary_address_removal_clears_primary_ip(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"] = [] + data["NetworkInterfaces"][ENI]["Ipv6Addresses"] = [] + source.apply() + assert "primary_ip4" in vm.unset_items + assert "primary_ip6" in vm.unset_items + + +def test_ip_owned_by_other_source_is_not_reassigned(make_source, inventory, vpc_vrf): + source = make_source() + other = inventory.add_object(NBVM, data={"name": "other", "status": "active"}) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + with pytest.raises(SystemExit): + source.apply() + assert existing.get_interface() is other_interface + + +def test_tagging_and_successful_empty_discovery_use_existing_orphan_lifecycle( + make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + handler = object.__new__(NetBoxHandler) + handler.settings = SimpleNamespace( + prune_enabled=True, ignore_unknown_source_object_pruning=False, + orphaned_device_status=None, orphaned_vm_status=None, + ) + inventory.tag_all_the_things(handler) + vm, = inventory.get_all_items(NBVM) + assert source.source_tag in vm.get_tags() + assert handler.primary_tag in vm.get_tags() + for items in inventory.base_structure.values(): + for item in items: + item.source = None + data["instances"] = [] + source.apply() + inventory.tag_all_the_things(handler) + assert handler.orphaned_tag in vm.get_tags() + + +def test_customized_cpu_options_override_type_default(make_source, inventory, vpc_vrf): + data = inventory_data() + data["instances"][0]["CpuOptions"] = {"CoreCount": 1, "ThreadsPerCore": 1} + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["vcpus"] == 1 + + +def test_multiple_enis_only_primary_eni_sets_primary_ip(make_source, inventory, vpc_vrf): + data = inventory_data() + secondary_id = "eni-11111111111111111" + secondary = copy.deepcopy(data["NetworkInterfaces"][ENI]) + secondary.update(NetworkInterfaceId=secondary_id, MacAddress="02:00:00:00:00:01") + secondary["Attachment"]["DeviceIndex"] = 1 + secondary["PrivateIpAddresses"] = [{"PrivateIpAddress": "10.0.0.20", "Primary": True}] + secondary["Ipv6Addresses"] = [] + data["NetworkInterfaces"][secondary_id] = secondary + data["instances"][0]["NetworkInterfaces"].append({"NetworkInterfaceId": secondary_id}) + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + assert len(inventory.get_all_items(NBVMInterface)) == 2 + assert vm.data["primary_ip4"].data["address"] == "10.0.0.10/24" + + +def test_optional_network_import_does_not_require_vrf(make_source, inventory): + source = make_source("sync_private_ips = False") + source.apply() + assert len(inventory.get_all_items(NBVMInterface)) == 1 + assert inventory.get_all_items(NBIPAddress) == [] + + +def test_explicit_vrf_mapping(make_source, inventory): + inventory.add_object(NBVRF, data={"id": 1, "name": "Production"}, read_from_netbox=True) + source = make_source(f"vpc_vrf_relation = {REGION}/{VPC} = Production") + source.apply() + assert {item.data["vrf"].data["name"] for item in inventory.get_all_items(NBIPAddress)} == {"Production"} + + +def test_quoted_vrf_name_with_comma(make_source, inventory): + inventory.add_object(NBVRF, data={"id": 1, "name": "Production, New York"}, read_from_netbox=True) + source = make_source(f'vpc_vrf_relation = {REGION}/{VPC} = "Production, New York"') + source.apply() + assert {item.data["vrf"].data["name"] for item in inventory.get_all_items(NBIPAddress)} == { + "Production, New York", + } + + +def test_disallowed_addresses_are_not_imported(make_source, inventory, vpc_vrf): + source = make_source() + from module.sources.common.permitted_subnets import PermittedSubnets + source.settings.permitted_subnets = PermittedSubnets("192.168.0.0/16") + source.apply() + assert inventory.get_all_items(NBIPAddress) == [] + assert len(inventory.get_all_items(NBVMInterface)) == 1 + + +def test_unmapped_subnet_address_aborts_before_changes(make_source, inventory, vpc_vrf): + data = inventory_data() + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"][0]["PrivateIpAddress"] = "10.1.1.1" + source = make_source(data=data) + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBCluster) == [] + + +def test_missing_sdk_aborts_enabled_source_but_disabled_source_is_safe( + inventory, load_config, monkeypatch, caplog): + monkeypatch.setattr(aws_client, "boto3", None) + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION} +enabled = False +""") + assert instantiate_sources() == [] + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION} +""") + with pytest.raises(SystemExit): + instantiate_sources() + assert "requirements-aws.txt" in caplog.text + + +def test_yaml_lists_and_account_guard_parse(inventory, load_config, monkeypatch): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(f""" +source: + test-aws: + type: aws + regions: [us-east-1, us-west-2] + account_id: "{ACCOUNT}" + tag_keys: [Environment, Owner] +""", "settings.yaml") + source, = instantiate_sources() + assert source.settings.regions == ["us-east-1", "us-west-2"] + assert source.settings.tag_keys == ["Environment", "Owner"] + assert source.settings.account_id == ACCOUNT + + +@pytest.mark.parametrize("extra", [ + "account_id = not-an-account", + "vpc_vrf_relation = malformed", + f"vpc_vrf_relation = {REGION}/{VPC} = One, {REGION}/{VPC} = Two", +]) +def test_invalid_config_fails(make_source, extra): + with pytest.raises(SystemExit): + make_source(extra) + + +@pytest.fixture +def sdk(monkeypatch): + session = boto3.Session( + aws_access_key_id="test-only", aws_secret_access_key="test-only", region_name=REGION) + sts = session.client("sts") + ec2 = session.client("ec2") + sts_stub, ec2_stub = Stubber(sts), Stubber(ec2) + fake_session = Mock() + fake_session.region_name = REGION + fake_session.client.side_effect = lambda service, **kwargs: sts if service == "sts" else ec2 + monkeypatch.setattr(aws_client.boto3, "Session", Mock(return_value=fake_session)) + with sts_stub, ec2_stub: + yield sts_stub, ec2_stub + sts_stub.assert_no_pending_responses() + ec2_stub.assert_no_pending_responses() + + +def identity(stub, account=ACCOUNT, partition="aws"): + stub.add_response("get_caller_identity", { + "Account": account, "Arn": f"arn:{partition}:iam::{account}:user/test", "UserId": "test", + }, {}) + + +def stub_region(stub, data): + stub.add_response("describe_instances", {"Reservations": [{ + "Instances": data["instances"], + }], "NextToken": "next-page"}, {}) + stub.add_response("describe_instances", {"Reservations": []}, {"NextToken": "next-page"}) + for operation, result_key in ( + ("describe_network_interfaces", "NetworkInterfaces"), + ("describe_volumes", "Volumes"), ("describe_subnets", "Subnets"), + ("describe_vpcs", "Vpcs"), + ): + stub.add_response(operation, {result_key: list(data[result_key].values())}, {}) + if data["instances"]: + stub.add_response("describe_instance_types", { + "InstanceTypes": list(data["InstanceTypes"].values()), + }, {"InstanceTypes": ["t3.small"]}) + + +def test_sdk_paginated_discovery(sdk): + sts, ec2 = sdk + identity(sts) + data = inventory_data() + stub_region(ec2, data) + account, snapshot = AWSClient().discover([REGION], ACCOUNT) + assert account == ACCOUNT + assert snapshot[REGION]["instances"] == data["instances"] + assert snapshot[REGION]["Volumes"] == data["Volumes"] + + +def test_stubbed_sdk_to_real_inventory(sdk, inventory, load_config, vpc_vrf): + sts, ec2 = sdk + identity(sts) + stub_region(ec2, inventory_data()) + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION} +account_id = {ACCOUNT} +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +""") + source, = instantiate_sources() + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["name"] == INSTANCE + assert vm.data["primary_ip4"].data["vrf"] is vpc_vrf + assert len(inventory.get_all_items(NBVirtualDisk)) == 1 + + +def test_profile_is_forwarded_to_sdk(sdk): + AWSClient(profile="inventory") + aws_client.boto3.Session.assert_called_once_with(profile_name="inventory") + + +def test_sdk_error_after_first_region_aborts_entire_apply(sdk, inventory, load_config, caplog): + sts, ec2 = sdk + identity(sts) + stub_region(ec2, inventory_data()) + ec2.add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + service_message="not permitted", http_status_code=403, expected_params={}) + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION}, us-west-2 +sync_private_ips = False +""") + source, = instantiate_sources() + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert "No NetBox writes or pruning" in caplog.text + + +def test_wrong_account_aborts(sdk): + sts, _ = sdk + identity(sts) + with pytest.raises(aws_client.AWSDiscoveryError, match="account mismatch"): + AWSClient().discover([REGION], "999999999999") + + +def test_terminated_instances_are_excluded(sdk): + sts, ec2 = sdk + identity(sts) + terminated = inventory_data()["instances"][0] + terminated["State"]["Name"] = "terminated" + ec2.add_response("describe_instances", {"Reservations": [{"Instances": [terminated]}]}, {}) + for operation, result_key in ( + ("describe_network_interfaces", "NetworkInterfaces"), ("describe_volumes", "Volumes"), + ("describe_subnets", "Subnets"), ("describe_vpcs", "Vpcs"), + ): + ec2.add_response(operation, {result_key: []}, {}) + _, snapshot = AWSClient().discover([REGION]) + assert snapshot[REGION]["instances"] == [] + + +def test_missing_sdk_fails_explicitly(monkeypatch): + monkeypatch.setattr(aws_client, "boto3", None) + with pytest.raises(aws_client.AWSDiscoveryError, match="requirements-aws.txt"): + AWSClient() + + +def region_response(stub, regions): + stub.add_response("describe_regions", {"Regions": [ + {"RegionName": name, "OptInStatus": status} for name, status in regions + ]}, {"AllRegions": True}) + + +def test_auto_discovers_enabled_regions_and_preserves_region_identity(sdk): + sts, ec2 = sdk + identity(sts) + region_response(ec2, [ + ("us-west-2", "opted-in"), + (REGION, "opt-in-not-required"), + ("ap-east-1", "not-opted-in"), + (REGION, "opt-in-not-required"), + ]) + stub_region(ec2, inventory_data()) + stub_region(ec2, inventory_data()) + client = AWSClient() + account, snapshot = client.discover(["auto"], ACCOUNT) + assert account == ACCOUNT + assert list(snapshot) == [REGION, "us-west-2"] + ec2_regions = [ + call.kwargs["region_name"] for call in client.session.client.call_args_list + if call.args[0] == "ec2" + ] + assert ec2_regions == [REGION, REGION, "us-west-2"] + + +def test_auto_bootstrap_override(sdk): + sts, ec2 = sdk + identity(sts) + region_response(ec2, [(REGION, "opt-in-not-required")]) + stub_region(ec2, inventory_data()) + client = AWSClient() + client.session.region_name = None + client.discover(["auto"], discovery_region="us-west-2") + assert client.session.client.call_args_list[0].kwargs["region_name"] == "us-west-2" + assert client.session.client.call_args_list[1].kwargs["region_name"] == "us-west-2" + assert client.session.client.call_args_list[2].kwargs["region_name"] == REGION + + +def test_auto_without_bootstrap_fails_explicitly(sdk): + client = AWSClient() + client.session.region_name = None + with pytest.raises(aws_client.AWSDiscoveryError, match="AWS_DEFAULT_REGION"): + client.discover(["auto"]) + client.session.client.assert_not_called() + + +@pytest.mark.parametrize("regions", [[], [("ap-east-1", "not-opted-in")]]) +def test_auto_no_enabled_regions_aborts(sdk, regions): + sts, ec2 = sdk + identity(sts) + region_response(ec2, regions) + with pytest.raises(aws_client.AWSDiscoveryError, match="no enabled regions"): + AWSClient().discover(["auto"]) + + +def test_auto_permission_failure_aborts_before_inventory_changes( + sdk, inventory, load_config, caplog): + sts, ec2 = sdk + identity(sts) + ec2.add_client_error("describe_regions", service_error_code="UnauthorizedOperation", + service_message="not permitted", http_status_code=403, + expected_params={"AllRegions": True}) + load_config(""" +[source/test-aws] +type = aws +sync_private_ips = False +""") + source, = instantiate_sources() + assert source.settings.regions == ["auto"] + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert "No NetBox writes or pruning" in caplog.text + + +def test_auto_later_region_failure_does_not_return_partial_inventory( + sdk, inventory, load_config): + sts, ec2 = sdk + identity(sts) + region_response(ec2, [(REGION, "opt-in-not-required"), ("us-west-2", "opted-in")]) + stub_region(ec2, inventory_data()) + ec2.add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + http_status_code=403, expected_params={}) + load_config(""" +[source/test-aws] +type = aws +regions = auto +sync_private_ips = False +""") + source, = instantiate_sources() + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + + +@pytest.mark.parametrize("regions", ["auto, us-east-1", "us-east-1, auto"]) +def test_auto_cannot_be_mixed_with_explicit_regions(make_source, regions): + with pytest.raises(SystemExit): + make_source(regions=regions) + + +def test_auto_yaml_list_parses(inventory, load_config, monkeypatch): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(""" +source: + test-aws: + type: aws + regions: [auto] + discovery_region: us-west-2 +""", "settings.yaml") + source, = instantiate_sources() + assert source.settings.regions == ["auto"] + assert source.settings.discovery_region == "us-west-2" + + +def test_explicit_regions_do_not_need_bootstrap_or_describe_regions(sdk): + sts, ec2 = sdk + identity(sts) + stub_region(ec2, inventory_data()) + client = AWSClient() + client.session.region_name = None + _, snapshot = client.discover([REGION], discovery_region="us-west-2") + assert list(snapshot) == [REGION] + assert client.session.client.call_args_list[0].kwargs["region_name"] == REGION + + +MANAGEMENT = "999999999999" +SECOND_ACCOUNT = "234567890123" + + +@pytest.fixture +def org_sdk(monkeypatch): + real_session = boto3.Session( + aws_access_key_id="test-only", aws_secret_access_key="test-only", region_name=REGION) + stubs = [] + + def make_session(organization=False): + services = ("sts", "organizations") if organization else ("sts", "ec2") + clients = {service: real_session.client(service) for service in services} + session = Mock(region_name=REGION) + session.client.side_effect = lambda service, **kwargs: clients[service] + bundle = SimpleNamespace(session=session) + for service, client in clients.items(): + stub = Stubber(client) + stubs.append(stub) + setattr(bundle, service, stub) + return bundle + + central = make_session(organization=True) + members = {account: make_session() for account in (MANAGEMENT, ACCOUNT, SECOND_ACCOUNT)} + factory = Mock(side_effect=lambda **kwargs: ( + members[kwargs["aws_session_token"]].session if "aws_session_token" in kwargs else central.session)) + monkeypatch.setattr(aws_client.boto3, "Session", factory) + with ExitStack() as stack: + for stub in stubs: + stack.enter_context(stub) + yield SimpleNamespace(central=central, members=members, factory=factory) + for stub in stubs: + stub.assert_no_pending_responses() + + +def organization_accounts(sdk, accounts=None, partition="aws"): + identity(sdk.central.sts, MANAGEMENT, partition) + sdk.central.organizations.add_response("describe_organization", { + "Organization": {"MasterAccountId": MANAGEMENT}, + }, {}) + # An empty page with a continuation token must not terminate discovery. + sdk.central.organizations.add_response("list_accounts", { + "Accounts": [], "NextToken": "next-page", + }, {}) + sdk.central.organizations.add_response("list_accounts", { + "Accounts": accounts if accounts is not None else [ + {"Id": MANAGEMENT, "State": "ACTIVE"}, + {"Id": ACCOUNT, "State": "ACTIVE"}, + {"Id": SECOND_ACCOUNT, "State": "ACTIVE"}, + {"Id": "345678901234", "State": "SUSPENDED"}, + ], + }, {"NextToken": "next-page"}) + + +def assume_member(sdk, account, external_id=None, partition="aws", role="NetBoxInventoryReadOnly"): + request = {"RoleArn": f"arn:{partition}:iam::{account}:role/{role}", + "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} + if external_id: + request["ExternalId"] = external_id + sdk.central.sts.add_response("assume_role", {"Credentials": { + "AccessKeyId": "ASIA" + "A" * 16, "SecretAccessKey": "test-only-secret", + "SessionToken": account, "Expiration": datetime(2099, 1, 1, tzinfo=timezone.utc), + }}, request) + return sdk.members[account] + + +def organization_source(load_config, extra=""): + load_config(f""" +[source/test-aws] +type = aws +organization = True +regions = auto +account_id = {MANAGEMENT} +sync_private_ips = False +{extra} +""") + source, = instantiate_sources() + return source + + +def test_organization_scans_members_not_management(org_sdk, inventory, load_config): + organization_accounts(org_sdk) + for account in (ACCOUNT, SECOND_ACCOUNT): + member = assume_member(org_sdk, account) + identity(member.sts, account) + region_response(member.ec2, [(REGION, "opt-in-not-required")]) + stub_region(member.ec2, inventory_data()) + source = organization_source(load_config) + source.apply() + vms = inventory.get_all_items(NBVM) + assert len(vms) == 2 + assert {vm.data["custom_fields"]["aws_account_id"] for vm in vms} == {ACCOUNT, SECOND_ACCOUNT} + assert {vm.data["cluster"].data["name"] for vm in vms} == { + f"AWS {ACCOUNT}/{REGION}", f"AWS {SECOND_ACCOUNT}/{REGION}", + } + assert len(inventory.get_all_items(NBVMInterface)) == 2 + assert len(inventory.get_all_items(NBVirtualDisk)) == 2 + assert {item.data["virtual_machine"] for item in inventory.get_all_items(NBVMInterface)} == set(vms) + assert {item.data["virtual_machine"] for item in inventory.get_all_items(NBVirtualDisk)} == set(vms) + org_sdk.members[MANAGEMENT].session.client.assert_not_called() + assert [call.args[0] for call in org_sdk.central.session.client.call_args_list] == [ + "sts", "organizations", + ] + # Each account gets fresh temporary credentials, not the collector's IAM-user credentials. + assert [call.kwargs["aws_session_token"] for call in org_sdk.factory.call_args_list[1:]] == [ + ACCOUNT, SECOND_ACCOUNT, + ] + for account in (ACCOUNT, SECOND_ACCOUNT): + source.sync_snapshot(account, {REGION: inventory_data()}) + assert len(inventory.get_all_items(NBVMInterface)) == 2 + assert len(inventory.get_all_items(NBVirtualDisk)) == 2 + + +def test_organization_allowlist_role_path_external_id_and_partition(org_sdk): + organization_accounts(org_sdk, partition="aws-us-gov") + member = assume_member(org_sdk, SECOND_ACCOUNT, external_id="inventory-test", + partition="aws-us-gov", role="service-accounts/Inventory") + identity(member.sts, SECOND_ACCOUNT, partition="aws-us-gov") + stub_region(member.ec2, inventory_data()) + snapshots = AWSClient().discover_organization( + [REGION], "service-accounts/Inventory", external_id="inventory-test", + account_ids=[SECOND_ACCOUNT]) + assert list(snapshots) == [SECOND_ACCOUNT] + org_sdk.members[ACCOUNT].session.client.assert_not_called() + + +def test_organization_can_include_management(org_sdk): + organization_accounts(org_sdk) + member = assume_member(org_sdk, MANAGEMENT) + identity(member.sts, MANAGEMENT) + stub_region(member.ec2, inventory_data()) + snapshots = AWSClient().discover_organization( + [REGION], "NetBoxInventoryReadOnly", include_management=True, account_ids=[MANAGEMENT]) + assert list(snapshots) == [MANAGEMENT] + + +@pytest.mark.parametrize("failure", ["assume", "scan", "identity"]) +def test_organization_later_account_failure_prevents_all_mutations( + org_sdk, inventory, load_config, failure, caplog): + organization_accounts(org_sdk) + first = assume_member(org_sdk, ACCOUNT) + identity(first.sts, ACCOUNT) + region_response(first.ec2, [(REGION, "opt-in-not-required")]) + stub_region(first.ec2, inventory_data()) + if failure == "assume": + org_sdk.central.sts.add_client_error( + "assume_role", service_error_code="AccessDenied", http_status_code=403, + expected_params={"RoleArn": f"arn:aws:iam::{SECOND_ACCOUNT}:role/NetBoxInventoryReadOnly", + "RoleSessionName": "netbox-sync", "DurationSeconds": 3600}) + else: + second = assume_member(org_sdk, SECOND_ACCOUNT) + identity(second.sts, ACCOUNT if failure == "identity" else SECOND_ACCOUNT) + if failure == "scan": + region_response(second.ec2, [(REGION, "opt-in-not-required")]) + second.ec2.add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + http_status_code=403, expected_params={}) + source = organization_source(load_config) + sync = Mock() + source.sync_snapshot = sync + with pytest.raises(SystemExit): + source.apply() + sync.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert SECOND_ACCOUNT in caplog.text + assert "No NetBox writes or pruning" in caplog.text + + +def test_organization_listing_failure_aborts(org_sdk): + identity(org_sdk.central.sts, MANAGEMENT) + org_sdk.central.organizations.add_response("describe_organization", { + "Organization": {"MasterAccountId": MANAGEMENT}, + }, {}) + org_sdk.central.organizations.add_client_error( + "list_accounts", service_error_code="AccessDeniedException", http_status_code=403, expected_params={}) + with pytest.raises(aws_client.AWSDiscoveryError, match="AccessDeniedException"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + + +@pytest.mark.parametrize("selection", [[MANAGEMENT], ["999999999999"], ["345678901234"]]) +def test_organization_invalid_allowlist_is_not_silently_ignored(org_sdk, selection): + organization_accounts(org_sdk) + with pytest.raises(aws_client.AWSDiscoveryError, match="not active/eligible"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly", account_ids=selection) + + +def test_organization_no_eligible_accounts_aborts(org_sdk): + organization_accounts(org_sdk, accounts=[{"Id": MANAGEMENT, "State": "ACTIVE"}]) + with pytest.raises(aws_client.AWSDiscoveryError, match="no eligible accounts"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + + +def test_organization_wrong_collector_account_aborts_before_listing(org_sdk): + identity(org_sdk.central.sts, MANAGEMENT) + with pytest.raises(aws_client.AWSDiscoveryError, match="account mismatch"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly", expected_account=ACCOUNT) + assert org_sdk.central.session.client.call_count == 1 + + +def test_organization_missing_bootstrap_aborts(org_sdk): + org_sdk.central.session.region_name = None + with pytest.raises(aws_client.AWSDiscoveryError, match="AWS_DEFAULT_REGION"): + AWSClient().discover_organization(["auto"], "NetBoxInventoryReadOnly") + org_sdk.central.session.client.assert_not_called() + + +def test_organization_all_accounts_validated_before_sync(make_source, inventory, monkeypatch): + source = make_source("organization = True") + invalid = inventory_data() + invalid["Volumes"] = {} + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: inventory_data()}, SECOND_ACCOUNT: {REGION: invalid}, + })) + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + + +def test_organization_rejects_vrf_reuse_across_accounts(make_source, vpc_vrf, monkeypatch): + source = make_source(f""" +organization = True +vpc_vrf_relation = {REGION}/{VPC} = {vpc_vrf.data['name']} +""") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: inventory_data()}, SECOND_ACCOUNT: {REGION: inventory_data()}, + })) + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + + +def test_organization_account_specific_vrf_mapping(make_source, inventory, vpc_vrf, monkeypatch): + second_vrf = inventory.add_object(NBVRF, data={"id": 2, "name": "Second VPC"}, read_from_netbox=True) + source = make_source(f""" +organization = True +vpc_vrf_relation = {SECOND_ACCOUNT}/{REGION}/{VPC} = Second VPC +""") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: inventory_data()}, SECOND_ACCOUNT: {REGION: inventory_data()}, + })) + source.apply() + assert {address.data["vrf"] for address in inventory.get_all_items(NBIPAddress)} == { + vpc_vrf, second_vrf, + } + + +@pytest.mark.parametrize("extra", [ + "organization = True\norganization_role_name = arn:aws:iam::123456789012:role/Inventory", + "organization = True\norganization_role_name = invalid role", + "organization = True\norganization_external_id = x", + "organization = True\norganization_account_ids = 123", + "organization = True\norganization_account_ids = 123456789012, invalid", + "organization_account_ids = 123456789012", + "organization_include_management = True", + "organization_external_id = test-id", +]) +def test_organization_invalid_config(make_source, extra): + with pytest.raises(SystemExit): + make_source(extra) + + +def test_organization_yaml_options(inventory, load_config, monkeypatch): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(f""" +source: + test-aws: + type: aws + organization: true + organization_account_ids: ["{ACCOUNT}", "{SECOND_ACCOUNT}"] + organization_role_name: service-accounts/Inventory +""", "settings.yaml") + source, = instantiate_sources() + assert source.settings.organization_account_ids == [ACCOUNT, SECOND_ACCOUNT] + assert source.settings.organization_role_name == "service-accounts/Inventory" + + +def test_organization_empty_member_does_not_stop_next_account(org_sdk): + organization_accounts(org_sdk) + empty = inventory_data() + empty["instances"] = [] + for account, data in ((ACCOUNT, empty), (SECOND_ACCOUNT, inventory_data())): + member = assume_member(org_sdk, account) + identity(member.sts, account) + stub_region(member.ec2, data) + snapshots = AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + assert snapshots[ACCOUNT][REGION]["instances"] == [] + assert snapshots[SECOND_ACCOUNT][REGION]["instances"][0]["InstanceId"] == INSTANCE + + +def test_organization_paginates_150_accounts(org_sdk, monkeypatch): + accounts = [str(111100000000 + index) for index in range(150)] + identity(org_sdk.central.sts, MANAGEMENT) + org_sdk.central.organizations.add_response("describe_organization", { + "Organization": {"MasterAccountId": MANAGEMENT}, + }, {}) + for start in range(0, len(accounts), 20): + response = {"Accounts": [ + {"Id": account, "State": "ACTIVE"} for account in accounts[start:start + 20] + ]} + if start + 20 < len(accounts): + response["NextToken"] = f"page-{start + 20}" + params = {"NextToken": f"page-{start}"} if start else {} + org_sdk.central.organizations.add_response("list_accounts", response, params) + for account in accounts: + org_sdk.central.sts.add_response("assume_role", {"Credentials": { + "AccessKeyId": "ASIA" + "A" * 16, "SecretAccessKey": "test-only-secret", + "SessionToken": account, "Expiration": datetime(2099, 1, 1, tzinfo=timezone.utc), + }}, {"RoleArn": f"arn:aws:iam::{account}:role/NetBoxInventoryReadOnly", + "RoleSessionName": "netbox-sync", "DurationSeconds": 3600}) + org_sdk.factory.side_effect = lambda **kwargs: org_sdk.central.session + discovery = Mock(side_effect=lambda regions, expected_account, discovery_region: ( + expected_account, {REGION: inventory_data()})) + monkeypatch.setattr(AWSClient, "discover", discovery) + snapshots = AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + assert list(snapshots) == accounts + assert discovery.call_count == 150 + assert org_sdk.factory.call_count == 151 + + +def test_organization_source_forwards_options(make_source, monkeypatch): + source = make_source(f""" +organization = True +organization_include_management = True +organization_role_name = service-accounts/Inventory +organization_external_id = inventory-test +organization_account_ids = {ACCOUNT}, {SECOND_ACCOUNT} +discovery_region = us-west-2 +""") + discovery = Mock(return_value={ACCOUNT: {}}) + monkeypatch.setattr(AWSClient, "discover_organization", discovery) + source.apply() + discovery.assert_called_once_with( + [REGION], "service-accounts/Inventory", expected_account=None, + discovery_region="us-west-2", include_management=True, external_id="inventory-test", + account_ids=[ACCOUNT, SECOND_ACCOUNT]) + + +@pytest.mark.parametrize("accounts", ["[]", "[123456789012]", "123456789012", "true"]) +def test_organization_invalid_yaml_allowlist(inventory, load_config, monkeypatch, accounts): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(f""" +source: + test-aws: + type: aws + organization: true + organization_account_ids: {accounts} +""", "settings.yaml") + with pytest.raises(SystemExit): + instantiate_sources() diff --git a/uv.lock b/uv.lock index 0af21d5f..df188b4f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 -revision = 3 -requires-python = ">=3.13" +revision = 5 +requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", "python_full_version < '3.14'", @@ -18,6 +18,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/70/72e4ab117425ccdc4d10bd523a94c1baa051a15586057d64a4c6888f9e3f/aiodns-4.0.4-py3-none-any.whl", hash = "sha256:c24dd605bac70a1676ce503f967a98483ff163507198557d8e9db16267e6cfd2", size = 12696, upload-time = "2026-05-20T01:54:14.134Z" }, ] +[[package]] +name = "boto3" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/8b/5694c8433b5202f34a82def4edee274f7df1eba2919c66d9d389d2202ec8/boto3-1.43.118.tar.gz", hash = "sha256:c0620f799e0f0c222ba42c337b65cdbcd3da38a3121187f634d8926cbbb28a9c", size = 112649, upload-time = "2026-10-22T19:22:09.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a3/65bb3d64928f3c2da17640b92044084150317f02adf22442cb0c52af2a8b/boto3-1.43.118-py3-none-any.whl", hash = "sha256:ab8f3d2c622ef039338b4f228d97ea55f1abd573d84bb9b9a5ee8cb6780c0ccd", size = 140045, upload-time = "2026-10-22T19:22:08.353Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b8/3e55154ddaf1a6d1d4ddda85c0e5983205a3b7657edacd8ebba8c6fd774c/botocore-1.43.118.tar.gz", hash = "sha256:7f3f9d88cc9e847587690f82a42e3c3c292c3026c92c1e60d68a9310018e003f", size = 16420126, upload-time = "2026-10-22T19:22:04.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/ce/61b39a8e7a69af5ea43c1c3e1b0065a4866b42ccddf107e16686a3143614/botocore-1.43.118-py3-none-any.whl", hash = "sha256:a45d7ae570674161e4a3542d8c68633bf7503f92d1de781ef7361b79488eda6f", size = 16122016, upload-time = "2026-10-22T19:22:00.414Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -36,6 +64,18 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, @@ -106,6 +146,22 @@ version = "3.5.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, @@ -255,9 +311,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "netbox-sync" -version = "1.9.0" +version = "1.9.1" source = { virtual = "." } dependencies = [ { name = "aiodns" }, @@ -280,6 +345,9 @@ dependencies = [ ] [package.optional-dependencies] +aws = [ + { name = "boto3" }, +] test = [ { name = "pytest" }, ] @@ -287,6 +355,7 @@ test = [ [package.metadata] requires-dist = [ { name = "aiodns", specifier = "==4.0.4" }, + { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.40,<2" }, { name = "certifi", specifier = "==2026.7.22" }, { name = "cffi", specifier = "==2.1.1" }, { name = "charset-normalizer", specifier = "==3.5.2" }, @@ -305,7 +374,7 @@ requires-dist = [ { name = "urllib3", specifier = "==2.8.0" }, { name = "wheel", specifier = "==0.48.0" }, ] -provides-extras = ["test"] +provides-extras = ["aws", "test"] [[package]] name = "packaging" @@ -334,6 +403,18 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/df/a0/9c823651872e6a0face3f0311de2a40c8bbcb9c8dcb15680bd019ac56ac7/pycares-5.0.1.tar.gz", hash = "sha256:5a3c249c830432631439815f9a818463416f2a8cbdb1e988e78757de9ae75081", size = 652222, upload-time = "2026-01-01T12:37:00.604Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/75/67/e84ba11d3fec3bf1322c3b302c4df13c85e0a1bc48f16d65cd0f59ad9853/pycares-5.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ee551be4f3f3ac814ac8547586c464c9035e914f5122a534d25de147fa745e1", size = 136241, upload-time = "2026-01-01T12:35:25.439Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ae/50fbb3b4e52b9f1d16a36ffabd051ef8b2106b3f0a0d1c1113904d187a9d/pycares-5.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:252d4e5a52a68f825eaa90e16b595f9baee22c760f51e286ab612c6829b96de3", size = 131069, upload-time = "2026-01-01T12:35:26.293Z" }, + { url = "https://files.pythonhosted.org/packages/0e/ea/f431599f1ac42149ea4768e516db7cdae3a503a6646319ae63ab66da1486/pycares-5.0.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c1aa549b8c2f2e224215c793d660270778dcba9abc3b85abbc7c41eabe4f1e5", size = 221120, upload-time = "2026-01-01T12:35:27.143Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4f/0a7a6c8b3a64ee5149e935c167cd8ba5d1fdd766ec03e273dbc7502f7bea/pycares-5.0.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:db7c9c9f16e8311998667a7488e817f8cbeedec2447bac827c71804663f1437e", size = 252228, upload-time = "2026-01-01T12:35:28.443Z" }, + { url = "https://files.pythonhosted.org/packages/49/3d/7f9fd20e97ee30c4b959f87ab26e47ddcef666e5e7717e45f2245fe9d70a/pycares-5.0.1-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9c4c8bb69bab863f677fa166653bb872bfa5d5a742f1f30bebc2d53b6e71db", size = 239473, upload-time = "2026-01-01T12:35:29.794Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d0/c67967a10abd89529cb9aded9d73f43e5de00cf21243638ef529f6757262/pycares-5.0.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09ef90da8da3026fcba4ed223bd71e8057608d5b3fec4f5990b52ae1e8c855cc", size = 223831, upload-time = "2026-01-01T12:35:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/4f/9a/94aacaf22a20b7d342c8f18bf006be57967beef6319adc668d4d86b627be/pycares-5.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ce193ebd54f4c74538b751ebb0923a9208c234ff180589d4d3cec134c001840e", size = 223963, upload-time = "2026-01-01T12:35:31.691Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e1/3666aab6fc5e7d0c669b981fe0407e6a4b67e4e6a37ac429d440274663d5/pycares-5.0.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36b9ff18ef231277f99a846feade50b417187a96f742689a9d08b9594e386de4", size = 251813, upload-time = "2026-01-01T12:35:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/94/44/ddab5fbc16ad0084a827167ae8628f54c7a55ce6b743585e6f47a5dd527e/pycares-5.0.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5e40ea4a0ef0c01a02ef7f7390a58c62d237d5ad48d36bc3245e9c2ac181cc22", size = 238181, upload-time = "2026-01-01T12:35:34.078Z" }, + { url = "https://files.pythonhosted.org/packages/66/27/05467933e0e5c4e712302a2d7499797bc3029bf4d0d8ffbfe737254482b7/pycares-5.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3f323b0ddfd2c7896af6fba4f8851d34d3d13387566aa573d93330fb01cb1038", size = 223552, upload-time = "2026-01-01T12:35:35.076Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e2/14f3837e943d46ee12441fe6aaa418fdb2f698d42e179f368eaa9829744b/pycares-5.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:bdc6bcafb72a97b3cdd529fc87210e59e67feb647a7e138110656023599b84da", size = 117478, upload-time = "2026-01-01T12:35:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c3/3284061f18188d5085338e1f1fd4f03d9c135657acf16f8020b9dd3be5fc/pycares-5.0.1-cp312-cp312-win_arm64.whl", hash = "sha256:f8ef4c70c1edaf022875a8f9ff6c0c064f82831225acc91aa1b4f4d389e2e03a", size = 108889, upload-time = "2026-01-01T12:35:37.135Z" }, { url = "https://files.pythonhosted.org/packages/92/0a/6bd9bdc2d0ee23ff3aabab7747212e2c5323a081b9b745624d62df88f7e9/pycares-5.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d1b2c6b152c65f14d0e12d741fabb78a487f0f0d22773eede8d8cfc97af612b", size = 136242, upload-time = "2026-01-01T12:35:38.372Z" }, { url = "https://files.pythonhosted.org/packages/18/2a/2e9f888fc076cfe7a3493a3c4113e787cc4b4533f531dfb562ac9b04898f/pycares-5.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8c8ffcc9a48cfc296fe1aefc07d2c8e29a7f97e4bb366ce17effea6a38825f70", size = 131070, upload-time = "2026-01-01T12:35:39.262Z" }, { url = "https://files.pythonhosted.org/packages/ec/5b/83b5aaf7b6ed102f63cd768a747b6cb5d4624f2eaecd84868d103b9dbf39/pycares-5.0.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8efc38c2703e3530b823a4165a7b28d7ce0fdcf41960fb7a4ca834a0f8cfe79", size = 221137, upload-time = "2026-01-01T12:35:40.155Z" }, @@ -432,6 +513,16 @@ version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, @@ -477,6 +568,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "setuptools" version = "84.0.0" From 7028c13ab505d892ce2bfe75e9e0bec5926db1ce Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 10:19:50 -0500 Subject: [PATCH 02/28] Enhance AWS source documentation with account_name Added account_name configuration for friendly labels in cluster names and updated documentation on instance naming and account handling. --- docs/source_aws.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/source_aws.md b/docs/source_aws.md index d31b9939..c7e8b7ca 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -81,6 +81,7 @@ permitted_subnets = 10.0.0.0/8, 2001:db8::/32 vpc_vrf_relation = us-east-1/vpc-0123456789abcdef0 = Production VPC tag_keys = Environment, Owner tenant = Operations +account_name = Production ``` All options also support the application's usual `NBS_SOURCE__*` @@ -415,23 +416,34 @@ private AWS DNS is often inaccessible from the sync host. | AWS | NetBox | | --- | --- | -| Account ID | Cluster group `AWS ` | -| Account and region | Cluster `AWS /`, type `AWS EC2` | +| Account name | Cluster group `AWS ` | +| Account name and region | Cluster `AWS /`, type `AWS EC2` | | Region | Logical site `AWS ` | -| Instance ID | VM name; serial on NetBox 4.1+ | +| EC2 Name tag | VM name and `aws_name` custom field; falls back to instance ID when missing | +| Instance ID | `aws_instance_id` custom field; serial on NetBox 4.1+ | | Instance type | CPU and memory, plus custom field | -| Name tag / Availability Zone | Custom fields, not identity or physical sites | +| AWS account ID | `aws_account_id` custom field | +| Availability Zone | Custom field, not a physical site | | ENI ID | VM interface name and custom field | | ENI MAC | MAC object on NetBox 4.2+; legacy interface field on older versions | | Private IPv4 / IPv6 | IP in the VPC's existing VRF, assigned to the ENI | | EBS volume | Virtual disk named by volume ID | -Region sites and clusters are logical containers, not claims about physical -AWS facilities or hypervisor placement. Instance IDs within account/region -clusters avoid duplicate or changing Name tags merging unrelated VMs. Memory -uses the instance type's MiB capacity; CPU uses instance CPU options when -reported, otherwise the type's default vCPU count. This is provisioned capacity, -not utilization. +In single-account mode, set `account_name` to use a friendly label in the +cluster group and cluster names; it defaults to the account ID. Organization +mode uses the account name returned by AWS Organizations, falling back to the +account ID when unavailable. The account ID remains available on each VM as +`aws_account_id`. Region sites and clusters are logical containers, not claims +about physical AWS facilities or hypervisor placement. + +VM names come from the EC2 `Name` tag, falling back to the instance ID if the +tag is missing or empty. If multiple instances in the same account and region +have the same Name tag, the instance ID is appended to each VM name to keep the +names distinct. Reconciliation uses the account ID and instance ID custom +fields, so a changed Name tag updates the existing VM rather than creating a +second one. Memory uses the instance type's MiB capacity; CPU uses instance CPU +options when reported, otherwise the type's default vCPU count. This is +provisioned capacity, not utilization. Running instances are `active`; stopped, pending, stopping, and shutting-down instances are `offline`. Terminated instances are excluded and subsequently From 561e7c588bc5a14238e32c296fd39dc2aa19375e Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 10:21:46 -0500 Subject: [PATCH 03/28] Update AWS source tests with new assertions --- tests/test_aws_source.py | 58 +++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/tests/test_aws_source.py b/tests/test_aws_source.py index d93b537b..041a797d 100644 --- a/tests/test_aws_source.py +++ b/tests/test_aws_source.py @@ -116,11 +116,12 @@ def test_ec2_ebs_eni_import(make_source, inventory, vpc_vrf, api_version): vm, = inventory.get_all_items(NBVM) cluster, = inventory.get_all_items(NBCluster) interface, = inventory.get_all_items(NBVMInterface) - assert vm.data["name"] == INSTANCE assert vm.data["status"] == "active" + assert vm.data["name"] == "web" assert vm.data["vcpus"] == 2 assert vm.data["memory"] == 2048 assert vm.data["custom_fields"]["aws_name"] == "web" + assert vm.data["custom_fields"]["aws_instance_id"] == INSTANCE assert vm.data["tenant"].data["name"] == "Operations" assert "AWS:Environment=test" in vm.get_tags() assert interface.data["name"] == ENI @@ -213,10 +214,53 @@ def test_name_tag_rename_does_not_duplicate_vm(make_source, inventory, vpc_vrf): data["instances"][0]["Tags"][0]["Value"] = "renamed" source.apply() vm, = inventory.get_all_items(NBVM) - assert vm.data["name"] == INSTANCE + assert vm.data["name"] == "renamed" assert vm.data["custom_fields"]["aws_name"] == "renamed" +def test_missing_name_tag_uses_instance_id(make_source, inventory, vpc_vrf): + data = inventory_data() + data["instances"][0]["Tags"] = [{"Key": "Environment", "Value": "test"}] + make_source(data=data).apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["name"] == INSTANCE + assert vm.data["custom_fields"]["aws_instance_id"] == INSTANCE + + +def test_duplicate_name_tags_include_instance_ids(make_source, inventory, vpc_vrf): + data = inventory_data() + duplicate = copy.deepcopy(data["instances"][0]) + duplicate_id = "i-11111111111111111" + duplicate["InstanceId"] = duplicate_id + duplicate["NetworkInterfaces"] = [] + duplicate["BlockDeviceMappings"] = [] + data["instances"].append(duplicate) + make_source(data=data).apply() + assert {vm.data["name"] for vm in inventory.get_all_items(NBVM)} == { + f"web ({INSTANCE})", f"web ({duplicate_id})", + } + + +def test_account_name_is_used_for_cluster_labels(make_source, inventory, vpc_vrf): + make_source("account_name = Production", data=inventory_data()).apply() + cluster, = inventory.get_all_items(NBCluster) + assert cluster.data["name"] == f"AWS Production/{REGION}" + + +def test_duplicate_organization_account_names_remain_distinct(make_source, inventory): + snapshot = {REGION: inventory_data()} + source = make_source("organization = True\nsync_private_ips = False") + source.client.account_names = {ACCOUNT: "Shared", SECOND_ACCOUNT: "Shared"} + source.client.discover_organization = Mock(return_value={ + ACCOUNT: snapshot, SECOND_ACCOUNT: snapshot, + }) + source.apply() + assert {cluster.data["name"] for cluster in inventory.get_all_items(NBCluster)} == { + f"AWS Shared ({ACCOUNT})/{REGION}", f"AWS Shared ({SECOND_ACCOUNT})/{REGION}", + } + assert len(inventory.get_all_items(NBVM)) == 2 + + def test_removed_aws_tag_is_reconciled_but_user_tag_is_kept(make_source, inventory, vpc_vrf): data = inventory_data() source = make_source(data=data) @@ -707,10 +751,10 @@ def organization_accounts(sdk, accounts=None, partition="aws"): }, {}) sdk.central.organizations.add_response("list_accounts", { "Accounts": accounts if accounts is not None else [ - {"Id": MANAGEMENT, "State": "ACTIVE"}, - {"Id": ACCOUNT, "State": "ACTIVE"}, - {"Id": SECOND_ACCOUNT, "State": "ACTIVE"}, - {"Id": "345678901234", "State": "SUSPENDED"}, + {"Id": MANAGEMENT, "Name": "Management", "State": "ACTIVE"}, + {"Id": ACCOUNT, "Name": "Production", "State": "ACTIVE"}, + {"Id": SECOND_ACCOUNT, "Name": "Development", "State": "ACTIVE"}, + {"Id": "345678901234", "Name": "Suspended", "State": "SUSPENDED"}, ], }, {"NextToken": "next-page"}) @@ -754,7 +798,7 @@ def test_organization_scans_members_not_management(org_sdk, inventory, load_conf assert len(vms) == 2 assert {vm.data["custom_fields"]["aws_account_id"] for vm in vms} == {ACCOUNT, SECOND_ACCOUNT} assert {vm.data["cluster"].data["name"] for vm in vms} == { - f"AWS {ACCOUNT}/{REGION}", f"AWS {SECOND_ACCOUNT}/{REGION}", + f"AWS Production/{REGION}", f"AWS Development/{REGION}", } assert len(inventory.get_all_items(NBVMInterface)) == 2 assert len(inventory.get_all_items(NBVirtualDisk)) == 2 From d39e605f2449d5e5351da3789f1069fcf1b8593e Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 10:22:27 -0500 Subject: [PATCH 04/28] Add account_names dictionary for account tracking --- module/sources/aws/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py index 8fc2f308..685af9e0 100644 --- a/module/sources/aws/client.py +++ b/module/sources/aws/client.py @@ -24,6 +24,7 @@ def __init__(self, profile=None, *, session=None): self.session = session if session is not None else boto3.Session(profile_name=profile) except BotoCoreError as exc: raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc + self.account_names = {} def discover_organization(self, regions, role_name, expected_account=None, discovery_region=None, include_management=False, external_id=None, account_ids=None): @@ -49,6 +50,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco clients.append(organizations) management_account = organizations.describe_organization()["Organization"]["MasterAccountId"] accounts = set() + account_names = {} for page in organizations.get_paginator("list_accounts").paginate(): for account in page["Accounts"]: account_id = account["Id"] @@ -58,6 +60,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco if account_id == management_account and not include_management: continue accounts.add(account_id) + account_names[account_id] = account.get("Name") or account_id if account_ids: missing = set(account_ids) - accounts if missing: @@ -80,6 +83,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco aws_session_token=credentials["SessionToken"], region_name=bootstrap)) _, snapshot = member.discover(regions, current_account, discovery_region) snapshots[current_account] = snapshot + self.account_names = account_names return snapshots except (BotoCoreError, ClientError, KeyError, AWSDiscoveryError) as exc: context = f" for account '{current_account}'" if current_account else "" From 2e34144c7a410de899159405781901998937f23f Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 10:23:37 -0500 Subject: [PATCH 05/28] Add account_name option to AWS config --- module/sources/aws/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/module/sources/aws/config.py b/module/sources/aws/config.py index 75bfc774..50173e03 100644 --- a/module/sources/aws/config.py +++ b/module/sources/aws/config.py @@ -35,6 +35,8 @@ def __init__(self): ConfigOption("account_id", str, description="""Optional expected 12-digit credential account ID. In organization mode this guards the collector account, not member accounts."""), + ConfigOption("account_name", str, + description="Optional friendly account name for NetBox cluster labels in single-account mode."), ConfigOption("organization", bool, default_value=False, description="Discover active AWS Organizations accounts and assume an inventory role in each."), ConfigOption("organization_role_name", str, default_value="NetBoxInventoryReadOnly", @@ -64,7 +66,7 @@ def __init__(self): super().__init__() def validate_options(self): - for key in ("profile", "tenant", "discovery_region", + for key in ("profile", "tenant", "discovery_region", "account_name", "organization_role_name", "organization_external_id"): value = self.get_option_by_name(key).value if value is not None and not isinstance(value, str): From f80b15c1c4d9a3acd7c36dde28f05dc18d01afd4 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 10:23:57 -0500 Subject: [PATCH 06/28] Enhance sync_snapshot and sync_instance methods --- module/sources/aws/connection.py | 47 ++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 913df14f..e55548df 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -58,15 +58,28 @@ def apply(self): include_management=self.settings.organization_include_management, external_id=self.settings.organization_external_id, account_ids=self.settings.organization_account_ids) + account_names = getattr(self.client, "account_names", {}) else: account, snapshot = self.client.discover( self.settings.regions, self.settings.account_id, self.settings.discovery_region) snapshots = {account: snapshot} + account_names = {} vrfs = {} for account, snapshot in snapshots.items(): self.validate_snapshot(account, snapshot, vrfs) + account_labels = { + account: account_names.get(account) or self.settings.account_name or account + for account in snapshots + } + label_counts = {} + for label in account_labels.values(): + key = label.casefold() + label_counts[key] = label_counts.get(key, 0) + 1 + for account, label in account_labels.items(): + if label_counts[label.casefold()] > 1: + account_labels[account] = f"{label} ({account})" for account, snapshot in snapshots.items(): - self.sync_snapshot(account, snapshot) + self.sync_snapshot(account, snapshot, account_name=account_labels[account]) except (AWSDiscoveryError, ValueError, KeyError) as exc: self.abort(exc) @@ -165,16 +178,17 @@ def custom_fields(self): "description": f"Reported by AWS ({self.name})", }) - def sync_snapshot(self, account, snapshot): + def sync_snapshot(self, account, snapshot, account_name=None): self.custom_fields() + account_name = account_name or account group = self.inventory.add_update_object( - NBClusterGroup, data={"name": f"AWS {account}"}, source=self) + NBClusterGroup, data={"name": f"AWS {account_name}"}, source=self) api_version = Version(self.inventory.netbox_api_version) for region, data in snapshot.items(): site = self.inventory.add_update_object( NBSite, data={"name": f"AWS {region}"}, source=self) cluster_data = { - "name": f"AWS {account}/{region}", "group": group, + "name": f"AWS {account_name}/{region}", "group": group, "type": {"name": "AWS EC2"}, } if api_version >= Version("4.2"): @@ -182,15 +196,26 @@ def sync_snapshot(self, account, snapshot): else: cluster_data["site"] = site cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) + name_counts = {} + for instance in data["instances"]: + name = next((tag["Value"].strip() for tag in instance.get("Tags", []) + if tag["Key"] == "Name" and tag["Value"].strip()), instance["InstanceId"]) + key = name.casefold() + name_counts[key] = name_counts.get(key, 0) + 1 for instance in data["instances"]: - self.sync_instance(account, region, instance, data, site, cluster, api_version) + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + vm_name = tags.get("Name", "").strip() or instance["InstanceId"] + if name_counts[vm_name.casefold()] > 1: + vm_name = f"{vm_name} ({instance['InstanceId']})" + self.sync_instance( + account, region, instance, data, site, cluster, api_version, vm_name=vm_name) - def sync_instance(self, account, region, instance, data, site, cluster, api_version): + def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): instance_id = instance["InstanceId"] tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} spec = data["InstanceTypes"][instance["InstanceType"]] vm_data = { - "name": instance_id, "cluster": cluster, + "name": vm_name or tags.get("Name", "").strip() or instance_id, "cluster": cluster, "status": "active" if instance["State"]["Name"] == "running" else "offline", "vcpus": spec["VCpuInfo"]["DefaultVCpus"], "memory": spec["MemoryInfo"]["SizeInMiB"], @@ -218,7 +243,13 @@ def sync_instance(self, account, region, instance, data, site, cluster, api_vers ] if api_version < Version("3.7"): vm_data["disk"] = sum(volume["Size"] for volume in disks) - vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) + vm = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if vm is None: + vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) + else: + vm.update(data=vm_data, source=self) for tag in vm.get_tags(): if any(tag.startswith(f"AWS:{key}=") for key in self.settings.tag_keys) and tag not in vm_data["tags"]: vm.remove_tags(tag) From bebc9035193347a1851bb8e3f396ee818eb9928c Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 16:50:55 -0500 Subject: [PATCH 07/28] Rename test for missing VRF to reflect default creation --- tests/test_aws_source.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_aws_source.py b/tests/test_aws_source.py index 041a797d..883d5ba6 100644 --- a/tests/test_aws_source.py +++ b/tests/test_aws_source.py @@ -177,13 +177,21 @@ def test_overlapping_vpcs_have_distinct_ips(make_source, inventory, vpc_vrf): assert len(inventory.get_all_items(NBIPAddress)) == 6 -def test_missing_vrf_aborts_before_inventory_changes(make_source, inventory, caplog): +def test_missing_default_vrf_is_created_for_vpc(make_source, inventory): source = make_source() + source.apply() + vrf, = inventory.get_all_items(NBVRF) + assert vrf.data["name"] == f"AWS {ACCOUNT}/{REGION}/{VPC}" + assert inventory.get_all_items(NBVM) + assert len(inventory.get_all_items(NBIPAddress)) == 3 + + +def test_missing_mapped_vrf_aborts_before_vm_import(make_source, inventory): + source = make_source(f"vpc_vrf_relation = {REGION}/{VPC} = Production") with pytest.raises(SystemExit): source.apply() assert inventory.get_all_items(NBVM) == [] assert inventory.get_all_items(NBCluster) == [] - assert "Create NetBox VRF" in caplog.text def test_missing_volume_aborts_before_inventory_changes(make_source, inventory, vpc_vrf): From 24b0d3d497af8c04c6d50bfb618facbf3ed6fe74 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 16:52:14 -0500 Subject: [PATCH 08/28] Update VRF creation logic in connection.py Refactor error handling for VRF creation and add mapping check. --- module/sources/aws/connection.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index e55548df..859932a3 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -89,10 +89,16 @@ def vrf_name(self, account, region, vpc): f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) def get_vrf(self, account, region, vpc): + account_region_vpc = f"{account}/{region}/{vpc}" + region_vpc = f"{region}/{vpc}" + has_mapping = account_region_vpc in self.settings.vpc_vrf_relation or \ + region_vpc in self.settings.vpc_vrf_relation name = self.vrf_name(account, region, vpc) vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) if vrf is None: - raise ValueError(f"Create NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") + if has_mapping: + raise ValueError(f"Create mapped NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") + vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) return vrf def addresses(self, eni, subnet): From e38f4779e2c5482a06e0d823c2d3f9c1c5ab2841 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 16:52:30 -0500 Subject: [PATCH 09/28] Update descriptions for config options in config.py --- module/sources/aws/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/module/sources/aws/config.py b/module/sources/aws/config.py index 50173e03..2c6e227d 100644 --- a/module/sources/aws/config.py +++ b/module/sources/aws/config.py @@ -50,11 +50,12 @@ def __init__(self): Unset selects all active member accounts. Management inclusion is controlled separately."""), ConfigOption(**config_option_permitted_subnets_definition), ConfigOption("sync_private_ips", bool, default_value=True, - description="Import private IPv4 and IPv6 into pre-existing VPC-specific VRFs."), + description="Import private IPv4 and IPv6 into VPC-specific VRFs, creating default VRFs as needed."), ConfigOption("vpc_vrf_relation", str, description="""Comma-separated region/vpc-id = NetBox VRF name mappings. Use account-id/region/vpc-id keys for account-specific mappings in organization mode. - Unmapped VPCs require a VRF named AWS //. + Unmapped VPCs automatically create a VRF named AWS //. + Mapped VRF names must already exist in NetBox. Each VPC must have a distinct VRF; global address space is never used.""", config_example="us-east-1/vpc-0123456789abcdef0 = Production VPC"), ConfigOption("tag_keys", str, From 73aa78a6945e483c34e5a8600715c08b7c74b80e Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sun, 25 Oct 2026 16:56:42 -0500 Subject: [PATCH 10/28] Update documentation on VPC VRF handling Clarify VRF creation and mapping for VPCs in NetBox integration. --- docs/source_aws.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/source_aws.md b/docs/source_aws.md index c7e8b7ca..c4bc4403 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -380,18 +380,19 @@ scope. Avoid overlapping single-account and organization source definitions. ### VPC address-space isolation -The current NetBox integration reads VRFs and prefixes but does not create them. -Before enabling IP import, create a **distinct VRF for every VPC** in NetBox. -The default VRF name is: +Each VPC uses a **distinct VRF**. When IP import encounters a VPC without an +explicit `vpc_vrf_relation`, the importer creates a VRF in NetBox using this +default name: ```text AWS // ``` For example: `AWS 123456789012/us-east-1/vpc-0123456789abcdef0`. -Use `vpc_vrf_relation` to select an existing differently named VRF. A mapping -key may be `account-id/region/vpc-id` in organization mode; this takes precedence -over a legacy `region/vpc-id` mapping. For example: +Use `vpc_vrf_relation` to select an existing differently named VRF; mapped +VRFs must already exist, and a missing mapped VRF aborts the run rather than +creating one. A mapping key may be `account-id/region/vpc-id` in organization +mode; this takes precedence over a legacy `region/vpc-id` mapping. For example: ```ini vpc_vrf_relation = 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC @@ -401,16 +402,16 @@ A mapping must not share a VRF between discovered VPCs. VRFs must remain distinc accounts and separately configured sources too. Organization mode validates VRF isolation across all selected accounts before reconciliation. -Missing VRFs for permitted addresses abort the run before inventory updates. There is no fallback to global address space. Subnet mask lengths come from AWS subnet CIDRs; NetBox prefixes are neither required nor created. Identical IPs in different VPCs remain separate NetBox IPs. `permitted_subnets` is required to populate IP addresses; leaving it undefined imports no IPs, matching other source behavior. `sync_private_ips = False` -imports VMs, disks, and interfaces without requiring VRFs; previously managed -IPs on discovered interfaces are retained. DNS lookup defaults to false because -private AWS DNS is often inaccessible from the sync host. +imports VMs, disks, and interfaces without creating or requiring VRFs; +previously managed IPs on discovered interfaces are retained. DNS lookup +defaults to false because private AWS DNS is often inaccessible from the sync +host. ## Object mapping and lifecycle From b0eeb653c0434559bc1abdb68f799d6c6c818306 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Mon, 26 Oct 2026 14:54:58 -0500 Subject: [PATCH 11/28] Enhance AWS connection handling for networks and addresses Added resolve_shared_networks method to handle VPC and subnet ownership conflicts. Updated addresses method to include IPv6 networks and modified VPC ownership retrieval logic. --- module/sources/aws/connection.py | 96 ++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 10 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 859932a3..df3e33ef 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -64,6 +64,8 @@ def apply(self): self.settings.regions, self.settings.account_id, self.settings.discovery_region) snapshots = {account: snapshot} account_names = {} + if self.settings.organization and self.settings.sync_private_ips: + self.resolve_shared_networks(snapshots) vrfs = {} for account, snapshot in snapshots.items(): self.validate_snapshot(account, snapshot, vrfs) @@ -83,6 +85,63 @@ def apply(self): except (AWSDiscoveryError, ValueError, KeyError) as exc: self.abort(exc) + def resolve_shared_networks(self, snapshots): + networks = {} + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + for kind in ("Vpcs", "Subnets"): + for resource_id, resource in data[kind].items(): + owner = resource.get("OwnerId", account) + key = (kind, region, resource_id) + owners = networks.setdefault(key, {}) + if resource.get("OwnerId") and any( + item.get("OwnerId") and previous_owner != owner + for previous_owner, item in owners.items()): + raise ValueError( + f"Conflicting AWS {kind} ownership for '{resource_id}' " + f"(region '{region}'): {', '.join(sorted(set(owners) | {owner}))}") + if owner in owners: + previous = owners[owner] + fields = ("VpcId", "CidrBlock") + if any(previous.get(field) != resource.get(field) for field in fields) or \ + set(self.ipv6_networks(previous)) != set(self.ipv6_networks(resource)): + raise ValueError( + f"Conflicting AWS {kind} discovery for '{resource_id}' " + f"(owner '{owner}', region '{region}')") + owners[owner] = resource + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + data["VpcOwners"] = {} + for instance in data["instances"]: + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + continue + vpc_id, subnet_id = eni["VpcId"], eni["SubnetId"] + vpcs = networks.get(("Vpcs", region, vpc_id), {}) + local_vpc = data["Vpcs"].get(vpc_id) + owner = local_vpc.get("OwnerId", account) if local_vpc is not None else None + if owner is None and len(vpcs) == 1: + owner = next(iter(vpcs)) + if owner is None and vpcs: + raise ValueError( + f"Ambiguous AWS VPC ownership for '{vpc_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}'): {', '.join(sorted(vpcs))}") + if owner is None: + continue + data["VpcOwners"][vpc_id] = owner + data["Vpcs"][vpc_id] = dict(vpcs[owner], OwnerId=owner) + subnets = networks.get(("Subnets", region, subnet_id), {}) + if subnets and owner not in subnets: + raise ValueError( + f"Conflicting AWS subnet ownership for '{subnet_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}', VPC owner '{owner}'): " + f"{', '.join(sorted(subnets))}") + if owner in subnets: + data["Subnets"][subnet_id] = dict(subnets[owner], OwnerId=owner) + def vrf_name(self, account, region, vpc): return self.settings.vpc_vrf_relation.get( f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( @@ -101,13 +160,15 @@ def get_vrf(self, account, region, vpc): vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) return vrf - def addresses(self, eni, subnet): - networks = [ip_network(subnet["CidrBlock"])] - networks.extend( + def ipv6_networks(self, resource): + return [ ip_network(entry["Ipv6CidrBlock"]) - for entry in subnet.get("Ipv6CidrBlockAssociationSet", []) + for entry in resource.get("Ipv6CidrBlockAssociationSet", []) if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" - ) + ] + + def addresses(self, eni, subnet): + networks = [ip_network(subnet["CidrBlock"]), *self.ipv6_networks(subnet)] addresses = [ (entry["PrivateIpAddress"], entry.get("Primary", False)) for entry in eni.get("PrivateIpAddresses", []) @@ -156,11 +217,25 @@ def validate_snapshot(self, account, snapshot, vrfs=None): if not self.settings.sync_private_ips: continue vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) - if vpc not in data["Vpcs"] or subnet is None or subnet["VpcId"] != vpc: - raise ValueError(f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}'") + network_errors = [] + if vpc not in data["Vpcs"]: + network_errors.append(f"VPC '{vpc}' was not returned by DescribeVpcs") + if subnet is None: + network_errors.append( + f"subnet '{eni['SubnetId']}' was not returned by DescribeSubnets") + elif subnet["VpcId"] != vpc: + network_errors.append( + f"subnet '{eni['SubnetId']}' belongs to VPC '{subnet['VpcId']}', " + f"not ENI VPC '{vpc}'") + if network_errors: + raise ValueError( + f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}' " + f"(account '{account}', region '{region}', instance '{instance['InstanceId']}', " + f"VPC '{vpc}', subnet '{eni['SubnetId']}'): {'; '.join(network_errors)}") if self.addresses(eni, subnet): - vrf = self.get_vrf(account, region, vpc) - identity = (account, region, vpc) + owner = data.get("VpcOwners", {}).get(vpc, account) + vrf = self.get_vrf(owner, region, vpc) + identity = (owner, region, vpc) if vrf in vrfs and vrfs[vrf] != identity: raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") vrfs[vrf] = identity @@ -321,7 +396,8 @@ def sync_interface(self, account, region, vm, eni, data, api_version): addresses = self.addresses(eni, subnet) discovered = [] for address, primary in addresses: - vrf = self.get_vrf(account, region, eni["VpcId"]) + owner = data.get("VpcOwners", {}).get(eni["VpcId"], account) + vrf = self.get_vrf(owner, region, eni["VpcId"]) parsed = ip_interface(address) candidates = [ item for item in self.inventory.get_all_items(NBIPAddress) From 55a3404f5a20551460553288859104c475949a75 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Mon, 26 Oct 2026 14:55:48 -0500 Subject: [PATCH 12/28] Remove read_only attribute from VRF object class --- module/netbox/object_classes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dacc36fe..7ee564e4 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1522,7 +1522,6 @@ class NBVRF(NetBoxObject): object_type = "ipam.vrf" primary_key = "name" prune = False - read_only = True def __init__(self, *args, **kwargs): self.data_model = { From 00be56eeaef9ccba4e4e22b231772e553e134f7a Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Mon, 26 Oct 2026 14:56:31 -0500 Subject: [PATCH 13/28] Add tests for incomplete network discovery and organization --- tests/test_aws_source.py | 181 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/tests/test_aws_source.py b/tests/test_aws_source.py index 883d5ba6..2bb0f546 100644 --- a/tests/test_aws_source.py +++ b/tests/test_aws_source.py @@ -203,6 +203,39 @@ def test_missing_volume_aborts_before_inventory_changes(make_source, inventory, assert inventory.get_all_items(NBVM) == [] +@pytest.mark.parametrize("failure", ["vpc", "subnet", "both", "mismatch"]) +def test_incomplete_network_discovery_reports_context_and_aborts( + make_source, inventory, caplog, failure): + data = inventory_data() + other_vpc = "vpc-11111111111111111" + reasons = [] + if failure in ("vpc", "both"): + data["Vpcs"].clear() + reasons.append(f"VPC '{VPC}' was not returned by DescribeVpcs") + if failure in ("subnet", "both"): + data["Subnets"].clear() + reasons.append(f"subnet '{SUBNET}' was not returned by DescribeSubnets") + if failure == "mismatch": + data["Subnets"][SUBNET]["VpcId"] = other_vpc + reasons.append(f"subnet '{SUBNET}' belongs to VPC '{other_vpc}', not ENI VPC '{VPC}'") + source = make_source(data=data) + source.sync_snapshot = Mock() + + with pytest.raises(SystemExit): + source.apply() + + assert ( + f"Incomplete VPC/subnet discovery for ENI '{ENI}' " + f"(account '{ACCOUNT}', region '{REGION}', instance '{INSTANCE}', " + f"VPC '{VPC}', subnet '{SUBNET}'): {'; '.join(reasons)}" + ) in caplog.text + assert "No NetBox writes or pruning will be performed." in caplog.text + source.sync_snapshot.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert inventory.get_all_items(NBIPAddress) == [] + + def test_stopped_instance_without_public_ip_is_retained(make_source, inventory, vpc_vrf): data = inventory_data() data["instances"][0]["State"]["Name"] = "stopped" @@ -962,6 +995,154 @@ def test_organization_account_specific_vrf_mapping(make_source, inventory, vpc_v } +def shared_network_snapshots(): + owner = inventory_data() + owner["Vpcs"][VPC]["OwnerId"] = ACCOUNT + owner["Subnets"][SUBNET]["OwnerId"] = ACCOUNT + participant = copy.deepcopy(owner) + instance = participant["instances"][0] + instance["InstanceId"] = "i-11111111111111111" + instance["NetworkInterfaces"] = [{"NetworkInterfaceId": "eni-11111111111111111"}] + eni = participant["NetworkInterfaces"].pop(ENI) + eni["NetworkInterfaceId"] = "eni-11111111111111111" + eni["Attachment"]["InstanceId"] = instance["InstanceId"] + eni["MacAddress"] = "02:aa:bb:cc:dd:ff" + eni["PrivateIpAddresses"] = [{"PrivateIpAddress": "10.0.0.20", "Primary": True}] + eni["Ipv6Addresses"] = [{"Ipv6Address": "2001:db8::20", "IsPrimaryIpv6": True}] + participant["NetworkInterfaces"][eni["NetworkInterfaceId"]] = eni + return {ACCOUNT: {REGION: owner}, SECOND_ACCOUNT: {REGION: participant}} + + +@pytest.mark.parametrize("missing", ["both", "vpc", "subnet", "neither"]) +@pytest.mark.parametrize("participant_first", [False, True]) +def test_organization_shared_network_uses_owner_vrf( + make_source, inventory, monkeypatch, missing, participant_first): + snapshots = shared_network_snapshots() + participant = snapshots[SECOND_ACCOUNT][REGION] + if missing in ("both", "vpc"): + participant["Vpcs"].clear() + if missing in ("both", "subnet"): + participant["Subnets"].clear() + if participant_first: + snapshots = dict(reversed(list(snapshots.items()))) + source = make_source("organization = True") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value=snapshots)) + + source.apply() + source.apply() + + vrf, = inventory.get_all_items(NBVRF) + assert vrf.data["name"] == f"AWS {ACCOUNT}/{REGION}/{VPC}" + assert len(inventory.get_all_items(NBIPAddress)) == 5 + assert all(ip.data["vrf"] is vrf for ip in inventory.get_all_items(NBIPAddress)) + assert {vm.data["custom_fields"]["aws_account_id"] for vm in inventory.get_all_items(NBVM)} == { + ACCOUNT, SECOND_ACCOUNT, + } + + +def test_organization_shared_network_uses_owner_mapping(make_source, inventory, monkeypatch): + snapshots = shared_network_snapshots() + source = make_source(f""" +organization = True +vpc_vrf_relation = {ACCOUNT}/{REGION}/{VPC} = Shared Network +""") + vrf = inventory.add_object(NBVRF, data={"id": 1, "name": "Shared Network"}, read_from_netbox=True) + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value=snapshots)) + source.apply() + assert {ip.data["vrf"] for ip in inventory.get_all_items(NBIPAddress)} == {vrf} + + +@pytest.mark.parametrize("failure, message", [ + ("missing-subnet", "was not returned by DescribeSubnets"), + ("missing-vpc", "was not returned by DescribeVpcs"), + ("wrong-region", "was not returned by DescribeVpcs"), + ("subnet-owner", "Conflicting AWS subnet ownership"), + ("subnet-vpc", "belongs to VPC"), + ("ipv4-conflict", "Conflicting AWS Subnets discovery"), + ("ipv6-conflict", "Conflicting AWS Subnets discovery"), + ("vpc-conflict", "Conflicting AWS Vpcs discovery"), + ("vpc-owner", "Conflicting AWS Vpcs ownership"), + ("ambiguous-owner", "Ambiguous AWS VPC ownership"), +]) +def test_organization_shared_network_failure_aborts_before_sync( + make_source, inventory, monkeypatch, caplog, failure, message): + snapshots = shared_network_snapshots() + owner, participant = snapshots[ACCOUNT][REGION], snapshots[SECOND_ACCOUNT][REGION] + if failure in ("missing-subnet", "missing-vpc", "wrong-region", "ambiguous-owner"): + participant["Vpcs"].clear() + participant["Subnets"].clear() + if failure == "missing-subnet": + owner["Subnets"].clear() + elif failure == "missing-vpc": + owner["Vpcs"].clear() + elif failure == "wrong-region": + snapshots[ACCOUNT] = {"us-west-2": owner} + elif failure == "subnet-owner": + owner["Subnets"][SUBNET]["OwnerId"] = SECOND_ACCOUNT + participant["Subnets"][SUBNET]["OwnerId"] = SECOND_ACCOUNT + elif failure == "subnet-vpc": + for data in (owner, participant): + data["Subnets"][SUBNET]["VpcId"] = "vpc-11111111111111111" + elif failure == "ipv4-conflict": + participant["Subnets"][SUBNET]["CidrBlock"] = "10.0.0.0/25" + elif failure == "ipv6-conflict": + participant["Subnets"][SUBNET]["Ipv6CidrBlockAssociationSet"].clear() + elif failure == "vpc-conflict": + participant["Vpcs"][VPC]["CidrBlock"] = "10.0.0.0/16" + elif failure == "vpc-owner": + participant["Vpcs"][VPC]["OwnerId"] = SECOND_ACCOUNT + elif failure == "ambiguous-owner": + owner["Vpcs"][VPC].pop("OwnerId") + third = copy.deepcopy(owner) + third["instances"] = [] + snapshots[MANAGEMENT] = {REGION: third} + source = make_source("organization = True") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value=snapshots)) + source.sync_snapshot = Mock() + + with pytest.raises(SystemExit): + source.apply() + + assert message in caplog.text + source.sync_snapshot.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBIPAddress) == [] + + +def test_shared_network_sdk_to_inventory(org_sdk, inventory, load_config): + snapshots = shared_network_snapshots() + owner, participant = snapshots[ACCOUNT][REGION], snapshots[SECOND_ACCOUNT][REGION] + owner["instances"] = [] + participant["Vpcs"].clear() + participant["Subnets"].clear() + organization_accounts(org_sdk) + for account, data in ((ACCOUNT, owner), (SECOND_ACCOUNT, participant)): + member = assume_member(org_sdk, account) + identity(member.sts, account) + region_response(member.ec2, [(REGION, "opt-in-not-required")]) + stub_region(member.ec2, data) + load_config(f""" +[source/test-aws] +type = aws +organization = True +regions = auto +account_id = {MANAGEMENT} +sync_private_ips = True +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +""") + source, = instantiate_sources() + source.apply() + + vm, = inventory.get_all_items(NBVM) + assert vm.data["custom_fields"]["aws_account_id"] == SECOND_ACCOUNT + vrf, = inventory.get_all_items(NBVRF) + assert vrf.data["name"] == f"AWS {ACCOUNT}/{REGION}/{VPC}" + assert {ip.data["address"] for ip in inventory.get_all_items(NBIPAddress)} == { + "10.0.0.20/24", "2001:db8::20/64", + } + assert vm.data["primary_ip4"].data["vrf"] is vrf + + @pytest.mark.parametrize("extra", [ "organization = True\norganization_role_name = arn:aws:iam::123456789012:role/Inventory", "organization = True\norganization_role_name = invalid role", From 6b913774a899127872b2cb985c1d3fb3e40e8391 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:17:17 -0500 Subject: [PATCH 14/28] Comment out optional AWS service sync settings Comment out optional AWS service sync settings in the configuration file. --- settings-example.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/settings-example.ini b/settings-example.ini index 6140ea7c..8d6b8254 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -632,6 +632,11 @@ regions = auto ;organization_external_id = ; See docs/source_aws.md for collector permissions and member-role trust policies. ;sync_private_ips = True +; Optional AWS services require additional read permissions; all default to False. +; S3 uses logical account/region VMs and metadata-only disks (NetBox 3.7+). +;sync_s3 = False +;sync_lambda = False +;sync_eks = False ;permitted_subnets = 10.0.0.0/8, fd00::/8 ;vpc_vrf_relation = us-east-1/vpc-0123456789abcdef0 = Production VPC ; Organization account-specific key: 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC From 2a7a1a3545ce38a3080687b1282279d3daea8f10 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:17:50 -0500 Subject: [PATCH 15/28] Enhance AWS source description in README Updated AWS source description to include S3, Lambda, and EKS. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a6efed89..711ca097 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a tool to sync data from different sources to a NetBox instance. Available source types: * VMware vCenter Server -* AWS EC2, EBS, and network interfaces (see [AWS source](docs/source_aws.md)) +* AWS EC2, EBS, and network interfaces, with opt-in S3, Lambda, and EKS inventory (see [AWS source](docs/source_aws.md)) * [bb-ricardo/check_redfish](https://github.com/bb-Ricardo/check_redfish) inventory files **IMPORTANT: READ INSTRUCTIONS CAREFULLY BEFORE RUNNING THIS PROGRAM** From 8dd6cbb39650fcce569c6df6b780d0263b75d3d1 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:18:36 -0500 Subject: [PATCH 16/28] Refactor AWS source tests for tag handling and errors Refactor AWS source tests to improve tag handling and error management. Update test cases to ensure proper handling of tags and permissions. --- tests/test_aws_source.py | 211 ++++++++++++++++++++++++++++++++++----- 1 file changed, 184 insertions(+), 27 deletions(-) diff --git a/tests/test_aws_source.py b/tests/test_aws_source.py index 2bb0f546..6c5e7f30 100644 --- a/tests/test_aws_source.py +++ b/tests/test_aws_source.py @@ -13,7 +13,7 @@ from module.config.file_output import ConfigFileOutput from module.config.parser import ConfigParser from module.netbox import ( - NBCluster, NBIPAddress, NBMACAddress, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, + NBCluster, NBIPAddress, NBMACAddress, NBTag, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, ) from module.netbox.connection import NetBoxHandler from module.sources import instantiate_sources, valid_sources @@ -302,6 +302,24 @@ def test_duplicate_organization_account_names_remain_distinct(make_source, inven assert len(inventory.get_all_items(NBVM)) == 2 +@pytest.mark.parametrize("tag_name", ["AWS:Environment=Dev", "aws:environment=Dev"]) +def test_aws_vm_tags_use_resolved_identity(make_source, inventory, vpc_vrf, tag_name): + tag = inventory.add_object(NBTag, data={"name": tag_name}) + data = inventory_data() + data["instances"][0]["Tags"] = [{"Key": "Environment", "Value": "dev"}] + source = make_source(data=data) + for _ in range(2): + source.apply() + vm, = inventory.get_all_items(NBVM) + assert tag in vm.data["tags"] + assert tag_name in vm.get_tags() + vm.add_tags("user-managed") + data["instances"][0]["Tags"] = [] + source.apply() + assert tag_name not in vm.get_tags() + assert "user-managed" in vm.get_tags() + + def test_removed_aws_tag_is_reconciled_but_user_tag_is_kept(make_source, inventory, vpc_vrf): data = inventory_data() source = make_source(data=data) @@ -343,17 +361,151 @@ def test_primary_address_removal_clears_primary_ip(make_source, inventory, vpc_v assert "primary_ip6" in vm.unset_items -def test_ip_owned_by_other_source_is_not_reassigned(make_source, inventory, vpc_vrf): +@pytest.mark.parametrize("tags", [ + [], + [NetBoxHandler.primary_tag], + [NetBoxHandler.orphaned_tag], + [NetBoxHandler.primary_tag, "Source: other"], +]) +def test_ip_owned_by_other_source_is_not_reassigned( + make_source, inventory, vpc_vrf, caplog, tags): source = make_source() - other = inventory.add_object(NBVM, data={"name": "other", "status": "active"}) + other = inventory.add_object(NBVM, data={"name": "other", "status": "active", "tags": tags}) other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) existing = inventory.add_object(NBIPAddress, data={ "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, "assigned_object_type": NBVMInterface, }) - with pytest.raises(SystemExit): - source.apply() + source.apply() assert existing.get_interface() is other_interface + assert "belongs to another source" in caplog.text + assert "Skipping reassignment" in caplog.text + vm = next(item for item in inventory.get_all_items(NBVM) if item is not other) + assert "primary_ip4" not in vm.data + assert vm.data["primary_ip6"].data["address"] == "2001:db8::10/64" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + + +@pytest.mark.parametrize("previous_source", [None, "Source: other"]) +def test_ip_on_managed_orphan_is_reclaimed( + make_source, inventory, vpc_vrf, caplog, previous_source): + source = make_source() + tags = [NetBoxHandler.primary_tag, NetBoxHandler.orphaned_tag] + if previous_source: + tags.append(previous_source) + other = inventory.add_object(NBVM, data={"name": "old", "status": "offline", "tags": tags}) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + other.update(data={"primary_ip4": existing}) + source.apply() + vm = next(item for item in inventory.get_all_items(NBVM) if item is not other) + assert existing.get_device_vm() is vm + assert vm.data["primary_ip4"] is existing + assert "primary_ip4" in other.unset_items + assert len(inventory.get_all_items(NBIPAddress)) == 3 + assert "Reclaiming IP" in caplog.text + + +def test_ip_with_unresolved_owner_is_not_reassigned(make_source, inventory, vpc_vrf, caplog): + source = make_source() + existing = inventory.add_object(NBIPAddress, data={ + "id": 99, "address": "10.0.0.10/24", "vrf": vpc_vrf, + "assigned_object_id": 999, "assigned_object_type": NBVMInterface.object_type, + }, read_from_netbox=True) + source.apply() + assert existing.data["assigned_object_id"] == 999 + assert "unresolved owner" in caplog.text + assert "Skipping reassignment" in caplog.text + vm, = inventory.get_all_items(NBVM) + assert "primary_ip4" not in vm.data + assert vm.data["primary_ip6"].data["address"] == "2001:db8::10/64" + + +@pytest.mark.parametrize("management", ["tag", "source", "source-tag", "unmanaged"]) +def test_offline_owner_ip_reclamation( + make_source, inventory, vpc_vrf, caplog, management): + source = make_source() + tags = { + "tag": [NetBoxHandler.primary_tag, "Source: other"], + "source-tag": [source.source_tag], + }.get(management, []) + other = inventory.add_object( + NBVM, data={"name": "down", "status": "offline", "tags": tags}, + source=source if management == "source" else None) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + other.update(data={"primary_ip4": existing}) + source.apply() + vm = next(item for item in inventory.get_all_items(NBVM) if item is not other) + if management == "unmanaged": + assert existing.get_device_vm() is other + assert other.data["primary_ip4"] is existing + assert "primary_ip4" not in vm.data + assert "Skipping reassignment" in caplog.text + else: + assert existing.get_device_vm() is vm + assert vm.data["primary_ip4"] is existing + assert "primary_ip4" in other.unset_items + assert "Reclaiming IP" in caplog.text + + +@pytest.mark.parametrize("owner_status,instance_state,reclaim", [ + ({"value": "offline", "label": "Offline"}, "running", True), + ("offline", "stopped", False), + ("active", "running", False), + ("failed", "running", False), + ("planned", "running", False), +]) +def test_cross_source_reclamation_requires_offline_to_active( + make_source, inventory, vpc_vrf, owner_status, instance_state, reclaim): + data = inventory_data() + data["instances"][0]["State"]["Name"] = instance_state + source = make_source(data=data) + other = inventory.add_object(NBVM, data={ + "id": 99, "name": "other", "status": owner_status, + }, read_from_netbox=True) + other.add_tags([NetBoxHandler.primary_tag, "Source: other"]) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + source.apply() + assert (existing.get_device_vm() is not other) is reclaim + + +@pytest.mark.parametrize("active_first", [False, True]) +def test_active_instance_keeps_shared_ip_regardless_of_discovery_order( + make_source, inventory, vpc_vrf, active_first, caplog): + data = inventory_data() + stopped = copy.deepcopy(data["instances"][0]) + stopped_id, stopped_eni = "i-11111111111111111", "eni-11111111111111111" + stopped.update(InstanceId=stopped_id, State={"Name": "stopped"}, + NetworkInterfaces=[{"NetworkInterfaceId": stopped_eni}], BlockDeviceMappings=[]) + eni = copy.deepcopy(data["NetworkInterfaces"][ENI]) + eni.update(NetworkInterfaceId=stopped_eni, MacAddress="02:00:00:00:00:01") + eni["Attachment"]["InstanceId"] = stopped_id + data["NetworkInterfaces"][stopped_eni] = eni + data["instances"].insert(1 if active_first else 0, stopped) + source = make_source(data=data) + for _ in range(2): + source.apply() + active = next(vm for vm in inventory.get_all_items(NBVM) if vm.data["status"] == "active") + offline = next(vm for vm in inventory.get_all_items(NBVM) if vm.data["status"] == "offline") + ips = inventory.get_all_items(NBIPAddress) + assert len(ips) == 3 + assert all(ip.get_device_vm() is active for ip in ips) + assert active.data["primary_ip4"].data["address"] == "10.0.0.10/24" + assert active.data["primary_ip6"].data["address"] == "2001:db8::10/64" + for field in ("primary_ip4", "primary_ip6"): + assert not offline.data.get(field) or field in offline.unset_items + assert "Skipping reassignment" in caplog.text or "Reclaiming IP" in caplog.text def test_tagging_and_successful_empty_discovery_use_existing_orphan_lifecycle( @@ -567,7 +719,7 @@ def test_profile_is_forwarded_to_sdk(sdk): aws_client.boto3.Session.assert_called_once_with(profile_name="inventory") -def test_sdk_error_after_first_region_aborts_entire_apply(sdk, inventory, load_config, caplog): +def test_sdk_access_denied_after_first_region_continues(sdk, inventory, load_config, caplog): sts, ec2 = sdk identity(sts) stub_region(ec2, inventory_data()) @@ -580,11 +732,10 @@ def test_sdk_error_after_first_region_aborts_entire_apply(sdk, inventory, load_c sync_private_ips = False """) source, = instantiate_sources() - with pytest.raises(SystemExit): - source.apply() - assert inventory.get_all_items(NBVM) == [] - assert inventory.get_all_items(NBCluster) == [] - assert "No NetBox writes or pruning" in caplog.text + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert "UnauthorizedOperation" in caplog.text + assert source.client.access_denied == [(ACCOUNT, "us-west-2", "ec2")] def test_wrong_account_aborts(sdk): @@ -673,7 +824,7 @@ def test_auto_no_enabled_regions_aborts(sdk, regions): AWSClient().discover(["auto"]) -def test_auto_permission_failure_aborts_before_inventory_changes( +def test_auto_permission_failure_logs_and_returns_without_inventory_changes( sdk, inventory, load_config, caplog): sts, ec2 = sdk identity(sts) @@ -687,14 +838,14 @@ def test_auto_permission_failure_aborts_before_inventory_changes( """) source, = instantiate_sources() assert source.settings.regions == ["auto"] - with pytest.raises(SystemExit): - source.apply() + source.apply() assert inventory.get_all_items(NBVM) == [] assert inventory.get_all_items(NBCluster) == [] - assert "No NetBox writes or pruning" in caplog.text + assert "UnauthorizedOperation" in caplog.text + assert source.client.access_denied -def test_auto_later_region_failure_does_not_return_partial_inventory( +def test_auto_later_region_denied_keeps_accessible_inventory( sdk, inventory, load_config): sts, ec2 = sdk identity(sts) @@ -709,10 +860,9 @@ def test_auto_later_region_failure_does_not_return_partial_inventory( sync_private_ips = False """) source, = instantiate_sources() - with pytest.raises(SystemExit): - source.apply() - assert inventory.get_all_items(NBVM) == [] - assert inventory.get_all_items(NBCluster) == [] + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert source.client.access_denied == [(ACCOUNT, "us-west-2", "ec2")] @pytest.mark.parametrize("regions", ["auto, us-east-1", "us-east-1, auto"]) @@ -883,7 +1033,7 @@ def test_organization_can_include_management(org_sdk): @pytest.mark.parametrize("failure", ["assume", "scan", "identity"]) -def test_organization_later_account_failure_prevents_all_mutations( +def test_organization_later_account_denied_continues_but_identity_failure_aborts( org_sdk, inventory, load_config, failure, caplog): organization_accounts(org_sdk) first = assume_member(org_sdk, ACCOUNT) @@ -905,24 +1055,31 @@ def test_organization_later_account_failure_prevents_all_mutations( source = organization_source(load_config) sync = Mock() source.sync_snapshot = sync - with pytest.raises(SystemExit): + if failure == "identity": + with pytest.raises(SystemExit): + source.apply() + sync.assert_not_called() + assert "No NetBox writes or pruning" in caplog.text + else: source.apply() - sync.assert_not_called() + assert sync.called + assert source.client.access_denied assert inventory.get_all_items(NBVM) == [] assert inventory.get_all_items(NBCluster) == [] assert SECOND_ACCOUNT in caplog.text - assert "No NetBox writes or pruning" in caplog.text -def test_organization_listing_failure_aborts(org_sdk): +def test_organization_listing_denied_returns_non_authoritative_empty_result(org_sdk, caplog): identity(org_sdk.central.sts, MANAGEMENT) org_sdk.central.organizations.add_response("describe_organization", { "Organization": {"MasterAccountId": MANAGEMENT}, }, {}) org_sdk.central.organizations.add_client_error( "list_accounts", service_error_code="AccessDeniedException", http_status_code=403, expected_params={}) - with pytest.raises(aws_client.AWSDiscoveryError, match="AccessDeniedException"): - AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + client = AWSClient() + assert client.discover_organization([REGION], "NetBoxInventoryReadOnly") == {} + assert client.access_denied + assert "AccessDeniedException" in caplog.text @pytest.mark.parametrize("selection", [[MANAGEMENT], ["999999999999"], ["345678901234"]]) From 65d6870a74ce5e0b97c67b7903d236676893cda1 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:19:25 -0500 Subject: [PATCH 17/28] Add tests for AWS services integration --- tests/test_aws_services.py | 660 +++++++++++++++++++++++++++++++++++++ 1 file changed, 660 insertions(+) create mode 100644 tests/test_aws_services.py diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py new file mode 100644 index 00000000..03b71281 --- /dev/null +++ b/tests/test_aws_services.py @@ -0,0 +1,660 @@ +import copy +from contextlib import ExitStack +from unittest.mock import Mock + +import pytest + +boto3 = pytest.importorskip("boto3") +from botocore.stub import Stubber + +from module.netbox import NBCustomField, NBCluster, NBIPAddress, NBTag, NBVirtualDisk, NBVM, NBVMInterface +from module.sources import instantiate_sources +from module.sources.aws import client as aws_client +from module.sources.aws.client import AWSClient, AWSDiscoveryError +from test_aws_source import ( + ACCOUNT, INSTANCE, REGION, SECOND_ACCOUNT, VPC, assume_member, identity, inventory_data, + make_source, org_sdk, organization_accounts, stub_region, windows_config_paths, +) +from test_orphaned_object_status import make_netbox_handler + +FUNCTION_ARN = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:inventory-function" +CLUSTER_ARN = f"arn:aws:eks:{REGION}:{ACCOUNT}:cluster/inventory-cluster" + + +def service_data(): + data = inventory_data() + data.update( + Buckets=[{ + "Name": "inventory-bucket", "Region": REGION, "Versioning": "Enabled", + "Encryption": {"Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}]}, + "Tags": [{"Key": "Environment", "Value": "test"}], + }], + Functions=[{ + "FunctionArn": FUNCTION_ARN, "FunctionName": "inventory-function", "MemorySize": 128, + "State": "Active", "Runtime": "python3.12", "Timeout": 30, "Handler": "app.handler", + "LastModified": "2026-10-27T00:00:00.000+0000", "Architectures": ["arm64"], + "Tags": {"Environment": "test"}, + }], + EKSClusters=[{ + "arn": CLUSTER_ARN, "name": "inventory-cluster", "version": "1.34", "status": "ACTIVE", + "resourcesVpcConfig": {"vpcId": VPC}, "endpoint": "https://example.invalid", + "nodegroups": [{ + "nodegroupName": "workers", "status": "ACTIVE", + "resources": {"autoScalingGroups": [{"name": "eks-workers"}]}, + }], + "fargateProfiles": [{"fargateProfileName": "serverless", "status": "ACTIVE"}], + "tags": {"Environment": "test"}, + }], + ) + data["instances"][0]["Tags"].append({"Key": "aws:autoscaling:groupName", "Value": "eks-workers"}) + return data + + +@pytest.mark.parametrize("version", ["3.7.0", "4.1.0", "4.3.0"]) +def test_services_import_and_repeat(make_source, inventory, version): + inventory.netbox_api_version = version + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + source.apply() + vms = inventory.get_all_items(NBVM) + assert len(vms) == 3 + worker = next(vm for vm in vms if vm.data["custom_fields"].get("aws_instance_id") == INSTANCE) + function = next(vm for vm in vms if vm.data["custom_fields"].get("aws_service") == "lambda") + holder = next(vm for vm in vms if vm.data["custom_fields"].get("aws_service") == "s3") + assert worker.data["cluster"].data["type"].data["name"] == "AWS EKS" + assert function.data["cluster"].data["type"].data["name"] == "AWS Lambda" + assert function.data["memory"] == 128 + assert function.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN + assert "vcpus" not in function.data and "vcpus" not in holder.data + assert "memory" not in holder.data + assert function.data["status"] == "active" + assert "AWS:logical-lambda" in function.get_tags() + assert "AWS:Environment=test" in function.get_tags() + assert not inventory.get_all_interfaces(function) + assert not inventory.get_all_interfaces(holder) + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + assert bucket.data["virtual_machine"] is holder + assert bucket.data["size"] == 0 + assert bucket.data["custom_fields"]["aws_metadata"]["usage_bytes"] is None + assert bucket.data["custom_fields"]["aws_metadata"]["usage_status"] == "unknown" + assert bucket.data["custom_fields"]["aws_metadata"]["Versioning"] == "Enabled" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + metadata = worker.data["cluster"].data["custom_fields"]["aws_metadata"] + assert metadata["nodegroups"][0]["nodegroupName"] == "workers" + assert metadata["fargateProfiles"][0]["fargateProfileName"] == "serverless" + source.client.discover.assert_called_with( + [REGION], None, None, services=("s3", "lambda", "eks")) + field = inventory.get_by_data(NBCustomField, data={"name": "aws_metadata"}) + assert field.data.get("object_types", field.data.get("content_types")) == [ + NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type, + ] + + +def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): + inventory.netbox_api_version = "3.6.0" + source = make_source("sync_s3 = True") + with pytest.raises(SystemExit): + source.apply() + assert "requires NetBox 3.7" in caplog.text + source.client.discover.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + + +@pytest.mark.parametrize("key,setting", [ + ("Buckets", "sync_s3"), ("Functions", "sync_lambda"), ("EKSClusters", "sync_eks"), +]) +def test_enabled_service_requires_complete_snapshot(make_source, inventory, key, setting, caplog): + source = make_source(f"{setting} = True") + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + assert f"Missing AWS {key} discovery" in caplog.text + + +def test_disabled_service_objects_are_retained(make_source, inventory): + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=service_data()) + source.apply() + managed = [] + for kind in (NBVM, NBVirtualDisk, NBCluster): + for item in inventory.get_all_items(kind): + if item.data.get("custom_fields", {}).get("aws_service"): + item.add_tags(source.source_tag) + item.source = None + managed.append(item) + source.settings.sync_s3 = False + source.settings.sync_lambda = False + source.settings.sync_eks = False + source.apply() + assert managed and all(item.source is source for item in managed) + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + assert all(handler.orphaned_tag not in item.get_tags() for item in managed) + worker = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_instance_id") == INSTANCE) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EC2" + + +def test_absent_enabled_resources_are_not_retained(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True", data=data) + source.apply() + resources = [item for kind in (NBVM, NBVirtualDisk) for item in inventory.get_all_items(kind) + if item.data.get("custom_fields", {}).get("aws_service")] + for item in resources: + item.add_tags([source.source_tag, make_netbox_handler().primary_tag]) + item.source = None + data["Buckets"] = [] + data["Functions"] = [] + source.apply() + assert resources and all(item.source is None for item in resources) + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + assert all(handler.orphaned_tag in item.get_tags() for item in resources) + + +@pytest.mark.parametrize("state,status", [ + ("Pending", "planned"), ("Inactive", "offline"), ("Failed", "failed"), +]) +def test_lambda_states(make_source, inventory, state, status): + data = service_data() + data["Functions"][0]["State"] = state + make_source("sync_lambda = True", data=data).apply() + function = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda") + assert function.data["status"] == status + + +def test_long_names_and_same_lambda_names_across_regions(make_source, inventory): + data = service_data() + function = data["Functions"][0] + function["FunctionName"] = "x" * 64 + function["FunctionArn"] = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:{function['FunctionName']}" + other = copy.deepcopy(data) + other["instances"] = [] + other["Functions"][0]["FunctionArn"] = function["FunctionArn"].replace(REGION, "us-west-2") + source = make_source("sync_lambda = True", data=data) + source.client.discover.return_value = (ACCOUNT, {REGION: data, "us-west-2": other}) + source.apply() + source.apply() + functions = [vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda"] + assert len(functions) == 2 + assert all(len(vm.data["name"]) <= 64 for vm in functions) + assert len({vm.data["name"] for vm in functions}) == 2 + + +def test_services_do_not_overwrite_unmanaged_objects(make_source, inventory): + data = service_data() + source = make_source("sync_lambda = True", data=data) + source.apply() + function = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda") + function.source = None + with pytest.raises(SystemExit): + source.apply() + + +@pytest.mark.parametrize("failure", ["arn", "membership", "tag", "state"]) +def test_invalid_service_metadata_aborts_before_sync(make_source, inventory, failure): + data = service_data() + if failure == "arn": + data["Functions"][0]["FunctionArn"] = FUNCTION_ARN.replace(ACCOUNT, SECOND_ACCOUNT) + elif failure == "membership": + data["instances"][0]["Tags"].append({"Key": "eks:cluster-name", "Value": "different"}) + elif failure == "tag": + data["Functions"][0]["Tags"]["Environment"] = "x" * 101 + else: + data["Functions"][0]["State"] = "Unknown" + source = make_source("sync_lambda = True\nsync_eks = True", data=data) + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + + +@pytest.fixture +def service_sdk(monkeypatch): + session = boto3.Session( + aws_access_key_id="test-only", aws_secret_access_key="test-only", region_name=REGION) + clients = {service: session.client(service) for service in ("sts", "ec2", "s3", "lambda", "eks")} + fake_session = Mock(region_name=REGION) + fake_session.client.side_effect = lambda service, **kwargs: clients[service] + monkeypatch.setattr(aws_client.boto3, "Session", Mock(return_value=fake_session)) + with ExitStack() as stack: + stubs = {service: stack.enter_context(Stubber(client)) for service, client in clients.items()} + yield stubs + for stub in stubs.values(): + stub.assert_no_pending_responses() + + +def stub_s3(stub, present=True): + stub.add_response("list_buckets", { + "Buckets": [{"Name": "inventory-bucket", "BucketRegion": REGION}], + "ContinuationToken": "next", + }, {"BucketRegion": REGION}) + params = {"Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT} + if present: + stub.add_response("get_bucket_tagging", { + "TagSet": [{"Key": "Environment", "Value": "test"}], + }, params) + else: + stub.add_client_error("get_bucket_tagging", service_error_code="NoSuchTagSet", expected_params=params) + stub.add_response("get_bucket_versioning", {"Status": "Enabled"}, params) + stub.add_response("get_bucket_encryption", { + "ServerSideEncryptionConfiguration": { + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}], + }, + }, params) + stub.add_response("list_buckets", {"Buckets": []}, {"BucketRegion": REGION, "ContinuationToken": "next"}) + + +def stub_lambda(stub): + config = service_data()["Functions"][0] + stub.add_response("list_functions", { + "Functions": [{"FunctionName": config["FunctionName"], "FunctionArn": FUNCTION_ARN}], + "NextMarker": "next", + }, {}) + stub.add_response("get_function_configuration", { + **{key: value for key, value in config.items() if key != "Tags"}, + "Environment": {"Variables": {"DO_NOT_IMPORT": "sensitive"}}, + }, {"FunctionName": FUNCTION_ARN}) + stub.add_response("list_tags", {"Tags": config["Tags"]}, {"Resource": FUNCTION_ARN}) + stub.add_response("list_functions", {"Functions": []}, {"Marker": "next"}) + + +def stub_eks(stub): + cluster = service_data()["EKSClusters"][0] + name = cluster["name"] + stub.add_response("list_clusters", {"clusters": [name], "nextToken": "next"}, {}) + stub.add_response("describe_cluster", { + "cluster": {key: value for key, value in cluster.items() if key not in ("nodegroups", "fargateProfiles")}, + }, {"name": name}) + stub.add_response("list_nodegroups", {"nodegroups": ["workers"], "nextToken": "next"}, {"clusterName": name}) + stub.add_response("describe_nodegroup", { + "nodegroup": {**cluster["nodegroups"][0], "clusterName": name}, + }, {"clusterName": name, "nodegroupName": "workers"}) + stub.add_response("list_nodegroups", {"nodegroups": []}, {"clusterName": name, "nextToken": "next"}) + stub.add_response("list_fargate_profiles", { + "fargateProfileNames": ["serverless"], "nextToken": "next", + }, {"clusterName": name}) + stub.add_response("describe_fargate_profile", { + "fargateProfile": {**cluster["fargateProfiles"][0], "clusterName": name}, + }, {"clusterName": name, "fargateProfileName": "serverless"}) + stub.add_response("list_fargate_profiles", {"fargateProfileNames": []}, {"clusterName": name, "nextToken": "next"}) + stub.add_response("list_clusters", {"clusters": []}, {"nextToken": "next"}) + + +def test_sdk_services_to_real_inventory(service_sdk, inventory, load_config): + identity(service_sdk["sts"]) + stub_region(service_sdk["ec2"], service_data()) + stub_s3(service_sdk["s3"]) + stub_lambda(service_sdk["lambda"]) + stub_eks(service_sdk["eks"]) + load_config(f""" +[source/services] +type = aws +regions = {REGION} +sync_s3 = True +sync_lambda = True +sync_eks = True +sync_private_ips = False +""") + source, = instantiate_sources() + source.apply() + assert len(inventory.get_all_items(NBVM)) == 3 + function = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda") + assert "Environment" not in function.data["custom_fields"]["aws_metadata"] + assert "sensitive" not in str(function.data) + + +@pytest.mark.parametrize("service,operation", [ + ("s3", "list_buckets"), ("lambda", "list_functions"), ("eks", "list_clusters"), +]) +def test_service_access_denied_continues(service_sdk, inventory, load_config, service, operation, caplog): + identity(service_sdk["sts"]) + stub_region(service_sdk["ec2"], inventory_data()) + service_sdk[service].add_client_error(operation, service_error_code="AccessDenied", http_status_code=403) + load_config(f""" +[source/services] +type = aws +regions = {REGION} +sync_{service} = True +""") + source, = instantiate_sources() + source.apply() + assert "AccessDenied" in caplog.text and "Retaining all previously managed source objects" in caplog.text + assert len(inventory.get_all_items(NBVM)) == 1 + assert source.client.access_denied == [(ACCOUNT, REGION, service)] + + +def test_optional_services_propagate_to_member_accounts(org_sdk, monkeypatch): + organization_accounts(org_sdk) + for account in (ACCOUNT, SECOND_ACCOUNT): + assume_member(org_sdk, account) + discovery = Mock(side_effect=lambda regions, account, discovery_region, **kwargs: (account, {})) + monkeypatch.setattr(AWSClient, "discover", discovery) + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly", services=("s3", "lambda", "eks")) + assert discovery.call_count == 2 + for call in discovery.call_args_list: + assert call.kwargs == {"services": ("s3", "lambda", "eks")} + + +def test_service_defaults_preserve_ec2_only(make_source): + source = make_source() + assert not any((source.settings.sync_s3, source.settings.sync_lambda, source.settings.sync_eks)) + source.apply() + source.client.discover.assert_called_once_with([REGION], None, None) + + +def test_incompatible_metadata_custom_field_aborts(make_source, inventory, caplog): + inventory.add_object(NBCustomField, data={ + "id": 1, "name": "aws_metadata", "type": "text", "object_types": [NBVM.object_type], + }, read_from_netbox=True) + source = make_source("sync_s3 = True", data=service_data()) + with pytest.raises(SystemExit): + source.apply() + assert "must have type 'json'" in caplog.text + assert inventory.get_all_items(NBVM) == [] + + +def test_s3_holders_are_per_account_and_region(make_source, inventory, monkeypatch): + data = service_data() + data["instances"] = [] + second = copy.deepcopy(data) + second["Buckets"][0]["Name"] = "second-account-bucket" + other_region = copy.deepcopy(data) + other_region["Buckets"][0].update(Name="other-region-bucket", Region="us-west-2") + source = make_source("organization = True\nsync_s3 = True") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: data, "us-west-2": other_region}, + SECOND_ACCOUNT: {REGION: second}, + })) + source.apply() + assert len(inventory.get_all_items(NBVM)) == 3 + assert len(inventory.get_all_items(NBVirtualDisk)) == 3 + assert {vm.data["name"] for vm in inventory.get_all_items(NBVM)} == { + f"S3 inventory {ACCOUNT}/{REGION}", f"S3 inventory {ACCOUNT}/us-west-2", + f"S3 inventory {SECOND_ACCOUNT}/{REGION}", + } + + +@pytest.mark.parametrize("tag_name", ["AWS:Environment=Dev", "aws:environment=Dev"]) +def test_service_tags_use_resolved_identity(make_source, inventory, tag_name): + tag = inventory.add_object(NBTag, data={"name": tag_name}) + data = service_data() + data["Buckets"][0]["Tags"] = [{"Key": "Environment", "Value": "dev"}] + data["Functions"][0]["Tags"] = {"Environment": "dev"} + data["EKSClusters"][0]["tags"] = {"Environment": "dev"} + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + for _ in range(2): + source.apply() + tagged = [item for kind in (NBVM, NBVirtualDisk, NBCluster) + for item in inventory.get_all_items(kind) + if item.data.get("custom_fields", {}).get("aws_resource_id") in ( + FUNCTION_ARN, CLUSTER_ARN, f"s3/{ACCOUNT}/inventory-bucket")] + assert len(tagged) == 3 + assert all(tag in item.data["tags"] and tag_name in item.get_tags() for item in tagged) + for item in tagged: + item.add_tags("user-managed") + data["Buckets"][0]["Tags"] = [] + data["Functions"][0]["Tags"] = {} + data["EKSClusters"][0]["tags"] = {} + source.apply() + assert all(tag_name not in item.get_tags() and "user-managed" in item.get_tags() for item in tagged) + + +def test_optional_resource_tags_reconcile_without_losing_user_tags(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + resources = [item for kind in (NBVM, NBVirtualDisk, NBCluster) + for item in inventory.get_all_items(kind) + if item.data.get("custom_fields", {}).get("aws_service") and + "AWS:Environment=test" in item.get_tags()] + for item in resources: + item.add_tags("user-managed") + data["Buckets"][0]["Tags"] = [] + data["Functions"][0]["Tags"] = {} + data["EKSClusters"][0]["tags"] = {} + source.apply() + assert len(resources) == 3 + assert all("AWS:Environment=test" not in item.get_tags() and "user-managed" in item.get_tags() + for item in resources) + + +@pytest.mark.parametrize("key", [ + "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", +]) +def test_eks_worker_tag_membership(make_source, inventory, key): + data = service_data() + data["instances"][0]["Tags"] = [{"Key": key, "Value": "inventory-cluster"}] + source = make_source("sync_eks = True", data=data) + source.apply() + worker, = inventory.get_all_items(NBVM) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EKS" + + +def test_enabling_eks_moves_existing_worker_without_duplication(make_source, inventory): + source = make_source(data=service_data()) + source.apply() + worker, = inventory.get_all_items(NBVM) + disks = list(worker.get_virtual_disks()) + interfaces = list(inventory.get_all_items(NBVMInterface)) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EC2" + source.settings.sync_eks = True + source.apply() + same_worker, = inventory.get_all_items(NBVM) + assert same_worker is worker + assert same_worker.data["cluster"].data["type"].data["name"] == "AWS EKS" + assert worker.get_virtual_disks() == disks + assert inventory.get_all_items(NBVMInterface) == interfaces + + +def test_unknown_eks_cluster_tag_is_explicit(make_source, inventory, caplog): + data = service_data() + data["instances"][0]["Tags"] = [{"Key": "eks:cluster-name", "Value": "deleted-cluster"}] + source = make_source("sync_eks = True", data=data) + source.apply() + worker, = inventory.get_all_items(NBVM) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EC2" + assert "references undiscovered EKS cluster 'deleted-cluster'" in caplog.text + + +@pytest.mark.parametrize("missing", ["tags", "encryption", "versioning"]) +def test_s3_absent_optional_configuration_is_explicit(service_sdk, missing): + stub = service_sdk["s3"] + stub.add_response("list_buckets", {"Buckets": [{"Name": "inventory-bucket"}]}, {"BucketRegion": REGION}) + params = {"Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT} + stub.add_response("get_bucket_location", {"LocationConstraint": ""}, params) + if missing == "tags": + stub.add_client_error("get_bucket_tagging", service_error_code="NoSuchTagSet", expected_params=params) + else: + stub.add_response("get_bucket_tagging", {"TagSet": []}, params) + stub.add_response("get_bucket_versioning", {} if missing == "versioning" else {"Status": "Suspended"}, params) + if missing == "encryption": + stub.add_client_error( + "get_bucket_encryption", service_error_code="ServerSideEncryptionConfigurationNotFoundError", + expected_params=params) + else: + stub.add_response("get_bucket_encryption", { + "ServerSideEncryptionConfiguration": { + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}], + }, + }, params) + buckets = AWSClient().discover_s3(stub.client, REGION, ACCOUNT) + bucket, = buckets + assert bucket["Arn"] == "arn:aws:s3:::inventory-bucket" + if missing == "tags": + assert bucket["Tags"] == [] + elif missing == "encryption": + assert bucket["Encryption"] == {} + else: + assert bucket["Versioning"] == "Unversioned" + + +def test_s3_wrong_region_fails_closed(service_sdk): + service_sdk["s3"].add_response("list_buckets", { + "Buckets": [{"Name": "inventory-bucket", "BucketRegion": "us-west-2"}], + }, {"BucketRegion": REGION}) + with pytest.raises(AWSDiscoveryError, match="belongs to 'us-west-2'"): + AWSClient().discover_s3(service_sdk["s3"].client, REGION, ACCOUNT) + + +def test_s3_metadata_access_denied_skips_bucket_not_empty_configuration(service_sdk, caplog): + stub = service_sdk["s3"] + stub.add_response("list_buckets", { + "Buckets": [{"Name": "inventory-bucket", "BucketRegion": REGION}], + }, {"BucketRegion": REGION}) + stub.add_client_error("get_bucket_tagging", service_error_code="AccessDenied", expected_params={ + "Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT, + }) + client = AWSClient() + assert client.discover_s3(stub.client, REGION, ACCOUNT) == [] + assert client.access_denied == [(ACCOUNT, REGION, "s3")] + assert "AccessDenied" in caplog.text + + +@pytest.mark.parametrize("operation", [ + "get_bucket_tagging", "get_bucket_versioning", "get_bucket_encryption", +]) +def test_s3_denied_bucket_does_not_stop_next_bucket(service_sdk, operation, caplog): + stub = service_sdk["s3"] + stub.add_response("list_buckets", {"Buckets": [ + {"Name": "denied-bucket", "BucketRegion": REGION}, + {"Name": "accessible-bucket", "BucketRegion": REGION}, + ]}, {"BucketRegion": REGION}) + for name in ("denied-bucket", "accessible-bucket"): + params = {"Bucket": name, "ExpectedBucketOwner": ACCOUNT} + for request, response in ( + ("get_bucket_tagging", {"TagSet": []}), + ("get_bucket_versioning", {}), + ("get_bucket_encryption", {"ServerSideEncryptionConfiguration": { + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}], + }}), + ): + if name == "denied-bucket" and request == operation: + stub.add_client_error(request, service_error_code="AccessDenied", expected_params=params) + break + stub.add_response(request, response, params) + client = AWSClient() + buckets = client.discover_s3(stub.client, REGION, ACCOUNT) + assert [bucket["Name"] for bucket in buckets] == ["accessible-bucket"] + assert client.access_denied == [(ACCOUNT, REGION, "s3")] + assert operation.replace("_", "").lower() in caplog.text.replace("_", "").lower() + + +def test_access_denied_retains_objects_and_preserves_eks_placement(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + objects = [item for kind in (NBVM, NBVirtualDisk, NBVMInterface, NBIPAddress) + for item in inventory.get_all_items(kind)] + worker = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_instance_id") == INSTANCE) + cluster = worker.data["cluster"] + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + bucket_metadata = copy.deepcopy(bucket.data["custom_fields"]["aws_metadata"]) + for item in objects: + item.source = None + data["Buckets"] = [] + data.pop("Functions") + data.pop("EKSClusters") + data["InaccessibleServices"] = ["lambda", "eks"] + data["instances"][0]["Tags"][0]["Value"] = "updated-worker" + source.client.access_denied = [(ACCOUNT, REGION, "s3"), (ACCOUNT, REGION, "lambda"), (ACCOUNT, REGION, "eks")] + source.apply() + inventory.tag_all_the_things(handler) + assert all(item.source is source and handler.orphaned_tag not in item.get_tags() for item in objects) + assert bucket.data["custom_fields"]["aws_metadata"] == bucket_metadata + assert "AWS:Environment=test" in bucket.get_tags() + assert worker.data["cluster"] is cluster + assert worker.data["name"] == "updated-worker" + + +def test_denied_organization_listing_retains_existing_objects(make_source, inventory): + source = make_source(data=service_data()) + source.apply() + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + vm, = inventory.get_all_items(NBVM) + vm.source = None + source.settings.organization = True + source.client.discover_organization = Mock(return_value={}) + source.client.access_denied = [(None, None, "organizations")] + source.apply() + inventory.tag_all_the_things(handler) + assert vm.source is source and handler.orphaned_tag not in vm.get_tags() + + +@pytest.mark.parametrize("code", ["ExpiredToken", "ThrottlingException", "InternalError"]) +def test_non_authorization_errors_remain_fatal(service_sdk, code): + identity(service_sdk["sts"]) + service_sdk["ec2"].add_client_error("describe_instances", service_error_code=code, expected_params={}) + with pytest.raises(AWSDiscoveryError, match=code): + AWSClient().discover([REGION]) + + +def test_denied_shared_network_owner_defers_dependent_instances(make_source, inventory): + data = inventory_data() + data["Vpcs"].clear() + data["Subnets"].clear() + source = make_source("organization = True") + source.client.discover_organization = Mock(return_value={ACCOUNT: {REGION: data}}) + source.client.access_denied = [(SECOND_ACCOUNT, REGION, "ec2")] + source.apply() + assert inventory.get_all_items(NBVM) == [] + + +def test_denied_identity_without_expected_account_is_nonfatal(service_sdk, inventory, load_config, caplog): + service_sdk["sts"].add_client_error("get_caller_identity", service_error_code="AccessDenied", expected_params={}) + load_config(f""" +[source/services] +type = aws +regions = {REGION} +""") + source, = instantiate_sources() + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert "AccessDenied" in caplog.text + assert source.client.access_denied == [(None, None, "account")] + + +def test_authorized_run_resumes_orphaning_after_denial(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True", data=data) + source.apply() + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + data["Buckets"] = [] + bucket.source = None + source.client.access_denied = [(ACCOUNT, REGION, "s3")] + source.apply() + inventory.tag_all_the_things(handler) + assert handler.orphaned_tag not in bucket.get_tags() + bucket.source = None + source.client.access_denied = [] + source.apply() + inventory.tag_all_the_things(handler) + assert handler.orphaned_tag in bucket.get_tags() + + +def test_ec2_denial_still_discovers_optional_services(service_sdk): + identity(service_sdk["sts"]) + service_sdk["ec2"].add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + expected_params={}) + stub_s3(service_sdk["s3"]) + client = AWSClient() + account, snapshots = client.discover([REGION], services=("s3",)) + assert account == ACCOUNT + assert snapshots[REGION]["EC2AccessDenied"] is True + assert snapshots[REGION]["Buckets"][0]["Name"] == "inventory-bucket" + assert client.access_denied == [(ACCOUNT, REGION, "ec2")] From e4ba0044736ff59b0081c8604dbfbe50c19bb43e Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:32:07 -0500 Subject: [PATCH 18/28] Enhance AWS resource validation and syncing Added validation for AWS services and enhanced error handling for S3, Lambda, and EKS resources. Updated methods for syncing AWS resources with NetBox, including improved handling of tags and resource identification. --- module/sources/aws/connection.py | 311 ++++++++++++++++++++++++++++++- 1 file changed, 302 insertions(+), 9 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index df3e33ef..e3cebe1f 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -1,3 +1,4 @@ +from hashlib import sha256 from ipaddress import ip_address, ip_interface, ip_network from packaging.version import Version @@ -10,6 +11,7 @@ NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, ) from module.netbox.inventory import NetBoxInventory +from module.netbox.connection import NetBoxHandler from module.sources.aws.client import AWSClient, AWSDiscoveryError from module.sources.aws.config import AWSConfig from module.sources.common.source_base import SourceBase @@ -50,6 +52,11 @@ def abort(self, error): def apply(self): try: + if self.settings.sync_s3 and Version(self.inventory.netbox_api_version) < Version("3.7"): + raise ValueError("S3 bucket inventory requires NetBox 3.7 or newer") + services = tuple(service for service in ("s3", "lambda", "eks") + if getattr(self.settings, f"sync_{service}")) + options = {"services": services} if services else {} if self.settings.organization: snapshots = self.client.discover_organization( self.settings.regions, self.settings.organization_role_name, @@ -57,18 +64,23 @@ def apply(self): discovery_region=self.settings.discovery_region, include_management=self.settings.organization_include_management, external_id=self.settings.organization_external_id, - account_ids=self.settings.organization_account_ids) + account_ids=self.settings.organization_account_ids, **options) account_names = getattr(self.client, "account_names", {}) else: account, snapshot = self.client.discover( - self.settings.regions, self.settings.account_id, self.settings.discovery_region) - snapshots = {account: snapshot} + self.settings.regions, self.settings.account_id, self.settings.discovery_region, **options) + snapshots = {account: snapshot} if account is not None else {} account_names = {} if self.settings.organization and self.settings.sync_private_ips: self.resolve_shared_networks(snapshots) + self.defer_inaccessible_network_instances(snapshots) vrfs = {} for account, snapshot in snapshots.items(): + self.validate_services(account, snapshot) self.validate_snapshot(account, snapshot, vrfs) + self.retain_disabled_services() + if getattr(self.client, "access_denied", []): + self.retain_inaccessible_objects() account_labels = { account: account_names.get(account) or self.settings.account_name or account for account in snapshots @@ -85,6 +97,219 @@ def apply(self): except (AWSDiscoveryError, ValueError, KeyError) as exc: self.abort(exc) + def validate_services(self, account, snapshot): + for region, data in snapshot.items(): + inaccessible = data.get("InaccessibleServices", []) + for enabled, key in ( + (self.settings.sync_s3 and "s3" not in inaccessible, "Buckets"), + (self.settings.sync_lambda and "lambda" not in inaccessible, "Functions"), + (self.settings.sync_eks and "eks" not in inaccessible, "EKSClusters"), + ): + if enabled and not isinstance(data.get(key), list): + raise ValueError(f"Missing AWS {key} discovery (account '{account}', region '{region}')") + if enabled: + identity_key = {"Buckets": "Name", "Functions": "FunctionArn", "EKSClusters": "arn"}[key] + identities = [resource[identity_key] for resource in data[key]] + if len(set(identities)) != len(identities): + raise ValueError(f"Duplicate AWS {key} discovery (account '{account}', region '{region}')") + for bucket in data["Buckets"] if self.settings.sync_s3 and "s3" not in inaccessible else []: + if bucket["Region"] != region or not bucket["Name"] or len(bucket["Name"]) > 63: + raise ValueError(f"Invalid S3 bucket '{bucket['Name']}' for region '{region}'") + self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}) + if "Versioning" not in bucket or "Encryption" not in bucket: + raise ValueError(f"Incomplete S3 bucket metadata for '{bucket['Name']}'") + for function in data["Functions"] if self.settings.sync_lambda and "lambda" not in inaccessible else []: + self.validate_arn(function["FunctionArn"], "lambda", account, region) + if function["State"] not in ("Pending", "Active", "Inactive", "Failed"): + raise ValueError(f"Unknown Lambda state '{function['State']}' for '{function['FunctionArn']}'") + if not function["FunctionName"] or not isinstance(function["MemorySize"], int): + raise ValueError(f"Incomplete Lambda configuration for '{function['FunctionArn']}'") + self.service_tags(function["Tags"]) + if self.settings.sync_eks and "eks" not in inaccessible: + clusters = {cluster["name"]: cluster for cluster in data["EKSClusters"]} + if len(clusters) != len(data["EKSClusters"]): + raise ValueError(f"Duplicate EKS cluster discovery in account '{account}', region '{region}'") + asg_clusters = {} + for cluster in data["EKSClusters"]: + self.validate_arn(cluster["arn"], "eks", account, region) + for field in ("status", "version", "resourcesVpcConfig", "nodegroups", "fargateProfiles"): + if field not in cluster: + raise ValueError(f"Incomplete EKS {field} for '{cluster['arn']}'") + self.service_tags(cluster.get("tags", {})) + for group in cluster["nodegroups"]: + for asg in group.get("resources", {}).get("autoScalingGroups", []): + name = asg["name"] + if name in asg_clusters and asg_clusters[name] != cluster["name"]: + raise ValueError(f"EKS Auto Scaling group '{name}' belongs to multiple clusters") + asg_clusters[name] = cluster["name"] + data["EKSWorkers"] = {} + for instance in data["instances"]: + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + names = {tags[key] for key in ( + "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", + ) if tags.get(key)} + asg_cluster = asg_clusters.get(tags.get("aws:autoscaling:groupName")) + if asg_cluster: + names.add(asg_cluster) + if len(names) > 1: + raise ValueError(f"Conflicting EKS cluster membership for instance '{instance['InstanceId']}'") + if names: + name = next(iter(names)) + if name in clusters: + data["EKSWorkers"][instance["InstanceId"]] = name + else: + log.warning( + f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " + f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") + + def validate_arn(self, arn, service, account, region): + parts = arn.split(":", 5) + if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: + raise ValueError(f"Invalid {service} resource ARN '{arn}' for account '{account}', region '{region}'") + + def service_tags(self, tags): + result = [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags] + if any(len(tag) > 100 for tag in result): + raise ValueError("AWS service tag exceeds the NetBox tag name limit (100 characters)") + return result + + def reconcile_tags(self, item, tags): + # Tag lookup can reuse an existing name with different capitalization. + desired_tags = {self.inventory.get_by_data(NBTag, data={"name": name}) for name in tags} + managed_prefixes = tuple(f"AWS:{key}=".casefold() for key in self.settings.tag_keys) + for tag in item.data.get("tags", []): + if tag.get_display_name().casefold().startswith(managed_prefixes) and tag not in desired_tags: + item.remove_tags(tag) + + def retain_disabled_services(self): + for object_type in (NBVM, NBVirtualDisk, NBCluster): + for item in self.inventory.get_all_items(object_type): + service = (item.data.get("custom_fields") or {}).get("aws_service") + if service in ("s3", "lambda", "eks") and not getattr(self.settings, f"sync_{service}") and \ + (item.source is self or self.source_tag in item.get_tags()): + item.set_source(self) + + def retain_inaccessible_objects(self): + # Source-wide retention also protects children and cross-account shared-network dependencies. + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if self.source_tag in item.get_tags(): + item.set_source(self) + log.warning( + f"AWS source '{self.name}': authorization-denied discovery is incomplete. " + "Retaining all previously managed source objects this run; accessible resources will still be updated.") + + def defer_inaccessible_network_instances(self, snapshots): + if not any(service in ("ec2", "account", "sts", "organizations") + for _, _, service in getattr(self.client, "access_denied", [])): + return + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + retained = [] + for instance in data["instances"]: + missing_network = False + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is not None and (eni["VpcId"] not in data["Vpcs"] or + eni["SubnetId"] not in data["Subnets"]): + missing_network = True + if missing_network: + log.warning( + f"AWS instance '{instance['InstanceId']}' (account '{account}', region '{region}') " + "has missing network dependencies during authorization-denied discovery. " + "Deferring its update and retaining existing objects.") + else: + retained.append(instance) + data["instances"] = retained + + def resource_name(self, name, identity, limit): + if len(name) <= limit: + return name + suffix = sha256(identity.encode("utf-8")).hexdigest()[:12] + return f"{name[:limit - 13]}-{suffix}" + + def service_object(self, object_type, identity, data): + matches = [item for item in self.inventory.get_all_items(object_type) + if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + if len(matches) > 1: + raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") + if matches: + item = matches[0] + if item.source is not self and self.source_tag not in item.get_tags(): + raise ValueError(f"AWS resource '{identity}' belongs to another source") + item.update(data=data, source=self) + else: + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" + if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) + for item in self.inventory.get_all_items(object_type)): + raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") + item = self.inventory.add_object(object_type, data=data, source=self) + self.reconcile_tags(item, data["tags"]) + return item + + def service_cluster(self, account, region, site, group, service, identity, name, + metadata=None, tags=None): + data = { + "name": self.resource_name(name, identity, 100), "group": group, + "type": {"name": f"AWS {service.upper() if service != 'lambda' else 'Lambda'}"}, + "custom_fields": {"aws_service": service, "aws_resource_id": identity, + "aws_metadata": metadata or {"account": account, "region": region}}, + "tags": self.service_tags(tags or {}), + } + if Version(self.inventory.netbox_api_version) >= Version("4.2"): + data.update(scope_type="dcim.site", scope_id=site) + else: + data["site"] = site + return self.service_object(NBCluster, identity, data) + + def logical_vm(self, account, region, site, cluster, service, identity, name, metadata, tags, **attributes): + data = { + "name": self.resource_name(name, identity, 64), "cluster": cluster, "status": "active", + "comments": f"Logical AWS {service} inventory object; not an EC2 instance.", + "custom_fields": { + "aws_account_id": account, "aws_region": region, "aws_service": service, + "aws_resource_id": identity, "aws_metadata": metadata, + }, + "tags": [f"AWS:logical-{service}", *self.service_tags(tags)], + **attributes, + } + if Version(self.inventory.netbox_api_version) >= Version("3.3"): + data["site"] = site + if self.settings.tenant: + data["tenant"] = {"name": self.settings.tenant} + return self.service_object(NBVM, identity, data) + + def sync_lambda(self, account, region, site, cluster, function): + status = {"Pending": "planned", "Active": "active", "Inactive": "offline", "Failed": "failed"} + self.logical_vm( + account, region, site, cluster, "lambda", function["FunctionArn"], + f"Lambda {function['FunctionName']}", + {key: value for key, value in function.items() if key != "Tags"}, + function["Tags"], memory=function["MemorySize"], status=status[function["State"]]) + + def sync_s3(self, account, region, site, group, buckets): + if not buckets: + return + identity = f"s3/{account}/{region}" + cluster = self.service_cluster( + account, region, site, group, "s3", identity, f"S3 {account}/{region}") + holder = self.logical_vm( + account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", + {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) + for bucket in buckets: + bucket_id = f"s3/{account}/{bucket['Name']}" + self.service_object(NBVirtualDisk, bucket_id, { + "name": bucket["Name"], "virtual_machine": holder, + # NetBox requires size; zero is explicitly unknown, never a usage measurement. + "size": 0, "description": "AWS S3 bucket; usage unknown (size 0 is a placeholder, not measured).", + "custom_fields": { + "aws_service": "s3", "aws_resource_id": bucket_id, + "aws_metadata": {**{key: value for key, value in bucket.items() if key != "Tags"}, + "usage_status": "unknown", "usage_bytes": None}, + }, + "tags": self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}), + }) + def resolve_shared_networks(self, snapshots): networks = {} for account, snapshot in snapshots.items(): @@ -258,6 +483,23 @@ def custom_fields(self): "name": name, "label": label, "object_types": types, "type": "text", "description": f"Reported by AWS ({self.name})", }) + if any((self.settings.sync_s3, self.settings.sync_lambda, self.settings.sync_eks)): + types = [NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type] + for name, field_type in ( + ("aws_service", "text"), ("aws_resource_id", "text"), + ("aws_metadata", "json"), + ): + field = self.add_update_custom_field({ + "name": name, "label": name.replace("_", " ").upper(), + "object_types": types, "type": field_type, + "description": f"Reported by AWS ({self.name})", + }) + existing_type = field.data.get("type") + if isinstance(existing_type, dict): + existing_type = existing_type.get("value") + if existing_type != field_type: + raise ValueError( + f"NetBox custom field '{name}' must have type '{field_type}', not '{existing_type}'") def sync_snapshot(self, account, snapshot, account_name=None): self.custom_fields() @@ -277,6 +519,24 @@ def sync_snapshot(self, account, snapshot, account_name=None): else: cluster_data["site"] = site cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) + eks_clusters = {} + inaccessible = data.get("InaccessibleServices", []) + if self.settings.sync_eks and "eks" not in inaccessible: + for resource in data["EKSClusters"]: + eks_cluster = self.service_cluster( + account, region, site, group, "eks", resource["arn"], + f"EKS {account}/{region}/{resource['name']}", + metadata={key: value for key, value in resource.items() if key != "tags"}, + tags=resource.get("tags", {})) + eks_clusters[resource["name"]] = eks_cluster + if self.settings.sync_lambda and "lambda" not in inaccessible: + lambda_cluster = self.service_cluster( + account, region, site, group, "lambda", f"lambda/{account}/{region}", + f"Lambda {account}/{region}") + for function in data["Functions"]: + self.sync_lambda(account, region, site, lambda_cluster, function) + if self.settings.sync_s3 and "s3" not in inaccessible: + self.sync_s3(account, region, site, group, data["Buckets"]) name_counts = {} for instance in data["instances"]: name = next((tag["Value"].strip() for tag in instance.get("Tags", []) @@ -289,10 +549,18 @@ def sync_snapshot(self, account, snapshot, account_name=None): if name_counts[vm_name.casefold()] > 1: vm_name = f"{vm_name} ({instance['InstanceId']})" self.sync_instance( - account, region, instance, data, site, cluster, api_version, vm_name=vm_name) + account, region, instance, data, site, + eks_clusters.get(data.get("EKSWorkers", {}).get(instance["InstanceId"]), cluster), + api_version, vm_name=vm_name) def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): instance_id = instance["InstanceId"] + if "eks" in data.get("InaccessibleServices", []): + existing = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if existing is not None and existing.data.get("cluster") is not None: + cluster = existing.data["cluster"] tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} spec = data["InstanceTypes"][instance["InstanceType"]] vm_data = { @@ -331,9 +599,7 @@ def sync_instance(self, account, region, instance, data, site, cluster, api_vers vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) else: vm.update(data=vm_data, source=self) - for tag in vm.get_tags(): - if any(tag.startswith(f"AWS:{key}=") for key in self.settings.tag_keys) and tag not in vm_data["tags"]: - vm.remove_tags(tag) + self.reconcile_tags(vm, vm_data["tags"]) if api_version >= Version("3.7"): for volume in disks: self.add_update_vm_child(NBVirtualDisk, vm, { @@ -409,8 +675,35 @@ def sync_interface(self, account, region, vm, eni, data, api_version): if existing is not None and existing.data.get("assigned_object_id") is not None and \ existing.get_interface() is not interface: owner = existing.get_device_vm() - if owner is None or (owner.source is not self and self.source_tag not in owner.get_tags()): - raise ValueError(f"IP '{address}' in VRF '{vrf.data['name']}' belongs to another source") + owner_tags = owner.get_tags() if owner is not None else [] + same_source = owner is not None and ( + owner.source is self or self.source_tag in owner_tags) + managed_owner = same_source or NetBoxHandler.primary_tag in owner_tags + managed_orphan = NetBoxHandler.primary_tag in owner_tags and \ + NetBoxHandler.orphaned_tag in owner_tags + owner_status = owner.data.get("status") if owner is not None else None + if isinstance(owner_status, dict): + owner_status = owner_status.get("value") + if managed_owner and owner_status == "active" and vm.data["status"] == "offline": + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"is assigned to active owner '{owner.get_display_name()}'. " + f"Skipping reassignment to offline ENI '{eni['NetworkInterfaceId']}'.") + continue + offline_to_active = managed_owner and owner_status == "offline" and vm.data["status"] == "active" + if not (same_source or managed_orphan or offline_to_active): + owner_name = owner.get_display_name() if owner is not None else "unresolved owner" + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"belongs to another source (owner '{owner_name}'). " + f"Skipping reassignment to ENI '{eni['NetworkInterfaceId']}'.") + continue + if managed_orphan or offline_to_active: + reason = "orphaned" if managed_orphan else "offline" + log.warning( + f"AWS source '{self.name}': Reclaiming IP '{address}' in VRF '{vrf.data['name']}' " + f"from {reason} owner '{owner.get_display_name()}' " + f"for ENI '{eni['NetworkInterfaceId']}'.") ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, "assigned_object_type": NBVMInterface} if self.settings.tenant: From 0ba0be76bd3dc56ad771c68c3e8eced40514a3fd Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:32:23 -0500 Subject: [PATCH 19/28] Add sync options for S3, Lambda, and EKS --- module/sources/aws/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module/sources/aws/config.py b/module/sources/aws/config.py index 2c6e227d..89e79c14 100644 --- a/module/sources/aws/config.py +++ b/module/sources/aws/config.py @@ -51,6 +51,14 @@ def __init__(self): ConfigOption(**config_option_permitted_subnets_definition), ConfigOption("sync_private_ips", bool, default_value=True, description="Import private IPv4 and IPv6 into VPC-specific VRFs, creating default VRFs as needed."), + ConfigOption("sync_s3", bool, default_value=False, + description="""Import S3 buckets as metadata-only virtual disks on logical account/region VMs. + Requires NetBox 3.7+. Disk size zero is a placeholder for unknown usage, not empty storage."""), + ConfigOption("sync_lambda", bool, default_value=False, + description="Import Lambda functions as logical VMs in AWS Lambda clusters."), + ConfigOption("sync_eks", bool, default_value=False, + description="""Import EKS clusters, node-group/Fargate metadata, and associate existing + EC2 workers with their EKS cluster. Does not discover Kubernetes pods."""), ConfigOption("vpc_vrf_relation", str, description="""Comma-separated region/vpc-id = NetBox VRF name mappings. Use account-id/region/vpc-id keys for account-specific mappings in organization mode. From 1db80ada8d22ce76e928e9bc5d642a22febc6a71 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:32:43 -0500 Subject: [PATCH 20/28] Implement access denial handling in AWS client Added context manager to handle AWS access denial gracefully and updated discovery methods to record denied scopes. --- module/sources/aws/client.py | 221 +++++++++++++++++++++++++++++------ 1 file changed, 187 insertions(+), 34 deletions(-) diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py index 685af9e0..7d749ff2 100644 --- a/module/sources/aws/client.py +++ b/module/sources/aws/client.py @@ -1,3 +1,5 @@ +from contextlib import contextmanager + from module.common.logging import get_logger log = get_logger() @@ -25,10 +27,27 @@ def __init__(self, profile=None, *, session=None): except BotoCoreError as exc: raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc self.account_names = {} + self.access_denied = [] + + @contextmanager + def tolerate_access_denied(self, account=None, region=None, service=None, resource=None): + try: + yield + except ClientError as exc: + if exc.response["Error"]["Code"] not in ( + "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", + "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException"): + raise AWSDiscoveryError(f"Incomplete AWS discovery: {exc}") from exc + self.access_denied.append((account, region, service)) + log.warning( + f"AWS access denied (account '{account}', region '{region}', service '{service}', " + f"resource '{resource}'): {exc}. " + "Skipping inaccessible discovery; existing source objects will be retained.") def discover_organization(self, regions, role_name, expected_account=None, discovery_region=None, - include_management=False, external_id=None, account_ids=None): - """Collect all selected accounts; never return a partial organization.""" + include_management=False, external_id=None, account_ids=None, services=()): + """Collect selected accounts, recording authorization-denied scopes for retention.""" + self.access_denied = [] clients = [] current_account = None try: @@ -76,25 +95,37 @@ def discover_organization(self, regions, role_name, expected_account=None, disco request = {"RoleArn": role_arn, "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} if external_id: request["ExternalId"] = external_id - credentials = sts.assume_role(**request)["Credentials"] + credentials = None + with self.tolerate_access_denied(current_account, service="sts"): + credentials = sts.assume_role(**request)["Credentials"] + if credentials is None: + continue member = AWSClient(session=boto3.Session( aws_access_key_id=credentials["AccessKeyId"], aws_secret_access_key=credentials["SecretAccessKey"], aws_session_token=credentials["SessionToken"], region_name=bootstrap)) - _, snapshot = member.discover(regions, current_account, discovery_region) + options = {"services": services} if services else {} + _, snapshot = member.discover(regions, current_account, discovery_region, **options) + self.access_denied.extend(member.access_denied) snapshots[current_account] = snapshot self.account_names = account_names return snapshots - except (BotoCoreError, ClientError, KeyError, AWSDiscoveryError) as exc: + except ClientError as exc: + with self.tolerate_access_denied(service="organizations"): + raise exc + return {} + except (BotoCoreError, KeyError, AWSDiscoveryError) as exc: context = f" for account '{current_account}'" if current_account else "" raise AWSDiscoveryError(f"Incomplete AWS organization discovery{context}: {exc}") from exc finally: for client in clients: client.close() - def discover(self, regions, expected_account=None, discovery_region=None): - """Fully collect all configured regions before any inventory changes.""" + def discover(self, regions, expected_account=None, discovery_region=None, services=()): + """Collect accessible regions and record denied scopes before inventory changes.""" clients = [] + self.access_denied = [] + account = expected_account try: config = Config(retries={"mode": "standard", "max_attempts": 5}, connect_timeout=10, read_timeout=60) @@ -125,36 +156,158 @@ def discover(self, regions, expected_account=None, discovery_region=None): for region in regions: ec2 = self.session.client("ec2", region_name=region, config=config) clients.append(ec2) - instances = [ - instance - for page in ec2.get_paginator("describe_instances").paginate() - for reservation in page.get("Reservations", []) - for instance in reservation.get("Instances", []) - if instance["State"]["Name"] != "terminated" - ] - data = {"instances": instances} - for operation, result_key, key in ( - ("describe_network_interfaces", "NetworkInterfaces", "NetworkInterfaceId"), - ("describe_volumes", "Volumes", "VolumeId"), - ("describe_subnets", "Subnets", "SubnetId"), - ("describe_vpcs", "Vpcs", "VpcId"), - ): - data[result_key] = { - item[key]: item - for page in ec2.get_paginator(operation).paginate() - for item in page.get(result_key, []) - } - instance_types = {} - names = sorted({item["InstanceType"] for item in instances}) - for start in range(0, len(names), 100): - for page in ec2.get_paginator("describe_instance_types").paginate( - InstanceTypes=names[start:start + 100]): - instance_types.update({item["InstanceType"]: item for item in page["InstanceTypes"]}) - data["InstanceTypes"] = instance_types + data = None + with self.tolerate_access_denied(account, region, "ec2"): + data = self.discover_ec2(ec2) + if data is None: + data = {"instances": [], "InstanceTypes": {}, "Vpcs": {}, "Subnets": {}, + "NetworkInterfaces": {}, "Volumes": {}, "EC2AccessDenied": True} + data["InaccessibleServices"] = [] + for service in services: + service_client = self.session.client(service, region_name=region, config=config) + clients.append(service_client) + try: + with self.tolerate_access_denied(account, region, service): + if service == "s3": + data["Buckets"] = self.discover_s3(service_client, region, account) + elif service == "lambda": + data["Functions"] = self.discover_lambda(service_client) + elif service == "eks": + data["EKSClusters"] = self.discover_eks(service_client) + else: + raise AWSDiscoveryError(f"Unsupported AWS service '{service}'") + key = {"s3": "Buckets", "lambda": "Functions", "eks": "EKSClusters"}[service] + if key not in data: + data["InaccessibleServices"].append(service) + except (BotoCoreError, ClientError, KeyError) as exc: + raise AWSDiscoveryError( + f"Incomplete {service} discovery (account '{account}', region '{region}'): {exc}") from exc snapshot[region] = data return account, snapshot - except (BotoCoreError, ClientError, KeyError) as exc: + except ClientError as exc: + with self.tolerate_access_denied(account, service="account"): + raise exc + return account, {} + except (BotoCoreError, KeyError) as exc: raise AWSDiscoveryError(f"Incomplete AWS discovery; synchronization aborted: {exc}") from exc finally: for client in clients: client.close() + + def discover_ec2(self, ec2): + instances = [ + instance + for page in ec2.get_paginator("describe_instances").paginate() + for reservation in page.get("Reservations", []) + for instance in reservation.get("Instances", []) + if instance["State"]["Name"] != "terminated" + ] + data = {"instances": instances} + for operation, result_key, key in ( + ("describe_network_interfaces", "NetworkInterfaces", "NetworkInterfaceId"), + ("describe_volumes", "Volumes", "VolumeId"), + ("describe_subnets", "Subnets", "SubnetId"), + ("describe_vpcs", "Vpcs", "VpcId"), + ): + data[result_key] = { + item[key]: item + for page in ec2.get_paginator(operation).paginate() + for item in page.get(result_key, []) + } + instance_types = {} + names = sorted({item["InstanceType"] for item in instances}) + for start in range(0, len(names), 100): + for page in ec2.get_paginator("describe_instance_types").paginate(InstanceTypes=names[start:start + 100]): + instance_types.update({item["InstanceType"]: item for item in page["InstanceTypes"]}) + data["InstanceTypes"] = instance_types + return data + + def discover_s3(self, client, region, account): + buckets = [] + for page in client.get_paginator("list_buckets").paginate(BucketRegion=region): + for bucket in page["Buckets"]: + with self.tolerate_access_denied(account, region, "s3", bucket["Name"]): + buckets.append(self.discover_s3_bucket(client, bucket, region, account)) + return buckets + + def discover_s3_bucket(self, client, bucket, region, account): + name = bucket["Name"] + bucket_region = bucket.get("BucketRegion") + if bucket_region is None: + location = client.get_bucket_location(Bucket=name, ExpectedBucketOwner=account) + bucket_region = location["LocationConstraint"] or "us-east-1" + if bucket_region == "EU": + bucket_region = "eu-west-1" + if bucket_region != region: + raise AWSDiscoveryError( + f"S3 bucket '{name}' returned for region '{region}' belongs to '{bucket_region}'") + request = {"Bucket": name, "ExpectedBucketOwner": account} + try: + tags = client.get_bucket_tagging(**request)["TagSet"] + except ClientError as exc: + if exc.response["Error"]["Code"] != "NoSuchTagSet": + raise + tags = [] + versioning = client.get_bucket_versioning(**request).get("Status", "Unversioned") + try: + encryption = client.get_bucket_encryption(**request)["ServerSideEncryptionConfiguration"] + except ClientError as exc: + if exc.response["Error"]["Code"] != "ServerSideEncryptionConfigurationNotFoundError": + raise + encryption = {} + return { + "Name": name, "Region": region, "Tags": tags, "Versioning": versioning, + "Encryption": encryption, "Arn": f"arn:{client.meta.partition}:s3:::{name}", + } + + def discover_lambda(self, client): + functions = [] + for page in client.get_paginator("list_functions").paginate(): + for reference in page["Functions"]: + config = client.get_function_configuration(FunctionName=reference["FunctionArn"]) + # Do not retain environment variables or other unneeded configuration. + function = {key: config[key] for key in ( + "FunctionArn", "FunctionName", "MemorySize", "State", "Runtime", "Handler", + "Timeout", "LastModified", "Architectures", "PackageType", "VpcConfig", + ) if key in config} + if function["FunctionArn"] != reference["FunctionArn"]: + raise AWSDiscoveryError(f"Lambda function identity changed for '{reference['FunctionArn']}'") + function["Tags"] = client.list_tags(Resource=function["FunctionArn"]).get("Tags", {}) + functions.append(function) + return functions + + def discover_eks(self, client): + clusters = [] + for page in client.get_paginator("list_clusters").paginate(): + for name in page["clusters"]: + cluster = client.describe_cluster(name=name)["cluster"] + if cluster["name"] != name: + raise AWSDiscoveryError(f"EKS cluster identity changed for '{name}'") + groups = [] + for group_page in client.get_paginator("list_nodegroups").paginate(clusterName=name): + for group_name in group_page["nodegroups"]: + group = client.describe_nodegroup(clusterName=name, nodegroupName=group_name)["nodegroup"] + if group["clusterName"] != name or group["nodegroupName"] != group_name: + raise AWSDiscoveryError(f"EKS node group identity changed for '{group_name}'") + groups.append({key: group[key] for key in ( + "nodegroupName", "nodegroupArn", "status", "version", "capacityType", + "instanceTypes", "scalingConfig", "subnets", "resources", + ) if key in group}) + profiles = [] + for profile_page in client.get_paginator("list_fargate_profiles").paginate(clusterName=name): + for profile_name in profile_page["fargateProfileNames"]: + profile = client.describe_fargate_profile( + clusterName=name, fargateProfileName=profile_name)["fargateProfile"] + if profile["clusterName"] != name or profile["fargateProfileName"] != profile_name: + raise AWSDiscoveryError(f"EKS Fargate profile identity changed for '{profile_name}'") + profiles.append({key: profile[key] for key in ( + "fargateProfileName", "fargateProfileArn", "status", "subnets", "selectors", + ) if key in profile}) + clusters.append({ + **{key: cluster[key] for key in ( + "name", "arn", "status", "version", "endpoint", "resourcesVpcConfig", "tags", + ) if key in cluster}, + "nodegroups": sorted(groups, key=lambda group: group["nodegroupName"]), + "fargateProfiles": sorted(profiles, key=lambda profile: profile["fargateProfileName"]), + }) + return clusters From 0f49d922596933bb12648663ed0033187f112b88 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:33:23 -0500 Subject: [PATCH 21/28] Add custom_fields to object classes --- module/netbox/object_classes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 7ee564e4..37aeb407 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1323,6 +1323,8 @@ def __init__(self, *args, **kwargs): NBInventoryItem.object_type, NBPowerPort.object_type, NBClusterGroup.object_type, + NBCluster.object_type, + NBVirtualDisk.object_type, NBVMInterface.object_type, NBVM.object_type, NBModule.object_type @@ -1914,7 +1916,8 @@ def __init__(self, *args, **kwargs): "scope_id": self.scopes, # supports pre4.2.0 clusters with site "site": NBSite, - "tags": NBTagList + "tags": NBTagList, + "custom_fields": NBCustomField } super().__init__(*args, **kwargs) @@ -2134,7 +2137,8 @@ def __init__(self, *args, **kwargs): "virtual_machine": NBVM, "description": 200, "size": int, # in MB - "tags": NBTagList + "tags": NBTagList, + "custom_fields": NBCustomField } super().__init__(*args, **kwargs) From 8b5c9f932c04039634e2ca3f9eae50286d274134 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Tue, 27 Oct 2026 20:33:46 -0500 Subject: [PATCH 22/28] Revise AWS source documentation for optional services Updated AWS source documentation to reflect optional service discovery for S3, Lambda, and EKS. Clarified permissions and added sections on authorization-denied discovery and troubleshooting. --- docs/source_aws.md | 272 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 256 insertions(+), 16 deletions(-) diff --git a/docs/source_aws.md b/docs/source_aws.md index c4bc4403..af0d1b6b 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -1,7 +1,8 @@ -# AWS EC2 source +# AWS source The `aws` source imports EC2 instances, attached EBS volumes, attached network interfaces (ENIs), MAC addresses, and permitted private IPv4 / IPv6 addresses. +Optional discovery adds S3 buckets, Lambda functions, and EKS clusters. AWS is queried read-only; writes are only made to NetBox. This branch is based on current main and does not depend on the oVirt or OpenStack branches. @@ -38,7 +39,8 @@ Profile-based assume-role credential refresh is handled by boto3. Credentials must allow access to every configured region. No SSH access, guest agents, inbound connections, or AWS write permissions are -needed. Outbound HTTPS to the regional EC2 and STS endpoints is required. +needed. Outbound HTTPS to the regional EC2 and STS endpoints is required, +plus the S3, Lambda, and EKS endpoints for enabled optional services. Example IAM policy (attach to a dedicated role or non-root identity): @@ -68,6 +70,101 @@ An assume-role profile additionally needs `sts:AssumeRole` permission on the target role and a matching target-account trust policy (and external ID where required). Single-account mode does not use Organizations permissions. +### Additional permissions for optional services + +The new services are **opt-in**. Add only the actions for the services you enable +to the single-account inventory identity or every selected member-account role. +For example, the following supplemental policy enables all three: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListAllMyBuckets", + "s3:GetBucketLocation", + "s3:GetBucketTagging", + "s3:GetBucketVersioning", + "s3:GetEncryptionConfiguration", + "lambda:ListFunctions", + "lambda:GetFunctionConfiguration", + "lambda:ListTags", + "eks:ListClusters", + "eks:DescribeCluster", + "eks:ListNodegroups", + "eks:DescribeNodegroup", + "eks:ListFargateProfiles", + "eks:DescribeFargateProfile" + ], + "Resource": "*" + } + ] +} +``` + +Where your IAM policy supports resource-level permissions, restrict the +resource-specific actions to the intended buckets, functions, and EKS resources. +Listing actions still need their documented wildcard scope. No S3 object reads, +Lambda code downloads, Kubernetes credentials, or CloudWatch permissions are +required. Authorization denials are logged and skipped as described below; +other discovery failures still abort before NetBox writes and pruning. + +### Authorization-denied discovery + +Recognized authorization errors (`AccessDenied`, `AccessDeniedException`, +`UnauthorizedOperation`, `AuthorizationError`, `AuthorizationErrorException`, +`Forbidden`, and `ForbiddenException`) produce warnings and allow accessible +discovery to continue. Warnings include account, region, service, AWS operation, +and the bucket name when applicable. + +* S3 bucket metadata denial skips that bucket and continues with subsequent + buckets. Existing bucket metadata and tags are not cleared. +* Denied S3/Lambda/EKS service discovery skips that service in that region. + Partially collected service results are discarded if listing or detail + discovery fails outside the per-bucket handling. +* Denied EC2 discovery discards the regional EC2 snapshot, but still attempts + optional services and other regions. Dependent instances whose shared + network metadata is unavailable during denied organization discovery are + deferred rather than imported with guessed masks. +* Denied member-role assumption skips that account and scans remaining + accounts. Denied identity, region enumeration, or organization enumeration + skips the affected scan without treating its inventory as empty. + +**Any authorization denial conservatively retains all objects previously +managed by this AWS source for that run**, including children and shared-network +dependencies. Accessible objects still update. This also postpones orphaning +and pruning for genuinely deleted objects in otherwise accessible scopes of +this source; a later fully authorized run resumes normal lifecycle processing. +Objects from other sources are unaffected. + +Existing EKS worker cluster placement is preserved when regional EKS discovery +is denied. Authentication/credential expiration, throttling exhaustion, service +errors, account mismatches, and invalid or conflicting data remain fatal. +Successful empty results are authoritative only when discovery has no denials. + +### Troubleshooting incomplete network discovery + +With private IP synchronization enabled, each attached ENI must have a discovered +VPC and subnet, and the subnet's VPC must match the ENI's VPC. If validation fails, +the error includes the account, region, instance, ENI, VPC, and subnet IDs. It +identifies resources missing from `DescribeVpcs` or `DescribeSubnets`, or reports +the conflicting VPC ID for a subnet mismatch. Multiple failures for the same ENI +are reported together. + +Check those resource IDs in the reported account and region using the same role +or profile as discovery, including any cross-account network sharing setup. +In organization mode, VPCs and subnets discovered in another selected account +can satisfy these checks as described under shared VPCs below. Missing resources +must still be discovered somewhere in the selected organization scope. +Missing or mismatched network data normally aborts without NetBox writes or +pruning. When denied organization discovery leaves shared-network dependencies +unavailable, affected instance updates are instead deferred with a warning and +existing objects retained. Address-space checks are not bypassed for imports. +Earlier "Created new VRF object" log messages refer to in-memory inventory +objects created during validation, not completed NetBox API writes. + ## Configuration ```ini @@ -89,6 +186,91 @@ environment variables. `regions` and `tag_keys` accept YAML lists as well as comma-separated strings. Quote the account ID in YAML so it remains a string. `python netbox-sync.py -g` generates the full configuration. +### Optional S3, Lambda, and EKS discovery + +These settings default to `False`, preserving existing EC2-only discovery and +permissions. Add them to the same AWS source for single-account or organization +mode: + +```ini +sync_s3 = True +sync_lambda = True +sync_eks = True +``` + +All three follow the source's selected account and region scope. Pagination is +completed before reconciliation. Disabling an optional service retains its +previously managed logical VMs, disks, and clusters instead of orphaning them. +While a service is enabled, successfully discovered empty results are +authoritative when discovery has no denials: removed functions, buckets, and empty S3 holders follow the +normal orphaning/pruning policy. Clusters remain subject to the existing +non-pruning cluster policy. + +#### S3 buckets + +Requires **NetBox 3.7+**; older versions abort before discovery when `sync_s3` +is enabled. Each account/region with buckets gets: + +* Cluster `S3 /`, type `AWS S3`. +* Logical VM `S3 inventory /`, tagged `AWS:logical-s3`. +* One virtual disk per general-purpose bucket, named by its bucket name. + +The holder VM is explicitly an inventory container, not a server. It has no +invented CPU, memory, MAC address, or IP. Bucket metadata includes ARN, region, +versioning, and encryption configuration in the disk's `aws_metadata` JSON +custom field. Configured `tag_keys` are copied to the bucket disk. + +**Disk size zero is a required-field placeholder, not a measurement.** +`aws_metadata.usage_status` is `unknown` and `usage_bytes` is JSON null; the +description also states that usage is unknown. Empty and unmeasured buckets +are therefore not treated as equivalent usage measurements. No provisioned +capacity or CloudWatch usage is inferred, and S3 disk totals must not be used +as storage-capacity reporting. Bucket contents, directory buckets, access +points, and S3-compatible non-AWS services are not discovered. + +#### Lambda functions + +Each account/region gets a logical `AWS Lambda` cluster and one logical VM per +function, tagged `AWS:logical-lambda`. Matching uses the unqualified function ARN, +not the display name. Long display names use a deterministic hash suffix to fit +the importer's name limit without losing resource identity. + +Configured function memory is VM memory. vCPUs and interfaces are not invented, +and shared Lambda VPC ENIs are not assigned to a function VM. `aws_metadata` +records runtime, handler, architecture, timeout, modification time, package +type, VPC configuration, and AWS state when returned. Environment variables, +code, credentials, invocation instances, aliases, and published versions are +not imported. + +State maps as follows: `Active` to `active`, `Pending` to `planned`, `Inactive` +to `offline`, and `Failed` to `failed`. These describe AWS function state, not +whether an invocation is currently executing. + +#### EKS clusters and workers + +Each real EKS cluster becomes a NetBox cluster of type `AWS EKS`. Its ARN is +the stable identity. `aws_metadata` records Kubernetes version, AWS status, +endpoint/VPC configuration, managed node groups (including scaling and Auto +Scaling group information), and Fargate profile metadata. + +Existing discovered EC2 worker VMs move into that cluster; duplicate worker VMs +and fabricated control-plane VMs are not created. Worker membership is matched +using managed node-group Auto Scaling group names or the EC2 tags +`aws:eks:cluster-name`, `eks:cluster-name`, and +`eksctl.cluster.k8s.io/v1alpha1/cluster-name`. Conflicting membership aborts the +run. A reference to a cluster no longer discovered emits a warning and leaves +the instance in its regular EC2 cluster. Untagged self-managed workers without +a discovered node-group relationship remain ordinary EC2 VMs. + +Disabling EKS discovery returns discovered EC2 workers to their normal EC2 +clusters. Kubernetes API access, pods, workloads, and individual Fargate runtime +instances are outside this discovery scope. + +Optional resources use `aws_service`, `aws_resource_id`, and `aws_metadata` +custom fields on VMs, clusters, and virtual disks. Existing unrelated tags are +preserved. These objects are logical documentation of AWS resources, not +claims that serverless services are virtual servers. + ### Automatic or explicit regions `regions = auto` (also the default when omitted) calls EC2 `DescribeRegions` and @@ -107,9 +289,9 @@ mode does not call `DescribeRegions`, does not need that permission, and ignores `discovery_region`. Do not combine `auto` and named regions in the same list. Auto mode requires `ec2:DescribeRegions` in addition to the inventory permissions. -Failure to discover regions, an empty enabled-region result, or failure to query -any enabled region aborts synchronization before NetBox writes or pruning. -There is no silent fallback to a partial region list. Use explicit mode when +Authorization denial during region discovery is logged and preserves existing +objects. Other failures, or an empty enabled-region result, abort synchronization +before NetBox writes or pruning. There is no silent fallback to a partial region list. Use explicit mode when credentials are intentionally restricted to a subset of enabled regions. Switching between modes changes discovery scope; inspect a dry run before allowing writes, particularly if narrowing to an explicit list. @@ -151,10 +333,11 @@ there. A role path such as `service-accounts/NetBoxInventoryReadOnly` is support Role ARNs use the collector's AWS partition; cross-partition discovery is not supported. -All selected accounts are collected and validated before reconciliation. An -Organizations API error, role-assumption failure, account mismatch, or regional -discovery failure aborts the entire run before NetBox writes or pruning. No -inaccessible account is treated as empty. A selection with no eligible accounts +All accessible selected accounts are collected and validated before reconciliation. +Authorization denials are logged and inaccessible scopes skipped with source-wide +retention. Other Organizations API errors, role-assumption failures, account +mismatches, or regional discovery failures abort before NetBox writes or pruning. +No inaccessible account is treated as empty. A selection with no eligible accounts also fails. Account discovery and scanning are sequential. Fresh one-hour STS credentials are obtained immediately before each account scan, so time spent scanning earlier accounts does not consume later accounts' credential lifetime. @@ -398,9 +581,39 @@ mode; this takes precedence over a legacy `region/vpc-id` mapping. For example: vpc_vrf_relation = 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC ``` -A mapping must not share a VRF between discovered VPCs. VRFs must remain distinct across -accounts and separately configured sources too. Organization mode validates -VRF isolation across all selected accounts before reconciliation. +A mapping must not share a VRF between distinct discovered VPCs. VRFs must remain +distinct for different VPC owners and separately configured sources too. +Organization mode validates VRF isolation across all selected accounts before +reconciliation. + +#### Shared VPCs in organization mode + +With private IP synchronization enabled, organization mode resolves attached +ENIs' network resources across all selected account snapshots. A participant +account may omit the VPC or subnet from its EC2 responses if another selected +account discovers them in the **same region**. Resolution uses exact resource +IDs and AWS `OwnerId`, never IP overlap or guessed subnet masks. In the absence +of `OwnerId`, a resource remains associated with the account that discovered it; +an ambiguous cross-account match aborts the run. + +All participants of the same shared VPC use **one owner-account VRF**: +`AWS //`. Account-specific +`vpc_vrf_relation` mappings must use the **VPC owner's account ID**, not the +participant's. VM account custom fields and cluster membership still use the +instance account. The owner account can supply network metadata even when it +has no EC2 instances. + +Include the network owner account in the same organization source when its +resources are not visible to participants. Resolution cannot borrow metadata +from another source or region. Missing resources, conflicting ownership or +CIDRs, and subnet/VPC mismatches still abort before reconciliation. If +authorization-denied discovery leaves network dependencies unavailable, affected +instance updates are deferred and existing objects retained instead. + +This changes IP placement for shared VPCs previously imported into separate +participant-account VRFs. Review a dry run and existing mappings before enabling +pruning; previously imported IPs may remain in the old VRFs until reconciled +under the normal lifecycle rules. There is no fallback to global address space. Subnet mask lengths come from AWS subnet CIDRs; NetBox prefixes are neither required nor created. Identical IPs in @@ -429,6 +642,10 @@ host. | ENI MAC | MAC object on NetBox 4.2+; legacy interface field on older versions | | Private IPv4 / IPv6 | IP in the VPC's existing VRF, assigned to the ENI | | EBS volume | Virtual disk named by volume ID | +| S3 bucket (opt-in) | Metadata-only virtual disk on an account/region logical VM | +| Lambda function (opt-in) | Logical VM in an `AWS Lambda` cluster | +| EKS cluster (opt-in) | `AWS EKS` cluster with node-group and Fargate profile metadata | +| EKS EC2 worker (opt-in) | Existing EC2 VM associated with its EKS cluster | In single-account mode, set `account_name` to use a friendly label in the cluster group and cluster names; it defaults to the account ID. Organization @@ -449,8 +666,9 @@ provisioned capacity, not utilization. Running instances are `active`; stopped, pending, stopping, and shutting-down instances are `offline`. Terminated instances are excluded and subsequently follow the existing NetBox orphaning/pruning configuration. An empty, successfully -queried account/region is authoritative. All configured regions and paginated -responses are collected before reconciliation. AWS access failures, credential +queried account/region is authoritative when discovery has no denials. All accessible +configured regions and paginated responses are collected before reconciliation. +Authorization denials log warnings and retain existing objects. Other AWS failures, credential failures, missing attached resources, and missing VRF mappings abort the run before the application's NetBox write/prune phase. Transient attachment changes may therefore require retrying the run rather than importing partial data. @@ -466,11 +684,33 @@ are stored on the ENI as `aws_public_ipv4`, not assigned as guest IPs or made primary. IPv6 addresses are actual interface addresses even when globally routable. Delegated address prefixes are not imported. +An IP already assigned to another interface can be reassigned when its current +VM/device belongs to this source, or carries both `NetBox-synced` and +`NetBox-synced: Orphaned` tags. Reclamation from an orphaned owner logs a warning; +the orphan lifecycle removes source tags, so the former source may be different. +An active EC2 VM can also reclaim an IP from an `offline` owner managed by +netbox-sync (the current source's ownership or a `NetBox-synced` tag), even if +another source originally managed it. This logs a warning and clears the old +owner's primary-IP reference when applicable. An offline EC2 VM cannot take an +IP from an active managed owner, regardless of discovery order. Other non-active +statuses do not qualify as offline, and manually managed offline owners are not +eligible for this exception. +Other ownership conflicts, including unresolved owners, log an error and skip +that IP without aborting the sync. Its existing assignment is left untouched, +and it is not set as the discovered VM's primary IP. Normal write and pruning +phases still proceed. + Only explicitly selected `tag_keys` become tags (`AWS:=`). Their combined names must fit NetBox's 100-character limit. Existing unrelated NetBox tags are preserved. Metadata custom fields use the `aws_` prefix. Their names should not conflict with user-defined fields of incompatible types. +Tag reconciliation uses the same inventory lookup as tag assignment. When that +lookup reuses an existing NetBox tag with different capitalization, its existing +name is preserved and the assigned tag is not removed as stale. This applies to +EC2 VMs, S3 bucket disks, Lambda VMs, and EKS clusters. Tags for configured keys +are still removed when they are no longer reported by AWS. + Avoid overlapping source definitions for the same account/region. Narrowing regions, disabling IP import, or changing permitted networks affects the source's managed scope; review a dry run before making such changes. The @@ -489,9 +729,9 @@ AWS credentials or network calls: ```sh pip install -r requirements-aws.txt pytest -python -m pytest tests/test_aws_source.py +python -m pytest tests/test_aws_source.py tests/test_aws_services.py ``` Live AWS/NetBox validation is still required before production use. This source -does not inventory RDS, load balancers, ECS/EKS workloads, security groups, +does not inventory RDS, load balancers, ECS/Kubernetes workloads, security groups, software, or guest operating-system configuration. From 948d1703089a17822b87a44c833b3968a2c96a4d Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 28 Oct 2026 13:17:53 -0500 Subject: [PATCH 23/28] Enhance AWS source documentation on failures and conflicts Expand documentation on authorization failures and resource conflicts in AWS source. --- docs/source_aws.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/source_aws.md b/docs/source_aws.md index af0d1b6b..91b9071a 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -144,6 +144,32 @@ is denied. Authentication/credential expiration, throttling exhaustion, service errors, account mismatches, and invalid or conflicting data remain fatal. Successful empty results are authoritative only when discovery has no denials. +Authorization failures also add operation-specific tags to matching managed +NetBox objects, for example `Sync:AccessDenied:GetBucketTagging`. Bucket-specific +failures tag the bucket's virtual disk. Failures without an individual resource +identifier tag existing managed objects in the affected account/region/service +scope; account/role or organization failures cover their broader scopes. Objects +not managed by this AWS source are not tagged. Resources that have never been +imported are not created as placeholders, so those failures remain log-only. + +Permission tags clear automatically after the same operation succeeds for the +relevant resource or scope. A disabled service, unattempted operation, or success +for another resource does not clear a tag. A denial takes precedence over success +in the same applicable scope during a run. Other user tags are preserved. + +### NetBox resource conflicts + +Logical service resources with duplicate NetBox resource identities, ownership +by another source, or a NetBox name collision are logged and skipped, without +aborting unrelated resources, regions, or accounts. Conflicting objects are not +adopted or overwritten. A skipped cluster also skips its dependent resources, +including EKS workers; a skipped S3 inventory VM skips its buckets. + +If a resource conflict occurs, all previously managed objects for this source +are retained for the run, protecting skipped resources and their children from +orphaning/pruning. Discovery and snapshot-validation errors retain their existing +failure handling. + ### Troubleshooting incomplete network discovery With private IP synchronization enabled, each attached ENI must have a discovered From 5e428e90365b2716bb3a170590970a0f643a834e Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 28 Oct 2026 13:18:30 -0500 Subject: [PATCH 24/28] Add AWSAccessCheck dataclass and access tracking Added AWSAccessCheck dataclass to track access checks and modified AWSClient to record access checks during AWS service calls. --- module/sources/aws/client.py | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py index 7d749ff2..38002b6f 100644 --- a/module/sources/aws/client.py +++ b/module/sources/aws/client.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +from dataclasses import dataclass from module.common.logging import get_logger @@ -18,6 +19,16 @@ class AWSDiscoveryError(RuntimeError): """Discovery failed; do not reconcile this run with NetBox.""" +@dataclass(frozen=True) +class AWSAccessCheck: + account: str | None + region: str | None + service: str + resource: str | None + operation: str + denied: bool + + class AWSClient: def __init__(self, profile=None, *, session=None): if boto3 is None: @@ -28,6 +39,31 @@ def __init__(self, profile=None, *, session=None): raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc self.account_names = {} self.access_denied = [] + self.access_checks = [] + + def track_access(self, client, account, region, service): + def capture_resource(params, context, **kwargs): + context["netbox_sync_resource"] = next(( + params[key] for key in ("Bucket", "clusterName", "FunctionName", "Resource", "name") + if key in params + ), None) + + def record_access(model, parsed, context, **kwargs): + code = parsed.get("Error", {}).get("Code") + denied = code in ( + "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", + "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException") + if code is None or denied or code in ( + "NoSuchTagSet", "ServerSideEncryptionConfigurationNotFoundError"): + check_account = parsed.get("Account", account) if model.name == "GetCallerIdentity" else account + self.access_checks.append(AWSAccessCheck( + check_account, region, service, context.get("netbox_sync_resource"), model.name, denied)) + + for event, handler in ( + ("before-parameter-build.*.*", capture_resource), ("after-call.*.*", record_access)): + unique_id = f"netbox-sync-access-{event}" + client.meta.events.unregister(event, unique_id=unique_id) + client.meta.events.register(event, handler, unique_id=unique_id) @contextmanager def tolerate_access_denied(self, account=None, region=None, service=None, resource=None): @@ -48,6 +84,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco include_management=False, external_id=None, account_ids=None, services=()): """Collect selected accounts, recording authorization-denied scopes for retention.""" self.access_denied = [] + self.access_checks = [] clients = [] current_account = None try: @@ -59,6 +96,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco config = Config(retries={"mode": "standard", "max_attempts": 5}, connect_timeout=10, read_timeout=60) sts = self.session.client("sts", region_name=bootstrap, config=config) + self.track_access(sts, None, None, "sts") clients.append(sts) identity = sts.get_caller_identity() if expected_account is not None and identity["Account"] != expected_account: @@ -66,6 +104,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco f"AWS account mismatch: expected {expected_account}, authenticated as {identity['Account']}.") partition = identity["Arn"].split(":")[1] organizations = self.session.client("organizations", region_name=bootstrap, config=config) + self.track_access(organizations, None, None, "organizations") clients.append(organizations) management_account = organizations.describe_organization()["Organization"]["MasterAccountId"] accounts = set() @@ -96,6 +135,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco if external_id: request["ExternalId"] = external_id credentials = None + self.track_access(sts, current_account, None, "sts") with self.tolerate_access_denied(current_account, service="sts"): credentials = sts.assume_role(**request)["Credentials"] if credentials is None: @@ -107,6 +147,7 @@ def discover_organization(self, regions, role_name, expected_account=None, disco options = {"services": services} if services else {} _, snapshot = member.discover(regions, current_account, discovery_region, **options) self.access_denied.extend(member.access_denied) + self.access_checks.extend(member.access_checks) snapshots[current_account] = snapshot self.account_names = account_names return snapshots @@ -125,6 +166,7 @@ def discover(self, regions, expected_account=None, discovery_region=None, servic """Collect accessible regions and record denied scopes before inventory changes.""" clients = [] self.access_denied = [] + self.access_checks = [] account = expected_account try: config = Config(retries={"mode": "standard", "max_attempts": 5}, @@ -136,6 +178,7 @@ def discover(self, regions, expected_account=None, discovery_region=None, servic "Automatic region discovery requires discovery_region or an AWS profile/environment region " "(AWS_DEFAULT_REGION).") sts = self.session.client("sts", region_name=bootstrap, config=config) + self.track_access(sts, account, None, "account") clients.append(sts) account = sts.get_caller_identity()["Account"] if expected_account is not None and account != expected_account: @@ -143,6 +186,7 @@ def discover(self, regions, expected_account=None, discovery_region=None, servic f"AWS account mismatch: expected {expected_account}, authenticated as {account}.") if automatic: discovery_client = self.session.client("ec2", region_name=bootstrap, config=config) + self.track_access(discovery_client, account, None, "account") clients.append(discovery_client) response = discovery_client.describe_regions(AllRegions=True) regions = sorted({ @@ -155,6 +199,7 @@ def discover(self, regions, expected_account=None, discovery_region=None, servic snapshot = {} for region in regions: ec2 = self.session.client("ec2", region_name=region, config=config) + self.track_access(ec2, account, region, "ec2") clients.append(ec2) data = None with self.tolerate_access_denied(account, region, "ec2"): @@ -165,6 +210,7 @@ def discover(self, regions, expected_account=None, discovery_region=None, servic data["InaccessibleServices"] = [] for service in services: service_client = self.session.client(service, region_name=region, config=config) + self.track_access(service_client, account, region, service) clients.append(service_client) try: with self.tolerate_access_denied(account, region, service): From 8dab31782ba3c15ae5905157f7f95e0ac3aa40e2 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 28 Oct 2026 13:18:45 -0500 Subject: [PATCH 25/28] Update print statement to say 'Goodbye World' --- module/sources/aws/client.py | 1154 ++++++++++++++++++++++++---------- 1 file changed, 811 insertions(+), 343 deletions(-) diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py index 38002b6f..c3559320 100644 --- a/module/sources/aws/client.py +++ b/module/sources/aws/client.py @@ -1,359 +1,827 @@ -from contextlib import contextmanager -from dataclasses import dataclass +from hashlib import sha256 +from ipaddress import ip_address, ip_interface, ip_network +from itertools import product + +from packaging.version import Version from module.common.logging import get_logger +from module.common.misc import do_error_exit +from module.common.support import normalize_mac_address +from module.netbox import ( + NBCustomField, NBCluster, NBClusterGroup, NBClusterType, NBIPAddress, + NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) +from module.netbox.inventory import NetBoxInventory +from module.netbox.connection import NetBoxHandler +from module.sources.aws.client import AWSClient, AWSDiscoveryError +from module.sources.aws.config import AWSConfig +from module.sources.common.source_base import SourceBase log = get_logger() -try: - import boto3 - from botocore.config import Config - from botocore.exceptions import BotoCoreError, ClientError -except ModuleNotFoundError as exc: - if exc.name != "boto3": - raise - boto3 = None +class AWSHandler(SourceBase): + source_type = "aws" + dependent_netbox_objects = [ + NBTag, NBCustomField, NBTenant, NBSite, NBClusterGroup, NBClusterType, + NBCluster, NBVM, NBVMInterface, NBMACAddress, NBIPAddress, NBVRF, NBVirtualDisk, + ] -class AWSDiscoveryError(RuntimeError): - """Discovery failed; do not reconcile this run with NetBox.""" + def __init__(self, name=None): + if not name: + raise ValueError(f"Invalid source name '{name}'.") + self.name = name + self.inventory = NetBoxInventory() + config = AWSConfig() + config.source_name = name + self.settings = config.parse() + self.set_source_tag() + self.init_successful = False + if not self.settings.enabled: + log.info(f"Source '{name}' is disabled. Skipping.") + return + try: + self.client = AWSClient(profile=self.settings.profile) + except AWSDiscoveryError as exc: + self.abort(exc) + self.init_successful = True + def abort(self, error): + message = f"AWS source '{self.name}': {error}. No NetBox writes or pruning will be performed." + log.error(message) + do_error_exit(message) -@dataclass(frozen=True) -class AWSAccessCheck: - account: str | None - region: str | None - service: str - resource: str | None - operation: str - denied: bool + def apply(self): + try: + if self.settings.sync_s3 and Version(self.inventory.netbox_api_version) < Version("3.7"): + raise ValueError("S3 bucket inventory requires NetBox 3.7 or newer") + services = tuple(service for service in ("s3", "lambda", "eks") + if getattr(self.settings, f"sync_{service}")) + options = {"services": services} if services else {} + if self.settings.organization: + snapshots = self.client.discover_organization( + self.settings.regions, self.settings.organization_role_name, + expected_account=self.settings.account_id, + discovery_region=self.settings.discovery_region, + include_management=self.settings.organization_include_management, + external_id=self.settings.organization_external_id, + account_ids=self.settings.organization_account_ids, **options) + account_names = getattr(self.client, "account_names", {}) + else: + account, snapshot = self.client.discover( + self.settings.regions, self.settings.account_id, self.settings.discovery_region, **options) + snapshots = {account: snapshot} if account is not None else {} + account_names = {} + if self.settings.organization and self.settings.sync_private_ips: + self.resolve_shared_networks(snapshots) + self.defer_inaccessible_network_instances(snapshots) + vrfs = {} + for account, snapshot in snapshots.items(): + self.validate_services(account, snapshot) + self.validate_snapshot(account, snapshot, vrfs) + self.retain_disabled_services() + if getattr(self.client, "access_denied", []): + self.retain_inaccessible_objects() + account_labels = { + account: account_names.get(account) or self.settings.account_name or account + for account in snapshots + } + label_counts = {} + for label in account_labels.values(): + key = label.casefold() + label_counts[key] = label_counts.get(key, 0) + 1 + for account, label in account_labels.items(): + if label_counts[label.casefold()] > 1: + account_labels[account] = f"{label} ({account})" + for account, snapshot in snapshots.items(): + self.sync_snapshot(account, snapshot, account_name=account_labels[account]) + self.reconcile_access_tags() + except (AWSDiscoveryError, ValueError, KeyError) as exc: + self.abort(exc) + def validate_services(self, account, snapshot): + for region, data in snapshot.items(): + inaccessible = data.get("InaccessibleServices", []) + for enabled, key in ( + (self.settings.sync_s3 and "s3" not in inaccessible, "Buckets"), + (self.settings.sync_lambda and "lambda" not in inaccessible, "Functions"), + (self.settings.sync_eks and "eks" not in inaccessible, "EKSClusters"), + ): + if enabled and not isinstance(data.get(key), list): + raise ValueError(f"Missing AWS {key} discovery (account '{account}', region '{region}')") + if enabled: + identity_key = {"Buckets": "Name", "Functions": "FunctionArn", "EKSClusters": "arn"}[key] + identities = [resource[identity_key] for resource in data[key]] + if len(set(identities)) != len(identities): + raise ValueError(f"Duplicate AWS {key} discovery (account '{account}', region '{region}')") + for bucket in data["Buckets"] if self.settings.sync_s3 and "s3" not in inaccessible else []: + if bucket["Region"] != region or not bucket["Name"] or len(bucket["Name"]) > 63: + raise ValueError(f"Invalid S3 bucket '{bucket['Name']}' for region '{region}'") + self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}) + if "Versioning" not in bucket or "Encryption" not in bucket: + raise ValueError(f"Incomplete S3 bucket metadata for '{bucket['Name']}'") + for function in data["Functions"] if self.settings.sync_lambda and "lambda" not in inaccessible else []: + self.validate_arn(function["FunctionArn"], "lambda", account, region) + if function["State"] not in ("Pending", "Active", "Inactive", "Failed"): + raise ValueError(f"Unknown Lambda state '{function['State']}' for '{function['FunctionArn']}'") + if not function["FunctionName"] or not isinstance(function["MemorySize"], int): + raise ValueError(f"Incomplete Lambda configuration for '{function['FunctionArn']}'") + self.service_tags(function["Tags"]) + if self.settings.sync_eks and "eks" not in inaccessible: + clusters = {cluster["name"]: cluster for cluster in data["EKSClusters"]} + if len(clusters) != len(data["EKSClusters"]): + raise ValueError(f"Duplicate EKS cluster discovery in account '{account}', region '{region}'") + asg_clusters = {} + for cluster in data["EKSClusters"]: + self.validate_arn(cluster["arn"], "eks", account, region) + for field in ("status", "version", "resourcesVpcConfig", "nodegroups", "fargateProfiles"): + if field not in cluster: + raise ValueError(f"Incomplete EKS {field} for '{cluster['arn']}'") + self.service_tags(cluster.get("tags", {})) + for group in cluster["nodegroups"]: + for asg in group.get("resources", {}).get("autoScalingGroups", []): + name = asg["name"] + if name in asg_clusters and asg_clusters[name] != cluster["name"]: + raise ValueError(f"EKS Auto Scaling group '{name}' belongs to multiple clusters") + asg_clusters[name] = cluster["name"] + data["EKSWorkers"] = {} + for instance in data["instances"]: + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + names = {tags[key] for key in ( + "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", + ) if tags.get(key)} + asg_cluster = asg_clusters.get(tags.get("aws:autoscaling:groupName")) + if asg_cluster: + names.add(asg_cluster) + if len(names) > 1: + raise ValueError(f"Conflicting EKS cluster membership for instance '{instance['InstanceId']}'") + if names: + name = next(iter(names)) + if name in clusters: + data["EKSWorkers"][instance["InstanceId"]] = name + else: + log.warning( + f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " + f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") -class AWSClient: - def __init__(self, profile=None, *, session=None): - if boto3 is None: - raise AWSDiscoveryError("AWS SDK missing. Install dependencies from requirements-aws.txt.") - try: - self.session = session if session is not None else boto3.Session(profile_name=profile) - except BotoCoreError as exc: - raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc - self.account_names = {} - self.access_denied = [] - self.access_checks = [] - - def track_access(self, client, account, region, service): - def capture_resource(params, context, **kwargs): - context["netbox_sync_resource"] = next(( - params[key] for key in ("Bucket", "clusterName", "FunctionName", "Resource", "name") - if key in params - ), None) - - def record_access(model, parsed, context, **kwargs): - code = parsed.get("Error", {}).get("Code") - denied = code in ( - "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", - "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException") - if code is None or denied or code in ( - "NoSuchTagSet", "ServerSideEncryptionConfigurationNotFoundError"): - check_account = parsed.get("Account", account) if model.name == "GetCallerIdentity" else account - self.access_checks.append(AWSAccessCheck( - check_account, region, service, context.get("netbox_sync_resource"), model.name, denied)) - - for event, handler in ( - ("before-parameter-build.*.*", capture_resource), ("after-call.*.*", record_access)): - unique_id = f"netbox-sync-access-{event}" - client.meta.events.unregister(event, unique_id=unique_id) - client.meta.events.register(event, handler, unique_id=unique_id) - - @contextmanager - def tolerate_access_denied(self, account=None, region=None, service=None, resource=None): - try: - yield - except ClientError as exc: - if exc.response["Error"]["Code"] not in ( - "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", - "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException"): - raise AWSDiscoveryError(f"Incomplete AWS discovery: {exc}") from exc - self.access_denied.append((account, region, service)) - log.warning( - f"AWS access denied (account '{account}', region '{region}', service '{service}', " - f"resource '{resource}'): {exc}. " - "Skipping inaccessible discovery; existing source objects will be retained.") - - def discover_organization(self, regions, role_name, expected_account=None, discovery_region=None, - include_management=False, external_id=None, account_ids=None, services=()): - """Collect selected accounts, recording authorization-denied scopes for retention.""" - self.access_denied = [] - self.access_checks = [] - clients = [] - current_account = None - try: - bootstrap = (discovery_region or self.session.region_name) if regions == ["auto"] else regions[0] - if not bootstrap: - raise AWSDiscoveryError( - "Organization discovery requires discovery_region or an AWS profile/environment region " - "(AWS_DEFAULT_REGION).") - config = Config(retries={"mode": "standard", "max_attempts": 5}, - connect_timeout=10, read_timeout=60) - sts = self.session.client("sts", region_name=bootstrap, config=config) - self.track_access(sts, None, None, "sts") - clients.append(sts) - identity = sts.get_caller_identity() - if expected_account is not None and identity["Account"] != expected_account: - raise AWSDiscoveryError( - f"AWS account mismatch: expected {expected_account}, authenticated as {identity['Account']}.") - partition = identity["Arn"].split(":")[1] - organizations = self.session.client("organizations", region_name=bootstrap, config=config) - self.track_access(organizations, None, None, "organizations") - clients.append(organizations) - management_account = organizations.describe_organization()["Organization"]["MasterAccountId"] - accounts = set() - account_names = {} - for page in organizations.get_paginator("list_accounts").paginate(): - for account in page["Accounts"]: - account_id = account["Id"] - if account["State"] != "ACTIVE": - log.info(f"AWS organization: excluding account '{account_id}' in state '{account['State']}'.") - continue - if account_id == management_account and not include_management: - continue - accounts.add(account_id) - account_names[account_id] = account.get("Name") or account_id - if account_ids: - missing = set(account_ids) - accounts - if missing: - raise AWSDiscoveryError( - f"Requested organization accounts are not active/eligible: {', '.join(sorted(missing))}.") - accounts.intersection_update(account_ids) - if not accounts: - raise AWSDiscoveryError("Organization discovery returned no eligible accounts.") - snapshots = {} - for current_account in sorted(accounts): - role_arn = f"arn:{partition}:iam::{current_account}:role/{role_name}" - log.info(f"AWS organization: discovering account '{current_account}' using '{role_arn}'.") - request = {"RoleArn": role_arn, "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} - if external_id: - request["ExternalId"] = external_id - credentials = None - self.track_access(sts, current_account, None, "sts") - with self.tolerate_access_denied(current_account, service="sts"): - credentials = sts.assume_role(**request)["Credentials"] - if credentials is None: + def access_scope(self, item): + fields = item.data.get("custom_fields") or {} + metadata = fields.get("aws_metadata") or {} + identity = fields.get("aws_resource_id") or "" + service = "ec2" if fields.get("aws_instance_id") else fields.get("aws_service") + account = fields.get("aws_account_id") or metadata.get("account") + region = fields.get("aws_region") or metadata.get("region") + resource = None + if identity.startswith("arn:"): + parts = identity.split(":", 5) + if len(parts) == 6: + service, region, account, resource = parts[2:] + if service == "lambda": + resource = identity + elif service == "eks": + resource = resource.removeprefix("cluster/") + elif identity.startswith("s3/"): + parts = identity.split("/") + account = parts[1] + if isinstance(item, NBVirtualDisk): + resource = parts[2] + else: + region = parts[2] + for parent_key in ("virtual_machine", "cluster"): + parent = item.data.get(parent_key) + if parent is not None: + parent_account, parent_region, parent_service, _ = self.access_scope(parent) + account = account or parent_account + region = region or parent_region + service = service or parent_service + break + if isinstance(item, (NBIPAddress, NBMACAddress)): + parent = item.get_device_vm() + if parent is not None: + account, region, service, _ = self.access_scope(parent) + if account is None: + group = item if isinstance(item, NBClusterGroup) else item.data.get("group") + name = group.data.get("name", "") if group is not None else "" + if name.startswith("AWS "): + label = name[4:] + account = next((key for key, value in getattr(self.client, "account_names", {}).items() + if label in (value, f"{value} ({key})")), None) + if account is None and label.isdigit(): + account = label + if region is None: + site = item.data.get("scope_id") or item.data.get("site") + name = site.data.get("name", "") if isinstance(site, NBSite) else "" + if name.startswith("AWS "): + region = name[4:] + return account, region, service or "ec2", resource + + def reconcile_access_tags(self): + checks = getattr(self.client, "access_checks", []) + if not checks: + return + prefix = "Sync:AccessDenied:" + scopes = {} + for check in checks: + outcomes = scopes.setdefault( + (check.account, check.region, check.service, check.resource), (set(), set())) + outcomes[0 if check.denied else 1].add(check.operation) + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if object_type in (NBTag, NBCustomField, NBTenant) or "tags" not in item.data_model or not ( + item.source is self or self.source_tag in item.get_tags()): continue - member = AWSClient(session=boto3.Session( - aws_access_key_id=credentials["AccessKeyId"], - aws_secret_access_key=credentials["SecretAccessKey"], - aws_session_token=credentials["SessionToken"], region_name=bootstrap)) - options = {"services": services} if services else {} - _, snapshot = member.discover(regions, current_account, discovery_region, **options) - self.access_denied.extend(member.access_denied) - self.access_checks.extend(member.access_checks) - snapshots[current_account] = snapshot - self.account_names = account_names - return snapshots - except ClientError as exc: - with self.tolerate_access_denied(service="organizations"): - raise exc - return {} - except (BotoCoreError, KeyError, AWSDiscoveryError) as exc: - context = f" for account '{current_account}'" if current_account else "" - raise AWSDiscoveryError(f"Incomplete AWS organization discovery{context}: {exc}") from exc - finally: - for client in clients: - client.close() - - def discover(self, regions, expected_account=None, discovery_region=None, services=()): - """Collect accessible regions and record denied scopes before inventory changes.""" - clients = [] - self.access_denied = [] - self.access_checks = [] - account = expected_account - try: - config = Config(retries={"mode": "standard", "max_attempts": 5}, - connect_timeout=10, read_timeout=60) - automatic = regions == ["auto"] - bootstrap = (discovery_region or self.session.region_name) if automatic else regions[0] - if not bootstrap: - raise AWSDiscoveryError( - "Automatic region discovery requires discovery_region or an AWS profile/environment region " - "(AWS_DEFAULT_REGION).") - sts = self.session.client("sts", region_name=bootstrap, config=config) - self.track_access(sts, account, None, "account") - clients.append(sts) - account = sts.get_caller_identity()["Account"] - if expected_account is not None and account != expected_account: - raise AWSDiscoveryError( - f"AWS account mismatch: expected {expected_account}, authenticated as {account}.") - if automatic: - discovery_client = self.session.client("ec2", region_name=bootstrap, config=config) - self.track_access(discovery_client, account, None, "account") - clients.append(discovery_client) - response = discovery_client.describe_regions(AllRegions=True) - regions = sorted({ - entry["RegionName"] for entry in response["Regions"] - if entry["OptInStatus"] in ("opt-in-not-required", "opted-in") - }) - if not regions: - raise AWSDiscoveryError("Automatic region discovery returned no enabled regions.") - log.info(f"AWS account '{account}' enabled regions: {', '.join(regions)}") - snapshot = {} - for region in regions: - ec2 = self.session.client("ec2", region_name=region, config=config) - self.track_access(ec2, account, region, "ec2") - clients.append(ec2) - data = None - with self.tolerate_access_denied(account, region, "ec2"): - data = self.discover_ec2(ec2) - if data is None: - data = {"instances": [], "InstanceTypes": {}, "Vpcs": {}, "Subnets": {}, - "NetworkInterfaces": {}, "Volumes": {}, "EC2AccessDenied": True} - data["InaccessibleServices"] = [] - for service in services: - service_client = self.session.client(service, region_name=region, config=config) - self.track_access(service_client, account, region, service) - clients.append(service_client) - try: - with self.tolerate_access_denied(account, region, service): - if service == "s3": - data["Buckets"] = self.discover_s3(service_client, region, account) - elif service == "lambda": - data["Functions"] = self.discover_lambda(service_client) - elif service == "eks": - data["EKSClusters"] = self.discover_eks(service_client) - else: - raise AWSDiscoveryError(f"Unsupported AWS service '{service}'") - key = {"s3": "Buckets", "lambda": "Functions", "eks": "EKSClusters"}[service] - if key not in data: - data["InaccessibleServices"].append(service) - except (BotoCoreError, ClientError, KeyError) as exc: - raise AWSDiscoveryError( - f"Incomplete {service} discovery (account '{account}', region '{region}'): {exc}") from exc - snapshot[region] = data - return account, snapshot - except ClientError as exc: - with self.tolerate_access_denied(account, service="account"): - raise exc - return account, {} - except (BotoCoreError, KeyError) as exc: - raise AWSDiscoveryError(f"Incomplete AWS discovery; synchronization aborted: {exc}") from exc - finally: - for client in clients: - client.close() - - def discover_ec2(self, ec2): - instances = [ - instance - for page in ec2.get_paginator("describe_instances").paginate() - for reservation in page.get("Reservations", []) - for instance in reservation.get("Instances", []) - if instance["State"]["Name"] != "terminated" + account, region, service, resource = self.access_scope(item) + denied, succeeded = set(), set() + for key in product( + (None, account), (None, region), ("organizations", "sts", "account", service), + (None, resource)): + if key in scopes: + failures, successes = scopes[key] + denied.update(failures) + succeeded.update(successes) + for operation in denied: + item.add_tags(f"{prefix}{operation}") + for tag in item.get_tags(): + if tag.startswith(prefix) and tag[len(prefix):] in succeeded - denied: + item.remove_tags(tag) + + def validate_arn(self, arn, service, account, region): + parts = arn.split(":", 5) + if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: + raise ValueError(f"Invalid {service} resource ARN '{arn}' for account '{account}', region '{region}'") + + def service_tags(self, tags): + result = [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags] + if any(len(tag) > 100 for tag in result): + raise ValueError("AWS service tag exceeds the NetBox tag name limit (100 characters)") + return result + + def reconcile_tags(self, item, tags): + # Tag lookup can reuse an existing name with different capitalization. + desired_tags = {self.inventory.get_by_data(NBTag, data={"name": name}) for name in tags} + managed_prefixes = tuple(f"AWS:{key}=".casefold() for key in self.settings.tag_keys) + for tag in item.data.get("tags", []): + if tag.get_display_name().casefold().startswith(managed_prefixes) and tag not in desired_tags: + item.remove_tags(tag) + + def retain_disabled_services(self): + for object_type in (NBVM, NBVirtualDisk, NBCluster): + for item in self.inventory.get_all_items(object_type): + service = (item.data.get("custom_fields") or {}).get("aws_service") + if service in ("s3", "lambda", "eks") and not getattr(self.settings, f"sync_{service}") and \ + (item.source is self or self.source_tag in item.get_tags()): + item.set_source(self) + + def retain_source_objects(self): + # Source-wide retention also protects children and cross-account shared-network dependencies. + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if self.source_tag in item.get_tags(): + item.set_source(self) + + def retain_inaccessible_objects(self): + self.retain_source_objects() + log.warning( + f"AWS source '{self.name}': authorization-denied discovery is incomplete. " + "Retaining all previously managed source objects this run; accessible resources will still be updated.") + + def defer_inaccessible_network_instances(self, snapshots): + if not any(service in ("ec2", "account", "sts", "organizations") + for _, _, service in getattr(self.client, "access_denied", [])): + return + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + retained = [] + for instance in data["instances"]: + missing_network = False + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is not None and (eni["VpcId"] not in data["Vpcs"] or + eni["SubnetId"] not in data["Subnets"]): + missing_network = True + if missing_network: + log.warning( + f"AWS instance '{instance['InstanceId']}' (account '{account}', region '{region}') " + "has missing network dependencies during authorization-denied discovery. " + "Deferring its update and retaining existing objects.") + else: + retained.append(instance) + data["instances"] = retained + + def resource_name(self, name, identity, limit): + if len(name) <= limit: + return name + suffix = sha256(identity.encode("utf-8")).hexdigest()[:12] + return f"{name[:limit - 13]}-{suffix}" + + def service_object(self, object_type, identity, data): + matches = [item for item in self.inventory.get_all_items(object_type) + if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + error = None + if len(matches) > 1: + error = f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'" + elif matches: + item = matches[0] + if item.source is not self and self.source_tag not in item.get_tags(): + error = f"AWS resource '{identity}' belongs to another source" + else: + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" + if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) + for item in self.inventory.get_all_items(object_type)): + error = f"NetBox {object_type.name} name collision for AWS resource '{identity}'" + if error is not None: + log.error( + f"AWS source '{self.name}': {error}. Skipping this resource and any dependent resources. " + "Retaining all previously managed source objects this run to prevent pruning skipped resources.") + self.retain_source_objects() + return None + if matches: + item.update(data=data, source=self) + else: + item = self.inventory.add_object(object_type, data=data, source=self) + self.reconcile_tags(item, data["tags"]) + return item + + def service_cluster(self, account, region, site, group, service, identity, name, + metadata=None, tags=None): + data = { + "name": self.resource_name(name, identity, 100), "group": group, + "type": {"name": f"AWS {service.upper() if service != 'lambda' else 'Lambda'}"}, + "custom_fields": {"aws_service": service, "aws_resource_id": identity, + "aws_metadata": metadata or {"account": account, "region": region}}, + "tags": self.service_tags(tags or {}), + } + if Version(self.inventory.netbox_api_version) >= Version("4.2"): + data.update(scope_type="dcim.site", scope_id=site) + else: + data["site"] = site + return self.service_object(NBCluster, identity, data) + + def logical_vm(self, account, region, site, cluster, service, identity, name, metadata, tags, **attributes): + data = { + "name": self.resource_name(name, identity, 64), "cluster": cluster, "status": "active", + "comments": f"Logical AWS {service} inventory object; not an EC2 instance.", + "custom_fields": { + "aws_account_id": account, "aws_region": region, "aws_service": service, + "aws_resource_id": identity, "aws_metadata": metadata, + }, + "tags": [f"AWS:logical-{service}", *self.service_tags(tags)], + **attributes, + } + if Version(self.inventory.netbox_api_version) >= Version("3.3"): + data["site"] = site + if self.settings.tenant: + data["tenant"] = {"name": self.settings.tenant} + return self.service_object(NBVM, identity, data) + + def sync_lambda(self, account, region, site, cluster, function): + status = {"Pending": "planned", "Active": "active", "Inactive": "offline", "Failed": "failed"} + self.logical_vm( + account, region, site, cluster, "lambda", function["FunctionArn"], + f"Lambda {function['FunctionName']}", + {key: value for key, value in function.items() if key != "Tags"}, + function["Tags"], memory=function["MemorySize"], status=status[function["State"]]) + + def sync_s3(self, account, region, site, group, buckets): + if not buckets: + return + identity = f"s3/{account}/{region}" + cluster = self.service_cluster( + account, region, site, group, "s3", identity, f"S3 {account}/{region}") + if cluster is None: + return + holder = self.logical_vm( + account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", + {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) + if holder is None: + return + for bucket in buckets: + bucket_id = f"s3/{account}/{bucket['Name']}" + self.service_object(NBVirtualDisk, bucket_id, { + "name": bucket["Name"], "virtual_machine": holder, + # NetBox requires size; zero is explicitly unknown, never a usage measurement. + "size": 0, "description": "AWS S3 bucket; usage unknown (size 0 is a placeholder, not measured).", + "custom_fields": { + "aws_service": "s3", "aws_resource_id": bucket_id, + "aws_metadata": {**{key: value for key, value in bucket.items() if key != "Tags"}, + "usage_status": "unknown", "usage_bytes": None}, + }, + "tags": self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}), + }) + + def resolve_shared_networks(self, snapshots): + networks = {} + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + for kind in ("Vpcs", "Subnets"): + for resource_id, resource in data[kind].items(): + owner = resource.get("OwnerId", account) + key = (kind, region, resource_id) + owners = networks.setdefault(key, {}) + if resource.get("OwnerId") and any( + item.get("OwnerId") and previous_owner != owner + for previous_owner, item in owners.items()): + raise ValueError( + f"Conflicting AWS {kind} ownership for '{resource_id}' " + f"(region '{region}'): {', '.join(sorted(set(owners) | {owner}))}") + if owner in owners: + previous = owners[owner] + fields = ("VpcId", "CidrBlock") + if any(previous.get(field) != resource.get(field) for field in fields) or \ + set(self.ipv6_networks(previous)) != set(self.ipv6_networks(resource)): + raise ValueError( + f"Conflicting AWS {kind} discovery for '{resource_id}' " + f"(owner '{owner}', region '{region}')") + owners[owner] = resource + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + data["VpcOwners"] = {} + for instance in data["instances"]: + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + continue + vpc_id, subnet_id = eni["VpcId"], eni["SubnetId"] + vpcs = networks.get(("Vpcs", region, vpc_id), {}) + local_vpc = data["Vpcs"].get(vpc_id) + owner = local_vpc.get("OwnerId", account) if local_vpc is not None else None + if owner is None and len(vpcs) == 1: + owner = next(iter(vpcs)) + if owner is None and vpcs: + raise ValueError( + f"Ambiguous AWS VPC ownership for '{vpc_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}'): {', '.join(sorted(vpcs))}") + if owner is None: + continue + data["VpcOwners"][vpc_id] = owner + data["Vpcs"][vpc_id] = dict(vpcs[owner], OwnerId=owner) + subnets = networks.get(("Subnets", region, subnet_id), {}) + if subnets and owner not in subnets: + raise ValueError( + f"Conflicting AWS subnet ownership for '{subnet_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}', VPC owner '{owner}'): " + f"{', '.join(sorted(subnets))}") + if owner in subnets: + data["Subnets"][subnet_id] = dict(subnets[owner], OwnerId=owner) + + def vrf_name(self, account, region, vpc): + return self.settings.vpc_vrf_relation.get( + f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( + f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) + + def get_vrf(self, account, region, vpc): + account_region_vpc = f"{account}/{region}/{vpc}" + region_vpc = f"{region}/{vpc}" + has_mapping = account_region_vpc in self.settings.vpc_vrf_relation or \ + region_vpc in self.settings.vpc_vrf_relation + name = self.vrf_name(account, region, vpc) + vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) + if vrf is None: + if has_mapping: + raise ValueError(f"Create mapped NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") + vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) + return vrf + + def ipv6_networks(self, resource): + return [ + ip_network(entry["Ipv6CidrBlock"]) + for entry in resource.get("Ipv6CidrBlockAssociationSet", []) + if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" ] - data = {"instances": instances} - for operation, result_key, key in ( - ("describe_network_interfaces", "NetworkInterfaces", "NetworkInterfaceId"), - ("describe_volumes", "Volumes", "VolumeId"), - ("describe_subnets", "Subnets", "SubnetId"), - ("describe_vpcs", "Vpcs", "VpcId"), - ): - data[result_key] = { - item[key]: item - for page in ec2.get_paginator(operation).paginate() - for item in page.get(result_key, []) + + def addresses(self, eni, subnet): + networks = [ip_network(subnet["CidrBlock"]), *self.ipv6_networks(subnet)] + addresses = [ + (entry["PrivateIpAddress"], entry.get("Primary", False)) + for entry in eni.get("PrivateIpAddresses", []) + ] + addresses.extend( + (entry["Ipv6Address"], entry.get("IsPrimaryIpv6", False)) + for entry in eni.get("Ipv6Addresses", []) + ) + result = [] + for value, primary in addresses: + address = ip_address(value) + network = next((net for net in networks + if net.version == address.version and address in net), None) + if network is None: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' IP '{value}' has no matching AWS subnet") + value = f"{address}/{network.prefixlen}" + if self.settings.permitted_subnets.permitted(value, interface_name=eni["NetworkInterfaceId"]): + result.append((value, primary)) + return result + + def validate_snapshot(self, account, snapshot, vrfs=None): + if vrfs is None: + vrfs = {} + for region, data in snapshot.items(): + for instance in data["instances"]: + spec = data["InstanceTypes"].get(instance["InstanceType"]) + if spec is None: + raise ValueError(f"Missing AWS instance type '{instance['InstanceType']}'") + if "DefaultVCpus" not in spec["VCpuInfo"] or "SizeInMiB" not in spec["MemoryInfo"]: + raise ValueError(f"Incomplete AWS specifications for '{instance['InstanceType']}'") + for tag in instance.get("Tags", []): + if tag["Key"] in self.settings.tag_keys and len(f"AWS:{tag['Key']}={tag['Value']}") > 100: + raise ValueError(f"AWS tag '{tag['Key']}' exceeds the NetBox tag name limit (100 characters)") + for attachment in instance.get("BlockDeviceMappings", []): + volume_id = attachment.get("Ebs", {}).get("VolumeId") + if volume_id and volume_id not in data["Volumes"]: + raise ValueError(f"Missing attached EBS volume '{volume_id}'") + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + raise ValueError(f"Missing attached ENI '{reference['NetworkInterfaceId']}'") + if eni.get("Attachment", {}).get("InstanceId") != instance["InstanceId"]: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' changed attachment during discovery") + if not eni.get("MacAddress"): + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' has no MAC address") + if not self.settings.sync_private_ips: + continue + vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) + network_errors = [] + if vpc not in data["Vpcs"]: + network_errors.append(f"VPC '{vpc}' was not returned by DescribeVpcs") + if subnet is None: + network_errors.append( + f"subnet '{eni['SubnetId']}' was not returned by DescribeSubnets") + elif subnet["VpcId"] != vpc: + network_errors.append( + f"subnet '{eni['SubnetId']}' belongs to VPC '{subnet['VpcId']}', " + f"not ENI VPC '{vpc}'") + if network_errors: + raise ValueError( + f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}' " + f"(account '{account}', region '{region}', instance '{instance['InstanceId']}', " + f"VPC '{vpc}', subnet '{eni['SubnetId']}'): {'; '.join(network_errors)}") + if self.addresses(eni, subnet): + owner = data.get("VpcOwners", {}).get(vpc, account) + vrf = self.get_vrf(owner, region, vpc) + identity = (owner, region, vpc) + if vrf in vrfs and vrfs[vrf] != identity: + raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") + vrfs[vrf] = identity + + def custom_fields(self): + fields = ( + ("aws_account_id", "AWS account ID", [NBVM.object_type]), + ("aws_region", "AWS region", [NBVM.object_type]), + ("aws_instance_id", "AWS instance ID", [NBVM.object_type]), + ("aws_name", "AWS Name tag", [NBVM.object_type]), + ("aws_instance_type", "AWS instance type", [NBVM.object_type]), + ("aws_availability_zone", "AWS Availability Zone", [NBVM.object_type]), + ("aws_vpc_id", "AWS VPC ID", [NBVM.object_type, NBVMInterface.object_type]), + ("aws_eni_id", "AWS ENI ID", [NBVMInterface.object_type]), + ("aws_subnet_id", "AWS subnet ID", [NBVMInterface.object_type]), + ("aws_public_ipv4", "AWS public IPv4", [NBVMInterface.object_type]), + ) + for name, label, types in fields: + self.add_update_custom_field({ + "name": name, "label": label, "object_types": types, "type": "text", + "description": f"Reported by AWS ({self.name})", + }) + if any((self.settings.sync_s3, self.settings.sync_lambda, self.settings.sync_eks)): + types = [NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type] + for name, field_type in ( + ("aws_service", "text"), ("aws_resource_id", "text"), + ("aws_metadata", "json"), + ): + field = self.add_update_custom_field({ + "name": name, "label": name.replace("_", " ").upper(), + "object_types": types, "type": field_type, + "description": f"Reported by AWS ({self.name})", + }) + existing_type = field.data.get("type") + if isinstance(existing_type, dict): + existing_type = existing_type.get("value") + if existing_type != field_type: + raise ValueError( + f"NetBox custom field '{name}' must have type '{field_type}', not '{existing_type}'") + + def sync_snapshot(self, account, snapshot, account_name=None): + self.custom_fields() + account_name = account_name or account + group = self.inventory.add_update_object( + NBClusterGroup, data={"name": f"AWS {account_name}"}, source=self) + api_version = Version(self.inventory.netbox_api_version) + for region, data in snapshot.items(): + site = self.inventory.add_update_object( + NBSite, data={"name": f"AWS {region}"}, source=self) + cluster_data = { + "name": f"AWS {account_name}/{region}", "group": group, + "type": {"name": "AWS EC2"}, } - instance_types = {} - names = sorted({item["InstanceType"] for item in instances}) - for start in range(0, len(names), 100): - for page in ec2.get_paginator("describe_instance_types").paginate(InstanceTypes=names[start:start + 100]): - instance_types.update({item["InstanceType"]: item for item in page["InstanceTypes"]}) - data["InstanceTypes"] = instance_types - return data - - def discover_s3(self, client, region, account): - buckets = [] - for page in client.get_paginator("list_buckets").paginate(BucketRegion=region): - for bucket in page["Buckets"]: - with self.tolerate_access_denied(account, region, "s3", bucket["Name"]): - buckets.append(self.discover_s3_bucket(client, bucket, region, account)) - return buckets - - def discover_s3_bucket(self, client, bucket, region, account): - name = bucket["Name"] - bucket_region = bucket.get("BucketRegion") - if bucket_region is None: - location = client.get_bucket_location(Bucket=name, ExpectedBucketOwner=account) - bucket_region = location["LocationConstraint"] or "us-east-1" - if bucket_region == "EU": - bucket_region = "eu-west-1" - if bucket_region != region: - raise AWSDiscoveryError( - f"S3 bucket '{name}' returned for region '{region}' belongs to '{bucket_region}'") - request = {"Bucket": name, "ExpectedBucketOwner": account} - try: - tags = client.get_bucket_tagging(**request)["TagSet"] - except ClientError as exc: - if exc.response["Error"]["Code"] != "NoSuchTagSet": - raise - tags = [] - versioning = client.get_bucket_versioning(**request).get("Status", "Unversioned") - try: - encryption = client.get_bucket_encryption(**request)["ServerSideEncryptionConfiguration"] - except ClientError as exc: - if exc.response["Error"]["Code"] != "ServerSideEncryptionConfigurationNotFoundError": - raise - encryption = {} - return { - "Name": name, "Region": region, "Tags": tags, "Versioning": versioning, - "Encryption": encryption, "Arn": f"arn:{client.meta.partition}:s3:::{name}", - } + if api_version >= Version("4.2"): + cluster_data.update(scope_type="dcim.site", scope_id=site) + else: + cluster_data["site"] = site + cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) + eks_clusters = {} + inaccessible = data.get("InaccessibleServices", []) + if self.settings.sync_eks and "eks" not in inaccessible: + for resource in data["EKSClusters"]: + eks_cluster = self.service_cluster( + account, region, site, group, "eks", resource["arn"], + f"EKS {account}/{region}/{resource['name']}", + metadata={key: value for key, value in resource.items() if key != "tags"}, + tags=resource.get("tags", {})) + eks_clusters[resource["name"]] = eks_cluster + if self.settings.sync_lambda and "lambda" not in inaccessible: + lambda_cluster = self.service_cluster( + account, region, site, group, "lambda", f"lambda/{account}/{region}", + f"Lambda {account}/{region}") + if lambda_cluster is not None: + for function in data["Functions"]: + self.sync_lambda(account, region, site, lambda_cluster, function) + if self.settings.sync_s3 and "s3" not in inaccessible: + self.sync_s3(account, region, site, group, data["Buckets"]) + name_counts = {} + for instance in data["instances"]: + name = next((tag["Value"].strip() for tag in instance.get("Tags", []) + if tag["Key"] == "Name" and tag["Value"].strip()), instance["InstanceId"]) + key = name.casefold() + name_counts[key] = name_counts.get(key, 0) + 1 + for instance in data["instances"]: + instance_cluster = eks_clusters.get( + data.get("EKSWorkers", {}).get(instance["InstanceId"]), cluster) + if instance_cluster is None: + log.error( + f"AWS source '{self.name}': Skipping instance '{instance['InstanceId']}' " + f"(account '{account}', region '{region}') because its EKS cluster was skipped.") + continue + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + vm_name = tags.get("Name", "").strip() or instance["InstanceId"] + if name_counts[vm_name.casefold()] > 1: + vm_name = f"{vm_name} ({instance['InstanceId']})" + self.sync_instance( + account, region, instance, data, site, + instance_cluster, + api_version, vm_name=vm_name) - def discover_lambda(self, client): - functions = [] - for page in client.get_paginator("list_functions").paginate(): - for reference in page["Functions"]: - config = client.get_function_configuration(FunctionName=reference["FunctionArn"]) - # Do not retain environment variables or other unneeded configuration. - function = {key: config[key] for key in ( - "FunctionArn", "FunctionName", "MemorySize", "State", "Runtime", "Handler", - "Timeout", "LastModified", "Architectures", "PackageType", "VpcConfig", - ) if key in config} - if function["FunctionArn"] != reference["FunctionArn"]: - raise AWSDiscoveryError(f"Lambda function identity changed for '{reference['FunctionArn']}'") - function["Tags"] = client.list_tags(Resource=function["FunctionArn"]).get("Tags", {}) - functions.append(function) - return functions - - def discover_eks(self, client): - clusters = [] - for page in client.get_paginator("list_clusters").paginate(): - for name in page["clusters"]: - cluster = client.describe_cluster(name=name)["cluster"] - if cluster["name"] != name: - raise AWSDiscoveryError(f"EKS cluster identity changed for '{name}'") - groups = [] - for group_page in client.get_paginator("list_nodegroups").paginate(clusterName=name): - for group_name in group_page["nodegroups"]: - group = client.describe_nodegroup(clusterName=name, nodegroupName=group_name)["nodegroup"] - if group["clusterName"] != name or group["nodegroupName"] != group_name: - raise AWSDiscoveryError(f"EKS node group identity changed for '{group_name}'") - groups.append({key: group[key] for key in ( - "nodegroupName", "nodegroupArn", "status", "version", "capacityType", - "instanceTypes", "scalingConfig", "subnets", "resources", - ) if key in group}) - profiles = [] - for profile_page in client.get_paginator("list_fargate_profiles").paginate(clusterName=name): - for profile_name in profile_page["fargateProfileNames"]: - profile = client.describe_fargate_profile( - clusterName=name, fargateProfileName=profile_name)["fargateProfile"] - if profile["clusterName"] != name or profile["fargateProfileName"] != profile_name: - raise AWSDiscoveryError(f"EKS Fargate profile identity changed for '{profile_name}'") - profiles.append({key: profile[key] for key in ( - "fargateProfileName", "fargateProfileArn", "status", "subnets", "selectors", - ) if key in profile}) - clusters.append({ - **{key: cluster[key] for key in ( - "name", "arn", "status", "version", "endpoint", "resourcesVpcConfig", "tags", - ) if key in cluster}, - "nodegroups": sorted(groups, key=lambda group: group["nodegroupName"]), - "fargateProfiles": sorted(profiles, key=lambda profile: profile["fargateProfileName"]), + def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): + instance_id = instance["InstanceId"] + if "eks" in data.get("InaccessibleServices", []): + existing = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if existing is not None and existing.data.get("cluster") is not None: + cluster = existing.data["cluster"] + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + spec = data["InstanceTypes"][instance["InstanceType"]] + vm_data = { + "name": vm_name or tags.get("Name", "").strip() or instance_id, "cluster": cluster, + "status": "active" if instance["State"]["Name"] == "running" else "offline", + "vcpus": spec["VCpuInfo"]["DefaultVCpus"], + "memory": spec["MemoryInfo"]["SizeInMiB"], + "custom_fields": { + "aws_account_id": account, "aws_region": region, + "aws_instance_id": instance_id, "aws_name": tags.get("Name", ""), + "aws_instance_type": instance["InstanceType"], + "aws_availability_zone": instance["Placement"]["AvailabilityZone"], + "aws_vpc_id": instance.get("VpcId", ""), + }, + "tags": [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags], + } + cpu = instance.get("CpuOptions", {}) + if cpu.get("CoreCount") is not None and cpu.get("ThreadsPerCore") is not None: + vm_data["vcpus"] = cpu["CoreCount"] * cpu["ThreadsPerCore"] + if api_version >= Version("3.3"): + vm_data["site"] = site + if api_version >= Version("4.1"): + vm_data["serial"] = instance_id + if self.settings.tenant: + vm_data["tenant"] = {"name": self.settings.tenant} + disks = [ + data["Volumes"][entry["Ebs"]["VolumeId"]] + for entry in instance.get("BlockDeviceMappings", []) if "Ebs" in entry + ] + if api_version < Version("3.7"): + vm_data["disk"] = sum(volume["Size"] for volume in disks) + vm = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if vm is None: + vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) + else: + vm.update(data=vm_data, source=self) + self.reconcile_tags(vm, vm_data["tags"]) + if api_version >= Version("3.7"): + for volume in disks: + self.add_update_vm_child(NBVirtualDisk, vm, { + "name": volume["VolumeId"], "virtual_machine": vm, + "size": volume["Size"] * (1024 if api_version >= Version("4.1") else 1), + "description": f"AWS EBS {volume['VolumeType']}", }) - return clusters + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"][reference["NetworkInterfaceId"]] + self.sync_interface(account, region, vm, eni, data, api_version) + + def add_update_vm_child(self, object_type, vm, data): + # Display-name matching loses the VM's account/cluster when names repeat. + existing = next((item for item in self.inventory.get_all_items(object_type) + if item.data.get("virtual_machine") is vm and + item.data.get("name") == data["name"]), None) + if existing is None: + return self.inventory.add_object(object_type, data=data, source=self) + existing.update(data=data, source=self) + return existing + + def sync_interface(self, account, region, vm, eni, data, api_version): + interface_data = { + "name": eni["NetworkInterfaceId"], "virtual_machine": vm, "enabled": True, + "description": eni.get("Description", ""), + "custom_fields": { + "aws_eni_id": eni["NetworkInterfaceId"], "aws_vpc_id": eni.get("VpcId", ""), + "aws_subnet_id": eni.get("SubnetId", ""), + "aws_public_ipv4": ", ".join(sorted({ + address["Association"]["PublicIp"] + for address in eni.get("PrivateIpAddresses", []) + if address.get("Association", {}).get("PublicIp") + })), + }, + } + mac = normalize_mac_address(eni.get("MacAddress")) + if api_version < Version("4.2"): + interface_data["mac_address"] = mac + interface = self.add_update_vm_child(NBVMInterface, vm, interface_data) + if api_version >= Version("4.2") and mac is not None: + mac_object = next((item for item in self.inventory.get_all_items(NBMACAddress) + if item.data.get("mac_address") == mac and + item.data.get("assigned_object_id") in (None, interface)), None) + mac_data = {"mac_address": mac, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if mac_object is None: + mac_object = self.inventory.add_object(NBMACAddress, data=mac_data, source=self) + else: + mac_object.update(data=mac_data, source=self) + for old in interface.get_mac_addresses(): + if old is not mac_object: + old.remove_interface_association() + interface.update(data={"primary_mac_address": mac_object}, source=self) + if not self.settings.sync_private_ips: + for address in interface.get_ip_addresses(): + if self.source_tag in address.get_tags(): + address.set_source(self) + return + subnet = data["Subnets"][eni["SubnetId"]] + addresses = self.addresses(eni, subnet) + discovered = [] + for address, primary in addresses: + owner = data.get("VpcOwners", {}).get(eni["VpcId"], account) + vrf = self.get_vrf(owner, region, eni["VpcId"]) + parsed = ip_interface(address) + candidates = [ + item for item in self.inventory.get_all_items(NBIPAddress) + if ip_interface(item.data["address"]).ip == parsed.ip and item.data.get("vrf") is vrf + ] + if len(candidates) > 1: + raise ValueError(f"Multiple NetBox IPs match '{address}' in VRF '{vrf.data['name']}'") + existing = candidates[0] if candidates else None + if existing is not None and existing.data.get("assigned_object_id") is not None and \ + existing.get_interface() is not interface: + owner = existing.get_device_vm() + owner_tags = owner.get_tags() if owner is not None else [] + same_source = owner is not None and ( + owner.source is self or self.source_tag in owner_tags) + managed_owner = same_source or NetBoxHandler.primary_tag in owner_tags + managed_orphan = NetBoxHandler.primary_tag in owner_tags and \ + NetBoxHandler.orphaned_tag in owner_tags + owner_status = owner.data.get("status") if owner is not None else None + if isinstance(owner_status, dict): + owner_status = owner_status.get("value") + if managed_owner and owner_status == "active" and vm.data["status"] == "offline": + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"is assigned to active owner '{owner.get_display_name()}'. " + f"Skipping reassignment to offline ENI '{eni['NetworkInterfaceId']}'.") + continue + offline_to_active = managed_owner and owner_status == "offline" and vm.data["status"] == "active" + if not (same_source or managed_orphan or offline_to_active): + owner_name = owner.get_display_name() if owner is not None else "unresolved owner" + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"belongs to another source (owner '{owner_name}'). " + f"Skipping reassignment to ENI '{eni['NetworkInterfaceId']}'.") + continue + if managed_orphan or offline_to_active: + reason = "orphaned" if managed_orphan else "offline" + log.warning( + f"AWS source '{self.name}': Reclaiming IP '{address}' in VRF '{vrf.data['name']}' " + f"from {reason} owner '{owner.get_display_name()}' " + f"for ENI '{eni['NetworkInterfaceId']}'.") + ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if self.settings.tenant: + ip_data["tenant"] = {"name": self.settings.tenant} + if existing is None: + existing = self.inventory.add_object(NBIPAddress, data=ip_data, source=self) + else: + existing.update(data=ip_data, source=self) + discovered.append(existing) + if eni.get("Attachment", {}).get("DeviceIndex") == 0 and primary: + vm.update(data={f"primary_ip{parsed.version}": existing}, source=self) + for old in interface.get_ip_addresses(): + if old not in discovered and (old.source is self or self.source_tag in old.get_tags()): + old.remove_interface_association() From 68314b7feba0748fb6ff3f5943b3aa8df80f8153 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 28 Oct 2026 13:19:18 -0500 Subject: [PATCH 26/28] Refactor AWS service tests and add conflict handling Refactor AWS service tests to improve clarity and functionality. Added new tests for handling conflicts and access tracking. --- tests/test_aws_services.py | 213 ++++++++++++++++++++++++++++++++++++- 1 file changed, 209 insertions(+), 4 deletions(-) diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py index 03b71281..01cde538 100644 --- a/tests/test_aws_services.py +++ b/tests/test_aws_services.py @@ -7,10 +7,12 @@ boto3 = pytest.importorskip("boto3") from botocore.stub import Stubber -from module.netbox import NBCustomField, NBCluster, NBIPAddress, NBTag, NBVirtualDisk, NBVM, NBVMInterface +from module.netbox import ( + NBCustomField, NBCluster, NBClusterGroup, NBIPAddress, NBTag, NBVirtualDisk, NBVM, NBVMInterface, +) from module.sources import instantiate_sources from module.sources.aws import client as aws_client -from module.sources.aws.client import AWSClient, AWSDiscoveryError +from module.sources.aws.client import AWSAccessCheck, AWSClient, AWSDiscoveryError from test_aws_source import ( ACCOUNT, INSTANCE, REGION, SECOND_ACCOUNT, VPC, assume_member, identity, inventory_data, make_source, org_sdk, organization_accounts, stub_region, windows_config_paths, @@ -186,15 +188,218 @@ def test_long_names_and_same_lambda_names_across_regions(make_source, inventory) assert len({vm.data["name"] for vm in functions}) == 2 -def test_services_do_not_overwrite_unmanaged_objects(make_source, inventory): +def test_services_do_not_overwrite_unmanaged_objects(make_source, inventory, caplog): data = service_data() source = make_source("sync_lambda = True", data=data) source.apply() function = next(vm for vm in inventory.get_all_items(NBVM) if vm.data["custom_fields"].get("aws_service") == "lambda") function.source = None - with pytest.raises(SystemExit): + before = copy.deepcopy(function.data["custom_fields"]) + data["Functions"][0]["MemorySize"] = 256 + source.apply() + assert "belongs to another source" in caplog.text + assert function.source is None + assert function.data["custom_fields"] == before + assert function.data["memory"] == 128 + + +@pytest.mark.parametrize("conflict", ["name", "duplicate_identity", "other_source"]) +def test_lambda_cluster_conflict_continues_and_retains_children( + make_source, inventory, caplog, conflict): + source = make_source("sync_lambda = True\nsync_private_ips = False", data=service_data()) + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": f"AWS {ACCOUNT}"}) + fields = {} if conflict == "name" else {"aws_resource_id": f"lambda/{ACCOUNT}/{REGION}"} + cluster = inventory.add_object(NBCluster, data={ + "name": f"Lambda {ACCOUNT}/{REGION}", "group": group, "custom_fields": fields, + }) + if conflict == "duplicate_identity": + inventory.add_object(NBCluster, data={ + "name": "Duplicate", "group": group, "custom_fields": fields, + }) + retained = inventory.add_object(NBVM, data={ + "name": "Existing function", "cluster": cluster, "tags": [source.source_tag], + }) + source.apply() + inventory.tag_all_the_things(make_netbox_handler()) + assert "Skipping" in caplog.text + assert retained.source is source + assert make_netbox_handler().orphaned_tag not in retained.get_tags() + assert cluster.source is None + assert cluster.data.get("custom_fields", {}) == fields + assert any(vm.data.get("custom_fields", {}).get("aws_instance_id") == INSTANCE + for vm in inventory.get_all_items(NBVM)) + assert not any(vm.data.get("custom_fields", {}).get("aws_resource_id") == FUNCTION_ARN + for vm in inventory.get_all_items(NBVM)) + + +@pytest.mark.parametrize("stage", ["cluster", "holder", "bucket"]) +def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, caplog, stage): + data = service_data() + data["Buckets"].append({**data["Buckets"][0], "Name": "second"}) + source = make_source("sync_s3 = True\nsync_private_ips = False", data=data) + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": f"AWS {ACCOUNT}"}) + identity = f"s3/{ACCOUNT}/{REGION}" + cluster = inventory.add_object(NBCluster, data={ + "name": f"S3 {ACCOUNT}/{REGION}", "group": group, + "custom_fields": {} if stage == "cluster" else {"aws_resource_id": identity}, + }, source=source) + if stage != "cluster": + holder = inventory.add_object(NBVM, data={ + "name": f"S3 inventory {ACCOUNT}/{REGION}", "cluster": cluster, + "custom_fields": {} if stage == "holder" else {"aws_resource_id": identity}, + }, source=source) + if stage == "bucket": + inventory.add_object(NBVirtualDisk, data={ + "name": "inventory-bucket", "virtual_machine": holder, "size": 0, + }) + source.apply() + assert "name collision" in caplog.text + buckets = [disk.data["name"] for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data.get("custom_fields", {}).get("aws_service") == "s3"] + assert buckets == (["second"] if stage == "bucket" else []) + assert any(vm.data.get("custom_fields", {}).get("aws_instance_id") == INSTANCE + for vm in inventory.get_all_items(NBVM)) + + +def test_eks_conflict_skips_workers_and_continues_next_region(make_source, inventory, caplog): + data = service_data() + source = make_source("sync_eks = True\nsync_private_ips = False", data=data) + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": f"AWS {ACCOUNT}"}) + inventory.add_object(NBCluster, data={ + "name": f"EKS {ACCOUNT}/{REGION}/inventory-cluster", "group": group, + }) + next_region = inventory_data() + next_region["EKSClusters"] = [] + next_region["instances"][0]["InstanceId"] = "i-next-region" + for eni in next_region["NetworkInterfaces"].values(): + eni["Attachment"]["InstanceId"] = "i-next-region" + source.client.discover.return_value = ACCOUNT, {REGION: data, "us-west-2": next_region} + source.apply() + assert f"Skipping instance '{INSTANCE}'" in caplog.text + instances = [vm.data.get("custom_fields", {}).get("aws_instance_id") + for vm in inventory.get_all_items(NBVM)] + assert INSTANCE not in instances + assert "i-next-region" in instances + + +def test_lambda_function_conflict_continues_next_function(make_source, inventory, caplog): + data = service_data() + source = make_source("sync_lambda = True\nsync_private_ips = False", data=data) + source.apply() + cluster = next(item for item in inventory.get_all_items(NBCluster) + if item.data.get("custom_fields", {}).get("aws_service") == "lambda") + inventory.add_object(NBVM, data={"name": "Lambda second", "cluster": cluster}) + data["Functions"].extend([ + {**data["Functions"][0], "FunctionName": name, "FunctionArn": f"{FUNCTION_ARN}-{name}"} + for name in ("second", "third") + ]) + source.apply() + assert "name collision" in caplog.text + assert any(vm.data["name"] == "Lambda third" for vm in inventory.get_all_items(NBVM)) + + +def test_access_tracking_replaces_scope_and_records_operations(service_sdk): + client = AWSClient() + sts = service_sdk["sts"].client + client.track_access(sts, ACCOUNT, None, "sts") + client.track_access(sts, SECOND_ACCOUNT, None, "sts") + params = {"RoleArn": f"arn:aws:iam::{SECOND_ACCOUNT}:role/reports", "RoleSessionName": "test"} + service_sdk["sts"].add_client_error("assume_role", service_error_code="AccessDenied", expected_params=params) + with client.tolerate_access_denied(SECOND_ACCOUNT, service="sts"): + sts.assume_role(**params) + assert client.access_checks == [ + AWSAccessCheck(SECOND_ACCOUNT, None, "sts", None, "AssumeRole", True)] + + +@pytest.mark.parametrize("scope", ["resource", "service", "account", "organization"]) +def test_permission_tags_require_verified_recovery(make_source, inventory, scope): + source = make_source("sync_s3 = True", data=service_data()) + source.apply() + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + bucket.add_tags("user-tag") + unmanaged = inventory.add_object(NBVirtualDisk, data={ + "name": "unmanaged", "virtual_machine": bucket.data["virtual_machine"], "size": 0, + }) + account = None if scope == "organization" else ACCOUNT + region = REGION if scope in ("resource", "service") else None + service = {"resource": "s3", "service": "s3", "account": "sts", + "organization": "organizations"}[scope] + resource = "inventory-bucket" if scope == "resource" else None + operation = "GetBucketTagging" + tag = f"Sync:AccessDenied:{operation}" + failure = AWSAccessCheck(account, region, service, resource, operation, True) + success = AWSAccessCheck(account, region, service, resource, operation, False) + source.client.access_checks = [failure] + source.reconcile_access_tags() + assert tag in bucket.get_tags() + assert tag not in unmanaged.get_tags() + for checks in ([], [AWSAccessCheck(ACCOUNT, REGION, "s3", "other", operation, False)], + [success, failure]): + source.client.access_checks = checks + source.reconcile_access_tags() + assert tag in bucket.get_tags() + source.client.access_checks = [success] + source.reconcile_access_tags() + assert tag not in bucket.get_tags() + assert "user-tag" in bucket.get_tags() + + +def test_permission_tags_do_not_leak_across_scopes(make_source, inventory): + source = make_source("sync_s3 = True", data=service_data()) + source.custom_fields() + objects = [] + for account, region, service in ( + (ACCOUNT, REGION, "s3"), (SECOND_ACCOUNT, REGION, "s3"), + (ACCOUNT, "us-west-2", "s3"), (ACCOUNT, REGION, "lambda")): + objects.append(inventory.add_object(NBVM, data={ + "name": f"{account}-{region}-{service}", "tags": [source.source_tag], + "custom_fields": { + "aws_account_id": account, "aws_region": region, "aws_service": service, + }, + })) + source.client.access_checks = [AWSAccessCheck(ACCOUNT, REGION, "s3", None, "ListBuckets", True)] + source.reconcile_access_tags() + assert ["Sync:AccessDenied:ListBuckets" in item.get_tags() for item in objects] == [ + True, False, False, False] + + +def test_bucket_permission_tag_through_sdk_denial_and_recovery(service_sdk, inventory, load_config): + load_config(f""" +[source/services] +type = aws +regions = {REGION} +sync_s3 = True +sync_private_ips = False +""") + source, = instantiate_sources() + tag = "Sync:AccessDenied:GetBucketTagging" + for denied in (False, True, False): + identity(service_sdk["sts"]) + stub_region(service_sdk["ec2"], inventory_data()) + if denied: + service_sdk["s3"].add_response("list_buckets", {"Buckets": [{ + "Name": "inventory-bucket", "BucketRegion": REGION, + }]}, {"BucketRegion": REGION}) + service_sdk["s3"].add_client_error("get_bucket_tagging", service_error_code="AccessDenied", + expected_params={ + "Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT, + }) + else: + stub_s3(service_sdk["s3"]) source.apply() + inventory.tag_all_the_things(make_netbox_handler()) + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + assert (tag in bucket.get_tags()) is denied + assert make_netbox_handler().orphaned_tag not in bucket.get_tags() + assert any(check.operation == "GetBucketTagging" and check.resource == "inventory-bucket" + and check.denied is denied for check in source.client.access_checks) + bucket.source = None @pytest.mark.parametrize("failure", ["arn", "membership", "tag", "state"]) From 107c1f2866e805b2814c8823b8d51b0facdb5712 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 28 Oct 2026 23:43:03 -0500 Subject: [PATCH 27/28] bad paste correction. replacing file contents. --- module/sources/aws/client.py | 1154 ++++++++++------------------------ 1 file changed, 343 insertions(+), 811 deletions(-) diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py index c3559320..38002b6f 100644 --- a/module/sources/aws/client.py +++ b/module/sources/aws/client.py @@ -1,827 +1,359 @@ -from hashlib import sha256 -from ipaddress import ip_address, ip_interface, ip_network -from itertools import product - -from packaging.version import Version +from contextlib import contextmanager +from dataclasses import dataclass from module.common.logging import get_logger -from module.common.misc import do_error_exit -from module.common.support import normalize_mac_address -from module.netbox import ( - NBCustomField, NBCluster, NBClusterGroup, NBClusterType, NBIPAddress, - NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, -) -from module.netbox.inventory import NetBoxInventory -from module.netbox.connection import NetBoxHandler -from module.sources.aws.client import AWSClient, AWSDiscoveryError -from module.sources.aws.config import AWSConfig -from module.sources.common.source_base import SourceBase log = get_logger() +try: + import boto3 + from botocore.config import Config + from botocore.exceptions import BotoCoreError, ClientError +except ModuleNotFoundError as exc: + if exc.name != "boto3": + raise + boto3 = None -class AWSHandler(SourceBase): - source_type = "aws" - dependent_netbox_objects = [ - NBTag, NBCustomField, NBTenant, NBSite, NBClusterGroup, NBClusterType, - NBCluster, NBVM, NBVMInterface, NBMACAddress, NBIPAddress, NBVRF, NBVirtualDisk, - ] - - def __init__(self, name=None): - if not name: - raise ValueError(f"Invalid source name '{name}'.") - self.name = name - self.inventory = NetBoxInventory() - config = AWSConfig() - config.source_name = name - self.settings = config.parse() - self.set_source_tag() - self.init_successful = False - if not self.settings.enabled: - log.info(f"Source '{name}' is disabled. Skipping.") - return - try: - self.client = AWSClient(profile=self.settings.profile) - except AWSDiscoveryError as exc: - self.abort(exc) - self.init_successful = True - - def abort(self, error): - message = f"AWS source '{self.name}': {error}. No NetBox writes or pruning will be performed." - log.error(message) - do_error_exit(message) - - def apply(self): - try: - if self.settings.sync_s3 and Version(self.inventory.netbox_api_version) < Version("3.7"): - raise ValueError("S3 bucket inventory requires NetBox 3.7 or newer") - services = tuple(service for service in ("s3", "lambda", "eks") - if getattr(self.settings, f"sync_{service}")) - options = {"services": services} if services else {} - if self.settings.organization: - snapshots = self.client.discover_organization( - self.settings.regions, self.settings.organization_role_name, - expected_account=self.settings.account_id, - discovery_region=self.settings.discovery_region, - include_management=self.settings.organization_include_management, - external_id=self.settings.organization_external_id, - account_ids=self.settings.organization_account_ids, **options) - account_names = getattr(self.client, "account_names", {}) - else: - account, snapshot = self.client.discover( - self.settings.regions, self.settings.account_id, self.settings.discovery_region, **options) - snapshots = {account: snapshot} if account is not None else {} - account_names = {} - if self.settings.organization and self.settings.sync_private_ips: - self.resolve_shared_networks(snapshots) - self.defer_inaccessible_network_instances(snapshots) - vrfs = {} - for account, snapshot in snapshots.items(): - self.validate_services(account, snapshot) - self.validate_snapshot(account, snapshot, vrfs) - self.retain_disabled_services() - if getattr(self.client, "access_denied", []): - self.retain_inaccessible_objects() - account_labels = { - account: account_names.get(account) or self.settings.account_name or account - for account in snapshots - } - label_counts = {} - for label in account_labels.values(): - key = label.casefold() - label_counts[key] = label_counts.get(key, 0) + 1 - for account, label in account_labels.items(): - if label_counts[label.casefold()] > 1: - account_labels[account] = f"{label} ({account})" - for account, snapshot in snapshots.items(): - self.sync_snapshot(account, snapshot, account_name=account_labels[account]) - self.reconcile_access_tags() - except (AWSDiscoveryError, ValueError, KeyError) as exc: - self.abort(exc) - - def validate_services(self, account, snapshot): - for region, data in snapshot.items(): - inaccessible = data.get("InaccessibleServices", []) - for enabled, key in ( - (self.settings.sync_s3 and "s3" not in inaccessible, "Buckets"), - (self.settings.sync_lambda and "lambda" not in inaccessible, "Functions"), - (self.settings.sync_eks and "eks" not in inaccessible, "EKSClusters"), - ): - if enabled and not isinstance(data.get(key), list): - raise ValueError(f"Missing AWS {key} discovery (account '{account}', region '{region}')") - if enabled: - identity_key = {"Buckets": "Name", "Functions": "FunctionArn", "EKSClusters": "arn"}[key] - identities = [resource[identity_key] for resource in data[key]] - if len(set(identities)) != len(identities): - raise ValueError(f"Duplicate AWS {key} discovery (account '{account}', region '{region}')") - for bucket in data["Buckets"] if self.settings.sync_s3 and "s3" not in inaccessible else []: - if bucket["Region"] != region or not bucket["Name"] or len(bucket["Name"]) > 63: - raise ValueError(f"Invalid S3 bucket '{bucket['Name']}' for region '{region}'") - self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}) - if "Versioning" not in bucket or "Encryption" not in bucket: - raise ValueError(f"Incomplete S3 bucket metadata for '{bucket['Name']}'") - for function in data["Functions"] if self.settings.sync_lambda and "lambda" not in inaccessible else []: - self.validate_arn(function["FunctionArn"], "lambda", account, region) - if function["State"] not in ("Pending", "Active", "Inactive", "Failed"): - raise ValueError(f"Unknown Lambda state '{function['State']}' for '{function['FunctionArn']}'") - if not function["FunctionName"] or not isinstance(function["MemorySize"], int): - raise ValueError(f"Incomplete Lambda configuration for '{function['FunctionArn']}'") - self.service_tags(function["Tags"]) - if self.settings.sync_eks and "eks" not in inaccessible: - clusters = {cluster["name"]: cluster for cluster in data["EKSClusters"]} - if len(clusters) != len(data["EKSClusters"]): - raise ValueError(f"Duplicate EKS cluster discovery in account '{account}', region '{region}'") - asg_clusters = {} - for cluster in data["EKSClusters"]: - self.validate_arn(cluster["arn"], "eks", account, region) - for field in ("status", "version", "resourcesVpcConfig", "nodegroups", "fargateProfiles"): - if field not in cluster: - raise ValueError(f"Incomplete EKS {field} for '{cluster['arn']}'") - self.service_tags(cluster.get("tags", {})) - for group in cluster["nodegroups"]: - for asg in group.get("resources", {}).get("autoScalingGroups", []): - name = asg["name"] - if name in asg_clusters and asg_clusters[name] != cluster["name"]: - raise ValueError(f"EKS Auto Scaling group '{name}' belongs to multiple clusters") - asg_clusters[name] = cluster["name"] - data["EKSWorkers"] = {} - for instance in data["instances"]: - tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} - names = {tags[key] for key in ( - "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", - ) if tags.get(key)} - asg_cluster = asg_clusters.get(tags.get("aws:autoscaling:groupName")) - if asg_cluster: - names.add(asg_cluster) - if len(names) > 1: - raise ValueError(f"Conflicting EKS cluster membership for instance '{instance['InstanceId']}'") - if names: - name = next(iter(names)) - if name in clusters: - data["EKSWorkers"][instance["InstanceId"]] = name - else: - log.warning( - f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " - f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") - - def access_scope(self, item): - fields = item.data.get("custom_fields") or {} - metadata = fields.get("aws_metadata") or {} - identity = fields.get("aws_resource_id") or "" - service = "ec2" if fields.get("aws_instance_id") else fields.get("aws_service") - account = fields.get("aws_account_id") or metadata.get("account") - region = fields.get("aws_region") or metadata.get("region") - resource = None - if identity.startswith("arn:"): - parts = identity.split(":", 5) - if len(parts) == 6: - service, region, account, resource = parts[2:] - if service == "lambda": - resource = identity - elif service == "eks": - resource = resource.removeprefix("cluster/") - elif identity.startswith("s3/"): - parts = identity.split("/") - account = parts[1] - if isinstance(item, NBVirtualDisk): - resource = parts[2] - else: - region = parts[2] - for parent_key in ("virtual_machine", "cluster"): - parent = item.data.get(parent_key) - if parent is not None: - parent_account, parent_region, parent_service, _ = self.access_scope(parent) - account = account or parent_account - region = region or parent_region - service = service or parent_service - break - if isinstance(item, (NBIPAddress, NBMACAddress)): - parent = item.get_device_vm() - if parent is not None: - account, region, service, _ = self.access_scope(parent) - if account is None: - group = item if isinstance(item, NBClusterGroup) else item.data.get("group") - name = group.data.get("name", "") if group is not None else "" - if name.startswith("AWS "): - label = name[4:] - account = next((key for key, value in getattr(self.client, "account_names", {}).items() - if label in (value, f"{value} ({key})")), None) - if account is None and label.isdigit(): - account = label - if region is None: - site = item.data.get("scope_id") or item.data.get("site") - name = site.data.get("name", "") if isinstance(site, NBSite) else "" - if name.startswith("AWS "): - region = name[4:] - return account, region, service or "ec2", resource - - def reconcile_access_tags(self): - checks = getattr(self.client, "access_checks", []) - if not checks: - return - prefix = "Sync:AccessDenied:" - scopes = {} - for check in checks: - outcomes = scopes.setdefault( - (check.account, check.region, check.service, check.resource), (set(), set())) - outcomes[0 if check.denied else 1].add(check.operation) - for object_type in self.dependent_netbox_objects: - for item in self.inventory.get_all_items(object_type): - if object_type in (NBTag, NBCustomField, NBTenant) or "tags" not in item.data_model or not ( - item.source is self or self.source_tag in item.get_tags()): - continue - account, region, service, resource = self.access_scope(item) - denied, succeeded = set(), set() - for key in product( - (None, account), (None, region), ("organizations", "sts", "account", service), - (None, resource)): - if key in scopes: - failures, successes = scopes[key] - denied.update(failures) - succeeded.update(successes) - for operation in denied: - item.add_tags(f"{prefix}{operation}") - for tag in item.get_tags(): - if tag.startswith(prefix) and tag[len(prefix):] in succeeded - denied: - item.remove_tags(tag) - - def validate_arn(self, arn, service, account, region): - parts = arn.split(":", 5) - if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: - raise ValueError(f"Invalid {service} resource ARN '{arn}' for account '{account}', region '{region}'") - - def service_tags(self, tags): - result = [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags] - if any(len(tag) > 100 for tag in result): - raise ValueError("AWS service tag exceeds the NetBox tag name limit (100 characters)") - return result - - def reconcile_tags(self, item, tags): - # Tag lookup can reuse an existing name with different capitalization. - desired_tags = {self.inventory.get_by_data(NBTag, data={"name": name}) for name in tags} - managed_prefixes = tuple(f"AWS:{key}=".casefold() for key in self.settings.tag_keys) - for tag in item.data.get("tags", []): - if tag.get_display_name().casefold().startswith(managed_prefixes) and tag not in desired_tags: - item.remove_tags(tag) - - def retain_disabled_services(self): - for object_type in (NBVM, NBVirtualDisk, NBCluster): - for item in self.inventory.get_all_items(object_type): - service = (item.data.get("custom_fields") or {}).get("aws_service") - if service in ("s3", "lambda", "eks") and not getattr(self.settings, f"sync_{service}") and \ - (item.source is self or self.source_tag in item.get_tags()): - item.set_source(self) - - def retain_source_objects(self): - # Source-wide retention also protects children and cross-account shared-network dependencies. - for object_type in self.dependent_netbox_objects: - for item in self.inventory.get_all_items(object_type): - if self.source_tag in item.get_tags(): - item.set_source(self) - - def retain_inaccessible_objects(self): - self.retain_source_objects() - log.warning( - f"AWS source '{self.name}': authorization-denied discovery is incomplete. " - "Retaining all previously managed source objects this run; accessible resources will still be updated.") - - def defer_inaccessible_network_instances(self, snapshots): - if not any(service in ("ec2", "account", "sts", "organizations") - for _, _, service in getattr(self.client, "access_denied", [])): - return - for account, snapshot in snapshots.items(): - for region, data in snapshot.items(): - retained = [] - for instance in data["instances"]: - missing_network = False - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) - if eni is not None and (eni["VpcId"] not in data["Vpcs"] or - eni["SubnetId"] not in data["Subnets"]): - missing_network = True - if missing_network: - log.warning( - f"AWS instance '{instance['InstanceId']}' (account '{account}', region '{region}') " - "has missing network dependencies during authorization-denied discovery. " - "Deferring its update and retaining existing objects.") - else: - retained.append(instance) - data["instances"] = retained - - def resource_name(self, name, identity, limit): - if len(name) <= limit: - return name - suffix = sha256(identity.encode("utf-8")).hexdigest()[:12] - return f"{name[:limit - 13]}-{suffix}" - - def service_object(self, object_type, identity, data): - matches = [item for item in self.inventory.get_all_items(object_type) - if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] - error = None - if len(matches) > 1: - error = f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'" - elif matches: - item = matches[0] - if item.source is not self and self.source_tag not in item.get_tags(): - error = f"AWS resource '{identity}' belongs to another source" - else: - parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" - if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) - for item in self.inventory.get_all_items(object_type)): - error = f"NetBox {object_type.name} name collision for AWS resource '{identity}'" - if error is not None: - log.error( - f"AWS source '{self.name}': {error}. Skipping this resource and any dependent resources. " - "Retaining all previously managed source objects this run to prevent pruning skipped resources.") - self.retain_source_objects() - return None - if matches: - item.update(data=data, source=self) - else: - item = self.inventory.add_object(object_type, data=data, source=self) - self.reconcile_tags(item, data["tags"]) - return item - - def service_cluster(self, account, region, site, group, service, identity, name, - metadata=None, tags=None): - data = { - "name": self.resource_name(name, identity, 100), "group": group, - "type": {"name": f"AWS {service.upper() if service != 'lambda' else 'Lambda'}"}, - "custom_fields": {"aws_service": service, "aws_resource_id": identity, - "aws_metadata": metadata or {"account": account, "region": region}}, - "tags": self.service_tags(tags or {}), - } - if Version(self.inventory.netbox_api_version) >= Version("4.2"): - data.update(scope_type="dcim.site", scope_id=site) - else: - data["site"] = site - return self.service_object(NBCluster, identity, data) - - def logical_vm(self, account, region, site, cluster, service, identity, name, metadata, tags, **attributes): - data = { - "name": self.resource_name(name, identity, 64), "cluster": cluster, "status": "active", - "comments": f"Logical AWS {service} inventory object; not an EC2 instance.", - "custom_fields": { - "aws_account_id": account, "aws_region": region, "aws_service": service, - "aws_resource_id": identity, "aws_metadata": metadata, - }, - "tags": [f"AWS:logical-{service}", *self.service_tags(tags)], - **attributes, - } - if Version(self.inventory.netbox_api_version) >= Version("3.3"): - data["site"] = site - if self.settings.tenant: - data["tenant"] = {"name": self.settings.tenant} - return self.service_object(NBVM, identity, data) - - def sync_lambda(self, account, region, site, cluster, function): - status = {"Pending": "planned", "Active": "active", "Inactive": "offline", "Failed": "failed"} - self.logical_vm( - account, region, site, cluster, "lambda", function["FunctionArn"], - f"Lambda {function['FunctionName']}", - {key: value for key, value in function.items() if key != "Tags"}, - function["Tags"], memory=function["MemorySize"], status=status[function["State"]]) - def sync_s3(self, account, region, site, group, buckets): - if not buckets: - return - identity = f"s3/{account}/{region}" - cluster = self.service_cluster( - account, region, site, group, "s3", identity, f"S3 {account}/{region}") - if cluster is None: - return - holder = self.logical_vm( - account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", - {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) - if holder is None: - return - for bucket in buckets: - bucket_id = f"s3/{account}/{bucket['Name']}" - self.service_object(NBVirtualDisk, bucket_id, { - "name": bucket["Name"], "virtual_machine": holder, - # NetBox requires size; zero is explicitly unknown, never a usage measurement. - "size": 0, "description": "AWS S3 bucket; usage unknown (size 0 is a placeholder, not measured).", - "custom_fields": { - "aws_service": "s3", "aws_resource_id": bucket_id, - "aws_metadata": {**{key: value for key, value in bucket.items() if key != "Tags"}, - "usage_status": "unknown", "usage_bytes": None}, - }, - "tags": self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}), - }) +class AWSDiscoveryError(RuntimeError): + """Discovery failed; do not reconcile this run with NetBox.""" - def resolve_shared_networks(self, snapshots): - networks = {} - for account, snapshot in snapshots.items(): - for region, data in snapshot.items(): - for kind in ("Vpcs", "Subnets"): - for resource_id, resource in data[kind].items(): - owner = resource.get("OwnerId", account) - key = (kind, region, resource_id) - owners = networks.setdefault(key, {}) - if resource.get("OwnerId") and any( - item.get("OwnerId") and previous_owner != owner - for previous_owner, item in owners.items()): - raise ValueError( - f"Conflicting AWS {kind} ownership for '{resource_id}' " - f"(region '{region}'): {', '.join(sorted(set(owners) | {owner}))}") - if owner in owners: - previous = owners[owner] - fields = ("VpcId", "CidrBlock") - if any(previous.get(field) != resource.get(field) for field in fields) or \ - set(self.ipv6_networks(previous)) != set(self.ipv6_networks(resource)): - raise ValueError( - f"Conflicting AWS {kind} discovery for '{resource_id}' " - f"(owner '{owner}', region '{region}')") - owners[owner] = resource - for account, snapshot in snapshots.items(): - for region, data in snapshot.items(): - data["VpcOwners"] = {} - for instance in data["instances"]: - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) - if eni is None: - continue - vpc_id, subnet_id = eni["VpcId"], eni["SubnetId"] - vpcs = networks.get(("Vpcs", region, vpc_id), {}) - local_vpc = data["Vpcs"].get(vpc_id) - owner = local_vpc.get("OwnerId", account) if local_vpc is not None else None - if owner is None and len(vpcs) == 1: - owner = next(iter(vpcs)) - if owner is None and vpcs: - raise ValueError( - f"Ambiguous AWS VPC ownership for '{vpc_id}' " - f"(account '{account}', region '{region}', " - f"ENI '{eni['NetworkInterfaceId']}'): {', '.join(sorted(vpcs))}") - if owner is None: - continue - data["VpcOwners"][vpc_id] = owner - data["Vpcs"][vpc_id] = dict(vpcs[owner], OwnerId=owner) - subnets = networks.get(("Subnets", region, subnet_id), {}) - if subnets and owner not in subnets: - raise ValueError( - f"Conflicting AWS subnet ownership for '{subnet_id}' " - f"(account '{account}', region '{region}', " - f"ENI '{eni['NetworkInterfaceId']}', VPC owner '{owner}'): " - f"{', '.join(sorted(subnets))}") - if owner in subnets: - data["Subnets"][subnet_id] = dict(subnets[owner], OwnerId=owner) - def vrf_name(self, account, region, vpc): - return self.settings.vpc_vrf_relation.get( - f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( - f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) +@dataclass(frozen=True) +class AWSAccessCheck: + account: str | None + region: str | None + service: str + resource: str | None + operation: str + denied: bool - def get_vrf(self, account, region, vpc): - account_region_vpc = f"{account}/{region}/{vpc}" - region_vpc = f"{region}/{vpc}" - has_mapping = account_region_vpc in self.settings.vpc_vrf_relation or \ - region_vpc in self.settings.vpc_vrf_relation - name = self.vrf_name(account, region, vpc) - vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) - if vrf is None: - if has_mapping: - raise ValueError(f"Create mapped NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") - vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) - return vrf - def ipv6_networks(self, resource): - return [ - ip_network(entry["Ipv6CidrBlock"]) - for entry in resource.get("Ipv6CidrBlockAssociationSet", []) - if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" - ] - - def addresses(self, eni, subnet): - networks = [ip_network(subnet["CidrBlock"]), *self.ipv6_networks(subnet)] - addresses = [ - (entry["PrivateIpAddress"], entry.get("Primary", False)) - for entry in eni.get("PrivateIpAddresses", []) - ] - addresses.extend( - (entry["Ipv6Address"], entry.get("IsPrimaryIpv6", False)) - for entry in eni.get("Ipv6Addresses", []) - ) - result = [] - for value, primary in addresses: - address = ip_address(value) - network = next((net for net in networks - if net.version == address.version and address in net), None) - if network is None: - raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' IP '{value}' has no matching AWS subnet") - value = f"{address}/{network.prefixlen}" - if self.settings.permitted_subnets.permitted(value, interface_name=eni["NetworkInterfaceId"]): - result.append((value, primary)) - return result - - def validate_snapshot(self, account, snapshot, vrfs=None): - if vrfs is None: - vrfs = {} - for region, data in snapshot.items(): - for instance in data["instances"]: - spec = data["InstanceTypes"].get(instance["InstanceType"]) - if spec is None: - raise ValueError(f"Missing AWS instance type '{instance['InstanceType']}'") - if "DefaultVCpus" not in spec["VCpuInfo"] or "SizeInMiB" not in spec["MemoryInfo"]: - raise ValueError(f"Incomplete AWS specifications for '{instance['InstanceType']}'") - for tag in instance.get("Tags", []): - if tag["Key"] in self.settings.tag_keys and len(f"AWS:{tag['Key']}={tag['Value']}") > 100: - raise ValueError(f"AWS tag '{tag['Key']}' exceeds the NetBox tag name limit (100 characters)") - for attachment in instance.get("BlockDeviceMappings", []): - volume_id = attachment.get("Ebs", {}).get("VolumeId") - if volume_id and volume_id not in data["Volumes"]: - raise ValueError(f"Missing attached EBS volume '{volume_id}'") - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) - if eni is None: - raise ValueError(f"Missing attached ENI '{reference['NetworkInterfaceId']}'") - if eni.get("Attachment", {}).get("InstanceId") != instance["InstanceId"]: - raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' changed attachment during discovery") - if not eni.get("MacAddress"): - raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' has no MAC address") - if not self.settings.sync_private_ips: +class AWSClient: + def __init__(self, profile=None, *, session=None): + if boto3 is None: + raise AWSDiscoveryError("AWS SDK missing. Install dependencies from requirements-aws.txt.") + try: + self.session = session if session is not None else boto3.Session(profile_name=profile) + except BotoCoreError as exc: + raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc + self.account_names = {} + self.access_denied = [] + self.access_checks = [] + + def track_access(self, client, account, region, service): + def capture_resource(params, context, **kwargs): + context["netbox_sync_resource"] = next(( + params[key] for key in ("Bucket", "clusterName", "FunctionName", "Resource", "name") + if key in params + ), None) + + def record_access(model, parsed, context, **kwargs): + code = parsed.get("Error", {}).get("Code") + denied = code in ( + "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", + "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException") + if code is None or denied or code in ( + "NoSuchTagSet", "ServerSideEncryptionConfigurationNotFoundError"): + check_account = parsed.get("Account", account) if model.name == "GetCallerIdentity" else account + self.access_checks.append(AWSAccessCheck( + check_account, region, service, context.get("netbox_sync_resource"), model.name, denied)) + + for event, handler in ( + ("before-parameter-build.*.*", capture_resource), ("after-call.*.*", record_access)): + unique_id = f"netbox-sync-access-{event}" + client.meta.events.unregister(event, unique_id=unique_id) + client.meta.events.register(event, handler, unique_id=unique_id) + + @contextmanager + def tolerate_access_denied(self, account=None, region=None, service=None, resource=None): + try: + yield + except ClientError as exc: + if exc.response["Error"]["Code"] not in ( + "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", + "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException"): + raise AWSDiscoveryError(f"Incomplete AWS discovery: {exc}") from exc + self.access_denied.append((account, region, service)) + log.warning( + f"AWS access denied (account '{account}', region '{region}', service '{service}', " + f"resource '{resource}'): {exc}. " + "Skipping inaccessible discovery; existing source objects will be retained.") + + def discover_organization(self, regions, role_name, expected_account=None, discovery_region=None, + include_management=False, external_id=None, account_ids=None, services=()): + """Collect selected accounts, recording authorization-denied scopes for retention.""" + self.access_denied = [] + self.access_checks = [] + clients = [] + current_account = None + try: + bootstrap = (discovery_region or self.session.region_name) if regions == ["auto"] else regions[0] + if not bootstrap: + raise AWSDiscoveryError( + "Organization discovery requires discovery_region or an AWS profile/environment region " + "(AWS_DEFAULT_REGION).") + config = Config(retries={"mode": "standard", "max_attempts": 5}, + connect_timeout=10, read_timeout=60) + sts = self.session.client("sts", region_name=bootstrap, config=config) + self.track_access(sts, None, None, "sts") + clients.append(sts) + identity = sts.get_caller_identity() + if expected_account is not None and identity["Account"] != expected_account: + raise AWSDiscoveryError( + f"AWS account mismatch: expected {expected_account}, authenticated as {identity['Account']}.") + partition = identity["Arn"].split(":")[1] + organizations = self.session.client("organizations", region_name=bootstrap, config=config) + self.track_access(organizations, None, None, "organizations") + clients.append(organizations) + management_account = organizations.describe_organization()["Organization"]["MasterAccountId"] + accounts = set() + account_names = {} + for page in organizations.get_paginator("list_accounts").paginate(): + for account in page["Accounts"]: + account_id = account["Id"] + if account["State"] != "ACTIVE": + log.info(f"AWS organization: excluding account '{account_id}' in state '{account['State']}'.") continue - vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) - network_errors = [] - if vpc not in data["Vpcs"]: - network_errors.append(f"VPC '{vpc}' was not returned by DescribeVpcs") - if subnet is None: - network_errors.append( - f"subnet '{eni['SubnetId']}' was not returned by DescribeSubnets") - elif subnet["VpcId"] != vpc: - network_errors.append( - f"subnet '{eni['SubnetId']}' belongs to VPC '{subnet['VpcId']}', " - f"not ENI VPC '{vpc}'") - if network_errors: - raise ValueError( - f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}' " - f"(account '{account}', region '{region}', instance '{instance['InstanceId']}', " - f"VPC '{vpc}', subnet '{eni['SubnetId']}'): {'; '.join(network_errors)}") - if self.addresses(eni, subnet): - owner = data.get("VpcOwners", {}).get(vpc, account) - vrf = self.get_vrf(owner, region, vpc) - identity = (owner, region, vpc) - if vrf in vrfs and vrfs[vrf] != identity: - raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") - vrfs[vrf] = identity - - def custom_fields(self): - fields = ( - ("aws_account_id", "AWS account ID", [NBVM.object_type]), - ("aws_region", "AWS region", [NBVM.object_type]), - ("aws_instance_id", "AWS instance ID", [NBVM.object_type]), - ("aws_name", "AWS Name tag", [NBVM.object_type]), - ("aws_instance_type", "AWS instance type", [NBVM.object_type]), - ("aws_availability_zone", "AWS Availability Zone", [NBVM.object_type]), - ("aws_vpc_id", "AWS VPC ID", [NBVM.object_type, NBVMInterface.object_type]), - ("aws_eni_id", "AWS ENI ID", [NBVMInterface.object_type]), - ("aws_subnet_id", "AWS subnet ID", [NBVMInterface.object_type]), - ("aws_public_ipv4", "AWS public IPv4", [NBVMInterface.object_type]), - ) - for name, label, types in fields: - self.add_update_custom_field({ - "name": name, "label": label, "object_types": types, "type": "text", - "description": f"Reported by AWS ({self.name})", - }) - if any((self.settings.sync_s3, self.settings.sync_lambda, self.settings.sync_eks)): - types = [NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type] - for name, field_type in ( - ("aws_service", "text"), ("aws_resource_id", "text"), - ("aws_metadata", "json"), - ): - field = self.add_update_custom_field({ - "name": name, "label": name.replace("_", " ").upper(), - "object_types": types, "type": field_type, - "description": f"Reported by AWS ({self.name})", + if account_id == management_account and not include_management: + continue + accounts.add(account_id) + account_names[account_id] = account.get("Name") or account_id + if account_ids: + missing = set(account_ids) - accounts + if missing: + raise AWSDiscoveryError( + f"Requested organization accounts are not active/eligible: {', '.join(sorted(missing))}.") + accounts.intersection_update(account_ids) + if not accounts: + raise AWSDiscoveryError("Organization discovery returned no eligible accounts.") + snapshots = {} + for current_account in sorted(accounts): + role_arn = f"arn:{partition}:iam::{current_account}:role/{role_name}" + log.info(f"AWS organization: discovering account '{current_account}' using '{role_arn}'.") + request = {"RoleArn": role_arn, "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} + if external_id: + request["ExternalId"] = external_id + credentials = None + self.track_access(sts, current_account, None, "sts") + with self.tolerate_access_denied(current_account, service="sts"): + credentials = sts.assume_role(**request)["Credentials"] + if credentials is None: + continue + member = AWSClient(session=boto3.Session( + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], region_name=bootstrap)) + options = {"services": services} if services else {} + _, snapshot = member.discover(regions, current_account, discovery_region, **options) + self.access_denied.extend(member.access_denied) + self.access_checks.extend(member.access_checks) + snapshots[current_account] = snapshot + self.account_names = account_names + return snapshots + except ClientError as exc: + with self.tolerate_access_denied(service="organizations"): + raise exc + return {} + except (BotoCoreError, KeyError, AWSDiscoveryError) as exc: + context = f" for account '{current_account}'" if current_account else "" + raise AWSDiscoveryError(f"Incomplete AWS organization discovery{context}: {exc}") from exc + finally: + for client in clients: + client.close() + + def discover(self, regions, expected_account=None, discovery_region=None, services=()): + """Collect accessible regions and record denied scopes before inventory changes.""" + clients = [] + self.access_denied = [] + self.access_checks = [] + account = expected_account + try: + config = Config(retries={"mode": "standard", "max_attempts": 5}, + connect_timeout=10, read_timeout=60) + automatic = regions == ["auto"] + bootstrap = (discovery_region or self.session.region_name) if automatic else regions[0] + if not bootstrap: + raise AWSDiscoveryError( + "Automatic region discovery requires discovery_region or an AWS profile/environment region " + "(AWS_DEFAULT_REGION).") + sts = self.session.client("sts", region_name=bootstrap, config=config) + self.track_access(sts, account, None, "account") + clients.append(sts) + account = sts.get_caller_identity()["Account"] + if expected_account is not None and account != expected_account: + raise AWSDiscoveryError( + f"AWS account mismatch: expected {expected_account}, authenticated as {account}.") + if automatic: + discovery_client = self.session.client("ec2", region_name=bootstrap, config=config) + self.track_access(discovery_client, account, None, "account") + clients.append(discovery_client) + response = discovery_client.describe_regions(AllRegions=True) + regions = sorted({ + entry["RegionName"] for entry in response["Regions"] + if entry["OptInStatus"] in ("opt-in-not-required", "opted-in") }) - existing_type = field.data.get("type") - if isinstance(existing_type, dict): - existing_type = existing_type.get("value") - if existing_type != field_type: - raise ValueError( - f"NetBox custom field '{name}' must have type '{field_type}', not '{existing_type}'") - - def sync_snapshot(self, account, snapshot, account_name=None): - self.custom_fields() - account_name = account_name or account - group = self.inventory.add_update_object( - NBClusterGroup, data={"name": f"AWS {account_name}"}, source=self) - api_version = Version(self.inventory.netbox_api_version) - for region, data in snapshot.items(): - site = self.inventory.add_update_object( - NBSite, data={"name": f"AWS {region}"}, source=self) - cluster_data = { - "name": f"AWS {account_name}/{region}", "group": group, - "type": {"name": "AWS EC2"}, + if not regions: + raise AWSDiscoveryError("Automatic region discovery returned no enabled regions.") + log.info(f"AWS account '{account}' enabled regions: {', '.join(regions)}") + snapshot = {} + for region in regions: + ec2 = self.session.client("ec2", region_name=region, config=config) + self.track_access(ec2, account, region, "ec2") + clients.append(ec2) + data = None + with self.tolerate_access_denied(account, region, "ec2"): + data = self.discover_ec2(ec2) + if data is None: + data = {"instances": [], "InstanceTypes": {}, "Vpcs": {}, "Subnets": {}, + "NetworkInterfaces": {}, "Volumes": {}, "EC2AccessDenied": True} + data["InaccessibleServices"] = [] + for service in services: + service_client = self.session.client(service, region_name=region, config=config) + self.track_access(service_client, account, region, service) + clients.append(service_client) + try: + with self.tolerate_access_denied(account, region, service): + if service == "s3": + data["Buckets"] = self.discover_s3(service_client, region, account) + elif service == "lambda": + data["Functions"] = self.discover_lambda(service_client) + elif service == "eks": + data["EKSClusters"] = self.discover_eks(service_client) + else: + raise AWSDiscoveryError(f"Unsupported AWS service '{service}'") + key = {"s3": "Buckets", "lambda": "Functions", "eks": "EKSClusters"}[service] + if key not in data: + data["InaccessibleServices"].append(service) + except (BotoCoreError, ClientError, KeyError) as exc: + raise AWSDiscoveryError( + f"Incomplete {service} discovery (account '{account}', region '{region}'): {exc}") from exc + snapshot[region] = data + return account, snapshot + except ClientError as exc: + with self.tolerate_access_denied(account, service="account"): + raise exc + return account, {} + except (BotoCoreError, KeyError) as exc: + raise AWSDiscoveryError(f"Incomplete AWS discovery; synchronization aborted: {exc}") from exc + finally: + for client in clients: + client.close() + + def discover_ec2(self, ec2): + instances = [ + instance + for page in ec2.get_paginator("describe_instances").paginate() + for reservation in page.get("Reservations", []) + for instance in reservation.get("Instances", []) + if instance["State"]["Name"] != "terminated" + ] + data = {"instances": instances} + for operation, result_key, key in ( + ("describe_network_interfaces", "NetworkInterfaces", "NetworkInterfaceId"), + ("describe_volumes", "Volumes", "VolumeId"), + ("describe_subnets", "Subnets", "SubnetId"), + ("describe_vpcs", "Vpcs", "VpcId"), + ): + data[result_key] = { + item[key]: item + for page in ec2.get_paginator(operation).paginate() + for item in page.get(result_key, []) } - if api_version >= Version("4.2"): - cluster_data.update(scope_type="dcim.site", scope_id=site) - else: - cluster_data["site"] = site - cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) - eks_clusters = {} - inaccessible = data.get("InaccessibleServices", []) - if self.settings.sync_eks and "eks" not in inaccessible: - for resource in data["EKSClusters"]: - eks_cluster = self.service_cluster( - account, region, site, group, "eks", resource["arn"], - f"EKS {account}/{region}/{resource['name']}", - metadata={key: value for key, value in resource.items() if key != "tags"}, - tags=resource.get("tags", {})) - eks_clusters[resource["name"]] = eks_cluster - if self.settings.sync_lambda and "lambda" not in inaccessible: - lambda_cluster = self.service_cluster( - account, region, site, group, "lambda", f"lambda/{account}/{region}", - f"Lambda {account}/{region}") - if lambda_cluster is not None: - for function in data["Functions"]: - self.sync_lambda(account, region, site, lambda_cluster, function) - if self.settings.sync_s3 and "s3" not in inaccessible: - self.sync_s3(account, region, site, group, data["Buckets"]) - name_counts = {} - for instance in data["instances"]: - name = next((tag["Value"].strip() for tag in instance.get("Tags", []) - if tag["Key"] == "Name" and tag["Value"].strip()), instance["InstanceId"]) - key = name.casefold() - name_counts[key] = name_counts.get(key, 0) + 1 - for instance in data["instances"]: - instance_cluster = eks_clusters.get( - data.get("EKSWorkers", {}).get(instance["InstanceId"]), cluster) - if instance_cluster is None: - log.error( - f"AWS source '{self.name}': Skipping instance '{instance['InstanceId']}' " - f"(account '{account}', region '{region}') because its EKS cluster was skipped.") - continue - tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} - vm_name = tags.get("Name", "").strip() or instance["InstanceId"] - if name_counts[vm_name.casefold()] > 1: - vm_name = f"{vm_name} ({instance['InstanceId']})" - self.sync_instance( - account, region, instance, data, site, - instance_cluster, - api_version, vm_name=vm_name) - - def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): - instance_id = instance["InstanceId"] - if "eks" in data.get("InaccessibleServices", []): - existing = next((item for item in self.inventory.get_all_items(NBVM) - if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and - (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) - if existing is not None and existing.data.get("cluster") is not None: - cluster = existing.data["cluster"] - tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} - spec = data["InstanceTypes"][instance["InstanceType"]] - vm_data = { - "name": vm_name or tags.get("Name", "").strip() or instance_id, "cluster": cluster, - "status": "active" if instance["State"]["Name"] == "running" else "offline", - "vcpus": spec["VCpuInfo"]["DefaultVCpus"], - "memory": spec["MemoryInfo"]["SizeInMiB"], - "custom_fields": { - "aws_account_id": account, "aws_region": region, - "aws_instance_id": instance_id, "aws_name": tags.get("Name", ""), - "aws_instance_type": instance["InstanceType"], - "aws_availability_zone": instance["Placement"]["AvailabilityZone"], - "aws_vpc_id": instance.get("VpcId", ""), - }, - "tags": [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags], + instance_types = {} + names = sorted({item["InstanceType"] for item in instances}) + for start in range(0, len(names), 100): + for page in ec2.get_paginator("describe_instance_types").paginate(InstanceTypes=names[start:start + 100]): + instance_types.update({item["InstanceType"]: item for item in page["InstanceTypes"]}) + data["InstanceTypes"] = instance_types + return data + + def discover_s3(self, client, region, account): + buckets = [] + for page in client.get_paginator("list_buckets").paginate(BucketRegion=region): + for bucket in page["Buckets"]: + with self.tolerate_access_denied(account, region, "s3", bucket["Name"]): + buckets.append(self.discover_s3_bucket(client, bucket, region, account)) + return buckets + + def discover_s3_bucket(self, client, bucket, region, account): + name = bucket["Name"] + bucket_region = bucket.get("BucketRegion") + if bucket_region is None: + location = client.get_bucket_location(Bucket=name, ExpectedBucketOwner=account) + bucket_region = location["LocationConstraint"] or "us-east-1" + if bucket_region == "EU": + bucket_region = "eu-west-1" + if bucket_region != region: + raise AWSDiscoveryError( + f"S3 bucket '{name}' returned for region '{region}' belongs to '{bucket_region}'") + request = {"Bucket": name, "ExpectedBucketOwner": account} + try: + tags = client.get_bucket_tagging(**request)["TagSet"] + except ClientError as exc: + if exc.response["Error"]["Code"] != "NoSuchTagSet": + raise + tags = [] + versioning = client.get_bucket_versioning(**request).get("Status", "Unversioned") + try: + encryption = client.get_bucket_encryption(**request)["ServerSideEncryptionConfiguration"] + except ClientError as exc: + if exc.response["Error"]["Code"] != "ServerSideEncryptionConfigurationNotFoundError": + raise + encryption = {} + return { + "Name": name, "Region": region, "Tags": tags, "Versioning": versioning, + "Encryption": encryption, "Arn": f"arn:{client.meta.partition}:s3:::{name}", } - cpu = instance.get("CpuOptions", {}) - if cpu.get("CoreCount") is not None and cpu.get("ThreadsPerCore") is not None: - vm_data["vcpus"] = cpu["CoreCount"] * cpu["ThreadsPerCore"] - if api_version >= Version("3.3"): - vm_data["site"] = site - if api_version >= Version("4.1"): - vm_data["serial"] = instance_id - if self.settings.tenant: - vm_data["tenant"] = {"name": self.settings.tenant} - disks = [ - data["Volumes"][entry["Ebs"]["VolumeId"]] - for entry in instance.get("BlockDeviceMappings", []) if "Ebs" in entry - ] - if api_version < Version("3.7"): - vm_data["disk"] = sum(volume["Size"] for volume in disks) - vm = next((item for item in self.inventory.get_all_items(NBVM) - if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and - (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) - if vm is None: - vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) - else: - vm.update(data=vm_data, source=self) - self.reconcile_tags(vm, vm_data["tags"]) - if api_version >= Version("3.7"): - for volume in disks: - self.add_update_vm_child(NBVirtualDisk, vm, { - "name": volume["VolumeId"], "virtual_machine": vm, - "size": volume["Size"] * (1024 if api_version >= Version("4.1") else 1), - "description": f"AWS EBS {volume['VolumeType']}", - }) - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"][reference["NetworkInterfaceId"]] - self.sync_interface(account, region, vm, eni, data, api_version) - - def add_update_vm_child(self, object_type, vm, data): - # Display-name matching loses the VM's account/cluster when names repeat. - existing = next((item for item in self.inventory.get_all_items(object_type) - if item.data.get("virtual_machine") is vm and - item.data.get("name") == data["name"]), None) - if existing is None: - return self.inventory.add_object(object_type, data=data, source=self) - existing.update(data=data, source=self) - return existing - def sync_interface(self, account, region, vm, eni, data, api_version): - interface_data = { - "name": eni["NetworkInterfaceId"], "virtual_machine": vm, "enabled": True, - "description": eni.get("Description", ""), - "custom_fields": { - "aws_eni_id": eni["NetworkInterfaceId"], "aws_vpc_id": eni.get("VpcId", ""), - "aws_subnet_id": eni.get("SubnetId", ""), - "aws_public_ipv4": ", ".join(sorted({ - address["Association"]["PublicIp"] - for address in eni.get("PrivateIpAddresses", []) - if address.get("Association", {}).get("PublicIp") - })), - }, - } - mac = normalize_mac_address(eni.get("MacAddress")) - if api_version < Version("4.2"): - interface_data["mac_address"] = mac - interface = self.add_update_vm_child(NBVMInterface, vm, interface_data) - if api_version >= Version("4.2") and mac is not None: - mac_object = next((item for item in self.inventory.get_all_items(NBMACAddress) - if item.data.get("mac_address") == mac and - item.data.get("assigned_object_id") in (None, interface)), None) - mac_data = {"mac_address": mac, "assigned_object_id": interface, - "assigned_object_type": NBVMInterface} - if mac_object is None: - mac_object = self.inventory.add_object(NBMACAddress, data=mac_data, source=self) - else: - mac_object.update(data=mac_data, source=self) - for old in interface.get_mac_addresses(): - if old is not mac_object: - old.remove_interface_association() - interface.update(data={"primary_mac_address": mac_object}, source=self) - if not self.settings.sync_private_ips: - for address in interface.get_ip_addresses(): - if self.source_tag in address.get_tags(): - address.set_source(self) - return - subnet = data["Subnets"][eni["SubnetId"]] - addresses = self.addresses(eni, subnet) - discovered = [] - for address, primary in addresses: - owner = data.get("VpcOwners", {}).get(eni["VpcId"], account) - vrf = self.get_vrf(owner, region, eni["VpcId"]) - parsed = ip_interface(address) - candidates = [ - item for item in self.inventory.get_all_items(NBIPAddress) - if ip_interface(item.data["address"]).ip == parsed.ip and item.data.get("vrf") is vrf - ] - if len(candidates) > 1: - raise ValueError(f"Multiple NetBox IPs match '{address}' in VRF '{vrf.data['name']}'") - existing = candidates[0] if candidates else None - if existing is not None and existing.data.get("assigned_object_id") is not None and \ - existing.get_interface() is not interface: - owner = existing.get_device_vm() - owner_tags = owner.get_tags() if owner is not None else [] - same_source = owner is not None and ( - owner.source is self or self.source_tag in owner_tags) - managed_owner = same_source or NetBoxHandler.primary_tag in owner_tags - managed_orphan = NetBoxHandler.primary_tag in owner_tags and \ - NetBoxHandler.orphaned_tag in owner_tags - owner_status = owner.data.get("status") if owner is not None else None - if isinstance(owner_status, dict): - owner_status = owner_status.get("value") - if managed_owner and owner_status == "active" and vm.data["status"] == "offline": - log.error( - f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " - f"is assigned to active owner '{owner.get_display_name()}'. " - f"Skipping reassignment to offline ENI '{eni['NetworkInterfaceId']}'.") - continue - offline_to_active = managed_owner and owner_status == "offline" and vm.data["status"] == "active" - if not (same_source or managed_orphan or offline_to_active): - owner_name = owner.get_display_name() if owner is not None else "unresolved owner" - log.error( - f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " - f"belongs to another source (owner '{owner_name}'). " - f"Skipping reassignment to ENI '{eni['NetworkInterfaceId']}'.") - continue - if managed_orphan or offline_to_active: - reason = "orphaned" if managed_orphan else "offline" - log.warning( - f"AWS source '{self.name}': Reclaiming IP '{address}' in VRF '{vrf.data['name']}' " - f"from {reason} owner '{owner.get_display_name()}' " - f"for ENI '{eni['NetworkInterfaceId']}'.") - ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, - "assigned_object_type": NBVMInterface} - if self.settings.tenant: - ip_data["tenant"] = {"name": self.settings.tenant} - if existing is None: - existing = self.inventory.add_object(NBIPAddress, data=ip_data, source=self) - else: - existing.update(data=ip_data, source=self) - discovered.append(existing) - if eni.get("Attachment", {}).get("DeviceIndex") == 0 and primary: - vm.update(data={f"primary_ip{parsed.version}": existing}, source=self) - for old in interface.get_ip_addresses(): - if old not in discovered and (old.source is self or self.source_tag in old.get_tags()): - old.remove_interface_association() + def discover_lambda(self, client): + functions = [] + for page in client.get_paginator("list_functions").paginate(): + for reference in page["Functions"]: + config = client.get_function_configuration(FunctionName=reference["FunctionArn"]) + # Do not retain environment variables or other unneeded configuration. + function = {key: config[key] for key in ( + "FunctionArn", "FunctionName", "MemorySize", "State", "Runtime", "Handler", + "Timeout", "LastModified", "Architectures", "PackageType", "VpcConfig", + ) if key in config} + if function["FunctionArn"] != reference["FunctionArn"]: + raise AWSDiscoveryError(f"Lambda function identity changed for '{reference['FunctionArn']}'") + function["Tags"] = client.list_tags(Resource=function["FunctionArn"]).get("Tags", {}) + functions.append(function) + return functions + + def discover_eks(self, client): + clusters = [] + for page in client.get_paginator("list_clusters").paginate(): + for name in page["clusters"]: + cluster = client.describe_cluster(name=name)["cluster"] + if cluster["name"] != name: + raise AWSDiscoveryError(f"EKS cluster identity changed for '{name}'") + groups = [] + for group_page in client.get_paginator("list_nodegroups").paginate(clusterName=name): + for group_name in group_page["nodegroups"]: + group = client.describe_nodegroup(clusterName=name, nodegroupName=group_name)["nodegroup"] + if group["clusterName"] != name or group["nodegroupName"] != group_name: + raise AWSDiscoveryError(f"EKS node group identity changed for '{group_name}'") + groups.append({key: group[key] for key in ( + "nodegroupName", "nodegroupArn", "status", "version", "capacityType", + "instanceTypes", "scalingConfig", "subnets", "resources", + ) if key in group}) + profiles = [] + for profile_page in client.get_paginator("list_fargate_profiles").paginate(clusterName=name): + for profile_name in profile_page["fargateProfileNames"]: + profile = client.describe_fargate_profile( + clusterName=name, fargateProfileName=profile_name)["fargateProfile"] + if profile["clusterName"] != name or profile["fargateProfileName"] != profile_name: + raise AWSDiscoveryError(f"EKS Fargate profile identity changed for '{profile_name}'") + profiles.append({key: profile[key] for key in ( + "fargateProfileName", "fargateProfileArn", "status", "subnets", "selectors", + ) if key in profile}) + clusters.append({ + **{key: cluster[key] for key in ( + "name", "arn", "status", "version", "endpoint", "resourcesVpcConfig", "tags", + ) if key in cluster}, + "nodegroups": sorted(groups, key=lambda group: group["nodegroupName"]), + "fargateProfiles": sorted(profiles, key=lambda profile: profile["fargateProfileName"]), + }) + return clusters From 12b152920a699c58ceb49d8197dca5e3de56c559 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 28 Oct 2026 23:43:45 -0500 Subject: [PATCH 28/28] Enhance AWS connection with access and tagging methods Added access_scope and reconcile_access_tags methods to manage AWS resource access and tagging. Updated retain_inaccessible_objects to call retain_source_objects. --- module/sources/aws/connection.py | 125 ++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 9 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index e3cebe1f..c3559320 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -1,5 +1,6 @@ from hashlib import sha256 from ipaddress import ip_address, ip_interface, ip_network +from itertools import product from packaging.version import Version @@ -94,6 +95,7 @@ def apply(self): account_labels[account] = f"{label} ({account})" for account, snapshot in snapshots.items(): self.sync_snapshot(account, snapshot, account_name=account_labels[account]) + self.reconcile_access_tags() except (AWSDiscoveryError, ValueError, KeyError) as exc: self.abort(exc) @@ -162,6 +164,87 @@ def validate_services(self, account, snapshot): f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") + def access_scope(self, item): + fields = item.data.get("custom_fields") or {} + metadata = fields.get("aws_metadata") or {} + identity = fields.get("aws_resource_id") or "" + service = "ec2" if fields.get("aws_instance_id") else fields.get("aws_service") + account = fields.get("aws_account_id") or metadata.get("account") + region = fields.get("aws_region") or metadata.get("region") + resource = None + if identity.startswith("arn:"): + parts = identity.split(":", 5) + if len(parts) == 6: + service, region, account, resource = parts[2:] + if service == "lambda": + resource = identity + elif service == "eks": + resource = resource.removeprefix("cluster/") + elif identity.startswith("s3/"): + parts = identity.split("/") + account = parts[1] + if isinstance(item, NBVirtualDisk): + resource = parts[2] + else: + region = parts[2] + for parent_key in ("virtual_machine", "cluster"): + parent = item.data.get(parent_key) + if parent is not None: + parent_account, parent_region, parent_service, _ = self.access_scope(parent) + account = account or parent_account + region = region or parent_region + service = service or parent_service + break + if isinstance(item, (NBIPAddress, NBMACAddress)): + parent = item.get_device_vm() + if parent is not None: + account, region, service, _ = self.access_scope(parent) + if account is None: + group = item if isinstance(item, NBClusterGroup) else item.data.get("group") + name = group.data.get("name", "") if group is not None else "" + if name.startswith("AWS "): + label = name[4:] + account = next((key for key, value in getattr(self.client, "account_names", {}).items() + if label in (value, f"{value} ({key})")), None) + if account is None and label.isdigit(): + account = label + if region is None: + site = item.data.get("scope_id") or item.data.get("site") + name = site.data.get("name", "") if isinstance(site, NBSite) else "" + if name.startswith("AWS "): + region = name[4:] + return account, region, service or "ec2", resource + + def reconcile_access_tags(self): + checks = getattr(self.client, "access_checks", []) + if not checks: + return + prefix = "Sync:AccessDenied:" + scopes = {} + for check in checks: + outcomes = scopes.setdefault( + (check.account, check.region, check.service, check.resource), (set(), set())) + outcomes[0 if check.denied else 1].add(check.operation) + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if object_type in (NBTag, NBCustomField, NBTenant) or "tags" not in item.data_model or not ( + item.source is self or self.source_tag in item.get_tags()): + continue + account, region, service, resource = self.access_scope(item) + denied, succeeded = set(), set() + for key in product( + (None, account), (None, region), ("organizations", "sts", "account", service), + (None, resource)): + if key in scopes: + failures, successes = scopes[key] + denied.update(failures) + succeeded.update(successes) + for operation in denied: + item.add_tags(f"{prefix}{operation}") + for tag in item.get_tags(): + if tag.startswith(prefix) and tag[len(prefix):] in succeeded - denied: + item.remove_tags(tag) + def validate_arn(self, arn, service, account, region): parts = arn.split(":", 5) if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: @@ -189,12 +272,15 @@ def retain_disabled_services(self): (item.source is self or self.source_tag in item.get_tags()): item.set_source(self) - def retain_inaccessible_objects(self): + def retain_source_objects(self): # Source-wide retention also protects children and cross-account shared-network dependencies. for object_type in self.dependent_netbox_objects: for item in self.inventory.get_all_items(object_type): if self.source_tag in item.get_tags(): item.set_source(self) + + def retain_inaccessible_objects(self): + self.retain_source_objects() log.warning( f"AWS source '{self.name}': authorization-denied discovery is incomplete. " "Retaining all previously managed source objects this run; accessible resources will still be updated.") @@ -231,18 +317,27 @@ def resource_name(self, name, identity, limit): def service_object(self, object_type, identity, data): matches = [item for item in self.inventory.get_all_items(object_type) if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + error = None if len(matches) > 1: - raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") - if matches: + error = f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'" + elif matches: item = matches[0] if item.source is not self and self.source_tag not in item.get_tags(): - raise ValueError(f"AWS resource '{identity}' belongs to another source") - item.update(data=data, source=self) + error = f"AWS resource '{identity}' belongs to another source" else: parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) for item in self.inventory.get_all_items(object_type)): - raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") + error = f"NetBox {object_type.name} name collision for AWS resource '{identity}'" + if error is not None: + log.error( + f"AWS source '{self.name}': {error}. Skipping this resource and any dependent resources. " + "Retaining all previously managed source objects this run to prevent pruning skipped resources.") + self.retain_source_objects() + return None + if matches: + item.update(data=data, source=self) + else: item = self.inventory.add_object(object_type, data=data, source=self) self.reconcile_tags(item, data["tags"]) return item @@ -293,9 +388,13 @@ def sync_s3(self, account, region, site, group, buckets): identity = f"s3/{account}/{region}" cluster = self.service_cluster( account, region, site, group, "s3", identity, f"S3 {account}/{region}") + if cluster is None: + return holder = self.logical_vm( account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) + if holder is None: + return for bucket in buckets: bucket_id = f"s3/{account}/{bucket['Name']}" self.service_object(NBVirtualDisk, bucket_id, { @@ -533,8 +632,9 @@ def sync_snapshot(self, account, snapshot, account_name=None): lambda_cluster = self.service_cluster( account, region, site, group, "lambda", f"lambda/{account}/{region}", f"Lambda {account}/{region}") - for function in data["Functions"]: - self.sync_lambda(account, region, site, lambda_cluster, function) + if lambda_cluster is not None: + for function in data["Functions"]: + self.sync_lambda(account, region, site, lambda_cluster, function) if self.settings.sync_s3 and "s3" not in inaccessible: self.sync_s3(account, region, site, group, data["Buckets"]) name_counts = {} @@ -544,13 +644,20 @@ def sync_snapshot(self, account, snapshot, account_name=None): key = name.casefold() name_counts[key] = name_counts.get(key, 0) + 1 for instance in data["instances"]: + instance_cluster = eks_clusters.get( + data.get("EKSWorkers", {}).get(instance["InstanceId"]), cluster) + if instance_cluster is None: + log.error( + f"AWS source '{self.name}': Skipping instance '{instance['InstanceId']}' " + f"(account '{account}', region '{region}') because its EKS cluster was skipped.") + continue tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} vm_name = tags.get("Name", "").strip() or instance["InstanceId"] if name_counts[vm_name.casefold()] > 1: vm_name = f"{vm_name} ({instance['InstanceId']})" self.sync_instance( account, region, instance, data, site, - eks_clusters.get(data.get("EKSWorkers", {}).get(instance["InstanceId"]), cluster), + instance_cluster, api_version, vm_name=vm_name) def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None):