diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7c870977..661e28c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Test on: push: - branches: [ "main", "development" ] + branches: [ "main", "development", "AWS" ] pull_request: branches: [ "main", "development" ] workflow_dispatch: @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements-aws.txt pip install pytest - name: Install vcsim diff --git a/Dockerfile b/Dockerfile index 0852fa81..be3a2093 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,16 @@ FROM python:3.14-slim AS builder COPY requirements.txt . +COPY requirements-aws.txt . ARG VENV=/opt/netbox-sync/venv +ARG INSTALL_AWS=false # Install dependencies RUN python3 -m venv $VENV && \ $VENV/bin/python3 -m pip install --upgrade pip && \ $VENV/bin/pip install -r requirements.txt && \ + if [ "$INSTALL_AWS" = "true" ]; then $VENV/bin/pip install -r requirements-aws.txt; fi && \ $VENV/bin/pip install vmware-vcenter==9.1.1.0 && \ $VENV/bin/python3 -m pip uninstall -y pip && \ find $VENV -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf diff --git a/README.md b/README.md index f395b1e7..711ca097 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ This is a tool to sync data from different sources to a NetBox instance. Available source types: * VMware vCenter Server +* AWS EC2, EBS, and network interfaces, with opt-in S3, Lambda, and EKS inventory (see [AWS source](docs/source_aws.md)) * [bb-ricardo/check_redfish](https://github.com/bb-Ricardo/check_redfish) inventory files **IMPORTANT: READ INSTRUCTIONS CAREFULLY BEFORE RUNNING THIS PROGRAM** @@ -38,6 +39,10 @@ This ensures stale objects are removed from NetBox keeping an accurate current s * VMWare vCenter >= 6.0 #### Source: check_redfish (if used) * check_redfish >= 1.2.0 +#### Source: AWS (if used) +* Optional boto3 SDK, AWS read-only credentials, and automatic or explicit region discovery +* Optional AWS Organizations discovery with collector permissions and a trusted inventory role in each member account +* Existing VPC-specific NetBox VRFs for IP import; see [AWS setup](docs/source_aws.md) # Installing * here we assume we install in ```/opt``` diff --git a/docs/source_aws.md b/docs/source_aws.md new file mode 100644 index 00000000..91b9071a --- /dev/null +++ b/docs/source_aws.md @@ -0,0 +1,763 @@ +# AWS source + +The `aws` source imports EC2 instances, attached EBS volumes, attached network +interfaces (ENIs), MAC addresses, and permitted private IPv4 / IPv6 addresses. +Optional discovery adds S3 buckets, Lambda functions, and EKS clusters. +AWS is queried read-only; writes are only made to NetBox. This branch is based +on current main and does not depend on the oVirt or OpenStack branches. + +## Installation and authentication + +```sh +pip install -r requirements-aws.txt +# Alternatively: +uv sync --extra aws +``` + +Other sources do not require boto3. An enabled AWS source without its SDK or +credentials fails explicitly rather than continuing with empty inventory. +For Docker, enable the SDK explicitly: + +```sh +docker build --build-arg INSTALL_AWS=true -t netbox-sync:aws . +``` + +Use the standard boto3 credential chain: + +* EC2 instance roles, ECS task roles, or EKS workload identity. +* A shared AWS profile (`profile` below), including an assume-role profile for + cross-account access. +* `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`; temporary credentials also need + `AWS_SESSION_TOKEN`. Prefer roles over stored access keys. + +By default each configured source handles one AWS account. Regions are discovered +automatically by default (`regions = auto`); an explicit list is also supported. +For multiple accounts, use organization mode below or configure separate named +sources/profiles. The optional +`account_id` guard verifies the account returned by STS before importing anything. +Profile-based assume-role credential refresh is handled by boto3. Credentials +must allow access to every configured region. + +No SSH access, guest agents, inbound connections, or AWS write permissions are +needed. Outbound HTTPS to the regional EC2 and STS endpoints is required, +plus the S3, Lambda, and EKS endpoints for enabled optional services. + +Example IAM policy (attach to a dedicated role or non-root identity): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeVolumes", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeRegions", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +An assume-role profile additionally needs `sts:AssumeRole` permission on the +target role and a matching target-account trust policy (and external ID where +required). Single-account mode does not use Organizations permissions. + +### Additional permissions for optional services + +The new services are **opt-in**. Add only the actions for the services you enable +to the single-account inventory identity or every selected member-account role. +For example, the following supplemental policy enables all three: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListAllMyBuckets", + "s3:GetBucketLocation", + "s3:GetBucketTagging", + "s3:GetBucketVersioning", + "s3:GetEncryptionConfiguration", + "lambda:ListFunctions", + "lambda:GetFunctionConfiguration", + "lambda:ListTags", + "eks:ListClusters", + "eks:DescribeCluster", + "eks:ListNodegroups", + "eks:DescribeNodegroup", + "eks:ListFargateProfiles", + "eks:DescribeFargateProfile" + ], + "Resource": "*" + } + ] +} +``` + +Where your IAM policy supports resource-level permissions, restrict the +resource-specific actions to the intended buckets, functions, and EKS resources. +Listing actions still need their documented wildcard scope. No S3 object reads, +Lambda code downloads, Kubernetes credentials, or CloudWatch permissions are +required. Authorization denials are logged and skipped as described below; +other discovery failures still abort before NetBox writes and pruning. + +### Authorization-denied discovery + +Recognized authorization errors (`AccessDenied`, `AccessDeniedException`, +`UnauthorizedOperation`, `AuthorizationError`, `AuthorizationErrorException`, +`Forbidden`, and `ForbiddenException`) produce warnings and allow accessible +discovery to continue. Warnings include account, region, service, AWS operation, +and the bucket name when applicable. + +* S3 bucket metadata denial skips that bucket and continues with subsequent + buckets. Existing bucket metadata and tags are not cleared. +* Denied S3/Lambda/EKS service discovery skips that service in that region. + Partially collected service results are discarded if listing or detail + discovery fails outside the per-bucket handling. +* Denied EC2 discovery discards the regional EC2 snapshot, but still attempts + optional services and other regions. Dependent instances whose shared + network metadata is unavailable during denied organization discovery are + deferred rather than imported with guessed masks. +* Denied member-role assumption skips that account and scans remaining + accounts. Denied identity, region enumeration, or organization enumeration + skips the affected scan without treating its inventory as empty. + +**Any authorization denial conservatively retains all objects previously +managed by this AWS source for that run**, including children and shared-network +dependencies. Accessible objects still update. This also postpones orphaning +and pruning for genuinely deleted objects in otherwise accessible scopes of +this source; a later fully authorized run resumes normal lifecycle processing. +Objects from other sources are unaffected. + +Existing EKS worker cluster placement is preserved when regional EKS discovery +is denied. Authentication/credential expiration, throttling exhaustion, service +errors, account mismatches, and invalid or conflicting data remain fatal. +Successful empty results are authoritative only when discovery has no denials. + +Authorization failures also add operation-specific tags to matching managed +NetBox objects, for example `Sync:AccessDenied:GetBucketTagging`. Bucket-specific +failures tag the bucket's virtual disk. Failures without an individual resource +identifier tag existing managed objects in the affected account/region/service +scope; account/role or organization failures cover their broader scopes. Objects +not managed by this AWS source are not tagged. Resources that have never been +imported are not created as placeholders, so those failures remain log-only. + +Permission tags clear automatically after the same operation succeeds for the +relevant resource or scope. A disabled service, unattempted operation, or success +for another resource does not clear a tag. A denial takes precedence over success +in the same applicable scope during a run. Other user tags are preserved. + +### NetBox resource conflicts + +Logical service resources with duplicate NetBox resource identities, ownership +by another source, or a NetBox name collision are logged and skipped, without +aborting unrelated resources, regions, or accounts. Conflicting objects are not +adopted or overwritten. A skipped cluster also skips its dependent resources, +including EKS workers; a skipped S3 inventory VM skips its buckets. + +If a resource conflict occurs, all previously managed objects for this source +are retained for the run, protecting skipped resources and their children from +orphaning/pruning. Discovery and snapshot-validation errors retain their existing +failure handling. + +### Troubleshooting incomplete network discovery + +With private IP synchronization enabled, each attached ENI must have a discovered +VPC and subnet, and the subnet's VPC must match the ENI's VPC. If validation fails, +the error includes the account, region, instance, ENI, VPC, and subnet IDs. It +identifies resources missing from `DescribeVpcs` or `DescribeSubnets`, or reports +the conflicting VPC ID for a subnet mismatch. Multiple failures for the same ENI +are reported together. + +Check those resource IDs in the reported account and region using the same role +or profile as discovery, including any cross-account network sharing setup. +In organization mode, VPCs and subnets discovered in another selected account +can satisfy these checks as described under shared VPCs below. Missing resources +must still be discovered somewhere in the selected organization scope. +Missing or mismatched network data normally aborts without NetBox writes or +pruning. When denied organization discovery leaves shared-network dependencies +unavailable, affected instance updates are instead deferred with a warning and +existing objects retained. Address-space checks are not bypassed for imports. +Earlier "Created new VRF object" log messages refer to in-memory inventory +objects created during validation, not completed NetBox API writes. + +## Configuration + +```ini +[source/production-aws] +type = aws +regions = auto +discovery_region = us-east-1 +profile = inventory +account_id = 123456789012 +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +vpc_vrf_relation = us-east-1/vpc-0123456789abcdef0 = Production VPC +tag_keys = Environment, Owner +tenant = Operations +account_name = Production +``` + +All options also support the application's usual `NBS_SOURCE__*` +environment variables. `regions` and `tag_keys` accept YAML lists as well as +comma-separated strings. Quote the account ID in YAML so it remains a string. +`python netbox-sync.py -g` generates the full configuration. + +### Optional S3, Lambda, and EKS discovery + +These settings default to `False`, preserving existing EC2-only discovery and +permissions. Add them to the same AWS source for single-account or organization +mode: + +```ini +sync_s3 = True +sync_lambda = True +sync_eks = True +``` + +All three follow the source's selected account and region scope. Pagination is +completed before reconciliation. Disabling an optional service retains its +previously managed logical VMs, disks, and clusters instead of orphaning them. +While a service is enabled, successfully discovered empty results are +authoritative when discovery has no denials: removed functions, buckets, and empty S3 holders follow the +normal orphaning/pruning policy. Clusters remain subject to the existing +non-pruning cluster policy. + +#### S3 buckets + +Requires **NetBox 3.7+**; older versions abort before discovery when `sync_s3` +is enabled. Each account/region with buckets gets: + +* Cluster `S3 /`, type `AWS S3`. +* Logical VM `S3 inventory /`, tagged `AWS:logical-s3`. +* One virtual disk per general-purpose bucket, named by its bucket name. + +The holder VM is explicitly an inventory container, not a server. It has no +invented CPU, memory, MAC address, or IP. Bucket metadata includes ARN, region, +versioning, and encryption configuration in the disk's `aws_metadata` JSON +custom field. Configured `tag_keys` are copied to the bucket disk. + +**Disk size zero is a required-field placeholder, not a measurement.** +`aws_metadata.usage_status` is `unknown` and `usage_bytes` is JSON null; the +description also states that usage is unknown. Empty and unmeasured buckets +are therefore not treated as equivalent usage measurements. No provisioned +capacity or CloudWatch usage is inferred, and S3 disk totals must not be used +as storage-capacity reporting. Bucket contents, directory buckets, access +points, and S3-compatible non-AWS services are not discovered. + +#### Lambda functions + +Each account/region gets a logical `AWS Lambda` cluster and one logical VM per +function, tagged `AWS:logical-lambda`. Matching uses the unqualified function ARN, +not the display name. Long display names use a deterministic hash suffix to fit +the importer's name limit without losing resource identity. + +Configured function memory is VM memory. vCPUs and interfaces are not invented, +and shared Lambda VPC ENIs are not assigned to a function VM. `aws_metadata` +records runtime, handler, architecture, timeout, modification time, package +type, VPC configuration, and AWS state when returned. Environment variables, +code, credentials, invocation instances, aliases, and published versions are +not imported. + +State maps as follows: `Active` to `active`, `Pending` to `planned`, `Inactive` +to `offline`, and `Failed` to `failed`. These describe AWS function state, not +whether an invocation is currently executing. + +#### EKS clusters and workers + +Each real EKS cluster becomes a NetBox cluster of type `AWS EKS`. Its ARN is +the stable identity. `aws_metadata` records Kubernetes version, AWS status, +endpoint/VPC configuration, managed node groups (including scaling and Auto +Scaling group information), and Fargate profile metadata. + +Existing discovered EC2 worker VMs move into that cluster; duplicate worker VMs +and fabricated control-plane VMs are not created. Worker membership is matched +using managed node-group Auto Scaling group names or the EC2 tags +`aws:eks:cluster-name`, `eks:cluster-name`, and +`eksctl.cluster.k8s.io/v1alpha1/cluster-name`. Conflicting membership aborts the +run. A reference to a cluster no longer discovered emits a warning and leaves +the instance in its regular EC2 cluster. Untagged self-managed workers without +a discovered node-group relationship remain ordinary EC2 VMs. + +Disabling EKS discovery returns discovered EC2 workers to their normal EC2 +clusters. Kubernetes API access, pods, workloads, and individual Fargate runtime +instances are outside this discovery scope. + +Optional resources use `aws_service`, `aws_resource_id`, and `aws_metadata` +custom fields on VMs, clusters, and virtual disks. Existing unrelated tags are +preserved. These objects are logical documentation of AWS resources, not +claims that serverless services are virtual servers. + +### Automatic or explicit regions + +`regions = auto` (also the default when omitted) calls EC2 `DescribeRegions` and +queries regions whose opt-in status is `opt-in-not-required` or `opted-in`. +Disabled regions (`not-opted-in`) are excluded. Each resource is mapped to the +region that returned it, never to the bootstrap region. + +Auto discovery requires a bootstrap endpoint: set `discovery_region`, or provide +a region through the AWS profile or `AWS_DEFAULT_REGION`. There is no hardcoded +commercial-region fallback; choose an endpoint in the appropriate AWS partition +(for example GovCloud or China when applicable). The account's enabled regions +in that partition are discovered. A missing bootstrap region fails explicitly. + +To restrict scope, use `regions = us-east-1, us-west-2` or a YAML list. Explicit +mode does not call `DescribeRegions`, does not need that permission, and ignores +`discovery_region`. Do not combine `auto` and named regions in the same list. + +Auto mode requires `ec2:DescribeRegions` in addition to the inventory permissions. +Authorization denial during region discovery is logged and preserves existing +objects. Other failures, or an empty enabled-region result, abort synchronization +before NetBox writes or pruning. There is no silent fallback to a partial region list. Use explicit mode when +credentials are intentionally restricted to a subset of enabled regions. +Switching between modes changes discovery scope; inspect a dry run before +allowing writes, particularly if narrowing to an explicit list. + +### Organization / multi-account discovery + +For an organization with many member accounts, enable organization mode: + +```ini +[source/aws-organization] +type = aws +profile = inventory-collector +organization = True +organization_role_name = NetBoxInventoryReadOnly +discovery_region = us-east-1 +regions = auto +sync_private_ips = False +; Optional safety check: the account hosting the collector credentials +account_id = 123456789012 +; Start with one or two member accounts before enabling the full organization: +;organization_account_ids = 111122223333, 444455556666 +;organization_external_id = your-administrator-provided-external-id +;organization_include_management = False +``` + +The collector enumerates all account pages (including empty pages with a +continuation token), selects accounts with `State = ACTIVE`, and excludes the +organization's management account by default. It assumes the named role in each +selected account, verifies the resulting account identity, then discovers that +account's regions and resources. No EC2 permissions are needed on the collector +identity when management-account scanning is disabled. + +`organization_account_ids` is an optional allowlist, not an account-ID guard. +Quote IDs in YAML lists. An explicitly empty list is invalid; omit the option to +select all active members. Unknown, inactive, or excluded IDs in that list cause +an error rather than silently reducing scope. To scan the management account, set +`organization_include_management = True` and deploy the same inventory role +there. A role path such as `service-accounts/NetBoxInventoryReadOnly` is supported. +Role ARNs use the collector's AWS partition; cross-partition discovery is not +supported. + +All accessible selected accounts are collected and validated before reconciliation. +Authorization denials are logged and inaccessible scopes skipped with source-wide +retention. Other Organizations API errors, role-assumption failures, account +mismatches, or regional discovery failures abort before NetBox writes or pruning. +No inaccessible account is treated as empty. A selection with no eligible accounts +also fails. Account discovery and scanning are sequential. Fresh one-hour STS +credentials are obtained immediately before each account scan, so time spent +scanning earlier accounts does not consume later accounts' credential lifetime. +These member credentials are held only in memory and are not automatically +renewed within an individual account scan; expiration fails the run safely. +Collector profile credential refresh remains managed by boto3. + +#### AWS profile + +The profile supplies **collector credentials**, not a separate profile for every +member. No custom organization settings are needed in the AWS profile; the +organization options belong in the application's source section. + +For an existing named credentials entry `inventory-collector`, the AWS shared +config can contain: + +```ini +[profile inventory-collector] +region = us-east-1 +``` + +Store its credentials in the standard AWS shared credentials file under +`[inventory-collector]`, or use an existing assume-role/SSO profile supported by +boto3. On Windows the standard files are `%USERPROFILE%\.aws\config` and +`%USERPROFILE%\.aws\credentials`; on Linux they are `~/.aws/config` and +`~/.aws/credentials`. Authenticate SSO profiles before an interactive test. +For unattended jobs prefer a workload IAM role or an automatically refreshable +credential provider, not an expiring interactive SSO login. + +For example, if the collector itself uses an assume-role profile: + +```ini +[profile inventory-collector] +role_arn = arn:aws:iam::123456789012:role/NetBoxOrganizationCollector +source_profile = existing-base-profile +region = us-east-1 +``` + +The base profile must be authorized to assume that collector role, and the role +must trust it. The collector role then needs the permissions below. If the +collector runs with an attached IAM role or environment credentials, omit the +application `profile` setting entirely. Never store access keys in the source +configuration or commit credential files. + +#### Complete example: reporting roles in management and member accounts + +All account IDs, profile names, role names, and user names below are fictitious +examples. Replace them with your deployment's values. + +In this example, an IAM user's keys assume the management account's +`ExampleInventoryRole`, which then assumes a role also named +`ExampleInventoryRole` in each member account: + +```text +IAM user credentials (example-inventory-client profile) + -> arn:aws:iam::123456789012:role/ExampleInventoryRole + -> arn:aws:iam:::role/ExampleInventoryRole + -> member account EC2 discovery +``` + +Create these files for the OS user running the importer, on the machine or in +the container where it runs. Files on a workstation are not automatically +available to a remote host, service account, or container. Adjust account IDs +and role names for your deployment. + +**1. AWS shared credentials** (`%USERPROFILE%\.aws\credentials` on Windows, +`~/.aws/credentials` on Linux): + +```ini +[example-inventory-client] +aws_access_key_id = REPLACE_WITH_EXISTING_IAM_USER_ACCESS_KEY +aws_secret_access_key = REPLACE_WITH_EXISTING_IAM_USER_SECRET_KEY +; For temporary base credentials, also provide: +;aws_session_token = REPLACE_WITH_SESSION_TOKEN +``` + +Keep this file private and out of version control. The importer does not load +the standalone test script's `.env` file or its custom +`NETBOX_AWS_ACCESS_KEY_ID` / `NETBOX_AWS_SECRET_ACCESS_KEY` variables. +Alternatively, standard `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and +`AWS_SESSION_TOKEN` variables can supply base credentials; to use those for +the collector-role profile below, replace `source_profile` with +`credential_source = Environment`. Do not set both options. + +**2. AWS shared config** (`%USERPROFILE%\.aws\config` on Windows, +`~/.aws/config` on Linux): + +```ini +[profile example-inventory] +role_arn = arn:aws:iam::123456789012:role/ExampleInventoryRole +source_profile = example-inventory-client +region = us-east-1 +``` + +The `source_profile` must match the credentials section from step 1. The +`profile` prefix belongs in the shared config section, not the credentials +section. This profile performs the first role assumption; the importer performs +the member-account role assumptions. + +**3. Application settings** (add to your existing NetBox configuration): + +```ini +[source/aws] +type = aws +profile = example-inventory +organization = True +organization_role_name = ExampleInventoryRole +account_id = 123456789012 +regions = auto +discovery_region = us-east-1 +sync_private_ips = False + +; Optional: initially select one actual member account. +;organization_account_ids = 111122223333 +``` + +`profile` identifies the management-account AWS profile. +`organization_role_name` identifies the role inside each member account, even +though the profile and role names are related here. Setting `account_id` is an +optional guard for the collector account; it does not select member accounts. +The management account is excluded from EC2 scanning by default. +`sync_private_ips = False` allows the initial VM/disk/interface test without +pre-created VPC VRFs; configure VRFs and permitted subnets before enabling IP +import. + +Omit `organization_external_id` unless the member roles' trust policies require +an `sts:ExternalId` value. An external ID is not automatically the management +account ID. If the collector role itself requires an external ID, set +`external_id` in the AWS shared config profile; that is separate from the +application's member-role `organization_external_id`. + +Before running the importer, verify the first role assumption with the AWS CLI +under the same OS user and environment: + +```sh +aws sts get-caller-identity --profile example-inventory +``` + +The ARN should resemble +`arn:aws:sts::123456789012:assumed-role/ExampleInventoryRole/`, +not the original IAM user's ARN. This checks only the collector identity, not +member-role trust or all inventory permissions. Then run a dry run: + +```sh +python netbox-sync.py -n -c settings.ini +``` + +For this layout, adapt the policies below to use `ExampleInventoryRole` rather than +`NetBoxInventoryReadOnly`. The IAM user must be allowed to assume the management +role, and that role must trust the IAM user. The management role needs +Organizations access and permission to assume the member roles. Each member +role must trust +`arn:aws:iam::123456789012:role/ExampleInventoryRole` +and have the inventory read permissions (for example AWS-managed +`ReadOnlyAccess`). The importer does not create these roles or grants. + +#### Collector permissions + +Run from the management account or an appropriately delegated administrator +allowed to list organization accounts. Attach this policy to the collector user +or role, adjusting the role path and partition if needed: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "organizations:DescribeOrganization", + "organizations:ListAccounts" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::*:role/NetBoxInventoryReadOnly" + } + ] +} +``` + +Restrict the role ARN account list to your member accounts where feasible. +AWS-managed `ReadOnlyAccess` covers the EC2 inventory permissions but does not +replace the required cross-account `sts:AssumeRole` grant. + +#### Member-account inventory role + +Deploy `NetBoxInventoryReadOnly` in every selected member account, for example +with CloudFormation StackSets. Attach the EC2 read policy above or AWS-managed +`ReadOnlyAccess`. Set its trust policy to allow the **exact collector principal**. +For an example IAM user acting directly as the collector, that policy is: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::123456789012:user/example-inventory-client" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +If using the collector-role profile above, replace the principal with +`arn:aws:iam::123456789012:role/NetBoxOrganizationCollector`. +If your administrator requires an external ID, add a +`Condition: {"StringEquals": {"sts:ExternalId": ""}}` to the trust +statement and set the same `organization_external_id` in the application. +MFA-protected member-role assumption is not interactive; use an administrator- +approved unattended trust policy. SCPs and permissions boundaries must also +allow the calls. No roles or IAM policies are created by the importer. + +Adding an allowlist, excluding management, or removing/inactivating an account +changes the source's authoritative scope and can orphan previously imported +objects under the existing pruning rules. Review a dry run before changing +scope. Avoid overlapping single-account and organization source definitions. + +### VPC address-space isolation + +Each VPC uses a **distinct VRF**. When IP import encounters a VPC without an +explicit `vpc_vrf_relation`, the importer creates a VRF in NetBox using this +default name: + +```text +AWS // +``` + +For example: `AWS 123456789012/us-east-1/vpc-0123456789abcdef0`. +Use `vpc_vrf_relation` to select an existing differently named VRF; mapped +VRFs must already exist, and a missing mapped VRF aborts the run rather than +creating one. A mapping key may be `account-id/region/vpc-id` in organization +mode; this takes precedence over a legacy `region/vpc-id` mapping. For example: + +```ini +vpc_vrf_relation = 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC +``` + +A mapping must not share a VRF between distinct discovered VPCs. VRFs must remain +distinct for different VPC owners and separately configured sources too. +Organization mode validates VRF isolation across all selected accounts before +reconciliation. + +#### Shared VPCs in organization mode + +With private IP synchronization enabled, organization mode resolves attached +ENIs' network resources across all selected account snapshots. A participant +account may omit the VPC or subnet from its EC2 responses if another selected +account discovers them in the **same region**. Resolution uses exact resource +IDs and AWS `OwnerId`, never IP overlap or guessed subnet masks. In the absence +of `OwnerId`, a resource remains associated with the account that discovered it; +an ambiguous cross-account match aborts the run. + +All participants of the same shared VPC use **one owner-account VRF**: +`AWS //`. Account-specific +`vpc_vrf_relation` mappings must use the **VPC owner's account ID**, not the +participant's. VM account custom fields and cluster membership still use the +instance account. The owner account can supply network metadata even when it +has no EC2 instances. + +Include the network owner account in the same organization source when its +resources are not visible to participants. Resolution cannot borrow metadata +from another source or region. Missing resources, conflicting ownership or +CIDRs, and subnet/VPC mismatches still abort before reconciliation. If +authorization-denied discovery leaves network dependencies unavailable, affected +instance updates are deferred and existing objects retained instead. + +This changes IP placement for shared VPCs previously imported into separate +participant-account VRFs. Review a dry run and existing mappings before enabling +pruning; previously imported IPs may remain in the old VRFs until reconciled +under the normal lifecycle rules. + +There is no fallback to global address space. Subnet mask lengths come from AWS +subnet CIDRs; NetBox prefixes are neither required nor created. Identical IPs in +different VPCs remain separate NetBox IPs. + +`permitted_subnets` is required to populate IP addresses; leaving it undefined +imports no IPs, matching other source behavior. `sync_private_ips = False` +imports VMs, disks, and interfaces without creating or requiring VRFs; +previously managed IPs on discovered interfaces are retained. DNS lookup +defaults to false because private AWS DNS is often inaccessible from the sync +host. + +## Object mapping and lifecycle + +| AWS | NetBox | +| --- | --- | +| Account name | Cluster group `AWS ` | +| Account name and region | Cluster `AWS /`, type `AWS EC2` | +| Region | Logical site `AWS ` | +| EC2 Name tag | VM name and `aws_name` custom field; falls back to instance ID when missing | +| Instance ID | `aws_instance_id` custom field; serial on NetBox 4.1+ | +| Instance type | CPU and memory, plus custom field | +| AWS account ID | `aws_account_id` custom field | +| Availability Zone | Custom field, not a physical site | +| ENI ID | VM interface name and custom field | +| ENI MAC | MAC object on NetBox 4.2+; legacy interface field on older versions | +| Private IPv4 / IPv6 | IP in the VPC's existing VRF, assigned to the ENI | +| EBS volume | Virtual disk named by volume ID | +| S3 bucket (opt-in) | Metadata-only virtual disk on an account/region logical VM | +| Lambda function (opt-in) | Logical VM in an `AWS Lambda` cluster | +| EKS cluster (opt-in) | `AWS EKS` cluster with node-group and Fargate profile metadata | +| EKS EC2 worker (opt-in) | Existing EC2 VM associated with its EKS cluster | + +In single-account mode, set `account_name` to use a friendly label in the +cluster group and cluster names; it defaults to the account ID. Organization +mode uses the account name returned by AWS Organizations, falling back to the +account ID when unavailable. The account ID remains available on each VM as +`aws_account_id`. Region sites and clusters are logical containers, not claims +about physical AWS facilities or hypervisor placement. + +VM names come from the EC2 `Name` tag, falling back to the instance ID if the +tag is missing or empty. If multiple instances in the same account and region +have the same Name tag, the instance ID is appended to each VM name to keep the +names distinct. Reconciliation uses the account ID and instance ID custom +fields, so a changed Name tag updates the existing VM rather than creating a +second one. Memory uses the instance type's MiB capacity; CPU uses instance CPU +options when reported, otherwise the type's default vCPU count. This is +provisioned capacity, not utilization. + +Running instances are `active`; stopped, pending, stopping, and shutting-down +instances are `offline`. Terminated instances are excluded and subsequently +follow the existing NetBox orphaning/pruning configuration. An empty, successfully +queried account/region is authoritative when discovery has no denials. All accessible +configured regions and paginated responses are collected before reconciliation. +Authorization denials log warnings and retain existing objects. Other AWS failures, credential +failures, missing attached resources, and missing VRF mappings abort the run +before the application's NetBox write/prune phase. Transient attachment changes +may therefore require retrying the run rather than importing partial data. + +NetBox 3.7+ gets separate virtual disks; older versions get aggregate VM disk +size. Disk sizes use GiB before NetBox 4.1 and MiB from 4.1 onward, consistent +with the existing VMware source's binary-size convention. Only attached EBS +volumes are imported; instance-store disks and unattached volumes are excluded. + +The primary IPv4 is the primary private address on device-index-zero ENI; a +reported primary IPv6 on that ENI is handled similarly. Public IPv4 mappings +are stored on the ENI as `aws_public_ipv4`, not assigned as guest IPs or made +primary. IPv6 addresses are actual interface addresses even when globally +routable. Delegated address prefixes are not imported. + +An IP already assigned to another interface can be reassigned when its current +VM/device belongs to this source, or carries both `NetBox-synced` and +`NetBox-synced: Orphaned` tags. Reclamation from an orphaned owner logs a warning; +the orphan lifecycle removes source tags, so the former source may be different. +An active EC2 VM can also reclaim an IP from an `offline` owner managed by +netbox-sync (the current source's ownership or a `NetBox-synced` tag), even if +another source originally managed it. This logs a warning and clears the old +owner's primary-IP reference when applicable. An offline EC2 VM cannot take an +IP from an active managed owner, regardless of discovery order. Other non-active +statuses do not qualify as offline, and manually managed offline owners are not +eligible for this exception. +Other ownership conflicts, including unresolved owners, log an error and skip +that IP without aborting the sync. Its existing assignment is left untouched, +and it is not set as the discovered VM's primary IP. Normal write and pruning +phases still proceed. + +Only explicitly selected `tag_keys` become tags (`AWS:=`). Their +combined names must fit NetBox's 100-character limit. Existing unrelated NetBox +tags are preserved. Metadata custom fields use the `aws_` prefix. Their names +should not conflict with user-defined fields of incompatible types. + +Tag reconciliation uses the same inventory lookup as tag assignment. When that +lookup reuses an existing NetBox tag with different capitalization, its existing +name is preserved and the assigned tag is not removed as stale. This applies to +EC2 VMs, S3 bucket disks, Lambda VMs, and EKS clusters. Tags for configured keys +are still removed when they are no longer reported by AWS. + +Avoid overlapping source definitions for the same account/region. Narrowing +regions, disabling IP import, or changing permitted networks affects the +source's managed scope; review a dry run before making such changes. The +existing pruning settings determine what happens to objects no longer reported. + +## Validation + +Start with: + +```sh +python netbox-sync.py -n -c settings.ini +``` + +Tests use botocore `Stubber` and the real in-memory NetBox inventory, without +AWS credentials or network calls: + +```sh +pip install -r requirements-aws.txt pytest +python -m pytest tests/test_aws_source.py tests/test_aws_services.py +``` + +Live AWS/NetBox validation is still required before production use. This source +does not inventory RDS, load balancers, ECS/Kubernetes workloads, security groups, +software, or guest operating-system configuration. diff --git a/module/config/file_output.py b/module/config/file_output.py index 6a002f15..9acf4e5a 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -17,6 +17,7 @@ from module.sources.vmware.config import VMWareConfig from module.sources.check_redfish.config import CheckRedfishConfig from module.sources.hetzner.config import HetznerConfig +from module.sources.aws.config import AWSConfig from module.common.logging import get_logger from module.config import default_config_file_path, source_config_section_name from module.config.files import ConfigFile, ConfigFileINI, ConfigFileYAML @@ -35,7 +36,8 @@ class ConfigFileOutput(DescriptionFormatterMixin): source_config_list = [ VMWareConfig, CheckRedfishConfig, - HetznerConfig + HetznerConfig, + AWSConfig ] header = f"Welcome to the {__description__} configuration file." diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index e6e8c813..37aeb407 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1323,6 +1323,8 @@ def __init__(self, *args, **kwargs): NBInventoryItem.object_type, NBPowerPort.object_type, NBClusterGroup.object_type, + NBCluster.object_type, + NBVirtualDisk.object_type, NBVMInterface.object_type, NBVM.object_type, NBModule.object_type @@ -1522,7 +1524,6 @@ class NBVRF(NetBoxObject): object_type = "ipam.vrf" primary_key = "name" prune = False - read_only = True def __init__(self, *args, **kwargs): self.data_model = { @@ -1915,7 +1916,8 @@ def __init__(self, *args, **kwargs): "scope_id": self.scopes, # supports pre4.2.0 clusters with site "site": NBSite, - "tags": NBTagList + "tags": NBTagList, + "custom_fields": NBCustomField } super().__init__(*args, **kwargs) @@ -2032,6 +2034,7 @@ def __init__(self, *args, **kwargs): "untagged_vlan": NBVLAN, "tagged_vlans": NBVLANList, "description": 200, + "custom_fields": NBCustomField, "tags": NBTagList } super().__init__(*args, **kwargs) @@ -2134,7 +2137,8 @@ def __init__(self, *args, **kwargs): "virtual_machine": NBVM, "description": 200, "size": int, # in MB - "tags": NBTagList + "tags": NBTagList, + "custom_fields": NBCustomField } super().__init__(*args, **kwargs) diff --git a/module/sources/__init__.py b/module/sources/__init__.py index 0666da30..5cdc0f1b 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -10,6 +10,7 @@ # define all available sources here from module.sources.vmware.connection import VMWareHandler from module.sources.hetzner.connection import HetznerHandler +from module.sources.aws.connection import AWSHandler from module.sources.check_redfish.import_inventory import CheckRedfish from module.common.logging import get_logger @@ -19,7 +20,7 @@ from module.config import source_config_section_name # list of valid sources -valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler] +valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler, AWSHandler] def validate_source(source_class_object=None, state="pre"): diff --git a/module/sources/aws/client.py b/module/sources/aws/client.py new file mode 100644 index 00000000..38002b6f --- /dev/null +++ b/module/sources/aws/client.py @@ -0,0 +1,359 @@ +from contextlib import contextmanager +from dataclasses import dataclass + +from module.common.logging import get_logger + +log = get_logger() + +try: + import boto3 + from botocore.config import Config + from botocore.exceptions import BotoCoreError, ClientError +except ModuleNotFoundError as exc: + if exc.name != "boto3": + raise + boto3 = None + + +class AWSDiscoveryError(RuntimeError): + """Discovery failed; do not reconcile this run with NetBox.""" + + +@dataclass(frozen=True) +class AWSAccessCheck: + account: str | None + region: str | None + service: str + resource: str | None + operation: str + denied: bool + + +class AWSClient: + def __init__(self, profile=None, *, session=None): + if boto3 is None: + raise AWSDiscoveryError("AWS SDK missing. Install dependencies from requirements-aws.txt.") + try: + self.session = session if session is not None else boto3.Session(profile_name=profile) + except BotoCoreError as exc: + raise AWSDiscoveryError(f"Unable to initialize AWS credentials: {exc}") from exc + self.account_names = {} + self.access_denied = [] + self.access_checks = [] + + def track_access(self, client, account, region, service): + def capture_resource(params, context, **kwargs): + context["netbox_sync_resource"] = next(( + params[key] for key in ("Bucket", "clusterName", "FunctionName", "Resource", "name") + if key in params + ), None) + + def record_access(model, parsed, context, **kwargs): + code = parsed.get("Error", {}).get("Code") + denied = code in ( + "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", + "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException") + if code is None or denied or code in ( + "NoSuchTagSet", "ServerSideEncryptionConfigurationNotFoundError"): + check_account = parsed.get("Account", account) if model.name == "GetCallerIdentity" else account + self.access_checks.append(AWSAccessCheck( + check_account, region, service, context.get("netbox_sync_resource"), model.name, denied)) + + for event, handler in ( + ("before-parameter-build.*.*", capture_resource), ("after-call.*.*", record_access)): + unique_id = f"netbox-sync-access-{event}" + client.meta.events.unregister(event, unique_id=unique_id) + client.meta.events.register(event, handler, unique_id=unique_id) + + @contextmanager + def tolerate_access_denied(self, account=None, region=None, service=None, resource=None): + try: + yield + except ClientError as exc: + if exc.response["Error"]["Code"] not in ( + "AccessDenied", "AccessDeniedException", "UnauthorizedOperation", + "AuthorizationError", "AuthorizationErrorException", "Forbidden", "ForbiddenException"): + raise AWSDiscoveryError(f"Incomplete AWS discovery: {exc}") from exc + self.access_denied.append((account, region, service)) + log.warning( + f"AWS access denied (account '{account}', region '{region}', service '{service}', " + f"resource '{resource}'): {exc}. " + "Skipping inaccessible discovery; existing source objects will be retained.") + + def discover_organization(self, regions, role_name, expected_account=None, discovery_region=None, + include_management=False, external_id=None, account_ids=None, services=()): + """Collect selected accounts, recording authorization-denied scopes for retention.""" + self.access_denied = [] + self.access_checks = [] + clients = [] + current_account = None + try: + bootstrap = (discovery_region or self.session.region_name) if regions == ["auto"] else regions[0] + if not bootstrap: + raise AWSDiscoveryError( + "Organization discovery requires discovery_region or an AWS profile/environment region " + "(AWS_DEFAULT_REGION).") + config = Config(retries={"mode": "standard", "max_attempts": 5}, + connect_timeout=10, read_timeout=60) + sts = self.session.client("sts", region_name=bootstrap, config=config) + self.track_access(sts, None, None, "sts") + clients.append(sts) + identity = sts.get_caller_identity() + if expected_account is not None and identity["Account"] != expected_account: + raise AWSDiscoveryError( + f"AWS account mismatch: expected {expected_account}, authenticated as {identity['Account']}.") + partition = identity["Arn"].split(":")[1] + organizations = self.session.client("organizations", region_name=bootstrap, config=config) + self.track_access(organizations, None, None, "organizations") + clients.append(organizations) + management_account = organizations.describe_organization()["Organization"]["MasterAccountId"] + accounts = set() + account_names = {} + for page in organizations.get_paginator("list_accounts").paginate(): + for account in page["Accounts"]: + account_id = account["Id"] + if account["State"] != "ACTIVE": + log.info(f"AWS organization: excluding account '{account_id}' in state '{account['State']}'.") + continue + if account_id == management_account and not include_management: + continue + accounts.add(account_id) + account_names[account_id] = account.get("Name") or account_id + if account_ids: + missing = set(account_ids) - accounts + if missing: + raise AWSDiscoveryError( + f"Requested organization accounts are not active/eligible: {', '.join(sorted(missing))}.") + accounts.intersection_update(account_ids) + if not accounts: + raise AWSDiscoveryError("Organization discovery returned no eligible accounts.") + snapshots = {} + for current_account in sorted(accounts): + role_arn = f"arn:{partition}:iam::{current_account}:role/{role_name}" + log.info(f"AWS organization: discovering account '{current_account}' using '{role_arn}'.") + request = {"RoleArn": role_arn, "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} + if external_id: + request["ExternalId"] = external_id + credentials = None + self.track_access(sts, current_account, None, "sts") + with self.tolerate_access_denied(current_account, service="sts"): + credentials = sts.assume_role(**request)["Credentials"] + if credentials is None: + continue + member = AWSClient(session=boto3.Session( + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], region_name=bootstrap)) + options = {"services": services} if services else {} + _, snapshot = member.discover(regions, current_account, discovery_region, **options) + self.access_denied.extend(member.access_denied) + self.access_checks.extend(member.access_checks) + snapshots[current_account] = snapshot + self.account_names = account_names + return snapshots + except ClientError as exc: + with self.tolerate_access_denied(service="organizations"): + raise exc + return {} + except (BotoCoreError, KeyError, AWSDiscoveryError) as exc: + context = f" for account '{current_account}'" if current_account else "" + raise AWSDiscoveryError(f"Incomplete AWS organization discovery{context}: {exc}") from exc + finally: + for client in clients: + client.close() + + def discover(self, regions, expected_account=None, discovery_region=None, services=()): + """Collect accessible regions and record denied scopes before inventory changes.""" + clients = [] + self.access_denied = [] + self.access_checks = [] + account = expected_account + try: + config = Config(retries={"mode": "standard", "max_attempts": 5}, + connect_timeout=10, read_timeout=60) + automatic = regions == ["auto"] + bootstrap = (discovery_region or self.session.region_name) if automatic else regions[0] + if not bootstrap: + raise AWSDiscoveryError( + "Automatic region discovery requires discovery_region or an AWS profile/environment region " + "(AWS_DEFAULT_REGION).") + sts = self.session.client("sts", region_name=bootstrap, config=config) + self.track_access(sts, account, None, "account") + clients.append(sts) + account = sts.get_caller_identity()["Account"] + if expected_account is not None and account != expected_account: + raise AWSDiscoveryError( + f"AWS account mismatch: expected {expected_account}, authenticated as {account}.") + if automatic: + discovery_client = self.session.client("ec2", region_name=bootstrap, config=config) + self.track_access(discovery_client, account, None, "account") + clients.append(discovery_client) + response = discovery_client.describe_regions(AllRegions=True) + regions = sorted({ + entry["RegionName"] for entry in response["Regions"] + if entry["OptInStatus"] in ("opt-in-not-required", "opted-in") + }) + if not regions: + raise AWSDiscoveryError("Automatic region discovery returned no enabled regions.") + log.info(f"AWS account '{account}' enabled regions: {', '.join(regions)}") + snapshot = {} + for region in regions: + ec2 = self.session.client("ec2", region_name=region, config=config) + self.track_access(ec2, account, region, "ec2") + clients.append(ec2) + data = None + with self.tolerate_access_denied(account, region, "ec2"): + data = self.discover_ec2(ec2) + if data is None: + data = {"instances": [], "InstanceTypes": {}, "Vpcs": {}, "Subnets": {}, + "NetworkInterfaces": {}, "Volumes": {}, "EC2AccessDenied": True} + data["InaccessibleServices"] = [] + for service in services: + service_client = self.session.client(service, region_name=region, config=config) + self.track_access(service_client, account, region, service) + clients.append(service_client) + try: + with self.tolerate_access_denied(account, region, service): + if service == "s3": + data["Buckets"] = self.discover_s3(service_client, region, account) + elif service == "lambda": + data["Functions"] = self.discover_lambda(service_client) + elif service == "eks": + data["EKSClusters"] = self.discover_eks(service_client) + else: + raise AWSDiscoveryError(f"Unsupported AWS service '{service}'") + key = {"s3": "Buckets", "lambda": "Functions", "eks": "EKSClusters"}[service] + if key not in data: + data["InaccessibleServices"].append(service) + except (BotoCoreError, ClientError, KeyError) as exc: + raise AWSDiscoveryError( + f"Incomplete {service} discovery (account '{account}', region '{region}'): {exc}") from exc + snapshot[region] = data + return account, snapshot + except ClientError as exc: + with self.tolerate_access_denied(account, service="account"): + raise exc + return account, {} + except (BotoCoreError, KeyError) as exc: + raise AWSDiscoveryError(f"Incomplete AWS discovery; synchronization aborted: {exc}") from exc + finally: + for client in clients: + client.close() + + def discover_ec2(self, ec2): + instances = [ + instance + for page in ec2.get_paginator("describe_instances").paginate() + for reservation in page.get("Reservations", []) + for instance in reservation.get("Instances", []) + if instance["State"]["Name"] != "terminated" + ] + data = {"instances": instances} + for operation, result_key, key in ( + ("describe_network_interfaces", "NetworkInterfaces", "NetworkInterfaceId"), + ("describe_volumes", "Volumes", "VolumeId"), + ("describe_subnets", "Subnets", "SubnetId"), + ("describe_vpcs", "Vpcs", "VpcId"), + ): + data[result_key] = { + item[key]: item + for page in ec2.get_paginator(operation).paginate() + for item in page.get(result_key, []) + } + instance_types = {} + names = sorted({item["InstanceType"] for item in instances}) + for start in range(0, len(names), 100): + for page in ec2.get_paginator("describe_instance_types").paginate(InstanceTypes=names[start:start + 100]): + instance_types.update({item["InstanceType"]: item for item in page["InstanceTypes"]}) + data["InstanceTypes"] = instance_types + return data + + def discover_s3(self, client, region, account): + buckets = [] + for page in client.get_paginator("list_buckets").paginate(BucketRegion=region): + for bucket in page["Buckets"]: + with self.tolerate_access_denied(account, region, "s3", bucket["Name"]): + buckets.append(self.discover_s3_bucket(client, bucket, region, account)) + return buckets + + def discover_s3_bucket(self, client, bucket, region, account): + name = bucket["Name"] + bucket_region = bucket.get("BucketRegion") + if bucket_region is None: + location = client.get_bucket_location(Bucket=name, ExpectedBucketOwner=account) + bucket_region = location["LocationConstraint"] or "us-east-1" + if bucket_region == "EU": + bucket_region = "eu-west-1" + if bucket_region != region: + raise AWSDiscoveryError( + f"S3 bucket '{name}' returned for region '{region}' belongs to '{bucket_region}'") + request = {"Bucket": name, "ExpectedBucketOwner": account} + try: + tags = client.get_bucket_tagging(**request)["TagSet"] + except ClientError as exc: + if exc.response["Error"]["Code"] != "NoSuchTagSet": + raise + tags = [] + versioning = client.get_bucket_versioning(**request).get("Status", "Unversioned") + try: + encryption = client.get_bucket_encryption(**request)["ServerSideEncryptionConfiguration"] + except ClientError as exc: + if exc.response["Error"]["Code"] != "ServerSideEncryptionConfigurationNotFoundError": + raise + encryption = {} + return { + "Name": name, "Region": region, "Tags": tags, "Versioning": versioning, + "Encryption": encryption, "Arn": f"arn:{client.meta.partition}:s3:::{name}", + } + + def discover_lambda(self, client): + functions = [] + for page in client.get_paginator("list_functions").paginate(): + for reference in page["Functions"]: + config = client.get_function_configuration(FunctionName=reference["FunctionArn"]) + # Do not retain environment variables or other unneeded configuration. + function = {key: config[key] for key in ( + "FunctionArn", "FunctionName", "MemorySize", "State", "Runtime", "Handler", + "Timeout", "LastModified", "Architectures", "PackageType", "VpcConfig", + ) if key in config} + if function["FunctionArn"] != reference["FunctionArn"]: + raise AWSDiscoveryError(f"Lambda function identity changed for '{reference['FunctionArn']}'") + function["Tags"] = client.list_tags(Resource=function["FunctionArn"]).get("Tags", {}) + functions.append(function) + return functions + + def discover_eks(self, client): + clusters = [] + for page in client.get_paginator("list_clusters").paginate(): + for name in page["clusters"]: + cluster = client.describe_cluster(name=name)["cluster"] + if cluster["name"] != name: + raise AWSDiscoveryError(f"EKS cluster identity changed for '{name}'") + groups = [] + for group_page in client.get_paginator("list_nodegroups").paginate(clusterName=name): + for group_name in group_page["nodegroups"]: + group = client.describe_nodegroup(clusterName=name, nodegroupName=group_name)["nodegroup"] + if group["clusterName"] != name or group["nodegroupName"] != group_name: + raise AWSDiscoveryError(f"EKS node group identity changed for '{group_name}'") + groups.append({key: group[key] for key in ( + "nodegroupName", "nodegroupArn", "status", "version", "capacityType", + "instanceTypes", "scalingConfig", "subnets", "resources", + ) if key in group}) + profiles = [] + for profile_page in client.get_paginator("list_fargate_profiles").paginate(clusterName=name): + for profile_name in profile_page["fargateProfileNames"]: + profile = client.describe_fargate_profile( + clusterName=name, fargateProfileName=profile_name)["fargateProfile"] + if profile["clusterName"] != name or profile["fargateProfileName"] != profile_name: + raise AWSDiscoveryError(f"EKS Fargate profile identity changed for '{profile_name}'") + profiles.append({key: profile[key] for key in ( + "fargateProfileName", "fargateProfileArn", "status", "subnets", "selectors", + ) if key in profile}) + clusters.append({ + **{key: cluster[key] for key in ( + "name", "arn", "status", "version", "endpoint", "resourcesVpcConfig", "tags", + ) if key in cluster}, + "nodegroups": sorted(groups, key=lambda group: group["nodegroupName"]), + "fargateProfiles": sorted(profiles, key=lambda profile: profile["fargateProfileName"]), + }) + return clusters diff --git a/module/sources/aws/config.py b/module/sources/aws/config.py new file mode 100644 index 00000000..89e79c14 --- /dev/null +++ b/module/sources/aws/config.py @@ -0,0 +1,158 @@ +import re + +from module.common.logging import get_logger +from module.common.misc import quoted_split +from module.config import source_config_section_name +from module.config.base import ConfigBase +from module.config.option import ConfigOption +from module.sources.common.config import ( + config_option_enabled_definition, + config_option_permitted_subnets_definition, + config_option_type_definition, +) +from module.sources.common.permitted_subnets import PermittedSubnets + +log = get_logger() + + +class AWSConfig(ConfigBase): + section_name = source_config_section_name + source_name_example = "my-aws-example" + source_name = None + + def __init__(self): + self.options = [ + ConfigOption(**config_option_enabled_definition), + ConfigOption(**{**config_option_type_definition, "config_example": "aws"}), + ConfigOption("regions", str, default_value="auto", + description="""Use 'auto' to discover all account-enabled regions, or specify + regions to import (comma-separated or a YAML list). Do not mix auto with region names."""), + ConfigOption("discovery_region", str, + description="""Bootstrap region for automatic region discovery. Defaults to the + AWS profile/environment region. This selects the API endpoint, not resource placement."""), + ConfigOption("profile", str, + description="AWS SDK profile. Otherwise use the standard credential chain."), + ConfigOption("account_id", str, + description="""Optional expected 12-digit credential account ID. + In organization mode this guards the collector account, not member accounts."""), + ConfigOption("account_name", str, + description="Optional friendly account name for NetBox cluster labels in single-account mode."), + ConfigOption("organization", bool, default_value=False, + description="Discover active AWS Organizations accounts and assume an inventory role in each."), + ConfigOption("organization_role_name", str, default_value="NetBoxInventoryReadOnly", + description="Inventory role name (including any path) deployed in each selected account."), + ConfigOption("organization_external_id", str, sensitive=True, + description="Optional external ID required by the member inventory roles' trust policies."), + ConfigOption("organization_include_management", bool, default_value=False, + description="Also scan the organization management account using its inventory role."), + ConfigOption("organization_account_ids", str, + description="""Optional account allowlist (comma-separated or YAML list). + Unset selects all active member accounts. Management inclusion is controlled separately."""), + ConfigOption(**config_option_permitted_subnets_definition), + ConfigOption("sync_private_ips", bool, default_value=True, + description="Import private IPv4 and IPv6 into VPC-specific VRFs, creating default VRFs as needed."), + ConfigOption("sync_s3", bool, default_value=False, + description="""Import S3 buckets as metadata-only virtual disks on logical account/region VMs. + Requires NetBox 3.7+. Disk size zero is a placeholder for unknown usage, not empty storage."""), + ConfigOption("sync_lambda", bool, default_value=False, + description="Import Lambda functions as logical VMs in AWS Lambda clusters."), + ConfigOption("sync_eks", bool, default_value=False, + description="""Import EKS clusters, node-group/Fargate metadata, and associate existing + EC2 workers with their EKS cluster. Does not discover Kubernetes pods."""), + ConfigOption("vpc_vrf_relation", str, + description="""Comma-separated region/vpc-id = NetBox VRF name mappings. + Use account-id/region/vpc-id keys for account-specific mappings in organization mode. + Unmapped VPCs automatically create a VRF named AWS //. + Mapped VRF names must already exist in NetBox. + Each VPC must have a distinct VRF; global address space is never used.""", + config_example="us-east-1/vpc-0123456789abcdef0 = Production VPC"), + ConfigOption("tag_keys", str, + description="AWS tag keys to copy as namespaced NetBox tags; unset imports none."), + ConfigOption("tenant", str, description="Optional NetBox tenant for AWS VMs and IPs."), + ConfigOption("dns_name_lookup", bool, default_value=False, + description="Perform reverse DNS lookups for imported private IPs."), + ] + super().__init__() + + def validate_options(self): + for key in ("profile", "tenant", "discovery_region", "account_name", + "organization_role_name", "organization_external_id"): + value = self.get_option_by_name(key).value + if value is not None and not isinstance(value, str): + log.error(f"AWS '{self.source_name}' option '{key}' must be a string.") + self.set_validation_failed() + + role = self.get_option_by_name("organization_role_name").value + if isinstance(role, str) and ( + len(role) > 512 or not re.fullmatch(r"(?:[A-Za-z0-9_+=,.@-]+/)*[A-Za-z0-9_+=,.@-]{1,64}", role)): + log.error(f"AWS '{self.source_name}' organization_role_name must be a role name with an optional path.") + self.set_validation_failed() + external_id = self.get_option_by_name("organization_external_id").value + if isinstance(external_id, str) and not re.fullmatch(r"[A-Za-z0-9_+=,.@:/-]{2,1224}", external_id): + log.error(f"AWS '{self.source_name}' organization_external_id has an invalid format.") + self.set_validation_failed() + + for key in ("regions", "tag_keys", "organization_account_ids"): + option = self.get_option_by_name(key) + value = option.value + if value is not None and not isinstance(value, (str, list)): + log.error(f"AWS '{self.source_name}' option '{key}' must be a string or list.") + self.set_validation_failed() + continue + entries = value if isinstance(value, list) else quoted_split(value) if value else [] + if any(not isinstance(entry, str) or not entry.strip() for entry in entries): + log.error(f"AWS '{self.source_name}' option '{key}' contains an invalid entry.") + self.set_validation_failed() + continue + entries = list(dict.fromkeys(entry.strip() for entry in entries)) + if key == "regions" and not entries: + log.error(f"AWS '{self.source_name}' requires 'auto' or at least one explicit region.") + self.set_validation_failed() + if key == "regions" and "auto" in entries and entries != ["auto"]: + log.error(f"AWS '{self.source_name}' cannot mix 'auto' with explicit regions.") + self.set_validation_failed() + if key == "organization_account_ids" and any( + not re.fullmatch(r"[0-9]{12}", entry) for entry in entries): + log.error(f"AWS '{self.source_name}' organization_account_ids must contain 12-digit strings.") + self.set_validation_failed() + if key == "organization_account_ids" and value is not None and not entries: + log.error(f"AWS '{self.source_name}' organization_account_ids must be unset or nonempty.") + self.set_validation_failed() + option.set_value(entries) + + if not self.get_option_by_name("organization").value and ( + self.get_option_by_name("organization_account_ids").value or + self.get_option_by_name("organization_external_id").value or + self.get_option_by_name("organization_include_management").value): + log.error(f"AWS '{self.source_name}' organization options require organization = True.") + self.set_validation_failed() + + account = self.get_option_by_name("account_id").value + if account is not None and (not isinstance(account, str) or len(account) != 12 or not account.isdigit()): + log.error(f"AWS '{self.source_name}' account_id must be a 12-digit string.") + self.set_validation_failed() + + relation = self.get_option_by_name("vpc_vrf_relation") + mappings = {} + relation_value = relation.value + if relation_value is not None and not isinstance(relation_value, str): + log.error(f"AWS '{self.source_name}' vpc_vrf_relation must be a string.") + self.set_validation_failed() + relation_value = None + for entry in quoted_split(relation_value) if relation_value else []: + key, separator, value = entry.partition("=") + key, value = key.strip(' "\''), value.strip(' "\'') + parts = key.split("/") + if not separator or len(parts) not in (2, 3) or not all(parts) or not value or key in mappings or ( + len(parts) == 3 and not re.fullmatch(r"[0-9]{12}", parts[0])): + log.error(f"Malformed or duplicate AWS VPC/VRF mapping '{entry}'.") + self.set_validation_failed() + continue + mappings[key] = value + relation.set_value(mappings) + + permitted = self.get_option_by_name("permitted_subnets") + subnets = PermittedSubnets(permitted.value) + if subnets.validation_failed: + self.set_validation_failed() + permitted.set_value(subnets) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py new file mode 100644 index 00000000..c3559320 --- /dev/null +++ b/module/sources/aws/connection.py @@ -0,0 +1,827 @@ +from hashlib import sha256 +from ipaddress import ip_address, ip_interface, ip_network +from itertools import product + +from packaging.version import Version + +from module.common.logging import get_logger +from module.common.misc import do_error_exit +from module.common.support import normalize_mac_address +from module.netbox import ( + NBCustomField, NBCluster, NBClusterGroup, NBClusterType, NBIPAddress, + NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) +from module.netbox.inventory import NetBoxInventory +from module.netbox.connection import NetBoxHandler +from module.sources.aws.client import AWSClient, AWSDiscoveryError +from module.sources.aws.config import AWSConfig +from module.sources.common.source_base import SourceBase + +log = get_logger() + + +class AWSHandler(SourceBase): + source_type = "aws" + dependent_netbox_objects = [ + NBTag, NBCustomField, NBTenant, NBSite, NBClusterGroup, NBClusterType, + NBCluster, NBVM, NBVMInterface, NBMACAddress, NBIPAddress, NBVRF, NBVirtualDisk, + ] + + def __init__(self, name=None): + if not name: + raise ValueError(f"Invalid source name '{name}'.") + self.name = name + self.inventory = NetBoxInventory() + config = AWSConfig() + config.source_name = name + self.settings = config.parse() + self.set_source_tag() + self.init_successful = False + if not self.settings.enabled: + log.info(f"Source '{name}' is disabled. Skipping.") + return + try: + self.client = AWSClient(profile=self.settings.profile) + except AWSDiscoveryError as exc: + self.abort(exc) + self.init_successful = True + + def abort(self, error): + message = f"AWS source '{self.name}': {error}. No NetBox writes or pruning will be performed." + log.error(message) + do_error_exit(message) + + def apply(self): + try: + if self.settings.sync_s3 and Version(self.inventory.netbox_api_version) < Version("3.7"): + raise ValueError("S3 bucket inventory requires NetBox 3.7 or newer") + services = tuple(service for service in ("s3", "lambda", "eks") + if getattr(self.settings, f"sync_{service}")) + options = {"services": services} if services else {} + if self.settings.organization: + snapshots = self.client.discover_organization( + self.settings.regions, self.settings.organization_role_name, + expected_account=self.settings.account_id, + discovery_region=self.settings.discovery_region, + include_management=self.settings.organization_include_management, + external_id=self.settings.organization_external_id, + account_ids=self.settings.organization_account_ids, **options) + account_names = getattr(self.client, "account_names", {}) + else: + account, snapshot = self.client.discover( + self.settings.regions, self.settings.account_id, self.settings.discovery_region, **options) + snapshots = {account: snapshot} if account is not None else {} + account_names = {} + if self.settings.organization and self.settings.sync_private_ips: + self.resolve_shared_networks(snapshots) + self.defer_inaccessible_network_instances(snapshots) + vrfs = {} + for account, snapshot in snapshots.items(): + self.validate_services(account, snapshot) + self.validate_snapshot(account, snapshot, vrfs) + self.retain_disabled_services() + if getattr(self.client, "access_denied", []): + self.retain_inaccessible_objects() + account_labels = { + account: account_names.get(account) or self.settings.account_name or account + for account in snapshots + } + label_counts = {} + for label in account_labels.values(): + key = label.casefold() + label_counts[key] = label_counts.get(key, 0) + 1 + for account, label in account_labels.items(): + if label_counts[label.casefold()] > 1: + account_labels[account] = f"{label} ({account})" + for account, snapshot in snapshots.items(): + self.sync_snapshot(account, snapshot, account_name=account_labels[account]) + self.reconcile_access_tags() + except (AWSDiscoveryError, ValueError, KeyError) as exc: + self.abort(exc) + + def validate_services(self, account, snapshot): + for region, data in snapshot.items(): + inaccessible = data.get("InaccessibleServices", []) + for enabled, key in ( + (self.settings.sync_s3 and "s3" not in inaccessible, "Buckets"), + (self.settings.sync_lambda and "lambda" not in inaccessible, "Functions"), + (self.settings.sync_eks and "eks" not in inaccessible, "EKSClusters"), + ): + if enabled and not isinstance(data.get(key), list): + raise ValueError(f"Missing AWS {key} discovery (account '{account}', region '{region}')") + if enabled: + identity_key = {"Buckets": "Name", "Functions": "FunctionArn", "EKSClusters": "arn"}[key] + identities = [resource[identity_key] for resource in data[key]] + if len(set(identities)) != len(identities): + raise ValueError(f"Duplicate AWS {key} discovery (account '{account}', region '{region}')") + for bucket in data["Buckets"] if self.settings.sync_s3 and "s3" not in inaccessible else []: + if bucket["Region"] != region or not bucket["Name"] or len(bucket["Name"]) > 63: + raise ValueError(f"Invalid S3 bucket '{bucket['Name']}' for region '{region}'") + self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}) + if "Versioning" not in bucket or "Encryption" not in bucket: + raise ValueError(f"Incomplete S3 bucket metadata for '{bucket['Name']}'") + for function in data["Functions"] if self.settings.sync_lambda and "lambda" not in inaccessible else []: + self.validate_arn(function["FunctionArn"], "lambda", account, region) + if function["State"] not in ("Pending", "Active", "Inactive", "Failed"): + raise ValueError(f"Unknown Lambda state '{function['State']}' for '{function['FunctionArn']}'") + if not function["FunctionName"] or not isinstance(function["MemorySize"], int): + raise ValueError(f"Incomplete Lambda configuration for '{function['FunctionArn']}'") + self.service_tags(function["Tags"]) + if self.settings.sync_eks and "eks" not in inaccessible: + clusters = {cluster["name"]: cluster for cluster in data["EKSClusters"]} + if len(clusters) != len(data["EKSClusters"]): + raise ValueError(f"Duplicate EKS cluster discovery in account '{account}', region '{region}'") + asg_clusters = {} + for cluster in data["EKSClusters"]: + self.validate_arn(cluster["arn"], "eks", account, region) + for field in ("status", "version", "resourcesVpcConfig", "nodegroups", "fargateProfiles"): + if field not in cluster: + raise ValueError(f"Incomplete EKS {field} for '{cluster['arn']}'") + self.service_tags(cluster.get("tags", {})) + for group in cluster["nodegroups"]: + for asg in group.get("resources", {}).get("autoScalingGroups", []): + name = asg["name"] + if name in asg_clusters and asg_clusters[name] != cluster["name"]: + raise ValueError(f"EKS Auto Scaling group '{name}' belongs to multiple clusters") + asg_clusters[name] = cluster["name"] + data["EKSWorkers"] = {} + for instance in data["instances"]: + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + names = {tags[key] for key in ( + "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", + ) if tags.get(key)} + asg_cluster = asg_clusters.get(tags.get("aws:autoscaling:groupName")) + if asg_cluster: + names.add(asg_cluster) + if len(names) > 1: + raise ValueError(f"Conflicting EKS cluster membership for instance '{instance['InstanceId']}'") + if names: + name = next(iter(names)) + if name in clusters: + data["EKSWorkers"][instance["InstanceId"]] = name + else: + log.warning( + f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " + f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") + + def access_scope(self, item): + fields = item.data.get("custom_fields") or {} + metadata = fields.get("aws_metadata") or {} + identity = fields.get("aws_resource_id") or "" + service = "ec2" if fields.get("aws_instance_id") else fields.get("aws_service") + account = fields.get("aws_account_id") or metadata.get("account") + region = fields.get("aws_region") or metadata.get("region") + resource = None + if identity.startswith("arn:"): + parts = identity.split(":", 5) + if len(parts) == 6: + service, region, account, resource = parts[2:] + if service == "lambda": + resource = identity + elif service == "eks": + resource = resource.removeprefix("cluster/") + elif identity.startswith("s3/"): + parts = identity.split("/") + account = parts[1] + if isinstance(item, NBVirtualDisk): + resource = parts[2] + else: + region = parts[2] + for parent_key in ("virtual_machine", "cluster"): + parent = item.data.get(parent_key) + if parent is not None: + parent_account, parent_region, parent_service, _ = self.access_scope(parent) + account = account or parent_account + region = region or parent_region + service = service or parent_service + break + if isinstance(item, (NBIPAddress, NBMACAddress)): + parent = item.get_device_vm() + if parent is not None: + account, region, service, _ = self.access_scope(parent) + if account is None: + group = item if isinstance(item, NBClusterGroup) else item.data.get("group") + name = group.data.get("name", "") if group is not None else "" + if name.startswith("AWS "): + label = name[4:] + account = next((key for key, value in getattr(self.client, "account_names", {}).items() + if label in (value, f"{value} ({key})")), None) + if account is None and label.isdigit(): + account = label + if region is None: + site = item.data.get("scope_id") or item.data.get("site") + name = site.data.get("name", "") if isinstance(site, NBSite) else "" + if name.startswith("AWS "): + region = name[4:] + return account, region, service or "ec2", resource + + def reconcile_access_tags(self): + checks = getattr(self.client, "access_checks", []) + if not checks: + return + prefix = "Sync:AccessDenied:" + scopes = {} + for check in checks: + outcomes = scopes.setdefault( + (check.account, check.region, check.service, check.resource), (set(), set())) + outcomes[0 if check.denied else 1].add(check.operation) + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if object_type in (NBTag, NBCustomField, NBTenant) or "tags" not in item.data_model or not ( + item.source is self or self.source_tag in item.get_tags()): + continue + account, region, service, resource = self.access_scope(item) + denied, succeeded = set(), set() + for key in product( + (None, account), (None, region), ("organizations", "sts", "account", service), + (None, resource)): + if key in scopes: + failures, successes = scopes[key] + denied.update(failures) + succeeded.update(successes) + for operation in denied: + item.add_tags(f"{prefix}{operation}") + for tag in item.get_tags(): + if tag.startswith(prefix) and tag[len(prefix):] in succeeded - denied: + item.remove_tags(tag) + + def validate_arn(self, arn, service, account, region): + parts = arn.split(":", 5) + if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: + raise ValueError(f"Invalid {service} resource ARN '{arn}' for account '{account}', region '{region}'") + + def service_tags(self, tags): + result = [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags] + if any(len(tag) > 100 for tag in result): + raise ValueError("AWS service tag exceeds the NetBox tag name limit (100 characters)") + return result + + def reconcile_tags(self, item, tags): + # Tag lookup can reuse an existing name with different capitalization. + desired_tags = {self.inventory.get_by_data(NBTag, data={"name": name}) for name in tags} + managed_prefixes = tuple(f"AWS:{key}=".casefold() for key in self.settings.tag_keys) + for tag in item.data.get("tags", []): + if tag.get_display_name().casefold().startswith(managed_prefixes) and tag not in desired_tags: + item.remove_tags(tag) + + def retain_disabled_services(self): + for object_type in (NBVM, NBVirtualDisk, NBCluster): + for item in self.inventory.get_all_items(object_type): + service = (item.data.get("custom_fields") or {}).get("aws_service") + if service in ("s3", "lambda", "eks") and not getattr(self.settings, f"sync_{service}") and \ + (item.source is self or self.source_tag in item.get_tags()): + item.set_source(self) + + def retain_source_objects(self): + # Source-wide retention also protects children and cross-account shared-network dependencies. + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if self.source_tag in item.get_tags(): + item.set_source(self) + + def retain_inaccessible_objects(self): + self.retain_source_objects() + log.warning( + f"AWS source '{self.name}': authorization-denied discovery is incomplete. " + "Retaining all previously managed source objects this run; accessible resources will still be updated.") + + def defer_inaccessible_network_instances(self, snapshots): + if not any(service in ("ec2", "account", "sts", "organizations") + for _, _, service in getattr(self.client, "access_denied", [])): + return + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + retained = [] + for instance in data["instances"]: + missing_network = False + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is not None and (eni["VpcId"] not in data["Vpcs"] or + eni["SubnetId"] not in data["Subnets"]): + missing_network = True + if missing_network: + log.warning( + f"AWS instance '{instance['InstanceId']}' (account '{account}', region '{region}') " + "has missing network dependencies during authorization-denied discovery. " + "Deferring its update and retaining existing objects.") + else: + retained.append(instance) + data["instances"] = retained + + def resource_name(self, name, identity, limit): + if len(name) <= limit: + return name + suffix = sha256(identity.encode("utf-8")).hexdigest()[:12] + return f"{name[:limit - 13]}-{suffix}" + + def service_object(self, object_type, identity, data): + matches = [item for item in self.inventory.get_all_items(object_type) + if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + error = None + if len(matches) > 1: + error = f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'" + elif matches: + item = matches[0] + if item.source is not self and self.source_tag not in item.get_tags(): + error = f"AWS resource '{identity}' belongs to another source" + else: + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" + if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) + for item in self.inventory.get_all_items(object_type)): + error = f"NetBox {object_type.name} name collision for AWS resource '{identity}'" + if error is not None: + log.error( + f"AWS source '{self.name}': {error}. Skipping this resource and any dependent resources. " + "Retaining all previously managed source objects this run to prevent pruning skipped resources.") + self.retain_source_objects() + return None + if matches: + item.update(data=data, source=self) + else: + item = self.inventory.add_object(object_type, data=data, source=self) + self.reconcile_tags(item, data["tags"]) + return item + + def service_cluster(self, account, region, site, group, service, identity, name, + metadata=None, tags=None): + data = { + "name": self.resource_name(name, identity, 100), "group": group, + "type": {"name": f"AWS {service.upper() if service != 'lambda' else 'Lambda'}"}, + "custom_fields": {"aws_service": service, "aws_resource_id": identity, + "aws_metadata": metadata or {"account": account, "region": region}}, + "tags": self.service_tags(tags or {}), + } + if Version(self.inventory.netbox_api_version) >= Version("4.2"): + data.update(scope_type="dcim.site", scope_id=site) + else: + data["site"] = site + return self.service_object(NBCluster, identity, data) + + def logical_vm(self, account, region, site, cluster, service, identity, name, metadata, tags, **attributes): + data = { + "name": self.resource_name(name, identity, 64), "cluster": cluster, "status": "active", + "comments": f"Logical AWS {service} inventory object; not an EC2 instance.", + "custom_fields": { + "aws_account_id": account, "aws_region": region, "aws_service": service, + "aws_resource_id": identity, "aws_metadata": metadata, + }, + "tags": [f"AWS:logical-{service}", *self.service_tags(tags)], + **attributes, + } + if Version(self.inventory.netbox_api_version) >= Version("3.3"): + data["site"] = site + if self.settings.tenant: + data["tenant"] = {"name": self.settings.tenant} + return self.service_object(NBVM, identity, data) + + def sync_lambda(self, account, region, site, cluster, function): + status = {"Pending": "planned", "Active": "active", "Inactive": "offline", "Failed": "failed"} + self.logical_vm( + account, region, site, cluster, "lambda", function["FunctionArn"], + f"Lambda {function['FunctionName']}", + {key: value for key, value in function.items() if key != "Tags"}, + function["Tags"], memory=function["MemorySize"], status=status[function["State"]]) + + def sync_s3(self, account, region, site, group, buckets): + if not buckets: + return + identity = f"s3/{account}/{region}" + cluster = self.service_cluster( + account, region, site, group, "s3", identity, f"S3 {account}/{region}") + if cluster is None: + return + holder = self.logical_vm( + account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", + {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) + if holder is None: + return + for bucket in buckets: + bucket_id = f"s3/{account}/{bucket['Name']}" + self.service_object(NBVirtualDisk, bucket_id, { + "name": bucket["Name"], "virtual_machine": holder, + # NetBox requires size; zero is explicitly unknown, never a usage measurement. + "size": 0, "description": "AWS S3 bucket; usage unknown (size 0 is a placeholder, not measured).", + "custom_fields": { + "aws_service": "s3", "aws_resource_id": bucket_id, + "aws_metadata": {**{key: value for key, value in bucket.items() if key != "Tags"}, + "usage_status": "unknown", "usage_bytes": None}, + }, + "tags": self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}), + }) + + def resolve_shared_networks(self, snapshots): + networks = {} + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + for kind in ("Vpcs", "Subnets"): + for resource_id, resource in data[kind].items(): + owner = resource.get("OwnerId", account) + key = (kind, region, resource_id) + owners = networks.setdefault(key, {}) + if resource.get("OwnerId") and any( + item.get("OwnerId") and previous_owner != owner + for previous_owner, item in owners.items()): + raise ValueError( + f"Conflicting AWS {kind} ownership for '{resource_id}' " + f"(region '{region}'): {', '.join(sorted(set(owners) | {owner}))}") + if owner in owners: + previous = owners[owner] + fields = ("VpcId", "CidrBlock") + if any(previous.get(field) != resource.get(field) for field in fields) or \ + set(self.ipv6_networks(previous)) != set(self.ipv6_networks(resource)): + raise ValueError( + f"Conflicting AWS {kind} discovery for '{resource_id}' " + f"(owner '{owner}', region '{region}')") + owners[owner] = resource + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + data["VpcOwners"] = {} + for instance in data["instances"]: + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + continue + vpc_id, subnet_id = eni["VpcId"], eni["SubnetId"] + vpcs = networks.get(("Vpcs", region, vpc_id), {}) + local_vpc = data["Vpcs"].get(vpc_id) + owner = local_vpc.get("OwnerId", account) if local_vpc is not None else None + if owner is None and len(vpcs) == 1: + owner = next(iter(vpcs)) + if owner is None and vpcs: + raise ValueError( + f"Ambiguous AWS VPC ownership for '{vpc_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}'): {', '.join(sorted(vpcs))}") + if owner is None: + continue + data["VpcOwners"][vpc_id] = owner + data["Vpcs"][vpc_id] = dict(vpcs[owner], OwnerId=owner) + subnets = networks.get(("Subnets", region, subnet_id), {}) + if subnets and owner not in subnets: + raise ValueError( + f"Conflicting AWS subnet ownership for '{subnet_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}', VPC owner '{owner}'): " + f"{', '.join(sorted(subnets))}") + if owner in subnets: + data["Subnets"][subnet_id] = dict(subnets[owner], OwnerId=owner) + + def vrf_name(self, account, region, vpc): + return self.settings.vpc_vrf_relation.get( + f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( + f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) + + def get_vrf(self, account, region, vpc): + account_region_vpc = f"{account}/{region}/{vpc}" + region_vpc = f"{region}/{vpc}" + has_mapping = account_region_vpc in self.settings.vpc_vrf_relation or \ + region_vpc in self.settings.vpc_vrf_relation + name = self.vrf_name(account, region, vpc) + vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) + if vrf is None: + if has_mapping: + raise ValueError(f"Create mapped NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") + vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) + return vrf + + def ipv6_networks(self, resource): + return [ + ip_network(entry["Ipv6CidrBlock"]) + for entry in resource.get("Ipv6CidrBlockAssociationSet", []) + if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" + ] + + def addresses(self, eni, subnet): + networks = [ip_network(subnet["CidrBlock"]), *self.ipv6_networks(subnet)] + addresses = [ + (entry["PrivateIpAddress"], entry.get("Primary", False)) + for entry in eni.get("PrivateIpAddresses", []) + ] + addresses.extend( + (entry["Ipv6Address"], entry.get("IsPrimaryIpv6", False)) + for entry in eni.get("Ipv6Addresses", []) + ) + result = [] + for value, primary in addresses: + address = ip_address(value) + network = next((net for net in networks + if net.version == address.version and address in net), None) + if network is None: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' IP '{value}' has no matching AWS subnet") + value = f"{address}/{network.prefixlen}" + if self.settings.permitted_subnets.permitted(value, interface_name=eni["NetworkInterfaceId"]): + result.append((value, primary)) + return result + + def validate_snapshot(self, account, snapshot, vrfs=None): + if vrfs is None: + vrfs = {} + for region, data in snapshot.items(): + for instance in data["instances"]: + spec = data["InstanceTypes"].get(instance["InstanceType"]) + if spec is None: + raise ValueError(f"Missing AWS instance type '{instance['InstanceType']}'") + if "DefaultVCpus" not in spec["VCpuInfo"] or "SizeInMiB" not in spec["MemoryInfo"]: + raise ValueError(f"Incomplete AWS specifications for '{instance['InstanceType']}'") + for tag in instance.get("Tags", []): + if tag["Key"] in self.settings.tag_keys and len(f"AWS:{tag['Key']}={tag['Value']}") > 100: + raise ValueError(f"AWS tag '{tag['Key']}' exceeds the NetBox tag name limit (100 characters)") + for attachment in instance.get("BlockDeviceMappings", []): + volume_id = attachment.get("Ebs", {}).get("VolumeId") + if volume_id and volume_id not in data["Volumes"]: + raise ValueError(f"Missing attached EBS volume '{volume_id}'") + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + raise ValueError(f"Missing attached ENI '{reference['NetworkInterfaceId']}'") + if eni.get("Attachment", {}).get("InstanceId") != instance["InstanceId"]: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' changed attachment during discovery") + if not eni.get("MacAddress"): + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' has no MAC address") + if not self.settings.sync_private_ips: + continue + vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) + network_errors = [] + if vpc not in data["Vpcs"]: + network_errors.append(f"VPC '{vpc}' was not returned by DescribeVpcs") + if subnet is None: + network_errors.append( + f"subnet '{eni['SubnetId']}' was not returned by DescribeSubnets") + elif subnet["VpcId"] != vpc: + network_errors.append( + f"subnet '{eni['SubnetId']}' belongs to VPC '{subnet['VpcId']}', " + f"not ENI VPC '{vpc}'") + if network_errors: + raise ValueError( + f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}' " + f"(account '{account}', region '{region}', instance '{instance['InstanceId']}', " + f"VPC '{vpc}', subnet '{eni['SubnetId']}'): {'; '.join(network_errors)}") + if self.addresses(eni, subnet): + owner = data.get("VpcOwners", {}).get(vpc, account) + vrf = self.get_vrf(owner, region, vpc) + identity = (owner, region, vpc) + if vrf in vrfs and vrfs[vrf] != identity: + raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") + vrfs[vrf] = identity + + def custom_fields(self): + fields = ( + ("aws_account_id", "AWS account ID", [NBVM.object_type]), + ("aws_region", "AWS region", [NBVM.object_type]), + ("aws_instance_id", "AWS instance ID", [NBVM.object_type]), + ("aws_name", "AWS Name tag", [NBVM.object_type]), + ("aws_instance_type", "AWS instance type", [NBVM.object_type]), + ("aws_availability_zone", "AWS Availability Zone", [NBVM.object_type]), + ("aws_vpc_id", "AWS VPC ID", [NBVM.object_type, NBVMInterface.object_type]), + ("aws_eni_id", "AWS ENI ID", [NBVMInterface.object_type]), + ("aws_subnet_id", "AWS subnet ID", [NBVMInterface.object_type]), + ("aws_public_ipv4", "AWS public IPv4", [NBVMInterface.object_type]), + ) + for name, label, types in fields: + self.add_update_custom_field({ + "name": name, "label": label, "object_types": types, "type": "text", + "description": f"Reported by AWS ({self.name})", + }) + if any((self.settings.sync_s3, self.settings.sync_lambda, self.settings.sync_eks)): + types = [NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type] + for name, field_type in ( + ("aws_service", "text"), ("aws_resource_id", "text"), + ("aws_metadata", "json"), + ): + field = self.add_update_custom_field({ + "name": name, "label": name.replace("_", " ").upper(), + "object_types": types, "type": field_type, + "description": f"Reported by AWS ({self.name})", + }) + existing_type = field.data.get("type") + if isinstance(existing_type, dict): + existing_type = existing_type.get("value") + if existing_type != field_type: + raise ValueError( + f"NetBox custom field '{name}' must have type '{field_type}', not '{existing_type}'") + + def sync_snapshot(self, account, snapshot, account_name=None): + self.custom_fields() + account_name = account_name or account + group = self.inventory.add_update_object( + NBClusterGroup, data={"name": f"AWS {account_name}"}, source=self) + api_version = Version(self.inventory.netbox_api_version) + for region, data in snapshot.items(): + site = self.inventory.add_update_object( + NBSite, data={"name": f"AWS {region}"}, source=self) + cluster_data = { + "name": f"AWS {account_name}/{region}", "group": group, + "type": {"name": "AWS EC2"}, + } + if api_version >= Version("4.2"): + cluster_data.update(scope_type="dcim.site", scope_id=site) + else: + cluster_data["site"] = site + cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) + eks_clusters = {} + inaccessible = data.get("InaccessibleServices", []) + if self.settings.sync_eks and "eks" not in inaccessible: + for resource in data["EKSClusters"]: + eks_cluster = self.service_cluster( + account, region, site, group, "eks", resource["arn"], + f"EKS {account}/{region}/{resource['name']}", + metadata={key: value for key, value in resource.items() if key != "tags"}, + tags=resource.get("tags", {})) + eks_clusters[resource["name"]] = eks_cluster + if self.settings.sync_lambda and "lambda" not in inaccessible: + lambda_cluster = self.service_cluster( + account, region, site, group, "lambda", f"lambda/{account}/{region}", + f"Lambda {account}/{region}") + if lambda_cluster is not None: + for function in data["Functions"]: + self.sync_lambda(account, region, site, lambda_cluster, function) + if self.settings.sync_s3 and "s3" not in inaccessible: + self.sync_s3(account, region, site, group, data["Buckets"]) + name_counts = {} + for instance in data["instances"]: + name = next((tag["Value"].strip() for tag in instance.get("Tags", []) + if tag["Key"] == "Name" and tag["Value"].strip()), instance["InstanceId"]) + key = name.casefold() + name_counts[key] = name_counts.get(key, 0) + 1 + for instance in data["instances"]: + instance_cluster = eks_clusters.get( + data.get("EKSWorkers", {}).get(instance["InstanceId"]), cluster) + if instance_cluster is None: + log.error( + f"AWS source '{self.name}': Skipping instance '{instance['InstanceId']}' " + f"(account '{account}', region '{region}') because its EKS cluster was skipped.") + continue + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + vm_name = tags.get("Name", "").strip() or instance["InstanceId"] + if name_counts[vm_name.casefold()] > 1: + vm_name = f"{vm_name} ({instance['InstanceId']})" + self.sync_instance( + account, region, instance, data, site, + instance_cluster, + api_version, vm_name=vm_name) + + def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): + instance_id = instance["InstanceId"] + if "eks" in data.get("InaccessibleServices", []): + existing = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if existing is not None and existing.data.get("cluster") is not None: + cluster = existing.data["cluster"] + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + spec = data["InstanceTypes"][instance["InstanceType"]] + vm_data = { + "name": vm_name or tags.get("Name", "").strip() or instance_id, "cluster": cluster, + "status": "active" if instance["State"]["Name"] == "running" else "offline", + "vcpus": spec["VCpuInfo"]["DefaultVCpus"], + "memory": spec["MemoryInfo"]["SizeInMiB"], + "custom_fields": { + "aws_account_id": account, "aws_region": region, + "aws_instance_id": instance_id, "aws_name": tags.get("Name", ""), + "aws_instance_type": instance["InstanceType"], + "aws_availability_zone": instance["Placement"]["AvailabilityZone"], + "aws_vpc_id": instance.get("VpcId", ""), + }, + "tags": [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags], + } + cpu = instance.get("CpuOptions", {}) + if cpu.get("CoreCount") is not None and cpu.get("ThreadsPerCore") is not None: + vm_data["vcpus"] = cpu["CoreCount"] * cpu["ThreadsPerCore"] + if api_version >= Version("3.3"): + vm_data["site"] = site + if api_version >= Version("4.1"): + vm_data["serial"] = instance_id + if self.settings.tenant: + vm_data["tenant"] = {"name": self.settings.tenant} + disks = [ + data["Volumes"][entry["Ebs"]["VolumeId"]] + for entry in instance.get("BlockDeviceMappings", []) if "Ebs" in entry + ] + if api_version < Version("3.7"): + vm_data["disk"] = sum(volume["Size"] for volume in disks) + vm = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if vm is None: + vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) + else: + vm.update(data=vm_data, source=self) + self.reconcile_tags(vm, vm_data["tags"]) + if api_version >= Version("3.7"): + for volume in disks: + self.add_update_vm_child(NBVirtualDisk, vm, { + "name": volume["VolumeId"], "virtual_machine": vm, + "size": volume["Size"] * (1024 if api_version >= Version("4.1") else 1), + "description": f"AWS EBS {volume['VolumeType']}", + }) + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"][reference["NetworkInterfaceId"]] + self.sync_interface(account, region, vm, eni, data, api_version) + + def add_update_vm_child(self, object_type, vm, data): + # Display-name matching loses the VM's account/cluster when names repeat. + existing = next((item for item in self.inventory.get_all_items(object_type) + if item.data.get("virtual_machine") is vm and + item.data.get("name") == data["name"]), None) + if existing is None: + return self.inventory.add_object(object_type, data=data, source=self) + existing.update(data=data, source=self) + return existing + + def sync_interface(self, account, region, vm, eni, data, api_version): + interface_data = { + "name": eni["NetworkInterfaceId"], "virtual_machine": vm, "enabled": True, + "description": eni.get("Description", ""), + "custom_fields": { + "aws_eni_id": eni["NetworkInterfaceId"], "aws_vpc_id": eni.get("VpcId", ""), + "aws_subnet_id": eni.get("SubnetId", ""), + "aws_public_ipv4": ", ".join(sorted({ + address["Association"]["PublicIp"] + for address in eni.get("PrivateIpAddresses", []) + if address.get("Association", {}).get("PublicIp") + })), + }, + } + mac = normalize_mac_address(eni.get("MacAddress")) + if api_version < Version("4.2"): + interface_data["mac_address"] = mac + interface = self.add_update_vm_child(NBVMInterface, vm, interface_data) + if api_version >= Version("4.2") and mac is not None: + mac_object = next((item for item in self.inventory.get_all_items(NBMACAddress) + if item.data.get("mac_address") == mac and + item.data.get("assigned_object_id") in (None, interface)), None) + mac_data = {"mac_address": mac, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if mac_object is None: + mac_object = self.inventory.add_object(NBMACAddress, data=mac_data, source=self) + else: + mac_object.update(data=mac_data, source=self) + for old in interface.get_mac_addresses(): + if old is not mac_object: + old.remove_interface_association() + interface.update(data={"primary_mac_address": mac_object}, source=self) + if not self.settings.sync_private_ips: + for address in interface.get_ip_addresses(): + if self.source_tag in address.get_tags(): + address.set_source(self) + return + subnet = data["Subnets"][eni["SubnetId"]] + addresses = self.addresses(eni, subnet) + discovered = [] + for address, primary in addresses: + owner = data.get("VpcOwners", {}).get(eni["VpcId"], account) + vrf = self.get_vrf(owner, region, eni["VpcId"]) + parsed = ip_interface(address) + candidates = [ + item for item in self.inventory.get_all_items(NBIPAddress) + if ip_interface(item.data["address"]).ip == parsed.ip and item.data.get("vrf") is vrf + ] + if len(candidates) > 1: + raise ValueError(f"Multiple NetBox IPs match '{address}' in VRF '{vrf.data['name']}'") + existing = candidates[0] if candidates else None + if existing is not None and existing.data.get("assigned_object_id") is not None and \ + existing.get_interface() is not interface: + owner = existing.get_device_vm() + owner_tags = owner.get_tags() if owner is not None else [] + same_source = owner is not None and ( + owner.source is self or self.source_tag in owner_tags) + managed_owner = same_source or NetBoxHandler.primary_tag in owner_tags + managed_orphan = NetBoxHandler.primary_tag in owner_tags and \ + NetBoxHandler.orphaned_tag in owner_tags + owner_status = owner.data.get("status") if owner is not None else None + if isinstance(owner_status, dict): + owner_status = owner_status.get("value") + if managed_owner and owner_status == "active" and vm.data["status"] == "offline": + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"is assigned to active owner '{owner.get_display_name()}'. " + f"Skipping reassignment to offline ENI '{eni['NetworkInterfaceId']}'.") + continue + offline_to_active = managed_owner and owner_status == "offline" and vm.data["status"] == "active" + if not (same_source or managed_orphan or offline_to_active): + owner_name = owner.get_display_name() if owner is not None else "unresolved owner" + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"belongs to another source (owner '{owner_name}'). " + f"Skipping reassignment to ENI '{eni['NetworkInterfaceId']}'.") + continue + if managed_orphan or offline_to_active: + reason = "orphaned" if managed_orphan else "offline" + log.warning( + f"AWS source '{self.name}': Reclaiming IP '{address}' in VRF '{vrf.data['name']}' " + f"from {reason} owner '{owner.get_display_name()}' " + f"for ENI '{eni['NetworkInterfaceId']}'.") + ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if self.settings.tenant: + ip_data["tenant"] = {"name": self.settings.tenant} + if existing is None: + existing = self.inventory.add_object(NBIPAddress, data=ip_data, source=self) + else: + existing.update(data=ip_data, source=self) + discovered.append(existing) + if eni.get("Attachment", {}).get("DeviceIndex") == 0 and primary: + vm.update(data={f"primary_ip{parsed.version}": existing}, source=self) + for old in interface.get_ip_addresses(): + if old not in discovered and (old.source is self or self.source_tag in old.get_tags()): + old.remove_interface_association() diff --git a/pyproject.toml b/pyproject.toml index 9c284b23..0cf28b08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ dependencies = [ ] [project.optional-dependencies] +aws = [ + "boto3>=1.40,<2", +] test = [ "pytest>=9.1.1", ] diff --git a/requirements-aws.txt b/requirements-aws.txt new file mode 100644 index 00000000..368a9493 --- /dev/null +++ b/requirements-aws.txt @@ -0,0 +1,2 @@ +-r requirements.txt +boto3>=1.40,<2 diff --git a/settings-example.ini b/settings-example.ini index 75b725ff..8d6b8254 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -611,4 +611,61 @@ inventory_file_path = /full/path/to/inventory/files ; Hetzner Cloud API token used to authenticate against the Hetzner Cloud API. api_token = +[source/my-aws-example] + +; Install optional SDK: pip install -r requirements-aws.txt +; See docs/source_aws.md for read-only IAM permissions and VPC VRF setup. +;enabled = True +;type = aws +regions = auto +; Bootstrap endpoint for auto; defaults to the AWS profile/environment region. +;discovery_region = us-east-1 +; Alternatively restrict discovery: regions = us-east-1, us-west-2 +;profile = inventory +;account_id = 123456789012 +; Enable organization scanning using the profile's collector credentials. +;organization = False +;organization_role_name = NetBoxInventoryReadOnly +;organization_include_management = False +; Optional allowlist for an initial member-account test; unset scans all active members. +;organization_account_ids = 111122223333, 444455556666 +;organization_external_id = +; See docs/source_aws.md for collector permissions and member-role trust policies. +;sync_private_ips = True +; Optional AWS services require additional read permissions; all default to False. +; S3 uses logical account/region VMs and metadata-only disks (NetBox 3.7+). +;sync_s3 = False +;sync_lambda = False +;sync_eks = False +;permitted_subnets = 10.0.0.0/8, fd00::/8 +;vpc_vrf_relation = us-east-1/vpc-0123456789abcdef0 = Production VPC +; Organization account-specific key: 111122223333/us-east-1/vpc-0123456789abcdef0 = Production VPC +;tag_keys = Environment, Owner +;tenant = Operations +;dns_name_lookup = False + +; Alternative organization setup: uncomment this source instead of the AWS +; source above, not alongside an overlapping source. +; All identifiers below are fictitious examples; replace with your deployment values. +; AWS shared credentials: [example-inventory-client] with the existing IAM user keys. +; AWS shared config: +; [profile example-inventory] +; role_arn = arn:aws:iam::123456789012:role/ExampleInventoryRole +; source_profile = example-inventory-client +; region = us-east-1 +; These files must be available to the OS user/environment running netbox-sync. +; See docs/source_aws.md for complete examples and member-role trust requirements. +;[source/aws-organization] +;type = aws +;profile = example-inventory +;organization = True +;organization_role_name = ExampleInventoryRole +;account_id = 123456789012 +;regions = auto +;discovery_region = us-east-1 +;sync_private_ips = False +; Optional initial scope: replace with an actual member account ID. +;organization_account_ids = 111122223333 +; Leave organization_external_id unset unless the member trust policy requires it. + ;EOF diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py new file mode 100644 index 00000000..01cde538 --- /dev/null +++ b/tests/test_aws_services.py @@ -0,0 +1,865 @@ +import copy +from contextlib import ExitStack +from unittest.mock import Mock + +import pytest + +boto3 = pytest.importorskip("boto3") +from botocore.stub import Stubber + +from module.netbox import ( + NBCustomField, NBCluster, NBClusterGroup, NBIPAddress, NBTag, NBVirtualDisk, NBVM, NBVMInterface, +) +from module.sources import instantiate_sources +from module.sources.aws import client as aws_client +from module.sources.aws.client import AWSAccessCheck, AWSClient, AWSDiscoveryError +from test_aws_source import ( + ACCOUNT, INSTANCE, REGION, SECOND_ACCOUNT, VPC, assume_member, identity, inventory_data, + make_source, org_sdk, organization_accounts, stub_region, windows_config_paths, +) +from test_orphaned_object_status import make_netbox_handler + +FUNCTION_ARN = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:inventory-function" +CLUSTER_ARN = f"arn:aws:eks:{REGION}:{ACCOUNT}:cluster/inventory-cluster" + + +def service_data(): + data = inventory_data() + data.update( + Buckets=[{ + "Name": "inventory-bucket", "Region": REGION, "Versioning": "Enabled", + "Encryption": {"Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}]}, + "Tags": [{"Key": "Environment", "Value": "test"}], + }], + Functions=[{ + "FunctionArn": FUNCTION_ARN, "FunctionName": "inventory-function", "MemorySize": 128, + "State": "Active", "Runtime": "python3.12", "Timeout": 30, "Handler": "app.handler", + "LastModified": "2026-10-27T00:00:00.000+0000", "Architectures": ["arm64"], + "Tags": {"Environment": "test"}, + }], + EKSClusters=[{ + "arn": CLUSTER_ARN, "name": "inventory-cluster", "version": "1.34", "status": "ACTIVE", + "resourcesVpcConfig": {"vpcId": VPC}, "endpoint": "https://example.invalid", + "nodegroups": [{ + "nodegroupName": "workers", "status": "ACTIVE", + "resources": {"autoScalingGroups": [{"name": "eks-workers"}]}, + }], + "fargateProfiles": [{"fargateProfileName": "serverless", "status": "ACTIVE"}], + "tags": {"Environment": "test"}, + }], + ) + data["instances"][0]["Tags"].append({"Key": "aws:autoscaling:groupName", "Value": "eks-workers"}) + return data + + +@pytest.mark.parametrize("version", ["3.7.0", "4.1.0", "4.3.0"]) +def test_services_import_and_repeat(make_source, inventory, version): + inventory.netbox_api_version = version + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + source.apply() + vms = inventory.get_all_items(NBVM) + assert len(vms) == 3 + worker = next(vm for vm in vms if vm.data["custom_fields"].get("aws_instance_id") == INSTANCE) + function = next(vm for vm in vms if vm.data["custom_fields"].get("aws_service") == "lambda") + holder = next(vm for vm in vms if vm.data["custom_fields"].get("aws_service") == "s3") + assert worker.data["cluster"].data["type"].data["name"] == "AWS EKS" + assert function.data["cluster"].data["type"].data["name"] == "AWS Lambda" + assert function.data["memory"] == 128 + assert function.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN + assert "vcpus" not in function.data and "vcpus" not in holder.data + assert "memory" not in holder.data + assert function.data["status"] == "active" + assert "AWS:logical-lambda" in function.get_tags() + assert "AWS:Environment=test" in function.get_tags() + assert not inventory.get_all_interfaces(function) + assert not inventory.get_all_interfaces(holder) + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + assert bucket.data["virtual_machine"] is holder + assert bucket.data["size"] == 0 + assert bucket.data["custom_fields"]["aws_metadata"]["usage_bytes"] is None + assert bucket.data["custom_fields"]["aws_metadata"]["usage_status"] == "unknown" + assert bucket.data["custom_fields"]["aws_metadata"]["Versioning"] == "Enabled" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + metadata = worker.data["cluster"].data["custom_fields"]["aws_metadata"] + assert metadata["nodegroups"][0]["nodegroupName"] == "workers" + assert metadata["fargateProfiles"][0]["fargateProfileName"] == "serverless" + source.client.discover.assert_called_with( + [REGION], None, None, services=("s3", "lambda", "eks")) + field = inventory.get_by_data(NBCustomField, data={"name": "aws_metadata"}) + assert field.data.get("object_types", field.data.get("content_types")) == [ + NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type, + ] + + +def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): + inventory.netbox_api_version = "3.6.0" + source = make_source("sync_s3 = True") + with pytest.raises(SystemExit): + source.apply() + assert "requires NetBox 3.7" in caplog.text + source.client.discover.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + + +@pytest.mark.parametrize("key,setting", [ + ("Buckets", "sync_s3"), ("Functions", "sync_lambda"), ("EKSClusters", "sync_eks"), +]) +def test_enabled_service_requires_complete_snapshot(make_source, inventory, key, setting, caplog): + source = make_source(f"{setting} = True") + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + assert f"Missing AWS {key} discovery" in caplog.text + + +def test_disabled_service_objects_are_retained(make_source, inventory): + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=service_data()) + source.apply() + managed = [] + for kind in (NBVM, NBVirtualDisk, NBCluster): + for item in inventory.get_all_items(kind): + if item.data.get("custom_fields", {}).get("aws_service"): + item.add_tags(source.source_tag) + item.source = None + managed.append(item) + source.settings.sync_s3 = False + source.settings.sync_lambda = False + source.settings.sync_eks = False + source.apply() + assert managed and all(item.source is source for item in managed) + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + assert all(handler.orphaned_tag not in item.get_tags() for item in managed) + worker = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_instance_id") == INSTANCE) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EC2" + + +def test_absent_enabled_resources_are_not_retained(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True", data=data) + source.apply() + resources = [item for kind in (NBVM, NBVirtualDisk) for item in inventory.get_all_items(kind) + if item.data.get("custom_fields", {}).get("aws_service")] + for item in resources: + item.add_tags([source.source_tag, make_netbox_handler().primary_tag]) + item.source = None + data["Buckets"] = [] + data["Functions"] = [] + source.apply() + assert resources and all(item.source is None for item in resources) + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + assert all(handler.orphaned_tag in item.get_tags() for item in resources) + + +@pytest.mark.parametrize("state,status", [ + ("Pending", "planned"), ("Inactive", "offline"), ("Failed", "failed"), +]) +def test_lambda_states(make_source, inventory, state, status): + data = service_data() + data["Functions"][0]["State"] = state + make_source("sync_lambda = True", data=data).apply() + function = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda") + assert function.data["status"] == status + + +def test_long_names_and_same_lambda_names_across_regions(make_source, inventory): + data = service_data() + function = data["Functions"][0] + function["FunctionName"] = "x" * 64 + function["FunctionArn"] = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:{function['FunctionName']}" + other = copy.deepcopy(data) + other["instances"] = [] + other["Functions"][0]["FunctionArn"] = function["FunctionArn"].replace(REGION, "us-west-2") + source = make_source("sync_lambda = True", data=data) + source.client.discover.return_value = (ACCOUNT, {REGION: data, "us-west-2": other}) + source.apply() + source.apply() + functions = [vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda"] + assert len(functions) == 2 + assert all(len(vm.data["name"]) <= 64 for vm in functions) + assert len({vm.data["name"] for vm in functions}) == 2 + + +def test_services_do_not_overwrite_unmanaged_objects(make_source, inventory, caplog): + data = service_data() + source = make_source("sync_lambda = True", data=data) + source.apply() + function = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda") + function.source = None + before = copy.deepcopy(function.data["custom_fields"]) + data["Functions"][0]["MemorySize"] = 256 + source.apply() + assert "belongs to another source" in caplog.text + assert function.source is None + assert function.data["custom_fields"] == before + assert function.data["memory"] == 128 + + +@pytest.mark.parametrize("conflict", ["name", "duplicate_identity", "other_source"]) +def test_lambda_cluster_conflict_continues_and_retains_children( + make_source, inventory, caplog, conflict): + source = make_source("sync_lambda = True\nsync_private_ips = False", data=service_data()) + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": f"AWS {ACCOUNT}"}) + fields = {} if conflict == "name" else {"aws_resource_id": f"lambda/{ACCOUNT}/{REGION}"} + cluster = inventory.add_object(NBCluster, data={ + "name": f"Lambda {ACCOUNT}/{REGION}", "group": group, "custom_fields": fields, + }) + if conflict == "duplicate_identity": + inventory.add_object(NBCluster, data={ + "name": "Duplicate", "group": group, "custom_fields": fields, + }) + retained = inventory.add_object(NBVM, data={ + "name": "Existing function", "cluster": cluster, "tags": [source.source_tag], + }) + source.apply() + inventory.tag_all_the_things(make_netbox_handler()) + assert "Skipping" in caplog.text + assert retained.source is source + assert make_netbox_handler().orphaned_tag not in retained.get_tags() + assert cluster.source is None + assert cluster.data.get("custom_fields", {}) == fields + assert any(vm.data.get("custom_fields", {}).get("aws_instance_id") == INSTANCE + for vm in inventory.get_all_items(NBVM)) + assert not any(vm.data.get("custom_fields", {}).get("aws_resource_id") == FUNCTION_ARN + for vm in inventory.get_all_items(NBVM)) + + +@pytest.mark.parametrize("stage", ["cluster", "holder", "bucket"]) +def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, caplog, stage): + data = service_data() + data["Buckets"].append({**data["Buckets"][0], "Name": "second"}) + source = make_source("sync_s3 = True\nsync_private_ips = False", data=data) + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": f"AWS {ACCOUNT}"}) + identity = f"s3/{ACCOUNT}/{REGION}" + cluster = inventory.add_object(NBCluster, data={ + "name": f"S3 {ACCOUNT}/{REGION}", "group": group, + "custom_fields": {} if stage == "cluster" else {"aws_resource_id": identity}, + }, source=source) + if stage != "cluster": + holder = inventory.add_object(NBVM, data={ + "name": f"S3 inventory {ACCOUNT}/{REGION}", "cluster": cluster, + "custom_fields": {} if stage == "holder" else {"aws_resource_id": identity}, + }, source=source) + if stage == "bucket": + inventory.add_object(NBVirtualDisk, data={ + "name": "inventory-bucket", "virtual_machine": holder, "size": 0, + }) + source.apply() + assert "name collision" in caplog.text + buckets = [disk.data["name"] for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data.get("custom_fields", {}).get("aws_service") == "s3"] + assert buckets == (["second"] if stage == "bucket" else []) + assert any(vm.data.get("custom_fields", {}).get("aws_instance_id") == INSTANCE + for vm in inventory.get_all_items(NBVM)) + + +def test_eks_conflict_skips_workers_and_continues_next_region(make_source, inventory, caplog): + data = service_data() + source = make_source("sync_eks = True\nsync_private_ips = False", data=data) + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": f"AWS {ACCOUNT}"}) + inventory.add_object(NBCluster, data={ + "name": f"EKS {ACCOUNT}/{REGION}/inventory-cluster", "group": group, + }) + next_region = inventory_data() + next_region["EKSClusters"] = [] + next_region["instances"][0]["InstanceId"] = "i-next-region" + for eni in next_region["NetworkInterfaces"].values(): + eni["Attachment"]["InstanceId"] = "i-next-region" + source.client.discover.return_value = ACCOUNT, {REGION: data, "us-west-2": next_region} + source.apply() + assert f"Skipping instance '{INSTANCE}'" in caplog.text + instances = [vm.data.get("custom_fields", {}).get("aws_instance_id") + for vm in inventory.get_all_items(NBVM)] + assert INSTANCE not in instances + assert "i-next-region" in instances + + +def test_lambda_function_conflict_continues_next_function(make_source, inventory, caplog): + data = service_data() + source = make_source("sync_lambda = True\nsync_private_ips = False", data=data) + source.apply() + cluster = next(item for item in inventory.get_all_items(NBCluster) + if item.data.get("custom_fields", {}).get("aws_service") == "lambda") + inventory.add_object(NBVM, data={"name": "Lambda second", "cluster": cluster}) + data["Functions"].extend([ + {**data["Functions"][0], "FunctionName": name, "FunctionArn": f"{FUNCTION_ARN}-{name}"} + for name in ("second", "third") + ]) + source.apply() + assert "name collision" in caplog.text + assert any(vm.data["name"] == "Lambda third" for vm in inventory.get_all_items(NBVM)) + + +def test_access_tracking_replaces_scope_and_records_operations(service_sdk): + client = AWSClient() + sts = service_sdk["sts"].client + client.track_access(sts, ACCOUNT, None, "sts") + client.track_access(sts, SECOND_ACCOUNT, None, "sts") + params = {"RoleArn": f"arn:aws:iam::{SECOND_ACCOUNT}:role/reports", "RoleSessionName": "test"} + service_sdk["sts"].add_client_error("assume_role", service_error_code="AccessDenied", expected_params=params) + with client.tolerate_access_denied(SECOND_ACCOUNT, service="sts"): + sts.assume_role(**params) + assert client.access_checks == [ + AWSAccessCheck(SECOND_ACCOUNT, None, "sts", None, "AssumeRole", True)] + + +@pytest.mark.parametrize("scope", ["resource", "service", "account", "organization"]) +def test_permission_tags_require_verified_recovery(make_source, inventory, scope): + source = make_source("sync_s3 = True", data=service_data()) + source.apply() + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + bucket.add_tags("user-tag") + unmanaged = inventory.add_object(NBVirtualDisk, data={ + "name": "unmanaged", "virtual_machine": bucket.data["virtual_machine"], "size": 0, + }) + account = None if scope == "organization" else ACCOUNT + region = REGION if scope in ("resource", "service") else None + service = {"resource": "s3", "service": "s3", "account": "sts", + "organization": "organizations"}[scope] + resource = "inventory-bucket" if scope == "resource" else None + operation = "GetBucketTagging" + tag = f"Sync:AccessDenied:{operation}" + failure = AWSAccessCheck(account, region, service, resource, operation, True) + success = AWSAccessCheck(account, region, service, resource, operation, False) + source.client.access_checks = [failure] + source.reconcile_access_tags() + assert tag in bucket.get_tags() + assert tag not in unmanaged.get_tags() + for checks in ([], [AWSAccessCheck(ACCOUNT, REGION, "s3", "other", operation, False)], + [success, failure]): + source.client.access_checks = checks + source.reconcile_access_tags() + assert tag in bucket.get_tags() + source.client.access_checks = [success] + source.reconcile_access_tags() + assert tag not in bucket.get_tags() + assert "user-tag" in bucket.get_tags() + + +def test_permission_tags_do_not_leak_across_scopes(make_source, inventory): + source = make_source("sync_s3 = True", data=service_data()) + source.custom_fields() + objects = [] + for account, region, service in ( + (ACCOUNT, REGION, "s3"), (SECOND_ACCOUNT, REGION, "s3"), + (ACCOUNT, "us-west-2", "s3"), (ACCOUNT, REGION, "lambda")): + objects.append(inventory.add_object(NBVM, data={ + "name": f"{account}-{region}-{service}", "tags": [source.source_tag], + "custom_fields": { + "aws_account_id": account, "aws_region": region, "aws_service": service, + }, + })) + source.client.access_checks = [AWSAccessCheck(ACCOUNT, REGION, "s3", None, "ListBuckets", True)] + source.reconcile_access_tags() + assert ["Sync:AccessDenied:ListBuckets" in item.get_tags() for item in objects] == [ + True, False, False, False] + + +def test_bucket_permission_tag_through_sdk_denial_and_recovery(service_sdk, inventory, load_config): + load_config(f""" +[source/services] +type = aws +regions = {REGION} +sync_s3 = True +sync_private_ips = False +""") + source, = instantiate_sources() + tag = "Sync:AccessDenied:GetBucketTagging" + for denied in (False, True, False): + identity(service_sdk["sts"]) + stub_region(service_sdk["ec2"], inventory_data()) + if denied: + service_sdk["s3"].add_response("list_buckets", {"Buckets": [{ + "Name": "inventory-bucket", "BucketRegion": REGION, + }]}, {"BucketRegion": REGION}) + service_sdk["s3"].add_client_error("get_bucket_tagging", service_error_code="AccessDenied", + expected_params={ + "Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT, + }) + else: + stub_s3(service_sdk["s3"]) + source.apply() + inventory.tag_all_the_things(make_netbox_handler()) + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + assert (tag in bucket.get_tags()) is denied + assert make_netbox_handler().orphaned_tag not in bucket.get_tags() + assert any(check.operation == "GetBucketTagging" and check.resource == "inventory-bucket" + and check.denied is denied for check in source.client.access_checks) + bucket.source = None + + +@pytest.mark.parametrize("failure", ["arn", "membership", "tag", "state"]) +def test_invalid_service_metadata_aborts_before_sync(make_source, inventory, failure): + data = service_data() + if failure == "arn": + data["Functions"][0]["FunctionArn"] = FUNCTION_ARN.replace(ACCOUNT, SECOND_ACCOUNT) + elif failure == "membership": + data["instances"][0]["Tags"].append({"Key": "eks:cluster-name", "Value": "different"}) + elif failure == "tag": + data["Functions"][0]["Tags"]["Environment"] = "x" * 101 + else: + data["Functions"][0]["State"] = "Unknown" + source = make_source("sync_lambda = True\nsync_eks = True", data=data) + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + + +@pytest.fixture +def service_sdk(monkeypatch): + session = boto3.Session( + aws_access_key_id="test-only", aws_secret_access_key="test-only", region_name=REGION) + clients = {service: session.client(service) for service in ("sts", "ec2", "s3", "lambda", "eks")} + fake_session = Mock(region_name=REGION) + fake_session.client.side_effect = lambda service, **kwargs: clients[service] + monkeypatch.setattr(aws_client.boto3, "Session", Mock(return_value=fake_session)) + with ExitStack() as stack: + stubs = {service: stack.enter_context(Stubber(client)) for service, client in clients.items()} + yield stubs + for stub in stubs.values(): + stub.assert_no_pending_responses() + + +def stub_s3(stub, present=True): + stub.add_response("list_buckets", { + "Buckets": [{"Name": "inventory-bucket", "BucketRegion": REGION}], + "ContinuationToken": "next", + }, {"BucketRegion": REGION}) + params = {"Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT} + if present: + stub.add_response("get_bucket_tagging", { + "TagSet": [{"Key": "Environment", "Value": "test"}], + }, params) + else: + stub.add_client_error("get_bucket_tagging", service_error_code="NoSuchTagSet", expected_params=params) + stub.add_response("get_bucket_versioning", {"Status": "Enabled"}, params) + stub.add_response("get_bucket_encryption", { + "ServerSideEncryptionConfiguration": { + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}], + }, + }, params) + stub.add_response("list_buckets", {"Buckets": []}, {"BucketRegion": REGION, "ContinuationToken": "next"}) + + +def stub_lambda(stub): + config = service_data()["Functions"][0] + stub.add_response("list_functions", { + "Functions": [{"FunctionName": config["FunctionName"], "FunctionArn": FUNCTION_ARN}], + "NextMarker": "next", + }, {}) + stub.add_response("get_function_configuration", { + **{key: value for key, value in config.items() if key != "Tags"}, + "Environment": {"Variables": {"DO_NOT_IMPORT": "sensitive"}}, + }, {"FunctionName": FUNCTION_ARN}) + stub.add_response("list_tags", {"Tags": config["Tags"]}, {"Resource": FUNCTION_ARN}) + stub.add_response("list_functions", {"Functions": []}, {"Marker": "next"}) + + +def stub_eks(stub): + cluster = service_data()["EKSClusters"][0] + name = cluster["name"] + stub.add_response("list_clusters", {"clusters": [name], "nextToken": "next"}, {}) + stub.add_response("describe_cluster", { + "cluster": {key: value for key, value in cluster.items() if key not in ("nodegroups", "fargateProfiles")}, + }, {"name": name}) + stub.add_response("list_nodegroups", {"nodegroups": ["workers"], "nextToken": "next"}, {"clusterName": name}) + stub.add_response("describe_nodegroup", { + "nodegroup": {**cluster["nodegroups"][0], "clusterName": name}, + }, {"clusterName": name, "nodegroupName": "workers"}) + stub.add_response("list_nodegroups", {"nodegroups": []}, {"clusterName": name, "nextToken": "next"}) + stub.add_response("list_fargate_profiles", { + "fargateProfileNames": ["serverless"], "nextToken": "next", + }, {"clusterName": name}) + stub.add_response("describe_fargate_profile", { + "fargateProfile": {**cluster["fargateProfiles"][0], "clusterName": name}, + }, {"clusterName": name, "fargateProfileName": "serverless"}) + stub.add_response("list_fargate_profiles", {"fargateProfileNames": []}, {"clusterName": name, "nextToken": "next"}) + stub.add_response("list_clusters", {"clusters": []}, {"nextToken": "next"}) + + +def test_sdk_services_to_real_inventory(service_sdk, inventory, load_config): + identity(service_sdk["sts"]) + stub_region(service_sdk["ec2"], service_data()) + stub_s3(service_sdk["s3"]) + stub_lambda(service_sdk["lambda"]) + stub_eks(service_sdk["eks"]) + load_config(f""" +[source/services] +type = aws +regions = {REGION} +sync_s3 = True +sync_lambda = True +sync_eks = True +sync_private_ips = False +""") + source, = instantiate_sources() + source.apply() + assert len(inventory.get_all_items(NBVM)) == 3 + function = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_service") == "lambda") + assert "Environment" not in function.data["custom_fields"]["aws_metadata"] + assert "sensitive" not in str(function.data) + + +@pytest.mark.parametrize("service,operation", [ + ("s3", "list_buckets"), ("lambda", "list_functions"), ("eks", "list_clusters"), +]) +def test_service_access_denied_continues(service_sdk, inventory, load_config, service, operation, caplog): + identity(service_sdk["sts"]) + stub_region(service_sdk["ec2"], inventory_data()) + service_sdk[service].add_client_error(operation, service_error_code="AccessDenied", http_status_code=403) + load_config(f""" +[source/services] +type = aws +regions = {REGION} +sync_{service} = True +""") + source, = instantiate_sources() + source.apply() + assert "AccessDenied" in caplog.text and "Retaining all previously managed source objects" in caplog.text + assert len(inventory.get_all_items(NBVM)) == 1 + assert source.client.access_denied == [(ACCOUNT, REGION, service)] + + +def test_optional_services_propagate_to_member_accounts(org_sdk, monkeypatch): + organization_accounts(org_sdk) + for account in (ACCOUNT, SECOND_ACCOUNT): + assume_member(org_sdk, account) + discovery = Mock(side_effect=lambda regions, account, discovery_region, **kwargs: (account, {})) + monkeypatch.setattr(AWSClient, "discover", discovery) + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly", services=("s3", "lambda", "eks")) + assert discovery.call_count == 2 + for call in discovery.call_args_list: + assert call.kwargs == {"services": ("s3", "lambda", "eks")} + + +def test_service_defaults_preserve_ec2_only(make_source): + source = make_source() + assert not any((source.settings.sync_s3, source.settings.sync_lambda, source.settings.sync_eks)) + source.apply() + source.client.discover.assert_called_once_with([REGION], None, None) + + +def test_incompatible_metadata_custom_field_aborts(make_source, inventory, caplog): + inventory.add_object(NBCustomField, data={ + "id": 1, "name": "aws_metadata", "type": "text", "object_types": [NBVM.object_type], + }, read_from_netbox=True) + source = make_source("sync_s3 = True", data=service_data()) + with pytest.raises(SystemExit): + source.apply() + assert "must have type 'json'" in caplog.text + assert inventory.get_all_items(NBVM) == [] + + +def test_s3_holders_are_per_account_and_region(make_source, inventory, monkeypatch): + data = service_data() + data["instances"] = [] + second = copy.deepcopy(data) + second["Buckets"][0]["Name"] = "second-account-bucket" + other_region = copy.deepcopy(data) + other_region["Buckets"][0].update(Name="other-region-bucket", Region="us-west-2") + source = make_source("organization = True\nsync_s3 = True") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: data, "us-west-2": other_region}, + SECOND_ACCOUNT: {REGION: second}, + })) + source.apply() + assert len(inventory.get_all_items(NBVM)) == 3 + assert len(inventory.get_all_items(NBVirtualDisk)) == 3 + assert {vm.data["name"] for vm in inventory.get_all_items(NBVM)} == { + f"S3 inventory {ACCOUNT}/{REGION}", f"S3 inventory {ACCOUNT}/us-west-2", + f"S3 inventory {SECOND_ACCOUNT}/{REGION}", + } + + +@pytest.mark.parametrize("tag_name", ["AWS:Environment=Dev", "aws:environment=Dev"]) +def test_service_tags_use_resolved_identity(make_source, inventory, tag_name): + tag = inventory.add_object(NBTag, data={"name": tag_name}) + data = service_data() + data["Buckets"][0]["Tags"] = [{"Key": "Environment", "Value": "dev"}] + data["Functions"][0]["Tags"] = {"Environment": "dev"} + data["EKSClusters"][0]["tags"] = {"Environment": "dev"} + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + for _ in range(2): + source.apply() + tagged = [item for kind in (NBVM, NBVirtualDisk, NBCluster) + for item in inventory.get_all_items(kind) + if item.data.get("custom_fields", {}).get("aws_resource_id") in ( + FUNCTION_ARN, CLUSTER_ARN, f"s3/{ACCOUNT}/inventory-bucket")] + assert len(tagged) == 3 + assert all(tag in item.data["tags"] and tag_name in item.get_tags() for item in tagged) + for item in tagged: + item.add_tags("user-managed") + data["Buckets"][0]["Tags"] = [] + data["Functions"][0]["Tags"] = {} + data["EKSClusters"][0]["tags"] = {} + source.apply() + assert all(tag_name not in item.get_tags() and "user-managed" in item.get_tags() for item in tagged) + + +def test_optional_resource_tags_reconcile_without_losing_user_tags(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + resources = [item for kind in (NBVM, NBVirtualDisk, NBCluster) + for item in inventory.get_all_items(kind) + if item.data.get("custom_fields", {}).get("aws_service") and + "AWS:Environment=test" in item.get_tags()] + for item in resources: + item.add_tags("user-managed") + data["Buckets"][0]["Tags"] = [] + data["Functions"][0]["Tags"] = {} + data["EKSClusters"][0]["tags"] = {} + source.apply() + assert len(resources) == 3 + assert all("AWS:Environment=test" not in item.get_tags() and "user-managed" in item.get_tags() + for item in resources) + + +@pytest.mark.parametrize("key", [ + "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", +]) +def test_eks_worker_tag_membership(make_source, inventory, key): + data = service_data() + data["instances"][0]["Tags"] = [{"Key": key, "Value": "inventory-cluster"}] + source = make_source("sync_eks = True", data=data) + source.apply() + worker, = inventory.get_all_items(NBVM) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EKS" + + +def test_enabling_eks_moves_existing_worker_without_duplication(make_source, inventory): + source = make_source(data=service_data()) + source.apply() + worker, = inventory.get_all_items(NBVM) + disks = list(worker.get_virtual_disks()) + interfaces = list(inventory.get_all_items(NBVMInterface)) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EC2" + source.settings.sync_eks = True + source.apply() + same_worker, = inventory.get_all_items(NBVM) + assert same_worker is worker + assert same_worker.data["cluster"].data["type"].data["name"] == "AWS EKS" + assert worker.get_virtual_disks() == disks + assert inventory.get_all_items(NBVMInterface) == interfaces + + +def test_unknown_eks_cluster_tag_is_explicit(make_source, inventory, caplog): + data = service_data() + data["instances"][0]["Tags"] = [{"Key": "eks:cluster-name", "Value": "deleted-cluster"}] + source = make_source("sync_eks = True", data=data) + source.apply() + worker, = inventory.get_all_items(NBVM) + assert worker.data["cluster"].data["type"].data["name"] == "AWS EC2" + assert "references undiscovered EKS cluster 'deleted-cluster'" in caplog.text + + +@pytest.mark.parametrize("missing", ["tags", "encryption", "versioning"]) +def test_s3_absent_optional_configuration_is_explicit(service_sdk, missing): + stub = service_sdk["s3"] + stub.add_response("list_buckets", {"Buckets": [{"Name": "inventory-bucket"}]}, {"BucketRegion": REGION}) + params = {"Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT} + stub.add_response("get_bucket_location", {"LocationConstraint": ""}, params) + if missing == "tags": + stub.add_client_error("get_bucket_tagging", service_error_code="NoSuchTagSet", expected_params=params) + else: + stub.add_response("get_bucket_tagging", {"TagSet": []}, params) + stub.add_response("get_bucket_versioning", {} if missing == "versioning" else {"Status": "Suspended"}, params) + if missing == "encryption": + stub.add_client_error( + "get_bucket_encryption", service_error_code="ServerSideEncryptionConfigurationNotFoundError", + expected_params=params) + else: + stub.add_response("get_bucket_encryption", { + "ServerSideEncryptionConfiguration": { + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}], + }, + }, params) + buckets = AWSClient().discover_s3(stub.client, REGION, ACCOUNT) + bucket, = buckets + assert bucket["Arn"] == "arn:aws:s3:::inventory-bucket" + if missing == "tags": + assert bucket["Tags"] == [] + elif missing == "encryption": + assert bucket["Encryption"] == {} + else: + assert bucket["Versioning"] == "Unversioned" + + +def test_s3_wrong_region_fails_closed(service_sdk): + service_sdk["s3"].add_response("list_buckets", { + "Buckets": [{"Name": "inventory-bucket", "BucketRegion": "us-west-2"}], + }, {"BucketRegion": REGION}) + with pytest.raises(AWSDiscoveryError, match="belongs to 'us-west-2'"): + AWSClient().discover_s3(service_sdk["s3"].client, REGION, ACCOUNT) + + +def test_s3_metadata_access_denied_skips_bucket_not_empty_configuration(service_sdk, caplog): + stub = service_sdk["s3"] + stub.add_response("list_buckets", { + "Buckets": [{"Name": "inventory-bucket", "BucketRegion": REGION}], + }, {"BucketRegion": REGION}) + stub.add_client_error("get_bucket_tagging", service_error_code="AccessDenied", expected_params={ + "Bucket": "inventory-bucket", "ExpectedBucketOwner": ACCOUNT, + }) + client = AWSClient() + assert client.discover_s3(stub.client, REGION, ACCOUNT) == [] + assert client.access_denied == [(ACCOUNT, REGION, "s3")] + assert "AccessDenied" in caplog.text + + +@pytest.mark.parametrize("operation", [ + "get_bucket_tagging", "get_bucket_versioning", "get_bucket_encryption", +]) +def test_s3_denied_bucket_does_not_stop_next_bucket(service_sdk, operation, caplog): + stub = service_sdk["s3"] + stub.add_response("list_buckets", {"Buckets": [ + {"Name": "denied-bucket", "BucketRegion": REGION}, + {"Name": "accessible-bucket", "BucketRegion": REGION}, + ]}, {"BucketRegion": REGION}) + for name in ("denied-bucket", "accessible-bucket"): + params = {"Bucket": name, "ExpectedBucketOwner": ACCOUNT} + for request, response in ( + ("get_bucket_tagging", {"TagSet": []}), + ("get_bucket_versioning", {}), + ("get_bucket_encryption", {"ServerSideEncryptionConfiguration": { + "Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}], + }}), + ): + if name == "denied-bucket" and request == operation: + stub.add_client_error(request, service_error_code="AccessDenied", expected_params=params) + break + stub.add_response(request, response, params) + client = AWSClient() + buckets = client.discover_s3(stub.client, REGION, ACCOUNT) + assert [bucket["Name"] for bucket in buckets] == ["accessible-bucket"] + assert client.access_denied == [(ACCOUNT, REGION, "s3")] + assert operation.replace("_", "").lower() in caplog.text.replace("_", "").lower() + + +def test_access_denied_retains_objects_and_preserves_eks_placement(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + objects = [item for kind in (NBVM, NBVirtualDisk, NBVMInterface, NBIPAddress) + for item in inventory.get_all_items(kind)] + worker = next(vm for vm in inventory.get_all_items(NBVM) + if vm.data["custom_fields"].get("aws_instance_id") == INSTANCE) + cluster = worker.data["cluster"] + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + bucket_metadata = copy.deepcopy(bucket.data["custom_fields"]["aws_metadata"]) + for item in objects: + item.source = None + data["Buckets"] = [] + data.pop("Functions") + data.pop("EKSClusters") + data["InaccessibleServices"] = ["lambda", "eks"] + data["instances"][0]["Tags"][0]["Value"] = "updated-worker" + source.client.access_denied = [(ACCOUNT, REGION, "s3"), (ACCOUNT, REGION, "lambda"), (ACCOUNT, REGION, "eks")] + source.apply() + inventory.tag_all_the_things(handler) + assert all(item.source is source and handler.orphaned_tag not in item.get_tags() for item in objects) + assert bucket.data["custom_fields"]["aws_metadata"] == bucket_metadata + assert "AWS:Environment=test" in bucket.get_tags() + assert worker.data["cluster"] is cluster + assert worker.data["name"] == "updated-worker" + + +def test_denied_organization_listing_retains_existing_objects(make_source, inventory): + source = make_source(data=service_data()) + source.apply() + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + vm, = inventory.get_all_items(NBVM) + vm.source = None + source.settings.organization = True + source.client.discover_organization = Mock(return_value={}) + source.client.access_denied = [(None, None, "organizations")] + source.apply() + inventory.tag_all_the_things(handler) + assert vm.source is source and handler.orphaned_tag not in vm.get_tags() + + +@pytest.mark.parametrize("code", ["ExpiredToken", "ThrottlingException", "InternalError"]) +def test_non_authorization_errors_remain_fatal(service_sdk, code): + identity(service_sdk["sts"]) + service_sdk["ec2"].add_client_error("describe_instances", service_error_code=code, expected_params={}) + with pytest.raises(AWSDiscoveryError, match=code): + AWSClient().discover([REGION]) + + +def test_denied_shared_network_owner_defers_dependent_instances(make_source, inventory): + data = inventory_data() + data["Vpcs"].clear() + data["Subnets"].clear() + source = make_source("organization = True") + source.client.discover_organization = Mock(return_value={ACCOUNT: {REGION: data}}) + source.client.access_denied = [(SECOND_ACCOUNT, REGION, "ec2")] + source.apply() + assert inventory.get_all_items(NBVM) == [] + + +def test_denied_identity_without_expected_account_is_nonfatal(service_sdk, inventory, load_config, caplog): + service_sdk["sts"].add_client_error("get_caller_identity", service_error_code="AccessDenied", expected_params={}) + load_config(f""" +[source/services] +type = aws +regions = {REGION} +""") + source, = instantiate_sources() + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert "AccessDenied" in caplog.text + assert source.client.access_denied == [(None, None, "account")] + + +def test_authorized_run_resumes_orphaning_after_denial(make_source, inventory): + data = service_data() + source = make_source("sync_s3 = True", data=data) + source.apply() + handler = make_netbox_handler() + inventory.tag_all_the_things(handler) + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + data["Buckets"] = [] + bucket.source = None + source.client.access_denied = [(ACCOUNT, REGION, "s3")] + source.apply() + inventory.tag_all_the_things(handler) + assert handler.orphaned_tag not in bucket.get_tags() + bucket.source = None + source.client.access_denied = [] + source.apply() + inventory.tag_all_the_things(handler) + assert handler.orphaned_tag in bucket.get_tags() + + +def test_ec2_denial_still_discovers_optional_services(service_sdk): + identity(service_sdk["sts"]) + service_sdk["ec2"].add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + expected_params={}) + stub_s3(service_sdk["s3"]) + client = AWSClient() + account, snapshots = client.discover([REGION], services=("s3",)) + assert account == ACCOUNT + assert snapshots[REGION]["EC2AccessDenied"] is True + assert snapshots[REGION]["Buckets"][0]["Name"] == "inventory-bucket" + assert client.access_denied == [(ACCOUNT, REGION, "ec2")] diff --git a/tests/test_aws_source.py b/tests/test_aws_source.py new file mode 100644 index 00000000..6c5e7f30 --- /dev/null +++ b/tests/test_aws_source.py @@ -0,0 +1,1405 @@ +import copy +import os +from contextlib import ExitStack +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +boto3 = pytest.importorskip("boto3") +from botocore.stub import Stubber + +from module.config.file_output import ConfigFileOutput +from module.config.parser import ConfigParser +from module.netbox import ( + NBCluster, NBIPAddress, NBMACAddress, NBTag, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) +from module.netbox.connection import NetBoxHandler +from module.sources import instantiate_sources, valid_sources +from module.sources.aws import client as aws_client +from module.sources.aws.client import AWSClient +from module.sources.aws.config import AWSConfig +from module.sources.aws.connection import AWSHandler + +ACCOUNT = "123456789012" +REGION = "us-east-1" +INSTANCE = "i-0123456789abcdef0" +VPC = "vpc-0123456789abcdef0" +SUBNET = "subnet-0123456789abcdef0" +ENI = "eni-0123456789abcdef0" +VOLUME = "vol-0123456789abcdef0" + + +def inventory_data(): + return { + "instances": [{ + "InstanceId": INSTANCE, "InstanceType": "t3.small", + "State": {"Name": "running"}, + "Placement": {"AvailabilityZone": "us-east-1a"}, + "VpcId": VPC, "SubnetId": SUBNET, + "Tags": [{"Key": "Name", "Value": "web"}, {"Key": "Environment", "Value": "test"}], + "NetworkInterfaces": [{"NetworkInterfaceId": ENI}], + "BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"VolumeId": VOLUME}}], + }], + "InstanceTypes": {"t3.small": { + "InstanceType": "t3.small", "VCpuInfo": {"DefaultVCpus": 2}, + "MemoryInfo": {"SizeInMiB": 2048}, + }}, + "NetworkInterfaces": {ENI: { + "NetworkInterfaceId": ENI, "VpcId": VPC, "SubnetId": SUBNET, + "Attachment": {"InstanceId": INSTANCE, "DeviceIndex": 0}, + "MacAddress": "02:aa:bb:cc:dd:ee", + "PrivateIpAddresses": [ + {"PrivateIpAddress": "10.0.0.10", "Primary": True, + "Association": {"PublicIp": "198.51.100.10"}}, + {"PrivateIpAddress": "10.0.0.11", "Primary": False}, + ], + "Ipv6Addresses": [{"Ipv6Address": "2001:db8::10", "IsPrimaryIpv6": True}], + }}, + "Volumes": {VOLUME: {"VolumeId": VOLUME, "Size": 20, "VolumeType": "gp3"}}, + "Subnets": {SUBNET: { + "SubnetId": SUBNET, "VpcId": VPC, "CidrBlock": "10.0.0.0/24", + "Ipv6CidrBlockAssociationSet": [{ + "Ipv6CidrBlock": "2001:db8::/64", "Ipv6CidrBlockState": {"State": "associated"}, + }], + }}, + "Vpcs": {VPC: {"VpcId": VPC}}, + } + + +@pytest.fixture(autouse=True) +def windows_config_paths(monkeypatch): + if os.name == "nt": + monkeypatch.setattr(ConfigParser, "get_config_file_path", staticmethod(os.path.abspath)) + + +@pytest.fixture +def make_source(inventory, load_config, monkeypatch): + def make(extra="", data=None, regions=REGION): + load_config(f""" +[source/test-aws] +type = aws +regions = {regions} +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +tag_keys = Environment +{extra} +""") + discovery = Mock(return_value=(ACCOUNT, {REGION: data if data is not None else inventory_data()})) + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + monkeypatch.setattr(AWSClient, "discover", discovery) + sources = instantiate_sources() + assert len(sources) == 1 and sources[0].init_successful + return sources[0] + return make + + +@pytest.fixture +def vpc_vrf(inventory): + return inventory.add_object(NBVRF, data={ + "id": 1, "name": f"AWS {ACCOUNT}/{REGION}/{VPC}", + }, read_from_netbox=True) + + +def test_registration(): + assert AWSHandler in valid_sources + assert AWSConfig in ConfigFileOutput.source_config_list + assert NBVRF in AWSHandler.dependent_netbox_objects + assert NBMACAddress in AWSHandler.dependent_netbox_objects + + +@pytest.mark.parametrize("api_version", ["3.6.0", "3.7.0", "4.1.0", "4.3.0"]) +def test_ec2_ebs_eni_import(make_source, inventory, vpc_vrf, api_version): + inventory.netbox_api_version = api_version + source = make_source("tenant = Operations") + source.apply() + vm, = inventory.get_all_items(NBVM) + cluster, = inventory.get_all_items(NBCluster) + interface, = inventory.get_all_items(NBVMInterface) + assert vm.data["status"] == "active" + assert vm.data["name"] == "web" + assert vm.data["vcpus"] == 2 + assert vm.data["memory"] == 2048 + assert vm.data["custom_fields"]["aws_name"] == "web" + assert vm.data["custom_fields"]["aws_instance_id"] == INSTANCE + assert vm.data["tenant"].data["name"] == "Operations" + assert "AWS:Environment=test" in vm.get_tags() + assert interface.data["name"] == ENI + assert interface.data["custom_fields"]["aws_public_ipv4"] == "198.51.100.10" + ips = inventory.get_all_items(NBIPAddress) + assert {item.data["address"] for item in ips} == { + "10.0.0.10/24", "10.0.0.11/24", "2001:db8::10/64", + } + assert all(item.data["vrf"] is vpc_vrf for item in ips) + assert vm.data["primary_ip4"].data["address"] == "10.0.0.10/24" + assert vm.data["primary_ip6"].data["address"] == "2001:db8::10/64" + if api_version >= "4.2": + assert cluster.data["scope_type"] == "dcim.site" + assert cluster.data["scope_id"].data["name"] == f"AWS {REGION}" + mac, = inventory.get_all_items(NBMACAddress) + assert mac is interface.data["primary_mac_address"] + if api_version < "3.7": + assert vm.data["disk"] == 20 + else: + disk, = inventory.get_all_items(NBVirtualDisk) + assert disk.data["name"] == VOLUME + assert disk.data["size"] == (20 if api_version < "4.1" else 20480) + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert len(inventory.get_all_items(NBVMInterface)) == 1 + assert len(inventory.get_all_items(NBIPAddress)) == 3 + + +def test_overlapping_vpcs_have_distinct_ips(make_source, inventory, vpc_vrf): + data = inventory_data() + second = copy.deepcopy(data) + other_instance, other_eni = "i-11111111111111111", "eni-11111111111111111" + other_vpc, other_subnet = "vpc-11111111111111111", "subnet-11111111111111111" + vm = second["instances"][0] + vm.update(InstanceId=other_instance, VpcId=other_vpc, SubnetId=other_subnet, + NetworkInterfaces=[{"NetworkInterfaceId": other_eni}], BlockDeviceMappings=[]) + eni = second["NetworkInterfaces"].pop(ENI) + eni.update(NetworkInterfaceId=other_eni, VpcId=other_vpc, SubnetId=other_subnet, + MacAddress="02:aa:bb:cc:dd:ff") + eni["Attachment"]["InstanceId"] = other_instance + subnet = second["Subnets"].pop(SUBNET) + subnet.update(SubnetId=other_subnet, VpcId=other_vpc) + data["instances"].append(vm) + data["NetworkInterfaces"][other_eni] = eni + data["Subnets"][other_subnet] = subnet + data["Vpcs"][other_vpc] = {"VpcId": other_vpc} + inventory.add_object(NBVRF, data={ + "id": 2, "name": f"AWS {ACCOUNT}/{REGION}/{other_vpc}", + }, read_from_netbox=True) + source = make_source(data=data) + source.apply() + assert len(inventory.get_all_items(NBVM)) == 2 + assert len(inventory.get_all_items(NBIPAddress)) == 6 + + +def test_missing_default_vrf_is_created_for_vpc(make_source, inventory): + source = make_source() + source.apply() + vrf, = inventory.get_all_items(NBVRF) + assert vrf.data["name"] == f"AWS {ACCOUNT}/{REGION}/{VPC}" + assert inventory.get_all_items(NBVM) + assert len(inventory.get_all_items(NBIPAddress)) == 3 + + +def test_missing_mapped_vrf_aborts_before_vm_import(make_source, inventory): + source = make_source(f"vpc_vrf_relation = {REGION}/{VPC} = Production") + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + + +def test_missing_volume_aborts_before_inventory_changes(make_source, inventory, vpc_vrf): + data = inventory_data() + data["Volumes"].clear() + source = make_source(data=data) + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBVM) == [] + + +@pytest.mark.parametrize("failure", ["vpc", "subnet", "both", "mismatch"]) +def test_incomplete_network_discovery_reports_context_and_aborts( + make_source, inventory, caplog, failure): + data = inventory_data() + other_vpc = "vpc-11111111111111111" + reasons = [] + if failure in ("vpc", "both"): + data["Vpcs"].clear() + reasons.append(f"VPC '{VPC}' was not returned by DescribeVpcs") + if failure in ("subnet", "both"): + data["Subnets"].clear() + reasons.append(f"subnet '{SUBNET}' was not returned by DescribeSubnets") + if failure == "mismatch": + data["Subnets"][SUBNET]["VpcId"] = other_vpc + reasons.append(f"subnet '{SUBNET}' belongs to VPC '{other_vpc}', not ENI VPC '{VPC}'") + source = make_source(data=data) + source.sync_snapshot = Mock() + + with pytest.raises(SystemExit): + source.apply() + + assert ( + f"Incomplete VPC/subnet discovery for ENI '{ENI}' " + f"(account '{ACCOUNT}', region '{REGION}', instance '{INSTANCE}', " + f"VPC '{VPC}', subnet '{SUBNET}'): {'; '.join(reasons)}" + ) in caplog.text + assert "No NetBox writes or pruning will be performed." in caplog.text + source.sync_snapshot.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert inventory.get_all_items(NBIPAddress) == [] + + +def test_stopped_instance_without_public_ip_is_retained(make_source, inventory, vpc_vrf): + data = inventory_data() + data["instances"][0]["State"]["Name"] = "stopped" + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"][0].pop("Association") + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["status"] == "offline" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + assert inventory.get_all_items(NBVMInterface)[0].data["custom_fields"]["aws_public_ipv4"] == "" + + +def test_name_tag_rename_does_not_duplicate_vm(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + data["instances"][0]["Tags"][0]["Value"] = "renamed" + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["name"] == "renamed" + assert vm.data["custom_fields"]["aws_name"] == "renamed" + + +def test_missing_name_tag_uses_instance_id(make_source, inventory, vpc_vrf): + data = inventory_data() + data["instances"][0]["Tags"] = [{"Key": "Environment", "Value": "test"}] + make_source(data=data).apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["name"] == INSTANCE + assert vm.data["custom_fields"]["aws_instance_id"] == INSTANCE + + +def test_duplicate_name_tags_include_instance_ids(make_source, inventory, vpc_vrf): + data = inventory_data() + duplicate = copy.deepcopy(data["instances"][0]) + duplicate_id = "i-11111111111111111" + duplicate["InstanceId"] = duplicate_id + duplicate["NetworkInterfaces"] = [] + duplicate["BlockDeviceMappings"] = [] + data["instances"].append(duplicate) + make_source(data=data).apply() + assert {vm.data["name"] for vm in inventory.get_all_items(NBVM)} == { + f"web ({INSTANCE})", f"web ({duplicate_id})", + } + + +def test_account_name_is_used_for_cluster_labels(make_source, inventory, vpc_vrf): + make_source("account_name = Production", data=inventory_data()).apply() + cluster, = inventory.get_all_items(NBCluster) + assert cluster.data["name"] == f"AWS Production/{REGION}" + + +def test_duplicate_organization_account_names_remain_distinct(make_source, inventory): + snapshot = {REGION: inventory_data()} + source = make_source("organization = True\nsync_private_ips = False") + source.client.account_names = {ACCOUNT: "Shared", SECOND_ACCOUNT: "Shared"} + source.client.discover_organization = Mock(return_value={ + ACCOUNT: snapshot, SECOND_ACCOUNT: snapshot, + }) + source.apply() + assert {cluster.data["name"] for cluster in inventory.get_all_items(NBCluster)} == { + f"AWS Shared ({ACCOUNT})/{REGION}", f"AWS Shared ({SECOND_ACCOUNT})/{REGION}", + } + assert len(inventory.get_all_items(NBVM)) == 2 + + +@pytest.mark.parametrize("tag_name", ["AWS:Environment=Dev", "aws:environment=Dev"]) +def test_aws_vm_tags_use_resolved_identity(make_source, inventory, vpc_vrf, tag_name): + tag = inventory.add_object(NBTag, data={"name": tag_name}) + data = inventory_data() + data["instances"][0]["Tags"] = [{"Key": "Environment", "Value": "dev"}] + source = make_source(data=data) + for _ in range(2): + source.apply() + vm, = inventory.get_all_items(NBVM) + assert tag in vm.data["tags"] + assert tag_name in vm.get_tags() + vm.add_tags("user-managed") + data["instances"][0]["Tags"] = [] + source.apply() + assert tag_name not in vm.get_tags() + assert "user-managed" in vm.get_tags() + + +def test_removed_aws_tag_is_reconciled_but_user_tag_is_kept(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + vm.add_tags("user-managed") + data["instances"][0]["Tags"] = [{"Key": "Environment", "Value": "production"}] + source.apply() + assert "AWS:Environment=test" not in vm.get_tags() + assert "AWS:Environment=production" in vm.get_tags() + assert "user-managed" in vm.get_tags() + + +def test_removed_secondary_ip_and_mac_are_detached(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + secondary = next(ip for ip in inventory.get_all_items(NBIPAddress) + if ip.data["address"] == "10.0.0.11/24") + old_mac, = inventory.get_all_items(NBMACAddress) + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"].pop() + data["NetworkInterfaces"][ENI]["MacAddress"] = "02:00:00:00:00:01" + source.apply() + assert "assigned_object_id" in secondary.unset_items + assert "assigned_object_id" in old_mac.unset_items + interface, = inventory.get_all_items(NBVMInterface) + assert interface.data["primary_mac_address"].data["mac_address"] == "02:00:00:00:00:01" + + +def test_primary_address_removal_clears_primary_ip(make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"] = [] + data["NetworkInterfaces"][ENI]["Ipv6Addresses"] = [] + source.apply() + assert "primary_ip4" in vm.unset_items + assert "primary_ip6" in vm.unset_items + + +@pytest.mark.parametrize("tags", [ + [], + [NetBoxHandler.primary_tag], + [NetBoxHandler.orphaned_tag], + [NetBoxHandler.primary_tag, "Source: other"], +]) +def test_ip_owned_by_other_source_is_not_reassigned( + make_source, inventory, vpc_vrf, caplog, tags): + source = make_source() + other = inventory.add_object(NBVM, data={"name": "other", "status": "active", "tags": tags}) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + source.apply() + assert existing.get_interface() is other_interface + assert "belongs to another source" in caplog.text + assert "Skipping reassignment" in caplog.text + vm = next(item for item in inventory.get_all_items(NBVM) if item is not other) + assert "primary_ip4" not in vm.data + assert vm.data["primary_ip6"].data["address"] == "2001:db8::10/64" + assert len(inventory.get_all_items(NBIPAddress)) == 3 + + +@pytest.mark.parametrize("previous_source", [None, "Source: other"]) +def test_ip_on_managed_orphan_is_reclaimed( + make_source, inventory, vpc_vrf, caplog, previous_source): + source = make_source() + tags = [NetBoxHandler.primary_tag, NetBoxHandler.orphaned_tag] + if previous_source: + tags.append(previous_source) + other = inventory.add_object(NBVM, data={"name": "old", "status": "offline", "tags": tags}) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + other.update(data={"primary_ip4": existing}) + source.apply() + vm = next(item for item in inventory.get_all_items(NBVM) if item is not other) + assert existing.get_device_vm() is vm + assert vm.data["primary_ip4"] is existing + assert "primary_ip4" in other.unset_items + assert len(inventory.get_all_items(NBIPAddress)) == 3 + assert "Reclaiming IP" in caplog.text + + +def test_ip_with_unresolved_owner_is_not_reassigned(make_source, inventory, vpc_vrf, caplog): + source = make_source() + existing = inventory.add_object(NBIPAddress, data={ + "id": 99, "address": "10.0.0.10/24", "vrf": vpc_vrf, + "assigned_object_id": 999, "assigned_object_type": NBVMInterface.object_type, + }, read_from_netbox=True) + source.apply() + assert existing.data["assigned_object_id"] == 999 + assert "unresolved owner" in caplog.text + assert "Skipping reassignment" in caplog.text + vm, = inventory.get_all_items(NBVM) + assert "primary_ip4" not in vm.data + assert vm.data["primary_ip6"].data["address"] == "2001:db8::10/64" + + +@pytest.mark.parametrize("management", ["tag", "source", "source-tag", "unmanaged"]) +def test_offline_owner_ip_reclamation( + make_source, inventory, vpc_vrf, caplog, management): + source = make_source() + tags = { + "tag": [NetBoxHandler.primary_tag, "Source: other"], + "source-tag": [source.source_tag], + }.get(management, []) + other = inventory.add_object( + NBVM, data={"name": "down", "status": "offline", "tags": tags}, + source=source if management == "source" else None) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + other.update(data={"primary_ip4": existing}) + source.apply() + vm = next(item for item in inventory.get_all_items(NBVM) if item is not other) + if management == "unmanaged": + assert existing.get_device_vm() is other + assert other.data["primary_ip4"] is existing + assert "primary_ip4" not in vm.data + assert "Skipping reassignment" in caplog.text + else: + assert existing.get_device_vm() is vm + assert vm.data["primary_ip4"] is existing + assert "primary_ip4" in other.unset_items + assert "Reclaiming IP" in caplog.text + + +@pytest.mark.parametrize("owner_status,instance_state,reclaim", [ + ({"value": "offline", "label": "Offline"}, "running", True), + ("offline", "stopped", False), + ("active", "running", False), + ("failed", "running", False), + ("planned", "running", False), +]) +def test_cross_source_reclamation_requires_offline_to_active( + make_source, inventory, vpc_vrf, owner_status, instance_state, reclaim): + data = inventory_data() + data["instances"][0]["State"]["Name"] = instance_state + source = make_source(data=data) + other = inventory.add_object(NBVM, data={ + "id": 99, "name": "other", "status": owner_status, + }, read_from_netbox=True) + other.add_tags([NetBoxHandler.primary_tag, "Source: other"]) + other_interface = inventory.add_object(NBVMInterface, data={"name": "eth0", "virtual_machine": other}) + existing = inventory.add_object(NBIPAddress, data={ + "address": "10.0.0.10/24", "vrf": vpc_vrf, "assigned_object_id": other_interface, + "assigned_object_type": NBVMInterface, + }) + source.apply() + assert (existing.get_device_vm() is not other) is reclaim + + +@pytest.mark.parametrize("active_first", [False, True]) +def test_active_instance_keeps_shared_ip_regardless_of_discovery_order( + make_source, inventory, vpc_vrf, active_first, caplog): + data = inventory_data() + stopped = copy.deepcopy(data["instances"][0]) + stopped_id, stopped_eni = "i-11111111111111111", "eni-11111111111111111" + stopped.update(InstanceId=stopped_id, State={"Name": "stopped"}, + NetworkInterfaces=[{"NetworkInterfaceId": stopped_eni}], BlockDeviceMappings=[]) + eni = copy.deepcopy(data["NetworkInterfaces"][ENI]) + eni.update(NetworkInterfaceId=stopped_eni, MacAddress="02:00:00:00:00:01") + eni["Attachment"]["InstanceId"] = stopped_id + data["NetworkInterfaces"][stopped_eni] = eni + data["instances"].insert(1 if active_first else 0, stopped) + source = make_source(data=data) + for _ in range(2): + source.apply() + active = next(vm for vm in inventory.get_all_items(NBVM) if vm.data["status"] == "active") + offline = next(vm for vm in inventory.get_all_items(NBVM) if vm.data["status"] == "offline") + ips = inventory.get_all_items(NBIPAddress) + assert len(ips) == 3 + assert all(ip.get_device_vm() is active for ip in ips) + assert active.data["primary_ip4"].data["address"] == "10.0.0.10/24" + assert active.data["primary_ip6"].data["address"] == "2001:db8::10/64" + for field in ("primary_ip4", "primary_ip6"): + assert not offline.data.get(field) or field in offline.unset_items + assert "Skipping reassignment" in caplog.text or "Reclaiming IP" in caplog.text + + +def test_tagging_and_successful_empty_discovery_use_existing_orphan_lifecycle( + make_source, inventory, vpc_vrf): + data = inventory_data() + source = make_source(data=data) + source.apply() + handler = object.__new__(NetBoxHandler) + handler.settings = SimpleNamespace( + prune_enabled=True, ignore_unknown_source_object_pruning=False, + orphaned_device_status=None, orphaned_vm_status=None, + ) + inventory.tag_all_the_things(handler) + vm, = inventory.get_all_items(NBVM) + assert source.source_tag in vm.get_tags() + assert handler.primary_tag in vm.get_tags() + for items in inventory.base_structure.values(): + for item in items: + item.source = None + data["instances"] = [] + source.apply() + inventory.tag_all_the_things(handler) + assert handler.orphaned_tag in vm.get_tags() + + +def test_customized_cpu_options_override_type_default(make_source, inventory, vpc_vrf): + data = inventory_data() + data["instances"][0]["CpuOptions"] = {"CoreCount": 1, "ThreadsPerCore": 1} + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["vcpus"] == 1 + + +def test_multiple_enis_only_primary_eni_sets_primary_ip(make_source, inventory, vpc_vrf): + data = inventory_data() + secondary_id = "eni-11111111111111111" + secondary = copy.deepcopy(data["NetworkInterfaces"][ENI]) + secondary.update(NetworkInterfaceId=secondary_id, MacAddress="02:00:00:00:00:01") + secondary["Attachment"]["DeviceIndex"] = 1 + secondary["PrivateIpAddresses"] = [{"PrivateIpAddress": "10.0.0.20", "Primary": True}] + secondary["Ipv6Addresses"] = [] + data["NetworkInterfaces"][secondary_id] = secondary + data["instances"][0]["NetworkInterfaces"].append({"NetworkInterfaceId": secondary_id}) + source = make_source(data=data) + source.apply() + vm, = inventory.get_all_items(NBVM) + assert len(inventory.get_all_items(NBVMInterface)) == 2 + assert vm.data["primary_ip4"].data["address"] == "10.0.0.10/24" + + +def test_optional_network_import_does_not_require_vrf(make_source, inventory): + source = make_source("sync_private_ips = False") + source.apply() + assert len(inventory.get_all_items(NBVMInterface)) == 1 + assert inventory.get_all_items(NBIPAddress) == [] + + +def test_explicit_vrf_mapping(make_source, inventory): + inventory.add_object(NBVRF, data={"id": 1, "name": "Production"}, read_from_netbox=True) + source = make_source(f"vpc_vrf_relation = {REGION}/{VPC} = Production") + source.apply() + assert {item.data["vrf"].data["name"] for item in inventory.get_all_items(NBIPAddress)} == {"Production"} + + +def test_quoted_vrf_name_with_comma(make_source, inventory): + inventory.add_object(NBVRF, data={"id": 1, "name": "Production, New York"}, read_from_netbox=True) + source = make_source(f'vpc_vrf_relation = {REGION}/{VPC} = "Production, New York"') + source.apply() + assert {item.data["vrf"].data["name"] for item in inventory.get_all_items(NBIPAddress)} == { + "Production, New York", + } + + +def test_disallowed_addresses_are_not_imported(make_source, inventory, vpc_vrf): + source = make_source() + from module.sources.common.permitted_subnets import PermittedSubnets + source.settings.permitted_subnets = PermittedSubnets("192.168.0.0/16") + source.apply() + assert inventory.get_all_items(NBIPAddress) == [] + assert len(inventory.get_all_items(NBVMInterface)) == 1 + + +def test_unmapped_subnet_address_aborts_before_changes(make_source, inventory, vpc_vrf): + data = inventory_data() + data["NetworkInterfaces"][ENI]["PrivateIpAddresses"][0]["PrivateIpAddress"] = "10.1.1.1" + source = make_source(data=data) + with pytest.raises(SystemExit): + source.apply() + assert inventory.get_all_items(NBCluster) == [] + + +def test_missing_sdk_aborts_enabled_source_but_disabled_source_is_safe( + inventory, load_config, monkeypatch, caplog): + monkeypatch.setattr(aws_client, "boto3", None) + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION} +enabled = False +""") + assert instantiate_sources() == [] + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION} +""") + with pytest.raises(SystemExit): + instantiate_sources() + assert "requirements-aws.txt" in caplog.text + + +def test_yaml_lists_and_account_guard_parse(inventory, load_config, monkeypatch): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(f""" +source: + test-aws: + type: aws + regions: [us-east-1, us-west-2] + account_id: "{ACCOUNT}" + tag_keys: [Environment, Owner] +""", "settings.yaml") + source, = instantiate_sources() + assert source.settings.regions == ["us-east-1", "us-west-2"] + assert source.settings.tag_keys == ["Environment", "Owner"] + assert source.settings.account_id == ACCOUNT + + +@pytest.mark.parametrize("extra", [ + "account_id = not-an-account", + "vpc_vrf_relation = malformed", + f"vpc_vrf_relation = {REGION}/{VPC} = One, {REGION}/{VPC} = Two", +]) +def test_invalid_config_fails(make_source, extra): + with pytest.raises(SystemExit): + make_source(extra) + + +@pytest.fixture +def sdk(monkeypatch): + session = boto3.Session( + aws_access_key_id="test-only", aws_secret_access_key="test-only", region_name=REGION) + sts = session.client("sts") + ec2 = session.client("ec2") + sts_stub, ec2_stub = Stubber(sts), Stubber(ec2) + fake_session = Mock() + fake_session.region_name = REGION + fake_session.client.side_effect = lambda service, **kwargs: sts if service == "sts" else ec2 + monkeypatch.setattr(aws_client.boto3, "Session", Mock(return_value=fake_session)) + with sts_stub, ec2_stub: + yield sts_stub, ec2_stub + sts_stub.assert_no_pending_responses() + ec2_stub.assert_no_pending_responses() + + +def identity(stub, account=ACCOUNT, partition="aws"): + stub.add_response("get_caller_identity", { + "Account": account, "Arn": f"arn:{partition}:iam::{account}:user/test", "UserId": "test", + }, {}) + + +def stub_region(stub, data): + stub.add_response("describe_instances", {"Reservations": [{ + "Instances": data["instances"], + }], "NextToken": "next-page"}, {}) + stub.add_response("describe_instances", {"Reservations": []}, {"NextToken": "next-page"}) + for operation, result_key in ( + ("describe_network_interfaces", "NetworkInterfaces"), + ("describe_volumes", "Volumes"), ("describe_subnets", "Subnets"), + ("describe_vpcs", "Vpcs"), + ): + stub.add_response(operation, {result_key: list(data[result_key].values())}, {}) + if data["instances"]: + stub.add_response("describe_instance_types", { + "InstanceTypes": list(data["InstanceTypes"].values()), + }, {"InstanceTypes": ["t3.small"]}) + + +def test_sdk_paginated_discovery(sdk): + sts, ec2 = sdk + identity(sts) + data = inventory_data() + stub_region(ec2, data) + account, snapshot = AWSClient().discover([REGION], ACCOUNT) + assert account == ACCOUNT + assert snapshot[REGION]["instances"] == data["instances"] + assert snapshot[REGION]["Volumes"] == data["Volumes"] + + +def test_stubbed_sdk_to_real_inventory(sdk, inventory, load_config, vpc_vrf): + sts, ec2 = sdk + identity(sts) + stub_region(ec2, inventory_data()) + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION} +account_id = {ACCOUNT} +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +""") + source, = instantiate_sources() + source.apply() + vm, = inventory.get_all_items(NBVM) + assert vm.data["name"] == INSTANCE + assert vm.data["primary_ip4"].data["vrf"] is vpc_vrf + assert len(inventory.get_all_items(NBVirtualDisk)) == 1 + + +def test_profile_is_forwarded_to_sdk(sdk): + AWSClient(profile="inventory") + aws_client.boto3.Session.assert_called_once_with(profile_name="inventory") + + +def test_sdk_access_denied_after_first_region_continues(sdk, inventory, load_config, caplog): + sts, ec2 = sdk + identity(sts) + stub_region(ec2, inventory_data()) + ec2.add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + service_message="not permitted", http_status_code=403, expected_params={}) + load_config(f""" +[source/test-aws] +type = aws +regions = {REGION}, us-west-2 +sync_private_ips = False +""") + source, = instantiate_sources() + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert "UnauthorizedOperation" in caplog.text + assert source.client.access_denied == [(ACCOUNT, "us-west-2", "ec2")] + + +def test_wrong_account_aborts(sdk): + sts, _ = sdk + identity(sts) + with pytest.raises(aws_client.AWSDiscoveryError, match="account mismatch"): + AWSClient().discover([REGION], "999999999999") + + +def test_terminated_instances_are_excluded(sdk): + sts, ec2 = sdk + identity(sts) + terminated = inventory_data()["instances"][0] + terminated["State"]["Name"] = "terminated" + ec2.add_response("describe_instances", {"Reservations": [{"Instances": [terminated]}]}, {}) + for operation, result_key in ( + ("describe_network_interfaces", "NetworkInterfaces"), ("describe_volumes", "Volumes"), + ("describe_subnets", "Subnets"), ("describe_vpcs", "Vpcs"), + ): + ec2.add_response(operation, {result_key: []}, {}) + _, snapshot = AWSClient().discover([REGION]) + assert snapshot[REGION]["instances"] == [] + + +def test_missing_sdk_fails_explicitly(monkeypatch): + monkeypatch.setattr(aws_client, "boto3", None) + with pytest.raises(aws_client.AWSDiscoveryError, match="requirements-aws.txt"): + AWSClient() + + +def region_response(stub, regions): + stub.add_response("describe_regions", {"Regions": [ + {"RegionName": name, "OptInStatus": status} for name, status in regions + ]}, {"AllRegions": True}) + + +def test_auto_discovers_enabled_regions_and_preserves_region_identity(sdk): + sts, ec2 = sdk + identity(sts) + region_response(ec2, [ + ("us-west-2", "opted-in"), + (REGION, "opt-in-not-required"), + ("ap-east-1", "not-opted-in"), + (REGION, "opt-in-not-required"), + ]) + stub_region(ec2, inventory_data()) + stub_region(ec2, inventory_data()) + client = AWSClient() + account, snapshot = client.discover(["auto"], ACCOUNT) + assert account == ACCOUNT + assert list(snapshot) == [REGION, "us-west-2"] + ec2_regions = [ + call.kwargs["region_name"] for call in client.session.client.call_args_list + if call.args[0] == "ec2" + ] + assert ec2_regions == [REGION, REGION, "us-west-2"] + + +def test_auto_bootstrap_override(sdk): + sts, ec2 = sdk + identity(sts) + region_response(ec2, [(REGION, "opt-in-not-required")]) + stub_region(ec2, inventory_data()) + client = AWSClient() + client.session.region_name = None + client.discover(["auto"], discovery_region="us-west-2") + assert client.session.client.call_args_list[0].kwargs["region_name"] == "us-west-2" + assert client.session.client.call_args_list[1].kwargs["region_name"] == "us-west-2" + assert client.session.client.call_args_list[2].kwargs["region_name"] == REGION + + +def test_auto_without_bootstrap_fails_explicitly(sdk): + client = AWSClient() + client.session.region_name = None + with pytest.raises(aws_client.AWSDiscoveryError, match="AWS_DEFAULT_REGION"): + client.discover(["auto"]) + client.session.client.assert_not_called() + + +@pytest.mark.parametrize("regions", [[], [("ap-east-1", "not-opted-in")]]) +def test_auto_no_enabled_regions_aborts(sdk, regions): + sts, ec2 = sdk + identity(sts) + region_response(ec2, regions) + with pytest.raises(aws_client.AWSDiscoveryError, match="no enabled regions"): + AWSClient().discover(["auto"]) + + +def test_auto_permission_failure_logs_and_returns_without_inventory_changes( + sdk, inventory, load_config, caplog): + sts, ec2 = sdk + identity(sts) + ec2.add_client_error("describe_regions", service_error_code="UnauthorizedOperation", + service_message="not permitted", http_status_code=403, + expected_params={"AllRegions": True}) + load_config(""" +[source/test-aws] +type = aws +sync_private_ips = False +""") + source, = instantiate_sources() + assert source.settings.regions == ["auto"] + source.apply() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert "UnauthorizedOperation" in caplog.text + assert source.client.access_denied + + +def test_auto_later_region_denied_keeps_accessible_inventory( + sdk, inventory, load_config): + sts, ec2 = sdk + identity(sts) + region_response(ec2, [(REGION, "opt-in-not-required"), ("us-west-2", "opted-in")]) + stub_region(ec2, inventory_data()) + ec2.add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + http_status_code=403, expected_params={}) + load_config(""" +[source/test-aws] +type = aws +regions = auto +sync_private_ips = False +""") + source, = instantiate_sources() + source.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert source.client.access_denied == [(ACCOUNT, "us-west-2", "ec2")] + + +@pytest.mark.parametrize("regions", ["auto, us-east-1", "us-east-1, auto"]) +def test_auto_cannot_be_mixed_with_explicit_regions(make_source, regions): + with pytest.raises(SystemExit): + make_source(regions=regions) + + +def test_auto_yaml_list_parses(inventory, load_config, monkeypatch): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(""" +source: + test-aws: + type: aws + regions: [auto] + discovery_region: us-west-2 +""", "settings.yaml") + source, = instantiate_sources() + assert source.settings.regions == ["auto"] + assert source.settings.discovery_region == "us-west-2" + + +def test_explicit_regions_do_not_need_bootstrap_or_describe_regions(sdk): + sts, ec2 = sdk + identity(sts) + stub_region(ec2, inventory_data()) + client = AWSClient() + client.session.region_name = None + _, snapshot = client.discover([REGION], discovery_region="us-west-2") + assert list(snapshot) == [REGION] + assert client.session.client.call_args_list[0].kwargs["region_name"] == REGION + + +MANAGEMENT = "999999999999" +SECOND_ACCOUNT = "234567890123" + + +@pytest.fixture +def org_sdk(monkeypatch): + real_session = boto3.Session( + aws_access_key_id="test-only", aws_secret_access_key="test-only", region_name=REGION) + stubs = [] + + def make_session(organization=False): + services = ("sts", "organizations") if organization else ("sts", "ec2") + clients = {service: real_session.client(service) for service in services} + session = Mock(region_name=REGION) + session.client.side_effect = lambda service, **kwargs: clients[service] + bundle = SimpleNamespace(session=session) + for service, client in clients.items(): + stub = Stubber(client) + stubs.append(stub) + setattr(bundle, service, stub) + return bundle + + central = make_session(organization=True) + members = {account: make_session() for account in (MANAGEMENT, ACCOUNT, SECOND_ACCOUNT)} + factory = Mock(side_effect=lambda **kwargs: ( + members[kwargs["aws_session_token"]].session if "aws_session_token" in kwargs else central.session)) + monkeypatch.setattr(aws_client.boto3, "Session", factory) + with ExitStack() as stack: + for stub in stubs: + stack.enter_context(stub) + yield SimpleNamespace(central=central, members=members, factory=factory) + for stub in stubs: + stub.assert_no_pending_responses() + + +def organization_accounts(sdk, accounts=None, partition="aws"): + identity(sdk.central.sts, MANAGEMENT, partition) + sdk.central.organizations.add_response("describe_organization", { + "Organization": {"MasterAccountId": MANAGEMENT}, + }, {}) + # An empty page with a continuation token must not terminate discovery. + sdk.central.organizations.add_response("list_accounts", { + "Accounts": [], "NextToken": "next-page", + }, {}) + sdk.central.organizations.add_response("list_accounts", { + "Accounts": accounts if accounts is not None else [ + {"Id": MANAGEMENT, "Name": "Management", "State": "ACTIVE"}, + {"Id": ACCOUNT, "Name": "Production", "State": "ACTIVE"}, + {"Id": SECOND_ACCOUNT, "Name": "Development", "State": "ACTIVE"}, + {"Id": "345678901234", "Name": "Suspended", "State": "SUSPENDED"}, + ], + }, {"NextToken": "next-page"}) + + +def assume_member(sdk, account, external_id=None, partition="aws", role="NetBoxInventoryReadOnly"): + request = {"RoleArn": f"arn:{partition}:iam::{account}:role/{role}", + "RoleSessionName": "netbox-sync", "DurationSeconds": 3600} + if external_id: + request["ExternalId"] = external_id + sdk.central.sts.add_response("assume_role", {"Credentials": { + "AccessKeyId": "ASIA" + "A" * 16, "SecretAccessKey": "test-only-secret", + "SessionToken": account, "Expiration": datetime(2099, 1, 1, tzinfo=timezone.utc), + }}, request) + return sdk.members[account] + + +def organization_source(load_config, extra=""): + load_config(f""" +[source/test-aws] +type = aws +organization = True +regions = auto +account_id = {MANAGEMENT} +sync_private_ips = False +{extra} +""") + source, = instantiate_sources() + return source + + +def test_organization_scans_members_not_management(org_sdk, inventory, load_config): + organization_accounts(org_sdk) + for account in (ACCOUNT, SECOND_ACCOUNT): + member = assume_member(org_sdk, account) + identity(member.sts, account) + region_response(member.ec2, [(REGION, "opt-in-not-required")]) + stub_region(member.ec2, inventory_data()) + source = organization_source(load_config) + source.apply() + vms = inventory.get_all_items(NBVM) + assert len(vms) == 2 + assert {vm.data["custom_fields"]["aws_account_id"] for vm in vms} == {ACCOUNT, SECOND_ACCOUNT} + assert {vm.data["cluster"].data["name"] for vm in vms} == { + f"AWS Production/{REGION}", f"AWS Development/{REGION}", + } + assert len(inventory.get_all_items(NBVMInterface)) == 2 + assert len(inventory.get_all_items(NBVirtualDisk)) == 2 + assert {item.data["virtual_machine"] for item in inventory.get_all_items(NBVMInterface)} == set(vms) + assert {item.data["virtual_machine"] for item in inventory.get_all_items(NBVirtualDisk)} == set(vms) + org_sdk.members[MANAGEMENT].session.client.assert_not_called() + assert [call.args[0] for call in org_sdk.central.session.client.call_args_list] == [ + "sts", "organizations", + ] + # Each account gets fresh temporary credentials, not the collector's IAM-user credentials. + assert [call.kwargs["aws_session_token"] for call in org_sdk.factory.call_args_list[1:]] == [ + ACCOUNT, SECOND_ACCOUNT, + ] + for account in (ACCOUNT, SECOND_ACCOUNT): + source.sync_snapshot(account, {REGION: inventory_data()}) + assert len(inventory.get_all_items(NBVMInterface)) == 2 + assert len(inventory.get_all_items(NBVirtualDisk)) == 2 + + +def test_organization_allowlist_role_path_external_id_and_partition(org_sdk): + organization_accounts(org_sdk, partition="aws-us-gov") + member = assume_member(org_sdk, SECOND_ACCOUNT, external_id="inventory-test", + partition="aws-us-gov", role="service-accounts/Inventory") + identity(member.sts, SECOND_ACCOUNT, partition="aws-us-gov") + stub_region(member.ec2, inventory_data()) + snapshots = AWSClient().discover_organization( + [REGION], "service-accounts/Inventory", external_id="inventory-test", + account_ids=[SECOND_ACCOUNT]) + assert list(snapshots) == [SECOND_ACCOUNT] + org_sdk.members[ACCOUNT].session.client.assert_not_called() + + +def test_organization_can_include_management(org_sdk): + organization_accounts(org_sdk) + member = assume_member(org_sdk, MANAGEMENT) + identity(member.sts, MANAGEMENT) + stub_region(member.ec2, inventory_data()) + snapshots = AWSClient().discover_organization( + [REGION], "NetBoxInventoryReadOnly", include_management=True, account_ids=[MANAGEMENT]) + assert list(snapshots) == [MANAGEMENT] + + +@pytest.mark.parametrize("failure", ["assume", "scan", "identity"]) +def test_organization_later_account_denied_continues_but_identity_failure_aborts( + org_sdk, inventory, load_config, failure, caplog): + organization_accounts(org_sdk) + first = assume_member(org_sdk, ACCOUNT) + identity(first.sts, ACCOUNT) + region_response(first.ec2, [(REGION, "opt-in-not-required")]) + stub_region(first.ec2, inventory_data()) + if failure == "assume": + org_sdk.central.sts.add_client_error( + "assume_role", service_error_code="AccessDenied", http_status_code=403, + expected_params={"RoleArn": f"arn:aws:iam::{SECOND_ACCOUNT}:role/NetBoxInventoryReadOnly", + "RoleSessionName": "netbox-sync", "DurationSeconds": 3600}) + else: + second = assume_member(org_sdk, SECOND_ACCOUNT) + identity(second.sts, ACCOUNT if failure == "identity" else SECOND_ACCOUNT) + if failure == "scan": + region_response(second.ec2, [(REGION, "opt-in-not-required")]) + second.ec2.add_client_error("describe_instances", service_error_code="UnauthorizedOperation", + http_status_code=403, expected_params={}) + source = organization_source(load_config) + sync = Mock() + source.sync_snapshot = sync + if failure == "identity": + with pytest.raises(SystemExit): + source.apply() + sync.assert_not_called() + assert "No NetBox writes or pruning" in caplog.text + else: + source.apply() + assert sync.called + assert source.client.access_denied + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBCluster) == [] + assert SECOND_ACCOUNT in caplog.text + + +def test_organization_listing_denied_returns_non_authoritative_empty_result(org_sdk, caplog): + identity(org_sdk.central.sts, MANAGEMENT) + org_sdk.central.organizations.add_response("describe_organization", { + "Organization": {"MasterAccountId": MANAGEMENT}, + }, {}) + org_sdk.central.organizations.add_client_error( + "list_accounts", service_error_code="AccessDeniedException", http_status_code=403, expected_params={}) + client = AWSClient() + assert client.discover_organization([REGION], "NetBoxInventoryReadOnly") == {} + assert client.access_denied + assert "AccessDeniedException" in caplog.text + + +@pytest.mark.parametrize("selection", [[MANAGEMENT], ["999999999999"], ["345678901234"]]) +def test_organization_invalid_allowlist_is_not_silently_ignored(org_sdk, selection): + organization_accounts(org_sdk) + with pytest.raises(aws_client.AWSDiscoveryError, match="not active/eligible"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly", account_ids=selection) + + +def test_organization_no_eligible_accounts_aborts(org_sdk): + organization_accounts(org_sdk, accounts=[{"Id": MANAGEMENT, "State": "ACTIVE"}]) + with pytest.raises(aws_client.AWSDiscoveryError, match="no eligible accounts"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + + +def test_organization_wrong_collector_account_aborts_before_listing(org_sdk): + identity(org_sdk.central.sts, MANAGEMENT) + with pytest.raises(aws_client.AWSDiscoveryError, match="account mismatch"): + AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly", expected_account=ACCOUNT) + assert org_sdk.central.session.client.call_count == 1 + + +def test_organization_missing_bootstrap_aborts(org_sdk): + org_sdk.central.session.region_name = None + with pytest.raises(aws_client.AWSDiscoveryError, match="AWS_DEFAULT_REGION"): + AWSClient().discover_organization(["auto"], "NetBoxInventoryReadOnly") + org_sdk.central.session.client.assert_not_called() + + +def test_organization_all_accounts_validated_before_sync(make_source, inventory, monkeypatch): + source = make_source("organization = True") + invalid = inventory_data() + invalid["Volumes"] = {} + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: inventory_data()}, SECOND_ACCOUNT: {REGION: invalid}, + })) + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + + +def test_organization_rejects_vrf_reuse_across_accounts(make_source, vpc_vrf, monkeypatch): + source = make_source(f""" +organization = True +vpc_vrf_relation = {REGION}/{VPC} = {vpc_vrf.data['name']} +""") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: inventory_data()}, SECOND_ACCOUNT: {REGION: inventory_data()}, + })) + source.sync_snapshot = Mock() + with pytest.raises(SystemExit): + source.apply() + source.sync_snapshot.assert_not_called() + + +def test_organization_account_specific_vrf_mapping(make_source, inventory, vpc_vrf, monkeypatch): + second_vrf = inventory.add_object(NBVRF, data={"id": 2, "name": "Second VPC"}, read_from_netbox=True) + source = make_source(f""" +organization = True +vpc_vrf_relation = {SECOND_ACCOUNT}/{REGION}/{VPC} = Second VPC +""") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value={ + ACCOUNT: {REGION: inventory_data()}, SECOND_ACCOUNT: {REGION: inventory_data()}, + })) + source.apply() + assert {address.data["vrf"] for address in inventory.get_all_items(NBIPAddress)} == { + vpc_vrf, second_vrf, + } + + +def shared_network_snapshots(): + owner = inventory_data() + owner["Vpcs"][VPC]["OwnerId"] = ACCOUNT + owner["Subnets"][SUBNET]["OwnerId"] = ACCOUNT + participant = copy.deepcopy(owner) + instance = participant["instances"][0] + instance["InstanceId"] = "i-11111111111111111" + instance["NetworkInterfaces"] = [{"NetworkInterfaceId": "eni-11111111111111111"}] + eni = participant["NetworkInterfaces"].pop(ENI) + eni["NetworkInterfaceId"] = "eni-11111111111111111" + eni["Attachment"]["InstanceId"] = instance["InstanceId"] + eni["MacAddress"] = "02:aa:bb:cc:dd:ff" + eni["PrivateIpAddresses"] = [{"PrivateIpAddress": "10.0.0.20", "Primary": True}] + eni["Ipv6Addresses"] = [{"Ipv6Address": "2001:db8::20", "IsPrimaryIpv6": True}] + participant["NetworkInterfaces"][eni["NetworkInterfaceId"]] = eni + return {ACCOUNT: {REGION: owner}, SECOND_ACCOUNT: {REGION: participant}} + + +@pytest.mark.parametrize("missing", ["both", "vpc", "subnet", "neither"]) +@pytest.mark.parametrize("participant_first", [False, True]) +def test_organization_shared_network_uses_owner_vrf( + make_source, inventory, monkeypatch, missing, participant_first): + snapshots = shared_network_snapshots() + participant = snapshots[SECOND_ACCOUNT][REGION] + if missing in ("both", "vpc"): + participant["Vpcs"].clear() + if missing in ("both", "subnet"): + participant["Subnets"].clear() + if participant_first: + snapshots = dict(reversed(list(snapshots.items()))) + source = make_source("organization = True") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value=snapshots)) + + source.apply() + source.apply() + + vrf, = inventory.get_all_items(NBVRF) + assert vrf.data["name"] == f"AWS {ACCOUNT}/{REGION}/{VPC}" + assert len(inventory.get_all_items(NBIPAddress)) == 5 + assert all(ip.data["vrf"] is vrf for ip in inventory.get_all_items(NBIPAddress)) + assert {vm.data["custom_fields"]["aws_account_id"] for vm in inventory.get_all_items(NBVM)} == { + ACCOUNT, SECOND_ACCOUNT, + } + + +def test_organization_shared_network_uses_owner_mapping(make_source, inventory, monkeypatch): + snapshots = shared_network_snapshots() + source = make_source(f""" +organization = True +vpc_vrf_relation = {ACCOUNT}/{REGION}/{VPC} = Shared Network +""") + vrf = inventory.add_object(NBVRF, data={"id": 1, "name": "Shared Network"}, read_from_netbox=True) + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value=snapshots)) + source.apply() + assert {ip.data["vrf"] for ip in inventory.get_all_items(NBIPAddress)} == {vrf} + + +@pytest.mark.parametrize("failure, message", [ + ("missing-subnet", "was not returned by DescribeSubnets"), + ("missing-vpc", "was not returned by DescribeVpcs"), + ("wrong-region", "was not returned by DescribeVpcs"), + ("subnet-owner", "Conflicting AWS subnet ownership"), + ("subnet-vpc", "belongs to VPC"), + ("ipv4-conflict", "Conflicting AWS Subnets discovery"), + ("ipv6-conflict", "Conflicting AWS Subnets discovery"), + ("vpc-conflict", "Conflicting AWS Vpcs discovery"), + ("vpc-owner", "Conflicting AWS Vpcs ownership"), + ("ambiguous-owner", "Ambiguous AWS VPC ownership"), +]) +def test_organization_shared_network_failure_aborts_before_sync( + make_source, inventory, monkeypatch, caplog, failure, message): + snapshots = shared_network_snapshots() + owner, participant = snapshots[ACCOUNT][REGION], snapshots[SECOND_ACCOUNT][REGION] + if failure in ("missing-subnet", "missing-vpc", "wrong-region", "ambiguous-owner"): + participant["Vpcs"].clear() + participant["Subnets"].clear() + if failure == "missing-subnet": + owner["Subnets"].clear() + elif failure == "missing-vpc": + owner["Vpcs"].clear() + elif failure == "wrong-region": + snapshots[ACCOUNT] = {"us-west-2": owner} + elif failure == "subnet-owner": + owner["Subnets"][SUBNET]["OwnerId"] = SECOND_ACCOUNT + participant["Subnets"][SUBNET]["OwnerId"] = SECOND_ACCOUNT + elif failure == "subnet-vpc": + for data in (owner, participant): + data["Subnets"][SUBNET]["VpcId"] = "vpc-11111111111111111" + elif failure == "ipv4-conflict": + participant["Subnets"][SUBNET]["CidrBlock"] = "10.0.0.0/25" + elif failure == "ipv6-conflict": + participant["Subnets"][SUBNET]["Ipv6CidrBlockAssociationSet"].clear() + elif failure == "vpc-conflict": + participant["Vpcs"][VPC]["CidrBlock"] = "10.0.0.0/16" + elif failure == "vpc-owner": + participant["Vpcs"][VPC]["OwnerId"] = SECOND_ACCOUNT + elif failure == "ambiguous-owner": + owner["Vpcs"][VPC].pop("OwnerId") + third = copy.deepcopy(owner) + third["instances"] = [] + snapshots[MANAGEMENT] = {REGION: third} + source = make_source("organization = True") + monkeypatch.setattr(AWSClient, "discover_organization", Mock(return_value=snapshots)) + source.sync_snapshot = Mock() + + with pytest.raises(SystemExit): + source.apply() + + assert message in caplog.text + source.sync_snapshot.assert_not_called() + assert inventory.get_all_items(NBVM) == [] + assert inventory.get_all_items(NBIPAddress) == [] + + +def test_shared_network_sdk_to_inventory(org_sdk, inventory, load_config): + snapshots = shared_network_snapshots() + owner, participant = snapshots[ACCOUNT][REGION], snapshots[SECOND_ACCOUNT][REGION] + owner["instances"] = [] + participant["Vpcs"].clear() + participant["Subnets"].clear() + organization_accounts(org_sdk) + for account, data in ((ACCOUNT, owner), (SECOND_ACCOUNT, participant)): + member = assume_member(org_sdk, account) + identity(member.sts, account) + region_response(member.ec2, [(REGION, "opt-in-not-required")]) + stub_region(member.ec2, data) + load_config(f""" +[source/test-aws] +type = aws +organization = True +regions = auto +account_id = {MANAGEMENT} +sync_private_ips = True +permitted_subnets = 10.0.0.0/8, 2001:db8::/32 +""") + source, = instantiate_sources() + source.apply() + + vm, = inventory.get_all_items(NBVM) + assert vm.data["custom_fields"]["aws_account_id"] == SECOND_ACCOUNT + vrf, = inventory.get_all_items(NBVRF) + assert vrf.data["name"] == f"AWS {ACCOUNT}/{REGION}/{VPC}" + assert {ip.data["address"] for ip in inventory.get_all_items(NBIPAddress)} == { + "10.0.0.20/24", "2001:db8::20/64", + } + assert vm.data["primary_ip4"].data["vrf"] is vrf + + +@pytest.mark.parametrize("extra", [ + "organization = True\norganization_role_name = arn:aws:iam::123456789012:role/Inventory", + "organization = True\norganization_role_name = invalid role", + "organization = True\norganization_external_id = x", + "organization = True\norganization_account_ids = 123", + "organization = True\norganization_account_ids = 123456789012, invalid", + "organization_account_ids = 123456789012", + "organization_include_management = True", + "organization_external_id = test-id", +]) +def test_organization_invalid_config(make_source, extra): + with pytest.raises(SystemExit): + make_source(extra) + + +def test_organization_yaml_options(inventory, load_config, monkeypatch): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(f""" +source: + test-aws: + type: aws + organization: true + organization_account_ids: ["{ACCOUNT}", "{SECOND_ACCOUNT}"] + organization_role_name: service-accounts/Inventory +""", "settings.yaml") + source, = instantiate_sources() + assert source.settings.organization_account_ids == [ACCOUNT, SECOND_ACCOUNT] + assert source.settings.organization_role_name == "service-accounts/Inventory" + + +def test_organization_empty_member_does_not_stop_next_account(org_sdk): + organization_accounts(org_sdk) + empty = inventory_data() + empty["instances"] = [] + for account, data in ((ACCOUNT, empty), (SECOND_ACCOUNT, inventory_data())): + member = assume_member(org_sdk, account) + identity(member.sts, account) + stub_region(member.ec2, data) + snapshots = AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + assert snapshots[ACCOUNT][REGION]["instances"] == [] + assert snapshots[SECOND_ACCOUNT][REGION]["instances"][0]["InstanceId"] == INSTANCE + + +def test_organization_paginates_150_accounts(org_sdk, monkeypatch): + accounts = [str(111100000000 + index) for index in range(150)] + identity(org_sdk.central.sts, MANAGEMENT) + org_sdk.central.organizations.add_response("describe_organization", { + "Organization": {"MasterAccountId": MANAGEMENT}, + }, {}) + for start in range(0, len(accounts), 20): + response = {"Accounts": [ + {"Id": account, "State": "ACTIVE"} for account in accounts[start:start + 20] + ]} + if start + 20 < len(accounts): + response["NextToken"] = f"page-{start + 20}" + params = {"NextToken": f"page-{start}"} if start else {} + org_sdk.central.organizations.add_response("list_accounts", response, params) + for account in accounts: + org_sdk.central.sts.add_response("assume_role", {"Credentials": { + "AccessKeyId": "ASIA" + "A" * 16, "SecretAccessKey": "test-only-secret", + "SessionToken": account, "Expiration": datetime(2099, 1, 1, tzinfo=timezone.utc), + }}, {"RoleArn": f"arn:aws:iam::{account}:role/NetBoxInventoryReadOnly", + "RoleSessionName": "netbox-sync", "DurationSeconds": 3600}) + org_sdk.factory.side_effect = lambda **kwargs: org_sdk.central.session + discovery = Mock(side_effect=lambda regions, expected_account, discovery_region: ( + expected_account, {REGION: inventory_data()})) + monkeypatch.setattr(AWSClient, "discover", discovery) + snapshots = AWSClient().discover_organization([REGION], "NetBoxInventoryReadOnly") + assert list(snapshots) == accounts + assert discovery.call_count == 150 + assert org_sdk.factory.call_count == 151 + + +def test_organization_source_forwards_options(make_source, monkeypatch): + source = make_source(f""" +organization = True +organization_include_management = True +organization_role_name = service-accounts/Inventory +organization_external_id = inventory-test +organization_account_ids = {ACCOUNT}, {SECOND_ACCOUNT} +discovery_region = us-west-2 +""") + discovery = Mock(return_value={ACCOUNT: {}}) + monkeypatch.setattr(AWSClient, "discover_organization", discovery) + source.apply() + discovery.assert_called_once_with( + [REGION], "service-accounts/Inventory", expected_account=None, + discovery_region="us-west-2", include_management=True, external_id="inventory-test", + account_ids=[ACCOUNT, SECOND_ACCOUNT]) + + +@pytest.mark.parametrize("accounts", ["[]", "[123456789012]", "123456789012", "true"]) +def test_organization_invalid_yaml_allowlist(inventory, load_config, monkeypatch, accounts): + monkeypatch.setattr(AWSClient, "__init__", lambda self, profile=None: None) + load_config(f""" +source: + test-aws: + type: aws + organization: true + organization_account_ids: {accounts} +""", "settings.yaml") + with pytest.raises(SystemExit): + instantiate_sources() diff --git a/uv.lock b/uv.lock index 0af21d5f..df188b4f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 -revision = 3 -requires-python = ">=3.13" +revision = 5 +requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", "python_full_version < '3.14'", @@ -18,6 +18,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/70/72e4ab117425ccdc4d10bd523a94c1baa051a15586057d64a4c6888f9e3f/aiodns-4.0.4-py3-none-any.whl", hash = "sha256:c24dd605bac70a1676ce503f967a98483ff163507198557d8e9db16267e6cfd2", size = 12696, upload-time = "2026-05-20T01:54:14.134Z" }, ] +[[package]] +name = "boto3" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/8b/5694c8433b5202f34a82def4edee274f7df1eba2919c66d9d389d2202ec8/boto3-1.43.118.tar.gz", hash = "sha256:c0620f799e0f0c222ba42c337b65cdbcd3da38a3121187f634d8926cbbb28a9c", size = 112649, upload-time = "2026-10-22T19:22:09.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a3/65bb3d64928f3c2da17640b92044084150317f02adf22442cb0c52af2a8b/boto3-1.43.118-py3-none-any.whl", hash = "sha256:ab8f3d2c622ef039338b4f228d97ea55f1abd573d84bb9b9a5ee8cb6780c0ccd", size = 140045, upload-time = "2026-10-22T19:22:08.353Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b8/3e55154ddaf1a6d1d4ddda85c0e5983205a3b7657edacd8ebba8c6fd774c/botocore-1.43.118.tar.gz", hash = "sha256:7f3f9d88cc9e847587690f82a42e3c3c292c3026c92c1e60d68a9310018e003f", size = 16420126, upload-time = "2026-10-22T19:22:04.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/ce/61b39a8e7a69af5ea43c1c3e1b0065a4866b42ccddf107e16686a3143614/botocore-1.43.118-py3-none-any.whl", hash = "sha256:a45d7ae570674161e4a3542d8c68633bf7503f92d1de781ef7361b79488eda6f", size = 16122016, upload-time = "2026-10-22T19:22:00.414Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -36,6 +64,18 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, @@ -106,6 +146,22 @@ version = "3.5.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, @@ -255,9 +311,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "netbox-sync" -version = "1.9.0" +version = "1.9.1" source = { virtual = "." } dependencies = [ { name = "aiodns" }, @@ -280,6 +345,9 @@ dependencies = [ ] [package.optional-dependencies] +aws = [ + { name = "boto3" }, +] test = [ { name = "pytest" }, ] @@ -287,6 +355,7 @@ test = [ [package.metadata] requires-dist = [ { name = "aiodns", specifier = "==4.0.4" }, + { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.40,<2" }, { name = "certifi", specifier = "==2026.7.22" }, { name = "cffi", specifier = "==2.1.1" }, { name = "charset-normalizer", specifier = "==3.5.2" }, @@ -305,7 +374,7 @@ requires-dist = [ { name = "urllib3", specifier = "==2.8.0" }, { name = "wheel", specifier = "==0.48.0" }, ] -provides-extras = ["test"] +provides-extras = ["aws", "test"] [[package]] name = "packaging" @@ -334,6 +403,18 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/df/a0/9c823651872e6a0face3f0311de2a40c8bbcb9c8dcb15680bd019ac56ac7/pycares-5.0.1.tar.gz", hash = "sha256:5a3c249c830432631439815f9a818463416f2a8cbdb1e988e78757de9ae75081", size = 652222, upload-time = "2026-01-01T12:37:00.604Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/75/67/e84ba11d3fec3bf1322c3b302c4df13c85e0a1bc48f16d65cd0f59ad9853/pycares-5.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ee551be4f3f3ac814ac8547586c464c9035e914f5122a534d25de147fa745e1", size = 136241, upload-time = "2026-01-01T12:35:25.439Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ae/50fbb3b4e52b9f1d16a36ffabd051ef8b2106b3f0a0d1c1113904d187a9d/pycares-5.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:252d4e5a52a68f825eaa90e16b595f9baee22c760f51e286ab612c6829b96de3", size = 131069, upload-time = "2026-01-01T12:35:26.293Z" }, + { url = "https://files.pythonhosted.org/packages/0e/ea/f431599f1ac42149ea4768e516db7cdae3a503a6646319ae63ab66da1486/pycares-5.0.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c1aa549b8c2f2e224215c793d660270778dcba9abc3b85abbc7c41eabe4f1e5", size = 221120, upload-time = "2026-01-01T12:35:27.143Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4f/0a7a6c8b3a64ee5149e935c167cd8ba5d1fdd766ec03e273dbc7502f7bea/pycares-5.0.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:db7c9c9f16e8311998667a7488e817f8cbeedec2447bac827c71804663f1437e", size = 252228, upload-time = "2026-01-01T12:35:28.443Z" }, + { url = "https://files.pythonhosted.org/packages/49/3d/7f9fd20e97ee30c4b959f87ab26e47ddcef666e5e7717e45f2245fe9d70a/pycares-5.0.1-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9c4c8bb69bab863f677fa166653bb872bfa5d5a742f1f30bebc2d53b6e71db", size = 239473, upload-time = "2026-01-01T12:35:29.794Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d0/c67967a10abd89529cb9aded9d73f43e5de00cf21243638ef529f6757262/pycares-5.0.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09ef90da8da3026fcba4ed223bd71e8057608d5b3fec4f5990b52ae1e8c855cc", size = 223831, upload-time = "2026-01-01T12:35:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/4f/9a/94aacaf22a20b7d342c8f18bf006be57967beef6319adc668d4d86b627be/pycares-5.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ce193ebd54f4c74538b751ebb0923a9208c234ff180589d4d3cec134c001840e", size = 223963, upload-time = "2026-01-01T12:35:31.691Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e1/3666aab6fc5e7d0c669b981fe0407e6a4b67e4e6a37ac429d440274663d5/pycares-5.0.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36b9ff18ef231277f99a846feade50b417187a96f742689a9d08b9594e386de4", size = 251813, upload-time = "2026-01-01T12:35:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/94/44/ddab5fbc16ad0084a827167ae8628f54c7a55ce6b743585e6f47a5dd527e/pycares-5.0.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5e40ea4a0ef0c01a02ef7f7390a58c62d237d5ad48d36bc3245e9c2ac181cc22", size = 238181, upload-time = "2026-01-01T12:35:34.078Z" }, + { url = "https://files.pythonhosted.org/packages/66/27/05467933e0e5c4e712302a2d7499797bc3029bf4d0d8ffbfe737254482b7/pycares-5.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3f323b0ddfd2c7896af6fba4f8851d34d3d13387566aa573d93330fb01cb1038", size = 223552, upload-time = "2026-01-01T12:35:35.076Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e2/14f3837e943d46ee12441fe6aaa418fdb2f698d42e179f368eaa9829744b/pycares-5.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:bdc6bcafb72a97b3cdd529fc87210e59e67feb647a7e138110656023599b84da", size = 117478, upload-time = "2026-01-01T12:35:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c3/3284061f18188d5085338e1f1fd4f03d9c135657acf16f8020b9dd3be5fc/pycares-5.0.1-cp312-cp312-win_arm64.whl", hash = "sha256:f8ef4c70c1edaf022875a8f9ff6c0c064f82831225acc91aa1b4f4d389e2e03a", size = 108889, upload-time = "2026-01-01T12:35:37.135Z" }, { url = "https://files.pythonhosted.org/packages/92/0a/6bd9bdc2d0ee23ff3aabab7747212e2c5323a081b9b745624d62df88f7e9/pycares-5.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d1b2c6b152c65f14d0e12d741fabb78a487f0f0d22773eede8d8cfc97af612b", size = 136242, upload-time = "2026-01-01T12:35:38.372Z" }, { url = "https://files.pythonhosted.org/packages/18/2a/2e9f888fc076cfe7a3493a3c4113e787cc4b4533f531dfb562ac9b04898f/pycares-5.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8c8ffcc9a48cfc296fe1aefc07d2c8e29a7f97e4bb366ce17effea6a38825f70", size = 131070, upload-time = "2026-01-01T12:35:39.262Z" }, { url = "https://files.pythonhosted.org/packages/ec/5b/83b5aaf7b6ed102f63cd768a747b6cb5d4624f2eaecd84868d103b9dbf39/pycares-5.0.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8efc38c2703e3530b823a4165a7b28d7ce0fdcf41960fb7a4ca834a0f8cfe79", size = 221137, upload-time = "2026-01-01T12:35:40.155Z" }, @@ -432,6 +513,16 @@ version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, @@ -477,6 +568,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "setuptools" version = "84.0.0"