266 lines
17 KiB
JavaScript
266 lines
17 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { execFileSync, spawnSync } from "node:child_process";
|
|
import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import test from "node:test";
|
|
|
|
const templateUrl = new URL("../infra/aws-agentcore-paperclip.yaml", import.meta.url);
|
|
const wrapperUrl = new URL("./aws-agentcore.sh", import.meta.url);
|
|
const labServerUrl = new URL("./capability-issue-thread-server.mjs", import.meta.url);
|
|
const liveSessionUrl = new URL("../src/live/live-session.ts", import.meta.url);
|
|
|
|
function parameterAllowedPattern(source, parameterName) {
|
|
const marker = ` ${parameterName}:\n`;
|
|
const start = source.indexOf(marker);
|
|
assert.notEqual(start, -1, `missing ${parameterName} parameter`);
|
|
const remaining = source.slice(start + marker.length);
|
|
const nextMatch = /\n \S/.exec(remaining);
|
|
const nextParameter = nextMatch ? start + marker.length + nextMatch.index : -1;
|
|
const block = source.slice(start, nextParameter === -1 ? undefined : nextParameter);
|
|
const match = block.match(/AllowedPattern: "([^"]+)"/);
|
|
assert.ok(match, `missing ${parameterName} AllowedPattern`);
|
|
return new RegExp(match[1]);
|
|
}
|
|
|
|
test("AgentCore template has closed development/private resources and explicit command denial", async () => {
|
|
const source = await readFile(templateUrl, "utf8");
|
|
for (const resource of [
|
|
"AWS::BedrockAgentCore::Harness",
|
|
"AWS::BedrockAgentCore::Memory",
|
|
"AWS::S3::Bucket",
|
|
"AWS::KMS::Key",
|
|
"AWS::EC2::VPC",
|
|
"com.amazonaws.${AWS::Region}.ecr.api",
|
|
"com.amazonaws.${AWS::Region}.ecr.dkr",
|
|
"com.amazonaws.${AWS::Region}.s3",
|
|
"com.amazonaws.${AWS::Region}.bedrock-runtime",
|
|
]) assert.match(source, new RegExp(resource.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")));
|
|
assert.match(source, /AllowedValues: \[development, private\]/);
|
|
assert.match(source, /Effect: Deny\s+Action: bedrock-agentcore:InvokeAgentRuntimeCommand/g);
|
|
assert.doesNotMatch(source, /Effect: Allow\s+Action: bedrock-agentcore:InvokeAgentRuntimeCommand/);
|
|
assert.doesNotMatch(source, /AWS::EC2::NatGateway|AWS::EC2::InternetGateway/);
|
|
assert.match(source, /EventExpiryDuration: 90/);
|
|
assert.match(source, /MaxIterations: 8/);
|
|
assert.match(source, /MaxTokens: 4096/);
|
|
assert.match(source, /TimeoutSeconds: 300/);
|
|
assert.match(source, /AllowedTools:\s+- "@\*\/pc_\*"/);
|
|
assert.match(source, /Tools: \[\]/);
|
|
assert.match(source, /Skills: \[\]/);
|
|
assert.doesNotMatch(source, /BedrockModelResourceArn:\s+[\s\S]{0,100}Default: "\*"/);
|
|
assert.match(source, /BedrockModelId:[\s\S]*AllowedPattern: "\^\[A-Za-z0-9\]\[A-Za-z0-9\._:-\]\{0,255\}\$"/);
|
|
assert.match(source, /BedrockModelResourceArn:[\s\S]*AllowedPattern:[^\n]+inference-profile[^\n]+\[A-Za-z0-9\]\[A-Za-z0-9\._:-\]\{0,255\}/);
|
|
assert.match(source, /BedrockFoundationModelResourceArn:[\s\S]*AllowedPattern:[^\n]+foundation-model\/\[A-Za-z0-9\]\[A-Za-z0-9\._:-\]\{0,255\}/);
|
|
const modelIdPattern = parameterAllowedPattern(source, "BedrockModelId");
|
|
assert.match("global.anthropic.claude-sonnet-4-6", modelIdPattern);
|
|
assert.match("anthropic.claude-3-5-sonnet-20241022-v2:0", modelIdPattern);
|
|
for (const unsafeModelId of ["custom/model", "custom*", "custom?", "arn:aws:bedrock:us-east-1::foundation-model/custom"]) {
|
|
assert.doesNotMatch(unsafeModelId, modelIdPattern);
|
|
}
|
|
const inferenceProfilePattern = parameterAllowedPattern(source, "BedrockModelResourceArn");
|
|
assert.match("arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.anthropic.claude-sonnet-4-6", inferenceProfilePattern);
|
|
assert.doesNotMatch("arn:aws:bedrock:us-east-1:123456789012:inference-profile/*", inferenceProfilePattern);
|
|
assert.doesNotMatch("arn:aws:bedrock:us-east-1:123456789012:inference-profile/custom/model", inferenceProfilePattern);
|
|
const foundationModelPattern = parameterAllowedPattern(source, "BedrockFoundationModelResourceArn");
|
|
assert.match("arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-4-6", foundationModelPattern);
|
|
assert.doesNotMatch("arn:aws:bedrock:*::foundation-model/*", foundationModelPattern);
|
|
assert.doesNotMatch("arn:aws:bedrock:*::foundation-model/custom/model", foundationModelPattern);
|
|
assert.match(source, /BedrockFoundationModelResourceArn/);
|
|
assert.match(source, /HarnessEndpointName/);
|
|
assert.match(source, /ContextPrefix/);
|
|
assert.match(source, /BucketOwnerEnforced/);
|
|
assert.match(source, /SSEAlgorithm: aws:kms/);
|
|
assert.match(source, /EnableKeyRotation: true/);
|
|
assert.match(source, /Sid: DenyInsecureTransport/);
|
|
assert.match(source, /Sid: DenyUnencryptedContextUploads/);
|
|
assert.match(source, /Sid: DenyContextUploadsWithAnotherKey/);
|
|
assert.match(source, /Sid: WriteAndVerifyPinnedRuntimeContext[\s\S]*s3:GetObject[\s\S]*s3:PutObject[\s\S]*s3:DeleteObject[\s\S]*Resource: !Sub \$\{ContextBucket\.Arn\}\/\$\{ContextPrefix\}\/assets\/\*/);
|
|
assert.match(source, /Sid: EncryptPinnedRuntimeContext[\s\S]*kms:Encrypt[\s\S]*kms:GenerateDataKey[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/);
|
|
assert.match(source, /Sid: ReadPinnedRuntimeContext[\s\S]*Action: s3:GetObject[\s\S]*Resource: !Sub \$\{ContextBucket\.Arn\}\/\$\{ContextPrefix\}\/assets\/\*/);
|
|
assert.match(source, /Sid: EnumeratePinnedRuntimeContext[\s\S]*Action: s3:ListBucket[\s\S]*Resource: !GetAtt ContextBucket\.Arn[\s\S]*s3:prefix: !Sub \$\{ContextPrefix\}\/assets\/\*/);
|
|
assert.match(source, /Sid: EnumeratePinnedRuntimeContextForTeardown[\s\S]*Action: s3:ListBucket[\s\S]*Resource: !GetAtt ContextBucket\.Arn[\s\S]*s3:prefix: !Sub \$\{ContextPrefix\}\/assets\/\*/);
|
|
assert.match(source, /Sid: DecryptPinnedRuntimeContext[\s\S]*Action: kms:Decrypt[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/);
|
|
const invocationRole = source.slice(source.indexOf("RunnerInvocationRole:"), source.indexOf("PrivateVpc:"));
|
|
assert.doesNotMatch(invocationRole, /Action:\s+(?:-\s+)?(?:s3|kms):\*/);
|
|
assert.match(invocationRole, /Action: sts:AssumeRoleWithWebIdentity/);
|
|
assert.match(invocationRole, /token\.actions\.githubusercontent\.com:aud["']?: sts\.amazonaws\.com/);
|
|
assert.match(invocationRole, /token\.actions\.githubusercontent\.com:sub["']?: !Sub repo:\$\{GitHubRepository\}:environment:\$\{GitHubEnvironment\}/);
|
|
assert.doesNotMatch(invocationRole, /repo:\*|environment:\*/);
|
|
assert.match(source, /\$\{AgentHarness\.Arn\}\/harness-endpoint\/\$\{HarnessEndpointName\}/);
|
|
assert.match(source, /\$\{AgentHarness\.Arn\}\/runtime-endpoint\/\$\{HarnessEndpointName\}/);
|
|
assert.match(source, /BedrockMarketplaceProductId/);
|
|
assert.match(source, /Sid: ViewMarketplaceSubscriptionsForModelAccess[\s\S]*Action: aws-marketplace:ViewSubscriptions[\s\S]*Sid: SubscribePinnedMarketplaceModel[\s\S]*Action: aws-marketplace:Subscribe[\s\S]*aws-marketplace:ProductId: !Ref BedrockMarketplaceProductId/);
|
|
assert.doesNotMatch(source, /aws-marketplace:Unsubscribe/);
|
|
});
|
|
|
|
test("AgentCore wrapper is valid shell and writes only nonsecret profile metadata", async () => {
|
|
execFileSync("bash", ["-n", wrapperUrl.pathname]);
|
|
const source = await readFile(wrapperUrl, "utf8");
|
|
const generatedBlock = source.slice(source.lastIndexOf('"AWS_PROFILE=$AWS_PROFILE_NAME"'), source.lastIndexOf('>"$tmp"'));
|
|
const teardownBlock = source.slice(source.indexOf("write_teardown_metadata()"), source.indexOf("assume_runner_role()"));
|
|
assert.ok(generatedBlock.length > 0);
|
|
assert.ok(teardownBlock.length > 0);
|
|
assert.doesNotMatch(generatedBlock, /AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|Authorization|X-Amz-Signature/);
|
|
assert.doesNotMatch(teardownBlock, /AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|Authorization|X-Amz-Signature/);
|
|
assert.match(teardownBlock, /PAPERCLIP_AWS_AGENTCORE_STACK_NAME=\$STACK_NAME/);
|
|
assert.match(teardownBlock, /chmod 600 "\$tmp"/);
|
|
assert.match(source, /chmod 600 "\$tmp"/);
|
|
assert.match(source, /printf '%q\\n'/);
|
|
assert.match(source, /printf 'AWS_CONFIG_FILE=%q\\n'/);
|
|
assert.match(source, /AWS_PROFILE_EXPLICIT=true/);
|
|
assert.match(source, /AWS_REGION_EXPLICIT=true/);
|
|
assert.match(source, /STACK_NAME_EXPLICIT=true/);
|
|
assert.equal((source.match(/local requested_profile=/g) ?? []).length, 1);
|
|
assert.equal((source.match(/requested_region=/g) ?? []).length, 1);
|
|
assert.equal((source.match(/requested_stack=/g) ?? []).length, 1);
|
|
const destroyBlock = source.slice(source.indexOf("destroy()"), source.indexOf('case "$ACTION"'));
|
|
assert.ok(destroyBlock.indexOf("load_local_env") < destroyBlock.indexOf("stack_output HarnessId"));
|
|
assert.match(source, /cloudformation deploy/);
|
|
const provisionBlock = source.slice(source.indexOf("provision()"), source.indexOf("lab()"));
|
|
assert.ok(provisionBlock.indexOf("cloudformation deploy") < provisionBlock.indexOf("write_teardown_metadata"));
|
|
assert.ok(provisionBlock.indexOf("write_teardown_metadata") < provisionBlock.indexOf("stack_output HarnessId"));
|
|
assert.ok(provisionBlock.indexOf("write_teardown_metadata") < provisionBlock.indexOf("wait_for_harness_status"));
|
|
assert.match(source, /CAPABILITY_NAMED_IAM/);
|
|
assert.match(source, /--\) shift ;;/);
|
|
assert.match(source, /iam get-role --role-name "\$role_name"/);
|
|
assert.doesNotMatch(source, /printf 'arn:%s:iam::%s:role\/%s/);
|
|
assert.match(source, /existing_status.*ROLLBACK_COMPLETE/s);
|
|
assert.match(source, /--replace-failed-stack/);
|
|
assert.match(provisionBlock, /existing_description.*STACK_DESCRIPTION/s);
|
|
assert.match(provisionBlock, /existing_owned.*paperclip:owned/s);
|
|
assert.match(provisionBlock, /existing_environment.*paperclip:environment/s);
|
|
assert.match(provisionBlock, /existing_cost_center.*paperclip:cost-center/s);
|
|
assert.ok(provisionBlock.indexOf("ownership tags or template provenance") < provisionBlock.indexOf("cloudformation delete-stack"));
|
|
assert.ok(provisionBlock.indexOf("REPLACE_FAILED_STACK") < provisionBlock.indexOf("cloudformation delete-stack"));
|
|
assert.match(provisionBlock, /Unable to verify whether stack .* exists and is owned by Paperclip/);
|
|
assert.match(source, /cloudformation wait stack-delete-complete/);
|
|
assert.match(source, /--query harness\.status/);
|
|
assert.match(source, /AgentCore tool allowlist drift/);
|
|
assert.match(source, /--query memory\.status/);
|
|
assert.match(source, /--query endpoint\.status/);
|
|
assert.match(source, /o\.endpoint\?\.arn/);
|
|
assert.match(source, /--marketplace-product-id/);
|
|
assert.match(source, /--github-oidc-provider-arn/);
|
|
assert.match(source, /GitHubOidcProviderArn=\$GITHUB_OIDC_PROVIDER_ARN/);
|
|
assert.match(source, /BedrockMarketplaceProductId=\$MARKETPLACE_PRODUCT_ID/);
|
|
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=\$context_bucket/);
|
|
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=\$context_prefix/);
|
|
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=\$context_kms_key_arn/);
|
|
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=\$role_arn/);
|
|
assert.match(source, /ContextPrefix=\$CONTEXT_PREFIX/);
|
|
assert.match(source, /s3 rm "s3:\/\/\$context_bucket\/\$context_prefix\/assets\/" --recursive/);
|
|
assert.ok(source.indexOf("delete-harness-endpoint") < source.lastIndexOf("cloudformation delete-stack"));
|
|
});
|
|
|
|
test("AgentCore wrapper rejects unsafe model IDs before AWS access", () => {
|
|
const result = spawnSync("bash", [
|
|
wrapperUrl.pathname,
|
|
"provision",
|
|
"--model",
|
|
"custom/model/*",
|
|
"--marketplace-product-id",
|
|
"prod-safe123",
|
|
], { encoding: "utf8" });
|
|
assert.equal(result.status, 2);
|
|
assert.match(result.stderr, /must not contain ARN, path, glob, or wildcard syntax/);
|
|
});
|
|
|
|
test("AgentCore wrapper never implicitly deletes a colliding failed stack", async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), "paperclip-agentcore-test-"));
|
|
const fakeAws = join(temp, "aws");
|
|
const commandLog = join(temp, "aws.log");
|
|
await writeFile(fakeAws, `#!/usr/bin/env bash
|
|
set -eu
|
|
printf '%s\\n' "$*" >>"$MOCK_AWS_LOG"
|
|
if [[ " $* " == *" --version "* || "$1" == "--version" ]]; then
|
|
printf 'aws-cli/2.31.0 Python/3.13.0\\n'
|
|
elif [[ " $* " == *" sts get-caller-identity "* ]]; then
|
|
printf '{"Account":"123456789012","Arn":"arn:aws:iam::123456789012:role/paperclip-test"}\\n'
|
|
elif [[ " $* " == *" cloudformation describe-stacks "* ]]; then
|
|
printf '{"Stacks":[{"StackStatus":"ROLLBACK_COMPLETE","Description":"%s","Tags":[{"Key":"paperclip:owned","Value":"%s"},{"Key":"paperclip:environment","Value":"development"},{"Key":"paperclip:cost-center","Value":"runner-lab"}]}]}\\n' "\${MOCK_STACK_DESCRIPTION}" "\${MOCK_STACK_OWNED}"
|
|
elif [[ " $* " == *" cloudformation deploy "* ]]; then
|
|
exit 42
|
|
fi
|
|
`, { mode: 0o700 });
|
|
await chmod(fakeAws, 0o700);
|
|
const baseEnv = {
|
|
...process.env,
|
|
PATH: `${temp}:${process.env.PATH}`,
|
|
MOCK_AWS_LOG: commandLog,
|
|
MOCK_STACK_DESCRIPTION: "Paperclip proof-of-concept Amazon Bedrock AgentCore Harness and least-privilege invocation roles.",
|
|
MOCK_STACK_OWNED: "true",
|
|
};
|
|
try {
|
|
const implicit = spawnSync("bash", [wrapperUrl.pathname, "provision"], { encoding: "utf8", env: baseEnv });
|
|
assert.equal(implicit.status, 1);
|
|
assert.match(implicit.stderr, /--replace-failed-stack/);
|
|
assert.doesNotMatch(await readFile(commandLog, "utf8"), /cloudformation delete-stack/);
|
|
|
|
await writeFile(commandLog, "");
|
|
const foreign = spawnSync("bash", [wrapperUrl.pathname, "provision", "--replace-failed-stack", "--yes"], {
|
|
encoding: "utf8",
|
|
env: { ...baseEnv, MOCK_STACK_OWNED: "false" },
|
|
});
|
|
assert.equal(foreign.status, 1);
|
|
assert.match(foreign.stderr, /ownership tags or template provenance do not match/);
|
|
assert.doesNotMatch(await readFile(commandLog, "utf8"), /cloudformation delete-stack/);
|
|
|
|
await writeFile(commandLog, "");
|
|
const explicit = spawnSync("bash", [wrapperUrl.pathname, "provision", "--replace-failed-stack", "--yes"], {
|
|
encoding: "utf8",
|
|
env: baseEnv,
|
|
});
|
|
assert.equal(explicit.status, 42);
|
|
const explicitLog = await readFile(commandLog, "utf8");
|
|
assert.match(explicitLog, /cloudformation delete-stack/);
|
|
assert.ok(explicitLog.indexOf("cloudformation delete-stack") < explicitLog.indexOf("cloudformation deploy"));
|
|
} finally {
|
|
await rm(temp, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("Runner Lab accepts and resolves the complete qualified AgentCore profile", async () => {
|
|
const source = await readFile(labServerUrl, "utf8");
|
|
assert.match(source, /provider !== "aws_agentcore"/);
|
|
assert.match(source, /AWS AgentCore requires exact model global\.anthropic\.claude-sonnet-4-6/);
|
|
assert.match(source, /function resolveAgentCoreProfile\(configuration\)/);
|
|
for (const field of [
|
|
"PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET",
|
|
"PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX",
|
|
"PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN",
|
|
]) assert.match(source, new RegExp(field));
|
|
assert.match(source, /agentCoreProfile: resolveAgentCoreProfile\(configuration\)/);
|
|
assert.match(source, /agentCoreProfileId: snapshot\.config\.agentCoreProfile\.profileId/);
|
|
assert.match(source, /configuration\.provider === "aws_agentcore" \? "remote_service"/);
|
|
});
|
|
|
|
test("Runner Lab qualifies Claude Managed and exposes remote governance for both remote providers", async () => {
|
|
const source = await readFile(labServerUrl, "utf8");
|
|
const liveSessionSource = await readFile(liveSessionUrl, "utf8");
|
|
assert.match(source, /provider !== "claude_managed"/);
|
|
assert.match(source, /Claude Managed requires exact model claude-sonnet-5/);
|
|
assert.match(source, /function resolveManagedProfile\(configuration\)/);
|
|
for (const field of [
|
|
"PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID",
|
|
"ANTHROPIC_API_KEY",
|
|
"ANTHROPIC_MANAGED_AGENT_ID",
|
|
"ANTHROPIC_MANAGED_AGENT_VERSION",
|
|
"ANTHROPIC_MANAGED_ENVIRONMENT_ID",
|
|
]) assert.match(source, new RegExp(field));
|
|
assert.match(source, /\^\[1-9\]\[0-9\]\*\$/);
|
|
assert.match(source, /BigInt\(agentVersion\) <= 2_147_483_647n/);
|
|
assert.match(source, /managedProfile: resolveManagedProfile\(configuration\)/);
|
|
assert.match(source, /managedProfileId: snapshot\.config\.managedProfile\.profileId/);
|
|
assert.match(source, /route === "managed-budget"/);
|
|
assert.match(source, /entry\.session\.increaseManagedSessionBudget\(nextCap\)/);
|
|
assert.match(source, /entry\.configuration\?\.provider === "claude_managed"/);
|
|
assert.match(source, /entry\.configuration\?\.provider === "aws_agentcore"/);
|
|
assert.match(source, /route === "managed-session-delete"/);
|
|
assert.match(source, /body\.confirm !== true/);
|
|
assert.match(source, /entry\.session\.deleteManagedRemoteSession\(\)/);
|
|
assert.match(liveSessionSource, /async increaseManagedSessionBudget\(/);
|
|
assert.match(liveSessionSource, /this\.#transport\.request\("session\/budget\/increase"/);
|
|
assert.match(liveSessionSource, /async deleteManagedRemoteSession\(\)/);
|
|
assert.match(liveSessionSource, /this\.#transport\.request\("session\/destroy"/);
|
|
});
|