<output_file> _merge_jobs() { local jobs_dir="$1" title="$2" output_file="$3" local _tmp _tmp="$(mktemp)" { echo "# $title" echo "Generated: $(date)" echo "Scanner: $(basename "$0")" echo "" echo "---" echo "" while IFS= read -r f; do [ -f "$f" ] || continue # skip sections whose code block is empty (## Title\n```\n```\n) if python3 -c " import re, sys t = open(sys.argv[1]).read() # match code block: backticks, optional content, backticks m = re.search(r'\x60{3}\n(.*?)\x60{3}', t, re.S) sys.exit(0 if m and m.group(1).strip() == '' else 1) " "$f" 2>/dev/null; then continue fi cat "$f" done < <(find "$jobs_dir" -maxdepth 1 -name '*.txt' | sort) } > "$_tmp" _redact_inventory < "$_tmp" > "$output_file" rm -f "$_tmp" } # ── elapsed time helper ────────────────────────────────────────────────────── _print_elapsed() { local start="$1" local elapsed=$(( $(date +%s) - start )) echo "Total time: ${elapsed}s" } FILE:scripts/aws-scan-enrich.sh #!/bin/bash # Per-resource deep AWS discovery — run after aws-scan-region.sh # # aws-scan-region.sh only runs single list/describe calls (one API call per service). # This script loops over every main resource and fetches its details: # # EventBridge — list-rules + list-targets-by-rule on every event bus # S3 — get-bucket-lifecycle-configuration + get-bucket-policy per bucket # SNS — list-subscriptions-by-topic per topic # IAM — list-role-policies + get-role-policy per role (inline policies) # Lambda — get-policy per function (resource-based / invoke permissions) # API Gateway — get-resources (embed methods) + get-stages per REST API # API GW v2 — get-routes + get-stages per HTTP API # ECS — list-services + describe-services per cluster # EKS — describe-cluster per cluster (version, VPC, addons) # ELB v2 — describe-listeners + describe-target-groups per LB # Route53 — list-resource-record-sets per hosted zone # CloudFront — get-distribution-config per distribution # DynamoDB — describe-table per table (capacity, GSI/LSI, streams) # SQS — get-queue-attributes per queue (DLQ, policy, encryption) # RDS — describe-db-subnet-groups + describe-db-parameter-groups # Step Functions — describe-state-machine per machine # ElastiCache — describe-replication-groups + describe-cache-parameters (user-modified) # MSK (Kafka) — describe-cluster per cluster (broker config, version) # Cognito — describe-user-pool + list-user-pool-clients per pool # EFS — describe-mount-targets + describe-access-points per file system # # All sections run in parallel (background subshells), each with a timeout guard. # # Usage: # ./aws-scan-enrich.sh <region> [path-to-existing-aws-scan-output-dir] # # If the optional directory is given and exists, writes: # <that-dir>/inventory-deep.md # Otherwise creates: # ./aws-scan-<region>-deep-<timestamp>/inventory-deep.md # # Environment (same semantics as aws-scan-region.sh): # AWS_SCAN_CMD_TIMEOUT default 120 # AWS_SCAN_REDACT default 1 # AWS_SCAN_MAX_PARALLEL default 20 # AWS_MAX_ATTEMPTS / AWS_RETRY_MODE # AWS_SCAN_SECTION_TIMEOUT Per-section overall timeout (default: 300) # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=_scan-common.sh source "$SCRIPT_DIR/_scan-common.sh" START_TIME=$(date +%s) AWS_SCAN_SECTION_TIMEOUT="${AWS_SCAN_SECTION_TIMEOUT:-300}" REGION="${1:?usage: $0 <region> [existing-scan-output-dir]}" OPTIONAL_DIR="${2:-}" validate_aws_region "$REGION" if [[ -n "$OPTIONAL_DIR" ]] && [[ -d "$OPTIONAL_DIR" ]]; then DEEP_DIR="$(cd "$OPTIONAL_DIR" && pwd)" else TIMESTAMP=$(date +%Y%m%d_%H%M%S) DEEP_DIR="$(pwd)/aws-scan-${REGION}-deep-${TIMESTAMP}" mkdir -p "$DEEP_DIR" fi JOBS_DIR="${DEEP_DIR}/.deep-jobs" OUTPUT_FILE="${DEEP_DIR}/inventory-deep.md" mkdir -p "$JOBS_DIR" echo "Deep scan region: $REGION" echo "Output: $OUTPUT_FILE" echo "Max parallel: $AWS_SCAN_MAX_PARALLEL Section timeout: ${AWS_SCAN_SECTION_TIMEOUT}s" echo "" # ── helpers ─────────────────────────────────────────────────────────────────── # deep_section <name> <commands...> — same parallel pattern as aws-scan-region.sh deep_section() { local name="$1" shift local safe safe="$(echo "$name" | tr ' /' '__')" local out="$JOBS_DIR/${safe}.txt" _wait_for_slot { echo "## $name" echo '```' if ! _run_scan_cmd env AWS_DEFAULT_REGION="$REGION" "$@" 2>/dev/null; then echo "(no resources, no access, or timeout)" fi echo '```' echo "" } > "$out" & } # _run_section_with_timeout <output_file> <script> — wraps bash -c with overall timeout _run_section_with_timeout() { local out="$1" script="$2" export AWS_SCAN_SECTION_TIMEOUT if command -v timeout >/dev/null 2>&1; then timeout --signal=TERM "$AWS_SCAN_SECTION_TIMEOUT" bash -c "$script" > "$out" 2>&1 || { if [[ ! -s "$out" ]]; then printf '## (section timed out)\n```\n(timed out after %ss)\n```\n\n' "$AWS_SCAN_SECTION_TIMEOUT" > "$out" fi } elif command -v perl >/dev/null 2>&1; then perl -e 'alarm $ENV{AWS_SCAN_SECTION_TIMEOUT}; exec "bash", "-c", $ARGV[0]' -- "$script" > "$out" 2>&1 || { if [[ ! -s "$out" ]]; then printf '## (section timed out)\n```\n(timed out after %ss)\n```\n\n' "$AWS_SCAN_SECTION_TIMEOUT" > "$out" fi } else bash -c "$script" > "$out" 2>&1 fi } # ── EventBridge: rules + targets on every event bus (single list-rules call) ── _wait_for_slot ( _out="$JOBS_DIR/EventBridge_Rules_and_Targets.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## EventBridge Rules and Targets (all event buses)" echo "\`\`\`" aws events list-event-buses --region "$REGION" \ --query "EventBuses[].Name" --output text 2>/dev/null \ | tr "\t" "\n" | sed "/^$/d" | while read -r bus; do echo "=== Event bus: $bus ===" # Single JSON call: display rules and extract names for target lookup rules_json=$(_run_scan_cmd aws events list-rules --region "$REGION" \ --event-bus-name "$bus" --output json 2>/dev/null) || continue echo "$rules_json" | python3 -c " import json, sys rules = json.load(sys.stdin).get(\"Rules\", []) for r in rules: print(f\" Rule: {r[\"Name\"]} State: {r.get(\"State\",\"?\")} Schedule: {r.get(\"ScheduleExpression\",\"-\")} Pattern: {str(r.get(\"EventPattern\",\"-\"))[:80]}\") " 2>/dev/null || echo "$rules_json" # Extract rule names from the same JSON echo "$rules_json" | python3 -c " import json, sys for r in json.load(sys.stdin).get(\"Rules\", []): print(r[\"Name\"]) " 2>/dev/null | while read -r rule; do [ -z "$rule" ] && continue echo "--- Targets: $bus / $rule ---" _run_scan_cmd aws events list-targets-by-rule --region "$REGION" \ --event-bus-name "$bus" --rule "$rule" \ --query "Targets[].[Id,Arn,Input,InputPath]" --output table 2>/dev/null || true done done echo "\`\`\`" echo "" ' ) & # ── S3: lifecycle + bucket policy per bucket ───────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/S3_Bucket_Config.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## S3 Bucket Config (Lifecycle + Policy)" echo "\`\`\`" aws s3api list-buckets --query "Buckets[].Name" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r bucket; do [ -z "$bucket" ] && continue echo "=== $bucket ===" echo "-- Lifecycle Rules --" _run_scan_cmd aws s3api get-bucket-lifecycle-configuration --bucket "$bucket" \ --query "Rules[].[ID,Status,Filter,Expiration,Transitions]" \ --output json 2>/dev/null || echo "(no lifecycle rules)" echo "-- Bucket Policy --" _run_scan_cmd aws s3api get-bucket-policy --bucket "$bucket" \ --query "Policy" --output text 2>/dev/null \ | python3 -c "import sys,json; p=json.load(sys.stdin); [print(\" Sid:\",s.get(\"Sid\"),\"Effect:\",s.get(\"Effect\"),\"Action:\",s.get(\"Action\"),\"Principal:\",s.get(\"Principal\")) for s in p.get(\"Statement\",[])]" \ 2>/dev/null || echo "(no bucket policy)" done echo "\`\`\`" echo "" ' ) & # ── SNS: subscriptions per topic ───────────────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/SNS_Subscriptions.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## SNS Subscriptions" echo "\`\`\`" aws sns list-topics --region "$REGION" \ --query "Topics[].TopicArn" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r arn; do [ -z "$arn" ] && continue echo "--- $arn ---" _run_scan_cmd aws sns list-subscriptions-by-topic --topic-arn "$arn" --region "$REGION" \ --query "Subscriptions[].[Protocol,Endpoint,SubscriptionArn]" \ --output table 2>/dev/null || true done echo "\`\`\`" echo "" ' ) & # ── IAM: inline policies per role ──────────────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/IAM_Inline_Policies.txt" _run_section_with_timeout "$_out" ' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## IAM Inline Policies per Role" echo "\`\`\`" aws iam list-roles --no-paginate \ --query "Roles[].RoleName" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r role; do [ -z "$role" ] && continue policies=$(aws iam list-role-policies --role-name "$role" \ --query "PolicyNames" --output text 2>/dev/null) [ -z "$policies" ] && continue echo "=== $role: $policies ===" for p in $policies; do _run_scan_cmd aws iam get-role-policy --role-name "$role" --policy-name "$p" \ --query "PolicyDocument.Statement[].[Effect,Action,Resource]" \ --output table 2>/dev/null || true done done echo "\`\`\`" echo "" ' ) & # ── Lambda: resource-based policies (invoke permissions) ───────────────────── _wait_for_slot ( _out="$JOBS_DIR/Lambda_Resource_Policies.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## Lambda Resource-based Policies" echo "\`\`\`" aws lambda list-functions --region "$REGION" \ --query "Functions[].FunctionName" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r fn; do [ -z "$fn" ] && continue policy=$(aws lambda get-policy --function-name "$fn" \ --region "$REGION" --query "Policy" --output text 2>/dev/null) || continue echo "--- $fn ---" echo "$policy" | python3 -c " import sys, json p = json.load(sys.stdin) for s in p[\"Statement\"]: print(\" Sid:\", s.get(\"Sid\"), \"| Principal:\", s.get(\"Principal\"), \"| Action:\", s.get(\"Action\")) " 2>/dev/null || echo "$policy" done echo "\`\`\`" echo "" ' ) & # ── API Gateway REST: get-resources + get-stages per API ───────────────────── _wait_for_slot ( _out="$JOBS_DIR/APIGateway_REST_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## API Gateway REST — Resources and Stages per API" echo "\`\`\`" aws apigateway get-rest-apis --region "$REGION" \ --query "items[].id" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r api_id; do [ -z "$api_id" ] && continue echo "=== API $api_id — get-resources ===" _run_scan_cmd aws apigateway get-resources --rest-api-id "$api_id" --region "$REGION" \ --embed "methods" --output json 2>/dev/null || echo "(no access or timeout)" echo "=== API $api_id — get-stages ===" _run_scan_cmd aws apigateway get-stages --rest-api-id "$api_id" --region "$REGION" \ --output table 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── API Gateway v2 (HTTP APIs): routes + stages per API ────────────────────── _wait_for_slot ( _out="$JOBS_DIR/APIGateway_HTTP_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## API Gateway v2 (HTTP) — Routes and Stages per API" echo "\`\`\`" aws apigatewayv2 get-apis --region "$REGION" \ --query "Items[].ApiId" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r api_id; do [ -z "$api_id" ] && continue echo "=== HTTP API $api_id — routes ===" _run_scan_cmd aws apigatewayv2 get-routes --api-id "$api_id" --region "$REGION" \ --query "Items[].[RouteKey,Target,AuthorizationType]" \ --output table 2>/dev/null || echo "(no access or timeout)" echo "=== HTTP API $api_id — stages ===" _run_scan_cmd aws apigatewayv2 get-stages --api-id "$api_id" --region "$REGION" \ --query "Items[].[StageName,AutoDeploy,LastUpdatedDate]" \ --output table 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── ECS: services + task definitions per cluster ───────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/ECS_Services_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## ECS Services and Tasks per Cluster" echo "\`\`\`" aws ecs list-clusters --region "$REGION" \ --query "clusterArns" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r cluster_arn; do [ -z "$cluster_arn" ] && continue echo "=== Cluster: $cluster_arn ===" services=$(_run_scan_cmd aws ecs list-services --region "$REGION" \ --cluster "$cluster_arn" --query "serviceArns" --output text 2>/dev/null) || continue [ -z "$services" ] && { echo "(no services)"; continue; } # describe-services accepts up to 10 at a time echo "$services" | tr "\t" "\n" | while read -r svc; do [ -z "$svc" ] && continue _run_scan_cmd aws ecs describe-services --region "$REGION" \ --cluster "$cluster_arn" --services "$svc" \ --query "services[].[serviceName,status,launchType,desiredCount,runningCount,taskDefinition]" \ --output table 2>/dev/null || true done done echo "\`\`\`" echo "" ' ) & # ── EKS: describe-cluster + addons per cluster ────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/EKS_Cluster_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## EKS Cluster Details and Addons" echo "\`\`\`" aws eks list-clusters --region "$REGION" \ --query "clusters" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r cluster; do [ -z "$cluster" ] && continue echo "=== Cluster: $cluster ===" _run_scan_cmd aws eks describe-cluster --region "$REGION" --name "$cluster" \ --query "cluster.[name,version,platformVersion,status,resourcesVpcConfig.[vpcId,subnetIds,securityGroupIds,endpointPublicAccess,endpointPrivateAccess],logging.clusterLogging]" \ --output json 2>/dev/null || echo "(no access or timeout)" echo "--- Addons ---" _run_scan_cmd aws eks list-addons --region "$REGION" --cluster-name "$cluster" \ --query "addons" --output table 2>/dev/null || echo "(no addons)" done echo "\`\`\`" echo "" ' ) & # ── ELB v2: listeners + target groups per load balancer ────────────────────── _wait_for_slot ( _out="$JOBS_DIR/ELBv2_Listeners_Targets.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## ELB v2 — Listeners and Target Groups per LB" echo "\`\`\`" aws elbv2 describe-load-balancers --region "$REGION" \ --query "LoadBalancers[].LoadBalancerArn" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r lb_arn; do [ -z "$lb_arn" ] && continue echo "=== LB: $lb_arn ===" echo "--- Listeners ---" _run_scan_cmd aws elbv2 describe-listeners --region "$REGION" \ --load-balancer-arn "$lb_arn" \ --query "Listeners[].[Port,Protocol,DefaultActions[0].Type,DefaultActions[0].TargetGroupArn]" \ --output table 2>/dev/null || echo "(no listeners)" done echo "--- All Target Groups ---" _run_scan_cmd aws elbv2 describe-target-groups --region "$REGION" \ --query "TargetGroups[].[TargetGroupName,Protocol,Port,TargetType,HealthCheckPath,VpcId]" \ --output table 2>/dev/null || echo "(no target groups)" echo "\`\`\`" echo "" ' ) & # ── Route53: record sets per hosted zone ───────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/Route53_Records.txt" _run_section_with_timeout "$_out" ' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## Route53 Record Sets per Hosted Zone" echo "\`\`\`" aws route53 list-hosted-zones \ --query "HostedZones[].[Id,Name]" --output text 2>/dev/null \ | while read -r zone_id zone_name; do [ -z "$zone_id" ] && continue echo "=== Zone: $zone_name ($zone_id) ===" _run_scan_cmd aws route53 list-resource-record-sets --hosted-zone-id "$zone_id" \ --query "ResourceRecordSets[].[Name,Type,TTL,AliasTarget.DNSName,ResourceRecords[0].Value]" \ --output table 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── CloudFront: distribution config per distribution ───────────────────────── _wait_for_slot ( _out="$JOBS_DIR/CloudFront_Config.txt" _run_section_with_timeout "$_out" ' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## CloudFront Distribution Config" echo "\`\`\`" aws cloudfront list-distributions \ --query "DistributionList.Items[].Id" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r dist_id; do [ -z "$dist_id" ] && continue echo "=== Distribution: $dist_id ===" _run_scan_cmd aws cloudfront get-distribution --id "$dist_id" \ --query "Distribution.DistributionConfig.[Origins.Items[*].[Id,DomainName,S3OriginConfig,CustomOriginConfig],DefaultCacheBehavior.[ViewerProtocolPolicy,AllowedMethods.Items,CachePolicyId,OriginRequestPolicyId],ViewerCertificate.[ACMCertificateArn,SSLSupportMethod,MinimumProtocolVersion],WebACLId]" \ --output json 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── DynamoDB: describe-table per table ─────────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/DynamoDB_Table_Details.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## DynamoDB Table Details" echo "\`\`\`" aws dynamodb list-tables --region "$REGION" \ --query "TableNames" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r table; do [ -z "$table" ] && continue echo "=== Table: $table ===" _run_scan_cmd aws dynamodb describe-table --region "$REGION" --table-name "$table" \ --query "Table.[TableName,TableStatus,BillingModeSummary.BillingMode,ProvisionedThroughput.[ReadCapacityUnits,WriteCapacityUnits],GlobalSecondaryIndexes[*].[IndexName,KeySchema,Projection.ProjectionType,ProvisionedThroughput],LocalSecondaryIndexes[*].[IndexName,KeySchema],StreamSpecification,SSEDescription.Status,TableSizeBytes,ItemCount]" \ --output json 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── SQS: queue attributes per queue ────────────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/SQS_Queue_Attributes.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## SQS Queue Attributes" echo "\`\`\`" aws sqs list-queues --region "$REGION" \ --query "QueueUrls" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r queue_url; do [ -z "$queue_url" ] && continue echo "=== $(basename "$queue_url") ===" _run_scan_cmd aws sqs get-queue-attributes --region "$REGION" \ --queue-url "$queue_url" --attribute-names All \ --query "Attributes.[QueueArn,VisibilityTimeout,MessageRetentionPeriod,DelaySeconds,RedrivePolicy,KmsMasterKeyId,FifoQueue,ContentBasedDeduplication]" \ --output json 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── RDS: subnet groups + parameter groups ──────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/RDS_Config_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## RDS Subnet Groups and Parameter Groups" echo "\`\`\`" echo "--- DB Subnet Groups ---" _run_scan_cmd aws rds describe-db-subnet-groups --region "$REGION" \ --query "DBSubnetGroups[].[DBSubnetGroupName,VpcId,DBSubnetGroupDescription,Subnets[*].[SubnetIdentifier,SubnetAvailabilityZone.Name]]" \ --output json 2>/dev/null || echo "(no subnet groups)" echo "--- DB Parameter Groups (non-default) ---" aws rds describe-db-instances --region "$REGION" \ --query "DBInstances[].[DBInstanceIdentifier,DBParameterGroups[0].DBParameterGroupName]" \ --output text 2>/dev/null \ | while read -r inst pg; do [ -z "$pg" ] && continue echo "=== $inst → $pg ===" _run_scan_cmd aws rds describe-db-parameters --region "$REGION" \ --db-parameter-group-name "$pg" \ --query "Parameters[?Source==\`user\`].[ParameterName,ParameterValue,ApplyType]" \ --output table 2>/dev/null || true done echo "\`\`\`" echo "" ' ) & # ── Step Functions: state machine definitions ──────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/StepFunctions_Definitions.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## Step Functions State Machine Definitions" echo "\`\`\`" aws stepfunctions list-state-machines --region "$REGION" \ --query "stateMachines[].stateMachineArn" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r sm_arn; do [ -z "$sm_arn" ] && continue echo "=== $(basename "$sm_arn") ===" _run_scan_cmd aws stepfunctions describe-state-machine --region "$REGION" \ --state-machine-arn "$sm_arn" \ --query "[name,status,type,loggingConfiguration,definition]" \ --output json 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── ElastiCache: replication groups + user-modified parameters ──────────────── _wait_for_slot ( _out="$JOBS_DIR/ElastiCache_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## ElastiCache Replication Groups and Parameters" echo "\`\`\`" echo "--- Replication Groups ---" _run_scan_cmd aws elasticache describe-replication-groups --region "$REGION" \ --query "ReplicationGroups[].[ReplicationGroupId,Description,Status,NodeGroups[0].NodeGroupMembers[0].CacheNodeId,CacheNodeType,SnapshotRetentionLimit,AutomaticFailover,TransitEncryptionEnabled,AtRestEncryptionEnabled]" \ --output table 2>/dev/null || echo "(no replication groups)" echo "--- User-Modified Cache Parameters ---" aws elasticache describe-cache-clusters --region "$REGION" \ --query "CacheClusters[].[CacheClusterId,CacheParameterGroup.CacheParameterGroupName]" \ --output text 2>/dev/null \ | while read -r cluster pg; do [ -z "$pg" ] && continue # Only show user-modified parameters (source=user) params=$(_run_scan_cmd aws elasticache describe-cache-parameters --region "$REGION" \ --cache-parameter-group-name "$pg" \ --query "Parameters[?Source==\`user\`].[ParameterName,ParameterValue]" \ --output text 2>/dev/null) [ -z "$params" ] && continue echo "=== $cluster → $pg ===" echo "$params" done echo "\`\`\`" echo "" ' ) & # ── MSK (Kafka): cluster details ───────────────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/MSK_Cluster_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## MSK (Kafka) Cluster Details" echo "\`\`\`" aws kafka list-clusters --region "$REGION" \ --query "ClusterInfoList[].ClusterArn" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r cluster_arn; do [ -z "$cluster_arn" ] && continue echo "=== $(basename "$cluster_arn") ===" _run_scan_cmd aws kafka describe-cluster --region "$REGION" \ --cluster-arn "$cluster_arn" \ --query "ClusterInfo.[ClusterName,State,CurrentBrokerSoftwareInfo.KafkaVersion,NumberOfBrokerNodes,BrokerNodeGroupInfo.[InstanceType,ClientSubnets,StorageInfo],EncryptionInfo,EnhancedMonitoring]" \ --output json 2>/dev/null || echo "(no access or timeout)" done echo "\`\`\`" echo "" ' ) & # ── Cognito: user pool config + app clients ────────────────────────────────── _wait_for_slot ( _out="$JOBS_DIR/Cognito_UserPool_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## Cognito User Pool Details" echo "\`\`\`" aws cognito-idp list-user-pools --region "$REGION" --max-results 60 \ --query "UserPools[].Id" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r pool_id; do [ -z "$pool_id" ] && continue echo "=== Pool: $pool_id ===" _run_scan_cmd aws cognito-idp describe-user-pool --region "$REGION" \ --user-pool-id "$pool_id" \ --query "UserPool.[Name,Status,MfaConfiguration,Policies.PasswordPolicy,LambdaConfig,SchemaAttributes[*].[Name,AttributeDataType,Required,Mutable]]" \ --output json 2>/dev/null || echo "(no access or timeout)" echo "--- App Clients ---" _run_scan_cmd aws cognito-idp list-user-pool-clients --region "$REGION" \ --user-pool-id "$pool_id" \ --query "UserPoolClients[].[ClientId,ClientName]" \ --output table 2>/dev/null || true done echo "\`\`\`" echo "" ' ) & # ── EFS: mount targets + access points per file system ─────────────────────── _wait_for_slot ( _out="$JOBS_DIR/EFS_Deep.txt" _run_section_with_timeout "$_out" ' REGION='"'$REGION'"' AWS_SCAN_CMD_TIMEOUT='"'$AWS_SCAN_CMD_TIMEOUT'"' source "'"$SCRIPT_DIR/_scan-common.sh"'" echo "## EFS Mount Targets and Access Points" echo "\`\`\`" aws efs describe-file-systems --region "$REGION" \ --query "FileSystems[].FileSystemId" --output text 2>/dev/null \ | tr "\t" "\n" | while read -r fs_id; do [ -z "$fs_id" ] && continue echo "=== FileSystem: $fs_id ===" echo "--- Mount Targets ---" _run_scan_cmd aws efs describe-mount-targets --region "$REGION" \ --file-system-id "$fs_id" \ --query "MountTargets[].[MountTargetId,SubnetId,IpAddress,LifeCycleState,AvailabilityZoneName]" \ --output table 2>/dev/null || echo "(no mount targets)" echo "--- Access Points ---" _run_scan_cmd aws efs describe-access-points --region "$REGION" \ --file-system-id "$fs_id" \ --query "AccessPoints[].[AccessPointId,Name,RootDirectory.Path,PosixUser,LifeCycleState]" \ --output table 2>/dev/null || echo "(no access points)" done echo "\`\`\`" echo "" ' ) & # ── Wait + merge ────────────────────────────────────────────────────────────── echo "Waiting for all deep scan jobs to complete..." wait echo "All deep scans done. Merging results..." _merge_jobs "$JOBS_DIR" "AWS Deep Inventory — Region: $REGION" "$OUTPUT_FILE" echo "" echo "Deep sections scanned:" grep "^## " "$OUTPUT_FILE" | sed 's/^## / - /' || true echo "" echo "Full deep inventory: $OUTPUT_FILE" echo "Note: AWS_SCAN_REDACT=${AWS_SCAN_REDACT} (set 0 for raw IDs/IPs in inventory)." _print_elapsed "$START_TIME" FILE:scripts/aws-scan-region.sh #!/bin/bash # Full-region parallel AWS resource scanner # Usage: ./aws-scan-region.sh [region] # # Runs 30+ service discovery commands in parallel (background jobs). # Typical runtime: 30-60 seconds for a full region scan. # Output: aws-scan-<region>-<timestamp>/inventory.md # # Environment: # AWS_SCAN_CMD_TIMEOUT Per-scan command timeout in seconds (default: 120). Uses GNU `timeout` when available, perl fallback on macOS. # AWS_SCAN_REDACT If 1 (default), redact IPs / instance ids in inventory.md. # AWS_SCAN_MAX_PARALLEL Max concurrent scan jobs (default: 20). # AWS_MAX_ATTEMPTS / AWS_RETRY_MODE — passed to AWS CLI to bound retries on slow APIs. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=_scan-common.sh source "$SCRIPT_DIR/_scan-common.sh" START_TIME=$(date +%s) # ── Pre-flight: AWS CLI + credentials ──────────────────────────────────────── if ! command -v aws >/dev/null 2>&1; then echo "ERROR: AWS CLI is not installed. Install it first:" >&2 echo " https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" >&2 exit 1 fi REGION="${1:-$(aws configure get region 2>/dev/null || echo 'us-east-1')}" validate_aws_region "$REGION" AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text 2>&1)" || { echo "ERROR: AWS credentials are invalid or not configured." >&2 echo " aws sts get-caller-identity returned: $AWS_ACCOUNT_ID" >&2 echo "" >&2 echo "Fix: run 'aws configure' or set AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars." >&2 echo "Do NOT proceed with mock or representative data." >&2 exit 1 } PREFIX_OWNER_ID="$AWS_ACCOUNT_ID" [[ "$PREFIX_OWNER_ID" =~ ^[0-9]{12}$ ]] || { echo "ERROR: aws sts get-caller-identity returned unexpected Account ID: '$PREFIX_OWNER_ID'" >&2 exit 1 } TIMESTAMP=$(date +%Y%m%d_%H%M%S) OUTPUT_DIR="aws-scan-${REGION}-${TIMESTAMP}" OUTPUT_FILE="${OUTPUT_DIR}/inventory.md" JOBS_DIR="${OUTPUT_DIR}/.jobs" mkdir -p "$JOBS_DIR" echo "Scanning region: $REGION" echo "Output: $OUTPUT_FILE" echo "Max parallel: $AWS_SCAN_MAX_PARALLEL" echo "" # ── helpers ─────────────────────────────────────────────────────────────────── HITS_DIR="${OUTPUT_DIR}/.hits" mkdir -p "$HITS_DIR" scan() { local name="$1" shift local safe safe="$(echo "$name" | tr ' /' '__')" local out="$JOBS_DIR/${safe}.txt" _wait_for_slot { echo "## $name" echo '```' local result if result=$(_run_scan_cmd env AWS_DEFAULT_REGION="$REGION" "$@" 2>/dev/null) && [[ -n "$result" ]]; then printf '%s\n' "$result" touch "$HITS_DIR/${safe}" else echo "(no resources, no access, or timeout)" fi echo '```' echo "" } > "$out" & } # ── Compute ─────────────────────────────────────────────────────────────────── scan "EC2 Instances" aws ec2 describe-instances --region "$REGION" \ --query 'Reservations[].Instances[].[InstanceId,InstanceType,State.Name,Tags[?Key==`Name`].Value|[0],PublicIpAddress,PrivateIpAddress]' \ --output table scan "EBS Volumes" aws ec2 describe-volumes --region "$REGION" \ --query 'Volumes[].[VolumeId,Size,VolumeType,State,Attachments[0].InstanceId]' \ --output table scan "ECS Clusters" aws ecs list-clusters --region "$REGION" --query 'clusterArns' --output table scan "EKS Clusters" aws eks list-clusters --region "$REGION" --query 'clusters' --output table scan "Elastic Beanstalk Environments" aws elasticbeanstalk describe-environments --region "$REGION" \ --query 'Environments[].[EnvironmentName,ApplicationName,Status,EnvironmentUrl]' \ --output table # ── Serverless ──────────────────────────────────────────────────────────────── scan "Lambda Functions" aws lambda list-functions --region "$REGION" \ --query 'Functions[].[FunctionName,Runtime,MemorySize,Timeout,Role]' \ --output table scan "API Gateway REST APIs" aws apigateway get-rest-apis --region "$REGION" \ --query 'items[].[id,name,createdDate]' \ --output table scan "API Gateway HTTP APIs v2" aws apigatewayv2 get-apis --region "$REGION" \ --query 'Items[].[ApiId,Name,ProtocolType]' \ --output table # ── EventBridge ─────────────────────────────────────────────────────────────── scan "EventBridge Event Buses" aws events list-event-buses --region "$REGION" \ --query 'EventBuses[].[Name,Arn]' --output table scan "EventBridge Rules (default bus)" aws events list-rules --region "$REGION" \ --query 'Rules[].[Name,State,ScheduleExpression,EventPattern]' --output table # ── Storage ─────────────────────────────────────────────────────────────────── scan "S3 Buckets (global)" aws s3api list-buckets --query 'Buckets[].[Name,CreationDate]' --output table scan "EFS File Systems" aws efs describe-file-systems --region "$REGION" \ --query 'FileSystems[].[FileSystemId,Name,LifeCycleState,SizeInBytes.Value]' \ --output table # ── Database ────────────────────────────────────────────────────────────────── scan "RDS Instances" aws rds describe-db-instances --region "$REGION" \ --query 'DBInstances[].[DBInstanceIdentifier,Engine,EngineVersion,DBInstanceClass,DBInstanceStatus]' \ --output table scan "RDS Clusters (Aurora)" aws rds describe-db-clusters --region "$REGION" \ --query 'DBClusters[].[DBClusterIdentifier,Engine,EngineVersion,Status]' \ --output table scan "ElastiCache Clusters" aws elasticache describe-cache-clusters --region "$REGION" \ --query 'CacheClusters[].[CacheClusterId,Engine,EngineVersion,CacheClusterStatus]' \ --output table scan "DynamoDB Tables" aws dynamodb list-tables --region "$REGION" --query 'TableNames' --output table scan "Redshift Clusters" aws redshift describe-clusters --region "$REGION" \ --query 'Clusters[].[ClusterIdentifier,NodeType,NumberOfNodes,ClusterStatus]' \ --output table # ── Messaging ───────────────────────────────────────────────────────────────── scan "SQS Queues" aws sqs list-queues --region "$REGION" --query 'QueueUrls' --output table scan "SNS Topics" aws sns list-topics --region "$REGION" --query 'Topics[].TopicArn' --output table scan "MSK Clusters (Kafka)" aws kafka list-clusters --region "$REGION" \ --query 'ClusterInfoList[].[ClusterName,State,NumberOfBrokerNodes]' \ --output table # ── Networking ──────────────────────────────────────────────────────────────── scan "VPCs (non-default, with IPv6)" aws ec2 describe-vpcs --region "$REGION" \ --filters Name=is-default,Values=false \ --query 'Vpcs[].[VpcId,CidrBlock,Ipv6CidrBlockAssociationSet[0].Ipv6CidrBlock,State,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "VPCs (default)" aws ec2 describe-vpcs --region "$REGION" \ --filters Name=is-default,Values=true \ --query 'Vpcs[].[VpcId,CidrBlock,State]' \ --output table scan "VPC Peering Connections" aws ec2 describe-vpc-peering-connections --region "$REGION" \ --query 'VpcPeeringConnections[].[VpcPeeringConnectionId,Status.Code,RequesterVpcInfo.VpcId,AccepterVpcInfo.VpcId,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "Subnets (non-default VPC, with IPv6)" aws ec2 describe-subnets --region "$REGION" \ --filters Name=default-for-az,Values=false \ --query 'Subnets[].[SubnetId,VpcId,CidrBlock,Ipv6CidrBlockAssociationSet[0].Ipv6CidrBlock,AvailabilityZone,MapPublicIpOnLaunch,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "Internet Gateways" aws ec2 describe-internet-gateways --region "$REGION" \ --query 'InternetGateways[].[InternetGatewayId,Attachments[0].VpcId,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "Egress-only Internet Gateways" aws ec2 describe-egress-only-internet-gateways --region "$REGION" \ --query 'EgressOnlyInternetGateways[].[EgressOnlyInternetGatewayId,Attachments[0].VpcId,Attachments[0].State]' \ --output table scan "NAT Gateways" aws ec2 describe-nat-gateways --region "$REGION" \ --query 'NatGateways[].[NatGatewayId,VpcId,State,NatGatewayAddresses[0].PublicIp,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "Route Tables (non-default, with routes)" aws ec2 describe-route-tables --region "$REGION" \ --filters Name=association.main,Values=false \ --query 'RouteTables[].[RouteTableId,VpcId,Tags[?Key==`Name`].Value|[0],Routes[*].[DestinationCidrBlock,DestinationIpv6CidrBlock,GatewayId,NatGatewayId,VpcPeeringConnectionId,State]]' \ --output json scan "Security Groups (with rules)" aws ec2 describe-security-groups --region "$REGION" \ --query 'SecurityGroups[?GroupName!=`default`].[GroupId,GroupName,VpcId,IpPermissions[*].[IpProtocol,FromPort,ToPort,IpRanges[*].CidrIp,UserIdGroupPairs[*].GroupId],IpPermissionsEgress[*].[IpProtocol,FromPort,ToPort,IpRanges[*].CidrIp,UserIdGroupPairs[*].GroupId]]' \ --output json scan "Network ACLs (custom only, with rules)" aws ec2 describe-network-acls --region "$REGION" \ --filters Name=default,Values=false \ --query 'NetworkAcls[].[NetworkAclId,VpcId,Tags[?Key==`Name`].Value|[0],Entries[*].[RuleNumber,Protocol,RuleAction,Egress,CidrBlock,Ipv6CidrBlock,PortRange.From,PortRange.To]]' \ --output json scan "VPC Endpoints" aws ec2 describe-vpc-endpoints --region "$REGION" \ --query 'VpcEndpoints[].[VpcEndpointId,VpcEndpointType,ServiceName,VpcId,State,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "Managed Prefix Lists (customer-owned)" aws ec2 describe-managed-prefix-lists --region "$REGION" \ --filters "Name=owner-id,Values=$PREFIX_OWNER_ID" \ --query 'PrefixLists[].[PrefixListId,PrefixListName,AddressFamily,MaxEntries]' \ --output table scan "DHCP Option Sets (with content)" aws ec2 describe-dhcp-options --region "$REGION" \ --query 'DhcpOptions[].[DhcpOptionsId,Tags[?Key==`Name`].Value|[0],DhcpConfigurations[*].[Key,Values[*].Value]]' \ --output json scan "Elastic Load Balancers (v2)" aws elbv2 describe-load-balancers --region "$REGION" \ --query 'LoadBalancers[].[LoadBalancerName,Type,State.Code,DNSName]' \ --output table scan "Elastic Load Balancers (classic)" aws elb describe-load-balancers --region "$REGION" \ --query 'LoadBalancerDescriptions[].[LoadBalancerName,DNSName,VPCId]' \ --output table scan "Route53 Hosted Zones (global)" aws route53 list-hosted-zones \ --query 'HostedZones[].[Name,Id,Config.PrivateZone,ResourceRecordSetCount]' \ --output table scan "CloudFront Distributions (global)" aws cloudfront list-distributions \ --query 'DistributionList.Items[].[Id,DomainName,Status]' \ --output table # ── IAM (global) ────────────────────────────────────────────────────────────── scan "IAM Users" aws iam list-users --query 'Users[].[UserName,CreateDate]' --output table scan "IAM Roles" aws iam list-roles --no-paginate \ --query 'Roles[].[RoleName,Arn]' --output table # ── Monitoring / Other ──────────────────────────────────────────────────────── scan "CloudWatch Alarms" aws cloudwatch describe-alarms --region "$REGION" \ --query 'MetricAlarms[].[AlarmName,StateValue,MetricName,Namespace]' \ --output table scan "Secrets Manager Secrets" aws secretsmanager list-secrets --region "$REGION" \ --query 'SecretList[].[Name,LastChangedDate,RotationEnabled]' \ --output table scan "Systems Manager Parameters" aws ssm describe-parameters --region "$REGION" \ --query 'Parameters[].[Name,Type,LastModifiedDate]' \ --output table scan "Step Functions State Machines" aws stepfunctions list-state-machines --region "$REGION" \ --query 'stateMachines[].[name,stateMachineArn,type]' \ --output table scan "Cognito User Pools" aws cognito-idp list-user-pools --region "$REGION" --max-results 60 \ --query 'UserPools[].[Id,Name,Status]' --output table scan "ACM Certificates" aws acm list-certificates --region "$REGION" \ --query 'CertificateSummaryList[].[CertificateArn,DomainName,Status]' \ --output table scan "CloudTrail Trails" aws cloudtrail describe-trails --region "$REGION" --include-shadow-trails false \ --query 'trailList[].[Name,S3BucketName,CloudWatchLogsLogGroupArn,IsMultiRegionTrail,LogFileValidationEnabled]' \ --output table scan "CloudWatch Log Groups" aws logs describe-log-groups --region "$REGION" \ --query 'logGroups[].[logGroupName,retentionInDays,storedBytes]' \ --output table scan "CloudWatch Metric Filters" aws logs describe-metric-filters --region "$REGION" \ --query 'metricFilters[].[filterName,logGroupName,filterPattern,metricTransformations[0].metricName,metricTransformations[0].metricNamespace]' \ --output table # ── Container ──────────────────────────────────────────────────────────────── scan "ECR Repositories" aws ecr describe-repositories --region "$REGION" \ --query 'repositories[].[repositoryName,repositoryUri,imageTagMutability,imageScanningConfiguration.scanOnPush]' \ --output table # ── Streaming ──────────────────────────────────────────────────────────────── scan "Kinesis Data Streams" aws kinesis list-streams --region "$REGION" \ --query 'StreamNames' --output table # ── Security (WAF) ────────────────────────────────────────────────────────── scan "WAF Web ACLs (v2)" aws wafv2 list-web-acls --region "$REGION" --scope REGIONAL \ --query 'WebACLs[].[Name,Id,ARN]' --output table # ── Hybrid Networking ──────────────────────────────────────────────────────── scan "VPN Connections" aws ec2 describe-vpn-connections --region "$REGION" \ --query 'VpnConnections[].[VpnConnectionId,State,CustomerGatewayId,VpnGatewayId,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "VPN Gateways" aws ec2 describe-vpn-gateways --region "$REGION" \ --query 'VpnGateways[].[VpnGatewayId,State,Type,VpcAttachments[0].VpcId,Tags[?Key==`Name`].Value|[0]]' \ --output table scan "Direct Connect Connections" aws directconnect describe-connections --region "$REGION" \ --query 'connections[].[connectionId,connectionName,connectionState,bandwidth,location]' \ --output table scan "Direct Connect Virtual Interfaces" aws directconnect describe-virtual-interfaces --region "$REGION" \ --query 'virtualInterfaces[].[virtualInterfaceId,virtualInterfaceName,virtualInterfaceType,virtualInterfaceState,vlan,connectionId]' \ --output table # ── Big Data ───────────────────────────────────────────────────────────────── scan "EMR Clusters" aws emr list-clusters --region "$REGION" --active \ --query 'Clusters[].[Id,Name,Status.State,NormalizedInstanceHours]' \ --output table # ── Wait for all background jobs ────────────────────────────────────────────── echo "Waiting for all scan jobs to complete..." wait echo "All scans done. Merging results..." # ── Merge output (sorted by section name, empty sections filtered) ──────────── _merge_jobs "$JOBS_DIR" "AWS Resource Inventory — Region: $REGION" "$OUTPUT_FILE" # ── Post-scan validation ───────────────────────────────────────────────────── TOTAL_SECTIONS=$(find "$JOBS_DIR" -maxdepth 1 -name '*.txt' | wc -l | tr -d ' ') HIT_COUNT=$(find "$HITS_DIR" -maxdepth 1 -type f | wc -l | tr -d ' ') rm -rf "$HITS_DIR" # ── Summary ─────────────────────────────────────────────────────────────────── echo "" echo "Resource sections scanned: $TOTAL_SECTIONS total, $HIT_COUNT with data" grep "^## " "$OUTPUT_FILE" | sed 's/^## / - /' || true if [[ "$HIT_COUNT" -eq 0 ]]; then echo "" >&2 echo "WARNING: Every section returned empty — this almost certainly means" >&2 echo " the IAM user/role lacks permissions for all services in $REGION." >&2 echo " Verify: aws sts get-caller-identity && aws ec2 describe-instances --region $REGION" >&2 echo " Do NOT treat this as 'no resources exist'." >&2 # Exit non-zero so the agent cannot silently continue echo "" echo "Full inventory: $OUTPUT_FILE" _print_elapsed "$START_TIME" exit 2 fi echo "" echo "Full inventory: $OUTPUT_FILE" echo "Raw job outputs: $JOBS_DIR/" echo "Note: AWS_SCAN_REDACT=${AWS_SCAN_REDACT} (set 0 for raw IDs/IPs in inventory)." _print_elapsed "$START_TIME" echo "Next: ./scripts/aws-scan-enrich.sh $REGION \"$(pwd)/$OUTPUT_DIR\" → per-resource deep scan → inventory-deep.md" FILE:scripts/terraform_runtime_online.sh #!/usr/bin/env bash set -euo pipefail # terraform_runtime_online.sh - Execute Terraform via Alibaba Cloud IaCService # # Usage: # terraform_runtime_online.sh validate <hcl_file_or_code> # terraform_runtime_online.sh plan <hcl_file_or_code> [existing_state_id] # terraform_runtime_online.sh apply <hcl_file_or_code> # fresh apply (first time) # terraform_runtime_online.sh apply <hcl_file_or_code> --state-id <id> # retry after FAILURE only (not for in-place updates) # terraform_runtime_online.sh apply --state-id <id> # apply a previously planned state (after plan) # terraform_runtime_online.sh destroy <state_id> # terraform_runtime_online.sh poll <state_id> [max_attempts] [interval_seconds] # # ⚠️ STATE_ID REUSE RULE: # - plan → apply: do NOT pass the plan stateId to apply (IaCService locks plan states) # - Once a STATE_ID exists (from a previous apply), ALL subsequent changes to the same # deployment MUST reuse it via --state-id, including: # • Retrying after a failed/partial apply # • Adding new resources to main.tf # • Modifying existing resource configuration # Only the very first apply of a brand-new deployment runs without --state-id. # A fresh apply without --state-id creates a NEW state and causes duplicate resources. # # Online runtime — changing existing infra (e.g. rename): # 1) plan <main.tf> <STATE_ID> # STATE_ID from last successful apply # 2) apply --state-id <STATE_ID> # use the STATE_ID printed by plan (materialize plan) # Do NOT use: apply main.tf --state-id ... after status is Applied — API returns InvalidOperation.JobStatus. ENDPOINT="iac.cn-zhangjiakou.aliyuncs.com" IAC_USER_AGENT="AlibabaCloud-Agent-Skills" SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" # ClientToken (IaC OpenAPI): retry after 5xx with the SAME token; after 2xx success or a 4xx # failure, the next API invocation must use a NEW token. (TaskStatus lock retry = new token.) # TF_LOG_REDACT=1 (default): mask IPs and long resource ids in stderr JSON snippets # Set TF_LOG_REDACT=0 to print raw API responses (avoid in shared logs). # --------------------------------------------------------------------------- # Safety: IaC state id format (defense-in-depth for --state-id injection) # --------------------------------------------------------------------------- _validate_state_id() { local s="${1:-}" if [[ ! "$s" =~ ^[A-Za-z0-9_-]+$ ]] || [[ ${#s} -lt 8 ]] || [[ ${#s} -gt 128 ]]; then echo "$(_red)Error: invalid state-id format$(_reset)" >&2 exit 1 fi } # --------------------------------------------------------------------------- # Optional redaction for multi-line strings printed to stderr # --------------------------------------------------------------------------- _redact_multiline() { if [[ "${TF_LOG_REDACT:-1}" != "1" ]]; then cat return 0 fi python3 -c ' import re, sys t = sys.stdin.read() t = re.sub(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", "[REDACTED-IP]", t) t = re.sub(r"\b(?:[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12})\b", "[REDACTED-UUID]", t, flags=re.I) t = re.sub(r"\b\d{12}\b", "[REDACTED-ACCOUNT]", t) sys.stdout.write(t) ' 2>/dev/null || cat } # --------------------------------------------------------------------------- # Color helpers (degrade gracefully if tput unavailable) # --------------------------------------------------------------------------- _green() { command -v tput &>/dev/null && tput setaf 2; } _red() { command -v tput &>/dev/null && tput setaf 1; } _yellow() { command -v tput &>/dev/null && tput setaf 3; } _reset() { command -v tput &>/dev/null && tput sgr0; } # --------------------------------------------------------------------------- # Shared helper: resolve file-or-inline input to CODE variable # --------------------------------------------------------------------------- _read_input() { local input="$1" if [[ -f "$input" ]]; then cat "$input" else printf '%s' "$input" fi } _iac_new_client_token() { uuidgen; } # First StatusCode: NNN in aliyun CLI / SDK error text; empty if unknown _iac_http_status_from_output() { printf '%s' "${1:-}" | python3 -c " import re, sys t = sys.stdin.read() m = re.search(r'StatusCode:\s*(\d{3})\b', t) print(m.group(1) if m else '') " 2>/dev/null || true } _iac_is_5xx() { [[ "${1:-}" =~ ^5[0-9]{2}$ ]]; } # stdin = execute-* API body (NDJSON lines and/or single JSON object, pretty or minified) # $1 = optional fallback when stateId missing (e.g. destroy passes original state_id) _iac_extract_state_id() { FALLBACK_ID="${1:-}" python3 -c " import json, os, sys raw = sys.stdin.read() fb = os.environ.get('FALLBACK_ID', '') or '' sid = '' for line in raw.splitlines(): line = line.strip() if not line: continue try: d = json.loads(line) inner = d.get('data', d) s = inner.get('stateId') if s: sid = str(s) except Exception: pass if not sid: try: d = json.loads(raw) inner = d.get('data', d) s = inner.get('stateId', '') if s: sid = str(s) except Exception: pass if not sid: i = raw.find('{') if i >= 0: try: d = json.loads(raw[i:]) inner = d.get('data', d) s = inner.get('stateId', '') if s: sid = str(s) except Exception: pass print(sid or fb) " 2>/dev/null || true } # --------------------------------------------------------------------------- # cmd: validate # --------------------------------------------------------------------------- cmd_validate() { if [[ $# -lt 1 || "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then echo "Usage: $0 validate <hcl_file_or_code>" echo "Exit 0 = Validated, 1 = Errored" exit 1 fi local input="$1" local code code=$(_read_input "$input") echo "Validating: $input" >&2 local token response status message ec http attempt token="$(_iac_new_client_token)" attempt=0 ec=1 while [[ $attempt -lt 10 ]]; do attempt=$((attempt + 1)) ec=0 response=$(aliyun iacservice validate-module \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --source Upload \ --code "$code" 2>&1) || ec=$? if [[ $ec -eq 0 ]]; then break fi http=$(_iac_http_status_from_output "$response") if _iac_is_5xx "$http"; then echo "$(_yellow)[Retry $attempt/10] validate-module HTTP $http — same ClientToken$(_reset)" >&2 sleep $((attempt * 2)) continue fi echo "$(_red)Error: validate-module failed$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 done if [[ $ec -ne 0 ]]; then echo "$(_red)Error: validate-module failed after HTTP 5xx retries$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 fi # validate-module may stream multiple JSON lines; status lives under data.* status=$(echo "$response" | python3 -c " import sys,json st='Unknown' for line in sys.stdin.read().splitlines(): line=line.strip() if not line: continue try: d=json.loads(line) inner=d.get('data',d) s=inner.get('status') if s: st=str(s) except Exception: pass print(st) " 2>/dev/null) || status="Unknown" message=$(echo "$response" | python3 -c " import sys,json msg='' for line in sys.stdin.read().splitlines(): line=line.strip() if not line: continue try: d=json.loads(line) inner=d.get('data',d) m=inner.get('message') if m is not None: msg=str(m) except Exception: pass print(msg) " 2>/dev/null) || message="" echo "" >&2 if [[ "$status" == "Validated" ]]; then echo "$(_green)Validated$(_reset)" >&2 [[ -n "$message" ]] && echo "$message" >&2 exit 0 else echo "$(_red)Validation failed: $status$(_reset)" >&2 [[ -n "$message" ]] && echo "$message" >&2 exit 1 fi } # --------------------------------------------------------------------------- # cmd: poll # --------------------------------------------------------------------------- cmd_poll() { if [[ $# -lt 1 || "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then echo "Usage: $0 poll <state_id> [max_attempts] [interval_seconds]" echo "Exit 0 = terminal state reached, 1 = timeout or Errored" exit 1 fi local state_id="$1" _validate_state_id "$state_id" local max="${2:-60}" local interval="${3:-10}" local terminal_states=("Planned" "PlannedAndFinished" "Applied" "Errored" "Canceled" "Discarded") _is_terminal() { local s="$1" for t in "${terminal_states[@]}"; do [[ "$s" == "$t" ]] && return 0; done return 1 } local attempt=0 response status while [[ $attempt -lt $max ]]; do attempt=$((attempt + 1)) response=$(aliyun iacservice get-execute-state --endpoint "$ENDPOINT" --user-agent "$IAC_USER_AGENT" --state-id "$state_id" 2>&1) || true status=$(echo "$response" | python3 -c " import sys,json try: print(json.load(sys.stdin).get('status','Unknown')) except: print('Unknown') " 2>/dev/null) || status="Unknown" if [[ "$status" == "Errored" ]]; then echo "[$attempt/$max] $(_red)Status: $status$(_reset)" >&2 elif _is_terminal "$status"; then echo "[$attempt/$max] $(_green)Status: $status$(_reset)" >&2 else echo "[$attempt/$max] $(_yellow)Status: $status$(_reset)" >&2 fi if _is_terminal "$status"; then if [[ "$status" == "Errored" ]]; then local errmsg errmsg=$(echo "$response" | python3 -c " import sys,json try: print(json.load(sys.stdin).get('errorMessage','Unknown error')) except: print('Unknown error') " 2>/dev/null) || errmsg="Unknown error" echo "$(_red)Error: $errmsg$(_reset)" >&2 exit 1 fi exit 0 fi [[ $attempt -lt $max ]] && sleep "$interval" done echo "$(_red)Timeout: $max attempts reached$(_reset)" >&2 exit 1 } # --------------------------------------------------------------------------- # cmd: plan # --------------------------------------------------------------------------- cmd_plan() { if [[ $# -lt 1 || "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then echo "Usage: $0 plan <hcl_file_or_code> [existing_state_id]" echo "Output: STATE_ID=<id> PLAN_OUTPUT_FILE=<path>" echo "Exit 0 = Planned/PlannedAndFinished, 1 = Errored" exit 1 fi local input="$1" local state_id="${2:-}" local code token response new_state_id attempt ec http code=$(_read_input "$input") echo "Planning: $input" >&2 [[ -n "$state_id" ]] && echo "Using existing state: $state_id" >&2 token="$(_iac_new_client_token)" attempt=0 ec=1 response="" [[ -n "$state_id" ]] && _validate_state_id "$state_id" while [[ $attempt -lt 10 ]]; do attempt=$((attempt + 1)) ec=0 if [[ -n "$state_id" ]]; then response=$( aliyun iacservice execute-terraform-plan \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --code "$code" \ --state-id "$state_id" 2>&1 ) || ec=$? else response=$( aliyun iacservice execute-terraform-plan \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --code "$code" 2>&1 ) || ec=$? fi if [[ $ec -eq 0 ]]; then break fi http=$(_iac_http_status_from_output "$response") if _iac_is_5xx "$http"; then echo "$(_yellow)[Retry $attempt/10] execute-terraform-plan HTTP $http — same ClientToken$(_reset)" >&2 sleep $((attempt * 2)) continue fi echo "$(_red)Error: execute-terraform-plan failed$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 done if [[ $ec -ne 0 ]]; then echo "$(_red)Error: execute-terraform-plan failed after HTTP 5xx retries$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 fi new_state_id=$(printf '%s' "$response" | _iac_extract_state_id) [[ -z "$new_state_id" ]] && { echo "$(_red)Error: no stateId in response$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 } echo "STATE_ID=$new_state_id" echo "" >&2; echo "Plan started. stateId: $new_state_id" >&2; echo "Polling..." >&2; echo "" >&2 "$SELF" poll "$new_state_id" || { echo "$(_red)Plan failed$(_reset)" >&2; exit 1; } local final_response final_status error_message final_response=$(aliyun iacservice get-execute-state --endpoint "$ENDPOINT" --user-agent "$IAC_USER_AGENT" --state-id "$new_state_id" 2>&1) || true final_status=$(echo "$final_response" | python3 -c " import sys,json try: print(json.load(sys.stdin).get('status','Unknown')) except: print('Unknown') " 2>/dev/null) || final_status="Unknown" echo "" >&2 if [[ "$final_status" == "Planned" || "$final_status" == "PlannedAndFinished" ]]; then echo "$(_green)Plan completed: $final_status$(_reset)" >&2 local plan_file="/tmp/tf_plan_${new_state_id}.txt" local plan_summary plan_summary=$(echo "$final_response" | python3 -c " import sys,json,re plan_file=sys.argv[1] try: data=json.loads(sys.stdin.read(),strict=False) lf=data.get('logFile',{}) log=lf.get('tf-plan.run.log','') if isinstance(lf,dict) else (lf if isinstance(lf,str) else '') if not log: print(' No plan details available'); sys.exit(0) clean=re.sub(r'\x1b\[[0-9;]*[a-zA-Z]','',log) open(plan_file,'w').write(clean) lines=clean.split('\n') summary=[l for l in lines if ('# ' in l and ('will be' in l or 'must be' in l)) or l.strip().startswith('Plan:') or 'No changes' in l] for s in (summary or [' (see full output for details)']): print(' '+s.strip()) except Exception as e: print(f' Could not parse: {e}') " "$plan_file" 2>/dev/null) || plan_summary=" Could not parse plan details" echo "" >&2 echo "=== Plan Summary ===" >&2 echo "$plan_summary" >&2 [[ -f "$plan_file" ]] && { echo "" >&2; echo "Full output: cat $plan_file" >&2; } echo "====================" >&2 echo "PLAN_OUTPUT_FILE=$plan_file" exit 0 elif [[ "$final_status" == "Errored" ]]; then error_message=$(echo "$final_response" | python3 -c " import sys,json try: m=json.load(sys.stdin).get('errorMessage',''); m and print(m) except: pass " 2>/dev/null) || error_message="" echo "$(_red)Plan failed: $final_status$(_reset)" >&2 [[ -n "$error_message" ]] && echo "$(_red)Error: $error_message$(_reset)" >&2 exit 1 else echo "$(_yellow)Plan status: $final_status$(_reset)" >&2 exit 0 fi } # --------------------------------------------------------------------------- # cmd: apply # --------------------------------------------------------------------------- cmd_apply() { if [[ $# -lt 1 || "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then echo "Usage:" echo " $0 apply <hcl_file_or_code> # first apply of a brand-new deployment" echo " $0 apply <hcl_file_or_code> --state-id <id> # any subsequent change to an existing deployment" echo " $0 apply --state-id <id> # apply a previously planned state" echo "" echo " STATE_ID REUSE RULE:" echo " Once a STATE_ID exists, ALL subsequent operations on the same deployment MUST" echo " pass --state-id, including: retry after failure, add resources, modify config." echo " Starting a fresh apply (without --state-id) creates a NEW state and causes" echo " duplicate resource creation." echo "" echo "Output: STATE_ID=<id>" echo "Exit 0 = Applied, 1 = Errored" exit 1 fi local input="" state_id="" code="" while [[ $# -gt 0 ]]; do case "$1" in --state-id) state_id="${2:-}"; shift 2 ;; --help|-h) cmd_apply --help ;; *) input="$1"; shift ;; esac done [[ -z "$input" && -z "$state_id" ]] && { echo "$(_red)Error: provide HCL code/file or --state-id$(_reset)" >&2; exit 1; } [[ -n "$input" ]] && { code=$(_read_input "$input"); echo "Applying: $input" >&2; } [[ -n "$state_id" ]] && echo "Using existing state: $state_id" >&2 local token response new_state_id local max_retries=6 retry_delay=10 retry=0 local r5=0 token="$(_iac_new_client_token)" _invoke_apply() { if [[ -n "$code" && -n "$state_id" ]]; then _validate_state_id "$state_id" aliyun iacservice execute-terraform-apply \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --code "$code" \ --state-id "$state_id" elif [[ -n "$code" ]]; then aliyun iacservice execute-terraform-apply \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --code "$code" elif [[ -n "$state_id" ]]; then _validate_state_id "$state_id" aliyun iacservice execute-terraform-apply \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --state-id "$state_id" else return 1 fi } _invoke_apply_capture() { _invoke_apply 2>&1; } while true; do response=$(_invoke_apply_capture) && break http=$(_iac_http_status_from_output "$response") if _iac_is_5xx "$http"; then r5=$((r5 + 1)) if [[ $r5 -ge 10 ]]; then echo "$(_red)Error: execute-terraform-apply failed after HTTP 5xx retries (same ClientToken)$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 fi echo "$(_yellow)[Retry $r5/10] execute-terraform-apply HTTP $http — same ClientToken$(_reset)" >&2 sleep $((r5 * 3)) continue fi if echo "$response" | grep -q "InvalidOperation.TaskStatus"; then retry=$((retry + 1)) if [[ $retry -ge $max_retries ]]; then echo "$(_red)Error: state lock not released after $max_retries retries$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 fi echo "$(_yellow)[Retry $retry/$max_retries] State lock (HTTP 4xx) — new ClientToken, waiting ${retry_delay}s...$(_reset)" >&2 sleep "$retry_delay" token="$(_iac_new_client_token)" continue else echo "$(_red)Error: execute-terraform-apply failed$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 fi done new_state_id=$(printf '%s' "$response" | _iac_extract_state_id) [[ -z "$new_state_id" ]] && { echo "$(_red)Error: no stateId in response$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 } echo "STATE_ID=$new_state_id" echo "" >&2; echo "Apply started. stateId: $new_state_id" >&2; echo "Polling..." >&2; echo "" >&2 "$SELF" poll "$new_state_id" || { echo "$(_red)Apply failed$(_reset)" >&2; exit 1; } local final_response final_status error_message final_response=$(aliyun iacservice get-execute-state --endpoint "$ENDPOINT" --user-agent "$IAC_USER_AGENT" --state-id "$new_state_id" 2>&1) || true final_status=$(echo "$final_response" | python3 -c " import sys,json try: print(json.load(sys.stdin).get('status','Unknown')) except: print('Unknown') " 2>/dev/null) || final_status="Unknown" echo "" >&2 if [[ "$final_status" == "Applied" ]]; then echo "$(_green)Apply completed: $final_status$(_reset)" >&2 echo "" >&2; echo "Resources:" >&2 echo "$final_response" | python3 -c " import sys,json try: data=json.load(sys.stdin) s=data.get('state','') state=json.loads(s) if isinstance(s,str) else s resources=state.get('resources',[]) if state else [] if resources: for r in resources: for i in r.get('instances',[]): print(f' {r[\"type\"]}.{r[\"name\"]}: {i.get(\"attributes\",{}).get(\"id\",\"N/A\")}') else: print(' No resources found') except Exception as e: print(f' Could not parse resources: {e}') " 2>/dev/null || echo " Could not parse resources" >&2 exit 0 elif [[ "$final_status" == "Errored" ]]; then error_message=$(echo "$final_response" | python3 -c " import sys,json try: m=json.load(sys.stdin).get('errorMessage',''); m and print(m) except: pass " 2>/dev/null) || error_message="" echo "$(_red)Apply failed: $final_status$(_reset)" >&2 [[ -n "$error_message" ]] && echo "$(_red)Error: $error_message$(_reset)" >&2 exit 1 else echo "$(_yellow)Apply status: $final_status$(_reset)" >&2; exit 0 fi } # --------------------------------------------------------------------------- # cmd: destroy # --------------------------------------------------------------------------- cmd_destroy() { if [[ $# -lt 1 || "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then echo "Usage: $0 destroy <state_id>" echo "Exit 0 = Destroyed, 1 = Failed" exit 1 fi local state_id="$1" [[ -z "$state_id" ]] && { echo "$(_red)Error: state_id required$(_reset)" >&2; exit 1; } _validate_state_id "$state_id" echo "Destroying resources for state: $state_id" >&2 local token response destroy_state_id attempt ec http token="$(_iac_new_client_token)" attempt=0 ec=1 response="" while [[ $attempt -lt 10 ]]; do attempt=$((attempt + 1)) ec=0 response=$(aliyun iacservice execute-terraform-destroy \ --endpoint "$ENDPOINT" \ --user-agent "$IAC_USER_AGENT" \ --client-token "$token" \ --state-id "$state_id" 2>&1) || ec=$? if [[ $ec -eq 0 ]]; then break fi http=$(_iac_http_status_from_output "$response") if _iac_is_5xx "$http"; then echo "$(_yellow)[Retry $attempt/10] execute-terraform-destroy HTTP $http — same ClientToken$(_reset)" >&2 sleep $((attempt * 2)) continue fi echo "$(_red)Error: execute-terraform-destroy failed$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 done if [[ $ec -ne 0 ]]; then echo "$(_red)Error: execute-terraform-destroy failed after HTTP 5xx retries$(_reset)" >&2 echo "$response" | _redact_multiline >&2 exit 1 fi destroy_state_id=$(printf '%s' "$response" | _iac_extract_state_id "$state_id") echo "Polling..." >&2; echo "" >&2 "$SELF" poll "$destroy_state_id" || { echo "$(_red)Destroy failed$(_reset)" >&2; exit 1; } local final_response final_status final_response=$(aliyun iacservice get-execute-state --endpoint "$ENDPOINT" --user-agent "$IAC_USER_AGENT" --state-id "$destroy_state_id" 2>&1) || true final_status=$(echo "$final_response" | python3 -c " import sys,json try: print(json.load(sys.stdin).get('status','Unknown')) except: print('Unknown') " 2>/dev/null) || final_status="Unknown" echo "" >&2 if [[ "$final_status" == "Applied" || "$final_status" == "Canceled" || "$final_status" == "Discarded" ]]; then echo "$(_green)Destroy completed: $final_status$(_reset)" >&2; exit 0 elif [[ "$final_status" == "Errored" ]]; then local errmsg errmsg=$(echo "$final_response" | python3 -c " import sys,json try: m=json.load(sys.stdin).get('errorMessage',''); m and print(m) except: pass " 2>/dev/null) || errmsg="" echo "$(_red)Destroy failed: $final_status$(_reset)" >&2 [[ -n "$errmsg" ]] && echo "$(_red)Error: $errmsg$(_reset)" >&2 exit 1 else echo "$(_yellow)Destroy status: $final_status$(_reset)" >&2; exit 0 fi } # --------------------------------------------------------------------------- # Dispatch # --------------------------------------------------------------------------- usage() { echo "Usage: $0 <command> [args]" echo "" echo "Commands:" echo " validate <hcl_file_or_code> Validate HCL syntax" echo " plan <hcl_file_or_code> [state_id] Preview changes" echo " apply <hcl_file_or_code> [--state-id id] Create/update infrastructure" echo " apply --state-id <id> Apply planned state" echo " destroy <state_id> Destroy resources" echo " poll <state_id> [max] [interval] Poll execution status" echo "" echo "Run '$0 <command> --help' for per-command usage." exit 1 } COMMAND="${1:-}" shift || true case "$COMMAND" in validate) cmd_validate "$@" ;; plan) cmd_plan "$@" ;; apply) cmd_apply "$@" ;; destroy) cmd_destroy "$@" ;; poll) cmd_poll "$@" ;; *) usage ;; esac </article></body></html>

--- name: alibabacloud-migrate description: | Assess and migrate workloads from AWS to Alibaba Cloud. Follows a 4-phase methodology: Phase 1 (source architecture assessment), Phase 2 (migration plan generation), Phase 3 (infrastructure deployment as empty shells via Terraform), Phase 4 (data and application migration + cutover). Covers EC2→ECS (AMI export + ImportImage), RDS→ApsaraDB (DTS), S3→OSS, Lambda→Function Compute FCv3, and network/DNS migration. Uses Terraform (default) for infrastructure provisioning and Alibaba Cloud CLI for migration-specific operations (DTS, data transfer). Triggers: "migrate from AWS", "AWS to Alibaba Cloud", "migrate EC2 to ECS", "migrate S3 to OSS", "migrate RDS to ApsaraDB", "migrate Lambda to Function Compute", "migrate Lambda to FCv3", "cloud migration assessment", "cross-cloud migration", "AWS migration", "ImportImage", "database migration DTS". --- # AWS to Alibaba Cloud Migration > This skill handles **assessment, planning, and execution** of workload migration from AWS to Alibaba Cloud. > **Default approach**: Terraform for infrastructure provisioning; CLI for migration-specific operations (DTS, data transfer, ImportImage). ## Architecture ``` AWS (Source) Alibaba Cloud (Target) Tool ──────────── ───────────────────── ──── VPC / Networking ──→ VPC / VSwitch / SG Terraform EC2 Instances ──→ ECS CLI (AMI export→OSS→ImportImage) + Terraform RDS / Aurora ──→ ApsaraDB RDS / PolarDB CLI (DTS full+incremental) + Terraform S3 Buckets ──→ OSS CLI (ossutil/DataOnline) + Terraform Lambda ──→ Function Compute (FC 3.0) Terraform + code refactor API Gateway ──→ API Gateway Terraform (OpenAPI import) EventBridge ──→ EventBridge Terraform (rule+target) Step Functions ──→ Serverless Workflow Flow definition rewrite SQS / SNS ──→ MNS Queue / Topic Code changes (SDK) ECS (Container) / EKS ─→ ACK (Kubernetes) Velero + Terraform ECR ──→ ACR docker pull/push DynamoDB ──→ Tablestore DTS/DataX + schema rewrite ElastiCache ──→ Tair (Redis/Memcached) CLI (DTS) + Terraform MSK (Kafka) ──→ Message Queue for Kafka Broker config migration Redshift ──→ MaxCompute / AnalyticDB DataX + SQL adaptation Route53 ──→ Alibaba Cloud DNS Terraform (zone+records) CloudFront ──→ CDN Terraform (origin+SSL) ELB / ALB / NLB ──→ SLB / ALB / NLB Terraform VPN Gateway ──→ VPN Gateway Terraform (IPsec) Direct Connect ──→ Express Connect Physical circuit setup IAM ──→ RAM Policy syntax rewrite Cognito ──→ IDaaS User pool migration WAF ──→ WAF Terraform (rule migration) CloudWatch ──→ CloudMonitor + SLS Metric/log remapping ``` ## Installation **Step 1 — Aliyun CLI** (requires >= 3.3.1; see [references/cli-installation-guide.md](references/cli-installation-guide.md)) **Step 2 — Terraform Runtime Detection** — Run once, record result as `TERRAFORM_MODE`: ```bash terraform version 2>/dev/null && echo "TERRAFORM_MODE=local" || echo "TERRAFORM_MODE=online" ``` - **`TERRAFORM_MODE=local`**: Use `terraform` CLI directly for all operations. - **`TERRAFORM_MODE=online`**: Prompt user to install Terraform (`brew install terraform` / https://developer.hashicorp.com/terraform/install), or use `terraform_runtime_online.sh` as fallback. See `## Terraform Online Runtime`. **Step 3 — Set Terraform User-Agent** — Required for all Terraform operations (both local and online modes): ```hcl provider "alicloud" { region = var.region configuration_source = "AlibabaCloud-Agent-Skills/alibabacloud-migrate" } ``` ## Authentication > **CRITICAL Security Rules — NEVER violate:** > - NEVER read, echo, or print AK/SK values > - NEVER ask for AK/SK in conversation or command line > - NEVER use `aliyun configure set` with literal credential values ### AWS Credentials (Phase 1 prerequisite) ```bash aws sts get-caller-identity # Must return AccountId before starting Phase 1. ``` > **[STOP — AWS Source Access Required for Phase 1]** Before running any scan, confirm one of the following is available: > 1. **AWS credentials configured** — `aws sts get-caller-identity` returns a valid `AccountId`. If not, ask the user to configure credentials, then re-run the check. > 2. **Complete AWS resource inventory provided manually** — User supplies a full description of all resources (VPC, subnets, SGs, EC2, RDS, S3, Lambda, API Gateway, EventBridge, IAM roles, etc.) that substitutes the scan output. > > **Do NOT start Phase 1 scans or produce any output until one of the above conditions is confirmed.** ### Alibaba Cloud Credentials (Phase 3 prerequisite) ```bash aliyun configure list # Must show a valid profile (AK, STS, or OAuth). STOP if none found. ``` > **[STOP — Alibaba Cloud Credentials Required for Phase 3]** If no valid profile: > 1. Inform the user that Alibaba Cloud credentials are required before infrastructure deployment. > 2. Suggest: obtain credentials from [Alibaba Cloud Console](https://ram.console.aliyun.com/manage/ak), configure via `aliyun configure` in a terminal **outside this session**, then return. > 3. **Do NOT proceed to Phase 3 (Terraform apply) without verified credentials.** Phase 1-2 (assessment and planning) can proceed with AWS credentials only. ## CLI operation idempotency (DTS, ImportImage, SMC, etc.) Terraform state and `STATE_ID` (online runtime) prevent duplicate *Terraform-managed* resources, but **Alibaba creation APIs** for DTS tasks, image import, SMC jobs, and similar may still create **new** records if invoked repeatedly with different client tokens or names. **Agent / operator rules:** 1. **Describe / list before create:** Query existing tasks, jobs, or imports by name prefix or tags; if a matching **Succeeded** or **Running** resource exists, reuse its ID instead of creating another. 2. **Stable identifiers:** Prefer `ClientToken` / unique job names derived from a deterministic key (e.g., source server id + target region), documented in the migration runbook. 3. **Retries:** On timeout or ambiguous error, **do not** blindly re-run “create”; poll status first, then either wait, cancel, or resume per product documentation. 4. **User confirmation:** Before second creation of the same logical migration object, surface the duplicate risk and require explicit approval. ## Interaction Mode > **[MANDATORY FIRST STEP — STOP]** Present the mode choice below and **wait for user response** before ANY other action. Do NOT read files, run scans, or begin assessment until the user has chosen a mode. This is a blocking gate. Present this choice and wait for response: ``` Before starting the migration, choose the agent work mode: A) Interactive mode (recommended) Agent asks before every significant decision. Full control over every migration detail. Best for: first-time migrations, complex environments, learning the process. B) Autonomous mode Agent handles all low-risk decisions automatically (naming, sizing, transfer path, etc.). Only confirms at mandatory checkpoints: Phase 1/2 architecture review, DNS cutover, source decommission, instance type > ecs.g6.2xlarge. Each checkpoint presents [Recommended] option with reasoning; user can accept or override. Best for: experienced users, repeat migrations with known patterns. Choose mode (A/B, default A — press Enter to confirm A): ``` **Interactive mode — agent confirms before acting:** All significant decisions are presented to the user with context and a [Recommended] option. The user can accept, override, or ask for more information before proceeding. This includes: - Resource naming, VPC CIDR, VSwitch layout - Disk type, image format, instance type sizing - Data transfer path selection - Phase checkpoints (same as autonomous mode) **Autonomous mode defaults** (all shown in Phase 2 summary; user can override): naming `-` · VPC `10.0.0.0/16` · disk `cloud_essd` · image `VHD` · instance type closest match · transfer path Option B (HK relay ECS) **Autonomous mode — always confirm** (present [Recommended] + reasoning): Phase 1 architecture review · Phase 2 plan review · DNS cutover (4.6) · source decommission (4.8) · instance type > ecs.g6.2xlarge ## Parameter Confirmation > Parameters are confirmed at the **end of Phase 2** — autonomous mode as a single summary block; interactive mode one-by-one. See [references/parameter-reference.md](references/parameter-reference.md) for the full parameter table with agent defaults. ## Output Isolation > All generated migration files MUST go in `-alicloud/` directory (created in working directory). NEVER modify user's existing source files. If no project name provided, ask before creating the directory. ## Destructive Action Policy > **NEVER** delete, stop, or modify AWS source resources until ALL of the following are met: > 1. Target resources verified working — see [references/verification-method.md](references/verification-method.md) > 2. Data integrity confirmed (checksums, row counts, object counts match) > 3. DNS cutover complete and traffic flowing to Alibaba Cloud > 4. 24-hour observation period passed > 5. User explicitly approves decommissioning (Phase 4.8) ## Migration Status Tracking > **MANDATORY**: Create `migration-status.md` from [references/migration-status-template.md](references/migration-status-template.md) at the start of Phase 1. Update after every operation: ⬜ Not Started → 🔄 In Progress → ✅ Completed / ❌ Failed. Record STATE_IDs and error details. All items in current phase must show ✅ before advancing to the next phase. ## RAM Permissions See [references/ram-policies.md](references/ram-policies.md) for the complete per-service permission list and the recommended custom least-privilege policy. > **[MUST]** On any permission error: read `references/ram-policies.md`, use `ram-permission-diagnose` skill, and wait for user to confirm permissions are granted before continuing. ## Terraform Online Runtime (TERRAFORM_MODE=online only) > Skip this section when `TERRAFORM_MODE=local`. Full usage guide: [references/terraform-online-runtime.md](references/terraform-online-runtime.md) **Rules (online mode only):** - Consolidate all HCL into a **single** `main.tf` (IaCService requirement) - **STATE_ID** is the deployment identity. Only the **first** apply of a new deployment runs **without** `--state-id`. After a run reaches **Applied**, **in-place updates** must use **plan → apply(plan)** (IaCService rejects `apply main.tf --state-id` on an already-Applied job). ```bash export TF="$SKILL_DIR/scripts/terraform_runtime_online.sh" apply_output=$($TF apply main.tf) STATE_ID=$(echo "$apply_output" | grep '^STATE_ID=' | cut -d= -f2) echo "STATE_ID=$STATE_ID" >> terraform_state_ids.env # In-place change after Applied: plan then apply the plan state (same ID typically echoed twice) plan_output=$($TF plan main.tf "$STATE_ID") PLAN_SID=$(echo "$plan_output" | grep '^STATE_ID=' | cut -d= -f2) $TF apply --state-id "$PLAN_SID" # Destroy: $TF destroy "$STATE_ID" ``` **Multi-STATE_ID (production):** Use separate HCL files per resource layer (`network-main.tf`, `compute-main.tf`, etc.) and apply each independently for independent lifecycle management. ## Core Workflow Four phases, executed **strictly sequentially**. Each phase ends with a **[CHECKPOINT — STOP]** gate. These are hard barriers — the agent MUST stop, present the checkpoint output, and wait for the user to explicitly confirm before any Phase N+1 action. Merging phases, skipping checkpoints, or proceeding without user confirmation is a critical violation. ``` Phase 1: Source Assessment → Phase 2: Migration Plan → Phase 3: Infra Deploy → Phase 4: Data Migration [STOP: Confirm AWS state] [STOP: Confirm plan] [STOP: Confirm infra] [STOP: DNS cutover] ``` > **Anti-pattern — NEVER do this:** Combine multiple phases in a single response, generate Terraform code before Phase 1 checkpoint is confirmed, or execute `terraform apply` before Phase 2 plan is approved. --- ### Phase 1: Source Architecture Assessment **Goal**: Fully understand the AWS current state. Identify migration complexity and risks. No design or planning in this phase. 1. **Discover AWS resources** — Two steps, **both mandatory**, per [references/aws-discovery-commands.md](references/aws-discovery-commands.md): - **Step 1 broad scan** (~30–60s): `scripts/aws-scan-region.sh ` — one list/describe per service, outputs `inventory.md` - **Step 2 deep scan** (must run after Step 1): `scripts/aws-scan-enrich.sh

` — loops over each main resource for per-resource detail, outputs `inventory-deep.md` > **[PAUSE — AWS access unavailable: ask user before proceeding]** > > **Trigger conditions** (any one is sufficient): > - AWS CLI is not installed or not found on PATH > - Scan scripts return credential errors (`NoCredentialProviders`, `InvalidClientTokenId`, `ExpiredTokenException`, `AuthFailure`, etc.) > - API connectivity failures (network unreachable, timeout on all services) > - All service categories in `inventory.md` show "(no resources, no access, or timeout)" > > **Required action — pause the task, present the two options below, and wait for the user's reply before doing anything else:** > > > "AWS access is not available (AWS CLI missing / credentials not configured / API unreachable). I cannot proceed without real resource data. Please choose: > > > > **Option A** — Configure AWS credentials and re-run the scan scripts: > > `aws configure` (or set `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_DEFAULT_REGION`), then re-run `aws-scan-region.sh` and `aws-scan-enrich.sh`. > > > > **Option B** — Provide a complete AWS resource inventory manually (VPC, subnets, SGs, EC2, RDS, S3, Lambda, API Gateway, EventBridge, IAM roles, etc.) and I will continue from there. > > > > Which option would you like to proceed with?" > > **Unconditional prohibitions — no exceptions, no rationalisations:** > - **Do NOT fabricate** any AWS resource, architecture, topology, or configuration — not even as "representative", "typical", "simulated", "example", or "for evaluation purposes". > - **Do NOT create any output files** (`inventory.md`, `inventory-deep.md`, `migration-assessment-report.md`, `main.tf`, `migration-status.md`, runbooks, etc.) based on invented data. > - **Do NOT advance to Phase 2, 3, or 4** under any circumstances until real resource data is confirmed. > - Being a test, evaluation, or demo environment is **not** a valid reason to bypass this rule. > **[STOP — Both scans required]** Do NOT proceed to Phase 1 step 2 (dependency mapping) until **both** `inventory.md` and `inventory-deep.md` have been generated and read. If Step 1 returns resources but Step 2 is skipped, the assessment will miss critical migration details (triggers, policies, listener configs, DNS records, etc.). If the user cannot run the scripts, they must manually provide equivalent information for each resource category below. **[MUST]** The deep scan (Step 2) reveals details invisible in broad scan — do not skip: - **Lambda** `get-policy` per function (push triggers NOT returned by `list-event-source-mappings`) - **API Gateway** REST `get-resources`+`get-stages`; HTTP v2 routes+stages - **Step Functions** state machine definitions · **EventBridge** `list-targets-by-rule` on every bus - **SNS** per-topic subscriptions · **SQS** per-queue attributes (DLQ, encryption, FIFO) - **S3** per-bucket lifecycle+policy · **DynamoDB** per-table capacity/GSI/streams · **EFS** mount targets+access points - **ECS** per-cluster services · **EKS** per-cluster details+addons · **ElastiCache** replication groups+parameters - **ELB v2** per-LB listeners+target groups · **Route53** per-zone record sets · **CloudFront** per-distribution config - **RDS** subnet groups+parameter groups · **IAM** inline policies · **MSK** broker config · **Cognito** user pool config **[MUST]** VPC topology (broad scan — verify completeness): subnets, route tables, SG rules, NACLs, endpoints, peering, prefix lists, VPN, Direct Connect 2. **Map dependencies and data flows** — Service call chains, data origins/destinations, external integrations, migration ordering constraints. 3. **Risk and complexity assessment** — Rate each dimension High/Medium/Low: data volume, downtime tolerance (RTO/RPO), compliance requirements, custom AMI/OS compatibility, Lambda→FCv3 code refactor scope, security group complexity. 4. **Generate assessment report (Phase 1 sections)** — Create `migration-assessment-report.md` using [references/assessment-report-template.md](references/assessment-report-template.md). Complete: Executive Summary, Resource Inventory, Integration & Dependency Mapping, Risk Assessment. Remaining sections completed in Phase 2. 5. **Initialize status tracker** — Create `migration-status.md` from [references/migration-status-template.md](references/migration-status-template.md). Populate all discovered resources, set all statuses to ⬜. 6. **[CHECKPOINT — Phase 1 end]** Output the following three items **directly in the conversation** (do not write to a file): - **Structured resource inventory** — one table per category, one row per discovered resource. Do not omit any category that has resources. Format: ``` AWS Resource Discovery Results ## 1. VPC & Networking ┌──────────────────┬──────────────────────────┬──────────────────────────────────────────────┐ │ Resource Type │ Resource ID │ Details │ ├──────────────────┼──────────────────────────┼──────────────────────────────────────────────┤ │ VPC │ vpc-xxxxxxxxxxxxxxxxx │ CIDR x.x.x.x/x, state available, default/custom │ │ Subnet │ subnet-xxxxxxxxxxxxxxx │ AZ, CIDR, public/private │ │ Security Group │ sg-xxxxxxxxxxxxxxxxx │ inbound rules summary / outbound rules summary │ │ NAT Gateway │ nat-xxxxxxxxxxxxxxxxx │ public IP, associated subnet │ │ Route Table │ rtb-xxxxxxxxxxxxxxxxx │ key route entries │ └──────────────────┴──────────────────────────┴──────────────────────────────────────────────┘ ## 2. Compute ┌──────────────────┬──────────────────────────┬──────────────────────────────────────────────┐ │ EC2 Instance │ i-xxxxxxxxxxxxxxxxx │ type, OS, state, private IP │ │ Lambda Function │ │ runtime, memory, timeout, triggers (deep scan) │ └──────────────────┴──────────────────────────┴──────────────────────────────────────────────┘ ## 3. Storage ## 4. Database ## 5. ... (only include categories with discovered resources) ``` - **Architecture topology** (ASCII, not Mermaid): ``` ┌──────────────────── AWS ─────────────────────────┐ │ Region: VPC: │ │ EC2: ──▶ RDS: │ │ S3: buckets Lambda: SQS: │ │ Route53: → CloudFront → EC2 │ └──────────────────────────────────────────────────┘ ``` - **Risk assessment**: | Risk Dimension | Rating | Notes | |----------------|--------|-------| | Data volume | H/M/L | | | Downtime tolerance (RTO/RPO) | H/M/L | | | Code refactor scope | H/M/L | | | Compliance requirements | H/M/L | | > **Is the above AWS current state and risk assessment accurate? Confirm to proceed to Phase 2.** > **[DO NOT proceed to Phase 2 until the user explicitly confirms]** --- ### Phase 2: Migration Plan Generation **Goal**: Map AWS services to Alibaba Cloud equivalents, design target architecture, confirm all parameters, and produce a complete migration plan. 1. **Service mapping** — Use [references/service-mapping.md](references/service-mapping.md). For missing mappings follow the "Adding a New Mapping" procedure in that file. Record the planned Alibaba Cloud resource type for every AWS resource — these names will be validated against [references/terraform-providers/alicloud.md](references/terraform-providers/alicloud.md) at the Phase 3 pre-HCL gate (existence, deprecation, usage example). 2. **Target architecture design** — Region (proximity to source), VPC/VSwitch CIDRs, ECS/RDS sizing, OSS bucket policies, FCv3 RAM role + trigger mapping, CDN config, CloudWatch→Cloud Monitor mapping, IAM→RAM mapping. 3. **Data migration strategy** — Confirm tool per resource type and S3→OSS transfer path: | Option | Method | Speed | Est. time (2.5 GB) | |--------|--------|-------|-------------------| | A | Local relay | ~1–2 MB/s | ~30–45 min | | **B ← Recommended** | **HK relay ECS** | **~50–100 MB/s** | **~1–3 min, <¥1** | | C | Alibaba Cloud Online Migration | ~20–50 MB/s | ~3–8 min | Autonomous: default to B, notify user. Interactive: present table and wait for choice. 4. **Downtime window planning** — Maintenance windows, AMI export timing, DTS cutover window after sync lag stabilizes. 5. **Cost estimation** — Monthly cost for all target resources (ECS + RDS + OSS + FCv3) and data transfer costs during migration; compare to AWS baseline. 6. **Rollback plan** — DNS rollback (keep AWS endpoints running), DTS bidirectional sync (database fallback), Terraform destroy per STATE_ID (per-layer infrastructure rollback). 7. **Complete assessment report (Phase 2 sections)** — Add to `migration-assessment-report.md`: Service Mapping, Network Topology (target state), IAM & Security Mapping, Monitoring Mapping, Data Migration Strategy, Cost Estimation, Migration Plan, Rollback Plan, Next Steps + sign-off. 8. **[CHECKPOINT — Phase 2 end]** Render target architecture + migration plan + parameter summary **directly in the conversation**. Wait for explicit confirmation before Phase 3. ``` ┌──────────────── Alibaba Cloud ───────────────────┐ │ Region: VPC: │ │ ECS: (via ImportImage) ──▶ ApsaraDB │ │ OSS: buckets FCv3: functions │ │ Alibaba Cloud DNS: → CDN → ECS │ └──────────────────────────────────────────────────┘ ``` | Phase | Content | Tool | Est. Time | |-------|---------|------|-----------| | ✅ 1 | Source assessment | — | Done | | ✅ 2 | Migration plan | — | Done | | ⬜ 3 | Infrastructure deploy (empty shells) | Terraform | ~10–20 min | | ⬜ 4 | Data migration + cutover | CLI + DTS + Terraform | ~varies | Autonomous mode: include parameter summary (see [references/parameter-reference.md](references/parameter-reference.md) for full defaults table). > **Is the above target architecture and plan accurate? Confirm to proceed to Phase 3.** > **[DO NOT proceed to Phase 3 until the user explicitly confirms]** --- ### Phase 3: Infrastructure Deployment **Goal**: Deploy all target resources on Alibaba Cloud as empty shells. No data migration in this phase. Verify all infrastructure is ready before Phase 4. > **[MUST — Pre-HCL gate]** Before writing **any** Terraform resource block, look up **every** planned resource type in [references/terraform-providers/alicloud.md](references/terraform-providers/alicloud.md) and confirm **all three** of the following: > > 1. **Exists** — the resource name is listed in `alicloud.md` (do not invent or guess resource names). > 2. **Not deprecated** — the entry has no ⚠️ deprecated marker. If deprecated, switch to the replacement resource name listed there (e.g., use `alicloud_fcv3_function` not `alicloud_fc_function`). > 3. **Usage example reviewed** — read the example block in `alicloud.md` for each resource to confirm required arguments, argument names, and value formats before writing HCL. > > **Violation of this gate** (writing HCL without checking) is a critical error — deprecated resources cause silent apply failures and the resulting infrastructure cannot be used for migration. **Step 1 — Network layer first (required before all other layers):** ```bash $TF apply network-main.tf # → NETWORK_STATE_ID echo "NETWORK_STATE_ID=$NETWORK_STATE_ID" >> terraform_state_ids.env ``` HCL template: [references/migration-guides/network-migration.md](references/migration-guides/network-migration.md) **Step 2 — Remaining layers in parallel (after network is ready):** ```bash $TF apply compute-main.tf # → COMPUTE_STATE_ID (ECS instance — no data; image imported in Phase 4) $TF apply database-main.tf # → DATABASE_STATE_ID (ApsaraDB RDS — empty DB; data migrated via DTS in Phase 4) $TF apply storage-main.tf # → STORAGE_STATE_ID (OSS bucket — empty; data transferred in Phase 4) $TF apply serverless-main.tf # → SERVERLESS_STATE_ID (FCv3 RAM Role + empty function; code deployed in Phase 4) ``` HCL templates: [server](references/migration-guides/server-migration-importimage.md) · [database](references/migration-guides/database-migration-dts.md) · [storage](references/migration-guides/storage-migration-oss.md) · [serverless](references/migration-guides/serverless-migration-fc.md) > **[MUST] FCv3 RAM Role**: Create a RAM Role trusting `fc.aliyuncs.com` with least-privilege policies for services the function accesses (OSS, SLS, etc.). Set ARN as `role` on `alicloud_fcv3_function` — without this, the function cannot authenticate to other Alibaba Cloud services at runtime. **[CHECKPOINT — Phase 3 end]** Verify all infra is ready: VPC/SG applied, ECS defined, RDS connectable (empty), OSS accessible, FCv3 role + function ready. All items in `migration-status.md` show ✅. Present resource list and estimated ongoing cost. > **Is all target infrastructure ready? Confirm to proceed to Phase 4.** > **[DO NOT proceed to Phase 4 until the user explicitly confirms]** --- ### Phase 4: Application and Data Migration **Goal**: Migrate actual data and applications to Phase 3 infrastructure, complete cutover, and verify. Execute in dependency order; update `migration-status.md` after each sub-step. **4.1 Server migration (AMI export + ImportImage)** Export EC2 AMI → S3 → transfer to OSS → ImportImage → attach to Phase 3 ECS instance and start. No agent installation required on source server. See [references/migration-guides/server-migration-importimage.md](references/migration-guides/server-migration-importimage.md). **4.2 Database migration (DTS full + incremental sync)** Start DTS job against Phase 3 ApsaraDB instance: full migration + continuous incremental sync. Cut over during maintenance window once sync lag falls below acceptable threshold. See [references/migration-guides/database-migration-dts.md](references/migration-guides/database-migration-dts.md). **4.3 Storage migration (S3 → OSS)** Transfer all S3 objects to Phase 3 OSS bucket using the transfer path confirmed in Phase 2. Verify object count and total size match after transfer. See [references/migration-guides/storage-migration-oss.md](references/migration-guides/storage-migration-oss.md). **4.4 Serverless deployment (Lambda → FCv3)** Export Lambda code, adapt handler signatures, deploy to Phase 3 FCv3 functions. Configure triggers (API GW / EventBridge / SQS→MNS). Terraform resources: `alicloud_fcv3_function`, `alicloud_fcv3_trigger`. See [references/migration-guides/serverless-migration-fc.md](references/migration-guides/serverless-migration-fc.md). **4.5 Pre-cutover validation** Before DNS cutover, verify all workloads on Alibaba Cloud (see [references/verification-method.md](references/verification-method.md)): - [ ] ECS: application running from imported image - [ ] RDS: row counts match, DTS sync lag < threshold - [ ] OSS: object count matches source S3 (checksum spot-check) - [ ] FCv3: functions invocable, output correct - [ ] End-to-end functional and performance tests pass **4.6 [MUST CONFIRM] DNS cutover** Migrate Route53 → Alibaba Cloud DNS; CloudFront → Alibaba Cloud CDN (Terraform). See [references/migration-guides/network-migration.md](references/migration-guides/network-migration.md). > **This will immediately affect live traffic. Confirm DNS cutover?** > Lower TTL to 60s beforehand. Keep AWS endpoints running for instant DNS rollback. **4.7 Post-migration observation (minimum 24 hours)** Monitor: traffic routing, error rates, response times, DB write operations. Stop DTS sync after confirming data consistency. **4.8 [MUST CONFIRM] Source resource decommission** Decommission AWS resources only after observation period and user confirms. **Irreversible.** > **Observation period complete. Confirm decommission of AWS source resources?** > Retain RDS snapshots and EC2 AMI for at least 30 days as final backup. ## Success Verification See [references/verification-method.md](references/verification-method.md) for detailed steps. - [ ] VPC/VSwitch/SG created (Terraform state = `Applied`) - [ ] ECS image imported (`DescribeImages` status = `Available`) - [ ] ECS instance running from imported image - [ ] RDS accessible; DTS sync lag < threshold - [ ] OSS bucket: object count and size match source - [ ] FCv3 functions deployed and invocable - [ ] DNS resolving correctly; CDN active ## Cleanup ```bash # Destroy Terraform-managed infra (load STATE_IDs from terraform_state_ids.env) $TF destroy "$NETWORK_STATE_ID" $TF destroy "$COMPUTE_STATE_ID" $TF destroy "$DATABASE_STATE_ID" $TF destroy "$STORAGE_STATE_ID" $TF destroy "$SERVERLESS_STATE_ID" # Delete imported ECS image and OSS staging file (not Terraform-managed) aliyun ecs DeleteImage --RegionId --ImageId --Force true --user-agent AlibabaCloud-Agent-Skills ossutil rm oss:///migrated-image.vhd # Release DTS job (not Terraform-managed) aliyun dts DeleteMigrationJob --MigrationJobId --user-agent AlibabaCloud-Agent-Skills ``` > **WARNING**: Only clean up migration-related intermediate resources. Do NOT destroy target ECS/RDS/OSS unless decommissioning. ## Best Practices 1. **Never add `terraform {}` / `required_providers` block** — Provider is pre-initialized; this block causes `Failed to load plugin schemas`. Start HCL with `provider "alicloud" { region = var.region; configuration_source = "AlibabaCloud-Agent-Skills/alibabacloud-migrate" }` directly. See `references/acceptance-criteria.md §4`. 2. **Save and reuse every STATE_ID** — Record immediately after every apply. Always pass `--state-id` for any subsequent operation on existing infrastructure. 3. **Single `main.tf` (online mode only)** — Required by IaCService. Local mode can split HCL files freely. 4. **Consult error remediation first** — Check [references/error-remediation.md](references/error-remediation.md) before ad-hoc debugging. ## References All references are linked inline throughout this document. Key directories: - `references/migration-guides/` — Per-service migration workflows (server, database, storage, serverless, network) - `references/terraform-providers/` — Alicloud and AWS Terraform resource catalogs - `references/source-mappings/` — Raw source mapping documents FILE:references/acceptance-criteria.md # Acceptance Criteria: alibabacloud-migrate **Scenario**: AWS to Alibaba Cloud Migration **Purpose**: Skill testing acceptance criteria for CLI commands and SDK usage patterns --- # Correct CLI Command Patterns ## 1. SMC (Server Migration Center) ### Product Name - ✅ CORRECT: `aliyun smc` - ❌ INCORRECT: `aliyun SMC`, `aliyun server-migration-center`, `aliyun smc create-replication-job` ### Commands SMC uses **RPC-style API** (PascalCase API names, not plugin mode lowercase). - ✅ CORRECT: `aliyun smc CreateReplicationJob --RegionId cn-hangzhou --SourceId s-xxx --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun smc DescribeReplicationJobs --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun smc StartReplicationJob --RegionId cn-hangzhou --JobId j-xxx --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun smc create-replication-job` (wrong style - SMC uses RPC, not plugin mode) - ❌ INCORRECT: Missing `--user-agent AlibabaCloud-Agent-Skills` flag ### Parameters - ✅ CORRECT: `--RegionId cn-hangzhou` (PascalCase for RPC APIs) - ❌ INCORRECT: `--region-id cn-hangzhou` (wrong case for RPC style) - ✅ CORRECT: `--SourceId s-xxx` (valid source server ID format) - ❌ INCORRECT: `--SourceServerId s-xxx` (wrong parameter name) - ✅ CORRECT: `--TargetType Image` (valid enum: Image, ContainerImage, TargetInstance) - ❌ INCORRECT: `--TargetType ECS` (invalid enum value) - ✅ CORRECT: `--ImageName my-migrated-image` (custom image name) - ❌ INCORRECT: `--ImageId` in CreateReplicationJob (ImageId is output, not input) ## 2. DTS (Data Transmission Service) ### Product Name - ✅ CORRECT: `aliyun dts` - ❌ INCORRECT: `aliyun DTS`, `aliyun data-transmission` ### Commands DTS uses **RPC-style API** (PascalCase API names). - ✅ CORRECT: `aliyun dts CreateMigrationJob --Region cn-hangzhou --MigrationJobClass medium --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun dts ConfigureMigrationJob --MigrationJobId dts-xxx --MigrationJobName my-migration --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun dts DescribeMigrationJobs --Region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun dts StartMigrationJob --MigrationJobId dts-xxx --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun dts create-migration-job` (wrong style - DTS uses RPC, not plugin mode) - ❌ INCORRECT: `aliyun dts CreateMigrationJob --RegionId` (DTS uses `--Region` not `--RegionId` in CreateMigrationJob) ### Parameters - ✅ CORRECT: `--Region cn-hangzhou` (DTS CreateMigrationJob uses `Region` not `RegionId`) - ❌ INCORRECT: `--region cn-hangzhou` (wrong case) - ✅ CORRECT: `--MigrationJobClass medium` (valid: small, medium, large, xlarge, 2xlarge) - ❌ INCORRECT: `--MigrationJobClass Medium` (case-sensitive, must be lowercase) - ✅ CORRECT: `--SourceEndpoint.InstanceType other` (for external databases like AWS RDS) - ❌ INCORRECT: `--SourceEndpoint.InstanceType AWS` (invalid value) - ✅ CORRECT: `--SourceEndpoint.InstanceType MySQL` (for MySQL databases) - ✅ CORRECT: `--MigrationMode.StructureIntialization true` (note: "Intialization" not "Initialization" - official API typo) - ❌ INCORRECT: `--MigrationMode.StructureInitialization true` (wrong spelling - API has typo) - ✅ CORRECT: `--MigrationMode.DataIntialization true` (same typo pattern) - ✅ CORRECT: `--MigrationMode.DataSynchronization true` - ❌ INCORRECT: `--MigrationJobId` in CreateMigrationJob (JobId is returned, not input for creation) ## 3. VPC (Virtual Private Cloud) ### Product Name - ✅ CORRECT: `aliyun vpc` - ❌ INCORRECT: `aliyun VPC`, `aliyun virtual-private-cloud` ### Commands VPC uses **RPC-style API** (PascalCase API names). - ✅ CORRECT: `aliyun vpc CreateVpc --RegionId cn-hangzhou --CidrBlock 10.0.0.0/8 --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun vpc CreateVSwitch --RegionId cn-hangzhou --VpcId vpc-xxx --ZoneId cn-hangzhou-i --CidrBlock 10.0.0.0/24 --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun vpc DescribeVpcs --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun vpc DescribeVSwitches --RegionId cn-hangzhou --VpcId vpc-xxx --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun vpc create-vpc` (wrong style - VPC uses RPC, not plugin mode) - ❌ INCORRECT: `aliyun vpc create-vswitch` (wrong style) ### Parameters - ✅ CORRECT: `--RegionId cn-hangzhou` (PascalCase) - ❌ INCORRECT: `--region-id cn-hangzhou` (wrong case) - ✅ CORRECT: `--CidrBlock 10.0.0.0/8` (valid VPC CIDR: /8 to /28) - ✅ CORRECT: `--CidrBlock 172.16.0.0/16` (valid private CIDR) - ✅ CORRECT: `--CidrBlock 192.168.0.0/16` (valid private CIDR) - ❌ INCORRECT: `--CidrBlock 100.64.0.0/10` (reserved CIDR, not allowed) - ❌ INCORRECT: `--CidrBlock 224.0.0.0/4` (multicast range, not allowed) - ✅ CORRECT: `--VpcId vpc-xxx` (valid VPC ID format) - ✅ CORRECT: `--ZoneId cn-hangzhou-i` (valid zone ID format) - ❌ INCORRECT: `--ZoneId hangzhou-i` (missing region prefix) - ✅ CORRECT: `--VSwitchName my-vswitch` (optional descriptive name) - ✅ CORRECT: `--VpcName my-vpc` (optional descriptive name) ## 4. ECS (Elastic Compute Service) ### Product Name - ✅ CORRECT: `aliyun ecs` - ❌ INCORRECT: `aliyun ECS`, `aliyun elastic-compute-service` ### Commands ECS uses **RPC-style API** (PascalCase API names). - ✅ CORRECT: `aliyun ecs RunInstances --RegionId cn-hangzhou --ImageId m-xxx --InstanceType ecs.g6.large --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun ecs CreateSecurityGroup --RegionId cn-hangzhou --VpcId vpc-xxx --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun ecs DescribeInstances --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun ecs DescribeSecurityGroups --RegionId cn-hangzhou --VpcId vpc-xxx --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun ecs DescribeImages --RegionId cn-hangzhou --ImageId m-xxx --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun ecs run-instances` (wrong style - ECS uses RPC, not plugin mode) - ❌ INCORRECT: `aliyun ecs create-security-group` (wrong style) ### Parameters - ✅ CORRECT: `--RegionId cn-hangzhou` (PascalCase) - ❌ INCORRECT: `--region-id cn-hangzhou` (wrong case) - ✅ CORRECT: `--ImageId m-xxx` (valid image ID format) - ❌ INCORRECT: `--Image m-xxx` (wrong parameter name) - ✅ CORRECT: `--InstanceType ecs.g6.large` (valid instance type format) - ❌ INCORRECT: `--InstanceType g6.large` (missing ecs. prefix) - ✅ CORRECT: `--SecurityGroupId sg-xxx` (valid security group ID) - ✅ CORRECT: `--VpcId vpc-xxx` (valid VPC ID) - ✅ CORRECT: `--VSwitchId vsw-xxx` (valid vSwitch ID) - ✅ CORRECT: `--Amount 1` (number of instances, 1-100) - ❌ INCORRECT: `--Count 1` (wrong parameter name) - ✅ CORRECT: `--InstanceName my-ecs-instance` (optional descriptive name) - ✅ CORRECT: `--InternetChargeType PayByTraffic` (valid: PayByBandwidth, PayByTraffic) - ✅ CORRECT: `--InternetMaxBandwidthOut 5` (bandwidth in Mbps, 0-100) - ✅ CORRECT: `--SystemDisk.Category cloud_essd` (valid: cloud, cloud_efficiency, cloud_ssd, cloud_essd) - ❌ INCORRECT: `--SystemDisk.Category ssd` (wrong value format) - ✅ CORRECT: `--SecurityGroupIds.1 sg-xxx` (for multiple security groups) ## 5. OSS (Object Storage Service) ### Product Name - ✅ CORRECT: `aliyun oss` (ossutil-style subcommands) - ❌ INCORRECT: `aliyun OSS`, `aliyun object-storage-service` ### Commands OSS uses **ossutil-style subcommands** (lowercase, not RPC style). - ✅ CORRECT: `aliyun oss mb oss://bucket-name --region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun oss ls oss://bucket-name --region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun oss cp localfile.txt oss://bucket-name/key --region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun oss rm oss://bucket-name/key --region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun oss stat oss://bucket-name/key --region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun oss CreateBucket` (OSS uses ossutil subcommands, not RPC style) - ❌ INCORRECT: `aliyun oss PutObject` (wrong style) - ❌ INCORRECT: `aliyun oss MB oss://bucket` (subcommands must be lowercase) ### Parameters - ✅ CORRECT: `--region cn-hangzhou` (lowercase for ossutil) - ❌ INCORRECT: `--RegionId cn-hangzhou` (wrong parameter name for OSS) - ✅ CORRECT: `oss://bucket-name/key` (valid OSS URI format) - ❌ INCORRECT: `oss://bucket-name` without key for cp/rm operations (needs full path) - ✅ CORRECT: `--recursive` (for recursive operations on directories) - ✅ CORRECT: `--force` (for force delete operations) ## 6. FC (Function Compute 3.0) ### Product Name - ✅ CORRECT: `aliyun fc` (Function Compute 3.0 API) - ❌ INCORRECT: `aliyun fc-open` (deprecated), `aliyun FC` ### Commands FC 3.0 uses **ROA-style API** (RESTful path patterns with HTTP methods). - ✅ CORRECT: `aliyun fc create-function --function-name my-func --runtime nodejs20 --handler index.handler --code zipFile=base64encoded== --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun fc delete-function --function-name my-func --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun fc get-function --function-name my-func --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun fc create-trigger --function-name my-func --trigger-name my-trigger --trigger-type oss --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun fc CreateFunction` (FC 3.0 uses plugin mode lowercase, not RPC style) - ❌ INCORRECT: `aliyun fc POST /2023-03-30/functions` (FC CLI uses subcommands, not raw HTTP) ### Parameters - ✅ CORRECT: `--function-name my-func` (lowercase with hyphens) - ❌ INCORRECT: `--FunctionName my-func` (wrong case) - ❌ INCORRECT: `--functionName my-func` (wrong case) - ✅ CORRECT: `--runtime nodejs20` (valid: nodejs8/10/12/14/16/18/20, python3/3.9/3.10, java8/11, go1, php7.2, dotnetcore3.1, custom) - ❌ INCORRECT: `--runtime Node.js20` (wrong format) - ✅ CORRECT: `--handler index.handler` (language-specific handler format) - ✅ CORRECT: `--code zipFile=base64encoded==` (code as base64) - ✅ CORRECT: `--code ossBucketName=my-bucket,ossObjectName=code.zip` (code from OSS) - ❌ INCORRECT: `--Code` (wrong case) - ✅ CORRECT: `--memory-size 128` (in MB, multiples of 64) - ✅ CORRECT: `--timeout 60` (in seconds, 1-600) - ✅ CORRECT: `--trigger-type oss` (valid: oss, timer, http, log, cdn_events) - ❌ INCORRECT: `--triggerType oss` (wrong case) ## 7. Alibaba Cloud DNS (alidns) ### Product Name - ✅ CORRECT: `aliyun alidns` - ❌ INCORRECT: `aliyun dns`, `aliyun DNS`, `aliyun cloud-dns` ### Commands Alidns uses **RPC-style API** (PascalCase API names). - ✅ CORRECT: `aliyun alidns AddDomainRecord --DomainName example.com --RR www --Type A --Value 1.2.3.4 --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun alidns DescribeDomainRecords --DomainName example.com --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun alidns AddDomain --DomainName example.com --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `aliyun alidns DescribeDomains --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun alidns add-domain-record` (wrong style - alidns uses RPC, not plugin mode) - ❌ INCORRECT: `aliyun dns AddDomainRecord` (product code is alidns, not dns) ### Parameters - ✅ CORRECT: `--DomainName example.com` (full domain name) - ❌ INCORRECT: `--Domain example.com` (wrong parameter name) - ✅ CORRECT: `--RR www` (hostname part, e.g., www, @, mail) - ✅ CORRECT: `--RR @` (for root domain) - ❌ INCORRECT: `--RR` empty (cannot be empty) - ✅ CORRECT: `--Type A` (valid: A, AAAA, CNAME, MX, TXT, NS, SRV, CAA, PTR) - ❌ INCORRECT: `--TYPE A` (wrong case) - ❌ INCORRECT: `--Type a` (type values are uppercase) - ✅ CORRECT: `--Value 1.2.3.4` (record value, format depends on type) - ❌ INCORRECT: `--value 1.2.3.4` (wrong case) - ✅ CORRECT: `--TTL 600` (in seconds, default 600) - ✅ CORRECT: `--Priority 10` (required for MX records, 1-50) - ✅ CORRECT: `--Line default` (resolution line: default, telecom, unicom, mobile, etc.) --- # Global Rules ## User-Agent Flag - ✅ CORRECT: Every `aliyun` command includes `--user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: Any `aliyun` command missing `--user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT: `--user-agent AlibabaCloud-Agent-Skills` (exact string, case-sensitive) - ❌ INCORRECT: `--user-agent alibabacloud-agent-skills` (wrong case) - ❌ INCORRECT: `--UserAgent AlibabaCloud-Agent-Skills` (wrong parameter name) ## Credential Safety - ✅ CORRECT: `aliyun configure list` (check credential status only) - ❌ INCORRECT: `echo $ALIBABA_CLOUD_ACCESS_KEY_ID` (never print credentials) - ❌ INCORRECT: `echo $ALIBABA_CLOUD_ACCESS_KEY_SECRET` (never print credentials) - ❌ INCORRECT: `aliyun configure set --access-key-id LTAI5tXXX` (never hardcode AK in skill) - ❌ INCORRECT: `aliyun configure set --access-key-secret xxx` (never hardcode SK in skill) - ✅ CORRECT: Prompt user to configure credentials outside the session - ✅ CORRECT: Check credential status and stop if invalid, directing user to configure ## Parameter Placeholders - ✅ CORRECT: `--RegionId ` (placeholder indicates user must provide) - ✅ CORRECT: `--VpcId ` (placeholder with description) - ✅ CORRECT: `--ImageId ` (placeholder with context) - ❌ INCORRECT: `--RegionId cn-hangzhou` hardcoded without user confirmation prompt - ❌ INCORRECT: `--VpcId vpc-12345` hardcoded example without clear placeholder notation - ✅ CORRECT: Include parameter confirmation instruction before execution - ❌ INCORRECT: Assume default values for RegionId, instance names, CIDR blocks, passwords ## API Style Recognition ### RPC-Style APIs (PascalCase API names, --ParameterName format) - ✅ CORRECT for SMC: `aliyun smc CreateReplicationJob --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT for DTS: `aliyun dts CreateMigrationJob --Region cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT for VPC: `aliyun vpc CreateVpc --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT for ECS: `aliyun ecs RunInstances --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT for Alidns: `aliyun alidns AddDomainRecord --DomainName example.com --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT: `aliyun smc create-replication-job` (plugin mode lowercase wrong for RPC) - ❌ INCORRECT: `--region-id` (lowercase with hyphens wrong for RPC) ### Plugin Mode (lowercase with hyphens) - ✅ CORRECT for FC: `aliyun fc create-function --function-name my-func --user-agent AlibabaCloud-Agent-Skills` - ✅ CORRECT for OSS: `aliyun oss mb oss://bucket-name --user-agent AlibabaCloud-Agent-Skills` - ❌ INCORRECT for FC: `aliyun fc CreateFunction` (RPC style wrong for FC 3.0) - ❌ INCORRECT for OSS: `aliyun oss CreateBucket` (RPC style wrong for OSS) ### Parameter Case Sensitivity - ✅ CORRECT for RPC APIs: `--RegionId`, `--ImageId`, `--VpcId` (PascalCase) - ✅ CORRECT for FC: `--function-name`, `--runtime`, `--handler` (lowercase with hyphens) - ✅ CORRECT for OSS: `--region` (lowercase) - ❌ INCORRECT: Mixing styles (e.g., `--regionId` camelCase) ## Enum Value Validation ### SMC TargetType - ✅ CORRECT: `Image`, `ContainerImage`, `TargetInstance` - ❌ INCORRECT: `ECS`, `VM`, `ecs` ### DTS MigrationJobClass - ✅ CORRECT: `small`, `medium`, `large`, `xlarge`, `2xlarge` (lowercase) - ❌ INCORRECT: `Small`, `MEDIUM`, `Large` ### ECS InstanceType - ✅ CORRECT: `ecs.g6.large`, `ecs.c6.xlarge`, `ecs.r6.2xlarge` (with ecs. prefix) - ❌ INCORRECT: `g6.large`, `c6.xlarge` (missing prefix) ### ECS InternetChargeType - ✅ CORRECT: `PayByBandwidth`, `PayByTraffic` - ❌ INCORRECT: `pay-by-traffic`, `PAYBYBANDWIDTH` ### ECS SystemDisk.Category - ✅ CORRECT: `cloud`, `cloud_efficiency`, `cloud_ssd`, `cloud_essd`, `cloud_auto`, `cloud_essd_entry` - ❌ INCORRECT: `ssd`, `essd`, `hdd` (incomplete names) ### DNS Record Type - ✅ CORRECT: `A`, `AAAA`, `CNAME`, `MX`, `TXT`, `NS`, `SRV`, `CAA`, `PTR` (uppercase) - ❌ INCORRECT: `a`, `cname`, `Mx` ### FC Runtime - ✅ CORRECT: `nodejs20`, `python3.10`, `java11`, `go1`, `custom` - ❌ INCORRECT: `Node.js20`, `Python3`, `Java` --- # SDK Usage Patterns (Python Common SDK) ## Import Patterns - ✅ CORRECT: ```python from alibabacloud_tea_openapi.client import Client as OpenApiClient from alibabacloud_credentials.client import Client as CredentialClient from alibabacloud_smc20190601 import models as smc_models ``` - ❌ INCORRECT: ```python import aliyun # Wrong SDK package from alibabacloud import smc # Wrong import path ``` ## Authentication - ✅ CORRECT: ```python credential = CredentialClient() config = open_api_models.Config(credential=credential) client = smc_models.Client(config) ``` - ❌ INCORRECT: ```python # Never hardcode credentials config = open_api_models.Config( access_key_id="LTAI5tXXX", access_key_secret="xxx" ) ``` ## Client Initialization - ✅ CORRECT: ```python config = open_api_models.Config( region_id='cn-hangzhou', credential=credential ) client = smc20190601.Client(config) ``` - ❌ INCORRECT: ```python # Missing region_id or credential client = smc20190601.Client() ``` ## API Invocation - ✅ CORRECT: ```python request = smc_models.CreateReplicationJobRequest( region_id='cn-hangzhou', source_id='s-xxx', target_type='Image' ) response = client.create_replication_job(request) ``` - ❌ INCORRECT: ```python # Wrong method name (should be snake_case) response = client.CreateReplicationJob(request) # Wrong parameter case (should be snake_case) request = smc_models.CreateReplicationJobRequest( RegionId='cn-hangzhou', # Wrong SourceId='s-xxx' # Wrong ) ``` ## Error Handling - ✅ CORRECT: ```python from tea.exceptions import TeaException try: response = client.create_replication_job(request) except TeaException as e: print(f"Error code: {e.code}, message: {e.message}") ``` - ❌ INCORRECT: ```python # No error handling response = client.create_replication_job(request) # Or catching generic Exception only try: response = client.create_replication_job(request) except Exception as e: pass # Silent failure ``` --- # Common Anti-Patterns ## 1. Mixing API Styles - ❌ INCORRECT: Using plugin mode for RPC APIs ```bash aliyun smc create-replication-job --region-id cn-hangzhou # WRONG ``` - ✅ CORRECT: ```bash aliyun smc CreateReplicationJob --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills # RIGHT ``` ## 2. Wrong Product Codes - ❌ INCORRECT: `aliyun dns` (should be `alidns`) - ❌ INCORRECT: `aliyun server-migration` (should be `smc`) - ❌ INCORRECT: `aliyun function-compute` (should be `fc`) ## 3. Hardcoded Values - ❌ INCORRECT: Hardcoding region without user confirmation ```bash aliyun vpc CreateVpc --RegionId cn-hangzhou --user-agent AlibabaCloud-Agent-Skills # Assumes region ``` - ✅ CORRECT: Use placeholder and confirm with user ```bash aliyun vpc CreateVpc --RegionId --user-agent AlibabaCloud-Agent-Skills # Confirm with user first ``` ## 4. Missing Required Parameters - ❌ INCORRECT: `aliyun smc CreateReplicationJob` (missing SourceId, RegionId) - ❌ INCORRECT: `aliyun ecs RunInstances` (missing ImageId, InstanceType, RegionId) ## 5. Wrong Parameter Names - ❌ INCORRECT: `--source-server-id` (should be `--SourceId` for SMC) - ❌ INCORRECT: `--migration-class` (should be `--MigrationJobClass` for DTS) - ❌ INCORRECT: `--instance-type` in wrong context ## 6. Credential Exposure - ❌ INCORRECT: Printing environment variables ```bash echo "AK: $ALIBABA_CLOUD_ACCESS_KEY_ID" ``` - ❌ INCORRECT: Logging credentials ```python print(f"Using AK: {credential.access_key_id}") ``` --- # Verification Checklist Before considering any CLI command or SDK usage correct: 1. **Product Code**: Verify product code exists via `aliyun --help` 2. **API Style**: Confirm RPC vs plugin mode based on product 3. **Parameter Names**: Verify exact parameter names via `--help` 4. **Parameter Case**: RPC uses PascalCase, plugin uses lowercase-hyphen 5. **Enum Values**: Verify enum values match `--help` output exactly 6. **User-Agent**: Every command includes `--user-agent AlibabaCloud-Agent-Skills` 7. **Credential Safety**: No credential values printed or hardcoded 8. **Parameter Confirmation**: All user-specific parameters confirmed before execution --- # Terraform HCL Patterns ## 1. Terraform Online Runtime Usage #### ✅ CORRECT ```bash $TF apply main.tf $TF apply main.tf --state-id $EXISTING_STATE_ID $TF destroy "$STATE_ID" ``` #### ❌ INCORRECT ```bash # Never use inline aliyun iacservice commands aliyun iacservice execute-terraform-apply --code "..." # Never run local terraform CLI (unless explicitly requested) terraform apply # Never start fresh apply when STATE_ID exists (causes duplicates) $TF apply main.tf # when $STATE_ID already exists ``` ## 2. HCL File Consolidation #### ✅ CORRECT — Single main.tf ```hcl # main.tf contains ALL resources resource "alicloud_vpc" "migration" { ... } resource "alicloud_vswitch" "migration" { ... } resource "alicloud_instance" "migrated" { ... } ``` #### ❌ INCORRECT — Split across files ``` # network.tf, compute.tf, database.tf — IaCService only accepts one file ``` ## 3. State ID Management #### ✅ CORRECT ```bash STATE_ID=$($TF apply main.tf | grep '^STATE_ID=' | cut -d= -f2) echo "STATE_ID=$STATE_ID" >> terraform_state_ids.env # Subsequent changes reuse STATE_ID: $TF apply main.tf --state-id $STATE_ID ``` #### ❌ INCORRECT ```bash # Not saving STATE_ID — cannot cleanup later $TF apply main.tf # Fresh apply when state exists — creates duplicate resources $TF apply main.tf # without --state-id when STATE_ID exists ``` ## 4. Provider Configuration #### ✅ CORRECT — Direct provider block only, no terraform{} wrapper ```hcl provider "alicloud" { region = var.region configuration_source = "AlibabaCloud-Agent-Skills/alibabacloud-migrate" } ``` #### ❌ INCORRECT — `terraform {}` / `required_providers` block (causes plugin load failure) ```hcl # DO NOT add this block — the provider is pre-initialized by the environment. # Adding required_providers triggers plugin schema loading via the registry, # which fails with "Unrecognized remote plugin message" in this setup. terraform { required_providers { alicloud = { source = "aliyun/alicloud" version = "~> 1.220" } } } provider "alicloud" { region = var.region } ``` #### ❌ INCORRECT — Hardcoded credentials ```hcl provider "alicloud" { access_key = "LTAI..." secret_key = "abc123..." region = "cn-hangzhou" } ``` ## 5. Placeholder Values #### ✅ CORRECT ```hcl resource "alicloud_vpc" "migration" { vpc_name = "migration-vpc" cidr_block = var.vpc_cidr # or "" in examples } ``` #### ❌ INCORRECT ```hcl resource "alicloud_vpc" "migration" { vpc_name = "migration-vpc" cidr_block = "172.16.0.0/12" # hardcoded user value } ``` FILE:references/assessment-report-template.md # AWS → Alibaba Cloud Migration Assessment Report **Project Name:** [Enter project name] **Assessment Date:** [YYYY-MM-DD] **Assessed By:** [Name/Team] **Version:** 1.0 --- ## 1. Executive Summary ### Migration Scope - **Source AWS Account:** [Account ID or name] - **Source AWS Region:** [e.g., us-east-1] - **Target Alibaba Cloud Account:** [Account ID or name] - **Target Alibaba Cloud Region:** [e.g., ap-southeast-1] ### Resource Summary | Resource Type | Count | |---------------|-------| | Compute (EC2/ECS) | [number] | | Storage (S3/OSS) | [number] | | Database (RDS) | [number] | | Networking (VPC/ELB) | [number] | | Serverless (Lambda/FC) | [number] | | DNS/CDN (Route53/DCDN) | [number] | | Other | [number] | | **Total** | [total] | ### Overall Readiness Assessment **Status:** [Ready / Ready with Caveats / Not Ready] **Justification:** [Brief explanation of readiness level] ### Estimated Migration Timeline - **Start Date:** [YYYY-MM-DD] - **End Date:** [YYYY-MM-DD] - **Total Duration:** [X weeks/months] ### Key Risks and Mitigations | Risk | Summary | Mitigation | |------|---------|------------| | [Risk 1] | [Brief description] | [Mitigation strategy] | | [Risk 2] | [Brief description] | [Mitigation strategy] | --- ## 2. Resource Inventory ### Compute | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | [e.g., web-server-1] | EC2 | [e.g., i-0abc123def] | [e.g., t3.medium, 2 vCPU, 4GB] | ECS [e.g., ecs.g6.large] | SMC | Low | [Any notes] | | | | | | | | | | ### Storage | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | [e.g., app-assets] | S3 | [e.g., my-bucket-123] | [e.g., Standard, 500GB] | OSS [e.g., Standard] | ossutil/rsync | Low | [Any notes] | | | | | | | | | | ### Database | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | [e.g., main-db] | RDS MySQL | [e.g., db-abc123] | [e.g., db.t3.medium, 100GB] | RDS MySQL [e.g., mysql.n2.medium.2c] | DTS | Medium | [Any notes] | | | | | | | | | | ### Networking | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | [e.g., main-vpc] | VPC | [e.g., vpc-abc123] | [e.g., 10.0.0.0/16] | VPC [e.g., 10.1.0.0/16] | Terraform | Low | [Any notes] | | | | | | | | | | ### Serverless | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | [e.g., image-processor] | Lambda | [e.g., img-proc-func] | [e.g., 128MB, 3s timeout] | Function Compute [e.g., 128MB, 3s] | Manual/Terraform | Medium | [Any notes] | | | | | | | | | | ### DNS/CDN | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | [e.g., main-domain] | Route53 | [e.g., Z123ABC] | [e.g., Hosted Zone] | Alibaba Cloud DNS | Manual | Low | [Any notes] | | | | | | | | | | ### Other | Resource Name | AWS Service | AWS Resource ID | Specs (type/size/config) | Alibaba Cloud Target | Migration Tool | Complexity | Notes | |---------------|-------------|-----------------|--------------------------|----------------------|----------------|------------|-------| | | | | | | | | | --- ## 3. Service Mapping ### Mapped Services | AWS Service | Alibaba Cloud Equivalent | Mapping Confidence | Gap/Limitation | |-------------|--------------------------|-------------------|----------------| | [e.g., EC2] | ECS | High | [e.g., Some instance types unavailable] | | [e.g., S3] | OSS | High | [e.g., API differences] | | [e.g., RDS] | RDS | High | [e.g., Version compatibility] | | [e.g., Lambda] | Function Compute | Medium | [e.g., Different runtime support] | | | | | | ### Unmapped Services (Manual Handling Required) | AWS Service | Reason Unmapped | Manual Approach | |-------------|-----------------|-----------------| | [e.g., AWS-specific service] | [e.g., No direct equivalent] | [e.g., Replace with third-party or custom solution] | | | | | --- ## 4. Integration & Dependency Mapping ### Resource Dependencies | Source Resource | Depends On | Dependency Type | Migration Impact | |-----------------|------------|-----------------|------------------| | [e.g., web-server-1] | [e.g., main-db] | Network (port 3306) | Migrate DB first | | [e.g., api-lambda] | [e.g., app-assets] | API (S3 GET) | Migrate S3 first | | [e.g., worker-ec2] | [e.g., SQS queue] | API (SQS poll) | Migrate queue service first | | | | | | ### External Integrations | External System | Integration Type | AWS Resource | Alibaba Cloud Resource | Notes | |-----------------|------------------|--------------|------------------------|-------| | [e.g., Stripe API] | HTTPS API | [e.g., NAT Gateway] | [e.g., NAT Gateway] | [e.g., No changes needed] | | [e.g., On-prem DB] | VPN/Direct Connect | [e.g., VGW] | [e.g., Express Connect] | [e.g., Reconfigure tunnel] | | | | | | | --- ## 5. Architecture Diagrams ### Current State Architecture (AWS) ```mermaid graph TB subgraph "AWS [Region]" subgraph "VPC [CIDR]" LB[ELB/ALB] subgraph "Public Subnet" EC2_1[EC2 Instance 1] EC2_2[EC2 Instance 2] end subgraph "Private Subnet" RDS[(RDS MySQL)] CACHE[ElastiCache] end S3[(S3 Bucket)] LAMBDA[Lambda Function] end Route53[Route53 DNS] CloudWatch[CloudWatch] end LB --> EC2_1 LB --> EC2_2 EC2_1 --> RDS EC2_2 --> RDS EC2_1 --> CACHE EC2_2 --> CACHE EC2_1 --> S3 LAMBDA --> S3 Route53 --> LB EC2_1 --> CloudWatch RDS --> CloudWatch ``` **Replace above with actual AWS architecture. Include:** - All compute resources (EC2, Lambda, ECS, etc.) - All data stores (RDS, S3, DynamoDB, etc.) - Network topology (VPC, subnets, load balancers) - External integrations ### Target State Architecture (Alibaba Cloud) ```mermaid graph TB subgraph "Alibaba Cloud [Region]" subgraph "VPC [CIDR]" SLB[SLB/ALB] subgraph "Public Subnet" ECS_1[ECS Instance 1] ECS_2[ECS Instance 2] end subgraph "Private Subnet" RDS_ALI[(RDS MySQL)] REDIS[Redis] end OSS[(OSS Bucket)] FC[Function Compute] end DNS[Alibaba Cloud DNS] CloudMonitor[CloudMonitor] end SLB --> ECS_1 SLB --> ECS_2 ECS_1 --> RDS_ALI ECS_2 --> RDS_ALI ECS_1 --> REDIS ECS_2 --> REDIS ECS_1 --> OSS FC --> OSS DNS --> SLB ECS_1 --> CloudMonitor RDS_ALI --> CloudMonitor ``` **Replace above with actual target architecture. Ensure:** - Equivalent services are mapped correctly - Network topology maintains security posture - All dependencies are preserved --- ## 6. Network Topology ### VPC/Subnet Layout #### Current AWS VPC | VPC Name | CIDR | Subnet Name | Subnet CIDR | Type (Public/Private) | |----------|------|-------------|-------------|----------------------| | [e.g., main-vpc] | [e.g., 10.0.0.0/16] | [e.g., public-subnet-1a] | [e.g., 10.0.1.0/24] | Public | | | | [e.g., private-subnet-1a] | [e.g., 10.0.2.0/24] | Private | | | | | | | #### Target Alibaba Cloud VPC | VPC Name | CIDR | Subnet Name | Subnet CIDR | Type (Public/Private) | |----------|------|-------------|-------------|----------------------| | [e.g., main-vpc] | [e.g., 10.1.0.0/16] | [e.g., public-subnet-1a] | [e.g., 10.1.1.0/24] | Public | | | | [e.g., private-subnet-1a] | [e.g., 10.1.2.0/24] | Private | | | | | | | **Note:** Ensure CIDR ranges do not conflict if running hybrid during migration. ### Security Group Mapping | AWS Security Group | Purpose | Inbound Rules | Outbound Rules | Alibaba Cloud Security Group | |--------------------|---------|---------------|----------------|------------------------------| | [e.g., sg-web] | Web servers | 80, 443 from 0.0.0.0/0 | All | [e.g., sg-web-ali] | | [e.g., sg-db] | Database | 3306 from sg-web | All | [e.g., sg-db-ali] | | | | | | | ### Hybrid Connectivity (During Migration) | Requirement | AWS Side | Alibaba Cloud Side | Tool/Service | Status | |-------------|----------|-------------------|--------------|--------| | Site-to-Site VPN | [e.g., VGW] | [e.g., IPsec-VPN] | [e.g., IPsec tunnel] | [Planned/Configured] | | Direct Connect | [e.g., DX] | [e.g., Express Connect] | [e.g., Dedicated line] | [Planned/Configured] | | None (online only) | N/A | N/A | N/A | N/A | --- ## 7. IAM & Security Mapping ### IAM/RAM Entity Mapping | AWS IAM Entity | Type (User/Role/Policy) | Alibaba Cloud RAM Equivalent | Notes | |----------------|-------------------------|------------------------------|-------| | [e.g., ec2-admin-role] | Role | [e.g., ecs-admin-role] | [e.g., Custom least-privilege policy scoped to required ECS actions] | | [e.g., s3-read-policy] | Policy | [e.g., oss-read-policy] | [e.g., Custom policy for OSS read] | | [e.g., lambda-exec-role] | Role | [e.g., fc-exec-role] | [e.g., Custom least-privilege policy scoped to accessed services] | | | | | | ### Function Compute Execution Role Requirements > **CRITICAL**: Every FC function that accesses other Alibaba Cloud services (OSS, SLS, Tablestore, MNS, etc.) **MUST** have a RAM role assigned via the `role` parameter. Without this, the function will fail at runtime with authentication errors (e.g., OSS `AccessDenied`). | Lambda Function | AWS Execution Role | Services Accessed | Required RAM Policy Actions | Alibaba Cloud RAM Role Name | |-----------------|-------------------|-------------------|----------------------------|----------------------------| | [e.g., my-func] | [e.g., lambda-s3-role] | [e.g., S3 read/write] | [e.g., oss:GetObject, oss:PutObject] | [e.g., fc-my-func-role] | | | | | | | ### Encryption Mapping | AWS Service | Encryption Method | Alibaba Cloud Equivalent | Notes | |-------------|-------------------|--------------------------|-------| | S3 | SSE-S3 / SSE-KMS | OSS Server-Side Encryption | [e.g., Use KMS key] | | RDS | KMS key | RDS TDE with KMS | [e.g., Key ID mapping] | | EBS | KMS key | Cloud Disk Encryption | [e.g., Key ID mapping] | | | | | | ### Compliance Requirements | Requirement | AWS Implementation | Alibaba Cloud Implementation | Status | |-------------|--------------------|------------------------------|--------| | [e.g., Data residency] | [e.g., us-east-1 region] | [e.g., ap-southeast-1 region] | [Compliant/Review needed] | | [e.g., Encryption at rest] | [e.g., KMS for all storage] | [e.g., KMS for all storage] | [Compliant/Review needed] | | | | | | --- ## 8. Monitoring & Observability Mapping | AWS Service | Alibaba Cloud Equivalent | Config Notes | |-------------|--------------------------|--------------| | CloudWatch Metrics | CloudMonitor | [e.g., Set up custom metrics for ECS] | | CloudWatch Logs | SLS (Simple Log Service) | [e.g., Configure Logtail on ECS] | | CloudWatch Alarms | CloudMonitor Alerts | [e.g., Recreate alarm thresholds] | | X-Ray | ARMS (Application Real-Time Monitoring Service) | [e.g., Install ARMS agent] | | CloudTrail | ActionTrail | [e.g., Enable for all regions] | | AWS Config | Config Audit | [e.g., Set up compliance rules] | | | | | ### Dashboard Migration | AWS Dashboard | Purpose | Alibaba Cloud Dashboard | Status | |---------------|---------|-------------------------|--------| | [e.g., Main Ops Dashboard] | [e.g., Overall system health] | [e.g., CloudMonitor Dashboard] | [Planned/Complete] | | | | | | --- ## 9. Data Migration Strategy ### Database Migration | Database Name | AWS RDS Instance | Alibaba Cloud RDS Instance | Migration Approach | Data Volume | Est. Transfer Time | Cutover Window | |---------------|------------------|----------------------------|--------------------|-------------|--------------------|----------------| | [e.g., main-db] | [e.g., db-abc123] | [e.g., rm-xyz789] | [Full + Incremental via DTS] | [e.g., 100GB] | [e.g., 4 hours] | [e.g., 2 hours] | | | | | | | | | **Migration Steps:** 1. [ ] Create DTS migration task (full data migration) 2. [ ] Enable incremental sync 3. [ ] Monitor sync lag 4. [ ] Schedule cutover window 5. [ ] Stop writes to source 6. [ ] Wait for sync completion 7. [ ] Update application connection strings 8. [ ] Verify data integrity 9. [ ] Decommission source (after validation period) ### Storage Migration | Storage Name | AWS S3 Bucket | Alibaba Cloud OSS Bucket | Migration Approach | Data Volume | Est. Transfer Time | Cutover Window | |--------------|---------------|--------------------------|--------------------|-------------|--------------------|----------------| | [e.g., app-assets] | [e.g., my-bucket] | [e.g., my-bucket-ali] | [ossutil sync / online] | [e.g., 500GB] | [e.g., 8 hours] | [e.g., None (online)] | | | | | | | | | **Migration Steps:** 1. [ ] Create OSS bucket 2. [ ] Configure ossutil 3. [ ] Run initial sync: `ossutil sync oss://source oss://target -r` 4. [ ] Schedule periodic syncs for delta 5. [ ] Final sync during cutover (if needed) 6. [ ] Update application endpoints 7. [ ] Verify file integrity 8. [ ] Decommission source (after validation period) --- ## 10. Cost Estimation ### Resource Cost Comparison | Resource | AWS Monthly Cost (USD) | Alibaba Cloud Monthly Cost (USD) | Notes | |----------|------------------------|----------------------------------|-------| | [e.g., EC2 t3.medium x4] | [$XXX] | [$XXX] | [e.g., ECS g6.large equivalent] | | [e.g., RDS db.t3.medium] | [$XXX] | [$XXX] | [e.g., Similar specs] | | [e.g., S3 500GB] | [$XXX] | [$XXX] | [e.g., OSS Standard] | | [e.g., ELB] | [$XXX] | [$XXX] | [e.g., SLB equivalent] | | [e.g., Data transfer] | [$XXX] | [$XXX] | [e.g., Egress costs] | | **Total Monthly** | **[$XXX]** | **[$XXX]** | **Savings: $XXX (XX%)** | ### Migration Tool Costs | Tool | Cost Type | Estimated Cost | Notes | |------|-----------|----------------|-------| | SMC (Server Migration Center) | Free | $0 | [e.g., No charge for migration] | | DTS (Data Transmission Service) | Free + Data Transfer | $XXX | [e.g., Pay for data transfer only] | | ossutil | Free | $0 | [e.g., No charge, pay for egress] | | AWS Data Egress | Per GB | $XXX | [e.g., $0.09/GB for us-east-1] | | **One-Time Migration Cost** | | **[$XXX]** | | ### Ongoing Costs Post-Migration | Cost Category | Monthly Estimate | Notes | |---------------|------------------|-------| | Compute | [$XXX] | | | Storage | [$XXX] | | | Database | [$XXX] | | | Network | [$XXX] | | | Monitoring | [$XXX] | | | **Total** | **[$XXX]** | | --- ## 11. Risk Assessment | Risk | Probability | Impact | Mitigation | |------|-------------|--------|------------| | Data loss during migration | Low | High | [Use DTS with verification; keep source intact until validation complete] | | Extended downtime | Medium | High | [Plan cutover during low-traffic window; test rollback procedure] | | Performance degradation | Medium | Medium | [Benchmark target resources before cutover; scale up if needed] | | IAM/permission issues | Medium | Medium | [Test RAM policies in staging; use least-privilege principle] | | DNS propagation delay | Low | Medium | [Lower TTL 48h before cutover; use health checks] | | Application compatibility | Medium | High | [Test thoroughly in staging; have rollback ready] | | Cost overrun | Low | Low | [Monitor costs daily; set budget alerts] | | Third-party integration failure | Low | High | [Test all external integrations in staging; have fallback] | **Probability:** Low (<20%), Medium (20-50%), High (>50%) **Impact:** Low (minor inconvenience), Medium (significant disruption), High (business-critical failure) --- ## 12. Migration Plan (Phase Summary) | Phase | Resources | Tool | Dependencies | Estimated Duration | Status | |-------|-----------|------|--------------|--------------------|--------| | 1. Network Setup | VPC, Subnets, Security Groups, NAT Gateway | Terraform | None | [e.g., 1 week] | [Not Started] | | 2. Server Migration | EC2 instances → ECS | SMC | Phase 1 complete | [e.g., 2 weeks] | [Not Started] | | 3. Database Migration | RDS → RDS | DTS | Phase 1 complete | [e.g., 1 week + cutover] | [Not Started] | | 4. Storage Migration | S3 → OSS | ossutil | Phase 1 complete | [e.g., 1 week] | [Not Started] | | 5. Serverless Migration | Lambda → Function Compute | Manual/Terraform | Phase 1, 4 complete | [e.g., 1 week] | [Not Started] | | 6. DNS/CDN Cutover | Route53 → Alibaba Cloud DNS | Manual | All phases complete | [e.g., 1 day] | [Not Started] | | 7. Validation & Decommission | All resources | Manual | All phases complete | [e.g., 1 week] | [Not Started] | ### Rollback Strategy per Phase | Phase | Rollback Trigger | Rollback Procedure | |-------|------------------|--------------------| | 1. Network Setup | Configuration errors | Destroy and recreate VPC with corrected config | | 2. Server Migration | Application fails on ECS | Revert DNS to AWS ELB; terminate ECS instances | | 3. Database Migration | Data integrity issues | Stop DTS; revert application to AWS RDS endpoint | | 4. Storage Migration | File access issues | Revert application to S3 endpoints | | 5. Serverless Migration | Function failures | Revert to AWS Lambda; update API Gateway | | 6. DNS/CDN Cutover | Critical failures | Update DNS records back to AWS (respect TTL) | | 7. Validation | Any critical issue | Execute phase-specific rollback | --- ## 13. Rollback Plan ### Point of No Return Criteria **Do NOT proceed past this point until:** - [ ] All data migrations verified (checksums match) - [ ] All applications tested in staging - [ ] All stakeholders signed off - [ ] Rollback procedure tested - [ ] Backup of all source resources confirmed ### DNS Rollback (Fastest Path) 1. Update DNS records to point back to AWS ELB/CloudFront 2. Wait for TTL expiration (reduced to [X minutes] before cutover) 3. Verify traffic routing to AWS 4. Monitor application health **Estimated DNS Rollback Time:** [X minutes to X hours depending on TTL] ### DTS Reverse Sync Capability - **Supported:** Yes (if DTS task configured bidirectionally) - **Procedure:** 1. Stop writes to Alibaba Cloud RDS 2. Enable reverse sync in DTS (Alibaba Cloud → AWS) 3. Wait for sync completion 4. Update application connection strings to AWS RDS 5. Verify data integrity ### Data Rollback per Phase | Phase | Rollback Data Source | Rollback Time Estimate | |-------|----------------------|------------------------| | 2. Server Migration | AWS EC2 (still running) | Immediate (DNS switch) | | 3. Database Migration | AWS RDS (via DTS reverse sync) | [X hours] | | 4. Storage Migration | AWS S3 (still intact) | Immediate (endpoint switch) | | 5. Serverless Migration | AWS Lambda (still deployed) | [X minutes] | --- ## 14. Next Steps ### Pre-Migration Checklist - [ ] Complete resource inventory (Section 2) - [ ] Validate service mappings (Section 3) - [ ] Document all dependencies (Section 4) - [ ] Create architecture diagrams (Section 5) - [ ] Configure network topology (Section 6) - [ ] Set up IAM/RAM policies (Section 7) - [ ] Configure monitoring (Section 8) - [ ] Test data migration in staging (Section 9) - [ ] Review cost estimates (Section 10) - [ ] Review and accept risks (Section 11) - [ ] Finalize migration plan (Section 12) - [ ] Test rollback procedure (Section 13) - [ ] Schedule cutover window with stakeholders - [ ] Prepare communication plan for downtime ### Sign-Off **Migration Lead:** ________________________ **Date:** _______________ **Technical Approver:** ________________________ **Date:** _______________ **Business Stakeholder:** ________________________ **Date:** _______________ **Security/Compliance:** ________________________ **Date:** _______________ --- **Document Version History:** | Version | Date | Author | Changes | |---------|------|--------|---------| | 1.0 | [YYYY-MM-DD] | [Name] | Initial assessment | | | | | | FILE:references/aws-discovery-commands.md # AWS Resource Discovery (Script-First) Migration assessment requires a reproducible inventory. All discovery that involves looping over resources or making multiple API calls is encapsulated in scripts. This document explains **how to run the scripts**, **required permissions**, and **fallback options** — it does not list per-service AWS CLI commands. --- ## Quick Start Two steps for a complete discovery: ```bash # Step 1: Broad scan — one list/describe per service category, ~30–60 seconds ./scripts/aws-scan-region.sh # Step 2: Deep scan — loops over each main resource to fetch per-resource details ./scripts/aws-scan-enrich.sh aws-scan-- ``` - Step 1 output: `aws-scan--/inventory.md` - Step 2 writes `inventory-deep.md` into the same directory; omitting the second argument creates a new directory. ### Environment Variables (both scripts) | Variable | Default | Description | |----------|---------|-------------| | `AWS_SCAN_CMD_TIMEOUT` | `120` | Per-command timeout in seconds; requires GNU `timeout` | | `AWS_SCAN_REDACT` | `1` | When `1`, redacts IPs, resource IDs, and account numbers in output | | `AWS_MAX_ATTEMPTS` / `AWS_RETRY_MODE` | CLI standard | Passed to AWS CLI to prevent slow APIs from stalling the scan | --- ## Prerequisites - **AWS CLI v2** installed and configured via `aws configure` (or equivalent credential environment variables). - Verify access: `aws sts get-caller-identity` ### IAM Recommendations Attach the **`ReadOnlyAccess`** managed policy when possible. For least-privilege, the minimum permissions must cover all `Describe*` / `List*` / `Get*` calls in the scripts, plus: `lambda:GetPolicy` · `apigateway:GET` · `events:ListEventBuses` · `events:ListRules` · `events:ListTargetsByRule` · `s3:GetBucket*` · `sns:ListSubscriptionsByTopic` · `iam:GetRolePolicy` · `ecr:DescribeRepositories` · `kinesis:ListStreams` · `wafv2:ListWebACLs` · `directconnect:Describe*` · `elasticache:DescribeReplicationGroups` · `elasticache:DescribeCacheParameters` · `kafka:DescribeCluster` · `cognito-idp:DescribeUserPool` · `cognito-idp:ListUserPoolClients` · `elasticfilesystem:DescribeMountTargets` · `elasticfilesystem:DescribeAccessPoints` --- ## Script Responsibilities | Discovery requirement | Covered by | |-----------------------|------------| | Whether main resources exist per service (EC2, Lambda, S3, SNS, EventBridge buses, ECR, Kinesis, WAF, VPN, Direct Connect, EMR, etc.) | `aws-scan-region.sh` | | EventBridge rules **and** `list-targets-by-rule` per rule, on every event bus | `aws-scan-enrich.sh` | | S3 per-bucket lifecycle rules + bucket policy | `aws-scan-enrich.sh` | | SNS per-topic subscriptions | `aws-scan-enrich.sh` | | IAM inline policy content per role | `aws-scan-enrich.sh` | | Lambda `get-policy` per function (invoke permissions / push triggers) | `aws-scan-enrich.sh` | | API Gateway REST: `get-resources` + `get-stages` per API; HTTP v2: routes + stages | `aws-scan-enrich.sh` | | ECS per-cluster services + task definitions | `aws-scan-enrich.sh` | | EKS per-cluster describe + addons | `aws-scan-enrich.sh` | | ELB v2 per-LB listeners + target groups | `aws-scan-enrich.sh` | | Route53 per-zone record sets | `aws-scan-enrich.sh` | | CloudFront per-distribution origins + cache behaviors + SSL | `aws-scan-enrich.sh` | | DynamoDB per-table capacity, GSI/LSI, streams | `aws-scan-enrich.sh` | | SQS per-queue attributes (DLQ, encryption, FIFO) | `aws-scan-enrich.sh` | | RDS subnet groups + user-modified parameter groups | `aws-scan-enrich.sh` | | Step Functions per-state-machine definition + logging | `aws-scan-enrich.sh` | | ElastiCache replication groups + user-modified parameters | `aws-scan-enrich.sh` | | MSK (Kafka) per-cluster broker config and version | `aws-scan-enrich.sh` | | Cognito per-user-pool config + app clients | `aws-scan-enrich.sh` | | EFS per-file-system mount targets + access points | `aws-scan-enrich.sh` | --- ## Optional: Resource Explorer Overview If **Resource Explorer** is already enabled in the account, a single query gives a high-level resource count across all types (does not replace the per-field detail from the scripts): ```bash aws resourcegroupstaggingapi get-resources --region ``` Refer to current AWS documentation for the aggregation region and enablement steps. --- ## No CLI or Insufficient Permissions - Export or screenshot resource lists from the AWS Console per region, then fill in the resource table in `migration-assessment-report.md` manually. - Both scripts annotate `AccessDenied` and timeout conditions inside each output block, making it easy to distinguish "no resources" from "no permission". FILE:references/cli-installation-guide.md # Aliyun CLI Installation & Configuration Guide Complete guide for installing and configuring Aliyun CLI. > **Aliyun CLI 3.3.1+**: Supports installing and using all published Alibaba Cloud product plugins. Make sure to upgrade to 3.3.1 or later for full plugin ecosystem coverage. ## Installation ### macOS **Using Homebrew (Recommended)** ```bash brew install aliyun-cli # Upgrade to latest brew upgrade aliyun-cli # Verify version (>= 3.3.1) aliyun version ``` **Using Binary** ```bash # Download wget https://aliyuncli.alicdn.com/aliyun-cli-macosx-latest-amd64.tgz # Extract tar -xzf aliyun-cli-macosx-latest-amd64.tgz # Move to PATH sudo mv aliyun /usr/local/bin/ # Verify aliyun version ``` ### Linux **Debian/Ubuntu** ```bash # Download wget https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-amd64.tgz # Extract and install tar -xzf aliyun-cli-linux-latest-amd64.tgz sudo mv aliyun /usr/local/bin/ # Verify aliyun version ``` **CentOS/RHEL** ```bash # Download wget https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-amd64.tgz # Extract and install tar -xzf aliyun-cli-linux-latest-amd64.tgz sudo mv aliyun /usr/local/bin/ # Verify aliyun version ``` **ARM64 Architecture** ```bash # Download ARM64 version wget https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-arm64.tgz # Extract and install tar -xzf aliyun-cli-linux-latest-arm64.tgz sudo mv aliyun /usr/local/bin/ ``` ### Windows **Using Binary** 1. Download from: https://aliyuncli.alicdn.com/aliyun-cli-windows-latest-amd64.zip 2. Extract the ZIP file 3. Add the directory to your PATH environment variable 4. Open new Command Prompt or PowerShell 5. Verify: `aliyun version` **Using PowerShell** ```powershell # Download Invoke-WebRequest -Uri "https://aliyuncli.alicdn.com/aliyun-cli-windows-latest-amd64.zip" -OutFile "aliyun-cli.zip" # Extract Expand-Archive -Path aliyun-cli.zip -DestinationPath C:\aliyun-cli # Add to PATH (requires admin privileges) $env:Path += ";C:\aliyun-cli" [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) # Verify aliyun version ``` ## Configuration ### Quick Start ```bash aliyun configure set \ --mode AK \ --access-key-id \ --access-key-secret \ --region cn-hangzhou ``` All `aliyun configure` commands support non-interactive flags, which is the recommended approach — it works in scripts, CI/CD pipelines, and agent-driven automation without hanging on stdin prompts. **Where to Get Access Keys** 1. Log in to Aliyun Console: https://ram.console.aliyun.com/ 2. Navigate to: AccessKey Management 3. Create a new AccessKey pair 4. Save the secret immediately — it's only shown once ### Configuration Modes Aliyun CLI supports 6 authentication modes. All examples below use non-interactive flags. #### 1. AK Mode (Access Key) Most common mode for personal accounts and scripts. ```bash aliyun configure set \ --mode AK \ --access-key-id LTAI5tXXXXXXXX \ --access-key-secret 8dXXXXXXXXXXXXXXXXXXXXXXXX \ --region cn-hangzhou ``` Configuration is stored in `~/.aliyun/config.json`: ```json { "current": "default", "profiles": [ { "name": "default", "mode": "AK", "access_key_id": "LTAI5tXXXXXXXX", "access_key_secret": "8dXXXXXXXXXXXXXXXXXXXXXXXX", "region_id": "cn-hangzhou", "output_format": "json", "language": "en" } ] } ``` #### 2. StsToken Mode (Temporary Credentials) For short-lived access (tokens expire in 1-12 hours). ```bash aliyun configure set \ --mode StsToken \ --access-key-id LTAI5tXXXXXXXX \ --access-key-secret 8dXXXXXXXXXXXXXXXXXXXXXXXX \ --sts-token v1.0:XXXXXXXXXXXXXXXX \ --region cn-hangzhou ``` Use cases: CI/CD pipelines, temporary access for external contractors, cross-account access. #### 3. RamRoleArn Mode (Assume RAM Role) Assume a RAM role for elevated or cross-account access. ```bash aliyun configure set \ --mode RamRoleArn \ --access-key-id LTAI5tXXXXXXXX \ --access-key-secret 8dXXXXXXXXXXXXXXXXXXXXXXXX \ --ram-role-arn acs:ram::123456789012:role/AdminRole \ --role-session-name my-session \ --region cn-hangzhou ``` Use cases: cross-account resource access, temporary elevated privileges, role-based access control. #### 4. EcsRamRole Mode (ECS Instance RAM Role) Use the RAM role attached to an ECS instance — no credentials needed. ```bash aliyun configure set \ --mode EcsRamRole \ --ram-role-name MyEcsRole \ --region cn-hangzhou ``` Requirements: must be running on an ECS instance with a RAM role attached. Use cases: scripts and automation running on ECS instances. #### 5. RsaKeyPair Mode (RSA Key Pair) Use RSA key pair for authentication (generate key pair in Aliyun Console first). ```bash aliyun configure set \ --mode RsaKeyPair \ --private-key /path/to/private-key.pem \ --key-pair-name my-key-pair \ --region cn-hangzhou ``` #### 6. RamRoleArnWithEcs Mode (ECS + RAM Role) Combine ECS instance role with RAM role assumption for cross-account access from ECS. ```bash aliyun configure set \ --mode RamRoleArnWithEcs \ --ram-role-name MyEcsRole \ --ram-role-arn acs:ram::123456789012:role/TargetRole \ --role-session-name my-session \ --region cn-hangzhou ``` ### Environment Variables **Highest priority** - overrides config file **Access Key Mode** ```bash export ALIBABA_CLOUD_ACCESS_KEY_ID=your_access_key_id export ALIBABA_CLOUD_ACCESS_KEY_SECRET=your_access_key_secret export ALIBABA_CLOUD_REGION_ID=cn-hangzhou ``` **STS Token Mode** ```bash export ALIBABA_CLOUD_ACCESS_KEY_ID=your_access_key_id export ALIBABA_CLOUD_ACCESS_KEY_SECRET=your_access_key_secret export ALIBABA_CLOUD_SECURITY_TOKEN=your_sts_token export ALIBABA_CLOUD_REGION_ID=cn-hangzhou ``` **ECS RAM Role Mode** ```bash export ALIBABA_CLOUD_ECS_METADATA=role_name ``` **Use Case**: - CI/CD pipelines - Docker containers - Temporary credential override ### Managing Multiple Profiles **Create Named Profiles** ```bash aliyun configure set --profile projectA \ --mode AK \ --access-key-id LTAI5tAAAAAAAA \ --access-key-secret 8dAAAAAAAAAAAAAAAAAAAAAAAA \ --region cn-hangzhou aliyun configure set --profile projectB \ --mode AK \ --access-key-id LTAI5tBBBBBBBB \ --access-key-secret 8dBBBBBBBBBBBBBBBBBBBBBBBB \ --region cn-shanghai ``` **Use Specific Profile** ```bash aliyun ecs describe-instances --profile projectA export ALIBABA_CLOUD_PROFILE=projectA aliyun ecs describe-instances # Uses projectA ``` **List and Switch Profiles** ```bash aliyun configure list # List all profiles aliyun configure set --current projectA # Switch default profile ``` ### Credential Priority Credentials are loaded in this order (first found wins): 1. **Command-line flag**: `--profile ` 2. **Environment variable**: `ALIBABA_CLOUD_PROFILE` 3. **Environment credentials**: `ALIBABA_CLOUD_ACCESS_KEY_ID`, etc. 4. **Configuration file**: `~/.aliyun/config.json` (current profile) 5. **ECS Instance RAM Role**: If running on ECS with attached role ## Verification ### Test Authentication ```bash # Basic test - list regions aliyun ecs describe-regions # Expected output: JSON array of regions ``` **If successful**, you'll see: ```json { "Regions": { "Region": [ { "RegionId": "cn-hangzhou", "RegionEndpoint": "ecs.cn-hangzhou.aliyuncs.com", "LocalName": "华东 1（杭州）" }, ... ] }, "RequestId": "..." } ``` **If failed**, you'll see error messages: - `InvalidAccessKeyId.NotFound` - Wrong Access Key ID - `SignatureDoesNotMatch` - Wrong Access Key Secret - `InvalidSecurityToken.Expired` - STS token expired (for StsToken mode) - `Forbidden.RAM` - Insufficient permissions ### Debug Configuration ```bash # Show current configuration aliyun configure get # Test with debug logging aliyun ecs describe-regions --log-level=debug # Check credential provider aliyun configure get mode ``` ## Security Best Practices ### 1. Use RAM Users (Not Root Account) ❌ **Don't**: Use Aliyun root account credentials ✅ **Do**: Create RAM users with specific permissions ```bash # Create RAM user in console # Attach only necessary policies # Use RAM user's access keys ``` ### 2. Principle of Least Privilege Grant only the minimum permissions needed: ```bash # Example: Read-only ECS access # Attach policy: AliyunECSReadOnlyAccess ``` ### 3. Rotate Access Keys Regularly ```bash # Create new access key in RAM Console, then update configuration aliyun configure set --access-key-id NEW_KEY --access-key-secret NEW_SECRET # Delete old access key from console ``` ### 4. Use STS Tokens for Temporary Access ```bash aliyun configure set --mode StsToken \ --access-key-id XXXX --access-key-secret XXXX \ --sts-token XXXX --region cn-hangzhou ``` ### 5. Use ECS RAM Roles When Possible ```bash aliyun configure set --mode EcsRamRole --ram-role-name MyRole --region cn-hangzhou ``` ### 6. Never Commit Credentials ```bash # Add to .gitignore echo "~/.aliyun/config.json" >> .gitignore # Use environment variables in CI/CD instead ``` ### 7. Secure Config File ```bash # Restrict permissions chmod 600 ~/.aliyun/config.json ``` ## Troubleshooting ### Issue: Command Not Found ```bash # Check installation which aliyun # Check PATH echo $PATH # Reinstall or add to PATH ``` ### Issue: Authentication Failed ```bash # Verify configuration aliyun configure get # Test with debug aliyun ecs describe-regions --log-level=debug # Check credentials in console # Verify access key is active ``` ### Issue: Permission Denied ```bash # Error: Forbidden.RAM # Check RAM user permissions # Attach necessary policies in RAM console # Example: AliyunECSFullAccess for ECS operations ``` ### Issue: STS Token Expired ```bash # Error: InvalidSecurityToken.Expired # Reconfigure with new token aliyun configure set --mode StsToken \ --access-key-id XXXX --access-key-secret XXXX \ --sts-token NEW_TOKEN --region cn-hangzhou ``` ### Issue: Wrong Region ```bash # Some resources may not exist in the specified region # Check available regions aliyun ecs describe-regions # Update default region aliyun configure set region cn-shanghai ``` ## Advanced Configuration ### Custom Endpoint ```bash # Use custom or private endpoint export ALIBABA_CLOUD_ECS_ENDPOINT=ecs-vpc.cn-hangzhou.aliyuncs.com ``` ### Proxy Settings ```bash # HTTP proxy export HTTP_PROXY=http://proxy.example.com:8080 export HTTPS_PROXY=http://proxy.example.com:8080 # No proxy for specific domains export NO_PROXY=localhost,127.0.0.1,.aliyuncs.com ``` ### Timeout Settings ```bash # Connection timeout (default: 10s) export ALIBABA_CLOUD_CONNECT_TIMEOUT=30 # Read timeout (default: 10s) export ALIBABA_CLOUD_READ_TIMEOUT=30 ``` ## Next Steps After installation and configuration: 1. **Install plugins** for services you need (v3.3.1+ supports all published product plugins): ```bash aliyun plugin install --names ecs vpc rds # List all available plugins aliyun plugin list-remote ``` 2. **Explore commands**: ```bash aliyun ecs --help aliyun fc --help ``` 3. **Read documentation**: - [Command Syntax Guide](./command-syntax.md) - [Global Flags Reference](./global-flags.md) - [Common Scenarios](./common-scenarios.md) ## References - Official Documentation: https://help.aliyun.com/zh/cli/ - RAM Console: https://ram.console.aliyun.com/ - Access Key Management: https://ram.console.aliyun.com/manage/ak - Plugin Repository: https://github.com/aliyun/aliyun-cli FILE:references/error-remediation.md # Common Migration Errors and Remediation This document provides a structured reference for common failures encountered during AWS → Alibaba Cloud migration, organized by migration tool and phase. Each error includes the root cause and actionable remediation steps. --- ## 1. Terraform (IaCService) Errors | Error Pattern | Cause | Remediation | |---|---|---| | `Error: creating VPC: Forbidden.RAM` | Missing RAM permissions for VPC operations | Attach `AliyunVPCFullAccess` policy to the RAM user. See `ram-policies.md` for complete policy list | | `Error: InvalidCidrBlock` | CIDR block format invalid or conflicts with existing VPC | Check CIDR format (e.g., `10.0.0.0/8`). Ensure no overlap with existing VPCs using `aliyun vpc DescribeVpcs --user-agent AlibabaCloud-Agent-Skills` | | `Error: QuotaExceeded.xxx` | Resource quota limit reached for the account/region | Request quota increase via Alibaba Cloud Console → Quota Center. Check current quota: `aliyun ecs DescribeAccountAttributes --user-agent AlibabaCloud-Agent-Skills` | | `Error: IdempotentParameterMismatch` | Same resource name used with different parameters in subsequent apply | Use unique resource names per environment, or import existing resource with `terraform import . ` | | `Error: OperationDenied.xxx` | Resource in wrong state for the requested operation | Wait for resource to reach target state, then retry. Check resource status in console or via CLI: `aliyun Describe --user-agent AlibabaCloud-Agent-Skills` | | `Error: InvalidZone.NotOnSale` | Instance type not available in chosen zone | Change ZoneId or InstanceType. Check availability: `aliyun ecs DescribeAvailableResource --RegionId --InstanceType --user-agent AlibabaCloud-Agent-Skills` | | State lock error / STATE_ID conflict | Concurrent operations on same Terraform state file | Wait for previous IaCService task to complete. Check IaCService task status. If stuck, contact admin to release lock | | `Error: timeout while waiting for state` | Resource creation/update timed out waiting for target state | Check resource status in console. If created successfully, import into state. If not, check event logs and retry | | `Error: Provider produced inconsistent result` | Provider returned different resource attributes than expected | Run `terraform refresh` to sync state. If persists, check provider version compatibility in `terraform-providers/alicloud.md` | | `Error: InvalidAccessKeyId.NotFound` | AccessKey ID does not exist or is inactive | Verify AccessKey is active in RAM console. Ensure correct AK/SK configured in IaCService task | --- ## 2. SMC (Server Migration) Errors | Error Pattern | Cause | Remediation | |---|---|---| | Migration stuck at progress < 100% for extended period | Large disk size, slow network bandwidth, or SMC agent issue | Check SMC agent logs on source server (`/root/go2aliyun_client.log`). Consider increasing bandwidth or using VPC-based migration (NetMode=2) for better throughput | | `SourceServerNotFound` | SMC agent not registered with Alibaba Cloud SMC service | Install and start SMC agent on source EC2. Verify agent connectivity to Alibaba Cloud endpoint: `telnet smc..aliyuncs.com 8080` | | `Forbidden.InstanceExist` | Intermediate migration instance already exists from previous job | Delete existing intermediate instance via ECS console, or use a different migration job name | | Image creation failed | Disk size exceeds limits or unsupported filesystem type | Check source disk size (<500GB recommended for single migration). Verify filesystem type is supported (ext3/ext4/xfs/ntfs). For larger disks, use incremental migration | | `IntranetIpConflict` | Target IP conflicts with existing IP in VSwitch | Adjust VSwitch CIDR or specify a different VSwitch for the migration job. Check available IPs: `aliyun vpc DescribeVSwitches --VSwitchId --user-agent AlibabaCloud-Agent-Skills` | | Agent disconnected during migration | Network interruption between source and Alibaba Cloud | Restart SMC agent on source server. Migration supports resumption — it will continue from last checkpoint automatically | | `InvalidAccountStatus.NotEnoughBalance` | Insufficient account balance for pay-as-you-go resources | Top up Alibaba Cloud account. SMC creates intermediate pay-as-you-go instances during migration | | `InvalidParameter.Encrypted` | Source disk is encrypted and key not available | Export encrypted AMI with shared KMS key, or create unencrypted copy first. Alibaba Cloud does not support direct encrypted AMI import | | `ReplicationJobNotReady` | Replication job not in correct state for operation | Wait for job to reach `Ready` state. Check status: `aliyun smc DescribeReplicationJobs --ReplicationJobId --user-agent AlibabaCloud-Agent-Skills` | | `Go2AliyunClientError` | SMC agent encountered internal error | Check agent logs at `/root/go2aliyun_client.log`. Common fixes: restart agent, verify network connectivity, ensure sufficient disk space | --- ## 3. DTS (Database Migration) Errors | Error Pattern | Cause | Remediation | |---|---|---| | Pre-check failed: connectivity | Source RDS not accessible from DTS service | Configure AWS RDS security group to allow DTS IP ranges (check DTS console for IPs). Or use public endpoint with SSL enabled | | Pre-check failed: privileges | Database user lacks required replication permissions | Grant REPLICATION SLAVE, REPLICATION CLIENT privileges on source MySQL. For PostgreSQL: grant SUPERUSER or specific replication role | | Pre-check failed: binlog | Binary logging not enabled on source RDS | Enable binary logging on source RDS: set `binlog_format=ROW`, `binlog_row_image=FULL`, `expire_logs_days>=7`. Requires RDS parameter group modification | | Sync lag increasing continuously | High write volume on source exceeding DTS throughput | Increase DTS instance specification (Small→Medium→Large). Consider scheduling migration during low-traffic window | | Schema migration failed: DDL syntax error | Source DDL uses syntax incompatible with target | Manually create the table on target with compatible syntax, then configure DTS to skip schema migration for that table | | `OperationDenied.JobStatus` | DTS job in wrong state for requested operation | Check job status in console. May need to stop running job before reconfiguration. Some operations only allowed in `NotStarted` state | | Character set mismatch | Source uses charset not supported by target RDS | Verify charset compatibility before migration. Use `utf8mb4` for broad compatibility. Check target supported charsets in RDS console | | `DtsJobIdNotExist` | DTS job ID does not exist or was deleted | Verify job ID in DTS console. Job may have been deleted or never created. Re-create migration job if needed | | `TargetDbNotReady` | Target RDS instance not in running state | Start target RDS instance. Ensure instance is accessible and in `Running` state before starting DTS job | | Data consistency check failed | Source and target data mismatch after full migration | Run DTS data consistency check. For mismatches, use DTS repair function or manually reconcile affected tables | --- ## 4. Storage Migration (OSS) Errors | Error Pattern | Cause | Remediation | |---|---|---| | `AccessDenied` on source S3 | AWS IAM user lacks required S3 permissions | Ensure AWS IAM user has `s3:GetObject`, `s3:ListBucket`, `s3:GetBucketLocation` on source bucket. Add bucket policy if cross-account | | `BucketAlreadyExists` | OSS bucket name already taken globally | Choose a different bucket name (OSS bucket names are globally unique across all Alibaba Cloud accounts). Use naming convention: `--` | | Transfer speed too slow | Single-threaded transfer or suboptimal part size | Use `--part-size` and `--parallel` flags with ossutil (e.g., `ossutil cp -r --parallel 10 --part-size 100M`). Or use Data Online Migration for server-side transfer | | Large file upload timeout | File exceeds single upload timeout threshold | Use multipart upload (automatic with ossutil for files >100MB). Increase timeout: `ossutil config --timeout 3600` | | Object count mismatch after migration | Hidden objects, versioned objects, or delete markers not migrated | Enable versioning on source listing. Check for delete markers. Re-run with `--include-all-versions` flag or use Data Online Migration | | `InvalidObjectState` | Source object in Glacier/cold storage tier | Restore object from Glacier before migration. ossutil cannot directly migrate objects in cold storage | | `NoSuchBucket` | Source S3 bucket does not exist or name typo | Verify bucket name and region. Check bucket exists: `aws s3 ls s3:// --region ` | | MD5 checksum mismatch | Object corrupted during transfer | Re-transfer affected objects. Use `ossutil cp --checksum` to verify integrity. Check network stability | | `RequestTimeTooSkewed` | Source server clock significantly out of sync | Synchronize source server clock with NTP. Alibaba Cloud requires clock skew < 15 minutes | --- ## 5. Function Compute (FC) Errors | Error Pattern | Cause | Remediation | |---|---|---| | `InvalidArgument.handler` | Handler format doesn't match FC convention | FC handler format: `index.handler` (file.function). Adjust from Lambda format if different. Check `terraform-providers/alicloud.md` for FC resource specs | | `ServiceNotFound` / `FunctionNotFound` | Wrong service or function name in invocation | Verify service and function names match Terraform resource names. List functions: `aliyun fc list-functions --service-name --user-agent AlibabaCloud-Agent-Skills` | | Timeout during invocation | Function execution timeout too low for workload | Increase `timeout` in Terraform config (FC max: 600s for on-demand, 86400s for async tasks). Consider async invocation for long-running tasks | | `FunctionNotFound` after terraform apply | Using deprecated resource type | Use `alicloud_fcv3_function` (not deprecated `alicloud_fc_function`). Check `terraform-providers/alicloud.md` for current resource types | | Permission denied accessing OSS/RDS from FC | Missing RAM role or service-linked role | Create service-linked role for FC with access to required resources. Attach `AliyunFCDefaultRole` or custom policy with specific resource permissions | | `InvalidArgument.Runtime` | Runtime version not supported by FC | Check supported runtimes in FC console. Common: `python3.9`, `nodejs18`, `java11`, `custom-container`. Update Terraform config accordingly | | `CodeSizeExceeded` | Deployment package exceeds size limit | FC code size limit: 50MB (zip), 500MB (layer). Use layers for dependencies, or store large assets in OSS and download at runtime | | `ConcurrentInvocationExceeded` | Function concurrent execution limit reached | Request quota increase for concurrent invocations. Implement request queuing or increase provisioned concurrency | | `ServiceQuotaExceeded` | Account exceeded FC resource quotas | Check quotas in FC console. Request increase for: functions per service, memory per function, or total storage | --- ## 6. DNS & CDN Errors | Error Pattern | Cause | Remediation | |---|---|---| | `DomainNotFound` | Domain not added to Alibaba Cloud DNS | Add domain first via `alicloud_dns_domain` resource or DNS console. Verify domain ownership if required | | DNS propagation delay causing downtime | TTL not expired before cutover | Lower TTL to 60s at least 24-48 hours before migration. Wait for old TTL to expire, then migrate. Raise TTL after verification | | CDN `OriginNotAccessible` | Origin server not reachable from Alibaba Cloud CDN nodes | Check origin server firewall/security group allows Alibaba Cloud CDN IP ranges. Add health check endpoint | | Certificate error on CDN domain | SSL certificate not uploaded or associated with CDN domain | Upload certificate to Certificate Management Service, then associate with CDN domain. Ensure cert matches domain exactly | | `DomainAlreadyExists` | Domain already configured in another Alibaba Cloud account | Domain must be transferred or removed from other account. Contact Alibaba Cloud support if account access unavailable | | `RecordConflict` | DNS record conflicts with existing record | Delete or modify conflicting record. CNAME cannot coexist with other records at same name | | CDN cache not refreshing after origin update | Cache TTL not expired or purge not triggered | Manually purge CDN cache via console or API: `aliyun cdn RefreshObjectCaches --ObjectPath --user-agent AlibabaCloud-Agent-Skills` | | `Forbidden.Domain` | Domain blocked or requires ICP filing | For China regions, ensure domain has valid ICP filing. Check domain status in ICP management console | --- ## Quick Reference: Error → Phase Mapping | If you see... | Check phase... | |---|---| | `RAM`, `Forbidden`, `Access Denied`, `Unauthorized` | All phases — RAM permission issue | | `SMC`, `ReplicationJob`, `SourceServer`, `Go2Aliyun` | Phase 3: Server Migration | | `DTS`, `MigrationJob`, `binlog`, `sync`, `precheck` | Phase 4: Database Migration | | `OSS`, `Bucket`, `Object`, `ossutil` | Phase 5: Storage Migration | | `FC`, `Function`, `Service`, `handler`, `runtime` | Phase 6: Serverless Migration | | `DNS`, `Domain`, `CDN`, `CNAME`, `TTL` | Phase 7: DNS & CDN | | `Terraform`, `STATE_ID`, `HCL`, `Provider` | All phases — Terraform/IaCService issue | | `QuotaExceeded`, `LimitExceeded` | All phases — Resource quota issue | | `InvalidParameter`, `InvalidArgument` | All phases — Configuration error | | `Timeout`, `ConnectionRefused`, `NetworkUnreachable` | All phases — Network/connectivity issue | --- ## General Troubleshooting Principles ### 1. Check Logs First - **Terraform**: Review IaCService task logs in console - **SMC**: Check `/root/go2aliyun_client.log` on source server - **DTS**: View task logs in DTS console → Task Details → Logs - **OSS**: Use `ossutil ls oss://bucket --all-versions` for detailed listing - **FC**: Check function logs in Log Service (SLS) linked to FC ### 2. Verify Permissions Most `Forbidden` or `AccessDenied` errors stem from RAM policy gaps. Reference `ram-policies.md` for required policies per service. ### 3. Check Resource State Before retrying operations, verify current resource state: ```bash # ECS instance status aliyun ecs DescribeInstances \ --InstanceIds '["i-xxx"]' \ --user-agent AlibabaCloud-Agent-Skills # DTS job status aliyun dts DescribeMigrationJobs \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills # SMC job status aliyun smc DescribeReplicationJobs \ --ReplicationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 4. Use Dry-Run When Available - Terraform: `terraform plan` before `apply` - DTS: Run pre-check before starting migration - SMC: Validate migration job before execution ### 5. Contact Support When - Error persists after following remediation steps - Quota increase required - Cross-account or cross-region complexity - Data consistency issues after migration --- ## Related Documents - `verification-method.md` — Success verification commands and expected outputs - `ram-policies.md` — Required RAM policies for migration operations - `terraform-providers/alicloud.md` — Terraform provider reference - `terraform-online-runtime.md` — IaCService Terraform execution guide FILE:references/migration-guides/database-migration-dts.md # Database Migration with DTS (Data Transmission Service) ## Overview Alibaba Cloud Data Transmission Service (DTS) provides comprehensive database migration capabilities supporting: - **Schema Migration** — Automatically migrate database schemas (tables, indexes, constraints, views, stored procedures) - **Full Data Migration** — Migrate existing data from source to destination - **Incremental Data Synchronization** — Capture and replicate ongoing changes during migration for minimal downtime DTS supports homogeneous migrations (same database engine) and heterogeneous migrations (different engines) with automatic type mapping. ## Supported Scenarios | Source | Destination | Supported Migration Types | |--------|-------------|--------------------------| | Amazon RDS MySQL | ApsaraDB RDS MySQL | Schema + Full + Incremental | | Amazon RDS PostgreSQL | ApsaraDB RDS PostgreSQL | Schema + Full + Incremental | | Amazon RDS SQL Server | ApsaraDB RDS SQL Server | Schema + Full + Incremental | | Amazon Aurora MySQL | ApsaraDB RDS MySQL | Schema + Full + Incremental | | Amazon RDS MySQL | ApsaraDB RDS PostgreSQL | Schema + Full + Incremental (heterogeneous) | | Self-managed MySQL on EC2 | ApsaraDB RDS MySQL | Schema + Full + Incremental | | Self-managed PostgreSQL on EC2 | ApsaraDB RDS PostgreSQL | Schema + Full + Incremental | ## Prerequisites ### Source Database (Amazon RDS) - **Publicly Accessible**: Set to `Yes` in RDS configuration - **Security Group**: Allow inbound connections from DTS server IP ranges - **Database User**: Create a dedicated migration user with sufficient privileges: ```sql -- MySQL CREATE USER 'dts_user'@'%' IDENTIFIED BY 'password'; GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'dts_user'@'%'; FLUSH PRIVILEGES; -- PostgreSQL CREATE USER dts_user WITH PASSWORD 'password'; GRANT rds_superuser TO dts_user; ``` - **Binary Logging**: Enable for MySQL (required for incremental sync) ```sql -- Check binary log status SHOW VARIABLES LIKE 'log_bin'; -- Should return: ON ``` ### Destination Database (ApsaraDB RDS) - **Instance Created**: Provision ApsaraDB RDS instance with sufficient storage - **Version Compatibility**: Destination version >= source version recommended - **Whitelist Configuration**: Add DTS server IP ranges to RDS whitelist - **Storage**: Ensure destination has at least 1.5x source database size ### Network Requirements - **DTS IP Ranges**: Whitelist Alibaba Cloud DTS server IP ranges in source security group - Check current DTS IP ranges in DTS console or documentation - IP ranges vary by region - **Connectivity Test**: Verify network connectivity before starting migration ## Step 1: Purchase DTS Instance Create a DTS migration job instance: ```bash aliyun dts CreateMigrationJob \ --RegionId \ --MigrationJobClass medium \ --MigrationJobName "aws-rds-to-alicloud-migration" \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `RegionId` | Yes | Region where DTS instance will be created | `cn-hangzhou`, `ap-southeast-1` | | `MigrationJobClass` | Yes | Instance specification (micro, small, medium, large, xlarge) | `medium` | | `MigrationJobName` | No | Descriptive name for the migration job | `aws-rds-to-alicloud-migration` | **Response:** ```json { "MigrationJobId": "dts-xxxxxxxxxxxxx", "RequestId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" } ``` **Save the `MigrationJobId`** — required for subsequent steps. ### 1.1 Terraform Alternative for RDS Instance ```hcl resource "alicloud_db_instance" "migration" { engine = "" engine_version = "" instance_type = "" instance_storage = instance_charge_type = "Postpaid" instance_name = "" vswitch_id = "" security_group_ids = [""] db_instance_storage_type = "cloud_essd" } ``` **Note:** Use Terraform for RDS instance creation. DTS migration operations (`CreateMigrationJob`, `ConfigureMigrationJob`, `StartMigrationJob`) have no Terraform equivalent and must use CLI. ## Step 2: Configure Migration Task Configure source and destination endpoints, migration objects, and migration types: ```bash aliyun dts ConfigureMigrationJob \ --MigrationJobId \ --MigrationJobName "aws-rds-to-alicloud" \ --SourceEndpoint.InstanceType other \ --SourceEndpoint.EngineName MySQL \ --SourceEndpoint.IP \ --SourceEndpoint.Port 3306 \ --SourceEndpoint.UserName \ --SourceEndpoint.Password \ --SourceEndpoint.DatabaseName \ --DestinationEndpoint.InstanceType RDS \ --DestinationEndpoint.InstanceID \ --DestinationEndpoint.EngineName MySQL \ --MigrationMode.StructureIntialization true \ --MigrationMode.DataIntialization true \ --MigrationMode.DataSynchronization true \ --MigrationObject '[{"DBName":"mydb","SchemaName":"mydb","TableIncludes":[{"TableName":"*"}]}]' \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `MigrationJobId` | Yes | DTS job ID from Step 1 | `dts-xxxxxxxxxxxxx` | | `SourceEndpoint.InstanceType` | Yes | Source type (`other` for external databases) | `other` | | `SourceEndpoint.EngineName` | Yes | Source database engine | `MySQL`, `PostgreSQL`, `SQLServer` | | `SourceEndpoint.IP` | Yes | Source RDS endpoint | `mydb.xxxxxx.us-east-1.rds.amazonaws.com` | | `SourceEndpoint.Port` | Yes | Source database port | `3306` (MySQL), `5432` (PostgreSQL) | | `SourceEndpoint.UserName` | Yes | Migration user | `dts_user` | | `SourceEndpoint.Password` | Yes | Migration user password | `` | | `DestinationEndpoint.InstanceType` | Yes | Destination type (`RDS` for ApsaraDB) | `RDS` | | `DestinationEndpoint.InstanceID` | Yes | Destination RDS instance ID | `rm-xxxxxxxxxxxxx` | | `DestinationEndpoint.EngineName` | Yes | Destination database engine | `MySQL`, `PostgreSQL` | | `MigrationMode.StructureIntialization` | Yes | Enable schema migration | `true` or `false` | | `MigrationMode.DataIntialization` | Yes | Enable full data migration | `true` or `false` | | `MigrationMode.DataSynchronization` | Yes | Enable incremental sync | `true` or `false` | | `MigrationObject` | Yes | JSON array of migration objects | See below | **MigrationObject Format:** ```json [ { "DBName": "mydb", "SchemaName": "mydb", "TableIncludes": [ {"TableName": "*"} ] } ] ``` **Selective Table Migration:** ```json [ { "DBName": "mydb", "SchemaName": "mydb", "TableIncludes": [ {"TableName": "users"}, {"TableName": "orders"}, {"TableName": "products"} ] } ] ``` ## Step 3: Start Migration After configuration passes pre-check, start the migration job: ```bash aliyun dts StartMigrationJob \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` **Pre-check Items:** - Source database connectivity - Destination database connectivity - Source database permissions - Destination database permissions - Binary log status (for incremental sync) - Storage space on destination **If pre-check fails:** - Review error details in DTS console - Fix issues (permissions, network, etc.) - Re-run pre-check before starting ## Step 4: Monitor Progress Monitor migration job status and progress: ```bash aliyun dts DescribeMigrationJobStatus \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` **Response:** ```json { "MigrationJobStatus": { "MigrationJobId": "dts-xxxxxxxxxxxxx", "Status": "Synchronizing", "MigrationJobName": "aws-rds-to-alicloud", "Progress": { "StructureIntialization": { "Status": "Finished", "Progress": "100%" }, "DataIntialization": { "Status": "Finished", "Progress": "100%" }, "DataSynchronization": { "Status": "Synchronizing", "Progress": "95%", "Delay": "2" } }, "CreateTime": "2024-01-15T10:30:00Z" } } ``` **Status Values:** - `NotStarted` — Job created but not started - `Prechecking` — Running pre-checks - `Failed` — Pre-check or migration failed - `Initializing` — Performing initial full data sync - `Synchronizing` — Incremental sync in progress - `Suspending` — Job paused - `Finished` — Migration completed **Monitor Delay:** - `Delay` field shows replication lag in seconds - Wait for delay to stabilize at low values (< 10 seconds) before cutover ## Step 5: Cutover ### 5.1 Stop Application Writes - Put application in maintenance mode - Stop all write operations to source database - Wait for DTS to catch up (delay = 0) ### 5.2 Verify Data Consistency ```bash # Check row counts on source and destination # Source (MySQL) mysql -h -u -p -e "SELECT COUNT(*) FROM mydb.users;" # Destination (ApsaraDB RDS) mysql -h -u -p -e "SELECT COUNT(*) FROM mydb.users;" ``` ### 5.3 Stop DTS Job ```bash aliyun dts StopMigrationJob \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 5.4 Update Application Configuration - Update database connection strings to point to ApsaraDB RDS endpoint - Update environment variables or configuration files - Restart application services ### 5.5 Verify Application Functionality - Test read operations - Test write operations - Verify data integrity in production ## Error Handling ### Common Errors and Solutions | Error | Cause | Solution | |-------|-------|----------| | `Connection refused` | Security group blocking DTS IP | Add DTS IP ranges to source security group | | `Access denied for user` | Insufficient permissions | Grant required permissions to migration user | | `Binary log not enabled` | MySQL binary logging disabled | Enable binary log in RDS parameter group | | `Schema conflict` | Table already exists on destination | Choose "Ignore Errors" or clean destination schema | | `Data type mismatch` | Incompatible data types between engines | Review type mapping, adjust schema if needed | | `Network timeout` | Network latency or firewall | Check network connectivity, increase timeout | | `Insufficient storage` | Destination storage full | Expand RDS storage capacity | ### View Error Details ```bash aliyun dts DescribeMigrationJobStatus \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` Check `ErrorMessage` and `ErrorDetails` fields in response. ### Retry Failed Jobs ```bash # Resume migration after fixing issues aliyun dts StartMigrationJob \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` ## DTS Reverse Sync (Database Rollback) DTS supports **bidirectional synchronization** — if issues are discovered after cutover, you can create a reverse sync task (Alibaba Cloud → AWS) to stream changes back to the source database and revert traffic. ### When to Use - Post-cutover issues detected (data corruption, application incompatibility) - Need to fall back to AWS RDS while keeping data written to ApsaraDB in sync - Parallel-run validation: run both databases simultaneously and compare results ### Step 1: Create Reverse Sync Task ```bash aliyun dts CreateMigrationJob \ --RegionId \ --MigrationJobClass medium \ --MigrationJobName "alicloud-to-aws-reverse-sync" \ --user-agent AlibabaCloud-Agent-Skills ``` ### Step 2: Configure Reverse Direction Swap source and destination — ApsaraDB RDS becomes source, AWS RDS becomes destination: ```bash aliyun dts ConfigureMigrationJob \ --MigrationJobId \ --MigrationJobName "alicloud-to-aws-reverse-sync" \ --SourceEndpoint.InstanceType RDS \ --SourceEndpoint.InstanceID \ --SourceEndpoint.EngineName MySQL \ --DestinationEndpoint.InstanceType other \ --DestinationEndpoint.EngineName MySQL \ --DestinationEndpoint.IP \ --DestinationEndpoint.Port 3306 \ --DestinationEndpoint.UserName \ --DestinationEndpoint.Password \ --DestinationEndpoint.DatabaseName \ --MigrationMode.StructureIntialization false \ --MigrationMode.DataIntialization false \ --MigrationMode.DataSynchronization true \ --MigrationObject '[{"DBName":"mydb","SchemaName":"mydb","TableIncludes":[{"TableName":"*"}]}]' \ --user-agent AlibabaCloud-Agent-Skills ``` > **Note:** Schema and full data initialization are set to `false` — the source schema already exists on AWS. Only incremental sync is needed. ### Step 3: Start Reverse Sync and Monitor ```bash aliyun dts StartMigrationJob \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills # Monitor until Delay stabilizes at 0 aliyun dts DescribeMigrationJobStatus \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` ### Step 4: Execute Rollback Once reverse sync is caught up (`Delay: 0`): 1. Switch application traffic back to AWS RDS (DNS rollback) 2. Verify application works correctly with AWS RDS 3. Stop the reverse sync job 4. Stop the forward sync job (if still running) ### Prerequisites for Reverse Sync | Requirement | Details | |-------------|---------| | ApsaraDB RDS binary logging | Must be enabled (default for most versions) | | AWS RDS network access | ApsaraDB must be able to reach AWS RDS (public endpoint or VPN) | | AWS RDS user permissions | Same as forward migration: `SELECT, REPLICATION SLAVE, REPLICATION CLIENT` | | AWS RDS security group | Allow inbound from DTS IP ranges | ### Limitations - Reverse sync adds extra cost (additional DTS instance) - DDL changes during reverse sync may cause conflicts - Schema changes on either side after initial migration require manual resolution - Reverse sync only captures changes made **after** the reverse task starts — not retroactive ## Cleanup ### Delete DTS Migration Job ```bash aliyun dts DeleteMigrationJob \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` ### Delete DTS Instance (if no longer needed) ```bash aliyun dts DeleteMigrationJob \ --MigrationJobId \ --user-agent AlibabaCloud-Agent-Skills ``` ### Remove Source Database Access - Revoke DTS user permissions on source RDS - Remove DTS IP ranges from security group - Delete migration user if no longer needed ```sql -- MySQL REVOKE ALL PRIVILEGES ON *.* FROM 'dts_user'@'%'; DROP USER 'dts_user'@'%'; FLUSH PRIVILEGES; -- PostgreSQL REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM dts_user; DROP USER dts_user; ``` ### Decommission Source RDS (Optional) - Create final snapshot before deletion - Delete RDS instance after successful migration verification - Release Elastic IP if associated ## Best Practices 1. **Test Migration First** - Run a test migration with a subset of data - Validate schema, data, and application compatibility - Estimate migration duration 2. **Schedule Maintenance Window** - Plan cutover during low-traffic periods - Communicate downtime expectations to stakeholders - Prepare rollback plan 3. **Monitor Continuously** - Set up CloudWatch alarms for DTS metrics - Monitor replication lag during incremental sync - Track migration progress regularly 4. **Optimize Performance** - Choose appropriate DTS instance class based on data volume - Use multiple migration jobs for large databases (split by schema) - Enable compression for network transfer 5. **Data Validation** - Use DTS built-in data validation feature - Compare row counts and checksums post-migration - Run application-level validation tests 6. **Incremental Sync Strategy** - Start incremental sync days before cutover - Monitor and reduce replication lag over time - Plan cutover when lag is consistently minimal ## Related APIs | API Action | Description | CLI Command | |------------|-------------|-------------| | `CreateMigrationJob` | Create DTS migration job | `aliyun dts CreateMigrationJob ... --user-agent AlibabaCloud-Agent-Skills` | | `ConfigureMigrationJob` | Configure migration job | `aliyun dts ConfigureMigrationJob ... --user-agent AlibabaCloud-Agent-Skills` | | `StartMigrationJob` | Start migration job | `aliyun dts StartMigrationJob ... --user-agent AlibabaCloud-Agent-Skills` | | `StopMigrationJob` | Stop migration job | `aliyun dts StopMigrationJob ... --user-agent AlibabaCloud-Agent-Skills` | | `DescribeMigrationJobStatus` | Get migration job status | `aliyun dts DescribeMigrationJobStatus ... --user-agent AlibabaCloud-Agent-Skills` | | `DeleteMigrationJob` | Delete migration job | `aliyun dts DeleteMigrationJob ... --user-agent AlibabaCloud-Agent-Skills` | ## References - [DTS Product Documentation](https://www.alibabacloud.com/help/en/dts) - [DTS API Reference](https://www.alibabacloud.com/help/en/dts/api-reference) - [Database Migration Best Practices](https://www.alibabacloud.com/help/en/dts/user-guide/migration-best-practices) - [Supported Databases](https://www.alibabacloud.com/help/en/dts/user-guide/supported-databases) FILE:references/migration-guides/network-migration.md # Network Migration Reference Complete guide for setting up Alibaba Cloud networking during AWS migration. ## 1. VPC Setup Create VPC and VSwitch infrastructure for migration. ### 1.1 Create VPC ```bash aliyun vpc CreateVpc \ --RegionId \ --CidrBlock 10.0.0.0/8 \ --VpcName migration-vpc \ --Description "VPC for AWS migration" \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `RegionId` | Yes | Region where VPC is created | `cn-hangzhou` | | `CidrBlock` | Yes | VPC CIDR block (must be /8 to /24) | `10.0.0.0/8` | | `VpcName` | No | VPC name | `migration-vpc` | | `Description` | No | VPC description | `VPC for AWS migration` | **Response:** ```json { "VpcId": "vpc-bp1abc123def456789", "VRouterId": "vrt-bp1abc123def456789", "RouteTableId": "vtb-bp1abc123def456789" } ``` ### 1.2 Create VSwitch ```bash aliyun vpc CreateVSwitch \ --RegionId \ --VpcId \ --ZoneId \ --CidrBlock 10.0.0.0/24 \ --VSwitchName migration-vsw \ --Description "VSwitch for migration" \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `RegionId` | Yes | Region where VSwitch is created | `cn-hangzhou` | | `VpcId` | Yes | VPC ID from CreateVpc | `vpc-bp1abc123def456789` | | `ZoneId` | Yes | Availability zone | `cn-hangzhou-i` | | `CidrBlock` | Yes | VSwitch CIDR (must be within VPC CIDR) | `10.0.0.0/24` | | `VSwitchName` | No | VSwitch name | `migration-vsw` | | `Description` | No | VSwitch description | `VSwitch for migration` | **Response:** ```json { "VSwitchId": "vsw-bp1abc123def456789" } ``` ### 1.3 Create Multiple VSwitches (Multi-AZ) For high availability, create VSwitches in multiple zones: ```bash # VSwitch in Zone A aliyun vpc CreateVSwitch \ --RegionId cn-hangzhou \ --VpcId vpc-bp1abc123def456789 \ --ZoneId cn-hangzhou-i \ --CidrBlock 10.0.1.0/24 \ --VSwitchName migration-vsw-a \ --user-agent AlibabaCloud-Agent-Skills # VSwitch in Zone B aliyun vpc CreateVSwitch \ --RegionId cn-hangzhou \ --VpcId vpc-bp1abc123def456789 \ --ZoneId cn-hangzhou-j \ --CidrBlock 10.0.2.0/24 \ --VSwitchName migration-vsw-b \ --user-agent AlibabaCloud-Agent-Skills ``` ### 1.4 Describe VPC ```bash aliyun vpc DescribeVpcs \ --RegionId \ --VpcId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 1.5 Describe VSwitches ```bash aliyun vpc DescribeVSwitches \ --RegionId \ --VpcId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 1.6 Terraform Alternative for VPC and VSwitch ```hcl resource "alicloud_vpc" "migration" { vpc_name = "" cidr_block = "" } resource "alicloud_vswitch" "migration" { vpc_id = alicloud_vpc.migration.id cidr_block = "" zone_id = "" vswitch_name = "" } resource "alicloud_security_group" "migration" { name = "" vpc_id = alicloud_vpc.migration.id } resource "alicloud_security_group_rule" "ingress_ssh" { type = "ingress" ip_protocol = "tcp" port_range = "22/22" cidr_ip = "0.0.0.0/0" security_group_id = alicloud_security_group.migration.id priority = 1 } resource "alicloud_security_group_rule" "ingress_http" { type = "ingress" ip_protocol = "tcp" port_range = "80/80" cidr_ip = "0.0.0.0/0" security_group_id = alicloud_security_group.migration.id priority = 1 } resource "alicloud_security_group_rule" "ingress_https" { type = "ingress" ip_protocol = "tcp" port_range = "443/443" cidr_ip = "0.0.0.0/0" security_group_id = alicloud_security_group.migration.id priority = 1 } ``` ## 2. Security Group Create and configure security groups for migrated ECS instances. ### 2.1 Create Security Group ```bash aliyun ecs CreateSecurityGroup \ --RegionId \ --VpcId \ --SecurityGroupName migration-sg \ --Description "Security group for migrated ECS" \ --user-agent AlibabaCloud-Agent-Skills ``` **Response:** ```json { "SecurityGroupId": "sg-bp1abc123def456789" } ``` ### 2.2 Ingress Rules (Allow Inbound Traffic) **Allow SSH (Linux):** ```bash aliyun ecs AuthorizeSecurityGroup \ --RegionId \ --SecurityGroupId \ --IpProtocol tcp \ --PortRange 22/22 \ --SourceCidrIp 0.0.0.0/0 \ --Description "Allow SSH" \ --user-agent AlibabaCloud-Agent-Skills ``` **Allow RDP (Windows):** ```bash aliyun ecs AuthorizeSecurityGroup \ --RegionId \ --SecurityGroupId \ --IpProtocol tcp \ --PortRange 3389/3389 \ --SourceCidrIp 0.0.0.0/0 \ --Description "Allow RDP" \ --user-agent AlibabaCloud-Agent-Skills ``` **Allow HTTP:** ```bash aliyun ecs AuthorizeSecurityGroup \ --RegionId \ --SecurityGroupId \ --IpProtocol tcp \ --PortRange 80/80 \ --SourceCidrIp 0.0.0.0/0 \ --Description "Allow HTTP" \ --user-agent AlibabaCloud-Agent-Skills ``` **Allow HTTPS:** ```bash aliyun ecs AuthorizeSecurityGroup \ --RegionId \ --SecurityGroupId \ --IpProtocol tcp \ --PortRange 443/443 \ --SourceCidrIp 0.0.0.0/0 \ --Description "Allow HTTPS" \ --user-agent AlibabaCloud-Agent-Skills ``` **Allow ICMP (Ping):** ```bash aliyun ecs AuthorizeSecurityGroup \ --RegionId \ --SecurityGroupId \ --IpProtocol icmp \ --PortRange -1/-1 \ --SourceCidrIp 0.0.0.0/0 \ --Description "Allow ICMP" \ --user-agent AlibabaCloud-Agent-Skills ``` ### 2.3 Egress Rules (Allow Outbound Traffic) By default, all outbound traffic is allowed. To restrict: ```bash aliyun ecs AuthorizeSecurityGroupEgress \ --RegionId \ --SecurityGroupId \ --IpProtocol tcp \ --PortRange 443/443 \ --DestCidrIp 0.0.0.0/0 \ --Description "Allow HTTPS outbound" \ --user-agent AlibabaCloud-Agent-Skills ``` ### 2.4 Describe Security Group Rules ```bash aliyun ecs DescribeSecurityGroupAttribute \ --RegionId \ --SecurityGroupId \ --user-agent AlibabaCloud-Agent-Skills ``` ## 3. VPN Gateway Set up VPN Gateway for hybrid connectivity during migration. ### 3.1 Create VPN Gateway ```bash aliyun vpc CreateVpnGateway \ --RegionId \ --VpcId \ --Bandwidth \ --VpnGatewayName migration-vpngw \ --Description "VPN Gateway for AWS migration" \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `RegionId` | Yes | Region | `cn-hangzhou` | | `VpcId` | Yes | VPC ID | `vpc-bp1abc123def456789` | | `Bandwidth` | Yes | VPN bandwidth in Mbps (10, 20, 50, 100, 200, 500) | `50` | | `VpnGatewayName` | No | VPN Gateway name | `migration-vpngw` | | `Description` | No | Description | `VPN Gateway for AWS migration` | **Response:** ```json { "VpnGatewayId": "vpn-bp1abc123def456789", "OrderInstanceId": "order-bp1abc123def456789" } ``` ### 3.2 Create Customer Gateway (AWS Side) ```bash aliyun vpc CreateCustomerGateway \ --RegionId \ --IpAddress \ --Name aws-customer-gw \ --Description "AWS VPN Gateway" \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `RegionId` | Yes | Region | `cn-hangzhou` | | `IpAddress` | Yes | Public IP of AWS VPN Gateway | `54.123.45.67` | | `Name` | No | Customer Gateway name | `aws-customer-gw` | | `Description` | No | Description | `AWS VPN Gateway` | **Response:** ```json { "CustomerGatewayId": "cgw-bp1abc123def456789" } ``` ### 3.3 Create IPsec Connection ```bash aliyun vpc CreateVpnConnection \ --RegionId \ --VpnGatewayId \ --CustomerGatewayId \ --Name aws-vpn-connection \ --Description "VPN connection to AWS" \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `RegionId` | Yes | Region | `cn-hangzhou` | | `VpnGatewayId` | Yes | VPN Gateway ID | `vpn-bp1abc123def456789` | | `CustomerGatewayId` | Yes | Customer Gateway ID | `cgw-bp1abc123def456789` | | `Name` | No | Connection name | `aws-vpn-connection` | | `Description` | No | Description | `VPN connection to AWS` | **Response:** ```json { "VpnConnectionId": "vco-bp1abc123def456789" } ``` ### 3.4 Configure IPsec Parameters ```bash aliyun vpc ModifyVpnConnectionAttribute \ --RegionId \ --VpnConnectionId \ --LocalSubnet 10.0.0.0/8 \ --RemoteSubnet 172.31.0.0/16 \ --IpsecConfig.IpsecEncAlg AES-128-CBC \ --IpsecConfig.IpsecAuthAlg SHA1 \ --IpsecConfig.IpsecLifetime 86400 \ --IpsecConfig.IpsecPfs group2 \ --IkeConfig.IkeEncAlg AES-128-CBC \ --IkeConfig.IkeAuthAlg SHA1 \ --IkeConfig.IkeLifetime 86400 \ --IkeConfig.IkePfs group2 \ --IkeConfig.IkeVersion IKEv1 \ --user-agent AlibabaCloud-Agent-Skills ``` ### 3.5 Describe VPN Connection ```bash aliyun vpc DescribeVpnConnection \ --RegionId \ --VpnConnectionId \ --user-agent AlibabaCloud-Agent-Skills ``` ## 4. Express Connect Dedicated connection for high-bandwidth migration. ### 4.1 Create Physical Connection ```bash aliyun vpc CreatePhysicalConnection \ --RegionId \ --AccessPointId \ --LineOperator \ --Bandwidth \ --Name migration-express \ --Description "Express Connect for migration" \ --user-agent AlibabaCloud-Agent-Skills ``` **Note:** Express Connect requires physical circuit setup through Alibaba Cloud sales team. ### 4.2 Create Virtual Border Router (VBR) ```bash aliyun vpc CreateVirtualBorderRouter \ --RegionId \ --PhysicalConnectionId \ --VlanId \ --LocalGatewayIp \ --PeerGatewayIp \ --PeeringSubnetMask \ --Name migration-vbr \ --user-agent AlibabaCloud-Agent-Skills ``` ### 4.3 Create CEN Instance ```bash aliyun cen CreateCen \ --CenName migration-cen \ --Description "CEN for Express Connect" \ --user-agent AlibabaCloud-Agent-Skills ``` ### 4.4 Attach VPC to CEN ```bash aliyun cen AttachCenChildInstance \ --CenId \ --ChildInstanceId \ --ChildInstanceType VPC \ --ChildInstanceRegionId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 4.5 Attach VBR to CEN ```bash aliyun cen AttachCenChildInstance \ --CenId \ --ChildInstanceId \ --ChildInstanceType VBR \ --ChildInstanceRegionId \ --user-agent AlibabaCloud-Agent-Skills ``` ## 5. CEN (Cloud Enterprise Network) Multi-region and cross-cloud networking. ### 5.1 Create CEN Instance ```bash aliyun cen CreateCen \ --CenName global-migration-cen \ --Description "Global CEN for multi-region migration" \ --user-agent AlibabaCloud-Agent-Skills ``` **Response:** ```json { "CenId": "cen-bp1abc123def456789" } ``` ### 5.2 Attach VPCs to CEN ```bash aliyun cen AttachCenChildInstance \ --CenId \ --ChildInstanceId \ --ChildInstanceType VPC \ --ChildInstanceRegionId cn-hangzhou \ --user-agent AlibabaCloud-Agent-Skills aliyun cen AttachCenChildInstance \ --CenId \ --ChildInstanceId \ --ChildInstanceType VPC \ --ChildInstanceRegionId cn-shanghai \ --user-agent AlibabaCloud-Agent-Skills ``` ### 5.3 Publish Routes to CEN ```bash aliyun cen PublishRouteEntriesToCen \ --CenId \ --ChildInstanceId \ --ChildInstanceType VPC \ --ChildInstanceRegionId \ --RouteTableId \ --DestinationCidrBlock 10.0.0.0/8 \ --user-agent AlibabaCloud-Agent-Skills ``` ### 5.5 Describe CEN ```bash aliyun cen DescribeCens \ --CenId \ --user-agent AlibabaCloud-Agent-Skills ``` ## 6. DNS Migration Migrate from Route 53 to Alibaba Cloud DNS. ### 6.1 Export Route 53 Zone ```bash # Using AWS CLI aws route53 list-resource-record-sets \ --hosted-zone-id \ --output json > route53-records.json ``` ### 6.2 Create DNS Domain ```bash aliyun alidns AddDomain \ --DomainName example.com \ --user-agent AlibabaCloud-Agent-Skills ``` **Response:** ```json { "DomainId": "12345678", "DomainName": "example.com" } ``` ### 6.3 Add DNS Records **A Record:** ```bash aliyun alidns AddDomainRecord \ --DomainName example.com \ --RR @ \ --Type A \ --Value \ --user-agent AlibabaCloud-Agent-Skills ``` **CNAME Record:** ```bash aliyun alidns AddDomainRecord \ --DomainName example.com \ --RR www \ --Type CNAME \ --Value example.com \ --user-agent AlibabaCloud-Agent-Skills ``` **MX Record:** ```bash aliyun alidns AddDomainRecord \ --DomainName example.com \ --RR @ \ --Type MX \ --Value 10 mx.example.com \ --user-agent AlibabaCloud-Agent-Skills ``` **TXT Record:** ```bash aliyun alidns AddDomainRecord \ --DomainName example.com \ --RR @ \ --Type TXT \ --Value "v=spf1 include:spf.example.com ~all" \ --user-agent AlibabaCloud-Agent-Skills ``` ### 6.4 Describe DNS Records ```bash aliyun alidns DescribeDomainRecords \ --DomainName example.com \ --user-agent AlibabaCloud-Agent-Skills ``` ### 6.5 Terraform Alternative for DNS Records ```hcl resource "alicloud_alidns_record" "www" { domain_name = "" rr = "" type = "" value = "" ttl = } ``` **Example - A Record:** ```hcl resource "alicloud_alidns_record" "www_a" { domain_name = "example.com" rr = "www" type = "A" value = "" ttl = 600 } ``` **Example - CNAME Record:** ```hcl resource "alicloud_alidns_record" "www_cname" { domain_name = "example.com" rr = "blog" type = "CNAME" value = "www.example.com" ttl = 600 } ``` **Example - MX Record:** ```hcl resource "alicloud_alidns_record" "mx" { domain_name = "example.com" rr = "@" type = "MX" value = "10 mx.example.com" ttl = 600 } ``` **Example - TXT Record:** ```hcl resource "alicloud_alidns_record" "txt" { domain_name = "example.com" rr = "@" type = "TXT" value = "v=spf1 include:spf.example.com ~all" ttl = 600 } ``` ### 6.5 Update Name Servers 1. Get Alibaba Cloud DNS name servers: ```bash aliyun alidns DescribeDomainInfo \ --DomainName example.com \ --user-agent AlibabaCloud-Agent-Skills ``` 2. Update domain registrar with new name servers 3. Lower TTL on Route 53 records before migration (e.g., 300 seconds) 4. Monitor DNS propagation ## 7. CDN Migration Migrate from CloudFront to Alibaba Cloud CDN. ### 7.1 Add CDN Domain ```bash aliyun cdn AddCdnDomain \ --DomainName www.example.com \ --SourceType oss \ --SourceContent .oss-.aliyuncs.com \ --Scope global \ --CertType free \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `DomainName` | Yes | CDN domain name | `www.example.com` | | `SourceType` | Yes | Origin type: `oss`, `ipaddr`, `domain` | `oss` | | `SourceContent` | Yes | Origin address | `my-bucket.oss-cn-hangzhou.aliyuncs.com` | | `Scope` | No | CDN scope: `global`, `domestic`, `overseas` | `global` | | `CertType` | No | Certificate type: `free`, `upload` | `free` | **Response:** ```json { "DomainId": "12345678", "DomainName": "www.example.com", "Cname": "www.example.com.w.kunlun.com" } ``` ### 7.2 Terraform Alternative for CDN Domain ```hcl resource "alicloud_cdn_domain_new" "migration" { domain_name = "" cdn_type = "" scope = "" sources { content = "" type = "" priority = "" } } ``` **Example - OSS Origin:** ```hcl resource "alicloud_cdn_domain_new" "oss_origin" { domain_name = "www.example.com" cdn_type = "web" scope = "global" sources { content = ".oss-.aliyuncs.com" type = "oss" priority = "20" } } ``` **Example - Custom Origin:** ```hcl resource "alicloud_cdn_domain_new" "custom_origin" { domain_name = "api.example.com" cdn_type = "web" scope = "global" sources { content = "" type = "ipaddr" priority = "20" } } ``` ### 7.2 Configure CDN Cache Rules ```bash aliyun cdn SetCdnDomainStagingConfig \ --DomainName www.example.com \ --Functions '[{"functionArgs":[{"argName":"cacheTTL","argValue":"3600"}],"functionName":"Cache"}]' \ --user-agent AlibabaCloud-Agent-Skills ``` ### 7.3 Configure CDN HTTPS ```bash aliyun cdn SetCdnDomainSSLCertificate \ --DomainName www.example.com \ --CertName my-cert \ --CertType upload \ --SSLProtocol TLSv1.2 \ --user-agent AlibabaCloud-Agent-Skills ``` ### 7.4 Start CDN Domain ```bash aliyun cdn StartCdnDomain \ --DomainName www.example.com \ --user-agent AlibabaCloud-Agent-Skills ``` ### 7.5 Describe CDN Domain ```bash aliyun cdn DescribeCdnDomainDetail \ --DomainName www.example.com \ --user-agent AlibabaCloud-Agent-Skills ``` ### 7.6 Update DNS to Point to CDN ```bash aliyun alidns UpdateDomainRecord \ --RecordId \ --RR www \ --Type CNAME \ --Value www.example.com.w.kunlun.com \ --user-agent AlibabaCloud-Agent-Skills ``` ## 8. Load Balancer Setup ### 8.1 Create SLB Instance ```bash aliyun slb CreateLoadBalancer \ --RegionId \ --LoadBalancerName migration-slb \ --AddressType internet \ --VpcId \ --VSwitchId \ --user-agent AlibabaCloud-Agent-Skills ``` **Response:** ```json { "LoadBalancerId": "lb-bp1abc123def456789", "Address": "47.100.123.45" } ``` ### 8.2 Create Listener ```bash aliyun slb CreateLoadBalancerHTTPListener \ --RegionId \ --LoadBalancerId \ --ListenerPort 80 \ --BackendServerPort 80 \ --Scheduler wrr \ --user-agent AlibabaCloud-Agent-Skills ``` ### 8.3 Add Backend Servers ```bash aliyun slb AddBackendServers \ --RegionId \ --LoadBalancerId \ --BackendServers '[{"ServerId":"i-bp1abc123","Weight":100},{"ServerId":"i-bp1def456","Weight":100}]' \ --user-agent AlibabaCloud-Agent-Skills ``` ## 9. Cleanup ### 9.1 Delete VPN Resources ```bash # Delete VPN Connection aliyun vpc DeleteVpnConnection \ --RegionId \ --VpnConnectionId \ --user-agent AlibabaCloud-Agent-Skills # Delete Customer Gateway aliyun vpc DeleteCustomerGateway \ --CustomerGatewayId \ --user-agent AlibabaCloud-Agent-Skills # Delete VPN Gateway aliyun vpc DeleteVpnGateway \ --RegionId \ --VpnGatewayId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 9.2 Delete CEN Resources ```bash # Detach VPC from CEN aliyun cen DetachCenChildInstance \ --CenId \ --ChildInstanceId \ --ChildInstanceType VPC \ --ChildInstanceRegionId \ --user-agent AlibabaCloud-Agent-Skills # Delete CEN aliyun cen DeleteCen \ --CenId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 9.3 Delete CDN Domain ```bash aliyun cdn DeleteCdnDomain \ --DomainName www.example.com \ --user-agent AlibabaCloud-Agent-Skills ``` ### 9.4 Delete DNS Records ```bash aliyun alidns DeleteDomainRecord \ --RecordId \ --user-agent AlibabaCloud-Agent-Skills # Delete domain aliyun alidns DeleteDomain \ --DomainName example.com \ --user-agent AlibabaCloud-Agent-Skills ``` ### 9.5 Delete SLB ```bash aliyun slb DeleteLoadBalancer \ --RegionId \ --LoadBalancerId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 9.6 Delete VSwitch ```bash aliyun vpc DeleteVSwitch \ --VSwitchId \ --user-agent AlibabaCloud-Agent-Skills ``` ### 9.7 Delete VPC ```bash aliyun vpc DeleteVpc \ --RegionId \ --VpcId \ --user-agent AlibabaCloud-Agent-Skills ``` ## 10. Best Practices 1. **Plan CIDR carefully**: Avoid overlap with existing networks 2. **Use multiple VSwitches**: Deploy across multiple AZs for HA 3. **Security group least privilege**: Only open required ports 4. **Test VPN before migration**: Verify connectivity before cutover 5. **Lower DNS TTL**: Reduce TTL 24-48 hours before migration 6. **Monitor CDN metrics**: Track cache hit ratio, bandwidth, requests 7. **Use CEN for multi-region**: Simplify network management 8. **Document network topology**: Keep updated network diagrams 9. **Test rollback**: Know how to revert DNS and network changes 10. **Clean up unused resources**: Delete VPN, CEN after migration complete ## 11. Related APIs | Category | API | CLI Command | |----------|-----|-------------| | VPC | CreateVpc | `aliyun vpc CreateVpc ... --user-agent AlibabaCloud-Agent-Skills` | | VPC | CreateVSwitch | `aliyun vpc CreateVSwitch ... --user-agent AlibabaCloud-Agent-Skills` | | VPC | DeleteVpc | `aliyun vpc DeleteVpc ... --user-agent AlibabaCloud-Agent-Skills` | | VPC | DeleteVSwitch | `aliyun vpc DeleteVSwitch ... --user-agent AlibabaCloud-Agent-Skills` | | Security Group | CreateSecurityGroup | `aliyun ecs CreateSecurityGroup ... --user-agent AlibabaCloud-Agent-Skills` | | Security Group | AuthorizeSecurityGroup | `aliyun ecs AuthorizeSecurityGroup ... --user-agent AlibabaCloud-Agent-Skills` | | VPN | CreateVpnGateway | `aliyun vpc CreateVpnGateway ... --user-agent AlibabaCloud-Agent-Skills` | | VPN | CreateCustomerGateway | `aliyun vpc CreateCustomerGateway ... --user-agent AlibabaCloud-Agent-Skills` | | VPN | CreateVpnConnection | `aliyun vpc CreateVpnConnection ... --user-agent AlibabaCloud-Agent-Skills` | | CEN | CreateCen | `aliyun cen CreateCen ... --user-agent AlibabaCloud-Agent-Skills` | | CEN | AttachCenChildInstance | `aliyun cen AttachCenChildInstance ... --user-agent AlibabaCloud-Agent-Skills` | | DNS | AddDomain | `aliyun alidns AddDomain ... --user-agent AlibabaCloud-Agent-Skills` | | DNS | AddDomainRecord | `aliyun alidns AddDomainRecord ... --user-agent AlibabaCloud-Agent-Skills` | | CDN | AddCdnDomain | `aliyun cdn AddCdnDomain ... --user-agent AlibabaCloud-Agent-Skills` | | SLB | CreateLoadBalancer | `aliyun slb CreateLoadBalancer ... --user-agent AlibabaCloud-Agent-Skills` | FILE:references/migration-guides/server-migration-importimage.md # Server Migration: EC2 → ECS via AMI Export + ImportImage Agent-free migration approach: export EC2 AMI to S3, transfer to OSS, import as ECS custom image, then provision ECS with Terraform. No agent installation on the source server required. ## 1. Overview ``` AWS Alibaba Cloud ────────────────────────────────────── ──────────────────────────────── EC2 Instance │ ▼ aws ec2 export-image AMI (.vmdk / .vhd) │ ▼ stored in S3 S3 Bucket ──── ossutil / Data Online ──→ OSS Bucket Migration │ ▼ aliyun ecs ImportImage ECS Custom Image │ ▼ Terraform ECS Instance ``` **Supported image formats**: VHD, VMDK, RAW, QCOW2 **Supported OS**: Linux (all mainstream distros), Windows Server 2008+ **System disk limit**: ≤ 500 GB ## 2. Prerequisites ### 2.1 AWS Side Requirements - AWS CLI configured with permissions: `ec2:ExportImage`, `ec2:DescribeExportImageTasks`, `s3:GetObject`, `s3:PutObject` - An S3 bucket to store the exported AMI ### 2.2 Alibaba Cloud RAM Permissions Attach to the RAM user performing migration: - `AliyunOSSFullAccess` — upload image file to OSS - `AliyunECSFullAccess` — import image and create ECS See [references/ram-policies.md](../ram-policies.md) for minimum custom policy. ### 2.3 Parameter Confirmation Confirm the following before starting (per [SKILL Parameter Confirmation rules](../../SKILL.md)): | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `SourceRegion` | Yes | AWS source region | `us-east-1` | | `TargetRegionId` | Yes | Alibaba Cloud target region | `cn-hangzhou` | | `S3BucketName` | Yes | S3 bucket to store exported AMI | `my-ami-exports` | | `OSSBucketName` | Yes | OSS bucket to store image file | `my-image-imports` | | `ImageName` | Yes | Name for the imported ECS image | `aws-migrated-ubuntu22` | | `InstanceType` | Yes | Target ECS instance type | `ecs.g6.large` | | `SystemDiskSize` | Yes | System disk size (≥ source AMI disk size) | `40` | | `DiskImageFormat` | No | Export format: `VHD` or `VMDK` | `VHD` | ## 3. Step 1: Export EC2 AMI to S3 ### 3.1 Create AMI from Running EC2 (if not already done) ```bash # Create AMI snapshot of the source EC2 instance AMI_ID=$(aws ec2 create-image \ --instance-id \ --name "migration-$(date +%Y%m%d)" \ --no-reboot \ --region \ --query 'ImageId' --output text) echo "AMI_ID=$AMI_ID" # Wait for AMI to be available aws ec2 wait image-available \ --image-ids $AMI_ID \ --region ``` ### 3.2 Grant EC2 Export Permission (S3 bucket policy) AWS requires a service role and bucket policy for image export. Add this to the S3 bucket policy: ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "vmie.amazonaws.com" }, "Action": [ "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:GetBucketAcl" ], "Resource": [ "arn:aws:s3:::", "arn:aws:s3:::/*" ] } ] } ``` ### 3.3 Export AMI to S3 ```bash # Export AMI as VHD (recommended — smaller file, widely supported by ImportImage) EXPORT_TASK=$(aws ec2 export-image \ --image-id $AMI_ID \ --disk-image-format VHD \ --s3-export-location S3Bucket=,S3Prefix=exports/ \ --region \ --query 'ExportImageTaskId' --output text) echo "EXPORT_TASK=$EXPORT_TASK" ``` ### 3.4 Monitor Export Progress ```bash # Poll until status = completed (may take 30 min – 2 hours for large disks) aws ec2 describe-export-image-tasks \ --export-image-task-ids $EXPORT_TASK \ --region \ --query 'ExportImageTasks[0].{Status:Status,Progress:Progress,S3Key:S3ExportLocation.S3Key}' # Get the exported file path once complete S3_KEY=$(aws ec2 describe-export-image-tasks \ --export-image-task-ids $EXPORT_TASK \ --region \ --query 'ExportImageTasks[0].S3ExportLocation.S3Key' --output text) echo "S3_KEY=$S3_KEY" # Example: exports/export-ami-xxxxxxxx.vhd ``` ### 3.5 Download from S3 ```bash aws s3 cp s3:///$S3_KEY /tmp/migrated-image.vhd \ --region ``` > **Large file tip**: For images > 20 GB, use `--storage-class STANDARD` and multipart download: > ```bash > aws s3 cp s3:///$S3_KEY /tmp/migrated-image.vhd \ > --region \ > --expected-size $(aws s3api head-object --bucket --key $S3_KEY --query ContentLength --output text) > ``` ## 4. Step 2: Create OSS Bucket and Upload Image ### 4.1 Create OSS Bucket (Terraform) The OSS bucket must be in the **same region** as where you will run `ImportImage`. ```hcl resource "alicloud_oss_bucket" "image_import" { bucket = "" acl = "private" tags = { Purpose = "EC2 to ECS image import" } } ``` Apply via IaCService: ```bash $TF apply main.tf ``` ### 4.2 Upload Image to OSS (ossutil) ```bash # Install ossutil if not present # macOS: brew install ossutil # Or download: # curl -fL --connect-timeout 10 --max-time 300 -o ossutil https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil64 # chmod +x ossutil # Configure ossutil (interactive; do not pass AK/SK on the command line) ossutil config # Upload (multipart for large files) ossutil cp /tmp/migrated-image.vhd \ oss:///migrated-image.vhd \ --part-size 500 \ -j 4 \ --checkpoint-dir /tmp/ossutil-checkpoint # Verify upload ossutil stat oss:///migrated-image.vhd ``` > **Direct S3 → OSS transfer** (skip local download): Use [Alibaba Cloud Data Online Migration](https://mgw.console.aliyun.com/) to migrate directly from S3 to OSS without downloading locally. Especially recommended for files > 10 GB. ## 5. Step 3: Import Image to ECS ### 5.1 Import Image via CLI ```bash aliyun ecs ImportImage \ --RegionId \ --OSType linux \ --Architecture x86_64 \ --ImageName "" \ --Description "Imported from AWS EC2 AMI $AMI_ID" \ --DiskDeviceMapping.1.Format VHD \ --DiskDeviceMapping.1.OSSBucket \ --DiskDeviceMapping.1.OSSObject migrated-image.vhd \ --DiskDeviceMapping.1.DiskImSize \ --user-agent AlibabaCloud-Agent-Skills ``` **Response:** ```json { "ImageId": "m-bp1xxxxxxxxxxxxxxxxx", "RequestId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" } ``` Record the `ImageId`. ### 5.2 Monitor Import Progress ```bash # Poll until Status = Available (may take 10–60 min depending on image size) aliyun ecs DescribeImages \ --RegionId \ --ImageId \ --user-agent AlibabaCloud-Agent-Skills ``` **Image Status Values:** | Status | Meaning | Action | |--------|---------|--------| | `Creating` | Import in progress | Keep polling | | `Available` | Import complete | Proceed to ECS creation | | `CreateFailed` | Import failed | Check error, see §6 | | `Deprecated` | Image deprecated | Use a different image | **Poll with loop:** ```bash IMAGE_ID="" for i in {1..60}; do STATUS=$(aliyun ecs DescribeImages \ --RegionId \ --ImageId $IMAGE_ID \ --user-agent AlibabaCloud-Agent-Skills 2>/dev/null | \ python3 -c "import json,sys; imgs=json.load(sys.stdin)['Images']['Image']; print(imgs[0]['Status'] if imgs else 'NotFound')") echo "[$i/60] Image status: $STATUS" [ "$STATUS" = "Available" ] && echo "Import complete!" && break [ "$STATUS" = "CreateFailed" ] && echo "Import FAILED!" && break sleep 30 done ``` ### 5.3 Windows Images — Additional Setup For Windows images, add the `Platform` parameter: ```bash aliyun ecs ImportImage \ --RegionId \ --OSType windows \ --Platform "Windows Server 2019" \ --Architecture x86_64 \ --ImageName "" \ --DiskDeviceMapping.1.Format VHD \ --DiskDeviceMapping.1.OSSBucket \ --DiskDeviceMapping.1.OSSObject migrated-windows.vhd \ --DiskDeviceMapping.1.DiskImSize \ --user-agent AlibabaCloud-Agent-Skills ``` After the ECS instance starts, connect via VNC to reset the password and activate Windows. See [ECS Windows activation docs](https://www.alibabacloud.com/help/en/ecs/user-guide/activate-a-windows-server-instance) for KMS activation. ## 6. Step 4: Create ECS from Imported Image (Terraform) Once the image status is `Available`, create ECS using Terraform: ```hcl provider "alicloud" { region = "" configuration_source = "AlibabaCloud-Agent-Skills/alibabacloud-migrate" } # Reference existing network (from Phase 2) data "alicloud_vpcs" "main" { name_regex = "" } data "alicloud_vswitches" "main" { vpc_id = data.alicloud_vpcs.main.ids[0] } data "alicloud_security_groups" "main" { name_regex = "" vpc_id = data.alicloud_vpcs.main.ids[0] } # Key Pair resource "alicloud_ecs_key_pair" "main" { key_pair_name = "" public_key = "" } # ECS Instance from imported image resource "alicloud_instance" "migrated" { instance_name = "" instance_type = "" image_id = "" # from ImportImage vswitch_id = data.alicloud_vswitches.main.ids[0] security_groups = [data.alicloud_security_groups.main.ids[0]] key_name = alicloud_ecs_key_pair.main.key_pair_name system_disk_category = "cloud_essd" system_disk_size = internet_max_bandwidth_out = 10 internet_charge_type = "PayByTraffic" tags = { Name = "" MigratedFrom = "aws-ec2" SourceAMI = "" } } output "instance_public_ip" { value = alicloud_instance.migrated.public_ip } output "instance_id" { value = alicloud_instance.migrated.id } ``` Apply: ```bash apply_output=$($TF apply main.tf) STATE_ID=$(echo "$apply_output" | grep '^STATE_ID=' | cut -d= -f2) echo "STATE_ID=$STATE_ID" >> terraform_state_ids.env ``` ## 7. Step 5: Verify Migration ```bash # 1. Verify instance is running aliyun ecs DescribeInstances \ --RegionId \ --InstanceIds '[""]' \ --user-agent AlibabaCloud-Agent-Skills # 2. SSH and verify OS and data ssh -i root@ " uname -a df -h cat /etc/os-release # Verify application data exists ls -la /data/ " # 3. Verify application health curl -f --connect-timeout 5 --max-time 30 http://:/health # 4. Check system services ssh -i root@ "systemctl list-units --state=failed" ``` ## 8. Error Handling | Error | Cause | Solution | |-------|-------|----------| | `InvalidOSSObject.NotFound` | OSS object path incorrect | Verify bucket/object names; check region matches | | `QuotaExceed.Image` | Image quota exceeded | Delete unused custom images | | `InvalidFormat.NotSupported` | Image format not supported | Convert to VHD/VMDK using qemu-img | | `DiskImSizeTooSmall` | `DiskImSize` smaller than image content | Increase `DiskImSize` to match source disk | | `InvalidOSSBucket.NotFound` | OSS bucket not found | Verify bucket exists in the **same region** as ImportImage | | `Forbidden.RiskControl` | Account security check pending | Complete real-name authentication in console | | `CreateFailed` (generic) | Various import errors | Check image format; re-export AMI with `--disk-image-format VHD` | | ECS boots but network fails | AWS-specific network config | Install cloud-init or update /etc/network config on ECS | | ECS boots but SSH fails | SSH key not in authorized_keys | Use VNC console to reset; ensure cloud-init installed | ### Format Conversion (if needed) If the export format is not compatible, convert locally before uploading: ```bash # Install qemu-img brew install qemu # macOS # or: apt install qemu-utils # VMDK → VHD qemu-img convert -f vmdk -O vpc source.vmdk target.vhd # RAW → VHD qemu-img convert -f raw -O vpc source.img target.vhd # Check converted image qemu-img info target.vhd ``` ## 9. Cleanup (After Migration Validated) ```bash # 1. Delete image file from OSS (no longer needed after import) ossutil rm oss:///migrated-image.vhd # 2. Delete OSS bucket (if only used for this migration) ossutil rm oss:// -b # 3. Delete imported image (only if replacing with newer version) # NOTE: Do NOT delete while ECS instances reference it aliyun ecs DeleteImage \ --RegionId \ --ImageId \ --Force true \ --user-agent AlibabaCloud-Agent-Skills # 4. Delete AWS export artifacts (from S3) aws s3 rm s3:///exports/ --recursive # 5. Deregister original AMI (only after migration fully validated) aws ec2 deregister-image --image-id $AMI_ID --region ``` ## 10. Best Practices 1. **Export as VHD**: VHD format is recommended over VMDK — smaller size and better compatibility with `ImportImage` 2. **Same region for OSS and ImportImage**: OSS bucket and `ImportImage` target region must match 3. **Disk size ≥ source**: Set `DiskImSize` to at least the source disk size (round up to nearest GB) 4. **Install cloud-init before export**: On the source EC2, ensure `cloud-init` is installed for automatic network and SSH key injection on first boot 5. **Use Data Online Migration for large images**: For images > 5 GB, use the [Alibaba Cloud Data Online Migration](https://mgw.console.aliyun.com/) service to transfer directly from S3 to OSS (avoids local storage) 6. **Keep the OSS file until ECS is validated**: Delete the OSS image file only after the ECS instance boots and passes verification 7. **Keep source EC2 running**: Do not stop/terminate the source EC2 until the ECS instance is fully validated ## 11. Transfer Path Optimization The VHD/VMDK file produced by `aws ec2 export-image` must travel from AWS S3 to Alibaba Cloud OSS. The transfer path you choose is the **biggest factor in total migration time**. ### 11.1 Transfer Paths Compared | Path | Measured Speed | 2.5 GB File | Notes | |------|---------------|-------------|-------| | S3 (overseas) → Local (mainland China) → OSS | ~1–2 MB/s | ~30–45 min | Cross-border GFW throttling | | S3 (overseas) → Local (mainland China) → OSS (HK/overseas) | ~3–8 MB/s | ~10–20 min | Upload to overseas OSS is faster | | S3 → **Relay ECS (cn-hongkong)** → OSS | ~50–100 MB/s | ~1–3 min | **Recommended for production** | | S3 → **Alibaba Cloud Data Online Migration** → OSS | ~20–50 MB/s | ~3–8 min | No relay server needed | > **Rule of thumb**: If the image is > 1 GB and the Alibaba Cloud target region is in mainland China, always use a relay ECS (Option C) or Data Online Migration (Option D). Direct local download through mainland China is ~20–50× slower. ### 11.2 Option A: Local Machine Relay (Simple, Slowest) Suitable only for images < 1 GB or when the local machine has a fast overseas connection. ```bash # Step 1: Download from S3 to local disk aws s3 cp s3:/// /tmp/migrated-image.vhd \ --region # Step 2: Upload from local disk to OSS aliyun oss cp /tmp/migrated-image.vhd \ oss:///images/migrated-image.vhd \ -e oss-.aliyuncs.com \ --jobs 5 --part-size 104857600 --user-agent AlibabaCloud-Agent-Skills ``` ### 11.3 Option B: Relay ECS in Alibaba Cloud HK (Recommended) A small ECS instance in `cn-hongkong` downloads from Singapore S3 at ~50–100 MB/s (same geographic area, no GFW), then uploads to mainland OSS via Alibaba Cloud's internal backbone at ~100–200 MB/s. **Total: 1–3 minutes for a 2.5 GB image.** ```bash # --- On relay ECS (cn-hongkong) --- # 1. Install AWS CLI on relay ECS curl -fL --connect-timeout 10 --max-time 600 "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip unzip /tmp/awscliv2.zip -d /tmp && sudo /tmp/aws/install # Configure AWS credentials on relay ECS aws configure set aws_access_key_id aws configure set aws_secret_access_key aws configure set region # 2. Download from S3 (fast — same region/nearby) aws s3 cp s3:/// /tmp/migrated-image.vhd # 3. Install ossutil on relay ECS curl -fL --connect-timeout 10 --max-time 300 -o /tmp/ossutil64 https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil64 chmod +x /tmp/ossutil64 /tmp/ossutil64 config -e oss-.aliyuncs.com # 4. Upload to OSS (fast — Alibaba Cloud internal network) /tmp/ossutil64 cp /tmp/migrated-image.vhd \ oss:///images/migrated-image.vhd \ --jobs 5 --part-size 104857600 # 5. Terminate relay ECS after upload ``` **Relay ECS spec**: `ecs.u1-c1m4.large` (1 vCPU, 4 GB RAM) costs ~¥0.1/hour in cn-hongkong and is sufficient. Total relay cost is usually < ¥1 per migration. > **Terraform snippet to provision relay ECS** (destroy after use): > ```hcl > resource "alicloud_instance" "relay" { > instance_name = "migration-relay" > image_id = "ubuntu_22_04_x64_20G_alibase_20240424.vhd" > instance_type = "ecs.u1-c1m4.large" > vswitch_id = > security_groups = [] > internet_max_bandwidth_out = 100 > system_disk_category = "cloud_essd" > system_disk_size = 40 > } > ``` ### 11.4 Option C: Alibaba Cloud Data Online Migration (No Relay Needed) For images > 5 GB or when you cannot provision a relay ECS, use Alibaba Cloud's [Data Online Migration](https://mgw.console.aliyun.com/) service to pull the image directly from S3 to OSS: 1. Go to [Data Online Migration console](https://mgw.console.aliyun.com/) 2. **Create Source** → AWS S3 → enter AWS AK/SK + bucket name 3. **Create Destination** → Alibaba Cloud OSS → select target bucket 4. **Create Migration Job** → choose the VHD file as the source object 5. Monitor progress in the console (no local bandwidth consumed) ``` AWS S3 (Singapore) ──── Alibaba internal transfer ────→ OSS (cn-hangzhou) No local machine involved — pure cloud-to-cloud ``` > Data Online Migration uses Alibaba Cloud's premium cross-border bandwidth. Speed is usually 20–50 MB/s. There is no additional charge beyond standard OSS PUT fees. ### 11.5 OSS Region Selection for Minimum Latency Choose the OSS region closest to the AWS source region for Step 2 (upload), then set `ImportImage` to the same region: | AWS Source Region | Recommended OSS Region | Reason | |-------------------|------------------------|--------| | `ap-southeast-1` (Singapore) | `cn-hongkong` or `ap-southeast-5` | Nearest to Singapore | | `us-east-1` (N. Virginia) | `us-east-1` (Alibaba Cloud US East) | Same metro area | | `eu-west-1` (Ireland) | `eu-central-1` (Frankfurt) | Nearest EU | | `ap-northeast-1` (Tokyo) | `ap-northeast-1` (Alibaba Cloud Japan) | Same region | > If your production workload must run in `cn-hangzhou`, use Option B (relay ECS in HK) or Option C (Data Online Migration) to first stage the image in `cn-hangzhou` OSS, then run `ImportImage` in `cn-hangzhou`. ### 11.6 Transfer Time Estimation Use this formula to pre-estimate transfer time before starting: ``` Time (minutes) = File size (MB) / Expected speed (MB/s) / 60 Example: 5 GB VHD via relay ECS (80 MB/s): 5120 MB / 80 MB/s / 60 = ~1 minute 5 GB VHD via local (mainland China): 5120 MB / 1.5 MB/s / 60 = ~57 minutes ``` | Image Size | Local (mainland) | Relay ECS (HK) | Data Online Migration | |------------|-----------------|----------------|----------------------| | 1 GB | ~10 min | < 1 min | ~1 min | | 5 GB | ~55 min | ~2 min | ~4 min | | 10 GB | ~110 min | ~4 min | ~8 min | | 50 GB | ~9 hours | ~18 min | ~40 min | ## 12. Comparison with SMC | Aspect | ImportImage (this guide) | SMC | |--------|--------------------------|-----| | Agent on source | Not required | Required | | Cross-region (overseas → mainland) | ✅ No network restriction | ⚠️ Security group whitelist (mainland IPs only by default) | | Migration time | Longer (export + transfer + import) | Faster for incremental sync | | Incremental sync | ❌ Not supported | ✅ Supported | | Windows support | ✅ | ✅ | | Cost | S3 export + data transfer fees | Free (pay only for resources) | | Best for | One-time migration, overseas sources | Frequent incremental sync, large fleets | ## External References - [AWS EC2 Export Image](https://docs.aws.amazon.com/vm-import/latest/userguide/vmexport_image.html) - [Alibaba Cloud ECS ImportImage](https://www.alibabacloud.com/help/en/ecs/developer-reference/api-importimage) - [Alibaba Cloud Data Online Migration](https://www.alibabacloud.com/help/en/data-online-migration) - [ossutil download](https://www.alibabacloud.com/help/en/oss/developer-reference/ossutil) FILE:references/migration-guides/serverless-migration-fc.md # Serverless Migration: AWS Lambda to Alibaba Cloud Function Compute ## Overview Alibaba Cloud Function Compute (FC) is a fully managed serverless compute service equivalent to AWS Lambda. This guide covers migration strategies, code conversion, and infrastructure mapping for migrating from AWS Lambda to Function Compute. ### Key Differences | Aspect | AWS Lambda | Alibaba Cloud Function Compute | |--------|-----------|-------------------------------| | **Execution Model** | Event-driven compute | Event-driven compute | | **Billing** | Pay per invocation + duration | Pay per invocation + duration | | **Scaling** | Automatic scaling | Automatic scaling | | **Cold Start** | ~100ms-1s depending on runtime | ~100ms-1s depending on runtime | | **Max Memory** | 10 GB | 10 GB | | **Max Timeout** | 15 minutes | 10 minutes (configurable up to 60 minutes for specific scenarios) | | **Max Package Size** | 250 MB (unzipped) | 500 MB (unzipped) | | **Concurrent Executions** | Account-level limit | Account-level limit | | **Layers** | Supported | Supported (called "Layers") | | **Container Image Support** | Yes (up to 10 GB) | Yes (up to 10 GB) | ## Feature Comparison Table | Feature | AWS Lambda | Alibaba Cloud FC | Migration Notes | |---------|-----------|-----------------|-----------------| | **Handler Signature** | `exports.handler(event, context)` | `exports.handler(event, context, callback)` | FC supports both callback and Promise styles | | **Max Timeout** | 15 minutes | 10 minutes (configurable) | Adjust long-running functions | | **Memory Range** | 128 MB - 10 GB | 128 MB - 10 GB | Direct mapping | | **Triggers** | API GW, S3, SQS, SNS, etc. | HTTP, OSS, Timer, MNS, etc. | See trigger mapping below | | **Layers** | Supported | Supported | Similar concept, different CLI | | **Runtime Versions** | Node.js 20, Python 3.12, etc. | Node.js 20, Python 3.10, etc. | Check runtime availability | | **Environment Variables** | 4 KB limit | 4 KB limit | Direct mapping | | **VPC Integration** | Yes | Yes | Similar configuration | | **Dead Letter Queue** | SQS/SNS | MNS Topic/Queue | Different service | | **Provisioned Concurrency** | Supported | Supported | Called "Provisioned Mode" | | **Container Image** | ECR | ACR | Use ACR instead of ECR | | **Logging** | CloudWatch Logs | SLS (Simple Log Service) | Different logging service | | **Monitoring** | CloudWatch Metrics | CloudMonitor | Different monitoring service | | **X-Ray Tracing** | AWS X-Ray | ARMS | Different tracing service | ## Trigger Mapping | AWS Trigger | FC Trigger | Notes | |-------------|------------|-------| | **API Gateway** | **HTTP Trigger** | Direct equivalent, similar event structure | | **S3 Event** | **OSS Trigger** | Similar event structure, minor format differences | | **CloudWatch Scheduled** | **Timer Trigger** | Cron syntax compatible | | **SQS** | **MNS Topic Trigger** | Different queue semantics, code changes needed | | **SNS** | **MNS Topic Trigger** | Similar pub/sub model | | **DynamoDB Streams** | **Tablestore Trigger** | Different event format, code changes needed | | **Kinesis** | **Log Service Trigger** | Different streaming service | | **EventBridge** | **EventBridge Trigger** | Similar event routing | | **Cognito** | **IDaaS** | Different identity service | | **ALB** | **ALB Trigger** | Direct equivalent | ### Trigger Configuration Examples #### HTTP Trigger (API Gateway Equivalent) **AWS Lambda:** ```python def lambda_handler(event, context): return { 'statusCode': 200, 'body': json.dumps({'message': 'Hello'}) } ``` **Function Compute:** ```python def handler(event, context): return { 'statusCode': 200, 'headers': {'Content-Type': 'application/json'}, 'body': json.dumps({'message': 'Hello'}) } ``` #### OSS Trigger (S3 Event Equivalent) **AWS S3 Event:** ```json { "Records": [ { "s3": { "bucket": {"name": "my-bucket"}, "object": {"key": "my-key"} } } ] } ``` **Alibaba Cloud OSS Event:** ```json { "events": [ { "oss": { "bucket": {"name": "my-bucket"}, "object": {"key": "my-key"} } } ] } ``` #### Timer Trigger (CloudWatch Scheduled Equivalent) **AWS CloudWatch Events:** ```json { "source": ["aws.events"], "detail-type": ["Scheduled Event"] } ``` **Function Compute Timer:** ```json { "triggerName": "timer-trigger", "triggerTime": "2024-01-15T10:30:00Z" } ``` ## Code Migration Examples ### Node.js Migration #### HTTP Handler Conversion **AWS Lambda:** ```javascript // index.js exports.handler = async (event, context) => { const { httpMethod, path, body } = event; try { // Business logic const result = { message: 'Success' }; return { statusCode: 200, headers: { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }, body: JSON.stringify(result) }; } catch (error) { return { statusCode: 500, body: JSON.stringify({ error: error.message }) }; } }; ``` **Function Compute:** ```javascript // index.js exports.handler = async (event, context) => { // Parse event body const body = JSON.parse(event.body || '{}'); const { httpMethod, path } = event; try { // Business logic (same as Lambda) const result = { message: 'Success' }; return { statusCode: 200, headers: { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }, body: JSON.stringify(result) }; } catch (error) { return { statusCode: 500, body: JSON.stringify({ error: error.message }) }; } }; ``` **Key Differences:** - Event structure is similar but may have minor field name differences - Response format is identical - Context object has similar properties but different service names #### Event-Driven Handler (S3/OSS) **AWS Lambda (S3):** ```javascript exports.handler = async (event) => { for (const record of event.Records) { const bucket = record.s3.bucket.name; const key = decodeURIComponent(record.s3.object.key); // Process S3 object await processObject(bucket, key); } return { success: true }; }; ``` **Function Compute (OSS):** ```javascript exports.handler = async (event, context) => { const eventObj = JSON.parse(event.toString()); for (const record of eventObj.events) { const bucket = record.oss.bucket.name; const key = decodeURIComponent(record.oss.object.key); // Process OSS object await processObject(bucket, key); } return { success: true }; }; ``` ### Python Migration #### HTTP Handler Conversion **AWS Lambda:** ```python import json def lambda_handler(event, context): http_method = event.get('httpMethod', event.get('requestContext', {}).get('http', {}).get('method')) path = event.get('path', event.get('rawPath')) body = json.loads(event.get('body', '{}')) if event.get('body') else {} try: # Business logic result = {'message': 'Success'} return { 'statusCode': 200, 'headers': { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }, 'body': json.dumps(result) } except Exception as e: return { 'statusCode': 500, 'body': json.dumps({'error': str(e)}) } ``` **Function Compute:** ```python import json def handler(event, context): # Parse event if isinstance(event, str): event = json.loads(event) http_method = event.get('method', event.get('httpMethod')) path = event.get('path', event.get('rawPath')) body = json.loads(event.get('body', '{}')) if event.get('body') else {} try: # Business logic (same as Lambda) result = {'message': 'Success'} return { 'statusCode': 200, 'headers': { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }, 'body': json.dumps(result) } except Exception as e: return { 'statusCode': 500, 'body': json.dumps({'error': str(e)}) } ``` #### SDK Migration: boto3 (S3) → oss2 (OSS) > **CRITICAL**: When migrating Lambda functions that use `boto3` to access AWS services (S3, DynamoDB, etc.), you must replace the SDK calls with the corresponding Alibaba Cloud SDK. For S3→OSS, use the `oss2` Python package. Authentication **MUST** use FC's built-in STS credentials via `context.credentials` — never hardcode AK/SK. **AWS Lambda (boto3 + S3):** ```python import json import boto3 import os s3_client = boto3.client('s3') BUCKET_NAME = os.environ['BUCKET_NAME'] def lambda_handler(event, context): """Lambda function: S3 file operations via HTTP API.""" body = json.loads(event.get('body', '{}')) action = body.get('action', 'list') if action == 'list': response = s3_client.list_objects_v2(Bucket=BUCKET_NAME) files = [obj['Key'] for obj in response.get('Contents', [])] return {'statusCode': 200, 'body': json.dumps({'files': files})} elif action == 'read': obj = s3_client.get_object(Bucket=BUCKET_NAME, Key=body['key']) content = obj['Body'].read().decode('utf-8') return {'statusCode': 200, 'body': json.dumps({'content': content})} elif action == 'write': s3_client.put_object(Bucket=BUCKET_NAME, Key=body['key'], Body=body['content'].encode()) return {'statusCode': 200, 'body': json.dumps({'message': 'Written'})} elif action == 'delete': s3_client.delete_object(Bucket=BUCKET_NAME, Key=body['key']) return {'statusCode': 200, 'body': json.dumps({'message': 'Deleted'})} ``` **Function Compute (oss2 + STS credentials):** ```python import json import oss2 import os BUCKET_NAME = os.environ['BUCKET_NAME'] ENDPOINT = os.environ['OSS_ENDPOINT'] # e.g., https://oss-cn-hangzhou-internal.aliyuncs.com def get_bucket(context): """Create OSS bucket client using FC context credentials (STS temporary credentials). IMPORTANT: FC automatically injects STS credentials when a RAM role is assigned to the function. Use context.credentials to access them — never hardcode AK/SK. """ creds = context.credentials auth = oss2.StsAuth( creds.access_key_id, creds.access_key_secret, creds.security_token, ) return oss2.Bucket(auth, ENDPOINT, BUCKET_NAME) def handler(event, context): """FC function: OSS file operations (migrated from AWS Lambda + S3).""" if isinstance(event, bytes): event = json.loads(event.decode()) elif isinstance(event, str): event = json.loads(event) body = event.get('body', '{}') if isinstance(body, str): body = json.loads(body) action = body.get('action', 'list') bucket = get_bucket(context) if action == 'list': files = [obj.key for obj in oss2.ObjectIterator(bucket)] return {'statusCode': 200, 'body': json.dumps({'files': files})} elif action == 'read': result = bucket.get_object(body['key']) content = result.read().decode('utf-8') return {'statusCode': 200, 'body': json.dumps({'content': content})} elif action == 'write': bucket.put_object(body['key'], body['content'].encode()) return {'statusCode': 200, 'body': json.dumps({'message': 'Written'})} elif action == 'delete': bucket.delete_object(body['key']) return {'statusCode': 200, 'body': json.dumps({'message': 'Deleted'})} ``` **Key Migration Points:** - `boto3.client('s3')` → `oss2.Bucket(auth, endpoint, bucket_name)` - Authentication: AWS IAM role (automatic) → FC RAM role + `context.credentials` + `oss2.StsAuth` - `s3.list_objects_v2()` → `oss2.ObjectIterator(bucket)` - `s3.get_object()` → `bucket.get_object(key)` - `s3.put_object()` → `bucket.put_object(key, data)` - `s3.delete_object()` → `bucket.delete_object(key)` - **Dependency**: Add `oss2` to `requirements.txt` (replaces `boto3`) - **Environment Variables**: Replace `AWS_REGION` with `OSS_ENDPOINT` (use internal endpoint for best performance) #### Environment Variable Mapping **AWS Lambda:** ```python import os def lambda_handler(event, context): db_host = os.environ['DB_HOST'] api_key = os.environ['API_KEY'] region = os.environ.get('AWS_REGION', 'us-east-1') ``` **Function Compute:** ```python import os def handler(event, context): db_host = os.environ['DB_HOST'] api_key = os.environ['API_KEY'] region = os.environ.get('FC_REGION', 'cn-hangzhou') ``` **Environment Variable Migration:** - Copy all environment variables from Lambda to FC - Update region-specific variables (AWS_REGION → FC_REGION) - Update service endpoint variables (e.g., S3_ENDPOINT → OSS_ENDPOINT) ### Context Object Mapping | AWS Lambda Context | Function Compute Context | Notes | |-------------------|-------------------------|-------| | `context.function_name` | `context.function_name` | Same | | `context.function_version` | `context.function_version` | Same | | `context.memory_limit_in_mb` | `context.memory_limit_in_mb` | Same | | `context.aws_request_id` | `context.request_id` | Different name | | `context.log_group_name` | `context.log_project` | Different service | | `context.log_stream_name` | `context.log_store` | Different service | | `context.identity` | `context.identity` | Similar structure | | `context.client_context` | `context.client_context` | Similar structure | ## CLI Commands ### Prerequisites ```bash # Verify aliyun CLI version (MUST >= 3.3.1) aliyun version # Configure credentials aliyun configure list # Install FC plugin (if not auto-installed) aliyun plugin install fc ``` ### Terraform Alternative for Function Compute ```hcl # ─── RAM Role (REQUIRED: FC needs a role to access other Alibaba Cloud services) ─── resource "alicloud_ram_role" "fc_role" { name = "-fc-role" document = jsonencode({ Statement = [{ Action = "sts:AssumeRole" Effect = "Allow" Principal = { Service = ["fc.aliyuncs.com"] } }] Version = "1" }) force = true } resource "alicloud_ram_policy" "fc_policy" { policy_name = "-fc-policy" policy_document = jsonencode({ Statement = [ { Action = ["oss:GetObject", "oss:PutObject", "oss:DeleteObject", "oss:ListObjects"] Effect = "Allow" Resource = ["acs:oss:*:*:", "acs:oss:*:*:/*"] } # Add more statements as needed for other services (SLS, Tablestore, MNS, etc.) ] Version = "1" }) } resource "alicloud_ram_role_policy_attachment" "fc_attach" { policy_name = alicloud_ram_policy.fc_policy.policy_name policy_type = "Custom" role_name = alicloud_ram_role.fc_role.name } # ─── Function Compute ─── resource "alicloud_fcv3_function" "migration" { function_name = "" handler = "" runtime = "" memory_size = timeout = role = alicloud_ram_role.fc_role.arn # CRITICAL: Without this, FC cannot access OSS/SLS/etc. code { zip_file = "" } } resource "alicloud_fcv3_trigger" "http" { function_name = alicloud_fcv3_function.migration.function_name trigger_name = "" trigger_type = "http" trigger_config = jsonencode({ authType = "anonymous" methods = ["GET", "POST"] }) } ``` **Trigger Type Examples:** **HTTP Trigger:** ```hcl resource "alicloud_fcv3_trigger" "http" { function_name = alicloud_fcv3_function.migration.function_name trigger_name = "" trigger_type = "http" trigger_config = jsonencode({ authType = "anonymous" methods = ["GET", "POST"] }) } ``` **Timer Trigger:** ```hcl resource "alicloud_fcv3_trigger" "timer" { function_name = alicloud_fcv3_function.migration.function_name trigger_name = "" trigger_type = "timer" trigger_config = jsonencode({ cronExpression = "0 0 * * * *" payload = "" }) } ``` **OSS Trigger:** ```hcl resource "alicloud_fcv3_trigger" "oss" { function_name = alicloud_fcv3_function.migration.function_name trigger_name = "" trigger_type = "oss" trigger_config = jsonencode({ events = ["oss:ObjectCreated:*"] filter = { key = { prefix = "uploads/" } } }) } ``` ### Create Function ```bash aliyun fc create-function \ --function-name \ --runtime nodejs20 \ --handler index.handler \ --code zipFile=base64encoded== \ --memory-size 1024 \ --timeout 60 \ --environment-variables '{"DB_HOST": "localhost", "API_KEY": "xxx"}' \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `function-name` | Yes | Function name | `my-function` | | `runtime` | Yes | Runtime environment | `nodejs20`, `python3.10`, `java11` | | `handler` | Yes | Entry point | `index.handler`, `app.main` | | `code` | Yes | Code as base64 or OSS reference | `zipFile=base64encoded==` or `ossBucketName=bucket,ossObjectName=code.zip` | | `memory-size` | No | Memory in MB (128-10240) | `1024` | | `timeout` | No | Timeout in seconds (1-600) | `60` | | `environment-variables` | No | JSON string of env vars | `{"KEY": "value"}` | ### Create HTTP Trigger ```bash aliyun fc create-trigger \ --function-name \ --trigger-name http-trigger \ --trigger-type http \ --trigger-config '{"authType": "anonymous", "methods": ["GET", "POST"]}' \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `function-name` | Yes | Target function name | `my-function` | | `trigger-name` | Yes | Trigger name | `http-trigger` | | `trigger-type` | Yes | Trigger type | `http`, `oss`, `timer`, `log`, `cdn_events` | | `trigger-config` | Yes | Trigger configuration JSON | `{"authType": "anonymous"}` | ### Create Timer Trigger ```bash aliyun fc create-trigger \ --function-name \ --trigger-name timer-trigger \ --trigger-type timer \ --trigger-config '{"cronExpression": "0 0 * * * *", "payload": ""}' \ --user-agent AlibabaCloud-Agent-Skills ``` **Cron Expression Format:** - Seconds Minutes Hours Day-of-month Month Day-of-week Year (optional) - Example: `0 0 10 * * *` = Every day at 10:00 AM ### Create OSS Trigger ```bash aliyun fc create-trigger \ --function-name \ --trigger-name oss-trigger \ --trigger-type oss \ --trigger-config '{"events": ["oss:ObjectCreated:*"], "filter": {"key": {"prefix": "uploads/"}}}' \ --user-agent AlibabaCloud-Agent-Skills ``` ### List Functions ```bash aliyun fc list-functions \ --user-agent AlibabaCloud-Agent-Skills ``` ### Get Function Details ```bash aliyun fc get-function \ --function-name \ --user-agent AlibabaCloud-Agent-Skills ``` ### Invoke Function ```bash aliyun fc invoke-function \ --function-name \ --payload '{"key": "value"}' \ --user-agent AlibabaCloud-Agent-Skills ``` ### Update Function ```bash aliyun fc update-function \ --function-name \ --memory-size 2048 \ --timeout 120 \ --user-agent AlibabaCloud-Agent-Skills ``` ### Delete Function ```bash aliyun fc delete-function \ --function-name \ --user-agent AlibabaCloud-Agent-Skills ``` ## IAM Mapping: AWS IAM Role → Alibaba Cloud RAM Role ### AWS Lambda Execution Role **AWS IAM Policy Example:** ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents" ], "Resource": "arn:aws:logs:*:*:*" }, { "Effect": "Allow", "Action": [ "s3:GetObject", "s3:PutObject" ], "Resource": "arn:aws:s3:::my-bucket/*" }, { "Effect": "Allow", "Action": [ "dynamodb:GetItem", "dynamodb:PutItem" ], "Resource": "arn:aws:dynamodb:*:*:table/my-table" } ] } ``` ### Alibaba Cloud RAM Role for FC **RAM Policy Example:** ```json { "Version": "1", "Statement": [ { "Effect": "Allow", "Action": [ "logs:CreateLogStore", "logs:CreateLogStream", "logs:PutLogs" ], "Resource": "acs:log:*:*:project/*/logstore/*" }, { "Effect": "Allow", "Action": [ "oss:GetObject", "oss:PutObject" ], "Resource": "acs:oss:*:*:my-bucket/*" }, { "Effect": "Allow", "Action": [ "ots:GetRow", "ots:PutRow" ], "Resource": "acs:ots:*:*:instance/my-instance/table/my-table" } ] } ``` ### Create RAM Role for FC ```bash aliyun ram CreateRole \ --RoleName fc-execution-role \ --AssumeRolePolicyDocument '{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":["fc.aliyuncs.com"]}}],"Version":"1"}' \ --user-agent AlibabaCloud-Agent-Skills ``` ### Attach Policy to RAM Role ```bash aliyun ram AttachPolicyToRole \ --PolicyType Custom \ --PolicyName fc-oss-access \ --RoleName fc-execution-role \ --user-agent AlibabaCloud-Agent-Skills ``` ### Terraform Alternative for RAM Role Setup > **Recommended**: Use Terraform for RAM role management to keep it in the same state file as the FC function. This ensures role and function are always created/destroyed together. ```hcl # 1. Create RAM Role with FC trust policy resource "alicloud_ram_role" "fc_role" { name = "fc-execution-role" document = jsonencode({ Statement = [{ Action = "sts:AssumeRole" Effect = "Allow" Principal = { Service = ["fc.aliyuncs.com"] } }] Version = "1" }) force = true } # 2. Create custom policy (adapt actions/resources to match the original AWS IAM policy) resource "alicloud_ram_policy" "fc_policy" { policy_name = "fc-oss-access" policy_document = jsonencode({ Statement = [ { Effect = "Allow" Action = ["oss:GetObject", "oss:PutObject", "oss:DeleteObject", "oss:ListObjects"] Resource = ["acs:oss:*:*:", "acs:oss:*:*:/*"] } ] Version = "1" }) } # 3. Attach policy to role resource "alicloud_ram_role_policy_attachment" "fc_attach" { policy_name = alicloud_ram_policy.fc_policy.policy_name policy_type = "Custom" role_name = alicloud_ram_role.fc_role.name } # 4. Reference in function: role = alicloud_ram_role.fc_role.arn ``` ### Resource ARN Mapping | AWS Resource | AWS ARN Format | Alibaba Cloud Resource | Alibaba Cloud ARN Format | |-------------|---------------|----------------------|-------------------------| | S3 Bucket | `arn:aws:s3:::bucket-name` | OSS Bucket | `acs:oss:*:*:bucket-name` | | S3 Object | `arn:aws:s3:::bucket/key` | OSS Object | `acs:oss:*:*:bucket-name/object-key` | | DynamoDB Table | `arn:aws:dynamodb:region:account:table/name` | Tablestore Table | `acs:ots:*:*:instance/name/table/name` | | SQS Queue | `arn:aws:sqs:region:account:queue` | MNS Queue | `acs:mns:*:*:/queues/queue-name` | | SNS Topic | `arn:aws:sns:region:account:topic` | MNS Topic | `acs:mns:*:*:/topics/topic-name` | | Lambda Function | `arn:aws:lambda:region:account:function:name` | FC Function | `acs:fc:*:*:functions/function-name` | ## Migration Checklist ### Pre-Migration - [ ] Inventory all Lambda functions (count, runtimes, memory, timeout) - [ ] Document all triggers and their configurations - [ ] List all environment variables - [ ] Map IAM roles and permissions to RAM policies - [ ] Identify dependencies (layers, VPC, external services) - [ ] Estimate costs on Function Compute - [ ] Set up Alibaba Cloud account and RAM users ### Code Migration - [ ] Update handler signatures if needed - [ ] Replace AWS SDK calls with Alibaba Cloud SDK - [ ] Update environment variable names - [ ] Modify event parsing for trigger differences - [ ] Update logging calls (CloudWatch → SLS) - [ ] Test code locally with FC local runtime (if available) ### Infrastructure Migration - [ ] **Create RAM role for FC** (trust policy: `fc.aliyuncs.com` as principal) - [ ] **Create RAM policy** with least-privilege access to required services (OSS, SLS, Tablestore, etc.) - [ ] **Attach RAM policy to RAM role** - [ ] Create functions with correct runtime, memory, and **`role` parameter pointing to RAM role ARN** - [ ] Upload code packages to OSS - [ ] Configure environment variables - [ ] Set up VPC configuration (if needed) - [ ] Create and configure triggers - [ ] Set up logging to SLS - [ ] Configure monitoring and alarms ### Testing - [ ] Unit test migrated functions - [ ] Integration test with triggers - [ ] Performance test (cold start, duration) - [ ] Security test (permissions, VPC) - [ ] Error handling test ### Cutover - [ ] Deploy to production FC environment - [ ] Update API endpoints (API Gateway → FC HTTP Trigger) - [ ] Update event source configurations - [ ] Monitor for errors and performance issues - [ ] Keep Lambda functions for rollback period ### Post-Migration - [ ] Verify all functions working correctly - [ ] Optimize memory and timeout settings - [ ] Review and optimize costs - [ ] Set up monitoring dashboards - [ ] Document new architecture - [ ] Decommission Lambda functions ## Cleanup ### Delete All Triggers ```bash # List triggers first aliyun fc list-triggers \ --function-name \ --user-agent AlibabaCloud-Agent-Skills # Delete each trigger aliyun fc delete-trigger \ --function-name \ --trigger-name \ --user-agent AlibabaCloud-Agent-Skills ``` ### Delete All Functions ```bash # List functions aliyun fc list-functions \ --user-agent AlibabaCloud-Agent-Skills # Delete each function aliyun fc delete-function \ --function-name \ --user-agent AlibabaCloud-Agent-Skills ``` ### Delete Code from OSS ```bash aliyun oss rm oss:/// -r \ --user-agent AlibabaCloud-Agent-Skills ``` ### Detach and Delete RAM Policies ```bash # Detach policy from role aliyun ram DetachPolicyFromRole \ --PolicyType Custom \ --PolicyName \ --RoleName fc-execution-role \ --user-agent AlibabaCloud-Agent-Skills # Delete policy aliyun ram DeletePolicy \ --PolicyName \ --user-agent AlibabaCloud-Agent-Skills # Delete role aliyun ram DeleteRole \ --RoleName fc-execution-role \ --user-agent AlibabaCloud-Agent-Skills ``` ### Delete Lambda Functions (AWS Side) ```bash # Delete function aws lambda delete-function --function-name # Delete Lambda execution role (if no longer needed) aws iam delete-role --role-name ``` ## Best Practices ### 1. Runtime Selection - Use latest stable runtime versions available on FC - Match Lambda runtime versions as closely as possible - Consider container images for complex dependencies ### 2. Memory and Timeout Tuning - Start with same memory as Lambda - Monitor actual memory usage and adjust - Keep timeout as low as possible for cost optimization ### 3. Cold Start Optimization - Use provisioned concurrency for latency-sensitive functions - Minimize package size (remove unused dependencies) - Use layers for shared dependencies ### 4. Security - Follow least privilege principle for RAM policies - Use VPC for functions accessing private resources - Enable function encryption for sensitive data ### 5. Monitoring and Observability - Configure SLS logging for all functions - Set up CloudMonitor alarms for errors and duration - Use ARMS for distributed tracing ### 6. Cost Optimization - Right-size memory allocation - Use reserved instances for predictable workloads - Clean up unused functions and versions ## Troubleshooting ### Common Issues | Issue | Cause | Solution | |-------|-------|----------| | `Handler not found` | Incorrect handler path | Verify handler format: `file.function` | | `Timeout exceeded` | Function takes too long | Increase timeout or optimize code | | `Out of memory` | Insufficient memory | Increase memory allocation | | `Permission denied` | RAM policy missing permissions | Add required permissions to role | | `Trigger not firing` | Misconfigured trigger | Verify trigger configuration and permissions | | `Cold start too slow` | Large package or cold environment | Use provisioned mode, optimize package | ### View Function Logs ```bash # Query SLS logs aliyun log GetLogs \ --project \ --logstore \ --from \ --to \ --query "functionName:" \ --user-agent AlibabaCloud-Agent-Skills ``` ### Check Function Status ```bash aliyun fc get-function \ --function-name \ --user-agent AlibabaCloud-Agent-Skills ``` ## Related APIs | API Action | CLI Command | Description | |------------|-------------|-------------| | `CreateFunction` | `aliyun fc create-function ... --user-agent AlibabaCloud-Agent-Skills` | Create function | | `UpdateFunction` | `aliyun fc update-function ... --user-agent AlibabaCloud-Agent-Skills` | Update function configuration | | `DeleteFunction` | `aliyun fc delete-function ... --user-agent AlibabaCloud-Agent-Skills` | Delete function | | `CreateTrigger` | `aliyun fc create-trigger ... --user-agent AlibabaCloud-Agent-Skills` | Create trigger | | `DeleteTrigger` | `aliyun fc delete-trigger ... --user-agent AlibabaCloud-Agent-Skills` | Delete trigger | | `InvokeFunction` | `aliyun fc invoke-function ... --user-agent AlibabaCloud-Agent-Skills` | Invoke function | | `ListFunctions` | `aliyun fc list-functions ... --user-agent AlibabaCloud-Agent-Skills` | List functions | | `GetFunction` | `aliyun fc get-function ... --user-agent AlibabaCloud-Agent-Skills` | Get function details | ## References - [Function Compute Documentation](https://www.alibabacloud.com/help/en/function-compute) - [FC API Reference](https://www.alibabacloud.com/help/en/function-compute/api-reference) - [Runtime Documentation](https://www.alibabacloud.com/help/en/function-compute/developer-reference/runtime) - [Trigger Configuration](https://www.alibabacloud.com/help/en/function-compute/developer-reference/trigger-overview) - [AWS Lambda to FC Migration Guide](https://www.alibabacloud.com/help/en/function-compute/user-guide/migrate-from-aws-lambda) FILE:references/migration-guides/storage-migration-oss.md # Storage Migration: Amazon S3 to Alibaba Cloud OSS ## Overview Primary paths for migrating data from Amazon S3 to Alibaba Cloud OSS: - **ossutil**: Direct S3→OSS copy/sync, incremental snapshots, scripting (typical for most filesystem-style migrations). - **Terraform** (`alicloud_oss_bucket`): Destination bucket and baseline policy/encryption (data transfer itself is not expressed in Terraform). ## ossutil CLI ### Installation ```bash # macOS brew install ossutil # Linux wget --timeout=30 --tries=3 http://gosspublic.alicdn.com/ossutil/1.7.18/ossutil64 chmod +x ossutil64 sudo mv ossutil64 /usr/local/bin/ossutil # Verify installation ossutil version ``` ### Configuration Run `ossutil config` once so OSS endpoint and credentials are available to the tool (follow the interactive prompts). AWS-side access for S3 uses the same default mechanisms as the AWS CLI (`aws configure`, instance profile, etc.). **Configuration file location:** `~/.ossutilconfig` ### Create OSS Bucket ```bash aliyun oss mb oss:// \ --region \ --user-agent AlibabaCloud-Agent-Skills ``` **Parameters:** | Parameter | Required | Description | Example | |-----------|----------|-------------|---------| | `bucket-name` | Yes | OSS bucket name (globally unique) | `my-company-data` | | `region` | Yes | Region where bucket will be created | `cn-hangzhou`, `us-west-1` | ### Terraform Alternative for OSS Bucket ```hcl resource "alicloud_oss_bucket" "migration" { bucket = "" acl = "private" versioning { status = "Enabled" } server_side_encryption_rule { sse_algorithm = "AES256" } } ``` **Note:** Use Terraform for OSS bucket creation. Data transfer operations (`ossutil cp`, `aws s3 sync`, etc.) use CLIs; there is no Terraform equivalent for copying object data. ### Migrate Data from S3 #### Option A: Direct S3 to OSS Transfer ```bash ossutil64 cp s3:/// oss:/// \ -r \ --update \ --snapshot-path=/tmp/snapshot \ --jobs=10 ``` **Parameters:** | Parameter | Description | Example | |-----------|-------------|---------| | `-r` | Recursive copy for directories | | | `--update` | Skip objects that already exist with same size | | | `--snapshot-path` | Enable incremental sync with snapshot file | `/tmp/snapshot` | | `--jobs` | Number of concurrent threads | `10` | #### Option B: Download then Upload ```bash # Step 1: Download from S3 to local aws s3 sync s3:/// /tmp/s3-data/ # Step 2: Upload to OSS ossutil64 cp /tmp/s3-data/ oss:/// \ -r \ --jobs=10 ``` ### Verify Migration ```bash # Count objects in S3 aws s3 ls s3:/// --recursive | wc -l # Count objects in OSS ossutil64 ls oss:/// -r | wc -l # Compare object sizes ossutil64 stat oss://// ``` ### Enable Incremental Sync ```bash # Run periodic sync with snapshot ossutil64 cp s3:/// oss:/// \ -r \ --update \ --snapshot-path=/tmp/s3-oss-snapshot \ --jobs=10 ``` **Cron Job Example:** ```bash # Run every hour 0 * * * * /usr/local/bin/ossutil64 cp s3:/// oss:/// -r --update --snapshot-path=/tmp/snapshot --jobs=10 ``` ## S3 API Compatibility ### OSS S3-Compatible API Alibaba Cloud OSS supports S3-compatible API, allowing applications using AWS SDK to work with minimal changes. ### Endpoint Mapping | AWS S3 | Alibaba Cloud OSS | |--------|-------------------| | `s3.us-east-1.amazonaws.com` | `oss-us-east-1.aliyuncs.com` | | `s3.us-west-2.amazonaws.com` | `oss-us-west-1.aliyuncs.com` | | `s3.ap-southeast-1.amazonaws.com` | `oss-ap-southeast-1.aliyuncs.com` | | `s3.eu-west-1.amazonaws.com` | `oss-eu-west-1.aliyuncs.com` | ### SDK Configuration Changes Use each SDK’s **default credential provider chain** for both AWS S3 and OSS (S3-compatible endpoint). Only `endpoint` / `region` need to change for OSS. #### AWS SDK for Python (boto3) **Before (S3):** ```python import boto3 s3 = boto3.client('s3', region_name='us-east-1') ``` **After (OSS):** ```python import boto3 s3 = boto3.client( 's3', endpoint_url='https://oss-.aliyuncs.com', region_name='oss', ) ``` #### AWS SDK for Java **Before (S3):** ```java AmazonS3 s3 = AmazonS3ClientBuilder.standard() .withRegion("us-east-1") .build(); ``` **After (OSS):** ```java AmazonS3 s3 = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( "https://oss-.aliyuncs.com", "oss")) .build(); ``` #### AWS SDK for Node.js **Before (S3):** ```javascript const s3 = new AWS.S3({ region: 'us-east-1' }); ``` **After (OSS):** ```javascript const s3 = new AWS.S3({ endpoint: 'https://oss-.aliyuncs.com', region: 'oss', }); ``` ### Compatibility Notes | Feature | Compatibility | Notes | |---------|--------------|-------| | PUT/GET/DELETE Object | ✅ Fully Compatible | Direct replacement | | List Objects | ✅ Compatible | Minor parameter differences | | Multipart Upload | ✅ Compatible | Same API structure | | Presigned URLs | ✅ Compatible | Different signing algorithm | | Bucket Policies | ⚠️ Partial | Syntax differences | | S3 Select | ⚠️ Partial | Check OSS documentation | | S3 Inventory | ❌ Not Compatible | Use OSS inventory instead | | S3 Replication | ❌ Not Compatible | Use OSS cross-region replication | ## Handling S3 Object Versioning S3 buckets with versioning enabled store multiple versions of each object, plus delete markers. Standard `ossutil cp` and `aws s3 sync` only transfer **current versions** — previous versions and delete markers are silently skipped, causing object count mismatches during verification. ### Pre-Migration: Check Versioning Status ```bash aws s3api get-bucket-versioning --bucket # "Enabled" or "Suspended" means versioned objects may exist ``` ### Scenario 1: Only Current Versions Needed (Most Common) If you only need the latest version of each object (typical for migration): ```bash # Standard transfer — copies only current versions ossutil64 cp s3:/// oss:/// -r --update --jobs=10 ``` **Verification adjustment** — compare against current-version count only: ```bash # S3: count current versions only (exclude delete markers and non-current) aws s3api list-objects-v2 --bucket --query 'KeyCount' --output text # OSS: count objects ossutil64 ls oss:/// -r --only-count ``` ### Scenario 2: All Versions Needed (Compliance / Audit) If regulatory or audit requirements demand preserving version history: ```bash # Step 1: List all versions aws s3api list-object-versions --bucket \ --query '[Versions[].{Key:Key,VersionId:VersionId,IsLatest:IsLatest},DeleteMarkers[].{Key:Key,VersionId:VersionId}]' \ --output json > versions.json # Step 2: Download each version with version ID preserved in path python3 -c " import json, subprocess, os data = json.load(open('versions.json')) versions = data[0] or [] for v in versions: key, vid = v['Key'], v['VersionId'] dest = f'/tmp/s3-versioned/{key}/__v_{vid}' os.makedirs(os.path.dirname(dest), exist_ok=True) subprocess.run(['aws', 's3api', 'get-object', '--bucket', '', '--key', key, '--version-id', vid, dest], check=True) " # Step 3: Upload to OSS with version path encoding ossutil64 cp /tmp/s3-versioned/ oss:///versioned-archive/ -r --jobs=10 ``` > **Note:** OSS versioning (`versioning.status = "Enabled"`) tracks versions created *after* enabling. It does NOT reconstruct S3 version history. The approach above archives version history as separate objects. ### Scenario 3: Delete Markers S3 delete markers indicate soft-deleted objects. They are **not transferred** by default. ```bash # List delete markers aws s3api list-object-versions --bucket \ --query 'DeleteMarkers[].{Key:Key,VersionId:VersionId}' --output table ``` - If delete markers are for truly deleted data: **ignore** them (don't migrate). - If delete markers need preservation for audit: record them in a manifest file and store alongside migrated data. ### Object Count Reconciliation When source and destination counts don't match, check: | Difference | Cause | Action | |------------|-------|--------| | OSS count < S3 `list-objects-v2` count | Hidden objects (0-byte keys, special chars) | Check with `aws s3api list-objects-v2 --prefix ""` | | OSS count << S3 `list-object-versions` count | Non-current versions not transferred | Expected for Scenario 1; verify current-version count matches | | OSS count < expected after full transfer | Transfer errors / timeouts | Check ossutil logs, re-run with `--update` | ## Migration Best Practices ### 1. Pre-Migration Planning - **Inventory Assessment**: Catalog all S3 buckets, objects, and sizes - **Network Bandwidth**: Estimate migration duration based on available bandwidth - **Cost Estimation**: Calculate S3 egress costs and OSS storage costs - **Dependency Mapping**: Identify applications using S3 and update plans ### 2. Migration Strategy - **Full + Incremental**: Run full migration first, then incremental syncs - **Throttling**: Set speed limits to avoid impacting production workloads - **Parallel Migration**: Migrate multiple buckets simultaneously if bandwidth allows - **Validation**: Plan data validation steps before cutover ### 3. During Migration - **Monitor Progress**: Track transfer rates, error counts, and completion percentage - **Error Handling**: Review and retry failed objects - **Incremental Sync**: Run periodic syncs to minimize cutover window - **Communication**: Keep stakeholders informed of migration status ### 4. Cutover Execution - **Maintenance Window**: Schedule during low-traffic periods - **Stop Writes**: Disable all write operations to source S3 - **Final Sync**: Run final incremental sync to capture remaining changes - **Verification**: Validate object counts, sizes, and checksums - **DNS/Endpoint Update**: Update application endpoints to OSS - **Rollback Plan**: Keep S3 bucket available for emergency rollback ### 5. Post-Migration - **Application Testing**: Verify all application functionality with OSS - **Performance Monitoring**: Monitor OSS performance metrics - **Cost Optimization**: Review storage class and lifecycle policies - **Decommission S3**: Delete S3 bucket after successful migration ### 6. Security Considerations - **Encryption**: Enable server-side encryption on OSS buckets - **Access Control**: Configure RAM policies and bucket policies - **Network Security**: Use VPC endpoints for private network access - **Audit Logging**: Enable OSS access logging for compliance ## Cleanup ### Remove ossutil Configuration ```bash # Remove configuration file rm ~/.ossutilconfig ``` ### Decommission S3 Bucket ```bash # Empty bucket (remove all objects) aws s3 rm s3:// --recursive # Delete bucket aws s3 rb s3:// ``` ### Revoke AWS IAM Permissions ```bash # Delete access keys aws iam delete-access-key --access-key-id --user-name # Delete IAM user (if no longer needed) aws iam delete-user --user-name ``` ### Remove S3 Bucket Policy ```bash aws s3api delete-bucket-policy --bucket ``` ## Troubleshooting ### Common Issues | Issue | Cause | Solution | |-------|-------|----------| | `Access Denied` | Invalid credentials or permissions | Verify IAM/RAM policies and local credential configuration | | `Network Timeout` | Firewall or network connectivity | Check security groups and network ACLs | | `Slow Transfer Speed` | Bandwidth throttling or network latency | Increase parallelism, tune part size / job count | | `Object Upload Failed` | Object size exceeds limit or network issue | Check object size, retry with multipart upload | | `Checksum Mismatch` | Data corruption during transfer | Re-transfer affected objects | | `Permission Denied on OSS` | RAM policy missing required permissions | Add oss:PutObject, oss:GetObject permissions | ### Enable Debug Logging ```bash # ossutil verbose mode ossutil64 cp -r --loglevel=debug ``` ## Related APIs ### OSS | API Action | CLI Command | |------------|-------------| | Create Bucket | `aliyun oss mb oss:// ... --user-agent AlibabaCloud-Agent-Skills` | | List Objects | `aliyun oss ls oss:// ... --user-agent AlibabaCloud-Agent-Skills` | | Copy Object | `aliyun oss cp ... --user-agent AlibabaCloud-Agent-Skills` | | Delete Object | `aliyun oss rm oss:///

Alibabacloud Migrate

Comments (0)