GPU 시작 시간·캐시·비용 운영

set -euo pipefail
cd "$ISAAC_REPO"
read -r -p "측정할 실제 Isaac 실행 ID: " ISAAC_METRIC_ID
export ISAAC_METRIC_ID
isaac run --id "$ISAAC_METRIC_ID" > "$ISAAC_EVIDENCE/metrics-run.json"
jq '{id,state,createdAt,updatedAt,attempts,checkpointCompleted,gpu,cache,timings,
     readyEvidence,metrics}' "$ISAAC_EVIDENCE/metrics-run.json"
ik -n robot-atlas-isaac get pods -l "robot-atlas/run=$ISAAC_METRIC_ID" -o json \
  > "$ISAAC_EVIDENCE/metrics-pods.json"
ik -n robot-atlas-isaac get events -o json > "$ISAAC_EVIDENCE/namespace-events.json"

jq '{state,metrics,cacheCandidate}' "$ISAAC_EVIDENCE/metrics-run.json"
isaac promote-cache --id "$ISAAC_METRIC_ID" \
  > "$ISAAC_EVIDENCE/cache-promotion.json"
jq '{published,identity,sha256,files}' "$ISAAC_EVIDENCE/cache-promotion.json"

isaac submit --scenario transfer --seed 42 --order red \
  > "$ISAAC_EVIDENCE/cache-repeat-submit.json"
export ISAAC_REPEAT_ID="$(jq -er '.id' "$ISAAC_EVIDENCE/cache-repeat-submit.json")"
isaac_wait_run "$ISAAC_REPEAT_ID" 1800
isaac run --id "$ISAAC_REPEAT_ID" > "$ISAAC_EVIDENCE/cache-repeat-run.json"
jq '{id,state,gpu,cache,timings,metrics}' "$ISAAC_EVIDENCE/cache-repeat-run.json"

"$ISAAC_PYTHON" - <<'PY'
import json
from infra.isaac import deployment as d, operations
cfg = d.config()
aws = d.session(cfg)
images = operations.image_records()
deployment = operations.kubectl(
    aws, cfg, "-n", cfg["namespace"], "get", "deployment", "isaac-controller",
    "-o", "json", json_result=True)
container = next(c for c in deployment["spec"]["template"]["spec"]["containers"]
                 if c["name"] == "controller")
env = {e["name"]: e["value"] for e in container["env"] if "value" in e}
assert container["image"] == images["controller"]
assert env["ISAAC_WORKER_IMAGE"] == images["worker"]
print(json.dumps({"currentProfileImagesDeployed": True, "images": images}, indent=2))
PY

isaac enable-ondemand
ik get nodepool isaac-sim-ondemand -o json \
  > "$ISAAC_EVIDENCE/ondemand-pool.json"
jq '{name:.metadata.name,nodeClass:.spec.template.spec.nodeClassRef,
     requirements:.spec.template.spec.requirements,limits:.spec.limits}' \
  "$ISAAC_EVIDENCE/ondemand-pool.json"

isaac submit --scenario transfer --seed 42 --order red \
  --capacity-type on-demand --memory-profile rtx96 \
  > "$ISAAC_EVIDENCE/optional-submit.json"
export ISAAC_OPTIONAL_ID="$(jq -er '.id' "$ISAAC_EVIDENCE/optional-submit.json")"
jq -e '.operatorCapacityType == "on-demand" and .minimumVramMiB == 90000' \
  "$ISAAC_EVIDENCE/optional-submit.json"
isaac_wait_run "$ISAAC_OPTIONAL_ID" 1800
isaac run --id "$ISAAC_OPTIONAL_ID" > "$ISAAC_EVIDENCE/optional-run.json"
jq '{id,state,operatorCapacityType,minimumVramMiB,gpu,readyEvidence,metrics}' \
  "$ISAAC_EVIDENCE/optional-run.json"
ik -n robot-atlas-isaac get job "$ISAAC_OPTIONAL_ID" -o json \
  > "$ISAAC_EVIDENCE/optional-job.json"
jq -e '.spec.template.spec.nodeSelector["karpenter.sh/nodepool"] == "isaac-sim-ondemand"
       and ([.spec.template.spec.containers[].env[]? |
             select(.name == "ISAAC_MIN_VRAM_MIB") | .value] == ["90000"])' \
  "$ISAAC_EVIDENCE/optional-job.json"
jq -e '.state == "succeeded" and .gpu.memoryMiB >= 90000' \
  "$ISAAC_EVIDENCE/optional-run.json"

isaac warm --replicas 1
ik -n robot-atlas-isaac rollout status deployment/isaac-warm --timeout=900s
ik -n robot-atlas-isaac get deployment isaac-warm -o json \
  > "$ISAAC_EVIDENCE/warm-holder.json"
jq '{replicas:.spec.replicas,priority:.spec.template.spec.priorityClassName,
     containers:[.spec.template.spec.containers[] | {name,image,command,resources}]}' \
  "$ISAAC_EVIDENCE/warm-holder.json"

isaac warm --replicas 0
ik -n robot-atlas-isaac get deployment isaac-warm -o json \
  | jq '{replicas:.spec.replicas,readyReplicas:(.status.readyReplicas // 0)}'

"$ISAAC_PYTHON" - <<'PY'
from datetime import datetime, timezone
import json, os
from pathlib import Path
from infra.isaac import deployment as d
cfg = d.config()
aws = d.session(cfg)
now = datetime.now(timezone.utc)
types = [f"{family}.{size}" for family in ("g6e", "g7e")
         for size in ("2xlarge", "4xlarge", "8xlarge")]
ec2 = aws.client("ec2")
allowed_azs = {s["AvailabilityZone"] for s in ec2.describe_subnets(
    SubnetIds=cfg["privateSubnetIds"])["Subnets"]}
rows = []
pages = ec2.get_paginator("describe_spot_price_history").paginate(
    InstanceTypes=types, ProductDescriptions=["Linux/UNIX"], StartTime=now)
for page in pages:
    for row in page["SpotPriceHistory"]:
        if row["AvailabilityZone"] not in allowed_azs:
            continue
        rows.append({
            "instanceType": row["InstanceType"], "az": row["AvailabilityZone"],
            "priceUSDPerHour": row["SpotPrice"], "priceTimestamp": row["Timestamp"].isoformat(),
        })
assert rows, "No price records returned; do not assume zero cost or available capacity"
report = {"queriedAt": now.isoformat(), "region": cfg["region"], "prices": rows}
path = Path(os.environ["ISAAC_EVIDENCE"]) / "spot-prices-current.json"
path.write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
PY