Merge pull request #215 from spacetelescope/release/v0.4.48

Release/v0.4.48

Author: raswaters

calcloud/batch.py

@@ -13,7 +13,6 @@
 
 from . import common
 
-
 JOB_STATUSES = tuple("SUBMITTED|PENDING|RUNNABLE|STARTING|RUNNING|SUCCEEDED|FAILED".split("|"))
 
 KILL_STATUSES = tuple("SUBMITTED|PENDING|RUNNABLE|STARTING|RUNNING".split("|"))

changelog.md

@@ -1 +1,5 @@
 - update caldp version and changelog.md
+- add point-of-contact email tag
+- dependency updates
+- only retrieve latest 24 images
+- add EPH (ephemeral) to image names

lambda/JobPredict/Dockerfile

@@ -20,7 +20,9 @@ RUN mv /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-bundle.crt.org && ln -s /e
 RUN yum update -y java-1.8.0-openjdk
 
 #RUN python3.7 -m pip install --upgrade pip && python3.7 -m pip install -r requirements.txt && python3.7 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.6.2-cp37-cp37m-manylinux2010_x86_64.whl
-RUN python3.11 -m pip install --upgrade pip && python3.11 -m pip install -r requirements.txt && python3.11 -m pip install tensorflow-cpu==2.12.1
+# Install numpy, h5py, scipy, and ml_dtypes first from wheels so later deps don't trigger source builds (no gcc in image).
+RUN python3.11 -m pip install --upgrade pip && python3.11 -m pip install "numpy>=1.23,<2" "h5py==3.10.0" "scipy==1.11.4" "ml_dtypes==0.5.1"
+RUN python3.11 -m pip install -r requirements.txt
 ADD  models ./models/
 RUN chmod -R ugo+r ./models/
 RUN find ./models/ -type d -exec chmod og+x {} +

lambda/JobPredict/requirements.txt

@@ -1,3 +1,8 @@
+numpy>=1.23,<2
+h5py==3.10.0
+scipy==1.11.4
+ml_dtypes==0.5.1
+tensorflow-cpu==2.12.1
 scikit-learn==1.3.0
 boto3==1.28.56
 protobuf==4.24.3

terraform/batch.tf

@@ -1,7 +1,12 @@
 
 
 provider "aws" {
-  region  = var.region
+  region = var.region
+  default_tags {
+    tags = {
+      stsci-poc-email = var.stsci_poc_email
+    }
+  }
 }
 
 terraform {

terraform/deploy.sh

@@ -14,9 +14,14 @@ echo $aws_tfstate
 
 # get AMI id(s)
 cd $CALCLOUD_BUILD_DIR/ami_rotation
-ami_json=$(echo $(awsudo $ADMIN_ARN aws ec2 describe-images --region us-east-1 --executable-users self))
+# Limit responses to the most recent 24 images to prevent "Argument list too long"
+query='{Images: reverse(sort_by(Images, &CreationDate))[:24]}'
+ami_json="$(
+  awsudo "$ADMIN_ARN" \
+  "aws ec2 describe-images --region us-east-1 --executable-users self --query '$query' --output json"
+)"
 ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AMAZON-LINUX2023`
-ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-ECS-AL2023`
+ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-EPH-ECS-AL2023`
 
 if [[ "$ci_ami" =~ ^ami-[a-z0-9]+$ ]]; then
     echo $ci_ami
@@ -83,8 +88,8 @@ awsudo $ADMIN_ARN aws s3api put-object --bucket $bucket_url --key crds_env_vars/
 awsudo $ADMIN_ARN aws s3api put-object --bucket $bucket_url --key crds_env_vars/${crds_context}
 
 # tag images as in-use by this environment
-cd $CALCLOUD_BUILD_DIR/terraform 
-# this script doesn't replace "old-tag", it just takes that image, and adds this tag to it. 
+cd $CALCLOUD_BUILD_DIR/terraform
+# this script doesn't replace "old-tag", it just takes that image, and adds this tag to it.
 # And in fact, if another image exists with this tag, it's removed from that one; so this is image promotion all-in-one
 echo ${aws_env}
 ./deploy_image_promote.sh --old-tag $CALDP_ECR_TAG batch-${aws_env}

terraform/deploy_ami_rotate.sh

@@ -20,7 +20,7 @@ CSYS_VER=`echo $CSYS_VER | tr -d '",'`
 # i.e. CALCLOUD_BUILD_DIR="$HOME/deployer/calcloud"
 # these can be set as environment variables before running to avoid changing the script directly
 # (and avoid accidentally committing a custom path to the repo...)
-CALCLOUD_BUILD_DIR=${CALCLOUD_BUILD_DIR:-""} 
+CALCLOUD_BUILD_DIR=${CALCLOUD_BUILD_DIR:-""}
 CALDP_BUILD_DIR=${CALDP_BUILD_DIR:-""}
 
 # variables that will be changed less-frequently
@@ -50,7 +50,7 @@ fi
 
 # check for Batch jobs and exit if any exist that are running or should be soon
 cd ${CALCLOUD_BUILD_DIR}/scripts
-./check_batch_jobs.py 
+./check_batch_jobs.py
 batch_jobs=$?
 if [[ $batch_jobs -ne 0 ]]; then
     echo "there are running or submitted batch jobs; cannot rotate ami"
@@ -77,7 +77,7 @@ echo $aws_tfstate
 cd $CALCLOUD_BUILD_DIR/ami_rotation
 ami_json=$(echo $(awsudo $ADMIN_ARN aws ec2 describe-images --region us-east-1 --executable-users self))
 ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AMAZON-LINUX2023`
-ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-ECS-AL2023`
+ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-EPH-ECS-AL2023`
 
 if [[ "$ci_ami" =~ ^ami-[a-z0-9]+$ ]]; then
     echo $ci_ami

terraform/deploy_ami_rotate_codebuild_script.sh

@@ -21,7 +21,7 @@ CSYS_VER=`echo $CSYS_VER | tr -d '",'`
 # i.e. CALCLOUD_BUILD_DIR="$HOME/deployer/calcloud"
 # these can be set as environment variables before running to avoid changing the script directly
 # (and avoid accidentally committing a custom path to the repo...)
-CALCLOUD_BUILD_DIR=${CALCLOUD_BUILD_DIR:-""} 
+CALCLOUD_BUILD_DIR=${CALCLOUD_BUILD_DIR:-""}
 CALDP_BUILD_DIR=${CALDP_BUILD_DIR:-""}
 
 # variables that will be changed less-frequently
@@ -78,7 +78,7 @@ echo $aws_tfstate
 cd $CALCLOUD_BUILD_DIR/ami_rotation
 ami_json=$(echo $(aws ec2 describe-images --region us-east-1 --executable-users self))
 ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AMAZON-LINUX2023`
-ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-ECS-AL2023`
+ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-EPH-ECS-AL2023`
 
 if [[ "$ci_ami" =~ ^ami-[a-z0-9]+$ ]]; then
     echo $ci_ami

terraform/deploy_vars.sh

@@ -1,7 +1,7 @@
 #! /bin/bash -xu
-export CALCLOUD_VER="v0.4.47"
-export CALDP_VER="v0.2.29"
-export CAL_BASE_IMAGE="stsci/hst-pipeline:2025.3.1.7-alignmentmatches-py311"
+export CALCLOUD_VER="v0.4.48"
+export CALDP_VER="v0.2.30"
+export CAL_BASE_IMAGE="stsci/hst-pipeline:2026.2.1.7-arcticdrizzle-py312"
 
 export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} `
 

terraform/variables.tf

@@ -20,6 +20,12 @@ variable region {
   default = "us-east-1"
 }
 
+variable "stsci_poc_email" {
+  description = "Value for mandatory AWS tag stsci-poc-email for KMS, RDS, or S3 resources in STScI SDLC accounts."
+  type        = string
+  default     = "team-violet@stsci.edu"
+}
+
 variable awsdpver {
   description = "HST keyword for caldp repo tag version"
   type = string
@@ -117,10 +123,10 @@ variable crds_context {
   description = "the crds context to be the default for the environment"
   type = map(string)
   default = {
-    "-sb" = "hst_1241.pmap"
-    "-dev" = "hst_1241.pmap"
-    "-test" = "hst_1247.pmap"
-    "-ops" = "hst_1247.pmap"
+    "-sb" = "hst_1298.pmap"
+    "-dev" = "hst_1298.pmap"
+    "-test" = "hst_1323.pmap"
+    "-ops" = "hst_1323.pmap"
   }
 }