How to Configure 10 GB Memory for AWS Lambda Raster Processing

aws lambda update-function-configuration \
  --function-name raster-processor \
  --memory-size 10240 \
  --timeout 900 \
  --ephemeral-storage '{"Size": 10240}' \
  --environment 'Variables={
    GDAL_CACHEMAX=2048,
    GDAL_NUM_THREADS=4,
    GDAL_DATA=/opt/share/gdal,
    PROJ_LIB=/opt/share/proj,
    LD_LIBRARY_PATH=/opt/lib,
    GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
  }'

resource "aws_lambda_function" "raster_processor" {
  function_name = "raster-processor"
  runtime       = "python3.11"
  handler       = "handler.process_tile"
  memory_size   = 10240
  timeout       = 900

  ephemeral_storage {
    size = 10240
  }

  environment {
    variables = {
      # Hard limits: GDAL cache capped at 2 GB to leave headroom for numpy buffers
      GDAL_CACHEMAX                  = "2048"
      GDAL_NUM_THREADS               = "4"
      # Explicit paths — never rely on container defaults
      GDAL_DATA                      = "/opt/share/gdal"
      PROJ_LIB                       = "/opt/share/proj"
      LD_LIBRARY_PATH                = "/opt/lib"
      GDAL_DISABLE_READDIR_ON_OPEN   = "EMPTY_DIR"
    }
  }
}

Resources:
  RasterProcessor:
    Type: AWS::Serverless::Function
    Properties:
      Runtime: python3.11
      Handler: handler.process_tile
      MemorySize: 10240
      Timeout: 900
      EphemeralStorage:
        Size: 10240
      Environment:
        Variables:
          GDAL_CACHEMAX: "2048"
          GDAL_NUM_THREADS: "4"
          GDAL_DATA: "/opt/share/gdal"
          PROJ_LIB: "/opt/share/proj"
          LD_LIBRARY_PATH: "/opt/lib"
          GDAL_DISABLE_READDIR_ON_OPEN: "EMPTY_DIR"

import os
import gc
import uuid
import logging
import rasterio
from rasterio.windows import Window
import numpy as np

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def _check_tmp_headroom(required_mb: int = 2048) -> None:
    """Abort early if /tmp does not have enough free space."""
    stat = os.statvfs("/tmp")
    free_mb = (stat.f_frsize * stat.f_bavail) / (1024 * 1024)
    if free_mb < required_mb:
        raise RuntimeError(
            f"/tmp headroom too low: {free_mb:.0f} MB free, {required_mb} MB required"
        )


def process_tile(event: dict, context) -> dict:
    """
    Process a raster tile from S3 using windowed reads.
    Requires MemorySize=10240, EphemeralStorage=10240, and explicit GDAL env vars.
    """
    # Convert s3:// URI to the GDAL virtual filesystem path so rasterio
    # streams byte ranges directly without a full download to /tmp.
    src_path = event["s3_uri"].replace("s3://", "/vsis3/")

    # Isolate each invocation under a unique subdirectory so warm-start
    # container reuse does not corrupt previous outputs.
    invocation_dir = f"/tmp/{uuid.uuid4().hex}"
    os.makedirs(invocation_dir, exist_ok=True)
    dst_path = os.path.join(invocation_dir, os.path.basename(src_path))

    # Guard against /tmp exhaustion before writing any data.
    _check_tmp_headroom(required_mb=2048)

    window_size = 1024  # pixels per tile dimension; align with COG block size

    with rasterio.open(src_path) as src:
        profile = src.profile.copy()
        profile.update(
            dtype=rasterio.uint16,
            count=1,
            compress="deflate",
            tiled=True,
            blockxsize=512,
            blockysize=512,
            # Ensure GDAL writes the COG-compatible layout
            interleave="band",
        )

        with rasterio.open(dst_path, "w", **profile) as dst:
            for row_off in range(0, src.height, window_size):
                for col_off in range(0, src.width, window_size):
                    width  = min(window_size, src.width  - col_off)
                    height = min(window_size, src.height - row_off)
                    window = Window(col_off, row_off, width, height)

                    # Read only the current 1024×1024 window — not the full raster.
                    # At 10240 MB, ~6 vCPUs decode tiles in parallel across bands.
                    data = src.read(window=window)

                    # Band normalisation: scale uint16 to 0-100 integer range
                    processed = np.clip(
                        (data.astype(np.float32) / 65535.0 * 100),
                        0, 100
                    ).astype(np.uint16)

                    dst.write(processed, window=window)

                    # Explicit deallocation releases C-extension buffers before
                    # gc runs automatically — critical for back-to-back windows.
                    del data, processed
                    gc.collect()

    logger.info("Processed %s → %s", src_path, dst_path)
    return {"statusCode": 200, "output_path": dst_path}

# Invoke the function with a representative test event and capture the log output
aws lambda invoke \
  --function-name raster-processor \
  --cli-binary-format raw-in-base64-out \
  --payload '{"s3_uri":"s3://your-bucket/test-tile.tif"}' \
  --log-type Tail \
  response.json \
  --query 'LogResult' --output text | base64 --decode

REPORT RequestId: … Duration: 4321.00 ms  Billed Duration: 4400 ms
Memory Size: 10240 MB  Max Memory Used: 3847 MB  Init Duration: 2100.00 ms

import os
from osgeo import gdal

# Confirm GDAL sees the cache cap at runtime
cache_mb = gdal.GetCacheMax() / (1024 * 1024)
print(f"GDAL cache cap: {cache_mb:.0f} MB")  # Expected: 2048 MB
assert cache_mb == 2048, f"GDAL_CACHEMAX not applied — got {cache_mb} MB"

aws cloudwatch get-metric-statistics \
  --namespace AWS/Lambda \
  --metric-name MaxMemoryUsed \
  --dimensions Name=FunctionName,Value=raster-processor \
  --start-time "$(date -u -d '-1 hour' '+%Y-%m-%dT%H:%M:%SZ')" \
  --end-time "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
  --period 300 \
  --statistics Maximum \
  --output table