{ "cells": [ { "cell_type": "markdown", "id": "f5f4c479-c6f1-4d81-9241-bbfe542a5ca4", "metadata": {}, "source": [ "#### This script evaluates CMIP6 models over the 1985–2014 period by comparing daily maximum temperature (Tmax) to ERA5-Land reference data. It assesses their ability to reproduce the seasonal cycle (temporal correlation) and the spatial structure and bias of heat extremes using the Tx5day index. Models are then classified (Good, Reasonable, Bad) following the WWA protocol to retain a robust subset." ] }, { "cell_type": "code", "execution_count": 5, "id": "6ee817a9-1441-4eb7-ae53-1bc33fcedd06", "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "import numpy as np\n", "import pandas as pd\n", "import xarray as xr\n", "import xesmf as xe\n", "from scipy.stats import pearsonr\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 6, "id": "6d6b77aa-7a82-4ecb-a1de-a2128dfe2000", "metadata": {}, "outputs": [], "source": [ "ref_start = \"1985-01-01\"\n", "ref_end = \"2014-12-31\"\n", "var = \"tasmax\"\n", "\n", "# --- Rating functions ---\n", "\n", "def get_bias_rating(b):\n", " \"\"\"Rate model based on domain-mean Tx5day bias (°C).\"\"\"\n", " if -2.0 <= b <= 2.0:\n", " return \"Good\"\n", " elif -4.0 <= b <= 4.0:\n", " return \"Reasonable\"\n", " return \"Bad\"\n", "\n", "def get_spatial_rating(c):\n", " \"\"\"Rate model based on Tx5day spatial correlation.\"\"\"\n", " if c >= 0.80:\n", " return \"Good\"\n", " elif c >= 0.60:\n", " return \"Reasonable\"\n", " return \"Bad\"\n", "\n", "def get_seasonal_rating(c):\n", " \"\"\"Rate model based on 12-month seasonal cycle correlation.\"\"\"\n", " if c >= 0.90:\n", " return \"Good\"\n", " elif c >= 0.80:\n", " return \"Reasonable\"\n", " return \"Bad\"\n", "\n", "def get_overall_rating(ratings):\n", " \"\"\"Combine individual ratings into an overall rating.\"\"\"\n", " if \"Bad\" in ratings:\n", " return \"Bad\"\n", " if \"Reasonable\" in ratings:\n", " return \"Reasonable\"\n", " return \"Good\"\n", "\n", "# --- Helper functions ---\n", "\n", "def get_lat_lon_names(da):\n", " \"\"\"Detect lat/lon coordinate names in a DataArray.\"\"\"\n", " lat = next((c for c in [\"lat\", \"latitude\", \"y\"] if c in da.coords), None)\n", " lon = next((c for c in [\"lon\", \"longitude\", \"x\"] if c in da.coords), None)\n", " if lat is None or lon is None:\n", " raise ValueError(\"Lat/Lon not found\")\n", " return lat, lon\n", "\n", "def format_coords(da):\n", " \"\"\"Standardize lat/lon names, sort, wrap lon to [-180,180], convert K->°C.\n", " Matches exactly the format_coords from model_evaluation.ipynb.\"\"\"\n", " lat_n, lon_n = get_lat_lon_names(da)\n", " da = da.rename({lat_n: \"lat\", lon_n: \"lon\"})\n", " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(['lat', 'lon'])\n", " if float(da.max().compute()) > 150:\n", " da = da - 273.15\n", " return da.compute()\n", "\n", "def detect_tasmax_var(ds):\n", " \"\"\"Detect the tasmax variable name in a dataset (handles tmax, tasmax, etc.).\"\"\"\n", " for candidate in [\"tasmax\", \"tmax\", \"TMAX\", \"Tmax\", \"t2m\"]:\n", " if candidate in ds.data_vars:\n", " return candidate\n", " # Fallback: return the first data variable\n", " return list(ds.data_vars)[0]" ] }, { "cell_type": "code", "execution_count": 7, "id": "0afbcea3-08d0-462e-bdab-02a95238a97c", "metadata": {}, "outputs": [], "source": [ "# DEFINE OBSERVATIONAL REFERENCES\n", "# Each reference has:\n", "# - wcape_pattern: glob pattern for spatial/bias data (Western Cape domain)\n", "# - event_file: single merged file for temporal data (event domain)\n", "\n", "obs_references = {\n", " \"ERA5-Land\": {\n", " \"wcape_pattern\": os.path.expanduser(\n", " \"~/ctheatwave_2026/data/reanalysis/global/reanalysis/ECMWF/ERA5-Land/day/wcape/tasmax_*.nc\"\n", " ),\n", " \"event_file\": \"/terra/projects/ctheatwave_2026/data/reanalysis/global/reanalysis/ECMWF/ERA5-Land/day/event_domain/tasmax_day_ECMWF_ERA5-Land_merged.nc\",\n", " },\n", " \"CPC\": {\n", " \"wcape_pattern\": os.path.expanduser(\n", " \"~/ctheatwave_2026/data/observed/global/gridded/CPC/Global-Daily-Temperature/day/wcape/*max_*.nc\"\n", " ),\n", " \"event_file\": \"/terra/projects/ctheatwave_2026/data/observed/global/gridded/CPC/Global-Daily-Temperature/day/event_domain/tasmax_day_CPC_Global-Daily-Temperature_merged.nc\",\n", " },\n", " \"MSWX\": {\n", " \"wcape_pattern\": os.path.expanduser(\n", " \"~/ctheatwave_2026/data/observed/global/gridded/GloH2O/MSWX/day/wcape/tasmax_*.nc\"\n", " ),\n", " \"event_file\": \"/terra/projects/ctheatwave_2026/data/observed/global/gridded/GloH2O/MSWX/day/event_domain/tasmax_day_GloH2O_MSWX_merged.nc\",\n", " },\n", "}" ] }, { "cell_type": "code", "execution_count": 8, "id": "2823e994-34fd-469c-bc54-7b81ef780a96", "metadata": {}, "outputs": [], "source": [ "# CMIP6 MODEL FILES \n", "\n", "cmip6_wcape_pattern = \"/terra/projects/ctheatwave_2026/data/cmip6/global/historical/*/*/day/wcape/tasmax_*.nc\"\n", "cmip6_event_pattern = \"/terra/projects/ctheatwave_2026/data/cmip6/global/historical/*/*/day/event_domain/tasmax_*.nc\"\n", "\n", "cmip6_wcape_files = glob.glob(cmip6_wcape_pattern)\n", "cmip6_event_files = glob.glob(cmip6_event_pattern)\n", "\n", "model_wcape_files = {f.split(\"/\")[-5]: f for f in cmip6_wcape_files}\n", "model_event_files = {f.split(\"/\")[-5]: f for f in cmip6_event_files}\n", "\n", "selected_models = [\n", " \"EC-Earth3\", \"EC-Earth3-CC\", \"EC-Earth3-Veg\", \"ACCESS-CM2\", \"ACCESS-ESM1-5\", \n", " \"AWI-CM-1-1-MR\", \"BCC-CSM2-MR\", \"CMCC-ESM2\", \"EC-Earth3-Veg-LR\", \"FGOALS-g3\", \n", " \"GFDL-ESM4\", \"INM-CM4-8\", \"IPSL-CM6A-LR\", \"KACE-1-0-G\", \"MRI-ESM2-0\", \n", " \"NorESM2-LM\", \"NorESM2-MM\", \"TaiESM1\", \"CanESM5\", \"INM-CM5-0\", \"KIOST-ESM\", \n", " \"MIROC6\", \"MPI-ESM1-2-LR\", \"NESM3\"\n", "]\n", "\n", "common_models = sorted(set(model_wcape_files.keys()) & set(model_event_files.keys()) & set(selected_models))" ] }, { "cell_type": "code", "execution_count": 18, "id": "515c70a3-6083-4964-a091-da087d4cd436", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "######################################################################\n", "### REFERENCE: ERA5-Land\n", "######################################################################\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " [Spatial/Bias] ERA5-Land wcape Tx5day reference loaded.\n", " Tx5day climatology shape: (111, 121)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " [Temporal] ERA5-Land event_domain reference loaded.\n", " Seasonal cycle shape: (12,)\n", "\n", " ============================================================\n", " Evaluating selected models vs ERA5-Land: 24 / 24\n", " ============================================================\n", "\n", " Evaluating: ACCESS-CM2 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.6337°C (Good), Spat: 0.6920 (Reasonable), Seas: 0.9965 (Good) -> Reasonable\n", "\n", " Evaluating: ACCESS-ESM1-5 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.3822°C (Good), Spat: 0.7136 (Reasonable), Seas: 0.9959 (Good) -> Reasonable\n", "\n", " Evaluating: AWI-CM-1-1-MR vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.1028°C (Good), Spat: 0.7941 (Reasonable), Seas: 0.9984 (Good) -> Reasonable\n", "\n", " Evaluating: BCC-CSM2-MR vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.0240°C (Reasonable), Spat: 0.7564 (Reasonable), Seas: 0.9882 (Good) -> Reasonable\n", "\n", " Evaluating: CMCC-ESM2 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -1.3046°C (Good), Spat: 0.7143 (Reasonable), Seas: 0.9954 (Good) -> Reasonable\n", "\n", " Evaluating: CanESM5 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +6.5335°C (Bad), Spat: 0.2067 (Bad), Seas: 0.9928 (Good) -> Bad\n", "\n", " Evaluating: EC-Earth3 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.6756°C (Good), Spat: 0.8230 (Good), Seas: 0.9973 (Good) -> Good\n", "\n", " Evaluating: EC-Earth3-CC vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.8960°C (Good), Spat: 0.8186 (Good), Seas: 0.9979 (Good) -> Good\n", "\n", " Evaluating: EC-Earth3-Veg vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.4867°C (Good), Spat: 0.8017 (Good), Seas: 0.9968 (Good) -> Good\n", "\n", " Evaluating: EC-Earth3-Veg-LR vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.0445°C (Good), Spat: 0.6566 (Reasonable), Seas: 0.9927 (Good) -> Reasonable\n", "\n", " Evaluating: FGOALS-g3 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.3209°C (Good), Spat: 0.7684 (Reasonable), Seas: 0.9964 (Good) -> Reasonable\n", "\n", " Evaluating: GFDL-ESM4 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.0268°C (Good), Spat: 0.7254 (Reasonable), Seas: 0.9966 (Good) -> Reasonable\n", "\n", " Evaluating: INM-CM4-8 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.0329°C (Good), Spat: 0.6234 (Reasonable), Seas: 0.9986 (Good) -> Reasonable\n", "\n", " Evaluating: INM-CM5-0 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.2569°C (Good), Spat: 0.5944 (Bad), Seas: 0.9973 (Good) -> Bad\n", "\n", " Evaluating: IPSL-CM6A-LR vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -3.8154°C (Reasonable), Spat: 0.6894 (Reasonable), Seas: 0.9954 (Good) -> Reasonable\n", "\n", " Evaluating: KACE-1-0-G vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.9933°C (Good), Spat: 0.6229 (Reasonable), Seas: 0.9976 (Good) -> Reasonable\n", "\n", " Evaluating: KIOST-ESM vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +14.0499°C (Bad), Spat: 0.6069 (Reasonable), Seas: 0.9700 (Good) -> Bad\n", "\n", " Evaluating: MIROC6 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +12.4194°C (Bad), Spat: 0.6305 (Reasonable), Seas: 0.9881 (Good) -> Bad\n", "\n", " Evaluating: MPI-ESM1-2-LR vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.1346°C (Good), Spat: 0.5933 (Bad), Seas: 0.9980 (Good) -> Bad\n", "\n", " Evaluating: MRI-ESM2-0 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.8071°C (Reasonable), Spat: 0.8234 (Good), Seas: 0.9970 (Good) -> Reasonable\n", "\n", " Evaluating: NESM3 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -4.3075°C (Bad), Spat: 0.4867 (Bad), Seas: 0.9834 (Good) -> Bad\n", "\n", " Evaluating: NorESM2-LM vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.1480°C (Good), Spat: 0.7383 (Reasonable), Seas: 0.9966 (Good) -> Reasonable\n", "\n", " Evaluating: NorESM2-MM vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -1.3776°C (Good), Spat: 0.7718 (Reasonable), Seas: 0.9964 (Good) -> Reasonable\n", "\n", " Evaluating: TaiESM1 vs ERA5-Land\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -1.4984°C (Good), Spat: 0.7283 (Reasonable), Seas: 0.9954 (Good) -> Reasonable\n", "\n", " ============================================================\n", " === ERA5-Land EVALUATION COMPLETE ===\n", " ============================================================\n", " Models retained: 18 / 24\n", " WWA Table saved to: ../figures/evaluation_outputs/evaluation_cmip6_vs_ERA5-Land.csv\n", " Good: 3 model(s)\n", " Reasonable: 15 model(s)\n", " Bad: 6 model(s)\n", "\n", "######################################################################\n", "### REFERENCE: CPC\n", "######################################################################\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " [Spatial/Bias] CPC wcape Tx5day reference loaded.\n", " Tx5day climatology shape: (22, 24)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " [Temporal] CPC event_domain reference loaded.\n", " Seasonal cycle shape: (12,)\n", "\n", " ============================================================\n", " Evaluating selected models vs CPC: 24 / 24\n", " ============================================================\n", "\n", " Evaluating: ACCESS-CM2 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.8152°C (Good), Spat: 0.7498 (Reasonable), Seas: 0.9963 (Good) -> Reasonable\n", "\n", " Evaluating: ACCESS-ESM1-5 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.8028°C (Good), Spat: 0.7836 (Reasonable), Seas: 0.9965 (Good) -> Reasonable\n", "\n", " Evaluating: AWI-CM-1-1-MR vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.6926°C (Good), Spat: 0.7566 (Reasonable), Seas: 0.9989 (Good) -> Reasonable\n", "\n", " Evaluating: BCC-CSM2-MR vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +1.3700°C (Good), Spat: 0.7254 (Reasonable), Seas: 0.9909 (Good) -> Reasonable\n", "\n", " Evaluating: CMCC-ESM2 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -1.4158°C (Good), Spat: 0.6496 (Reasonable), Seas: 0.9957 (Good) -> Reasonable\n", "\n", " Evaluating: CanESM5 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +5.8913°C (Bad), Spat: 0.7010 (Reasonable), Seas: 0.9915 (Good) -> Bad\n", "\n", " Evaluating: EC-Earth3 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.6233°C (Good), Spat: 0.7623 (Reasonable), Seas: 0.9975 (Good) -> Reasonable\n", "\n", " Evaluating: EC-Earth3-CC vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.8665°C (Good), Spat: 0.7593 (Reasonable), Seas: 0.9983 (Good) -> Reasonable\n", "\n", " Evaluating: EC-Earth3-Veg vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.4417°C (Good), Spat: 0.7459 (Reasonable), Seas: 0.9971 (Good) -> Reasonable\n", "\n", " Evaluating: EC-Earth3-Veg-LR vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.3828°C (Good), Spat: 0.6561 (Reasonable), Seas: 0.9925 (Good) -> Reasonable\n", "\n", " Evaluating: FGOALS-g3 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.5863°C (Good), Spat: 0.6774 (Reasonable), Seas: 0.9966 (Good) -> Reasonable\n", "\n", " Evaluating: GFDL-ESM4 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.0589°C (Good), Spat: 0.6562 (Reasonable), Seas: 0.9979 (Good) -> Reasonable\n", "\n", " Evaluating: INM-CM4-8 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.0570°C (Good), Spat: 0.4247 (Bad), Seas: 0.9987 (Good) -> Bad\n", "\n", " Evaluating: INM-CM5-0 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.3468°C (Good), Spat: 0.3466 (Bad), Seas: 0.9968 (Good) -> Bad\n", "\n", " Evaluating: IPSL-CM6A-LR vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -3.5641°C (Reasonable), Spat: 0.5566 (Bad), Seas: 0.9969 (Good) -> Bad\n", "\n", " Evaluating: KACE-1-0-G vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.8338°C (Good), Spat: 0.6183 (Reasonable), Seas: 0.9985 (Good) -> Reasonable\n", "\n", " Evaluating: KIOST-ESM vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +13.4264°C (Bad), Spat: 0.8061 (Good), Seas: 0.9669 (Good) -> Bad\n", "\n", " Evaluating: MIROC6 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +11.8753°C (Bad), Spat: 0.6876 (Reasonable), Seas: 0.9874 (Good) -> Bad\n", "\n", " Evaluating: MPI-ESM1-2-LR vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.1553°C (Good), Spat: 0.5889 (Bad), Seas: 0.9981 (Good) -> Bad\n", "\n", " Evaluating: MRI-ESM2-0 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.4825°C (Reasonable), Spat: 0.7858 (Reasonable), Seas: 0.9957 (Good) -> Reasonable\n", "\n", " Evaluating: NESM3 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -4.2791°C (Bad), Spat: 0.3807 (Bad), Seas: 0.9862 (Good) -> Bad\n", "\n", " Evaluating: NorESM2-LM vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.2008°C (Good), Spat: 0.7020 (Reasonable), Seas: 0.9977 (Good) -> Reasonable\n", "\n", " Evaluating: NorESM2-MM vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -1.4562°C (Good), Spat: 0.6002 (Reasonable), Seas: 0.9976 (Good) -> Reasonable\n", "\n", " Evaluating: TaiESM1 vs CPC\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -1.6340°C (Good), Spat: 0.6821 (Reasonable), Seas: 0.9961 (Good) -> Reasonable\n", "\n", " ============================================================\n", " === CPC EVALUATION COMPLETE ===\n", " ============================================================\n", " Models retained: 16 / 24\n", " WWA Table saved to: ../figures/evaluation_outputs/evaluation_cmip6_vs_CPC.csv\n", " Good: 0 model(s)\n", " Reasonable: 16 model(s)\n", " Bad: 8 model(s)\n", "\n", "######################################################################\n", "### REFERENCE: MSWX\n", "######################################################################\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " [Spatial/Bias] MSWX wcape Tx5day reference loaded.\n", " Tx5day climatology shape: (110, 120)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " [Temporal] MSWX event_domain reference loaded.\n", " Seasonal cycle shape: (12,)\n", "\n", " ============================================================\n", " Evaluating selected models vs MSWX: 24 / 24\n", " ============================================================\n", "\n", " Evaluating: ACCESS-CM2 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.3331°C (Good), Spat: 0.9641 (Good), Seas: 0.9969 (Good) -> Good\n", "\n", " Evaluating: ACCESS-ESM1-5 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.2925°C (Good), Spat: 0.9458 (Good), Seas: 0.9971 (Good) -> Good\n", "\n", " Evaluating: AWI-CM-1-1-MR vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.8626°C (Good), Spat: 0.9740 (Good), Seas: 0.9989 (Good) -> Good\n", "\n", " Evaluating: BCC-CSM2-MR vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.1497°C (Reasonable), Spat: 0.9608 (Good), Seas: 0.9925 (Good) -> Reasonable\n", "\n", " Evaluating: CMCC-ESM2 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.4265°C (Good), Spat: 0.9364 (Good), Seas: 0.9962 (Good) -> Good\n", "\n", " Evaluating: CanESM5 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +5.4268°C (Bad), Spat: 0.9310 (Good), Seas: 0.9904 (Good) -> Bad\n", "\n", " Evaluating: EC-Earth3 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.4775°C (Reasonable), Spat: 0.9654 (Good), Seas: 0.9973 (Good) -> Reasonable\n", "\n", " Evaluating: EC-Earth3-CC vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.8373°C (Reasonable), Spat: 0.9656 (Good), Seas: 0.9985 (Good) -> Reasonable\n", "\n", " Evaluating: EC-Earth3-Veg vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.3078°C (Reasonable), Spat: 0.9662 (Good), Seas: 0.9969 (Good) -> Reasonable\n", "\n", " Evaluating: EC-Earth3-Veg-LR vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.2532°C (Reasonable), Spat: 0.9473 (Good), Seas: 0.9921 (Good) -> Reasonable\n", "\n", " Evaluating: FGOALS-g3 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +1.2510°C (Good), Spat: 0.9325 (Good), Seas: 0.9961 (Good) -> Good\n", "\n", " Evaluating: GFDL-ESM4 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +1.6314°C (Good), Spat: 0.9469 (Good), Seas: 0.9983 (Good) -> Good\n", "\n", " Evaluating: INM-CM4-8 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.1559°C (Reasonable), Spat: 0.8054 (Good), Seas: 0.9987 (Good) -> Reasonable\n", "\n", " Evaluating: INM-CM5-0 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.1715°C (Reasonable), Spat: 0.7916 (Reasonable), Seas: 0.9960 (Good) -> Reasonable\n", "\n", " Evaluating: IPSL-CM6A-LR vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.7846°C (Good), Spat: 0.8505 (Good), Seas: 0.9977 (Good) -> Good\n", "\n", " Evaluating: KACE-1-0-G vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +1.3605°C (Good), Spat: 0.9448 (Good), Seas: 0.9992 (Good) -> Good\n", "\n", " Evaluating: KIOST-ESM vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +10.7033°C (Bad), Spat: 0.9407 (Good), Seas: 0.9638 (Good) -> Bad\n", "\n", " Evaluating: MIROC6 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +8.9996°C (Bad), Spat: 0.9424 (Good), Seas: 0.9857 (Good) -> Bad\n", "\n", " Evaluating: MPI-ESM1-2-LR vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +2.1731°C (Reasonable), Spat: 0.8641 (Good), Seas: 0.9980 (Good) -> Reasonable\n", "\n", " Evaluating: MRI-ESM2-0 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +3.3292°C (Reasonable), Spat: 0.9251 (Good), Seas: 0.9950 (Good) -> Reasonable\n", "\n", " Evaluating: NESM3 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: -0.1663°C (Good), Spat: 0.8167 (Good), Seas: 0.9882 (Good) -> Good\n", "\n", " Evaluating: NorESM2-LM vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +1.3118°C (Good), Spat: 0.9140 (Good), Seas: 0.9981 (Good) -> Good\n", "\n", " Evaluating: NorESM2-MM vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.1385°C (Good), Spat: 0.9499 (Good), Seas: 0.9977 (Good) -> Good\n", "\n", " Evaluating: TaiESM1 vs MSWX\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "sh: 1: getfattr: not found\n", "sh: 1: getfattr: not found\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Bias: +0.2302°C (Good), Spat: 0.9340 (Good), Seas: 0.9964 (Good) -> Good\n", "\n", " ============================================================\n", " === MSWX EVALUATION COMPLETE ===\n", " ============================================================\n", " Models retained: 21 / 24\n", " WWA Table saved to: ../figures/evaluation_outputs/evaluation_cmip6_vs_MSWX.csv\n", " Good: 12 model(s)\n", " Reasonable: 9 model(s)\n", " Bad: 3 model(s)\n" ] } ], "source": [ "# EVALUATE AGAINST EACH REFERENCE\n", "\n", "for obs_name, obs_paths in obs_references.items():\n", "\n", " print(f\"\\n{'#'*70}\")\n", " print(f\"### REFERENCE: {obs_name}\")\n", " print(f\"{'#'*70}\")\n", "\n", " # ------------------- Load reference SPATIAL data -----------------------\n", " \n", " files_obs_wcape = glob.glob(obs_paths[\"wcape_pattern\"])\n", " if not files_obs_wcape:\n", " print(f\" [SKIP] {obs_name} wcape data not found: {obs_paths['wcape_pattern']}\")\n", " continue\n", "\n", " ds_obs_wcape = xr.open_dataset(files_obs_wcape[0], chunks={\"time\": 365})\n", " obs_var_wcape = detect_tasmax_var(ds_obs_wcape)\n", " da_obs_wcape = ds_obs_wcape[obs_var_wcape].sel(time=slice(ref_start, ref_end))\n", "\n", " da_roll_obs = da_obs_wcape.rolling(time=5, center=True).mean()\n", " tx5_obs = da_roll_obs.groupby(\"time.year\").max(dim=\"time\")\n", " obs_spatial_clim = tx5_obs.mean(dim=\"year\", keep_attrs=True)\n", " obs_spatial_clim = format_coords(obs_spatial_clim)\n", "\n", " print(f\" [Spatial/Bias] {obs_name} wcape Tx5day reference loaded.\")\n", " print(f\" Tx5day climatology shape: {obs_spatial_clim.shape}\")\n", "\n", " # ---------------------- Load reference TEMPORAL data --------------------\n", "\n", " event_file = obs_paths[\"event_file\"]\n", " if not os.path.exists(event_file):\n", " print(f\" [SKIP] {obs_name} event_domain file not found: {event_file}\")\n", " continue\n", "\n", " ds_obs_event = xr.open_dataset(event_file, use_cftime=True).sortby(\"time\")\n", " ds_obs_event = ds_obs_event.sel(time=slice(ref_start, ref_end))\n", " obs_var_event = detect_tasmax_var(ds_obs_event)\n", " da_obs_event = ds_obs_event[obs_var_event]\n", "\n", " if \"NAME\" in da_obs_event.dims:\n", " da_obs_event = da_obs_event.squeeze(dim=\"NAME\", drop=True)\n", " if float(da_obs_event.max()) > 150:\n", " da_obs_event = da_obs_event - 273.15\n", "\n", " # Seasonal cycle\n", " obs_seasonal = da_obs_event.groupby(\"time.month\").mean(\"time\").values.flatten()\n", "\n", " print(f\" [Temporal] {obs_name} event_domain reference loaded.\")\n", " print(f\" Seasonal cycle shape: {obs_seasonal.shape}\")\n", "\n", " # ------------------------ Evaluate each CMIP6 model ------------------\n", " \n", " print(f\"\\n {'='*60}\")\n", " print(f\" Evaluating selected models vs {obs_name}: {len(common_models)} / {len(selected_models)}\")\n", " print(f\" {'='*60}\")\n", "\n", " results = []\n", "\n", " for model in common_models:\n", " print(f\"\\n Evaluating: {model} vs {obs_name}\")\n", " try:\n", " # ======================= SEASONAL CYCLE CORRELATION ======================\n", "\n", " ds_event = xr.open_dataset(model_event_files[model])\n", " if hasattr(ds_event.time, 'encoding') and ds_event.time.encoding.get('calendar', '') in ['360_day', 'noleap', '365_day']:\n", " ds_event = ds_event.convert_calendar(\"standard\", dim=\"time\", align_on=\"date\")\n", " ds_event = ds_event.sel(time=slice(ref_start, ref_end))\n", " da_event = ds_event[var]\n", " if \"NAME\" in da_event.dims:\n", " da_event = da_event.squeeze(dim=\"NAME\", drop=True)\n", " if da_event.max() > 150:\n", " da_event = da_event - 273.15\n", "\n", " model_seasonal = da_event.groupby(\"time.month\").mean(\"time\").values.flatten()\n", " seas_corr, _ = pearsonr(obs_seasonal, model_seasonal)\n", "\n", " # ======================= SPATIAL CORRELATION & BIAS ======================\n", " \n", " ds_wcape = xr.open_dataset(model_wcape_files[model], chunks={\"time\": 365})\n", " if hasattr(ds_wcape.time, 'encoding') and ds_wcape.time.encoding.get('calendar', '') in ['360_day', 'noleap', '365_day']:\n", " ds_wcape = ds_wcape.convert_calendar(\"standard\", dim=\"time\", align_on=\"date\")\n", "\n", " da_wcape = ds_wcape[var]\n", " if \"NAME\" in da_wcape.dims:\n", " da_wcape = da_wcape.squeeze(\"NAME\", drop=True)\n", " da_wcape = da_wcape.sel(time=slice(ref_start, ref_end))\n", "\n", " da_roll_model = da_wcape.rolling(time=5, center=True).mean()\n", " tx5_model = da_roll_model.groupby(\"time.year\").max(dim=\"time\")\n", " model_spatial_clim = tx5_model.mean(dim=\"year\", keep_attrs=True)\n", "\n", " model_spatial_clim = format_coords(model_spatial_clim)\n", "\n", " regridder = xe.Regridder(\n", " obs_spatial_clim, model_spatial_clim,\n", " method=\"bilinear\", periodic=False, reuse_weights=False, extrap_method=\"nearest_s2d\"\n", " )\n", " obs_regridded = regridder(obs_spatial_clim)\n", "\n", " bias_map = model_spatial_clim - obs_regridded\n", " mask = ~np.isnan(obs_regridded)\n", " bias_map = bias_map.where(mask)\n", "\n", " weights = np.cos(np.deg2rad(bias_map.lat))\n", " bias_mean = float(bias_map.weighted(weights).mean(dim=[\"lat\", \"lon\"]).compute().values)\n", "\n", " obs_masked = obs_regridded.where(mask)\n", " model_masked = model_spatial_clim.where(mask)\n", " obs_flat = obs_masked.values.flatten()\n", " mod_flat = model_masked.values.flatten()\n", " valid = ~np.isnan(obs_flat) & ~np.isnan(mod_flat)\n", " spat_corr, _ = pearsonr(obs_flat[valid], mod_flat[valid])\n", "\n", " # ======================== CLASSIFICATION (3 criteria) =====================\n", " \n", " bias_rate = get_bias_rating(bias_mean)\n", " spat_rate = get_spatial_rating(spat_corr)\n", " seas_rate = get_seasonal_rating(seas_corr)\n", " \n", " overall = get_overall_rating([bias_rate, spat_rate, seas_rate])\n", "\n", " results.append({\n", " \"Model\": model,\n", " \"Reference\": obs_name,\n", " \"Domain Bias (°C)\": round(bias_mean, 4),\n", " \"Bias Rating\": bias_rate,\n", " \"Spatial Corr\": round(spat_corr, 4),\n", " \"Spatial Rating\": spat_rate,\n", " \"Seasonal Corr\": round(seas_corr, 4),\n", " \"Seasonal Rating\": seas_rate,\n", " \"Overall Rating\": overall\n", " })\n", "\n", " print(f\" Bias: {bias_mean:+.4f}°C ({bias_rate}), \"\n", " f\"Spat: {spat_corr:.4f} ({spat_rate}), \"\n", " f\"Seas: {seas_corr:.4f} ({seas_rate}) \"\n", " f\"-> {overall}\")\n", "\n", " except Exception as e:\n", " print(f\" ERROR with {model}: {str(e)}\")\n", "\n", " # ------------------------------- Export results for this reference ------------------------\n", " df = pd.DataFrame(results)\n", "\n", " if len(df) == 0:\n", " print(f\"\\n [WARNING] No results for {obs_name}. Skipping export.\")\n", " continue\n", "\n", " rating_order = {\"Good\": 0, \"Reasonable\": 1, \"Bad\": 2}\n", " df[\"sort_key\"] = df[\"Overall Rating\"].map(rating_order)\n", " df = df.sort_values([\"sort_key\", \"Model\"]).drop(columns=[\"sort_key\"])\n", " df = df.reset_index(drop=True)\n", "\n", " out_path = f\"../figures/evaluation_outputs/evaluation_cmip6_vs_{obs_name}.csv\"\n", " os.makedirs(os.path.dirname(out_path), exist_ok=True)\n", " df.to_csv(out_path, index=False)\n", "\n", " print(f\"\\n {'='*60}\")\n", " print(f\" === {obs_name} EVALUATION COMPLETE ===\")\n", " print(f\" {'='*60}\")\n", " print(f\" Models retained: {len(df[df['Overall Rating'] != 'Bad'])} / {len(df)}\")\n", " print(f\" WWA Table saved to: {out_path}\")\n", " for rating in [\"Good\", \"Reasonable\", \"Bad\"]:\n", " count = len(df[df[\"Overall Rating\"] == rating])\n", " print(f\" {rating}: {count} model(s)\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "1e5d669d-40ec-4d30-bd58-34dddb93429f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "### CROSS-REFERENCE COMPARISON SUMMARY\n", "Consistency: 6/24 models have identical ratings across all references.\n", "\n", " Model Rating_ERA5-Land Rating_CPC Rating_MSWX Consistent\n", " EC-Earth3 Good Reasonable Reasonable No\n", " EC-Earth3-CC Good Reasonable Reasonable No\n", " EC-Earth3-Veg Good Reasonable Reasonable No\n", " ACCESS-CM2 Reasonable Reasonable Good No\n", " ACCESS-ESM1-5 Reasonable Reasonable Good No\n", " AWI-CM-1-1-MR Reasonable Reasonable Good No\n", " BCC-CSM2-MR Reasonable Reasonable Reasonable Yes\n", " CMCC-ESM2 Reasonable Reasonable Good No\n", "EC-Earth3-Veg-LR Reasonable Reasonable Reasonable Yes\n", " FGOALS-g3 Reasonable Reasonable Good No\n", " GFDL-ESM4 Reasonable Reasonable Good No\n", " INM-CM4-8 Reasonable Bad Reasonable No\n", " IPSL-CM6A-LR Reasonable Bad Good No\n", " KACE-1-0-G Reasonable Reasonable Good No\n", " MRI-ESM2-0 Reasonable Reasonable Reasonable Yes\n", " NorESM2-LM Reasonable Reasonable Good No\n", " NorESM2-MM Reasonable Reasonable Good No\n", " TaiESM1 Reasonable Reasonable Good No\n", " CanESM5 Bad Bad Bad Yes\n", " INM-CM5-0 Bad Bad Reasonable No\n", " KIOST-ESM Bad Bad Bad Yes\n", " MIROC6 Bad Bad Bad Yes\n", " MPI-ESM1-2-LR Bad Bad Reasonable No\n", " NESM3 Bad Bad Good No\n", "=== ALL EVALUATIONS COMPLETE ===\n" ] } ], "source": [ "# 5. CROSS-REFERENCE COMPARISON SUMMARY\n", "print(\"### CROSS-REFERENCE COMPARISON SUMMARY\")\n", "\n", "# Load all 3 CSVs and compare overall ratings\n", "all_dfs = {}\n", "for obs_name in obs_references.keys():\n", " csv_path = f\"./Evaluation_Outputs/appendix_a1_wwa_evaluation_{obs_name}.csv\"\n", " if os.path.exists(csv_path):\n", " all_dfs[obs_name] = pd.read_csv(csv_path)\n", "\n", "if len(all_dfs) >= 2:\n", " # Build a comparison table: Model | Rating_ERA5 | Rating_CPC | Rating_MSWX | Consistent?\n", " comparison_rows = []\n", " all_models_union = sorted(set().union(*[set(d[\"Model\"]) for d in all_dfs.values()]))\n", " \n", " for model in all_models_union:\n", " row = {\"Model\": model}\n", " ratings = []\n", " for obs_name, df_obs in all_dfs.items():\n", " match = df_obs[df_obs[\"Model\"] == model]\n", " if len(match) > 0:\n", " r = match[\"Overall Rating\"].values[0]\n", " row[f\"Rating_{obs_name}\"] = r\n", " ratings.append(r)\n", " else:\n", " row[f\"Rating_{obs_name}\"] = \"N/A\"\n", " \n", " # Check consistency\n", " unique_ratings = set(r for r in ratings if r != \"N/A\")\n", " row[\"Consistent\"] = \"Yes\" if len(unique_ratings) <= 1 else \"No\"\n", " comparison_rows.append(row)\n", " \n", " df_comparison = pd.DataFrame(comparison_rows)\n", " \n", " # Sort by ERA5 rating if available, then model name\n", " if \"Rating_ERA5-Land\" in df_comparison.columns:\n", " rating_order = {\"Good\": 0, \"Reasonable\": 1, \"Bad\": 2, \"N/A\": 3}\n", " df_comparison[\"sort_key\"] = df_comparison[\"Rating_ERA5-Land\"].map(rating_order)\n", " df_comparison = df_comparison.sort_values([\"sort_key\", \"Model\"]).drop(columns=[\"sort_key\"])\n", " df_comparison = df_comparison.reset_index(drop=True)\n", " \n", " n_consistent = len(df_comparison[df_comparison[\"Consistent\"] == \"Yes\"])\n", " n_total = len(df_comparison)\n", " \n", " print(f\"Consistency: {n_consistent}/{n_total} models have identical ratings across all references.\\n\")\n", " crossref_path = f\"../figures/evaluation_outputs/synthese_cmip6_vs_obs.csv\"\n", " df_comparison.to_csv(crossref_path, index=False)\n", " print(df_comparison.to_string(index=False))\n", "\n", "print(\"=== ALL EVALUATIONS COMPLETE ===\")" ] }, { "cell_type": "code", "execution_count": null, "id": "37af4588-2113-47ef-9478-6ffb7bf8c822", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python (monenv)", "language": "python", "name": "monenv" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.13" } }, "nbformat": 4, "nbformat_minor": 5 }