ag first commit

This commit is contained in:
Andrea Gobbi committed 2024-02-20 11:50:01 +01:00
commit ea097a7b71
6 files changed
+4967

No files matched your search

File diff suppressed because it is too large. Load diff
+320
View File
@@ -0,0 +1,320 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 18,
"id": "e9ca0612-adb5-4d11-89a1-b224de3bd3b5",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append('../') ##accrocchio\n",
"from src.utils import retrive_data,split\n",
"import pickle"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dcb4d7b5-fe20-43f1-b988-51c9d595e06d",
"metadata": {},
"outputs": [],
"source": [
"labeled,labeled_small = retrive_data(False,threshold_under_represented=0.5,path='/home/agobbi/Projects/PID/datanalytics/PID/src')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "569dff52-2a78-43f4-ae40-07842d3669e3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" class p\n",
"0 0 0.203424\n",
"1 1 0.013596\n",
"2 2 0.056109\n",
"3 3 1.130661\n",
"4 4 16.206137\n"
]
}
],
"source": [
"dataset,dataset_test = split(labeled_small,\n",
" SKI_AREA_TEST= 'Klausberg',\n",
" SEASON_TEST_SKIAREA = 'Kronplatz',\n",
" SEASON_TEST_YEAR= 2023,\n",
" use_smote = False,\n",
" weight_type = 'sqrt' )"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "65405081-748d-43ef-aed6-2a58e38673b4",
"metadata": {},
"outputs": [],
"source": [
"with open('../src/best_params_and_final_model.pkl','rb') as f:\n",
" tot,bst_FS,FS = pickle.load(f) "
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "53434031-1a11-49f7-9481-de77f250d35b",
"metadata": {},
"outputs": [],
"source": [
"import shap\n",
"import xgboost as xgb\n",
"import numpy as np\n",
"dtest_FS = xgb.DMatrix(dataset_test.X_test_area[bst_FS.feature_names],dataset_test.y_test_area,enable_categorical=True,)\n",
"\n",
"explainer = shap.TreeExplainer(bst_FS,feature_names=bst_FS.feature_names)\n",
"shap_values = explainer.shap_values(dtest_FS)\n"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "f3b72453-27be-4db5-b40d-883454c294f9",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div align='center'><img src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABkAAAAWCAYAAAA1vze2AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAdxJREFUeNq0Vt1Rg0AQJjcpgBJiBWIFkgoMFYhPPAIVECogPuYpdJBYgXQQrMCUkA50V7+d2ZwXuXPGm9khHLu3f9+3l1nkWNvtNqfHLgpfQ1EUS3tz5nAQ0+NIsiAZSc6eDlI8M3J00B/mDuUKDk6kfOebAgW3pkdD0pFcODGW4gKKvOrAUm04MA4QDt1OEIXU9hDigfS5rC1eS5T90gltck1Xrizo257kgySZcNRzgCSxCvgiE9nckPJo2b/B2AcEkk2OwL8bD8gmOKR1GPbaCUqxEgTq0tLvgb6zfo7+DgYGkkWL2tqLDV4RSITfbHPPfJKIrWz4nJQTMPAWA7IbD6imcNaDeDfgk+4No+wZr40BL3g9eQJJCFqRQ54KiSt72lsLpE3o3MCBSxDuq4yOckU2hKXRuwBH3OyMR4g1UpyTYw6mlmBqNdUXRM1NfyF5EPI6JkcpIDBIX8jX6DR/6ckAZJ0wEAdLR8DEk6OfC1Pp8BKo6TQIwPJbvJ6toK5lmuvJoRtfK6Ym1iRYIarRo2UyYHvRN5qpakR3yoizWrouoyuXXQqI185LCw07op5ZyCRGL99h24InP0e9xdQukEKVmhzrqZuRIfwISB//cP3Wk3f8f/yR+BRgAHu00HjLcEQBAAAAAElFTkSuQmCC' /></div><script charset='utf-8'>/*! For license information please see bundle.js.LICENSE.txt */\n",
"(()=>{var e={486:function(e,t,n){var r;e=n.nmd(e),function(){var a,i=\"Expected a function\",o=\"__lodash_hash_undefined__\",u=\"__lodash_placeholder__\",l=32,s=128,c=1/0,f=9007199254740991,p=NaN,d=4294967295,h=[[\"ary\",s],[\"bind\",1],[\"bindKey\",2],[\"curry\",8],[\"curryRight\",16],[\"flip\",512],[\"partial\",l],[\"partialRight\",64],[\"rearg\",256]],v=\"[object Arguments]\",g=\"[object Array]\",y=\"[object Boolean]\",m=\"[object Date]\",b=\"[object Error]\",_=\"[object Function]\",w=\"[object GeneratorFunction]\",x=\"[object Map]\",k=\"[object Number]\",S=\"[object Object]\",E=\"[object Promise]\",C=\"[object RegExp]\",T=\"[object Set]\",M=\"[object String]\",N=\"[object Symbol]\",P=\"[object WeakMap]\",z=\"[object ArrayBuffer]\",L=\"[object DataView]\",O=\"[object Float32Array]\",A=\"[object Float64Array]\",F=\"[object Int8Array]\",D=\"[object Int16Array]\",R=\"[object Int32Array]\",j=\"[object Uint8Array]\",U=\"[object Uint8ClampedArray]\",I=\"[object Uint16Array]\",$=\"[object Uint32Array]\",B=/\\b__p \\+= '';/g,W=/\\b(__p \\+=) '' \\+/g,V=/(__e\\(.*?\\)|\\b__t\\)) \\+\\n'';/g,H=/&(?:amp|lt|gt|quot|#39);/g,q=/[&<>\"']/g,Q=RegExp(H.source),Y=RegExp(q.source),G=/<%-([\\s\\S]+?)%>/g,K=/<%([\\s\\S]+?)%>/g,Z=/<%=([\\s\\S]+?)%>/g,X=/\\.|\\[(?:[^[\\]]*|([\"'])(?:(?!\\1)[^\\\\]|\\\\.)*?\\1)\\]/,J=/^\\w*$/,ee=/[^.[\\]]+|\\[(?:(-?\\d+(?:\\.\\d+)?)|([\"'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2)\\]|(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))/g,te=/[\\\\^$.*+?()[\\]{}|]/g,ne=RegExp(te.source),re=/^\\s+/,ae=/\\s/,ie=/\\{(?:\\n\\/\\* \\[wrapped with .+\\] \\*\\/)?\\n?/,oe=/\\{\\n\\/\\* \\[wrapped with (.+)\\] \\*/,ue=/,? & /,le=/[^\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\x7f]+/g,se=/[()=,{}\\[\\]\\/\\s]/,ce=/\\\\(\\\\)?/g,fe=/\\$\\{([^\\\\}]*(?:\\\\.[^\\\\}]*)*)\\}/g,pe=/\\w*$/,de=/^[-+]0x[0-9a-f]+$/i,he=/^0b[01]+$/i,ve=/^\\[object .+?Constructor\\]$/,ge=/^0o[0-7]+$/i,ye=/^(?:0|[1-9]\\d*)$/,me=/[\\xc0-\\xd6\\xd8-\\xf6\\xf8-\\xff\\u0100-\\u017f]/g,be=/($^)/,_e=/['\\n\\r\\u2028\\u2029\\\\]/g,we=\"\\\\ud800-\\\\udfff\",xe=\"\\\\u0300-\\\\u036f\\\\ufe20-\\\\ufe2f\\\\u20d0-\\\\u20ff\",ke=\"\\\\u2700-\\\\u27bf\",Se=\"a-z\\\\xdf-\\\\xf6\\\\xf8-\\\\xff\",Ee=\"A-Z\\\\xc0-\\\\xd6\\\\xd8-\\\\xde\",Ce=\"\\\\ufe0e\\\\ufe0f\",Te=\"\\\\xac\\\\xb1\\\\xd7\\\\xf7\\\\x00-\\\\x2f\\\\x3a-\\\\x40\\\\x5b-\\\\x60\\\\x7b-\\\\xbf\\\\u2000-\\\\u206f \\\\t\\\\x0b\\\\f\\\\xa0\\\\ufeff\\\\n\\\\r\\\\u2028\\\\u2029\\\\u1680\\\\u180e\\\\u2000\\\\u2001\\\\u2002\\\\u2003\\\\u2004\\\\u2005\\\\u2006\\\\u2007\\\\u2008\\\\u2009\\\\u200a\\\\u202f\\\\u205f\\\\u3000\",Me=\"[\"+we+\"]\",Ne=\"[\"+Te+\"]\",Pe=\"[\"+xe+\"]\",ze=\"\\\\d+\",Le=\"[\"+ke+\"]\",Oe=\"[\"+Se+\"]\",Ae=\"[^\"+we+Te+ze+ke+Se+Ee+\"]\",Fe=\"\\\\ud83c[\\\\udffb-\\\\udfff]\",De=\"[^\"+we+\"]\",Re=\"(?:\\\\ud83c[\\\\udde6-\\\\uddff]){2}\",je=\"[\\\\ud800-\\\\udbff][\\\\udc00-\\\\udfff]\",Ue=\"[\"+Ee+\"]\",Ie=\"\\\\u200d\",$e=\"(?:\"+Oe+\"|\"+Ae+\")\",Be=\"(?:\"+Ue+\"|\"+Ae+\")\",We=\"(?:['’](?:d|ll|m|re|s|t|ve))?\",Ve=\"(?:['’](?:D|LL|M|RE|S|T|VE))?\",He=\"(?:\"+Pe+\"|\"+Fe+\")?\",qe=\"[\"+Ce+\"]?\",Qe=qe+He+\"(?:\"+Ie+\"(?:\"+[De,Re,je].join(\"|\")+\")\"+qe+He+\")*\",Ye=\"(?:\"+[Le,Re,je].join(\"|\")+\")\"+Qe,Ge=\"(?:\"+[De+Pe+\"?\",Pe,Re,je,Me].join(\"|\")+\")\",Ke=RegExp(\"['’]\",\"g\"),Ze=RegExp(Pe,\"g\"),Xe=RegExp(Fe+\"(?=\"+Fe+\")|\"+Ge+Qe,\"g\"),Je=RegExp([Ue+\"?\"+Oe+\"+\"+We+\"(?=\"+[Ne,Ue,\"$\"].join(\"|\")+\")\",Be+\"+\"+Ve+\"(?=\"+[Ne,Ue+$e,\"$\"].join(\"|\")+\")\",Ue+\"?\"+$e+\"+\"+We,Ue+\"+\"+Ve,\"\\\\d*(?:1ST|2ND|3RD|(?![123])\\\\dTH)(?=\\\\b|[a-z_])\",\"\\\\d*(?:1st|2nd|3rd|(?![123])\\\\dth)(?=\\\\b|[A-Z_])\",ze,Ye].join(\"|\"),\"g\"),et=RegExp(\"[\"+Ie+we+xe+Ce+\"]\"),tt=/[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,nt=[\"Array\",\"Buffer\",\"DataView\",\"Date\",\"Error\",\"Float32Array\",\"Float64Array\",\"Function\",\"Int8Array\",\"Int16Array\",\"Int32Array\",\"Map\",\"Math\",\"Object\",\"Promise\",\"RegExp\",\"Set\",\"String\",\"Symbol\",\"TypeError\",\"Uint8Array\",\"Uint8ClampedArray\",\"Uint16Array\",\"Uint32Array\",\"WeakMap\",\"_\",\"clearTimeout\",\"isFinite\",\"parseInt\",\"setTimeout\"],rt=-1,at={};at[O]=at[A]=at[F]=at[D]=at[R]=at[j]=at[U]=at[I]=at[$]=!0,at[v]=at[g]=at[z]=at[y]=at[L]=at[m]=at[b]=at[_]=at[x]=at[k]=at[S]=at[C]=at[T]=at[M]=at[P]=!1;var it={};it[v]=it[g]=it[z]=it[L]=it[y]=it[m]=it[O]=it[A]=it[F]=it[D]=it[R]=it[x]=it[k]=it[S]=it[C]=it[T]=it[M]=it[N]=it[j]=it[U]=it[I]=it[$]=!0,it[b]=it[_]=it[P]=!1;var ot={\"\\\\\":\"\\\\\",\"'\":\"'\",\"\\n\":\"n\",\"\\r\":\"r\",\"\\u2028\":\"u2028\",\"\\u2029\":\"u2029\"},ut=parseFloat,lt=parseInt,st=\"object\"==typeof n.g&&n.g&&n.g.Object===Object&&n.g,ct=\"object\"==typeof self&&self&&self.Object===Object&&self,ft=st||ct||Function(\"return this\")(),pt=t&&!t.nodeType&&t,dt=pt&&e&&!e.nodeType&&e,ht=dt&&dt.exports===pt,vt=ht&&st.process,gt=function(){try{return dt&&dt.require&&dt.require(\"util\").types||vt&&vt.binding&&vt.binding(\"util\")}catch(e){}}(),yt=gt&&gt.isArrayBuffer,mt=gt&&gt.isDate,bLine truncated
"</script>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"shap.initjs()"
]
},
{
"cell_type": "code",
"execution_count": 73,
"id": "129dd5fb-367e-493e-937c-12ae0727cf85",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([2.58451770e-03, 2.63414029e-02, 8.63697469e-01, 1.07182816e-01,\n",
" 1.93854154e-04], dtype=float32)"
]
},
"execution_count": 73,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred = bst_FS.predict(dtest_FS)\n",
"pred[2]"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "46bf2240-cfff-4c1b-a5db-87290bf566be",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"age 53.0\n",
"diagnosis dislocation\n",
"injury_general_location upper_limbs\n",
"country Germania\n",
"difficulty easy\n",
"cause fall_alone\n",
"destination hospital_emergency_room\n",
"injury_side R\n",
"equipment ski\n",
"gender F\n",
"snowmobile True\n",
"helicopter True\n",
"akja False\n",
"helmet True\n",
"ambulance False\n",
"snowmobile_sled False\n",
"ski_lift False\n",
"indipendently False\n",
"skiarea_ambulance False\n",
"other False\n",
"Name: 724, dtype: object"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset_test.X_test_area[bst_FS.feature_names].iloc[2]"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "27f3b14c-4917-45ff-93ee-234dbc392b20",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset_test.y_test_area.iloc[2]"
]
},
{
"cell_type": "code",
"execution_count": 64,
"id": "9799b767-15f2-403b-8535-eb471b862ab3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[-3.5973988, -1.704391, 1.8931085, -0.9031036, -7.5351233]"
]
},
"execution_count": 64,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[shap_values[i][2].sum() for i in range(5)]"
]
},
{
"cell_type": "code",
"execution_count": 71,
"id": "f4cac6a8-1ecc-435b-a7c0-5dff7df561f3",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<div id='i11854JF2PGG5AQIPP5RQ'>\n",
"<div style='color: #900; text-align: center;'>\n",
" <b>Visualization omitted, Javascript library not loaded!</b><br>\n",
" Have you run `initjs()` in this notebook? If this notebook was from another\n",
" user you must also trust this notebook (File -> Trust notebook). If you are viewing\n",
" this notebook on github the Javascript has been stripped for security. If you are using\n",
" JupyterLab this error is because a JupyterLab extension has not yet been written.\n",
"</div></div>\n",
" <script>\n",
" if (window.SHAP) SHAP.ReactDom.render(\n",
" SHAP.React.createElement(SHAP.AdditiveForceVisualizer, {\"outNames\": [\"f(x)\"], \"baseValue\": 0.552638053894043, \"outValue\": -1.1517529487609863, \"link\": \"identity\", \"featureNames\": [\"age\", \"diagnosis\", \"injury_general_location\", \"country\", \"difficulty\", \"cause\", \"destination\", \"injury_side\", \"equipment\", \"gender\", \"snowmobile\", \"helicopter\", \"akja\", \"helmet\", \"ambulance\", \"snowmobile_sled\", \"ski_lift\", \"indipendently\", \"skiarea_ambulance\", \"other\"], \"features\": {\"0\": {\"effect\": 0.09595096856355667, \"value\": \"\"}, \"1\": {\"effect\": -0.27542880177497864, \"value\": \"\"}, \"2\": {\"effect\": 0.5892016887664795, \"value\": \"\"}, \"3\": {\"effect\": -0.12579196691513062, \"value\": \"\"}, \"4\": {\"effect\": -0.01911946013569832, \"value\": \"\"}, \"5\": {\"effect\": 0.057017769664525986, \"value\": \"\"}, \"6\": {\"effect\": 0.0896696150302887, \"value\": \"\"}, \"7\": {\"effect\": 0.13361920416355133, \"value\": \"\"}, \"8\": {\"effect\": 0.0006894035032019019, \"value\": \"\"}, \"9\": {\"effect\": 0.11025260388851166, \"value\": \"\"}, \"10\": {\"effect\": 0.03937700018286705, \"value\": \"\"}, \"11\": {\"effect\": -2.0775811672210693, \"value\": \"\"}, \"12\": {\"effect\": 0.03850683569908142, \"value\": \"\"}, \"13\": {\"effect\": -0.005975374486297369, \"value\": \"\"}, \"14\": {\"effect\": -0.15785543620586395, \"value\": \"\"}, \"15\": {\"effect\": 0.00020830573339480907, \"value\": \"\"}, \"16\": {\"effect\": -0.07540600001811981, \"value\": \"\"}, \"17\": {\"effect\": -0.12225314229726791, \"value\": \"\"}, \"18\": {\"effect\": 0.00043001066660508513, \"value\": \"\"}, \"19\": {\"effect\": 9.698851499706507e-05, \"value\": \"\"}}, \"plot_cmap\": \"RdBu\", \"labelMargin\": 20}),\n",
" document.getElementById('i11854JF2PGG5AQIPP5RQ')\n",
" );\n",
"</script>"
],
"text/plain": [
"<shap.plots._force.AdditiveForceVisualizer at 0x7fb58227d450>"
]
},
"execution_count": 71,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i=1\n",
"shap.plots.force(explainer.expected_value[1], shap_values[i][2],feature_names=bst_FS.feature_names)\n"
]
},
{
"cell_type": "code",
"execution_count": 70,
"id": "3d56cbf0-ff20-4d3a-94b7-a0e07c5cc019",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<div id='iOOXREDIO2XGF7NOS1QU8'>\n",
"<div style='color: #900; text-align: center;'>\n",
" <b>Visualization omitted, Javascript library not loaded!</b><br>\n",
" Have you run `initjs()` in this notebook? If this notebook was from another\n",
" user you must also trust this notebook (File -> Trust notebook). If you are viewing\n",
" this notebook on github the Javascript has been stripped for security. If you are using\n",
" JupyterLab this error is because a JupyterLab extension has not yet been written.\n",
"</div></div>\n",
" <script>\n",
" if (window.SHAP) SHAP.ReactDom.render(\n",
" SHAP.React.createElement(SHAP.AdditiveForceVisualizer, {\"outNames\": [\"f(x)\"], \"baseValue\": 0.552638053894043, \"outValue\": 2.445746421813965, \"link\": \"identity\", \"featureNames\": [\"age\", \"diagnosis\", \"injury_general_location\", \"country\", \"difficulty\", \"cause\", \"destination\", \"injury_side\", \"equipment\", \"gender\", \"snowmobile\", \"helicopter\", \"akja\", \"helmet\", \"ambulance\", \"snowmobile_sled\", \"ski_lift\", \"indipendently\", \"skiarea_ambulance\", \"other\"], \"features\": {\"0\": {\"effect\": 0.16000449657440186, \"value\": \"\"}, \"1\": {\"effect\": 0.44064590334892273, \"value\": \"\"}, \"2\": {\"effect\": 0.3827016353607178, \"value\": \"\"}, \"3\": {\"effect\": 0.05334072560071945, \"value\": \"\"}, \"4\": {\"effect\": 0.17687170207500458, \"value\": \"\"}, \"5\": {\"effect\": 0.046136993914842606, \"value\": \"\"}, \"6\": {\"effect\": 0.362857460975647, \"value\": \"\"}, \"7\": {\"effect\": 0.1898631602525711, \"value\": \"\"}, \"8\": {\"effect\": 0.039788179099559784, \"value\": \"\"}, \"9\": {\"effect\": -0.024055209010839462, \"value\": \"\"}, \"10\": {\"effect\": 0.2057642787694931, \"value\": \"\"}, \"11\": {\"effect\": 0.08326093852519989, \"value\": \"\"}, \"12\": {\"effect\": -0.0021338891237974167, \"value\": \"\"}, \"13\": {\"effect\": -0.011541801504790783, \"value\": \"\"}, \"14\": {\"effect\": -0.12199775129556656, \"value\": \"\"}, \"15\": {\"effect\": -0.08116263896226883, \"value\": \"\"}, \"16\": {\"effect\": -0.030168458819389343, \"value\": \"\"}, \"17\": {\"effect\": 0.021043526008725166, \"value\": \"\"}, \"18\": {\"effect\": -0.0011919558746740222, \"value\": \"\"}, \"19\": {\"effect\": 0.003081258852034807, \"value\": \"\"}}, \"plot_cmap\": \"RdBu\", \"labelMargin\": 20}),\n",
" document.getElementById('iOOXREDIO2XGF7NOS1QU8')\n",
" );\n",
"</script>"
],
"text/plain": [
"<shap.plots._force.AdditiveForceVisualizer at 0x7fb5822b3190>"
]
},
"execution_count": 70,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i=2\n",
"shap.plots.force(explainer.expected_value[1], shap_values[i][2],feature_names=bst_FS.feature_names)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6bbece5c-75f4-4ba0-a44e-677d9819b17a",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}