ag first commit
This commit is contained in:
Andrea Gobbi committed
commit
ea097a7b71
6 files changed
+4967
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,320 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 18,
|
||||||
|
"id": "e9ca0612-adb5-4d11-89a1-b224de3bd3b5",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import sys\n",
|
||||||
|
"sys.path.append('../') ##accrocchio\n",
|
||||||
|
"from src.utils import retrive_data,split\n",
|
||||||
|
"import pickle"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "dcb4d7b5-fe20-43f1-b988-51c9d595e06d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"labeled,labeled_small = retrive_data(False,threshold_under_represented=0.5,path='/home/agobbi/Projects/PID/datanalytics/PID/src')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 16,
|
||||||
|
"id": "569dff52-2a78-43f4-ae40-07842d3669e3",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
" class p\n",
|
||||||
|
"0 0 0.203424\n",
|
||||||
|
"1 1 0.013596\n",
|
||||||
|
"2 2 0.056109\n",
|
||||||
|
"3 3 1.130661\n",
|
||||||
|
"4 4 16.206137\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"dataset,dataset_test = split(labeled_small,\n",
|
||||||
|
" SKI_AREA_TEST= 'Klausberg',\n",
|
||||||
|
" SEASON_TEST_SKIAREA = 'Kronplatz',\n",
|
||||||
|
" SEASON_TEST_YEAR= 2023,\n",
|
||||||
|
" use_smote = False,\n",
|
||||||
|
" weight_type = 'sqrt' )"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 19,
|
||||||
|
"id": "65405081-748d-43ef-aed6-2a58e38673b4",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"with open('../src/best_params_and_final_model.pkl','rb') as f:\n",
|
||||||
|
" tot,bst_FS,FS = pickle.load(f) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 49,
|
||||||
|
"id": "53434031-1a11-49f7-9481-de77f250d35b",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import shap\n",
|
||||||
|
"import xgboost as xgb\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"dtest_FS = xgb.DMatrix(dataset_test.X_test_area[bst_FS.feature_names],dataset_test.y_test_area,enable_categorical=True,)\n",
|
||||||
|
"\n",
|
||||||
|
"explainer = shap.TreeExplainer(bst_FS,feature_names=bst_FS.feature_names)\n",
|
||||||
|
"shap_values = explainer.shap_values(dtest_FS)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 33,
|
||||||
|
"id": "f3b72453-27be-4db5-b40d-883454c294f9",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/html": [
|
||||||
|
"<div align='center'><img src='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABkAAAAWCAYAAAA1vze2AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAdxJREFUeNq0Vt1Rg0AQJjcpgBJiBWIFkgoMFYhPPAIVECogPuYpdJBYgXQQrMCUkA50V7+d2ZwXuXPGm9khHLu3f9+3l1nkWNvtNqfHLgpfQ1EUS3tz5nAQ0+NIsiAZSc6eDlI8M3J00B/mDuUKDk6kfOebAgW3pkdD0pFcODGW4gKKvOrAUm04MA4QDt1OEIXU9hDigfS5rC1eS5T90gltck1Xrizo257kgySZcNRzgCSxCvgiE9nckPJo2b/B2AcEkk2OwL8bD8gmOKR1GPbaCUqxEgTq0tLvgb6zfo7+DgYGkkWL2tqLDV4RSITfbHPPfJKIrWz4nJQTMPAWA7IbD6imcNaDeDfgk+4No+wZr40BL3g9eQJJCFqRQ54KiSt72lsLpE3o3MCBSxDuq4yOckU2hKXRuwBH3OyMR4g1UpyTYw6mlmBqNdUXRM1NfyF5EPI6JkcpIDBIX8jX6DR/6ckAZJ0wEAdLR8DEk6OfC1Pp8BKo6TQIwPJbvJ6toK5lmuvJoRtfK6Ym1iRYIarRo2UyYHvRN5qpakR3yoizWrouoyuXXQqI185LCw07op5ZyCRGL99h24InP0e9xdQukEKVmhzrqZuRIfwISB//cP3Wk3f8f/yR+BRgAHu00HjLcEQBAAAAAElFTkSuQmCC' /></div><script charset='utf-8'>/*! For license information please see bundle.js.LICENSE.txt */\n",
|
||||||
|
"(()=>{var e={486:function(e,t,n){var r;e=n.nmd(e),function(){var a,i=\"Expected a function\",o=\"__lodash_hash_undefined__\",u=\"__lodash_placeholder__\",l=32,s=128,c=1/0,f=9007199254740991,p=NaN,d=4294967295,h=[[\"ary\",s],[\"bind\",1],[\"bindKey\",2],[\"curry\",8],[\"curryRight\",16],[\"flip\",512],[\"partial\",l],[\"partialRight\",64],[\"rearg\",256]],v=\"[object Arguments]\",g=\"[object Array]\",y=\"[object Boolean]\",m=\"[object Date]\",b=\"[object Error]\",_=\"[object Function]\",w=\"[object GeneratorFunction]\",x=\"[object Map]\",k=\"[object Number]\",S=\"[object Object]\",E=\"[object Promise]\",C=\"[object RegExp]\",T=\"[object Set]\",M=\"[object String]\",N=\"[object Symbol]\",P=\"[object WeakMap]\",z=\"[object ArrayBuffer]\",L=\"[object DataView]\",O=\"[object Float32Array]\",A=\"[object Float64Array]\",F=\"[object Int8Array]\",D=\"[object Int16Array]\",R=\"[object Int32Array]\",j=\"[object Uint8Array]\",U=\"[object Uint8ClampedArray]\",I=\"[object Uint16Array]\",$=\"[object Uint32Array]\",B=/\\b__p \\+= '';/g,W=/\\b(__p \\+=) '' \\+/g,V=/(__e\\(.*?\\)|\\b__t\\)) \\+\\n'';/g,H=/&(?:amp|lt|gt|quot|#39);/g,q=/[&<>\"']/g,Q=RegExp(H.source),Y=RegExp(q.source),G=/<%-([\\s\\S]+?)%>/g,K=/<%([\\s\\S]+?)%>/g,Z=/<%=([\\s\\S]+?)%>/g,X=/\\.|\\[(?:[^[\\]]*|([\"'])(?:(?!\\1)[^\\\\]|\\\\.)*?\\1)\\]/,J=/^\\w*$/,ee=/[^.[\\]]+|\\[(?:(-?\\d+(?:\\.\\d+)?)|([\"'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2)\\]|(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))/g,te=/[\\\\^$.*+?()[\\]{}|]/g,ne=RegExp(te.source),re=/^\\s+/,ae=/\\s/,ie=/\\{(?:\\n\\/\\* \\[wrapped with .+\\] \\*\\/)?\\n?/,oe=/\\{\\n\\/\\* \\[wrapped with (.+)\\] \\*/,ue=/,? & /,le=/[^\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\x7f]+/g,se=/[()=,{}\\[\\]\\/\\s]/,ce=/\\\\(\\\\)?/g,fe=/\\$\\{([^\\\\}]*(?:\\\\.[^\\\\}]*)*)\\}/g,pe=/\\w*$/,de=/^[-+]0x[0-9a-f]+$/i,he=/^0b[01]+$/i,ve=/^\\[object .+?Constructor\\]$/,ge=/^0o[0-7]+$/i,ye=/^(?:0|[1-9]\\d*)$/,me=/[\\xc0-\\xd6\\xd8-\\xf6\\xf8-\\xff\\u0100-\\u017f]/g,be=/($^)/,_e=/['\\n\\r\\u2028\\u2029\\\\]/g,we=\"\\\\ud800-\\\\udfff\",xe=\"\\\\u0300-\\\\u036f\\\\ufe20-\\\\ufe2f\\\\u20d0-\\\\u20ff\",ke=\"\\\\u2700-\\\\u27bf\",Se=\"a-z\\\\xdf-\\\\xf6\\\\xf8-\\\\xff\",Ee=\"A-Z\\\\xc0-\\\\xd6\\\\xd8-\\\\xde\",Ce=\"\\\\ufe0e\\\\ufe0f\",Te=\"\\\\xac\\\\xb1\\\\xd7\\\\xf7\\\\x00-\\\\x2f\\\\x3a-\\\\x40\\\\x5b-\\\\x60\\\\x7b-\\\\xbf\\\\u2000-\\\\u206f \\\\t\\\\x0b\\\\f\\\\xa0\\\\ufeff\\\\n\\\\r\\\\u2028\\\\u2029\\\\u1680\\\\u180e\\\\u2000\\\\u2001\\\\u2002\\\\u2003\\\\u2004\\\\u2005\\\\u2006\\\\u2007\\\\u2008\\\\u2009\\\\u200a\\\\u202f\\\\u205f\\\\u3000\",Me=\"[\"+we+\"]\",Ne=\"[\"+Te+\"]\",Pe=\"[\"+xe+\"]\",ze=\"\\\\d+\",Le=\"[\"+ke+\"]\",Oe=\"[\"+Se+\"]\",Ae=\"[^\"+we+Te+ze+ke+Se+Ee+\"]\",Fe=\"\\\\ud83c[\\\\udffb-\\\\udfff]\",De=\"[^\"+we+\"]\",Re=\"(?:\\\\ud83c[\\\\udde6-\\\\uddff]){2}\",je=\"[\\\\ud800-\\\\udbff][\\\\udc00-\\\\udfff]\",Ue=\"[\"+Ee+\"]\",Ie=\"\\\\u200d\",$e=\"(?:\"+Oe+\"|\"+Ae+\")\",Be=\"(?:\"+Ue+\"|\"+Ae+\")\",We=\"(?:['’](?:d|ll|m|re|s|t|ve))?\",Ve=\"(?:['’](?:D|LL|M|RE|S|T|VE))?\",He=\"(?:\"+Pe+\"|\"+Fe+\")?\",qe=\"[\"+Ce+\"]?\",Qe=qe+He+\"(?:\"+Ie+\"(?:\"+[De,Re,je].join(\"|\")+\")\"+qe+He+\")*\",Ye=\"(?:\"+[Le,Re,je].join(\"|\")+\")\"+Qe,Ge=\"(?:\"+[De+Pe+\"?\",Pe,Re,je,Me].join(\"|\")+\")\",Ke=RegExp(\"['’]\",\"g\"),Ze=RegExp(Pe,\"g\"),Xe=RegExp(Fe+\"(?=\"+Fe+\")|\"+Ge+Qe,\"g\"),Je=RegExp([Ue+\"?\"+Oe+\"+\"+We+\"(?=\"+[Ne,Ue,\"$\"].join(\"|\")+\")\",Be+\"+\"+Ve+\"(?=\"+[Ne,Ue+$e,\"$\"].join(\"|\")+\")\",Ue+\"?\"+$e+\"+\"+We,Ue+\"+\"+Ve,\"\\\\d*(?:1ST|2ND|3RD|(?![123])\\\\dTH)(?=\\\\b|[a-z_])\",\"\\\\d*(?:1st|2nd|3rd|(?![123])\\\\dth)(?=\\\\b|[A-Z_])\",ze,Ye].join(\"|\"),\"g\"),et=RegExp(\"[\"+Ie+we+xe+Ce+\"]\"),tt=/[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,nt=[\"Array\",\"Buffer\",\"DataView\",\"Date\",\"Error\",\"Float32Array\",\"Float64Array\",\"Function\",\"Int8Array\",\"Int16Array\",\"Int32Array\",\"Map\",\"Math\",\"Object\",\"Promise\",\"RegExp\",\"Set\",\"String\",\"Symbol\",\"TypeError\",\"Uint8Array\",\"Uint8ClampedArray\",\"Uint16Array\",\"Uint32Array\",\"WeakMap\",\"_\",\"clearTimeout\",\"isFinite\",\"parseInt\",\"setTimeout\"],rt=-1,at={};at[O]=at[A]=at[F]=at[D]=at[R]=at[j]=at[U]=at[I]=at[$]=!0,at[v]=at[g]=at[z]=at[y]=at[L]=at[m]=at[b]=at[_]=at[x]=at[k]=at[S]=at[C]=at[T]=at[M]=at[P]=!1;var it={};it[v]=it[g]=it[z]=it[L]=it[y]=it[m]=it[O]=it[A]=it[F]=it[D]=it[R]=it[x]=it[k]=it[S]=it[C]=it[T]=it[M]=it[N]=it[j]=it[U]=it[I]=it[$]=!0,it[b]=it[_]=it[P]=!1;var ot={\"\\\\\":\"\\\\\",\"'\":\"'\",\"\\n\":\"n\",\"\\r\":\"r\",\"\\u2028\":\"u2028\",\"\\u2029\":\"u2029\"},ut=parseFloat,lt=parseInt,st=\"object\"==typeof n.g&&n.g&&n.g.Object===Object&&n.g,ct=\"object\"==typeof self&&self&&self.Object===Object&&self,ft=st||ct||Function(\"return this\")(),pt=t&&!t.nodeType&&t,dt=pt&&e&&!e.nodeType&&e,ht=dt&&dt.exports===pt,vt=ht&&st.process,gt=function(){try{return dt&&dt.require&&dt.require(\"util\").types||vt&&vt.binding&&vt.binding(\"util\")}catch(e){}}(),yt=gt&>.isArrayBuffer,mt=gt&>.isDate,bLine truncated
|
||||||
|
"</script>"
|
||||||
|
],
|
||||||
|
"text/plain": [
|
||||||
|
"<IPython.core.display.HTML object>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"shap.initjs()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 73,
|
||||||
|
"id": "129dd5fb-367e-493e-937c-12ae0727cf85",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"array([2.58451770e-03, 2.63414029e-02, 8.63697469e-01, 1.07182816e-01,\n",
|
||||||
|
" 1.93854154e-04], dtype=float32)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 73,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"pred = bst_FS.predict(dtest_FS)\n",
|
||||||
|
"pred[2]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 53,
|
||||||
|
"id": "46bf2240-cfff-4c1b-a5db-87290bf566be",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"age 53.0\n",
|
||||||
|
"diagnosis dislocation\n",
|
||||||
|
"injury_general_location upper_limbs\n",
|
||||||
|
"country Germania\n",
|
||||||
|
"difficulty easy\n",
|
||||||
|
"cause fall_alone\n",
|
||||||
|
"destination hospital_emergency_room\n",
|
||||||
|
"injury_side R\n",
|
||||||
|
"equipment ski\n",
|
||||||
|
"gender F\n",
|
||||||
|
"snowmobile True\n",
|
||||||
|
"helicopter True\n",
|
||||||
|
"akja False\n",
|
||||||
|
"helmet True\n",
|
||||||
|
"ambulance False\n",
|
||||||
|
"snowmobile_sled False\n",
|
||||||
|
"ski_lift False\n",
|
||||||
|
"indipendently False\n",
|
||||||
|
"skiarea_ambulance False\n",
|
||||||
|
"other False\n",
|
||||||
|
"Name: 724, dtype: object"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 53,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"dataset_test.X_test_area[bst_FS.feature_names].iloc[2]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 51,
|
||||||
|
"id": "27f3b14c-4917-45ff-93ee-234dbc392b20",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"2"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 51,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"dataset_test.y_test_area.iloc[2]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 64,
|
||||||
|
"id": "9799b767-15f2-403b-8535-eb471b862ab3",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[-3.5973988, -1.704391, 1.8931085, -0.9031036, -7.5351233]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 64,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"[shap_values[i][2].sum() for i in range(5)]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 71,
|
||||||
|
"id": "f4cac6a8-1ecc-435b-a7c0-5dff7df561f3",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/html": [
|
||||||
|
"\n",
|
||||||
|
"<div id='i11854JF2PGG5AQIPP5RQ'>\n",
|
||||||
|
"<div style='color: #900; text-align: center;'>\n",
|
||||||
|
" <b>Visualization omitted, Javascript library not loaded!</b><br>\n",
|
||||||
|
" Have you run `initjs()` in this notebook? If this notebook was from another\n",
|
||||||
|
" user you must also trust this notebook (File -> Trust notebook). If you are viewing\n",
|
||||||
|
" this notebook on github the Javascript has been stripped for security. If you are using\n",
|
||||||
|
" JupyterLab this error is because a JupyterLab extension has not yet been written.\n",
|
||||||
|
"</div></div>\n",
|
||||||
|
" <script>\n",
|
||||||
|
" if (window.SHAP) SHAP.ReactDom.render(\n",
|
||||||
|
" SHAP.React.createElement(SHAP.AdditiveForceVisualizer, {\"outNames\": [\"f(x)\"], \"baseValue\": 0.552638053894043, \"outValue\": -1.1517529487609863, \"link\": \"identity\", \"featureNames\": [\"age\", \"diagnosis\", \"injury_general_location\", \"country\", \"difficulty\", \"cause\", \"destination\", \"injury_side\", \"equipment\", \"gender\", \"snowmobile\", \"helicopter\", \"akja\", \"helmet\", \"ambulance\", \"snowmobile_sled\", \"ski_lift\", \"indipendently\", \"skiarea_ambulance\", \"other\"], \"features\": {\"0\": {\"effect\": 0.09595096856355667, \"value\": \"\"}, \"1\": {\"effect\": -0.27542880177497864, \"value\": \"\"}, \"2\": {\"effect\": 0.5892016887664795, \"value\": \"\"}, \"3\": {\"effect\": -0.12579196691513062, \"value\": \"\"}, \"4\": {\"effect\": -0.01911946013569832, \"value\": \"\"}, \"5\": {\"effect\": 0.057017769664525986, \"value\": \"\"}, \"6\": {\"effect\": 0.0896696150302887, \"value\": \"\"}, \"7\": {\"effect\": 0.13361920416355133, \"value\": \"\"}, \"8\": {\"effect\": 0.0006894035032019019, \"value\": \"\"}, \"9\": {\"effect\": 0.11025260388851166, \"value\": \"\"}, \"10\": {\"effect\": 0.03937700018286705, \"value\": \"\"}, \"11\": {\"effect\": -2.0775811672210693, \"value\": \"\"}, \"12\": {\"effect\": 0.03850683569908142, \"value\": \"\"}, \"13\": {\"effect\": -0.005975374486297369, \"value\": \"\"}, \"14\": {\"effect\": -0.15785543620586395, \"value\": \"\"}, \"15\": {\"effect\": 0.00020830573339480907, \"value\": \"\"}, \"16\": {\"effect\": -0.07540600001811981, \"value\": \"\"}, \"17\": {\"effect\": -0.12225314229726791, \"value\": \"\"}, \"18\": {\"effect\": 0.00043001066660508513, \"value\": \"\"}, \"19\": {\"effect\": 9.698851499706507e-05, \"value\": \"\"}}, \"plot_cmap\": \"RdBu\", \"labelMargin\": 20}),\n",
|
||||||
|
" document.getElementById('i11854JF2PGG5AQIPP5RQ')\n",
|
||||||
|
" );\n",
|
||||||
|
"</script>"
|
||||||
|
],
|
||||||
|
"text/plain": [
|
||||||
|
"<shap.plots._force.AdditiveForceVisualizer at 0x7fb58227d450>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 71,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"i=1\n",
|
||||||
|
"shap.plots.force(explainer.expected_value[1], shap_values[i][2],feature_names=bst_FS.feature_names)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 70,
|
||||||
|
"id": "3d56cbf0-ff20-4d3a-94b7-a0e07c5cc019",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/html": [
|
||||||
|
"\n",
|
||||||
|
"<div id='iOOXREDIO2XGF7NOS1QU8'>\n",
|
||||||
|
"<div style='color: #900; text-align: center;'>\n",
|
||||||
|
" <b>Visualization omitted, Javascript library not loaded!</b><br>\n",
|
||||||
|
" Have you run `initjs()` in this notebook? If this notebook was from another\n",
|
||||||
|
" user you must also trust this notebook (File -> Trust notebook). If you are viewing\n",
|
||||||
|
" this notebook on github the Javascript has been stripped for security. If you are using\n",
|
||||||
|
" JupyterLab this error is because a JupyterLab extension has not yet been written.\n",
|
||||||
|
"</div></div>\n",
|
||||||
|
" <script>\n",
|
||||||
|
" if (window.SHAP) SHAP.ReactDom.render(\n",
|
||||||
|
" SHAP.React.createElement(SHAP.AdditiveForceVisualizer, {\"outNames\": [\"f(x)\"], \"baseValue\": 0.552638053894043, \"outValue\": 2.445746421813965, \"link\": \"identity\", \"featureNames\": [\"age\", \"diagnosis\", \"injury_general_location\", \"country\", \"difficulty\", \"cause\", \"destination\", \"injury_side\", \"equipment\", \"gender\", \"snowmobile\", \"helicopter\", \"akja\", \"helmet\", \"ambulance\", \"snowmobile_sled\", \"ski_lift\", \"indipendently\", \"skiarea_ambulance\", \"other\"], \"features\": {\"0\": {\"effect\": 0.16000449657440186, \"value\": \"\"}, \"1\": {\"effect\": 0.44064590334892273, \"value\": \"\"}, \"2\": {\"effect\": 0.3827016353607178, \"value\": \"\"}, \"3\": {\"effect\": 0.05334072560071945, \"value\": \"\"}, \"4\": {\"effect\": 0.17687170207500458, \"value\": \"\"}, \"5\": {\"effect\": 0.046136993914842606, \"value\": \"\"}, \"6\": {\"effect\": 0.362857460975647, \"value\": \"\"}, \"7\": {\"effect\": 0.1898631602525711, \"value\": \"\"}, \"8\": {\"effect\": 0.039788179099559784, \"value\": \"\"}, \"9\": {\"effect\": -0.024055209010839462, \"value\": \"\"}, \"10\": {\"effect\": 0.2057642787694931, \"value\": \"\"}, \"11\": {\"effect\": 0.08326093852519989, \"value\": \"\"}, \"12\": {\"effect\": -0.0021338891237974167, \"value\": \"\"}, \"13\": {\"effect\": -0.011541801504790783, \"value\": \"\"}, \"14\": {\"effect\": -0.12199775129556656, \"value\": \"\"}, \"15\": {\"effect\": -0.08116263896226883, \"value\": \"\"}, \"16\": {\"effect\": -0.030168458819389343, \"value\": \"\"}, \"17\": {\"effect\": 0.021043526008725166, \"value\": \"\"}, \"18\": {\"effect\": -0.0011919558746740222, \"value\": \"\"}, \"19\": {\"effect\": 0.003081258852034807, \"value\": \"\"}}, \"plot_cmap\": \"RdBu\", \"labelMargin\": 20}),\n",
|
||||||
|
" document.getElementById('iOOXREDIO2XGF7NOS1QU8')\n",
|
||||||
|
" );\n",
|
||||||
|
"</script>"
|
||||||
|
],
|
||||||
|
"text/plain": [
|
||||||
|
"<shap.plots._force.AdditiveForceVisualizer at 0x7fb5822b3190>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 70,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"i=2\n",
|
||||||
|
"shap.plots.force(explainer.expected_value[1], shap_values[i][2],feature_names=bst_FS.feature_names)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "6bbece5c-75f4-4ba0-a44e-677d9819b17a",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.11.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
+97
@@ -0,0 +1,97 @@
|
|||||||
|
from utils import retrive_data, split
|
||||||
|
from model import train, gain_accuracy_train
|
||||||
|
from sklearn.metrics import confusion_matrix,matthews_corrcoef,accuracy_score
|
||||||
|
import xgboost as xgb
|
||||||
|
import pandas as pd
|
||||||
|
import pickle
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
labeled,labeled_small = retrive_data(reload_data=args.reload_data,threshold_under_represented=0.5,path='/home/agobbi/Projects/PID/datanalytics/PID/src')
|
||||||
|
|
||||||
|
dataset,dataset_test = split(labeled_small if args.use_small else labeled ,
|
||||||
|
SKI_AREA_TEST= 'Klausberg',
|
||||||
|
SEASON_TEST_SKIAREA = 'Kronplatz',
|
||||||
|
SEASON_TEST_YEAR= 2023,
|
||||||
|
use_smote = args.use_smote,
|
||||||
|
weight_type = 'sqrt' )
|
||||||
|
if args.retrain:
|
||||||
|
|
||||||
|
|
||||||
|
print('OPTUNA hyperparameter tuning, please wait!')
|
||||||
|
best_model,params_final = train(dataset,n_trials=args.n_trials,timeout=600,num_boost_round=600)
|
||||||
|
feat_imp = pd.Series(best_model.get_fscore()).sort_values(ascending=False)
|
||||||
|
|
||||||
|
with open('best_params.pkl','wb') as f:
|
||||||
|
pickle.dump([params_final,feat_imp,best_model],f)
|
||||||
|
|
||||||
|
else:
|
||||||
|
with open('best_params.pkl','rb') as f:
|
||||||
|
params_final,feat_imp,best_model = pickle.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#for retriving prediction must convert to DMatrix type
|
||||||
|
tmp_train = xgb.DMatrix(dataset.X_train[best_model.feature_names],dataset.y_train,enable_categorical=True)
|
||||||
|
tmp_valid = xgb.DMatrix(dataset.X_valid[best_model.feature_names],dataset.y_valid,enable_categorical=True)
|
||||||
|
|
||||||
|
|
||||||
|
preds_class_valid = best_model.predict(tmp_valid)
|
||||||
|
preds_class_train= best_model.predict(tmp_train)
|
||||||
|
print('##################RESULT ON THE TRAIN SET#####################')
|
||||||
|
print(confusion_matrix(dataset.y_train,preds_class_train.argmax(1)))
|
||||||
|
print(f'MCC:{matthews_corrcoef(dataset.y_train,preds_class_train.argmax(1))}')
|
||||||
|
print(f'ACC:{accuracy_score(dataset.y_train,preds_class_train.argmax(1))}')
|
||||||
|
print('##################RESULT ON THE VALIDATION SET#####################')
|
||||||
|
print(confusion_matrix(dataset.y_valid,preds_class_valid.argmax(1)))
|
||||||
|
print(f'MCC:{matthews_corrcoef(dataset.y_valid,preds_class_valid.argmax(1))}')
|
||||||
|
print(f'ACC:{accuracy_score(dataset.y_valid,preds_class_valid.argmax(1))}')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if args.retrain_last_model:
|
||||||
|
tot,bst_FS,FS = gain_accuracy_train(dataset,feat_imp,num_boost_round=600,params=params_final)
|
||||||
|
with open('best_params_and_final_model.pkl','wb') as f:
|
||||||
|
pickle.dump([tot,bst_FS,FS],f)
|
||||||
|
else:
|
||||||
|
with open('best_params_and_final_model.pkl','rb') as f:
|
||||||
|
tot,bst_FS,FS = pickle.load(f)
|
||||||
|
|
||||||
|
dtest_FS = xgb.DMatrix(dataset_test.X_test_area[bst_FS.feature_names],dataset_test.y_test_area,enable_categorical=True,)
|
||||||
|
dtest_season_FS = xgb.DMatrix(dataset_test.X_test_season[bst_FS.feature_names],dataset_test.y_test_season,enable_categorical=True,)
|
||||||
|
preds_class_test = bst_FS.predict(dtest_FS)
|
||||||
|
preds_class_test_season = bst_FS.predict(dtest_season_FS)
|
||||||
|
|
||||||
|
mcc = matthews_corrcoef(dataset_test.y_test_area,preds_class_test.argmax(1))
|
||||||
|
acc = accuracy_score(dataset_test.y_test_area,preds_class_test.argmax(1))
|
||||||
|
cm = confusion_matrix(dataset_test.y_test_area,preds_class_test.argmax(1))
|
||||||
|
|
||||||
|
print(f'RESULT ON THE TEST SKI AREA {mcc=}, {acc=}, \n{cm=}')
|
||||||
|
mcc = matthews_corrcoef(dataset_test.y_test_season,preds_class_test_season.argmax(1))
|
||||||
|
acc = accuracy_score(dataset_test.y_test_season,preds_class_test_season.argmax(1))
|
||||||
|
cm = confusion_matrix(dataset_test.y_test_season,preds_class_test_season.argmax(1))
|
||||||
|
|
||||||
|
print(f'RESULT ON THE TEST SKI SEASON {mcc=}, {acc=}, {cm=}')
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description='Train Optuna XGBOOST model')
|
||||||
|
parser.add_argument('--use_small', action='store_true', help="Aggregate under represented input classes (es: rare country)")
|
||||||
|
parser.add_argument('--use_smote', action='store_true', help='oversampling underrperesented target labels')
|
||||||
|
parser.add_argument('--retrain', action='store_true', help='Retrain the optuna searcher')
|
||||||
|
parser.add_argument('--reload_data', action='store_true', help='Dowload data from db')
|
||||||
|
parser.add_argument('--retrain_last_model', action='store_true', help='retrain the last model')
|
||||||
|
parser.add_argument('--n_trials', type=int,default=1000, help='number of trials per optuna')
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
main(args)
|
||||||
|
|
||||||
|
#python main.py --use_small --retrain --retrain_last_model --n_trials=10 --reload_data
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
|
||||||
|
import xgboost as xgb
|
||||||
|
from sklearn.metrics import matthews_corrcoef, accuracy_score
|
||||||
|
import optuna
|
||||||
|
from utils import Dataset
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def objective(trial,dataset:Dataset,num_boost_round:int):
|
||||||
|
|
||||||
|
#These are the parameters usually used
|
||||||
|
params = dict(
|
||||||
|
learning_rate = trial.suggest_float("learning_rate", 0.01, 0.2),
|
||||||
|
max_depth= trial.suggest_int("max_depth",5, 15),
|
||||||
|
min_child_weight = trial.suggest_int("min_child_weight", 1, 8),
|
||||||
|
gamma = trial.suggest_float("gamma", 0, 10),
|
||||||
|
subsample = trial.suggest_float("subsample", 0.01,1),
|
||||||
|
colsample_bytree = trial.suggest_float("colsample_bytree", 0.01,1),
|
||||||
|
alpha = trial.suggest_float("alpha", 0, 10),
|
||||||
|
objective= 'multi:softprob',
|
||||||
|
nthread=4,
|
||||||
|
num_class= 5,
|
||||||
|
seed=27)
|
||||||
|
params['lambda'] = trial.suggest_float("lambda", 0, 10)
|
||||||
|
|
||||||
|
|
||||||
|
dtrain = xgb.DMatrix(dataset.X_train,dataset.y_train,
|
||||||
|
enable_categorical=True,
|
||||||
|
weight=dataset.weight_train)
|
||||||
|
dvalid = xgb.DMatrix(dataset.X_valid,dataset.y_valid,
|
||||||
|
enable_categorical=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
bst = xgb.train(params, dtrain,verbose_eval=False, num_boost_round=num_boost_round,
|
||||||
|
evals = [(dtrain, "train"), (dvalid, "valid")],
|
||||||
|
early_stopping_rounds=100)
|
||||||
|
|
||||||
|
preds = bst.predict(dvalid)
|
||||||
|
##MCC is more solid
|
||||||
|
mcc = matthews_corrcoef(dataset.y_valid,preds.argmax(1))
|
||||||
|
|
||||||
|
return mcc
|
||||||
|
|
||||||
|
|
||||||
|
def train(dataset,n_trials=1000,timeout=600,num_boost_round=600):
|
||||||
|
|
||||||
|
study = optuna.create_study(direction="maximize")
|
||||||
|
study.optimize(lambda trial: objective(trial,dataset,num_boost_round), n_trials=n_trials, timeout=timeout)
|
||||||
|
|
||||||
|
params_final = dict(
|
||||||
|
objective= 'multi:softprob',
|
||||||
|
nthread=4,
|
||||||
|
num_class= 5, ## 5 classi india
|
||||||
|
seed=27)
|
||||||
|
params_final.update(study.best_params)
|
||||||
|
dtrain = xgb.DMatrix(dataset.X_train,dataset.y_train,
|
||||||
|
enable_categorical=True,
|
||||||
|
weight=dataset.weight_train)
|
||||||
|
dvalid = xgb.DMatrix(dataset.X_valid,dataset.y_valid,
|
||||||
|
enable_categorical=True,
|
||||||
|
)
|
||||||
|
bst = xgb.train(params_final, dtrain,verbose_eval=False, num_boost_round=num_boost_round,
|
||||||
|
evals = [(dtrain, "train"), (dvalid, "valid")],
|
||||||
|
early_stopping_rounds=100,)
|
||||||
|
return bst,params_final
|
||||||
|
|
||||||
|
|
||||||
|
def gain_accuracy_train(dataset:Dataset,feat_imp:pd.DataFrame,num_boost_round:int,params:dict):
|
||||||
|
|
||||||
|
tot = []
|
||||||
|
for i in range(1,dataset.X_train.shape[1]):
|
||||||
|
|
||||||
|
dtrain_FS = xgb.DMatrix(dataset.X_train[list(feat_imp.head(i).index)],dataset.y_train,
|
||||||
|
enable_categorical=True,
|
||||||
|
weight=dataset.weight_train)
|
||||||
|
dvalid_FS = xgb.DMatrix(dataset.X_valid[list(feat_imp.head(i).index)],dataset.y_valid, enable_categorical=True,)
|
||||||
|
|
||||||
|
bst_FS = xgb.train(params, dtrain_FS,verbose_eval=False, num_boost_round=num_boost_round,
|
||||||
|
evals = [(dtrain_FS, "train"), (dvalid_FS, "valid")],
|
||||||
|
early_stopping_rounds=100,)
|
||||||
|
preds_class_valid = bst_FS.predict(dvalid_FS)
|
||||||
|
mcc = matthews_corrcoef(dataset.y_valid,preds_class_valid.argmax(1))
|
||||||
|
acc = accuracy_score(dataset.y_valid,preds_class_valid.argmax(1))
|
||||||
|
tot.append({'mcc':mcc,'acc':acc,'FS':i})
|
||||||
|
|
||||||
|
tot = pd.DataFrame(tot)
|
||||||
|
FS = int(tot.loc[tot.acc.argmax()].FS) ## get best
|
||||||
|
print(f'Best model with {FS} features, retraining....')
|
||||||
|
|
||||||
|
dtrain_FS = xgb.DMatrix(dataset.X_train[list(feat_imp.head(FS).index)],dataset.y_train, enable_categorical=True, weight=dataset.weight_train)
|
||||||
|
dvalid_FS = xgb.DMatrix(dataset.X_valid[list(feat_imp.head(FS).index)],dataset.y_valid,enable_categorical=True, )
|
||||||
|
bst_FS = xgb.train(params, dtrain_FS,verbose_eval=False, num_boost_round=num_boost_round,
|
||||||
|
evals = [(dtrain_FS, "train"), (dvalid_FS, "valid")],
|
||||||
|
early_stopping_rounds=100,)
|
||||||
|
|
||||||
|
|
||||||
|
return tot,bst_FS,FS
|
||||||
+149
@@ -0,0 +1,149 @@
|
|||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import psycopg2 as pg
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.model_selection import train_test_split
|
||||||
|
import pickle
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Union
|
||||||
|
import os
|
||||||
|
##AUXILIARY CLASSES
|
||||||
|
@dataclass
|
||||||
|
class Dataset:
|
||||||
|
X_train:Union[pd.DataFrame,None]
|
||||||
|
y_train:Union[pd.Series,None]
|
||||||
|
X_valid:Union[pd.DataFrame,None]
|
||||||
|
y_valid:Union[pd.Series,None]
|
||||||
|
weight_train:Union[np.array, None]
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Dataset_test:
|
||||||
|
X_test_area:Union[pd.DataFrame,None]
|
||||||
|
y_test_area:Union[pd.Series,None]
|
||||||
|
X_test_season:Union[pd.DataFrame,None]
|
||||||
|
y_test_season:Union[pd.Series,None]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def retrive_data(reload_data:bool,threshold_under_represented:float,path:str):
|
||||||
|
if reload_data:
|
||||||
|
engine = pg.connect("dbname='safeidx' user='fbk_mpba' host='172.104.247.67' port='5432' password='fbk2024$'")
|
||||||
|
df = pd.read_sql('select * from fbk_export_20240212', con=engine)
|
||||||
|
with open(os.path.join(path,'data.pkl'),'wb') as f:
|
||||||
|
pickle.dump(df,f)
|
||||||
|
else:
|
||||||
|
with open(os.path.join(path,'data.pkl'),'rb') as f:
|
||||||
|
df = pickle.load(f)
|
||||||
|
|
||||||
|
## these columns can lead to overfit!
|
||||||
|
df.drop(columns=['dateandtime','skiarea_id','day_of_year','minute_of_day','year'], inplace=True)
|
||||||
|
|
||||||
|
##evacuation_vehicles must be explicitated
|
||||||
|
ev = set({})
|
||||||
|
for i,row in df.iterrows():
|
||||||
|
ev = ev.union(set(row.evacuation_vehicles))
|
||||||
|
for c in ev:
|
||||||
|
df[c] = False
|
||||||
|
for i,row in df.iterrows():
|
||||||
|
for c in row.evacuation_vehicles:
|
||||||
|
df.loc[i,c] = True
|
||||||
|
df.drop(columns=['town','province','evacuation_vehicles'],inplace=True)
|
||||||
|
|
||||||
|
|
||||||
|
labeled = df[~pd.isna(df.india)].reset_index().drop(columns='index')
|
||||||
|
labeled['age'] = labeled['age'].astype(np.float32).fillna(np.nan)
|
||||||
|
|
||||||
|
|
||||||
|
## there are some under-represented categories,
|
||||||
|
## maybe it is possible to obtain a more stable model removing such classes
|
||||||
|
to_remove = {}
|
||||||
|
for c in labeled.columns:
|
||||||
|
if c not in ['india','age','season','skiarea_name']:
|
||||||
|
labeled[c] = labeled[c].astype('str')
|
||||||
|
tmp = labeled.groupby(c)[c].count()
|
||||||
|
tmp = 100*tmp/tmp.max()
|
||||||
|
tmp = tmp[tmp<threshold_under_represented]
|
||||||
|
if len(tmp)>0:
|
||||||
|
for k in tmp.index:
|
||||||
|
if c not in to_remove.keys():
|
||||||
|
to_remove[c]= []
|
||||||
|
to_remove[c].append(k)
|
||||||
|
|
||||||
|
## keep the datasets
|
||||||
|
labeled_small = labeled.copy()
|
||||||
|
for c in to_remove.keys():
|
||||||
|
for k in to_remove[c]:
|
||||||
|
labeled_small.loc[labeled_small[c]==k,c] = 'other'
|
||||||
|
for c in labeled_small.columns:
|
||||||
|
if c not in ['age','season','skiarea_name']:
|
||||||
|
labeled_small[c] = labeled_small[c].fillna('None').astype('category')
|
||||||
|
labeled[c] = labeled[c].fillna('None').astype('category')
|
||||||
|
labeled.dropna(inplace=True)
|
||||||
|
labeled_small.dropna(inplace=True)
|
||||||
|
|
||||||
|
#transform india into integers
|
||||||
|
labeled.india = labeled.india.apply(lambda x: x.replace('i','')).astype(int)
|
||||||
|
labeled_small.india = labeled_small.india.apply(lambda x: x.replace('i','')).astype(int)
|
||||||
|
|
||||||
|
return labeled,labeled_small
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def split(labeled:pd.DataFrame,
|
||||||
|
SKI_AREA_TEST: str = 'Klausberg',
|
||||||
|
SEASON_TEST_SKIAREA:str = 'Kronplatz',
|
||||||
|
SEASON_TEST_YEAR:int = 2023,
|
||||||
|
use_smote:bool = False,
|
||||||
|
weight_type:str = 'sqrt' ):
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
test_area = labeled[labeled.skiarea_name==SKI_AREA_TEST]
|
||||||
|
test_area_season = labeled[(labeled.skiarea_name==SEASON_TEST_SKIAREA)&(labeled.season>=SEASON_TEST_YEAR)]
|
||||||
|
|
||||||
|
##remove from dataset the corresponding test rows
|
||||||
|
labeled_tmp = labeled[labeled.skiarea_name!=SKI_AREA_TEST]
|
||||||
|
labeled_tmp = labeled_tmp[(labeled_tmp.skiarea_name!=SEASON_TEST_SKIAREA)|(labeled_tmp.season<SEASON_TEST_YEAR) ]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
X_train, X_valid, y_train, y_valid = train_test_split( labeled_tmp.drop(columns=['india','season','skiarea_name']),
|
||||||
|
labeled_tmp.india, test_size=0.33, random_state=0,stratify=labeled_tmp.india)
|
||||||
|
|
||||||
|
if use_smote:
|
||||||
|
from imblearn.over_sampling import RandomOverSampler
|
||||||
|
|
||||||
|
sm = RandomOverSampler()
|
||||||
|
X_train_smote,y_train_smote = sm.fit_resample(X_train,y_train)
|
||||||
|
X_train,y_train = sm.fit_resample(X_train,y_train)
|
||||||
|
|
||||||
|
##computed the weights for unbalanced dataset
|
||||||
|
|
||||||
|
w = pd.DataFrame(np.unique(y_train,return_counts=True)).T
|
||||||
|
w.columns = ['class','p']
|
||||||
|
|
||||||
|
##when computing the error, these are the weights used for each class: you can punish more errpr on most severe clases
|
||||||
|
if weight_type == 'sqrt':
|
||||||
|
w.p = np.sqrt(w.p.sum())/w.p
|
||||||
|
print(w)
|
||||||
|
elif weight_type == 'sum':
|
||||||
|
w.p = w.p.sum()/w.p/w.shape[0]
|
||||||
|
print(w)
|
||||||
|
else:
|
||||||
|
print(f'{weight_type=} not implemented please use a valid one: sqrt or sum, I will set all the weights to 0')
|
||||||
|
w.p = 1
|
||||||
|
|
||||||
|
if use_smote is False:
|
||||||
|
weight_train = pd.merge(pd.DataFrame({'class':y_train}),w).p.values
|
||||||
|
else:
|
||||||
|
w.p = 1
|
||||||
|
weight_train = pd.merge(pd.DataFrame({'class':y_train}),w).p.values
|
||||||
|
|
||||||
|
|
||||||
|
dataset = Dataset(X_train, y_train, X_valid, y_valid,weight_train)
|
||||||
|
dataset_test = Dataset_test(test_area,test_area.india,test_area_season,test_area_season.india)
|
||||||
|
|
||||||
|
return dataset,dataset_test
|
||||||
|
|
||||||
Reference in new issue
Block a user