agobbi/pid
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ag update readme and added documentation

Browse Source
This commit is contained in:
Andrea Gobbi committed 2024-02-22 14:39:44 +01:00
1 parent ea097a7b71
commit 7eb456384e
18 files changed
+14754 -28
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

notebooks/Variable_explanation.ipynb
+242 -15
View File
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 1,
"id": "e9ca0612-adb5-4d11-89a1-b224de3bd3b5",
"metadata": {},
"outputs": [],
@@ -15,17 +15,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "dcb4d7b5-fe20-43f1-b988-51c9d595e06d",
"metadata": {},
"outputs": [],
"source": [
"labeled,labeled_small = retrive_data(False,threshold_under_represented=0.5,path='/home/agobbi/Projects/PID/datanalytics/PID/src')"
"labeled,labeled_small,to_remove = retrive_data(False,threshold_under_represented=0.5,path='/home/agobbi/Projects/PID/datanalytics/PID/src')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 4,
"id": "569dff52-2a78-43f4-ae40-07842d3669e3",
"metadata": {},
"outputs": [
@@ -53,7 +53,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 5,
"id": "65405081-748d-43ef-aed6-2a58e38673b4",
"metadata": {},
"outputs": [],
@@ -64,10 +64,18 @@
},
{
"cell_type": "code",
"execution_count": 49,
"execution_count": 36,
"id": "53434031-1a11-49f7-9481-de77f250d35b",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n"
]
}
],
"source": [
"import shap\n",
"import xgboost as xgb\n",
@@ -80,7 +88,7 @@
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": 37,
"id": "f3b72453-27be-4db5-b40d-883454c294f9",
"metadata": {},
"outputs": [
@@ -105,7 +113,7 @@
},
{
"cell_type": "code",
"execution_count": 73,
"execution_count": 38,
"id": "129dd5fb-367e-493e-937c-12ae0727cf85",
"metadata": {},
"outputs": [
@@ -116,19 +124,18 @@
" 1.93854154e-04], dtype=float32)"
]
},
"execution_count": 73,
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred = bst_FS.predict(dtest_FS)\n",
"pred[2]"
"pred = bst_FS.predict(dtest_FS)\n"
]
},
{
"cell_type": "code",
"execution_count": 53,
"execution_count": 39,
"id": "46bf2240-cfff-4c1b-a5db-87290bf566be",
"metadata": {},
"outputs": [
@@ -158,7 +165,7 @@
"Name: 724, dtype: object"
]
},
"execution_count": 53,
"execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
@@ -188,6 +195,27 @@
"dataset_test.y_test_area.iloc[2]"
]
},
{
"cell_type": "code",
"execution_count": 44,
"id": "91430837-ad76-4835-afbb-3793bbed77fd",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0., 3., 86., 11., 0.], dtype=float32)"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.round(pred[2]*100)"
]
},
{
"cell_type": "code",
"execution_count": 64,
@@ -289,9 +317,208 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"id": "6bbece5c-75f4-4ba0-a44e-677d9819b17a",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_48291/444630369.py:4: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
" df = pd.read_sql('select * from fbk_export_20240212', con=engine)\n"
]
}
],
"source": [
"import psycopg2 as pg\n",
"import pandas as pd\n",
"engine = pg.connect(\"dbname='safeidx' user='fbk_mpba' host='172.104.247.67' port='5432' password='fbk2024$'\")\n",
"df = pd.read_sql('select * from fbk_export_20240212', con=engine) "
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "4b2cad74-76b2-4860-bacc-807156e2dfc0",
"metadata": {},
"outputs": [],
"source": [
"df = df[pd.isna(df.india)] ##solo quelli senza label"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "852e69e7-126d-4a27-beb7-9045e26f2f76",
"metadata": {},
"outputs": [],
"source": [
"from src.utils import prepare_new_data\n",
"with open('../src/to_remove.pkl','rb') as f:\n",
" to_remove = pickle.load(f)\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "1e6303a4-ab85-4476-8bdd-ee10e7f92e04",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/agobbi/Projects/PID/datanalytics/PID/notebooks/../src/utils.py:58: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value 'other' has dtype incompatible with bool, please explicitly cast to a compatible dtype first.\n",
" dataset_p_small.loc[dataset_p[c]==k,c] = 'other'\n",
"/home/agobbi/Projects/PID/datanalytics/PID/notebooks/../src/utils.py:61: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
" dataset_p_small[c] = dataset_p_small[c].fillna('None').astype('category')\n"
]
}
],
"source": [
"dataset_p,dataset_p_small = prepare_new_data(df,to_remove)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "e33a020d-d5af-4fbc-b5ba-3422ff87fd7f",
"metadata": {},
"outputs": [],
"source": [
"import xgboost as xgb\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"unlabeled_dm = xgb.DMatrix(dataset_p_small[bst_FS.feature_names],enable_categorical=True,)\n"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "396b40d7-8fd8-4145-ba1e-cd4fddcfa8a9",
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAEZCAYAAADxBRGtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbIUlEQVR4nO3dYWxV53kH8Mc4shkdOCFuTBGmTOoS1SKxU4wp3aRBaw1ZEWoSteJD1TmuRKTqOkp0m0rwBTYpFZESRazN1egyMTZNUVArQaSlzZJ5bdytRICRq3SMNEgkdZvaBFXY+EY1kX33oao3l1yKsa+P3+vfT7ofzrnH5zxHMg9/Xt7znppSqVQKAAAgCcuyLgAAALhxAjwAACREgAcAgIQI8AAAkBABHgAAEiLAAwBAQgR4AABIiAAPAAAJEeABACAhAjwAACREgAcAgIQI8ABU1Pvvvx8f//jH4/HHH8+6FICqIMADUFHf+MY34tOf/nTWZQBUDQEegIp566234ty5c9HV1ZV1KQBV45asC5itqampePfdd2PlypVRU1OTdTkAFVUqleLKlSuxdu3aWLZsYcdc+vv746mnnoqBgYH41a9+FceOHYv7779/xjGFQiGeeuqpGB4ejtbW1vjWt74VHR0d098//vjj8dRTT8WPf/zjm6pBzweWkhvt+ckF+HfffTeam5uzLgNgQQ0NDcW6desW9JrFYjFaW1vjK1/5Sjz44IPXfH/06NHI5/Nx6NCh2LJlSxw8eDB27NgRb775Ztxxxx3x4osvxp133hl33nnnDQf4iYmJmJiYmN7+5S9/GS0tLfN2TwAp+EM9v6ZUKpUWsJ45Gx0djVtvvTWGhoZi1apVWZcDUFFjY2PR3Nwcly9fjoaGhszqqKmpuWYEfsuWLbF58+Z49tlnI+K3o+XNzc3xyCOPxJ49e2Lv3r3xL//yL1FbWxvj4+PxwQcfxNe+9rXYt29f2ev89V//dfzN3/zNNfv1fGApuNGen1yAHxsbi4aGhhgdHdXMgaq3WHre7wf4q1evxooVK+K73/3ujFDf3d0dly9fjhdffHHGzx85ciR++tOfxtNPP33d6/z+CPzv/jLL+v4BFsKN9vzkptAAkL1Lly7F5ORkNDU1zdjf1NQU586du+nz1tfXR319/VzLA6hqyQT4QqEQhUIhJicnsy4FgFl66KGHZnW8ng9QXjLLSOZyuTh79mycOnUq61IAlrzGxsaora2NkZGRGftHRkZizZo1cz6/ng9QXjIBHoDFo66uLjZt2hR9fX3T+6ampqKvry+2bt2aYWUA1S+ZKTQALKzx8fE4f/789PaFCxdicHAwVq9eHevXr498Ph/d3d3R3t4eHR0dcfDgwSgWi9HT0zPna5tCA1BeMqvQ/P9m/rOf/cyKBMCSkOUqND/84Q9j+/bt1+zv7u6OI0eORETEs88+O/0ip7a2tvjmN78ZW7ZsmbcaFssqPAAL4UZ7XjIB/nc0c2ApWeo9b6nfP7C03GjPMwceAAASYg58lduw56WsS5iTt5+8L+sSgAyYAz97+j0sHcmMwBcKhWhpaYnNmzdnXQoAFWYZSYDykgnwmjkAACQU4AEAAAEeAACSIsADsOh47gmgvGQCvGYOsHR47gmgvGQCvGYOAAAJBXgAAECABwCApAjwACw6nnsCKE+AB2DR8dwTQHnJBHijMQAAkFCANxoDAAAJBXgAAECABwCApAjwAACQEAEegEXHwgUA5QnwACw6Fi4AKE+ABwCAhCQT4P13KgAAJBTg/XcqAAAkFOABAAABHgAAkiLAAwBAQgR4AABIiAAPwKJj5TGA8gR4ABYdK48BlCfAAwBAQgR4AABISDIB3nxIAACIuCXrAm5ULpeLXC4XY2Nj0dDQkHU5LFIb9ryUdQlz8vaT92VdAgCwyCUzAg8AAAjwAACQFAEeAAASIsADAEBCBHgAAEiIAA8AAAkR4AFYdLz7A6C8ZNaBnw+prxEeYZ1wYGnw7g+A8ozAAwBAQgR4AABIyJKaQgMAVIfUp8WaEstcJDMC74EmAABIKMDncrk4e/ZsnDp1KutSAAAgM8kEeAAAQIAHAICkCPAAAJAQAR4AABIiwAMAQEIEeAAASIgADwAACRHgAQAgIQI8ABVz+fLlaG9vj7a2tti4cWM899xzWZcEkLxbsi4AgOq1cuXK6O/vjxUrVkSxWIyNGzfGgw8+GLfffnvWpQEkywg8ABVTW1sbK1asiIiIiYmJKJVKUSqVMq4KIG0CPABl9ff3x86dO2Pt2rVRU1MTx48fv+aYQqEQGzZsiOXLl8eWLVvi5MmTM76/fPlytLa2xrp16+LrX/96NDY2LlD1ANVJgAegrGKxGK2trVEoFD70+6NHj0Y+n4/9+/fHmTNnorW1NXbs2BEXL16cPubWW2+Nn/zkJ3HhwoV4/vnnY2RkpOz1JiYmYmxsbMYHgJkEeADK6urqiieeeCIeeOCBD/3+mWeeid27d0dPT0+0tLTEoUOHYsWKFXH48OFrjm1qaorW1tb40Y9+VPZ6Bw4ciIaGhulPc3PzvN0LQLUQ4AG4KVevXo2BgYHo7Oyc3rds2bLo7OyMEydORETEyMhIXLlyJSIiRkdHo7+/P+66666y59y7d2+Mjo5Of4aGhip7EwAJsgoNADfl0qVLMTk5GU1NTTP2NzU1xblz5yIi4p133omHH354+uHVRx55JO6+++6y56yvr4/6+vqK1g2QuswC/Pvvvx+f/OQn44tf/GI8/fTTWZUBQAV1dHTE4ODgrH+uUChEoVCIycnJ+S8KIHGZTaH5xje+EZ/+9KezujwAc9TY2Bi1tbXXPJQ6MjISa9asmdO5c7lcnD17Nk6dOjWn8wBUo0wC/FtvvRXnzp2Lrq6uLC4PwDyoq6uLTZs2RV9f3/S+qamp6Ovri61bt2ZYGUB1m3WAn481gR9//PE4cODATRcNwMIYHx+PwcHB6WkwFy5ciMHBwfj5z38eERH5fD6ee+65+Kd/+qf4n//5n/jqV78axWIxenp6MqwaoLrNOsDPdU3gF198Me688864884751Y5ABV3+vTpuPfee+Pee++NiN8G9nvvvTf27dsXERG7du2Kp59+Ovbt2xdtbW0xODgYL7/88jUPts5WoVCIlpaW2Lx585zvAaDazPoh1q6urutOffn/awJHRBw6dCheeumlOHz4cOzZsydef/31eOGFF+I73/lOjI+PxwcffBCrVq2a/svg901MTMTExMT0tpd6ACycbdu2RalUuu4xvb290dvbO6/XzeVykcvlYmxsLBoaGub13ACpm9c58DeyJvCBAwdiaGgo3n777Xj66adj9+7dZcP77473Ug8AAPiteQ3w11sTeHh4+KbO6aUeAADwfzJ9kdNDDz30B4/xUg+Apcc68ADlzesIfCXXBPZAE8DSYR14gPLmNcBXck1gzRwAAG5iCs34+HicP39+evt3awKvXr061q9fH/l8Prq7u6O9vT06Ojri4MGD1gQGAIB5MusAf/r06di+ffv0dj6fj4iI7u7uOHLkSOzatSvee++92LdvXwwPD0dbW9u8rAkMwNJhDjxAebMO8FmtCayZAywd1oEHKG9e58BXkjnwAACQUIAHAAAEeAAASIoADwAACUkmwHuRE8DSoecDlDfrVWiyYkUCgKVjPnr+hj0vzXNVC+vtJ+/LugRgkUpmBB4AABDgAQAgKQI8AAAkJJkA74EmAABIKMB7EyvA0mHQBqC8ZAI8AEuHQRuA8gR4AABIiAAPAAAJEeABACAhyQR4DzQBAEBCAd4DTQAAkFCABwAABHgAAEiKAA/AouO5J4DyBHgAFh3PPQGUJ8ADAEBCBHgAAEjILVkXcKMKhUIUCoWYnJzMuhRYNDbseSnrEubk7Sfvy7oEAEhOMiPw5kMCAEBCAR4AABDgAQAgKQI8AAAkRIAHAICECPAAAJAQAR4AABIiwAOw6BQKhWhpaYnNmzdnXQrAopNMgNfMAZYO7/4AKC+ZAK+ZAwBAQgEeAAAQ4AEAICkCPAAAJESABwCAhAjwAACQEAEeAAASIsADAEBCBHgAAEiIAA8AAAkR4AEAICECPAAAJESABwCAhCQT4AuFQrS0tMTmzZuzLgWAGzQ0NBTbtm2LlpaWuOeee+I73/lO1iUBJC+ZAJ/L5eLs2bNx6tSprEsB4AbdcsstcfDgwTh79my88sor8dhjj0WxWMy6LICk3ZJ1AQBUr4997GPxsY99LCIi1qxZE42NjfHrX/86PvKRj2RcGUC6khmBB2Dh9ff3x86dO2Pt2rVRU1MTx48fv+aYQqEQGzZsiOXLl8eWLVvi5MmTH3qugYGBmJycjObm5gpXDVDdBHgAyioWi9Ha2hqFQuFDvz969Gjk8/nYv39/nDlzJlpbW2PHjh1x8eLFGcf9+te/jr/6q7+Kv//7v7/u9SYmJmJsbGzGB4CZBHgAyurq6oonnngiHnjggQ/9/plnnondu3dHT09PtLS0xKFDh2LFihVx+PDh6WMmJibi/vvvjz179sRnPvOZ617vwIED0dDQMP0xWg9wLQEegJty9erVGBgYiM7Ozul9y5Yti87Ozjhx4kRERJRKpXjooYfis5/9bHz5y1/+g+fcu3dvjI6OTn+GhoYqVj9AqgR4AG7KpUuXYnJyMpqammbsb2pqiuHh4YiI+K//+q84evRoHD9+PNra2qKtrS3eeOONsuesr6+PVatWzfgAMJNVaAComD//8z+PqampWf9coVCIQqEQk5OTFagKIG1G4AG4KY2NjVFbWxsjIyMz9o+MjMSaNWvmdG7v/gAozwg8ADelrq4uNm3aFH19fXH//fdHRMTU1FT09fVFb29vtsVBldmw56WsS5iTt5+8L+sSqooAD0BZ4+Pjcf78+entCxcuxODgYKxevTrWr18f+Xw+uru7o729PTo6OuLgwYNRLBajp6cnw6oBqpsAD0BZp0+fju3Line truncated
"text/plain": [
"<Figure size 900x300 with 2 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"fig, axs = plt.subplots(1, 2, figsize=(9, 3))\n",
"axs[0].bar(range(5),np.unique(np.argmax(bst_FS.predict(unlabeled_dm),1),return_counts=True)[1])\n",
"axs[0].set_yscale(\"log\", base=10)\n",
"axs[1].bar(range(5),np.unique(dataset.y_train,return_counts=True)[1])\n",
"axs[1].set_yscale(\"log\", base=10)"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "d1e3f7cb-e732-46f3-9d18-93eec0f3b90e",
"metadata": {},
"outputs": [],
"source": [
"with open('../src/best_params.pkl','rb') as f:\n",
" params_final,feat_imp,best_model = pickle.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "40d9658f-60c0-4c6a-b348-8bd1a307c1f4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"xgboost.core.Booster"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"xgb.Booster"
]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "2fe4e627-d4c9-484e-8912-7a12b0138dea",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<Axes: >"
]
},
"execution_count": 48,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAIwCAYAAACY8VFvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9OElEQVR4nO3dd1hUV/c24GdAmsBQBSygKBYQu1GJXYxEsZfE2Fvyaqxgf6OoWIOiYnntBjUaY6KxhFhRsaHYwN4LRkWNBhFREdjfH/44HyNYmDPDcPC5r2uuyJnDmgWBw5p99l5bJYQQICIiIlIQI0MnQERERJRbLGCIiIhIcVjAEBERkeKwgCEiIiLFYQFDREREisMChoiIiBSHBQwREREpTiFDJ6AvGRkZuHfvHqytraFSqQydDhEREX0EIQSePXuGYsWKwcjo3eMsBbaAuXfvHlxdXQ2dBhEREWnhzp07KFGixDufL7AFjLW1NYA33wC1Wm3gbIiIiOhjJCUlwdXVVfo7/i4FtoDJvG2kVqtZwBARESnMh6Z/cBIvERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIpTyNAJGEKpMRG5Ov/WDH89ZUJERETa4AgMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUR1YBM2PGDKhUKgwbNkw69vLlSwwcOBAODg6wsrJChw4d8ODBA43Pi4+Ph7+/PwoXLgwnJyeMHDkSaWlpGufs378f1atXh5mZGTw8PBAeHi4nVSIiIipAtC5gjh8/jiVLlqBy5coaxwMCArBt2zb89ttviIqKwr1799C+fXvp+fT0dPj7+yM1NRVHjhzBqlWrEB4ejqCgIOmcmzdvwt/fH40bN0ZsbCyGDRuGfv36YefOndqmS0RERAWIVgVMcnIyunbtimXLlsHOzk46/vTpU6xYsQKzZ89GkyZNUKNGDfz00084cuQIjh49CgDYtWsXLly4gJ9//hlVq1ZF8+bNMXnyZCxcuBCpqakAgMWLF8Pd3R2hoaHw9PTEoEGD0LFjR8yZM0cHXzIREREpnVYFzMCBA+Hv74+mTZtqHD958iRev36tcbxChQpwc3NDdHQ0ACA6OhqVKlWCs7OzdI6fnx+SkpJw/vx56Zy3Y/v5+UkxcvLq1SskJSVpPIiIiKhgKpTbT1i/fj1OnTqF48ePZ3suISEBpqamsLW11Tju7OyMhIQE6ZysxUvm85nPve+cpKQkvHjxAhYWFtlee/r06Zg0aVJuvxwiIiJSoFyNwNy5cwdDhw7F2rVrYW5urq+ctDJ27Fg8ffpUety5c8fQKREREZGe5KqAOXnyJB4+fIjq1aujUKFCKFSoEKKiojBv3jwUKlQIzs7OSE1NRWJiosbnPXjwAC4uLgAAFxeXbKuSMj/+0DlqtTrH0RcAMDMzg1qt1ngQERFRwZSrAsbX1xdnz55FbGys9KhZsya6du0q/dvExASRkZHS51y+fBnx8fHw8fEBAPj4+ODs2bN4+PChdM7u3buhVqvh5eUlnZM1RuY5mTGIiIjo05arOTDW1tbw9vbWOGZpaQkHBwfpeN++fREYGAh7e3uo1WoMHjwYPj4+qFOnDgCgWbNm8PLyQvfu3RESEoKEhASMGzcOAwcOhJmZGQCgf//+WLBgAUaNGoU+ffpg79692LBhAyIiInTxNRMREZHC5XoS74fMmTMHRkZG6NChA169egU/Pz/873//k543NjbGn3/+iQEDBsDHxweWlpbo2bMngoODpXPc3d0RERGBgIAAhIWFoUSJEli+fDn8/Px0nS4REREpkEoIIQydhD4kJSXBxsYGT58+zTYfptSY3I3k3Jrhr8vUiIiI6B3e9/c7K+6FRERERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFyVcAsWrQIlStXhlqthlqtho+PD7Zv3y49//LlSwwcOBAODg6wsrJChw4d8ODBA40Y8fHx8Pf3R+HCheHk5ISRI0ciLS1N45z9+/ejevXqMDMzg4eHB8LDw7X/ComIiKjAyVUBU6JECcyYMQMnT57EiRMn0KRJE7Rp0wbnz58HAAQEBGDbtm347bffEBUVhXv37qF9+/bS56enp8Pf3x+pqak4cuQIVq1ahfDwcAQFBUnn3Lx5E/7+/mjcuDFiY2MxbNgw9OvXDzt37tTRl0xERERKpxJCCDkB7O3tMXPmTHTs2BFFihTBunXr0LFjRwDApUuX4OnpiejoaNSpUwfbt29Hy5Ytce/ePTg7OwMAFi9ejNGjR+PRo0cwNTXF6NGjERERgXPnzkmv0blzZyQmJmLHjh0fnVdSUhJsbGzw9OlTqNVqjedKjYnI1dd4a4Z/rs4nIiIi7bzv73dWWs+BSU9Px/r16/H8+XP4+Pjg5MmTeP36NZo2bSqdU6FCBbi5uSE6OhoAEB0djUqVKknFCwD4+fkhKSlJGsWJjo7WiJF5TmaMd3n16hWSkpI0HkRERFQw5bqAOXv2LKysrGBmZob+/fvjjz/+gJeXFxISEmBqagpbW1uN852dnZGQkAAASEhI0CheMp/PfO595yQlJeHFixfvzGv69OmwsbGRHq6urrn90oiIiEghcl3AlC9fHrGxsTh27BgGDBiAnj174sKFC/rILVfGjh2Lp0+fSo87d+4YOiUiIiLSk0K5/QRTU1N4eHgAAGrUqIHjx48jLCwMX3/9NVJTU5GYmKgxCvPgwQO4uLgAAFxcXBATE6MRL3OVUtZz3l659ODBA6jValhYWLwzLzMzM5iZmeX2yyEiIiIFkt0HJiMjA69evUKNGjVgYmKCyMhI6bnLly8jPj4ePj4+AAAfHx+cPXsWDx8+lM7ZvXs31Go1vLy8pHOyxsg8JzMGERERUa5GYMaOHYvmzZvDzc0Nz549w7p167B//37s3LkTNjY26Nu3LwIDA2Fvbw+1Wo3BgwfDx8cHderUAQA0a9YMXl5e6N69O0JCQpCQkIBx48Zh4MCB0uhJ//79sWDBAowaNQp9+vTB3r17sWHDBkRE5G7lEBERERVcuSpgHj58iB49euD+/fuwsbFB5cqVsXPnTnzxxRcAgDlz5sDIyAgdOnTAq1ev4Ofnh//973/S5xsbG+PPP//EgAED4OPjA0tLS/Ts2RPBwcHSOe7u7oiIiEBAQADCwsJQokQJLF++HH5+fjr6komIiEjpZPeBya/YB4aIiEh59N4HhoiIiMhQWMAQERGR4rCAISIiIsVhAUNERESKwwKGiIiIFIcFDBERESkOCxgiIiJSHBYwREREpDgsYIiIiEhxWMAQERGR4rCAISIiIsVhAUNERESKwwKGiIiIFIcFDBERESkOCxgiIiJSHBYwREREpDgsYIiIiEhxWMAQERGR4rCAISIiIsVhAUNERESKwwKGiIiIFIcFDBERESkOCxgiIiJSHBYwREREpDgsYIiIiEhxWMAQERGR4rCAISIiIsVhAUNERESKwwKGiIiIFIcFDBERESkOCxgiIiJSHBYwREREpDgsYIiIiEhxWMAQERGR4rCAISIiIsVhAUNERESKwwKGiIiIFIcFDBERESkOCxgiIiJSHBYwREREpDgsYIiIiEhxWMAQERGR4rCAISIiIsVhAUNERESKwwKGiIiIFIcFDBERESlOIUMnUNCUGhORq/NvzfDXUyZEREQFF0dgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlIcFjBERESkOCxgiIiISHFYwBAREZHisIAhIiIixWEBQ0RERIrDAoaIiIgUhwUMERERKQ4LGCIiIlKcXBUw06dPx2effQZra2s4OTmhbdu2uHz5ssY5L1++xMCBA+Hg4AArKyt06NABDx480DgnPj4e/v7+KFy4MJycnDBy5EikpaVpnLN//35Ur14dZmZm8PDwQHh4uHZfIRERERU4uSpgoqKiMHDgQBw9ehS7d+/G69ev0axZMzx//lwLine truncated
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"feat_imp.plot(kind='bar')"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "2b89c4cc-92c6-416e-bb6e-2de31cf310a7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x7fc47694f650>"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRs0lEQVR4nO3deXhTZcI+/jtJm6RbUkppulBoWRSQpVpoxdFxq4IiiuJMZUSwKvMbBF+177wq40hlxrHi9q0zMjAi6DiILIqKylSdIo5otUJFQfZS9qYrJF2TNjm/P04TCG2ak61Jm/tzXbmSnJyc8ySxnptnlQmCIICIiIgoQOSBLgARERGFNoYRIiIiCiiGESIiIgoohhEiIiIKKIYRIiIiCiiGESIiIgoohhEiIiIKKIYRIiIiCqiwQBdACqvVitOnTyMmJgYymSzQxSEiIiIJBEFAY2MjkpOTIZc7r//oE2Hk9OnTSE1NDXQxiIiIyAMnTpzA4MGDnb7eJ8JITEwMAPHDaDSaAJeGiIiIpDAajUhNTbVfx53pE2HE1jSj0WgYRoiIiPoYV10s2IGViIiIAophhIiIiAKKYYSIiIgCimGEiIiIAophhIiIiAKKYYSIiIgCimGEiIiIAophhIiIiAKqT0x6RkRE5HdWC3DsG6CpGojWAUOvAOSKQJcqJDCMEBFR3+Pr4LB3M1D8OGA8fW6bJhmYuhQYc6v35aUeMYwQEfV1vr4w+6OGwJfH9HVw2LsZ2DAHgOC43Vglbv/1W54dN9i/R3+V0QMMI0REfZk/Lsy+riHw5TF9HRysFrFsFx4P6NwmA4qfAEZNc+8iHezfo7/K6CGZIAjd/QJBxWg0QqvVwmAwcKE8IiIbZxdmdC5K5u6F2dfH8/UxrRagaKzjxfPCY8YkAvf/B7C2Ax0moKNNvLeYHJ/b7mv2Ad+vdH3uWe8AF90EuFjwDUDwf4/+KmM3pF6/GUaIiPoilxdmABEDgJueB8IjgTBV503d9V6hBOThwN+zer7Qa5KBR3ZLryGQEh6iE4BZ64D2VsDcBJgaxZu5CTA1ndtmbgLOHAdOfS/t3P4gDweiBgFR8Z33FzyOThC/83W/EZs9uuWn79GdY/r6eD1gGCEi6q8EASh/C/jof3r/3BEDxRAjRYcJaK33b3m6I5OLAUyh7CZ8XRDK2oxA5Ze9X8aBIwFVjLR9TY1A/SHfHVPq8eZ+DKRf5Xq/Hki9frPPCBFRT4Kkgx8EAdD/BOz9EPj5A6ChQtr74i8G1FonzRRmoKMVEKzSy+GPcKGOFWsWVNGAMlq8oCqjuz5vrAK++avr483ZLP0iaq8lqEL3/UY6awkW7gBaG4DmWqC5rvO+FmiqcXx+9jjQdtb1eaWEAXf5+phOa3d8j2GEiAInWC70zvijg587n1kQgKofgb0fiCGk4ci51+ThYr8IV6a95PrCbOkAKrYCa3/l+njTXwGSL3W9HwCc/gH46GHX++WukRYerBZgz7uug8PQK6SVDxC/+6lLO/tPyC44bmf/ianPAcpI8aYd3PPxKr8C/nmL6/NetxjQXSKtjNU/A1v/5LtjSj1etM71Pj7CZhoiCoxAX+illM8fnRBdfWZBEC/iez8UQ8iZo+f2DVMDI28AxswARuQAyye7vjC73Y/AR8fz1zHtvwvQbXDwtONlt79NihhE3B2dE+zfoz/K6ITU67dH08EvW7YMaWlpUKvVyM7ORllZmdN9r7nmGshksi63adOmeXJqIuoPbBeUCzvQ2YZn7t3s2TGLxor/Kn3vfvG+aKxnx3I53BPicE+rxb3y9fSZt/8/4LOngFcmACuvBb4uEoNIWAQw+lbgztXA/1WItQjj7gQiYsUQA8B+IbY771/0Ui8mthoCXx3PX8ccc6sYODRJjts1yd6NABlzK/DIHrGfxMxV4v0ju90/Xl/4Hv1RRi+5XTOyfv16zJkzBytWrEB2djaKioqwceNGHDhwAAkJCV32b2hogNlstj+vr6/HhAkT8Prrr+Pee++VdE7WjBD1I/7oye9JLYYgAOZmoM0AmIzifVvn/akdwHcrXJ/38geBpIzOKvwosV+DMkq8hdvuI8Q+Ga5GvpwvPBIYeSMw5jbgoinicXr67L74F72/juevY/bJJr4g+x79UcYL+G00TXZ2NiZNmoRXX30VAGC1WpGamoqHHnoITzzxhMv3FxUVYfHixaiqqkJUVA9/YOdhGCHqR6S2qYdHihf3cLVYO3D+fXik2GQRHiGOmNi9URz66UyYGkiZCJgbO0NHZ/AQ3KjZ8JhMPH9Hq+td064EJs0Tm2J6CiAX6guzcgZ7ePCHvvA9+vl38ctoGrPZjJ07d2LRokX2bXK5HDk5OSgtLZV0jFWrVuGuu+7qMYiYTCaYTCb7c6PR6E4xiSiYNeql7dfeIt58oaMNOLa9+9fkYeJoE5VGvFdrAYsZOC7h/2mpl4uByNws3tqbz3tsK7sgLYgAQGYecMkMafs6fAaF10Mw/Xo8fx0z2PWF7zFIfhe3wkhdXR0sFgt0OscetjqdDvv373f5/rKyMuzZswerVq3qcb/CwkIsWbLEnaIRUV9Q/bPYN0KK218TRwZ0tIkTYrW3ihf19jbH+1PlwD4J/UKyfis2fZwfOtQasZblwlk1pXbwy9vi/F+RVqsYSMzNwJFtwPu/dV3GXhy9QBRMenVo76pVqzBu3DhkZWX1uN+iRYuQn59vf240GpGamurv4hGRv5gagW3PAd8u72wauXAI5fk6L/Tj7pRWXVz5lbQwMvpW6f8ClDrcs6fyyeXiPBmqaPGzlDzt2yGpRP2IW6Np4uPjoVAoUF3tOBFKdXU1EhMTe3xvc3Mz1q1bh/vvv9/leVQqFTQajcONiPogQQD2bAJenQSUvioGkdHTgVtehnhR90FP/qFXiBfyLsc675iaFPcv9L4ctRGEoxeIgolbNSNKpRKZmZkoKSnBjBkzAIgdWEtKSrBw4cIe37tx40aYTCbMnj3b48ISUR9SdwjY8nuxiQIABqQDN78IjMwRn0fGO5lzw82e/L6oxXBmzK3iaq2+6OBnCze++MxE/YxHQ3vnzp2Lf/zjH8jKykJRURE2bNiA/fv3Q6fTYc6cOUhJSUFhYaHD+6666iqkpKRg3bp1bheSo2mI+hBzC/DVS8DXr4gzhCpUwFX5wC8eEUfCnM/Xk5T5eZiiT4TiqBIKWX5bmyY3Nxe1tbVYvHgx9Ho9MjIyUFxcbO/Uevz4ccjljq0/Bw4cwPbt2/HZZ5+5ezoi6kv2bwH+/ThgOC4+H3kjcNNSIG5Y9/v7sie/L2sx/ClIRi8QBRNOB09E0vT0L/ozR8UQcrBYfK4ZDNz0HDDqlq4jVYgoZHDVXqJQ5/cmkGTgxmeA+iPAVy+KQ3Dl4cAVC4Ff/p97k3YRUUhjGCHyRLC3+/tyETpnU60bTwPv3nfuedpV4gqxgy72uNhEFJoYRojc1VdXm7UtyObOsNQeF4zrJJMDt/8DGPcrNskQkUcYRojc4csL/fnH9FW4cbnarAz492NA0gRxynNzc+fspp335pZz07C3twA1B1wv7iZYgZgkBhEi8hjDCAWfYG0CkbKs/Jb/A9J/KU41LuXi7E24aW8FmmrEW3ON+H2d2OEiPAhAYxXwynjXZXNHU7XrfYiInGAYoeDijyYQX2g9I05l7qqWoEkPLB0qduSMGHDuFhnn+DxigBhY/v0Yegw3mx8CavYBLXXiBb+pVrxvrhWXvfeUTCFOUx7eucS9MlJco8V2U0aK21vPAPs+cn08rqlCRF7g0F4KHs5qCWyzaHrSBAJ4VtNi6QBO7QAqtgKHS4DT5WJzRLBRqMTPFD1IvLdagEOfun7f3I+lzXUhdcG4R3YHR+0VEQUVDu2lvkVKX4fiJ8RJrdy56LlT03Lm6LnwUfnfrjUP2lTAcML1Oe9+D0gYJdYq2G4tDec9bwBazwJ1B8WbK0OvBIZki2EjqjN0RCeIN5XGsTlIaniQuk6LP6daJyLqxDBCweHYN677OhhPAfs/Bi6+GVCEuz6mq/4Yd/xDvJgfLhFDSEOF434RA4Bh1wLDrxNvMYnSLvTDrxUvztrBPZev8ivgn7e4/hzXPNG7q81eiGuqEJGfMYxQcJDaAXLDHPE+Ik6sJYgaBETFd9YYJJx7HBEHbPlf9NgfY9NvHTfLw4DBWWLwGHEdkJTR9aLtywu9bbVZXy8r74/w0FemWieiPol9Rig4SK0l6BICvBStE6csH3G9OGmXWsJ/X75ckM1eewN0G2487ScDBO+oJCIKGewzQn2LOlacPMtpJ9HOWoL/2SX25WiuFW9NNUBz3bnntsdnjoiPXZnyLDDuTvfK2leWleeCbETURzCMUOAdKwXeyT0viPTQBBKmBMLixeYYjHZ+TKk1LZ4OSQ3F1WaJiPyEYYQCa/8W4N08cZG11Gwg815g65+9ryXwV38Mf2EtBhGFMIYRCpzyt4CPHhZrRC6aCtz5hjjZ1vhc72sJOCSViKjPYBih3icIwPaXgZI/ic8zZgPTXwEUnf85+qqWgENSiYj6BIYR6l1WK/DpIuC7FeLzKx8Fri/w3yJr7I9BRBT0GEao93SYgQ9+B+x5T3w+pRCY/KD/z8v+GEREQY1hhHqHqRFYPxs4sk2cXGzGCmD8rwJdKiIiCgIMI+R/TbXA2l8Bp38QV4nN/Zc4yRgREREYRsjfzhwF/nWHuO5L5EDgNxuBwZmBLhUREQURhhHyH/1uYM1MseOodghwz/tA/IhAl4qIiIIMwwj5x9HtwDuzxKnbEy4BZr8HaJICXSoiIgpCDCPBrC8sdNZdGfd/Arz3AGAxAUOuAGa9A0TEBrqkREQUpBhGglW3K8Mmi7OKejNZly8DTndlVGuBNiMAQVwNd+brQHiE5+UlIqJ+j2EkGNmXlb9gTRVjlbjd02XlfRlwnJWxzSDep18D/Oqf52ZVJSIickIe6ALQBawWMTB0u7hLine truncated
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.plot(tot.FS,tot.mcc,'o-', label='MCC')\n",
"plt.plot(tot.FS,tot.acc,'o-', label='Accuracy')\n",
"plt.legend()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "27b66569-1e5b-4b76-ad38-8122253cef2a",
"metadata": {},
"outputs": [],
"source": []
}
notebooks/old_notebooks/.ipynb_checkpoints/test_binary-checkpoint.ipynb
+686
View File
@@ -0,0 +1,686 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"id": "7c5d059b-ed8a-4e2e-9420-25890f648895",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42878/2472232159.py:7: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
" df = pd.read_sql('select * from data_safeidx', con=engine)\n"
]
}
],
"source": [
"import pandas as pd\n",
"import psycopg2 as pg\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.model_selection import train_test_split\n",
"engine = pg.connect(\"dbname='safeidx' user='fbk_mpba' host='172.104.247.67' port='5432' password='fbk2024$'\")\n",
"df = pd.read_sql('select * from data_safeidx', con=engine)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "03aa2a04-93fa-469e-a678-685cacdebd6c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>difficulty</th>\n",
" <th>cause</th>\n",
" <th>town</th>\n",
" <th>province</th>\n",
" <th>gender</th>\n",
" <th>equipment</th>\n",
" <th>helmet</th>\n",
" <th>destination</th>\n",
" <th>diagnosis</th>\n",
" <th>india</th>\n",
" <th>age</th>\n",
" <th>country</th>\n",
" <th>injury_side</th>\n",
" <th>injury_general_location</th>\n",
" <th>evacuation_vehicles</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>novice</td>\n",
" <td>fall_alone</td>\n",
" <td>SIKLOS</td>\n",
" <td></td>\n",
" <td>F</td>\n",
" <td>ski</td>\n",
" <td>None</td>\n",
" <td>hospital_emergency_room</td>\n",
" <td>distortion</td>\n",
" <td>None</td>\n",
" <td>32.0</td>\n",
" <td>Ungheria</td>\n",
" <td>L</td>\n",
" <td>lower_limbs</td>\n",
" <td>[akja]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>advanced</td>\n",
" <td>fall_alone</td>\n",
" <td>MALMO</td>\n",
" <td></td>\n",
" <td>M</td>\n",
" <td>ski</td>\n",
" <td>None</td>\n",
" <td>hospital_emergency_room</td>\n",
" <td>bruise</td>\n",
" <td>None</td>\n",
" <td>32.0</td>\n",
" <td>Svezia</td>\n",
" <td>R</td>\n",
" <td>skull_or_face</td>\n",
" <td>[akja]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>advanced</td>\n",
" <td>fall_alone</td>\n",
" <td>CALDARO</td>\n",
" <td>BZ</td>\n",
" <td>F</td>\n",
" <td>ski</td>\n",
" <td>None</td>\n",
" <td>domicile</td>\n",
" <td>other</td>\n",
" <td>None</td>\n",
" <td>12.0</td>\n",
" <td>Italia</td>\n",
" <td>R</td>\n",
" <td>None</td>\n",
" <td>[snowmobile]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>advanced</td>\n",
" <td>collision_person</td>\n",
" <td>LINZ</td>\n",
" <td></td>\n",
" <td>M</td>\n",
" <td>ski</td>\n",
" <td>None</td>\n",
" <td>hospital_emergency_room</td>\n",
" <td>bruise</td>\n",
" <td>None</td>\n",
" <td>58.0</td>\n",
" <td>Austria</td>\n",
" <td>R</td>\n",
" <td>lower_limbs</td>\n",
" <td>[snowmobile]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>advanced</td>\n",
" <td>collision_person</td>\n",
" <td>RUSAVA</td>\n",
" <td></td>\n",
" <td>M</td>\n",
" <td>ski</td>\n",
" <td>None</td>\n",
" <td>other</td>\n",
" <td>bruise</td>\n",
" <td>None</td>\n",
" <td>25.0</td>\n",
" <td>Repubblica Ceca</td>\n",
" <td>L</td>\n",
" <td>lower_limbs</td>\n",
" <td>[other]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" difficulty cause town province gender equipment helmet \\\n",
"0 novice fall_alone SIKLOS F ski None \n",
"1 advanced fall_alone MALMO M ski None \n",
"2 advanced fall_alone CALDARO BZ F ski None \n",
"3 advanced collision_person LINZ M ski None \n",
"4 advanced collision_person RUSAVA M ski None \n",
"\n",
" destination diagnosis india age country \\\n",
"0 hospital_emergency_room distortion None 32.0 Ungheria \n",
"1 hospital_emergency_room bruise None 32.0 Svezia \n",
"2 domicile other None 12.0 Italia \n",
"3 hospital_emergency_room bruise None 58.0 Austria \n",
"4 other bruise None 25.0 Repubblica Ceca \n",
"\n",
" injury_side injury_general_location evacuation_vehicles \n",
"0 L lower_limbs [akja] \n",
"1 R skull_or_face [akja] \n",
"2 R None [snowmobile] \n",
"3 R lower_limbs [snowmobile] \n",
"4 L lower_limbs [other] "
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 99,
"id": "babc2e8b-1030-4e8a-aa41-6d2a788959a5",
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'Series' object has no attribute 'evacuation_vehicles'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipykernel_42878/3368331793.py\u001b[0m in \u001b[0;36m?\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mev\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevacuation_vehicles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/miniconda3/envs/pid/lib/python3.11/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 6289\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_accessors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6290\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6291\u001b[0m ):\n\u001b[1;32m 6292\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6293\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m: 'Series' object has no attribute 'evacuation_vehicles'"
]
}
],
"source": [
"ev = set({})\n",
"for i,row in df.iterrows():\n",
" ev = ev.union(set(row.evacuation_vehicles))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c8d6cc1c-f4f5-44ec-8652-b135963452ab",
"metadata": {},
"outputs": [],
"source": [
"for c in ev:\n",
" df[c] = False\n",
"for i,row in df.iterrows():\n",
" for c in row.evacuation_vehicles:\n",
" df.loc[i,c] = True"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "758c3317-1b02-4aed-b94d-7b6998d23797",
"metadata": {},
"outputs": [],
"source": [
"df.drop(columns=['town','province','evacuation_vehicles'],inplace=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "adadc0dc-9d6e-4277-8956-d1d4b2492e7e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 100,
"id": "33617e77-7c2b-41a3-96c0-8930aa5ac869",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(array([1.3808e+04, 0.0000e+00, 8.8100e+02, 0.0000e+00, 0.0000e+00,\n",
" 3.3690e+03, 0.0000e+00, 1.5200e+02, 0.0000e+00, 1.1000e+01]),\n",
" array([0. , 0.4, 0.8, 1.2, 1.6, 2. , 2.4, 2.8, 3.2, 3.6, 4. ]),\n",
" <BarContainer object of 10 artists>)"
]
},
"execution_count": 100,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApzUlEQVR4nO3df3AUdZ7/8VdCSALITAgcGWYNkDo5fiwcrKAQRVaWHGGJWrnF1UAWKDcLupcoCPJrhYiuLhIOEZQlx653oWrhROqEw6CBGFbiQggQzAERIlvHb2oSvZAZiUsIpL9/bKW/zIKQyIRhPjwfVV3l9Ofdn373NKm87PT0hFmWZQkAAMAw4cFuAAAAoDUQcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARooIdgPB1NjYqLNnz6pjx44KCwsLdjsAAKAZLMvS119/LbfbrfDwb79ec0eHnLNnzyo+Pj7YbQAAgO/g1KlTuvvuu791/I4OOR07dpT01zfJ4XAEuRsAANAcPp9P8fHx9u/xb3NHh5ymP1E5HA5CDgAAIeZGt5pw4zEAADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACO1OOQUFxfr0UcfldvtVlhYmDZt2vSttc8884zCwsL05ptv+q2vqalRenq6HA6HYmJilJGRofPnz/vVHDhwQA899JCio6MVHx+vnJycq+bfsGGD+vTpo+joaA0YMEAffvhhSw8HAAAYqsUhp66uTgMHDtTKlSuvW7dx40bt3r1bbrf7qrH09HRVVFSosLBQ+fn5Ki4u1tSpU+1xn8+n0aNHq0ePHiorK9OSJUu0cOFCrV692q7ZtWuXxo8fr4yMDH322WdKTU1VamqqDh061NJDAgAAJrJugiRr48aNV60/ffq09b3vfc86dOiQ1aNHD2vZsmX22Oeff25Jsvbu3Wuv++ijj6ywsDDrzJkzlmVZ1m9/+1urU6dOVn19vV0zZ84cq3fv3vbrJ554wkpJSfHb79ChQ62nn3662f17vV5LkuX1epu9DQAACK7m/v4O+D05jY2NmjhxombNmqXvf//7V42XlJQoJiZGQ4YMsdclJSUpPDxcpaWlds2IESMUGRlp1yQnJ6uyslLnzp2za5KSkvzmTk5OVklJybf2Vl9fL5/P57cAAAAzBTzkLF68WBEREXruueeuOe7xeNS1a1e/dREREYqNjZXH47Fr4uLi/GqaXt+opmn8WhYtWiSn02kvfG8VAADmCmjIKSsr0/Lly5WXl3dbfqv3vHnz5PV67eXUqVPBbgkAALSSgIacTz/9VNXV1erevbsiIiIUERGhEydOaObMmerZs6ckyeVyqbq62m+7S5cuqaamRi6Xy66pqqryq2l6faOapvFriYqKsr+niu+rAgDAbAENORMnTtSBAwdUXl5uL263W7NmzdLWrVslSYmJiaqtrVVZWZm93fbt29XY2KihQ4faNcXFxWpoaLBrCgsL1bt3b3Xq1MmuKSoq8tt/YWGhEhMTA3lIAAAgRLX4W8jPnz+vP//5z/brY8eOqby8XLGxserevbs6d+7sV9+2bVu5XC717t1bktS3b1+NGTNGU6ZMUW5urhoaGpSVlaW0tDT74+YTJkzQyy+/rIyMDM2ZM0eHDh3S8uXLtWzZMnveadOm6Yc//KGWLl2qlJQUvfvuu9q3b5/fx8wBAMCdq8UhZ9++fRo5cqT9esaMGZKkyZMnKy8vr1lzrF27VllZWRo1apTCw8M1btw4rVixwh53Op3atm2bMjMzNXjwYHXp0kXZ2dl+z9J54IEHtG7dOs2fP1+/+tWv1KtXL23atEn9+/dv6SG1ip5ztwS7hRY7/npKsFsAACBgwizLsoLdRLD4fD45nU55vd6A359DyAEAoHU09/c3310FAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGKnFIae4uFiPPvqo3G63wsLCtGnTJnusoaFBc+bM0YABA9ShQwe53W5NmjRJZ8+e9ZujpqZG6enpcjgciomJUUZGhs6fP+9Xc+DAAT300EOKjo5WfHy8cnJyruplw4YN6tOnj6KjozVgwAB9+OGHLT0cAABgqBaHnLq6Og0cOFArV668auybb77R/v37tWDBAu3fv1/vv/++Kisr9dhjj/nVpaenq6KiQoWFhcrPz1dxcbGmTp1qj/t8Po0ePVo9evRQWVmZlixZooULF2r16tV2za5duzR+/HhlZGTos88+U2pqqlJTU3Xo0KGWHhIAADBQmGVZ1nfeOCxMGzduVGpq6rfW7N27V/fff79OnDih7t276/Dhw+rXr5/27t2rIUOGSJIKCgo0duxYnT59Wm63W6tWrdKLL74oj8ejyMhISdLcuXO1adMmHTlyRJL05JNPqq6uTvn5+fa+hg0bpkGDBik3N7dZ/ft8PjmdTnm9Xjkcju/4Llxbz7lbAjrfrXD89ZRgtwAAwA019/d3q9+T4/V6FRYWppiYGElSSUmJYmJi7IAjSUlJSQoPD1dpaaldM2LECDvgSFJycrIqKyt17tw5uyYpKclvX8nJySopKfnWXurr6+Xz+fwWAABgplYNORcuXNCcOXM0fvx4O2l5PB517drVry4iIkKxsbHyeDx2TVxcnF9N0+sb1TSNX8uiRYvkdDrtJT4+/uYOEAAA3LZaLeQ0NDToiSeekGVZWrVqVWvtpkXmzZsnr9drL6dOnQp2SwAAoJVEtMakTQHnxIkT2r59u9/fy1wul6qrq/3qL126pJqaGrlcLrumqqrKr6bp9Y1qmsavJSoqSlFRUd/9wAAAQMgI+JWcpoBz9OhRffzxx+rcubPfeGJiompra1VWVmav2759uxobGzV06FC7pri4WA0NDXZNYWGhevfurU6dOtk1RUVFfnMXFhYqMTEx0IcEAABCUItDzvnz51VeXq7y8nJJ0rFjx1ReXq6TJ0+qoaFBjz/+uPbt26e1a9fq8uXL8ng88ng8unjxoiSpb9++GjNmjKZMmaI9e/Zo586dysrKUlpamtxutyRpwoQJioyMVEZGhioqKrR+/XotX75cM2bMsPuYNm2aCgoKtHTpUh05ckQLFy7Uvn37lJWVFYC3BQAAhLoWf4T8k08+0ciRI69aP3nyZC1cuFAJCQnX3O6Pf/yjHn74YUl/fRhgVlaWPvjgA4WHh2vcuHFasWKF7rrrLrv+wIEDyszM1N69e9WlSxc9++yzmjNnjt+cGzZs0Pz583X8+HH16tVLOTk5Gjt2bLOPhY+Q++Mj5ACAUNDc39839ZycUEfI8UfIAQCEgtvmOTkAAADBQMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGKnFIae4uFiPPvqo3G63wsLCtGnTJr9xy7KUnZ2tbt26qV27dkpKStLRo0f9ampqapSeni6Hw6GYmBhlZGTo/PnzfjUHDhzQQw89pOjoaMXHxysnJ+eqXjZs2KA+ffooOjpaAwYM0IcfftjSwwEAAIZqccipq6vTwIEDtXLlymuO5+TkaMWKFcrNzVVpaak6dOig5ORkXbhwwa5JT09XRUWFCgsLlZ+fr+LiYk2dOtUe9/l8Gj16tHr06KGysjItWbJECxcu1OrVq+2aXbt2afz48crIyNBnn32m1NRUpaam6tChQy09JAAAYKAwy7Ks77xxWJg2btyo1NRUSX+9iuN2uzVz5ky98MILkiSv16u4uDjl5eUpLS1Nhw8fVr9+/bR3714NGTJEklRQUKCxY8fq9OnTcrvdWrVqlV588UV5PB5FRkZKkubOnatNmzbpyJEjkqQnn3xSdXV1ys/Pt/sZNmyYBg0apNzc3Gb17/P55HQ65fV65XA4vuvbcE09524J6Hy3wvHXU4LdAgAAN9Tc398BvSfn2LFj8ng8SkpKstc5nU4NHTpUJSUlkqSSkhLFxMTYAUeSkpKSFB4ertLSUrtmxIgRdsCRpOTkZFVWVurcuXN2zZX7aapp2s+11NfXy+fz+S0AAMBMAQ05Ho9HkhQXF+e3Pi4uzh7zeDzq2rWr33hERIRiY2P9aq41x5X7+LaapvFrWbRokZxOp73Ex8e39BABAECIuKM+XTVv3jx5vV57OXXqVLBbAgAArSSgIcflckmSqqqq/NZXVVXZYy6XS9XV1X7jly5dUk1NjV/Ntea4ch/fVtM0fi1RUVFyOBx+CwAAMFNAQ05CQoJcLpeKiorsdT6fT6WlpUpMTJQkJSYmqra2VmVlZXbN9u3b1djYqKFDh9o1xcXFamhosGsKCwvVu3dvderUya65cj9NNU37AQAAd7YWh5zz58+rvLxc5eXlkv56s3F5eblOnjypsLAwTZ8+Xa+++qo2b96sgwcPatKkSXK73fYnsPr27asxY8ZoypQp2rNnj3bu3KmsrCylpaXJ7XZLkiZMmKDIyEhlZGSLine truncated
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"labeled = df[~pd.isna(df.india)]\n",
"plt.hist(labeled.india)"
]
},
{
"cell_type": "code",
"execution_count": 101,
"id": "2bda819b-4d4b-4e71-960a-53dd74d80b71",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42878/2289208715.py:1: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled['is_i1']=0\n"
]
},
{
"data": {
"text/plain": [
"(array([ 3532., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 14689.]),\n",
" array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),\n",
" <BarContainer object of 10 artists>)"
]
},
"execution_count": 101,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsVElEQVR4nO3df1RVdb7/8ReIgJnn4I8LeL6h0o9RKUdLJ6TSpokljeRc7thNkzHvDMk0QZNSKo6F9hOjrLQcuVYzutbo1ZyV3EIHZXCMSQkV5aqkVDdKHe+BWsg5Sokg+/vHLHadpBI7gOfT87HWXmvO/rz3Z7/3Z6zzWtt9dkGWZVkCAAAwTHB3NwAAANAZCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACOFdHcD3am1tVXHjx9Xnz59FBQU1N3tAACA82BZlk6ePCmXy6Xg4K+/X/O9DjnHjx9XTExMd7cBAAAuwNGjR3XZZZd97fj3OuT06dNH0j8XyeFwdHM3AADgfHi9XsXExNjf41/nex1y2v6KyuFwEHIAAAgw3/aoCQ8eAwAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABgppLsbAAAA325I9qbubqHDPlqc3K3n504OAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEbqcMgpLS3VpEmT5HK5FBQUpIKCgq+tvffeexUUFKQXXnjBZ399fb1SU1PlcDgUERGhtLQ0nTp1yqdm//79GjdunMLDwxUTE6O8vLxz5t+wYYOGDRum8PBwjRgxQps3b+7o5QAAAEN1OOQ0NjZq5MiRWr58+TfWbdy4Ue+8845cLtc5Y6mpqaqqqlJxcbEKCwtVWlqq9PR0e9zr9WrChAkaPHiwKioq9Mwzz2jRokVauXKlXbNz507dddddSktL0759+5SSkqKUlBQdPHiwo5cEAAAMFGRZlnXBBwcFaePGjUpJSfHZ/49//EPx8fHasmWLkpOTNWvWLM2aNUuSdOjQIcXFxWn37t0aM2aMJKmoqEgTJ07UsWPH5HK5tGLFCi1YsEBut1uhoaGSpOzsbBUUFOjw4cOSpClTpqixsVGFhYX2eceOHatRo0YpPz//vPr3er1yOp3yeDxyOBwXugwAAHS6IdmburuFDvtocXKnzHu+399+fyantbVV06dP15w5c3T11VefM15WVqaIiAg74EhSYmKigoODVV5ebteMHz/eDjiSlJSUpOrqap04ccKuSUxM9Jk7KSlJZWVlX9tbU1OTvF6vzwYAAMzk95Dz9NNPKyQkRL/97W/bHXe73YqMjPTZFxISon79+sntdts1UVFRPjVtn7+tpm28Pbm5uXI6nfYWExPTsYsDAAABw68hp6KiQkuXLtWqVasUFBTkz6n9Yv78+fJ4PPZ29OjR7m4JAAB0Er+GnL///e+qq6vToEGDFBISopCQEH388cd68MEHNWTIEElSdHS06urqfI5raWlRfX29oqOj7Zra2lqfmrbP31bTNt6esLAwORwOnw0AAJjJryFn+vTp2r9/vyorK+3N5XJpzpw52rJliyQpISFBDQ0NqqiosI/btm2bWltbFR8fb9eUlpaqubnZrikuLtbQoUPVt29fu6akpMTn/MXFxUpISPDnJQEAgAAV0tEDTp06pQ8++MD+XFNTo8rKSvXr10+DBg1S//79fep79uyp6OhoDR06VJI0fPhw3XbbbZo5c6by8/PV3NyszMxMTZ061f65+bRp0/Too48qLS1N8+bN08GDB7V06VI9//zz9rwPPPCAbr75Zi1ZskTJyclat26d9uzZ4/MzcwAA8P3V4Ts5e/bs0bXXXqtrr71WkpSVlaVrr71WOTk55z3HmjVrNGzYMN16662aOHGibrrpJp9w4nQ6tXXrVtXU1Gj06NF68MEHlZOT4/MunRtuuEFr167VypUrNXLkSP35z39WQUGBrrnmmo5eEgAAMNB3ek9OoOM9OQCAQMF7cr7Qbe/JAQAAuBgQcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwUodDTmlpqSZNmiSXy6WgoCAVFBTYY83NzZo3b55GjBih3r17y+Vy6e6779bx48d95qivr1dqaqocDociIiKUlpamU6dO+dTs379f48aNU3h4uGJiYpSXl3dOLxs2bNCwYcMUHh6uESNGaPPmzR29HAAAYKgOh5zGxkaNHDlSy5cvP2fss88+0969e/XII49o7969ev3111VdXa2f/exnPnWpqamqqqpScXGxCgsLVVpaqvT0dHvc6/VqwoQJGjx4sCoqKvTMM89o0aJFWrlypV2zc+dO3XXXXUpLS9O+ffuUkpKilJQUHTx4sKOXBAAADBRkWZZ1wQcHBWnjxo1KSUn52prdu3fr+uuv18cff6xBgwbp0KFDiouL0+7duzVmzBhJUlFRkSZOnKhjx47J5XJpxYoVWrBggdxut0JDQyVJ2dnZKigo0OHDhyVJU6ZMUWNjowoLC+1zjR07VqNGjVJ+fv559e/1euV0OuXxeORwOC5wFQAA6HxDsjd1dwsd9tHi5E6Z93y/vzv9mRyPx6OgoCBFRERIksrKyhQREWEHHElKTExUcHCwysvL7Zrx48fbAUeSkpKSVF1drRMnTtg1iYmJPudKSkpSWVnZ1/bS1NQkr9frswEAADN1asg5ffq05s2bp7vuustOWm63W5GRkT51ISEh6tevn9xut10TFRXlU9P2+dtq2sbbk5ubK6fTaW8xMTHf7QIBAMBFq9NCTnNzs+68805ZlqUVK1Z01mk6ZP78+fJ4PPZ29OjR7m4JAAB0kpDOmLQt4Hz88cfatm2bz9+XRUdHq66uzqe+paVF9fX1io6Otmtqa2t9ato+f1tN23h7wsLCFBYWduEXBgAAAobf7+S0BZz3339ff/3rX9W/f3+f8YSEBDU0NKiiosLet23bNrW2tio+Pt6uKS0tVXNzs11TXFysoUOHqm/fvnZNSUmJz9zFxcVKSEjw9yUBAIAA1OGQc+rUKVVWVqqyslKSVFNTo8rKSh05ckTNzc264447tGfPHq1Zs0Znz56V2+2W2+3WmTNnJEnDhw/XbbfdppkzZ2rXrl3asWOHMjMzNXXqVLlcLknStGnTFBoaqrS0NFVVVWn9+vVaunSpsrKy7D4eeOABFRUVacmSJTp8+LAWLVqkPXv2KDMz0w/LAgAAAl2Hf0K+fft23XLLLefsnzFjhhYtWqTY2Nh2j/vb3/6mH//4x5L++TLAzMxMvfnmmwoODtbkyZO1bNkyXXrppXb9/v37lZGRod27d2vAgAG6//77NW/ePJ85N2zYoIcfflgfffSRrrrqKuXl5WnixInnfS38hBwAECj4CfkXzvf7+zu9JyfQEXIAAIGCkPOFi+Y9OQAAAN2BkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAI3U45JSWlmrSpElyuVwKCgpSQUGBz7hlWcrJydHAgQPVq1cvJSYm6v333/epqa+vV2pqqhwOhyIiIpSWlqZTp0751Ozfv1/jxo1TeHi4YmJilJeXd04vGzZs0LBhwxQeHq4RI0Zo8+bNHb0cAABgqA6HnMbGRo0cOVLLly9vdzwvL0/Lli1Tfn6+ysvL1bt3byUlJen06dN2TWpqqqqqqlRcXKzCwkKVlpYqPT3dHvd6vZowYYIGDx6siooKPfPMM1q0aJFWrlxp1+zcuVN33XWX0tLStG/fPqWkpCglJUUHDx7s6CUBAAADBVmWZV3wwUFB2rhxo1JSUiT98y6Oy+XSgw8+qIceekiS5PF4FBUVpVWrVmnq1Kk6dOiQ4uLitHv3bo0ZM0aSVFRUpIkTJ+rYsWNyuVxasWKFFixYILfbrdDQUElSdna2CgoKdPjwYUnSlClT1NjYqMLCQrufsWPHatSoUcrPzz+v/r1er5xOpzwejxwOx4UuAwAAnW5I9qbubqHDPlqc3Cnznu/3t1+fyampqZHb7VZiYqK9z+l0Kj4+XmVlZZKksrIyRURE2AFHkhITExUcHKzy8nK7Zvz48XbAkaSkpCRVV1frxIkTds2Xz9NW03YeAADw/Rbiz8ncbrckKSoqymd/VFSUPeZ2uxUZGenbREiI+vXr51MTGxt7zhxtY3379pXb7f7G87SnqalJTU1N9mev19uRywMAAAHke/XrqtzcXDmdTnuLiYnp7pYAAEAn8WvIiY6OliTV1tb67K+trbXHoqOjVVdX5zPe0tKi+vp6n5r25vjyOb6upm28PfPnz5fH47G3o0ePdvQSAQBAgPBryImNjVV0dLRKSkrsfV6vV+Xl5UpISJAkJSQkqKGhQRUVFXbNtm3b1Nraqvj4eLumtLRLine truncated
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"labeled['is_i1']=0\n",
"labeled.loc[labeled.india.isin(['i1','i0']),'is_i1']=1\n",
"plt.hist(labeled.is_i1)"
]
},
{
"cell_type": "code",
"execution_count": 103,
"id": "e8c139d9-bf61-45ec-9da1-7eaf4ff754b4",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42878/1116779111.py:1: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].astype(np.float32).fillna(np.nan)\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n",
"/tmp/ipykernel_42878/1116779111.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n"
]
}
],
"source": [
"labeled[c] = labeled[c].astype(np.float32).fillna(np.nan)\n",
"for c in labeled.columns:\n",
" if c!='age':\n",
" labeled[c] = labeled[c].fillna('None').astype('category')\n"
]
},
{
"cell_type": "code",
"execution_count": 104,
"id": "fabf354e-f39e-4cde-af84-c65a277d309a",
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split( labeled.drop(columns=['india','is_i1']),\n",
" labeled.is_i1, test_size=0.33, random_state=0,stratify=labeled.is_i1)"
]
},
{
"cell_type": "code",
"execution_count": 105,
"id": "cf6cf5d8-d43e-499e-98a5-65ecd0b8ccda",
"metadata": {},
"outputs": [],
"source": [
"X_train, X_valid, y_train, y_valid = train_test_split(X_train,y_train, test_size=0.33, random_state=0,stratify=y_train)"
]
},
{
"cell_type": "code",
"execution_count": 147,
"id": "774ad570-a60b-475b-80cf-4f9b9949cc9d",
"metadata": {},
"outputs": [],
"source": [
"weight_train = (1-y_train.values.astype(int))*5+1 ## peso classi unbalanced"
]
},
{
"cell_type": "code",
"execution_count": 148,
"id": "4bed71a8-6420-4432-8362-3fe24eb394fb",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([6, 6, 1, ..., 1, 1, 1])"
]
},
"execution_count": 148,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"weight_train"
]
},
{
"cell_type": "code",
"execution_count": 149,
"id": "1fcc5234-abad-459a-9420-810833657796",
"metadata": {},
"outputs": [],
"source": [
"from catboost import CatBoostClassifier, Pool\n",
"\n",
"train_data = Pool(data=X_train,\n",
" label=y_train,\n",
" weight=weight_train,cat_features=[c for c in X_train.columns if c!='age'])\n",
"valid_data = Pool(data=X_valid,\n",
" label=y_valid,cat_features=[c for c in X_train.columns if c!='age']\n",
" )\n",
"model = CatBoostClassifier(iterations=1000)\n",
"\n",
"model.fit(train_data,eval_set=valid_data,verbose=False,early_stopping_rounds=100)\n",
"preds_class = model.predict(valid_data,)"
]
},
{
"cell_type": "code",
"execution_count": 150,
"id": "8df84007-110b-4f31-bc7d-53e7d5c9a178",
"metadata": {},
"outputs": [],
"source": [
"preds_class_valid = model.predict(valid_data)\n",
"preds_class_train= model.predict(train_data)"
]
},
{
"cell_type": "code",
"execution_count": 151,
"id": "55627ec4-fd24-4815-98d4-d8462bbfdd9a",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import confusion_matrix,matthews_corrcoef,accuracy_score"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "877e914d-44d4-4299-8d3c-24c4fc353317",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 152,
"id": "776395d9-a8e2-4fda-90e1-11d6dbe80de8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[ 622 159]\n",
" [ 778 2470]]\n",
"0.4623150659581546\n",
"0.7674360883593944\n",
"########################################\n",
"[[1436 149]\n",
" [1358 5236]]\n",
"0.5834604503367398\n",
"0.8157476464115417\n"
]
}
],
"source": [
"print(confusion_matrix(y_valid,preds_class_valid))\n",
"print(matthews_corrcoef(y_valid,preds_class_valid))\n",
"print(accuracy_score(y_valid,preds_class_valid))\n",
"print('########################################')\n",
"print(confusion_matrix(y_train,preds_class_train))\n",
"print(matthews_corrcoef(y_train,preds_class_train))\n",
"print(accuracy_score(y_train,preds_class_train))"
]
},
{
"cell_type": "code",
"execution_count": 157,
"id": "34c397bc-529a-4c52-b30e-957b28021200",
"metadata": {},
"outputs": [
{
"ename": "TypeError",
"evalue": "DMatrix.__init__() got an unexpected keyword argument 'scale_pos_weight'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[157], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mxgboost\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mxgb\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# Create regression matrices\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m dtrain \u001b[38;5;241m=\u001b[39m \u001b[43mxgb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDMatrix\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mastype\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menable_categorical\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43mscale_pos_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;66;03m#,weight=weight_train)\u001b[39;00m\n\u001b[1;32m 6\u001b[0m dvalid \u001b[38;5;241m=\u001b[39m xgb\u001b[38;5;241m.\u001b[39mDMatrix(X_valid, y_valid\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mint\u001b[39m), enable_categorical\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
"File \u001b[0;32m~/miniconda3/envs/pid/lib/python3.11/site-packages/xgboost/core.py:730\u001b[0m, in \u001b[0;36mrequire_keyword_args.<locals>.throw_if.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 728\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m k, arg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(sig\u001b[38;5;241m.\u001b[39mparameters, args):\n\u001b[1;32m 729\u001b[0m kwargs[k] \u001b[38;5;241m=\u001b[39m arg\n\u001b[0;32m--> 730\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mTypeError\u001b[0m: DMatrix.__init__() got an unexpected keyword argument 'scale_pos_weight'"
]
}
],
"source": [
"##try with xgboost\n",
"import xgboost as xgb\n",
"\n",
"# Create regression matrices\n",
"dtrain = xgb.DMatrix(X_train, y_train.astype(int), enable_categorical=True)#,weight=weight_train)\n",
"dvalid = xgb.DMatrix(X_valid, y_valid.astype(int), enable_categorical=True)"
]
},
{
"cell_type": "code",
"execution_count": 154,
"id": "7bb54c1d-ed2c-496a-b149-d567246bfee0",
"metadata": {},
"outputs": [],
"source": [
"params = {\"objective\": \"multi:softprob\", \"num_class\": 2}\n",
"n = 1000\n",
"\n",
"results = xgb.train(\n",
" params, dtrain,\n",
" num_boost_round=n,\n",
" evals = [(dtrain, \"train\"), (dvalid, \"valid\")],\n",
" verbose_eval=False,\n",
" early_stopping_rounds=100\n",
" # metrics=[\"mlogloss\", \"auc\", \"merror\"],\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": 155,
"id": "945e209e-1722-434e-b0bf-5104b3234eb9",
"metadata": {},
"outputs": [],
"source": [
"preds_class_valid = results.predict(dvalid)\n",
"preds_class_train= results.predict(dtrain)"
]
},
{
"cell_type": "code",
"execution_count": 156,
"id": "65228d59-e4c2-411d-8535-b36c15a55bfd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[ 492 289]\n",
" [ 471 2777]]\n",
"0.4495009568013873\n",
"0.811367585008687\n",
"########################################\n",
"[[1584 1]\n",
" [ 389 6205]]\n",
"0.8688028589259579\n",
"0.9523169091575987\n"
]
}
],
"source": [
"print(confusion_matrix(y_valid,preds_class_valid.argmax(1)))\n",
"print(matthews_corrcoef(y_valid,preds_class_valid.argmax(1)))\n",
"print(accuracy_score(y_valid,preds_class_valid.argmax(1)))\n",
"print('########################################')\n",
"print(confusion_matrix(y_train,preds_class_train.argmax(1)))\n",
"print(matthews_corrcoef(y_train,preds_class_train.argmax(1)))\n",
"print(accuracy_score(y_train,preds_class_train.argmax(1)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "95461970-fad9-4c50-84d3-e139bedfec3f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
notebooks/old_notebooks/.ipynb_checkpoints/test_clean-checkpoint.ipynb
+1180
View File
File diff suppressed because it is too large. Load diff
notebooks/old_notebooks/.ipynb_checkpoints/test_multi-checkpoint.ipynb
+1279
View File
File diff suppressed because it is too large. Load diff
notebooks/old_notebooks/.ipynb_checkpoints/test_multi_CV-checkpoint.ipynb
+3650
View File
File diff suppressed because it is too large. Load diff
notebooks/old_notebooks/test_binary.ipynb
+1190
View File
File diff suppressed because it is too large. Load diff
notebooks/old_notebooks/test_clean.ipynb
+1180
View File
File diff suppressed because it is too large. Load diff
notebooks/old_notebooks/test_multi.ipynb
+1554
View File
File diff suppressed because it is too large. Load diff
notebooks/old_notebooks/test_multi_CV.ipynb
+3650
View File
File diff suppressed because it is too large. Load diff