{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "import math\n",
    "import re\n",
    "\n",
    "from pandas import read_excel\n",
    "from pymongo import MongoClient\n",
    "from pyquery import PyQuery as pq\n",
    "from scipy import stats\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SHEETS = (\n",
    "    (\"神外-總院_1070920.xlsx\", \"總院\"), \n",
    "#     (\"(骨穩)總院_分科1070928.xlsx\", \"工作表1\")\n",
    "    (\"(骨穩)總院_分科1071002.xlsx\", \"工作表1\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MongoClient(\"mongodb.xiao.tw\", 27017)\n",
    "db = client.forteo\n",
    "posts = db.posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.font_manager import FontManager\n",
    "\n",
    "import subprocess\n",
    "\n",
    "def get_matplot_zh_font():\n",
    "    fm = FontManager()\n",
    "    mat_fonts = set(f.name for f in fm.ttflist)\n",
    "\n",
    "    output = subprocess.check_output('fc-list :lang=zh -f \"%{family}\\n\"', shell=True)\n",
    "    output=str(output)\n",
    "    zh_fonts = set(f.split(',', 1)[0] for f in output.split('\\n'))\n",
    "\n",
    "    print(mat_fonts)\n",
    "    print(zh_fonts)\n",
    "    available = list(mat_fonts & zh_fonts)\n",
    "\n",
    "    print ('*' * 10, '可用的字體', '*' * 10)\n",
    "    for f in available:\n",
    "        print(f)\n",
    "    return available\n",
    "\n",
    "def set_matplot_zh_font():\n",
    "    available = get_matplot_zh_font()\n",
    "    if len(available) > 0:\n",
    "        mpl.rcParams['font.sans-serif'] = [available[0]]    # 指定默認字體\n",
    "        mpl.rcParams['axes.unicode_minus'] = False          # 解決保存圖像是負號'-'顯示為方塊的問題\n",
    "        \n",
    "# set_matplot_zh_font()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file_name, sheet_name in SHEETS:\n",
    "    data = read_excel(file_name, sheet_name)\n",
    "    frames.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.concat(frames, ignore_index=True, sort=False)\n",
    "df.to_excel('concat.xls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.680568609022558\n",
      "6.696428571428571 0.14285714285714285 24.0\n"
     ]
    }
   ],
   "source": [
    "print(df['用藥時間'].mean())\n",
    "print(df['用藥時間'].median(), df['用藥時間'].min(), df['用藥時間'].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "antiresorptives = {\n",
    "    'Prolia': 'RANKL inhibitor',\n",
    "    'Evista': 'SERM',\n",
    "    'Fosamax': 'Bisphosphonate',\n",
    "    'Aclasta': 'Bisphosphonate',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "countDrug = Counter()\n",
    "\n",
    "column_ar = []\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    tmp_ar = None\n",
    "    post = posts.find_one({\"_id\": row['病歷號']})\n",
    "    if post is None:\n",
    "        continue\n",
    "        print(index, row['病歷號'], post)\n",
    "\n",
    "#     print(index, row['病歷號'])\n",
    "    drug_set = set()\n",
    "    if post['drug']:\n",
    "        \n",
    "        for drugs in post['drug']:\n",
    "#             print(drugs)\n",
    "            for tr in drugs['Drug']:\n",
    "                pqtr = pq(tr)\n",
    "                drugname = pqtr('td')[0].text\n",
    "                drugname = drugname.replace('(管4) ', '')\n",
    "                drug_set.add(drugname)\n",
    "#                 countDrug[pqtr('td')[0].text] += 1\n",
    "#                 print(pqtr('td')[0].text)\n",
    "#                 break\n",
    "#         print(drug_set)\n",
    "                if tmp_ar is None:\n",
    "                    for ar in antiresorptives:\n",
    "                        if drugname.startswith(ar):\n",
    "                            tmp_ar = ar\n",
    "\n",
    "        for d in drug_set:\n",
    "            countDrug[d] += 1\n",
    "    \n",
    "    column_ar.append(tmp_ar)\n",
    "#         break\n",
    "# print(countDrug)\n",
    "#     row['AR'] = column_ar\n",
    "    df.loc[index, 'AR'] = tmp_ar\n",
    "    if tmp_ar:\n",
    "        df.loc[index, 'AR2'] = antiresorptives[tmp_ar]\n",
    "        df.loc[index, 'antiresorptives'] = tmp_ar if tmp_ar == 'Prolia' else ' Others'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 27884 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cDrug = countDrug\n",
    "del cDrug['Forteo Prefilled Inj 600 mcg/2.4 mL /pen']\n",
    "del cDrug['(4mm) Micro-Fine Insulin Pen Needle 32G /set']\n",
    "del cDrug['(8mm) Micro-Fine Insulin Pen Needle 31G /set']\n",
    "labels, values = zip(*countDrug.most_common(10))\n",
    "# plt.rcParams['font.sans-serif']=['Arial Unicode MS'] #用来正常显示中文标签\n",
    "plt.pie(values, labels=labels)\n",
    "plt.savefig('drug.png', dpi=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "antiresorptives = {\n",
    "    'Prolia': 'RANKL inhibitor',\n",
    "    'Evista': 'SERM',\n",
    "    'Fosamax': 'Bisphosphonate',\n",
    "    'Aclasta': 'Bisphosphonate',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'Prolia': 30, 'Evista': 10, 'Fosamax': 6, 'Aclasta': 5})\n",
      "Counter({'RANKL inhibitor': 30, 'Bisphosphonate': 11, 'SERM': 10})\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 13220 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cAR = Counter()\n",
    "ccAR = Counter()\n",
    "\n",
    "for c in countDrug:\n",
    "    for ar in antiresorptives:\n",
    "        if c.startswith(ar):\n",
    "            cAR[ar]+=countDrug[c]\n",
    "            ccAR[antiresorptives[ar]]+=countDrug[c]\n",
    "            \n",
    "print(cAR)\n",
    "print(ccAR)\n",
    "labels, values = zip(*cAR.most_common())\n",
    "plt.pie(values, labels=labels, autopct='%.2f')\n",
    "plt.savefig('cAR.png', dpi=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATTERNS = (\n",
    "    'T-\\s*score\\s*=\\s*([-+]?\\d*\\.\\d+)',\n",
    "    'T-\\s*score\\s*=\\s*(- \\d*\\.\\d+)',\n",
    "    'T-score is\\s*([-+]?\\d*\\.\\d+)',\n",
    "    'T score =\\s*([-+]?\\d*\\.\\d+)',\n",
    ")\n",
    "def T_score(html):\n",
    "    min = None\n",
    "    for pattern in PATTERNS:\n",
    "        for m in re.findall(pattern, html):\n",
    "#             print(m)\n",
    "            t = float(m.replace(' ', ''))\n",
    "#             print (t)\n",
    "            if min and t > min:\n",
    "                continue\n",
    "            min = t            \n",
    "    return min \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 11676 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "PRE = []\n",
    "POST =[]\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    post = posts.find_one({\"_id\": row['病歷號']})\n",
    "    if post is None:\n",
    "        continue\n",
    "        print(index, row['病歷號'], post)\n",
    "\n",
    "#     print(index, row['病歷號'])\n",
    "#     print(post)\n",
    "    if post['report']:\n",
    "        BMD = {}\n",
    "        for r in post['report']:\n",
    "#             print(r)\n",
    "\n",
    "            r_class = r['報告類別'].upper()\n",
    "            r_date  = r['檢查日期']\n",
    "            if 'BMD' in r_class:\n",
    "#                 print(r_date, r_class)\n",
    "#                 print(r)\n",
    "#                 print(row['病歷號'], r_date, r_class)\n",
    "                t_score = T_score(r['html'])\n",
    "                if t_score is None:\n",
    "#                     # No T score in report\n",
    "#                     print(row['病歷號'], r_date, t_score, r_class)\n",
    "                    continue\n",
    "    \n",
    "                if r_date not in BMD or t_score < BMD[r_date]:\n",
    "                    BMD[r_date] = t_score\n",
    "                \n",
    "#         print(row['簽署日'], row['流失/停藥日期'])\n",
    "#         print(type(row['簽署日']), type(row['流失/停藥日期']))\n",
    "        \n",
    "        if type(row['簽署日']) is not str and math.isnan(row['簽署日']):\n",
    "            dStart = row['流失/停藥日期']\n",
    "        else:\n",
    "            dStart = row['簽署日']\n",
    "        if BMD:        \n",
    "#             print(row['病歷號'], dStart, BMD)\n",
    "            \n",
    "            pre = None\n",
    "            post = None\n",
    "            \n",
    "            for d, b in BMD.items():\n",
    "                if d > dStart:\n",
    "                    post = b\n",
    "                else:\n",
    "                    pre = b\n",
    "                    \n",
    "            if pre and post:\n",
    "                PRE.append(pre)\n",
    "                POST.append(post)\n",
    "#                 print(row['病歷號'], dStart, pre, post)\n",
    "                df.loc[index, 'BMD Change'] = post-pre\n",
    "\n",
    "df.to_excel('concat3.xls')\n",
    "    \n",
    "D=df.boxplot(column='BMD Change',by='antiresorptives')    \n",
    "D.get_figure().savefig('cAR3.png', dpi=400)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {\n",
    "    'bmd_before': PRE, \n",
    "    'bmd_after': POST,\n",
    "}\n",
    "\n",
    "df2 = pd.DataFrame(data=d)\n",
    "df2.describe()        \n",
    "df2.plot(kind='box')        \n",
    "# plt.savefig('boxplot_outliers.png')\n",
    "                \n",
    "# print(stats.ttest_rel(df2['bmd_before'], df2['bmd_after']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
