B2 - nSBM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cc8de91",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import graph_tool.all as gt\n",
    "\n",
    "import json\n",
    "import random\n",
    "import nmi\n",
    "import glob\n",
    "import string\n",
    "import statsmodels\n",
    "\n",
    "from pathlib import Path\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "from time import localtime, strftime\n",
    "from matplotlib.patches import Patch\n",
    "from sbmtm import sbmtm\n",
    "from nsbm import nsbm\n",
    "\n",
    "from helps import *\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "plt.style.use('seaborn')\n",
    "plt.rcParams['font.size'] = 15\n",
    "plt.rcParams[\"xtick.labelsize\"]=13\n",
    "plt.rcParams[\"ytick.labelsize\"]=13\n",
    "plt.rcParams[\"axes.titlesize\"]=15\n",
    "plt.rcParams[\"figure.dpi\"]=600\n",
    "plt.rcParams[\"savefig.format\"]=\"pdf\"\n",
    "plt.rcParams[\"savefig.bbox\"]=\"tight\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76691858",
   "metadata": {},
   "source": [
    "# triSBM: experiment run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21654ca8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mRNA=pd.read_csv(\"Results/hSBM-mRNA/hSBM-mRNA.csv.gz\", index_col=0)\n",
    "lncRNA=pd.read_csv(\"Results/hSBM-lncRNA/hSBM-lncRNA.csv.gz\", index_col=0)\n",
    "lncRNA=lncRNA[mRNA.columns]\n",
    "mRNA.shape, lncRNA.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e25bb511",
   "metadata": {},
   "outputs": [],
   "source": [
    "#It takes 2 minutes to build the graph on a i5-8265U 4 cores 1.60 GHz laptop\n",
    "model = nsbm()\n",
    "print(strftime(\"%Y-%m-%d %H:%M:%S\", localtime()))\n",
    "model.make_graph_multiple_df(mRNA, [lncRNA])\n",
    "print(strftime(\"%Y-%m-%d %H:%M:%S\", localtime()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed0664e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(strftime(\"%Y-%m-%d %H:%M:%S\", localtime()))\n",
    "model.fit(n_init=1,verbose=False)\n",
    "print(strftime(\"%Y-%m-%d %H:%M:%S\", localtime()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c877ef9d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path_to_save=\"Results/triSBM-mRNA-lncRNA/nSBM/triSBM-mRNA-lncRNA\"\n",
    "Path(path_to_save).mkdir(parents=True, exist_ok=True)\n",
    "save_levels_nSBM(model,path_to_save)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "776b2fef",
   "metadata": {},
   "source": [
    "# triSBM: analysis\n",
    "Here you can find all the steps that we followed to analyse the outcome of the trisbm-mRNA-lncRNA experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "377269b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mRNA=pd.read_csv(\"Results/hSBM-mRNA/hSBM-mRNA.csv.gz\", index_col=0)\n",
    "df_lncRNA=pd.read_csv(\"Results/hSBM-lncRNA/hSBM-lncRNA.csv.gz\", index_col=0)\n",
    "df_lncRNA=df_lncRNA[df_mRNA.columns]\n",
    "df_mRNA.shape, df_lncRNA.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b54dc13",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=pd.read_csv(\"HelperFiles/All-datasets-labels.csv\",index_col=0)\n",
    "labels=labels.loc[df_lncRNA.columns]\n",
    "subtypes=list(sorted(set(labels.typehisto)))\n",
    "info=pd.read_csv(\"HelperFiles/ENS-Info.txt\",sep=\"\\t\", index_col=0)\n",
    "info = info[~info.index.duplicated(keep='first')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51cc07ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "performances={}\n",
    "for level in range(0,3):\n",
    "    df_clu=pd.read_csv(f\"Results/triSBM-mRNA-lncRNA/triSBM/triSBM-mRNA-lncRNA-level-{level}-clusters.csv\",\n",
    "                       index_col=0) \n",
    "    labels=labels.loc[df_clu.columns]\n",
    "    labels[\"nSBM\"]=[str(np.array(df_clu[col]).argmax()) for col in df_clu.columns]\n",
    "    \n",
    "    NMI=np.around(nmi.compute_normalised_mutual_information(labels.typehisto,labels[\"nSBM\"]),decimals=3)\n",
    "    nmi_rand=0\n",
    "    for k in range(1000):\n",
    "        a=labels[\"nSBM\"].to_list()\n",
    "        np.random.shuffle(a)\n",
    "        nmi_rand+=nmi.compute_normalised_mutual_information(labels[\"typehisto\"],a)/1000\n",
    "\n",
    "    performances[f\"Level {level}\"]=[NMI,NMI/nmi_rand]\n",
    "with open(f\"Results/triSBM-mRNA-lncRNA/triSBM/triSBM-mRNA-lncRNA_NMI.json\", 'w') as fp:\n",
    "    json.dump(performances, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ee1dd45",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"Results/triSBM-mRNA-lncRNA/triSBM/triSBM-mRNA-lncRNA_NMI.json\") as f:\n",
    "        performances=json.load(f)\n",
    "for key in performances.keys():\n",
    "    print(f\"{key} NMI: {performances[key][0]}, NMI/NMI*: {int(performances[key][1])}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50e05ad7",
   "metadata": {},
   "source": [
    "## NMI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "313d2b63",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(1,3,figsize=(28,8))\n",
    "axs=axs.flatten()\n",
    "ax=0\n",
    "performances={}\n",
    "for level in range(0,3):\n",
    "    df_clu=pd.read_csv(f\"Results/triSBM-mRNA-lncRNA/nSBM/triSBM-mRNA-lncRNA-level-{level}-clusters.csv\",\n",
    "                       index_col=0) \n",
    "    labels=labels.loc[df_clu.columns]\n",
    "    labels[\"nSBM\"]=[str(np.array(df_clu[col]).argmax()) for col in df_clu.columns]\n",
    "    \n",
    "    labels[\"typehisto_1\"]=pd.Series(list(labels[\"typehisto\"])).astype('category').cat.codes.values    \n",
    "    fraction_sites = pd.DataFrame(index=labels[\"nSBM\"].unique(), columns=sorted(labels[\"typehisto_1\"].unique())[::-1]).fillna(0)\n",
    "    for sample in labels[[\"nSBM\",\"typehisto_1\"]].values:\n",
    "        fraction_sites.at[sample[0],sample[1]] += 1\n",
    "\n",
    "    fraction_sites = fraction_sites.sort_values(by=list(fraction_sites.columns), ascending=True)\n",
    "    fraction_sites.columns=subtypes[::-1]\n",
    "    fraction_sites.plot.bar(stacked=True, color=dict(zip(subtypes, nmi.set_colors(subtypes))),\n",
    "                           width=1, alpha=0.75, ax=axs[level])    \n",
    "    \n",
    "    NMI=np.around(nmi.compute_normalised_mutual_information(labels.typehisto,labels[\"nSBM\"]),decimals=3)\n",
    "    nmi_rand=0\n",
    "    for k in range(1000):\n",
    "        a=labels[\"nSBM\"].to_list()\n",
    "        np.random.shuffle(a)\n",
    "        nmi_rand+=nmi.compute_normalised_mutual_information(labels[\"typehisto\"],a)/1000\n",
    "    performances\n",
    "    performances[f\"Level {level}\"]=[NMI,NMI/nmi_rand]\n",
    "\n",
    "    axs[level].set_xlabel(\"cluster\", size=25, weight='bold')\n",
    "    axs[level].set_ylabel(\"number of cells\", size=25, weight='bold')\n",
    "    axs[level].yaxis.set_major_formatter(FormatStrFormatter('%.0f'))\n",
    "    axs[level].tick_params(axis='both', which='major', labelsize=25, rotation=0)\n",
    "    \n",
    "    legend_properties = {'weight':'bold', \"size\":\"x-large\"}\n",
    "    if level==1:\n",
    "        axs[level].legend(loc=(-0.025,0.7), prop=legend_properties)\n",
    "    else:\n",
    "        axs[level].get_legend().remove()        \n",
    "    \n",
    "    axs[level].text(-0.055, 1.1, string.ascii_uppercase[level],\n",
    "                 transform=axs[ax].transAxes, size=35, weight='bold',rotation=0)\n",
    "    axs[level].xaxis.set_major_locator(plt.MaxNLocator(min(10, len(set(labels.nSBM))+1)))\n",
    "    ax+=1   \n",
    "    \n",
    "title=f\"triSBM-mRNA-lncRNA cluster visualisation\"\n",
    "plt.savefig(f\"Results/Figures/{title}.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b01e1b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in performances.keys():\n",
    "    print(f\"{key} NMI: {performances[key][0]}, NMI/NMI*: {int(performances[key][1])}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09c13e63",
   "metadata": {},
   "source": [
    "## Data for analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "644abc7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "level=1\n",
    "dataset=\"triSBM-mRNA-lncRNA\"\n",
    "df_clu=pd.read_csv(f\"Results/{dataset}/nSBM/{dataset}-level-{level}-clusters.csv\",\n",
    "                       index_col=0)\n",
    "labels=labels.loc[df_clu.columns]\n",
    "labels[\"nSBM\"]=[str(np.array(df_clu[col]).argmax()) for col in df_clu.columns]\n",
    "    \n",
    "\n",
    "Path(f\"Results/{dataset}/nSBM/Data\").mkdir(parents=True, exist_ok=True)\n",
    "labels.to_csv(f\"Results/{dataset}/nSBM/{dataset}-level-{level}-sample-cluster.csv\")\n",
    "clusters=sorted(list(set(labels.nSBM)))\n",
    "subtypes=list(sorted(set(labels.typehisto)))\n",
    "\n",
    "clusters=sorted(list(set(labels.nSBM)))\n",
    "       \n",
    "path_to_save=f\"Results/{dataset}/nSBM/Outcome analysis/Level {level}\"\n",
    "Path(path_to_save).mkdir(parents=True, exist_ok=True)\n",
    "clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d771b3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "files=sorted(glob.glob(f\"Results/{dataset}/nSBM/{dataset}-level-{level}-*\"))\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4ec4bbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "mRNAtopic_gene, mRNAtopic_gene_prob, mRNAtopic_gene_genename, mRNAtopic_gene_raw = topic_gene_nSBM(files[1],\"mRNA\", info, 1, \"triSBM-mRNA-lncRNA\")\n",
    "mRNAtopic_gene_genename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f087083",
   "metadata": {},
   "outputs": [],
   "source": [
    "lncRNAtopic_gene, lncRNAtopic_gene_prob, lncRNAtopic_gene_genename, lncRNAtopic_gene_raw = topic_gene_nSBM(files[3],\"lncRNA\", info, 1, \"triSBM-mRNA-lncRNA\")\n",
    "lncRNAtopic_gene_genename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46f84dc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_c_mRNAtopic_class, p_c_mRNAtopic_cell, p_mRNAtopic_cell=p_c_t_c_nSBM(files[2], \"mRNA\", labels, 1, \"triSBM-mRNA-lncRNA\")\n",
    "p_c_mRNAtopic_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d199ce2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_c_lncRNAtopic_class, p_c_lncRNAtopic_cell, p_lncRNAtopic_cell=p_c_t_c_nSBM(files[4], \"lncRNA\", labels, 1, \"triSBM-mRNA-lncRNA\")\n",
    "p_c_lncRNAtopic_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd78745",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_arr_mRNA, threshold=loop_topics(clusters, p_c_mRNAtopic_class, direction=\"up\")\n",
    "with open(f\"{path_to_save}/{dataset} level {level} mRNA-topic up clusters threshold {np.around(threshold, decimals=2)}.json\", 'w') as fp:\n",
    "    json.dump(topic_arr_mRNA, fp)\n",
    "topic_arr_mRNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdc65e2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_arr_lncRNA, threshold=loop_topics(clusters, p_c_lncRNAtopic_class, direction=\"up\")\n",
    "with open(f\"{path_to_save}/{dataset} level {level} lncRNA-topic up clusters threshold {np.around(threshold, decimals=2)}.json\", 'w') as fp:\n",
    "    json.dump(topic_arr_lncRNA, fp)\n",
    "topic_arr_lncRNA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffcab1ba",
   "metadata": {},
   "source": [
    "## Enriched topics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dd578f3",
   "metadata": {},
   "source": [
    "### mRNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fba7a9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "path=f\"Results/{dataset}/nSBM/Outcome analysis/Level {level}\"\n",
    "Path(f\"{path}/Enrichment test topics\").mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daefb12b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"HelperFiles/GSEA.json\") as f:\n",
    "        all_lists_mRNA=json.load(f)\n",
    "len(all_lists_mRNA.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef2fa062",
   "metadata": {},
   "outputs": [],
   "source": [
    "mRNAtopic_gene_genename_ok=mRNAtopic_gene_genename[flat_list(topic_arr_mRNA.values())]\n",
    "mRNAtopic_gene_raw_ok=mRNAtopic_gene_raw[flat_list(topic_arr_mRNA.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6d485c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#It takes about 20-30 seconds for each topic on a i5-8265U 4 cores 1.60 GHz laptop\n",
    "enrichment_test(all_lists_mRNA, mRNAtopic_gene_genename_ok, mRNAtopic_gene_raw_ok, info, \"mRNA\",\n",
    "               f\"{path}/Enrichment test topics\", dataset, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a6ad3b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "path=f\"Results/{dataset}/nSBM/Outcome analysis/Level {level}\"\n",
    "dfs_hgt_mRNA={}\n",
    "for topic in mRNAtopic_gene_genename_ok.columns:\n",
    "    dfs_hgt_mRNA[topic]=pd.read_csv(f\"{path}/Enrichment test topics/{dataset} level {level} Enrichment Test mRNA-topic {topic}.csv\",index_col=0)\n",
    "    dfs_hgt_mRNA[topic].drop(dfs_hgt_mRNA[topic][dfs_hgt_mRNA[topic].fdr<3].dropna().index,inplace=True)\n",
    "    dfs_hgt_mRNA[topic].sort_values(by=\"fdr\",inplace=True,ascending=False)\n",
    "for key in dfs_hgt_mRNA.keys():\n",
    "    print(key, dfs_hgt_mRNA[key].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a504d75",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_name_mRNA=topics_names(topics=flat_list(topic_arr_mRNA.values()),enr_test_outcome=dfs_hgt_mRNA,\n",
    "                        database=all_lists_mRNA)\n",
    "topic_name_mRNA.to_csv(f\"{path_to_save}/{dataset} level {level} mRNA-topic-name_raw.csv\")\n",
    "topic_name_mRNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05d18c0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_change=\"mRNA-Topic 10\"\n",
    "new=\"c2_KOBAYASHI_EGFR_SIGNALING_24HR_DN\"\n",
    "topic_name_mRNA.loc[to_change]=[dfs_hgt_mRNA[to_change].loc[new].fdr,\n",
    "                                new,dfs_hgt_mRNA[to_change].loc[new].inter,\n",
    "                                dfs_hgt_mRNA[to_change].loc[new].len_inter, \n",
    "                                len(all_lists_mRNA[new]),\n",
    "                                dfs_hgt_mRNA[to_change].loc[new].p_gene_topic]\n",
    "\n",
    "to_change=\"mRNA-Topic 9\"\n",
    "new=\"c2_SMID_BREAST_CANCER_BASAL_DN\"\n",
    "topic_name_mRNA.loc[to_change]=[dfs_hgt_mRNA[to_change].loc[new].fdr,\n",
    "                                new,dfs_hgt_mRNA[to_change].loc[new].inter,\n",
    "                                dfs_hgt_mRNA[to_change].loc[new].len_inter, \n",
    "                                len(all_lists_mRNA[new]),\n",
    "                                dfs_hgt_mRNA[to_change].loc[new].p_gene_topic]\n",
    "\n",
    "topic_name_mRNA.to_csv(f\"{path_to_save}/{dataset} level {level} mRNA-topic-name.csv\")\n",
    "topic_name_mRNA.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c634ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_name_mRNA=pd.read_csv(f\"{path_to_save}/{dataset} level {level} mRNA-topic-name.csv\", index_col=0)\n",
    "heading_properties = [('font-size', '12px')]\n",
    "\n",
    "cell_properties = [('font-size', '16px')]\n",
    "\n",
    "dfstyle = [dict(selector=\"th\", props=heading_properties),\\\n",
    "dict(selector=\"td\", props=cell_properties)]\n",
    "\n",
    "topic_name_mRNA[[\"fdr\",\"name\",\"len_inter\", \"len_gene_set\"]].style.set_table_styles(dfstyle)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d262c695",
   "metadata": {},
   "source": [
    "### lncRNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f916bcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "path=f\"Results/{dataset}/nSBM/Outcome analysis/Level {level}\"\n",
    "Path(f\"{path}/Enrichment test topics\").mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81f77cee",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"HelperFiles/lncSEA_red.json\") as f:\n",
    "        all_lists_lncRNA=json.load(f)\n",
    "len(all_lists_lncRNA.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e761dffa",
   "metadata": {},
   "outputs": [],
   "source": [
    "lncRNAtopic_gene_genename_ok=lncRNAtopic_gene_genename[flat_list(topic_arr_lncRNA.values())]\n",
    "lncRNAtopic_gene_raw_ok=lncRNAtopic_gene_raw[flat_list(topic_arr_lncRNA.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e79e9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#It takes about 20-30 seconds for each topic on a i5-8265U 4 cores 1.60 GHz laptop\n",
    "enrichment_test(all_lists_lncRNA, lncRNAtopic_gene_genename_ok, lncRNAtopic_gene_raw_ok, info, \"lncRNA\",\n",
    "               f\"{path}/Enrichment test topics\", f\"{dataset}\", 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4bde32",
   "metadata": {},
   "outputs": [],
   "source": [
    "path=f\"Results/{dataset}/nSBM/Outcome analysis/Level {level}\"\n",
    "dfs_hgt_lncRNA={}\n",
    "for topic in sorted(flat_list(topic_arr_lncRNA.values())):\n",
    "    dfs_hgt_lncRNA[topic]=pd.read_csv(f\"{path}/Enrichment test topics/{dataset} level {level} Enrichment Test lncRNA-topic {topic}.csv\",index_col=0)\n",
    "    dfs_hgt_lncRNA[topic].drop(dfs_hgt_lncRNA[topic][dfs_hgt_lncRNA[topic].fdr<3].dropna().index,inplace=True)\n",
    "    dfs_hgt_lncRNA[topic].sort_values(by=\"fdr\",inplace=True,ascending=False)\n",
    "for key in dfs_hgt_lncRNA.keys():\n",
    "    print(key, dfs_hgt_lncRNA[key].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff38abf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_name_lncRNA=topics_names(topics=flat_list(topic_arr_lncRNA.values()),enr_test_outcome=dfs_hgt_lncRNA,\n",
    "                        database=all_lists_lncRNA)\n",
    "topic_name_lncRNA.to_csv(f\"{path_to_save}/{dataset} level {level} lncRNA-topic-name_raw.csv\")\n",
    "topic_name_lncRNA=pd.read_csv(f\"{path_to_save}/{dataset} level {level} lncRNA-topic-name_raw.csv\", index_col=0)\n",
    "heading_properties = [('font-size', '12px')]\n",
    "\n",
    "cell_properties = [('font-size', '16px')]\n",
    "\n",
    "dfstyle = [dict(selector=\"th\", props=heading_properties),\\\n",
    "dict(selector=\"td\", props=cell_properties)]\n",
    "\n",
    "topic_name_lncRNA[[\"fdr\",\"name\",\"len_inter\", \"len_gene_set\"]].style.set_table_styles(dfstyle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7efdc19c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"CTA-392C11\" in info.astype(str)[\"Gene name\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24b0cabf",
   "metadata": {},
   "outputs": [],
   "source": [
    "info[info[\"Gene name\"]==\"OSER1-AS1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63efc5db",
   "metadata": {},
   "outputs": [],
   "source": [
    "top=\"lncRNA-Topic 19\"\n",
    "d=pd.DataFrame(topic_name_lncRNA.loc[top][\"inter\"][3:-3].replace(\"'\", \"\").split(\", \"), columns=[\"genes\"])\n",
    "d[\"pgt\"]=topic_name_lncRNA.loc[top][\"p_gene_topic\"][1:-1].replace(\"'\", \"\").split(\", \")\n",
    "d.sort_values(by=\"pgt\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b8a7a41",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"PART1\" in d[\"genes\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d037de2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "lncRNAtopic_gene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf73c6f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_name_lncRNA.to_csv(f\"{path_to_save}/{dataset} level {level} lncRNA-topic-name.csv\")\n",
    "\n",
    "topic_name_lncRNA=pd.read_csv(f\"{path_to_save}/{dataset} level {level} lncRNA-topic-name.csv\", index_col=0)\n",
    "heading_properties = [('font-size', '12px')]\n",
    "\n",
    "cell_properties = [('font-size', '16px')]\n",
    "\n",
    "dfstyle = [dict(selector=\"th\", props=heading_properties),\\\n",
    "dict(selector=\"td\", props=cell_properties)]\n",
    "\n",
    "topic_name_lncRNA[[\"fdr\",\"name\",\"len_inter\", \"len_gene_set\"]].style.set_table_styles(dfstyle)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cb2d15e",
   "metadata": {},
   "source": [
    "### Topic cluster association"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fc61179",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_mRNAtopic_cell=pd.read_csv(f\"Results/{dataset}/nSBM/Data/{dataset}-level-{level}-mRNA-p_topic_sample.csv\", index_col=0)\n",
    "p_c_mRNAtopic_cell=p_mRNAtopic_cell-p_mRNAtopic_cell.mean()\n",
    "p_c_mRNAtopic_class=pd.DataFrame(index=clusters, columns=p_c_mRNAtopic_cell.columns)\n",
    "\n",
    "for cla in clusters:\n",
    "    insample=labels[labels[\"nSBM\"]==cla].index\n",
    "    p_c_mRNAtopic_class.loc[cla]=p_c_mRNAtopic_cell.loc[insample].mean()\n",
    "p_c_mRNAtopic_class.index=[f\"cluster {i}\" for i in range(len(p_c_mRNAtopic_class.index))]\n",
    "\n",
    "topic_name_mRNA=pd.read_csv(f\"{path_to_save}/{dataset} level {level} mRNA-topic-name.csv\", index_col=0)\n",
    "file=glob.glob(f\"{path_to_save}/{dataset} level {level} mRNA-topic up clusters threshold *.json\")[0]\n",
    "with open(file) as f:\n",
    "    topic_arr_mRNA=json.load(f)\n",
    "    \n",
    "topic_cluster_association_mRNA=pd.DataFrame(p_c_mRNAtopic_class)\n",
    "\n",
    "for key, value in zip(topic_arr_mRNA.keys(), topic_arr_mRNA.values()):\n",
    "    not_in=list(set(topic_cluster_association_mRNA.columns) - set(value))\n",
    "    topic_cluster_association_mRNA.loc[f\"cluster {key}\"][not_in]=0\n",
    "\n",
    "column=\"name\"\n",
    "new_cols=[f\"{col} - {topic_name_mRNA.loc[col][column]}\" if col in topic_name_mRNA.index else col for col in topic_cluster_association_mRNA]\n",
    "topic_cluster_association_mRNA.columns=new_cols\n",
    "topic_cluster_association_mRNA.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "717d5428",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_lncRNAtopic_cell=pd.read_csv(f\"Results/{dataset}/nSBM/Data/{dataset}-level-{level}-lncRNA-p_topic_sample.csv\", index_col=0)\n",
    "p_c_lncRNAtopic_cell=p_lncRNAtopic_cell-p_lncRNAtopic_cell.mean()\n",
    "p_c_lncRNAtopic_class=pd.DataFrame(index=clusters, columns=p_c_lncRNAtopic_cell.columns)\n",
    "\n",
    "for cla in clusters:\n",
    "    insample=labels[labels[\"nSBM\"]==cla].index\n",
    "    p_c_lncRNAtopic_class.loc[cla]=p_c_lncRNAtopic_cell.loc[insample].mean()\n",
    "p_c_lncRNAtopic_class.index=[f\"cluster {i}\" for i in range(len(p_c_lncRNAtopic_class.index))]\n",
    "\n",
    "topic_name_lncRNA=pd.read_csv(f\"{path_to_save}/{dataset} level {level} lncRNA-topic-name.csv\", index_col=0)\n",
    "file=glob.glob(f\"{path_to_save}/{dataset} level {level} lncRNA-topic up clusters threshold *.json\")[0]\n",
    "with open(file) as f:\n",
    "    topic_arr_lncRNA=json.load(f)\n",
    "    \n",
    "topic_cluster_association_lncRNA=pd.DataFrame(p_c_lncRNAtopic_class)\n",
    "\n",
    "for key, value in zip(topic_arr_lncRNA.keys(), topic_arr_lncRNA.values()):\n",
    "    not_in=list(set(topic_cluster_association_lncRNA.columns) - set(value))\n",
    "    topic_cluster_association_lncRNA.loc[f\"cluster {key}\"][not_in]=0\n",
    "\n",
    "column=\"name\"\n",
    "new_cols=[f\"{col} - {topic_name_lncRNA.loc[col][column]}\" if col in topic_name_lncRNA.index else col for col in topic_cluster_association_lncRNA]\n",
    "topic_cluster_association_lncRNA.columns=new_cols\n",
    "topic_cluster_association_lncRNA.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3c5c86b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb1b9785",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_cluster_association=pd.concat([topic_cluster_association_mRNA,topic_cluster_association_lncRNA], axis=1).astype(float)\n",
    "\n",
    "topic_cluster_association=pd.concat([topic_cluster_association_mRNA,topic_cluster_association_lncRNA], axis=1)\n",
    "topic_cluster_association=topic_cluster_association.astype(float)\n",
    "lut = dict(zip(topic_cluster_association.index, col_clusters(labels, \"nSBM\",\"typehisto\")))\n",
    "col_colors=pd.DataFrame.from_dict(lut, orient=\"index\", columns=[\"typehisto\"]).sort_values(by=\"typehisto\")\n",
    "\n",
    "col_colors[\"Second subpopulation\"]=[\"blue\",\"blue\",\"blue\",\"darkturquoise\",\"red\",\"orange\",\"orange\"]\n",
    "col_colors.columns=[\"First subpopulation\",\"Second subpopulation\"]\n",
    "\n",
    "\n",
    "row_colors=pd.DataFrame(index=topic_cluster_association.loc[col_colors.index].T.index, columns=[\"Gene type\", \"RNA family\"])\n",
    "row_colors[\"Gene type\"].loc[topic_cluster_association_mRNA.columns]=\"mRNA-topic\"\n",
    "row_colors[\"RNA family\"].loc[topic_cluster_association_mRNA.columns]=\"green\"\n",
    "row_colors[\"Gene type\"].loc[topic_cluster_association_lncRNA.columns]=\"lncRNA-topic\"\n",
    "row_colors[\"RNA family\"].loc[topic_cluster_association_lncRNA.columns]=\"violet\"\n",
    "\n",
    "\n",
    "vmin=topic_cluster_association.loc[col_colors.index].T.round(decimals=3).replace({'0':np.nan, 0:np.nan}).min().min()\n",
    "vmax=topic_cluster_association.loc[col_colors.index].T.round(decimals=3).replace({'0':np.nan, 0:np.nan}).max().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9bdf060",
   "metadata": {},
   "outputs": [],
   "source": [
    "kws = dict(cbar_kws=dict(ticks=[vmin, vmax], orientation='horizontal'), \n",
    "            annot_kws={\"fontsize\":20, \"fontweight\":\"bold\"}, figsize=(15,14))\n",
    "order=[\"cluster 4\",\"cluster 6\",\"cluster 3\",\"cluster 2\",\"cluster 1\",\"cluster 0\",\"cluster 5\"]\n",
    "cg=sns.clustermap(topic_cluster_association.loc[order].T.round(decimals=4).replace({'0':np.nan, 0:np.nan}).dropna(axis=0,  how='all'),\n",
    "                  row_cluster=False, col_cluster=False,  annot=True, vmin=vmin, vmax=vmax, fmt=\".3f\",\n",
    "                  row_colors=pd.DataFrame(row_colors[\"RNA family\"]),\n",
    "                  xticklabels=True, yticklabels=True, **kws)\n",
    "\n",
    "cg.ax_cbar.set_title(\"Pc(topic|cluster)\", fontsize=\"x-large\")\n",
    "cg.ax_cbar.set_position((0.6, 0.9, 0.2, 0.025))\n",
    "plt.setp(cg.ax_heatmap.xaxis.get_majorticklabels(), rotation=45, fontweight=\"bold\", fontsize=\"16\")\n",
    "plt.setp(cg.ax_heatmap.yaxis.get_majorticklabels(), fontweight=\"bold\", fontsize=\"20\")\n",
    "for spine in cg.ax_cbar.spines:\n",
    "    cg.ax_cbar.spines[spine].set_color('crimson')\n",
    "    cg.ax_cbar.spines[spine].set_linewidth(2)\n",
    "cg.ax_heatmap.axes.set_xticklabels([f\"cluster {i}\" for i in range(7)])\n",
    "\n",
    "leg=dict(zip([\"mRNA-topic\",\"lncRNA-topic\"],[\"green\",\"violet\"]))\n",
    "handles = [Patch(facecolor=leg[name]) for name in leg]\n",
    "leg_1=plt.legend(handles, leg, title=\"Partition (RNA Family)\", title_fontsize=\"x-large\", \n",
    "                fontsize=18,\n",
    "                bbox_transform=plt.gcf().transFigure, loc=(1.3,-1.75))\n",
    "plt.gca().add_artist(leg_1)\n",
    "\n",
    "title=f\"triSBM_Heatmap_HGT\"\n",
    "plt.savefig(f\"Results/Figures/{title}.jpeg\", dpi=600)\n",
    "#plt.savefig(f\"Results/Figures/{title}.pdf\", dpi=600)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44f6209a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "795px",
    "left": "21px",
    "top": "111.125px",
    "width": "295px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}