scraping_wiki_list.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "scrapingWikiList.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mattiapocci/PhilosopherRank/blob/master/scrapingWikiList.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jWGyinpWDSEy",
        "colab_type": "text"
      },
      "source": [
        "# ***Imports***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SANA3JojoGGn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import urllib.request\n",
        "import requests\n",
        "import json\n",
        "from bs4 import BeautifulSoup as bs\n",
        "import pandas as pd\n",
        "print('imports successful')\n",
        "starturl = 'https://en.wikipedia.org'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gvdE503SK3Th",
        "colab_type": "text"
      },
      "source": [
        "# ***Clustering***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jeZ3d8VLC3d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Define a cluster for every philosophy, in order to consider them for the influences\n",
        "import pprint\n",
        "url = 'https://en.wikipedia.org/wiki/List_of_philosophies'\n",
        "page = urllib.request.urlopen(url)\n",
        "soup = bs(page, \"lxml\")\n",
        "\n",
        "# return a list of titles for a given page\n",
        "def page_scrape(address):\n",
        "    page = urllib.request.urlopen(address)\n",
        "    soup = bs(page, \"lxml\")\n",
        "    a_list = soup.find_all('a')\n",
        "    res = []\n",
        "    for a in a_list:\n",
        "        if a.get('title') is not None:\n",
        "            res.append(a.get('title'))\n",
        "    return res\n",
        "\n",
        "raw_clusters = {}\n",
        "p_list = soup.find_all('p')\n",
        "# for every philosophy\n",
        "for p in p_list:\n",
        "    a_list = p.find_all('a')\n",
        "    # get a list of titles\n",
        "    for a in a_list:\n",
        "        try:\n",
        "            current_url = starturl + a.get('href')\n",
        "            raw_clusters[a.get('title')] = page_scrape(current_url)\n",
        "            #construct a list and return \n",
        "            print('Built element with title ' + a.get('title'))\n",
        "        except:\n",
        "            print(a)\n",
        "            print('anchor has no title')\n",
        "# at this point raw_clusters will contain a list of titles for every philosophy (for example raw_clusters['stoicism'] will contain [stoic_phil1, stoic_phil2, ...])\n",
        "pprint.pprint(raw_clusters)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vBIRDvG0Wbu-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# write clusters to file and download\n",
        "from google.colab import files\n",
        "with open('raw_clusters.json', 'w', encoding=\"utf-8\") as fp:\n",
        "    json.dump(raw_clusters, fp, indent=4)\n",
        "files.download('raw_clusters.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "enEqlgS9KvXy",
        "colab_type": "text"
      },
      "source": [
        "# ***Biography Table***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxcDBxQPZvZD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get data from wikipedias biography table\n",
        "# some philosophers have the fields 'influencers' and 'influenced'\n",
        "import pprint\n",
        "def bio_table(url):\n",
        "    # open url with bs\n",
        "    page = urllib.request.urlopen(url)\n",
        "    soup = bs(page, \"lxml\")\n",
        "    # get biography table\n",
        "    table = soup.find('table', class_='infobox biography vcard')\n",
        "    print(len(table.find_all('ul', class_='NavContent')))\n",
        "    try:\n",
        "        # get influencers unordered list\n",
        "        influencers = table.find_all('ul', class_='NavContent')[0]\n",
        "    except:\n",
        "        influencers = []\n",
        "    try:\n",
        "        # get influenced unordered list\n",
        "        influenced = table.find_all('ul', class_='NavContent')[1]\n",
        "    except:\n",
        "        influenced = []\n",
        "    print(influenced)\n",
        "    final_influencers = []\n",
        "    final_influenced = []\n",
        "    # We want a list of titles of wikipedia pages\n",
        "    if influencers != []:\n",
        "        for a in influencers.find_all('a'):\n",
        "            try:\n",
        "                # extract the title\n",
        "                final_influencers.append(a.get('title'))\n",
        "            except:\n",
        "                pass\n",
        "    # We want a list of titles of wikipedia pages\n",
        "    if influenced != []:\n",
        "        for a in influenced.find_all('a'):\n",
        "            try:\n",
        "                # extract the title\n",
        "                final_influenced.append(a.get('title'))\n",
        "            except:\n",
        "                pass\n",
        "\n",
        "    return final_influencers,final_influenced"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VaJAYfvM4_Ww",
        "colab_type": "text"
      },
      "source": [
        "# ***Scraping philosophers lists from wikipedia***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uE_qoGqx5EoC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Wikipedia has 4 pages listing philosophers, (A-C), (D-H), (I-Q), (R-Z)\n",
        "urls = [\"https://en.wikipedia.org/wiki/List_of_philosophers_(A%E2%80%93C)\", \"https://en.wikipedia.org/wiki/List_of_philosophers_(D%E2%80%93H)\", \"https://en.wikipedia.org/wiki/List_of_philosophers_(I%E2%80%93Q)\", \"https://en.wikipedia.org/wiki/List_of_philosophers_(R%E2%80%93Z)\"]\n",
        "\n",
        "# Initialising json\n",
        "# Philosopher\n",
        "phil=[]\n",
        "# Page url\n",
        "href=[]\n",
        "# Article\n",
        "raw=[]\n",
        "# Influence bio-table data\n",
        "influencers=[]\n",
        "influenced=[]\n",
        "# For every list of philosophers\n",
        "for url in urls:\n",
        "    # Parse page with beautiful soup\n",
        "    page = urllib.request.urlopen(url)\n",
        "    soup = bs(page, \"lxml\")\n",
        "    # Parse the list, unordered list\n",
        "    for ultag in soup.find_all('ul'):\n",
        "        # Access the list item\n",
        "        for litag in ultag.find_all('li'):\n",
        "            try:\n",
        "                # Get href\n",
        "                print('HREF: ' + litag.a.get('href'))\n",
        "                wikiurl = starturl + litag.a.get('href')\n",
        "                print(wikiurl)\n",
        "                href.append(wikiurl)\n",
        "            except:\n",
        "                # Invalid list item\n",
        "                print('No HREF')\n",
        "                href.append('NONE')\n",
        "            try:\n",
        "                # Use wikipedia api to get article\n",
        "                response = requests.get(\n",
        "                    'https://en.wikipedia.org/w/api.php',\n",
        "                    params={\n",
        "                    'action': 'query',\n",
        "                    'format': 'json',\n",
        "                    'titles': litag.a.get('title'),\n",
        "                    'prop': 'extracts',\n",
        "                    'explaintext': True,\n",
        "                    'exlimit': 'max',\n",
        "                }\n",
        "                ).json()\n",
        "                raw.append(response)\n",
        "            except:\n",
        "                # No article\n",
        "                raw.append('NONE')\n",
        "            try:\n",
        "                # Get influence bio-table data\n",
        "                tempinfluencers, tempinfluenced = bio_table(wikiurl)\n",
        "                influencers.append(tempinfluencers)\n",
        "                influenced.append(tempinfluenced)\n",
        "            except:\n",
        "                # No bio-table available\n",
        "                influencers.append([])\n",
        "                influenced.append([])\n",
        "            # Set philosophers name\n",
        "            phil.append(litag.text)\n",
        "            print(litag.text)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pnd7wuBOMVXT",
        "colab_type": "text"
      },
      "source": [
        "# ***Process the result***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ELs2e8ipKGaP",
        "colab_type": "text"
      },
      "source": [
        "Transform lists into pandas dataframe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gfpv34Al1xuZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.DataFrame(phil,columns=['Philosopher'])\n",
        "df['href']=href\n",
        "df['raw']=raw\n",
        "df['influencers']=influencers\n",
        "df['influenced']=influenced\n",
        "df\n",
        "df['raw'].sample(n=3)\n",
        "#with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
        "#    print(df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WbD0HiLzKLey",
        "colab_type": "text"
      },
      "source": [
        "Convert data frame to json"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "49PSjSEF2nsj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "result = df.to_json(orient='index')\n",
        "from google.colab import files\n",
        "open('result.json', 'w').write(result)\n",
        "files.download('result.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OSfsUBI96Dsm",
        "colab_type": "text"
      },
      "source": [
        "Transform the result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ubabtB316J3_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We want to transform the result\n",
        "# INPUT: json with fields [id][raw], [id][influencers], [id][influenced] -> we have to extract pageid, philosophers and article from raw (which is the raw response from the wikipedia api)\n",
        "# OUTPUT: json with fields [pageid], [philosopher], [article], [table] -> [table] will contain influencers and influenced (biography table data)\n",
        "with open('result.json', 'r') as f:\n",
        "  contents = json.loads(f.read())\n",
        "# prints for logging purposes\n",
        "print(contents['67']['raw']['query']['pages'])\n",
        "print(contents['67']['raw'])\n",
        "# json_list is a list of dictionaries\n",
        "json_list = []\n",
        "# for each entry\n",
        "for content in contents:\n",
        "  # define a json object\n",
        "  json_object = {}\n",
        "  json_object['table'] = {}\n",
        "  # populate the object\n",
        "  try:\n",
        "    json_object['pageid'] = list(contents[content]['raw']['query']['pages'].keys())[0]\n",
        "    json_object['philosopher'] = contents[content]['raw']['query']['pages'][json_object[\"pageid\"]]['title']\n",
        "    json_object['article'] = contents[content]['raw']['query']['pages'][json_object['pageid']]['extract']\n",
        "    json_object['table']['influencers'] = contents[content]['influencers']\n",
        "    json_object['table']['influenced'] = contents[content]['influenced']\n",
        "    json_list.append(json_object)\n",
        "  except:\n",
        "    print('Invalid page')\n",
        "# write and download final result\n",
        "with open('final_result.json', 'w', encoding='utf-8') as f:\n",
        "    json.dump(json_list, f, ensure_ascii=False, indent=4)\n",
        "files.download('final_result.json')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}