Merge pull request #1 from DriftAsimov/main

Experimental Python Doc Scraper

Author: FalseDev

brainfeed-soup.ipynb souppy.ipynb

@@ -2,20 +2,19 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
-   "id": "1a6cdb3f-759a-4e8c-a351-79934a29c1b4",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Imports\n",
     "from bs4 import BeautifulSoup\n",
-    "from requests import Session"
+    "from requests import Session\n",
+    "from urllib.parse import urlparse"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "id": "841f5dcf-80da-4fea-b866-94af3c490fe4",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -116,8 +115,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
-   "id": "716bc3dd-1d51-447b-ac13-61da1d859116",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -126,8 +124,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 59,
-   "id": "816e5cdc-fd61-41f0-8c11-d72cea22d1a4",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -136,8 +133,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 60,
-   "id": "36610e53-f418-4ee5-a9ab-07a68e73f65f",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -147,8 +143,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 61,
-   "id": "9d29e929-92d7-4024-9601-b04c384f9435",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -157,8 +152,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 120,
-   "id": "002dde50-0148-4278-89b0-00d9e8e0d1c2",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -167,8 +161,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 121,
-   "id": "ba80d72f-618f-4124-b7fc-c7b8c962afcf",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -185,8 +178,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 122,
-   "id": "fedb9f43-c929-42e6-a8f5-0c2e41c7849e",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -195,8 +187,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 141,
-   "id": "b03b0e44-3364-4180-b6ad-b8ec3ed3d46a",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -206,8 +197,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 142,
-   "id": "fff3107b-dddf-4fab-b3f3-4807510d63ea",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = urlparse(url).geturl().replace(\"\\\\\", \"/\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -216,98 +215,60 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 143,
-   "id": "2642b64b-c216-4ff9-b2db-359037d28582",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "soup = BeautifulSoup(res2.text)"
+    "soup = BeautifulSoup(res2.text, \"html.parser\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 147,
-   "id": "8e9736cf-382a-43f7-a404-282b23b29b6d",
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'Raised when indentation contains an inconsistent use of tabs and spaces.\\nThis is a subclass of'"
-      ]
-     },
-     "execution_count": 147,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "e = soup.find(id=id_)\n",
-    "e.parent.p.contents[0].strip()"
+    "base_element = soup.find(\"dt\", {'id' : id_})\r\n",
+    "base_parent = base_element.parent"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 148,
-   "id": "5fda45f4-b36e-4ca7-8921-1a30ae26849f",
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'TabError'"
-      ]
-     },
-     "execution_count": 148,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "id_"
+    "para = base_parent.find(\"dd\").text"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 149,
-   "id": "e6ebee8c-3ce6-4f2e-80e1-4d4002066a58",
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'Raised when indentation contains an inconsistent use of tabs and spaces.\\nThis is a subclass of IndentationError.'"
-      ]
-     },
-     "execution_count": 149,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "e.parent.p.text"
+    "print(para)"
    ]
   }
  ],
  "metadata": {
+  "interpreter": {
+   "hash": "63fd5069d213b44bf678585dea6b12cceca9941eaf7f819626cde1f2670de90d"
+  },
   "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
+   "display_name": "Python 3.9.5 64-bit",
    "name": "python3"
   },
   "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
    "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.5"
+   "version": ""
+  },
+  "metadata": {
+   "interpreter": {
+    "hash": "63fd5069d213b44bf678585dea6b12cceca9941eaf7f819626cde1f2670de90d"
+   }
   }
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}