model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import progressbar\n",
    "from random import random, choice\n",
    "from tokenizer.custom_tokenizer import CustomToken\n",
    "from sklearn.neighbors import KDTree\n",
    "import numpy as np\n",
    "import os\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node:\n",
    "    def __init__(self, value, p_space = 0.9):\n",
    "        self.value = value\n",
    "        self.input_edges = {}\n",
    "        self.in_degree = 0\n",
    "        self.out_degree = 0\n",
    "        self.output_edges = {}\n",
    "        self.types = {}\n",
    "        self.len_types = 0\n",
    "        self.cluster = 0\n",
    "        self.p_space = p_space\n",
    "\n",
    "    def add_type(self, type_tuple):\n",
    "        self.len_types += 1\n",
    "        try:\n",
    "            self.types[type_tuple] += 1\n",
    "        except KeyError:\n",
    "            self.types[type_tuple] = 1\n",
    "\n",
    "    def add_edge(self, n: \"Node\", out= False):\n",
    "        if out:\n",
    "            self.out_degree += 1\n",
    "            try:\n",
    "                self.output_edges[n.value] += 1\n",
    "            except KeyError:\n",
    "                self.output_edges[n.value] = 1\n",
    "        else:\n",
    "            self.in_degree += 1\n",
    "            try:\n",
    "                self.input_edges[n.value] += 1\n",
    "            except KeyError:\n",
    "                self.input_edges[n.value] = 1\n",
    "    \n",
    "    def __str__(self):\n",
    "        alpha = random()\n",
    "        return (' ' if self.p_space > alpha else '') + self.value\n",
    "\n",
    "\n",
    "def __token_struct__(t: CustomToken):\n",
    "    type_ = t.syntax[2]\n",
    "    spacing = 0.95\n",
    "    if t.is_emoji(): type_, spacing = 'emoji', 0.35\n",
    "    if t.is_url(): type_ = 'url'\n",
    "    if t.is_symbol: type_, spacing  = 'symbol', 0.2\n",
    "    \n",
    "    return (type_, t.syntax[3], t.text == '.'), spacing\n",
    "\n",
    "def sampler(list):\n",
    "    _len = sum([value for _, value in list])\n",
    "    list.sort(key=lambda x:x[1], reverse=True)\n",
    "    x = random()\n",
    "     \n",
    "    result, pivot = list[0]\n",
    "    for i in range(1, len(list)):\n",
    "        key, value = list[i]\n",
    "        if x < (pivot + value)/_len: return result\n",
    "        result, pivot = key, pivot + value\n",
    "    \n",
    "    return list[-1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StructureSelectByMarkovModel:\n",
    "    def __init__(self, p=1, k=1000000):\n",
    "        \"\"\"\n",
    "            params:\n",
    "                p: density of probability in that search next step \n",
    "                k: length of list in that search next step \n",
    "                sent_length: count of sent generate \n",
    "        \"\"\"\n",
    "        \n",
    "        self.p = p\n",
    "        self.k = k\n",
    "        self.structure_graph = {}\n",
    "\n",
    "\n",
    "    def __get_structure_node__(self, t: CustomToken) -> Node:\n",
    "        pivot_struct, _ = __token_struct__(t)\n",
    "        value = (pivot_struct, self.__sent_index)\n",
    "        try: \n",
    "            return self.structure_graph[pivot_struct][self.__sent_index]\n",
    "        except KeyError: \n",
    "            try:\n",
    "                self.structure_graph[pivot_struct][self.__sent_index] = Node(value)\n",
    "                return self.structure_graph[pivot_struct][self.__sent_index]\n",
    "            except KeyError:\n",
    "                self.structure_graph[pivot_struct] = {}\n",
    "                self.structure_graph[pivot_struct][self.__sent_index] = Node(value)\n",
    "                return self.structure_graph[pivot_struct][self.__sent_index]\n",
    "    \n",
    "    def fit(self, X: list[CustomToken]):\n",
    "        self.__current_sent = X[0].sent\n",
    "        self.__sent_index = 0\n",
    "        self.__previous_words = []\n",
    "        \n",
    "        pivot_struct =  self.__get_structure_node__(X[0])\n",
    "        self.__sent_index = 1\n",
    "        for i in range(1, len(X)):\n",
    "            if X[i].sent != self.__current_sent or self.__sent_index > len(self.__current_sent):\n",
    "                self.__previous_words = []\n",
    "                self.__sent_index = 0\n",
    "\n",
    "            new_struct = self.__get_structure_node__(X[i])\n",
    "            \n",
    "            pivot_struct.add_edge(new_struct, out=True)\n",
    "            new_struct.add_edge(pivot_struct)\n",
    "\n",
    "\n",
    "            pivot_struct = new_struct\n",
    "            self.__sent_index += 1\n",
    "            self.__previous_words.append(X[i].text)\n",
    "\n",
    "    def __next(self, struct, index):        \n",
    "        node : Node = self.structure_graph[struct][index]\n",
    "        next_list = list(node.output_edges.keys())\n",
    "        options = [(key, node.output_edges[(key, i)]) for key, i in next_list if i == index + 1]\n",
    "\n",
    "        if not any(options): return None\n",
    "        return sampler(options)\n",
    "        \n",
    "    def __previous(self, struct, index):        \n",
    "        node : Node = self.structure_graph[struct][index]\n",
    "        next_list = list(node.input_edges.keys())\n",
    "        options = [(key, node.input_edges[(key, i)]) for key, i in next_list if i == index - 1]\n",
    "\n",
    "        if not any(options): return None\n",
    "        return sampler(options)\n",
    "    \n",
    "    def __final_struct(self, struct):\n",
    "        return struct is None or struct[2]\n",
    "\n",
    "    def predict(self, struct):\n",
    "        next_structs, previous_structs = [struct], [struct]\n",
    "        options = list(self.structure_graph[struct].keys())\n",
    "        index = choice(options)\n",
    "        ni, pi = index, index\n",
    "\n",
    "        while pi > 0 or not self.__final_struct(next_structs[-1]):\n",
    "            if not next_structs[-1] is None and  not self.__final_struct(next_structs[-1]) :\n",
    "                n, ni = self.__next(next_structs[-1], ni), ni + 1\n",
    "                next_structs.append(n)\n",
    "            if pi > 0 and not previous_structs[-1] is None and not self.__final_struct(previous_structs[-1]):\n",
    "                p, pi = self.__previous(previous_structs[-1], pi), pi - 1\n",
    "                previous_structs.append(p)\n",
    "            \n",
    "        previous_structs.pop(0)\n",
    "        previous_structs.reverse()\n",
    "        return [value for value in previous_structs + next_structs if not value is None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StructureSelectByProbabilisticDict:\n",
    "    def __init__(self, p=1, k=1000000):\n",
    "        \"\"\"\n",
    "            params:\n",
    "                p: density of probability in that search next step \n",
    "                k: length of list in that search next step \n",
    "        \"\"\"\n",
    "        \n",
    "        self.p = p\n",
    "        self.k = k\n",
    "        self.structure_dict = {}\n",
    "\n",
    "\n",
    "    def fit(self, X: list[CustomToken]):\n",
    "        struct = []\n",
    "        sent = None\n",
    "        for t in X:\n",
    "            if sent != t.sent:\n",
    "                struct.append([])\n",
    "                sent = t.sent\n",
    "            s, _ = __token_struct__(t)\n",
    "            struct[-1].append(s)\n",
    "        \n",
    "        for s in struct:\n",
    "            s = tuple(s)  \n",
    "            try: \n",
    "                self.structure_dict[s] += 1\n",
    "            except KeyError: \n",
    "                self.structure_dict[s] = 1\n",
    "        \n",
    "    def predict(self, struct):\n",
    "        list_ = [(key, self.structure_dict[key]) for key in self.structure_dict if struct in key]\n",
    "        _len = sum([value for _, value in list_])\n",
    "        list_.sort(key=lambda x:x[1], reverse=True)\n",
    "        list_ = list_[0: self.k]\n",
    "        dist, pivot = [], 0\n",
    "        for key, value in list_:\n",
    "            if pivot/_len > self.p: \n",
    "                break\n",
    "            pivot += value\n",
    "            dist.append((key, value))\n",
    "\n",
    "        return sampler(dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MarkovGenerativeTextModel:\n",
    "    def __init__(self, p=1, k=None, sent_length = 2, strength_filter=0, memory_size=1):\n",
    "        ## TODO csp in structure constraint\n",
    "        ## TODO kbtree search and filter \n",
    "        \"\"\"\n",
    "            params:\n",
    "                p: density of probability in that search next step \n",
    "                k: length of list in that search next step \n",
    "                sent_length: count of sent generate \n",
    "                structure_graph: dict[(type_word, pos_word, is_end_word)] : Node\n",
    "                language_graph: dict[str] : Node\n",
    "        \"\"\"\n",
    "        \n",
    "        self.p = p\n",
    "        self.k = k\n",
    "        self.sent_length = sent_length\n",
    "        self.strength_filter = strength_filter\n",
    "        self.memory_size = memory_size\n",
    "        self.language_graph = {}\n",
    "        self.strength = {}\n",
    "\n",
    "    def __get_language_node__(self, t: CustomToken) -> Node:\n",
    "        pivot_struct, spacing = __token_struct__(t)\n",
    "        try: \n",
    "            self.language_graph[t.text].add_type(pivot_struct)\n",
    "        except: \n",
    "            self.language_graph[t.text] = Node(t.text, p_space=spacing)\n",
    "            self.language_graph[t.text].add_type(pivot_struct)\n",
    "            \n",
    "        return self.language_graph[t.text]\n",
    "    \n",
    "    def __strength_update__(self, t: CustomToken):\n",
    "        keys = [t.text] + [(word, t.text) for word in self.__previous_words if word != t.text]\n",
    "        for key in keys:\n",
    "            try:\n",
    "                self.strength[key] += 1\n",
    "            except KeyError:\n",
    "                self.strength[key] = 1\n",
    "        \n",
    "\n",
    "    def fit(self, X: list[CustomToken]):\n",
    "        self.__current_sent = X[0].sent\n",
    "        self.__previous_words = []\n",
    "        \n",
    "        pivot_word = self.__get_language_node__(X[0])\n",
    "        self.__strength_update__(X[0])\n",
    "\n",
    "        for i in range(1, len(X)):\n",
    "            if X[i].sent != self.__current_sent:\n",
    "                self.__previous_words = []\n",
    "\n",
    "            self.__strength_update__(X[i])\n",
    "            new_word = self.__get_language_node__(X[i])\n",
    "            \n",
    "            pivot_word.add_edge(new_word, out=True)\n",
    "            new_word.add_edge(pivot_word)\n",
    "\n",
    "\n",
    "            pivot_word = new_word\n",
    "            self.__previous_words.append(X[i].text)\n",
    "    \n",
    "    def __strength_filter(self, state, word_list, _next = True):\n",
    "        result = []\n",
    "        for key in word_list:\n",
    "            for word in state:\n",
    "                tupl = (word, key) if _next else (key, word)\n",
    "                try:\n",
    "                    value = self.strength[tupl]/ self.strength[key]\n",
    "                    result.append((key, value))\n",
    "                except KeyError: \n",
    "                    pass\n",
    "        \n",
    "        result.sort(key=lambda x:x[1], reverse=True)\n",
    "        return [key for key, _ in set(result[0:self.strength_filter])]\n",
    "    \n",
    "    def __struct_filter(self, struct, word_list):\n",
    "        return [word for word in word_list if struct in self.language_graph[word].types]\n",
    "\n",
    "    def __probabilistic_filter(self, p_dict):\n",
    "        p_dict.sort(key=lambda x:x[1], reverse=True)\n",
    "        _len = sum([value for _, value in p_dict])\n",
    "        result, pivot = [], 0\n",
    "        for key, value in p_dict:\n",
    "            if pivot/_len > self.p: break\n",
    "            result.append((key, value))\n",
    "        \n",
    "        return result\n",
    "\n",
    "    def __save_filter(self, filter, actual):\n",
    "        if any(filter): return filter\n",
    "    \n",
    "        return actual\n",
    "\n",
    "    def _next(self, actual_state, struct, dict_func, _next=True):\n",
    "        word = actual_state[-1]\n",
    "        _dict = dict_func(word)\n",
    "        word_list = list(_dict.keys())\n",
    "        word_init = word_list\n",
    "        \n",
    "        word_struct = self.__struct_filter(struct, word_list)\n",
    "        word_list = self.__save_filter(word_struct, word_list)\n",
    "\n",
    "        if self.strength_filter > 0:\n",
    "            word_strength = self.__strength_filter(actual_state[-self.memory_size:], word_list, _next)\n",
    "            word_list = self.__save_filter(word_strength, word_list)\n",
    "\n",
    "\n",
    "        p_dict = [(key, _dict[key]) for key in word_list]\n",
    "        p_dict = self.__save_filter(self.__probabilistic_filter(p_dict), p_dict)\n",
    "\n",
    "        if len(p_dict) > self.k:\n",
    "            p_dict = p_dict[0: self.k]\n",
    "        \n",
    "        return sampler(p_dict)\n",
    "        \n",
    "    def predict(self, word, structure_select_model):\n",
    "        node: Node = self.language_graph[word]\n",
    "        struct = sampler(list(node.types.items()))\n",
    "\n",
    "        sent_struct = structure_select_model.predict(struct)\n",
    "        index = sent_struct.index(struct)\n",
    "\n",
    "        previous = [word]\n",
    "        for i in range(index-1, -1, -1):\n",
    "            previous.append(\n",
    "                self._next(previous, sent_struct[i], lambda x: self.language_graph[x].input_edges, _next=False)\n",
    "            )\n",
    "        \n",
    "        _next = [word]\n",
    "        for i in range(index+1, len(sent_struct)):\n",
    "            _next.append(\n",
    "                self._next(_next, sent_struct[i], lambda x: self.language_graph[x].output_edges)\n",
    "            )\n",
    "\n",
    "        previous.pop(0)\n",
    "        previous.reverse()\n",
    "        result = ''\n",
    "        for word in previous + _next:\n",
    "            result += str(self.language_graph[word])\n",
    "        \n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def length(folder):\n",
    "    return len(os.listdir(f'{folder}'))\n",
    "\n",
    "\n",
    "def load(folder):\n",
    "    for filename in os.listdir(f'{folder}'):\n",
    "        with open(f'{folder}/{filename}', 'r') as f:\n",
    "            text = f.read()\n",
    "            f.close()\n",
    "            yield ast.literal_eval(text)\n",
    "        \n",
    "def pb(len_, name = \"\"):\n",
    "    bar = progressbar.ProgressBar(len_, widgets=[progressbar.Bar('🥵', '[', ']', ), name, progressbar.Percentage()])\n",
    "    bar.start()\n",
    "    return bar\n",
    "\n",
    "def data_to_token(data):\n",
    "    token = CustomToken(data[\"text\"], lex = data['lemma'], is_stop=data['is_stop'], is_sy=data['is_symbol'], sent=data['sent'])\n",
    "    token.syntax = data['syntax']\n",
    "    return token\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_generator = MarkovGenerativeTextModel(p=0.8, k=20, strength_filter=10, memory_size=4)\n",
    "structure_graph_model = StructureSelectByMarkovModel(p=0.7, k=20)\n",
    "probabilistic_structure = StructureSelectByProbabilisticDict(p=0.6, k=50)\n",
    "\n",
    "## TODO evaluate model\n",
    "## TODO parameter optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵] training model 324937 100%\n"
     ]
    }
   ],
   "source": [
    "_len_ = length('tokens')\n",
    "bar = pb(_len_, f\" training model {_len_} \")\n",
    "\n",
    "for i, data in enumerate(load('tokens')):\n",
    "    try:\n",
    "        tokens = [data_to_token(obj) for obj in data[1] if obj['text'] != '']\n",
    "    except:\n",
    "        continue\n",
    "    if not any(tokens): continue\n",
    "    text_generator.fit(tokens)\n",
    "    structure_graph_model.fit(tokens)\n",
    "    probabilistic_structure.fit(tokens)\n",
    "    # if i > _len_/100 : break\n",
    "    bar.update(i+1)\n",
    "bar.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' yanqui q me to What It’s making me of# CubaSalva# LoveDeathRobots este dia de @MemeruemXD @L_Lawlietc Soy fans roast-beafayant divulgué la targeta la alimetacion de CREO QUE RARA VEZ MAS ALLÁ PERO PARA QUE RESOLVER NADA DETODO PERO SI TIENE EN CUENTA PROPISTAS Y QUE HAY NI CUIDAR A mi vida nada incomoda mas bonito que codician entonces no nossigas yse logre salir pedrito el medio de paqar asociación que seincluyen algunos chubascos y siria Iraq para enfrentarla dialogarán hoy sea honesto trabajo para colmo entonces compraría toda vela la pesza de lis que cortarlo por tantas cosas perturbadoras: ARRIBA DE OTRA COSA QUE TAN LAMENTABLE LO JURO! traidores más sentidas condolencias y ventajistas'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_generator.predict('vida', structure_graph_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 10'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000011?line=0'>1</a>\u001b[0m text_generator\u001b[39m.\u001b[39;49mpredict(\u001b[39m'\u001b[39;49m\u001b[39mvida\u001b[39;49m\u001b[39m'\u001b[39;49m, structure_graph_model)\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 5'\u001b[0m in \u001b[0;36mMarkovGenerativeTextModel.predict\u001b[0;34m(self, word, structure_select_model)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=128'>129</a>\u001b[0m _next \u001b[39m=\u001b[39m [word]\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=129'>130</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(index\u001b[39m+\u001b[39m\u001b[39m1\u001b[39m, \u001b[39mlen\u001b[39m(sent_struct)):\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=130'>131</a>\u001b[0m     _next\u001b[39m.\u001b[39mappend(\n\u001b[0;32m--> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=131'>132</a>\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_next(_next, sent_struct[i], \u001b[39mlambda\u001b[39;49;00m x: \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlanguage_graph[x]\u001b[39m.\u001b[39;49moutput_edges)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=132'>133</a>\u001b[0m     )\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=134'>135</a>\u001b[0m previous\u001b[39m.\u001b[39mpop(\u001b[39m0\u001b[39m)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=135'>136</a>\u001b[0m previous\u001b[39m.\u001b[39mreverse()\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 5'\u001b[0m in \u001b[0;36mMarkovGenerativeTextModel._next\u001b[0;34m(self, actual_state, struct, dict_func, _next)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=110'>111</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(p_dict) \u001b[39m>\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mk:\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=111'>112</a>\u001b[0m     p_dict \u001b[39m=\u001b[39m p_dict[\u001b[39m0\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mk]\n\u001b[0;32m--> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=113'>114</a>\u001b[0m \u001b[39mreturn\u001b[39;00m sampler(p_dict)\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 2'\u001b[0m in \u001b[0;36msampler\u001b[0;34m(list)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=49'>50</a>\u001b[0m \u001b[39mlist\u001b[39m\u001b[39m.\u001b[39msort(key\u001b[39m=\u001b[39m\u001b[39mlambda\u001b[39;00m x:x[\u001b[39m1\u001b[39m], reverse\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=50'>51</a>\u001b[0m x \u001b[39m=\u001b[39m random()\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=52'>53</a>\u001b[0m result, pivot \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39;49m[\u001b[39m0\u001b[39;49m]\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=53'>54</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m1\u001b[39m, \u001b[39mlen\u001b[39m(\u001b[39mlist\u001b[39m)):\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=54'>55</a>\u001b[0m     key, value \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m[i]\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "text_generator.predict('vida', structure_graph_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "', para informarte q las prueva se deberían mejorar financieramente estábamos en vidasexual q usa armas a nuestros ancestros y vini xq planes de ésa es apercibida por EDICIONES ILCSA S I am up and you go to be clear as my passion as minhas músicas estridente hasta disimularlo. Momento oportuno hacer con pinzas y quimico contra trump amadrazos y electoras que le esta fuera y pósteriormentelos q usa nasobuco pueden tenerlas aqui comentan distorsionando los yanqui se puede defenderce como le hubieran más a nuestros qnunkllegara este bienése sector en cup q lienso se le retrocediera el banquero presidente y consultándolo todo alentando a q cr 7 Pues para todos lados y como otros usan poco mas cereyentes de ésa es ser desnazificada ydeivados y sr esto seremos escuchados y recluye en qtodo carismo una misma para colmo es repudiado por quésucedio en manzanillo que eldependiente quiera estremarse para cojer el monaco el usdc. mas pié porque él y q todo esmero a lo colgarón aqui como si percibes que pase en ves mas alargado. Fuerabueno para q edo hablan en Camaguey tiene hogar y ventajistasque secrefiere a q se alegraban y me puede disfrazar el comediante y como si otros ruines estén fuera d esos no capsitado que relamente fefleje nuestra está reinforzando algo bonito en dia luchan para atraparlos. nuestro todo fluya para jubilarnos como laminados metálicos y si aun no lo dudaban son lo prestasuna yavisandoles de paqar asociación que nuestro tiempo disciplinando mediante videoconferencia con apego'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_generator.predict('vida', structure_graph_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 12'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000013?line=0'>1</a>\u001b[0m text_generator\u001b[39m.\u001b[39;49mpredict(\u001b[39m'\u001b[39;49m\u001b[39mvida\u001b[39;49m\u001b[39m'\u001b[39;49m, structure_graph_model)\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 5'\u001b[0m in \u001b[0;36mMarkovGenerativeTextModel.predict\u001b[0;34m(self, word, structure_select_model)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=128'>129</a>\u001b[0m _next \u001b[39m=\u001b[39m [word]\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=129'>130</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(index\u001b[39m+\u001b[39m\u001b[39m1\u001b[39m, \u001b[39mlen\u001b[39m(sent_struct)):\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=130'>131</a>\u001b[0m     _next\u001b[39m.\u001b[39mappend(\n\u001b[0;32m--> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=131'>132</a>\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_next(_next, sent_struct[i], \u001b[39mlambda\u001b[39;49;00m x: \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlanguage_graph[x]\u001b[39m.\u001b[39;49moutput_edges)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=132'>133</a>\u001b[0m     )\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=134'>135</a>\u001b[0m previous\u001b[39m.\u001b[39mpop(\u001b[39m0\u001b[39m)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=135'>136</a>\u001b[0m previous\u001b[39m.\u001b[39mreverse()\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 5'\u001b[0m in \u001b[0;36mMarkovGenerativeTextModel._next\u001b[0;34m(self, actual_state, struct, dict_func, _next)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=110'>111</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(p_dict) \u001b[39m>\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mk:\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=111'>112</a>\u001b[0m     p_dict \u001b[39m=\u001b[39m p_dict[\u001b[39m0\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mk]\n\u001b[0;32m--> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=113'>114</a>\u001b[0m \u001b[39mreturn\u001b[39;00m sampler(p_dict)\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 2'\u001b[0m in \u001b[0;36msampler\u001b[0;34m(list)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=49'>50</a>\u001b[0m \u001b[39mlist\u001b[39m\u001b[39m.\u001b[39msort(key\u001b[39m=\u001b[39m\u001b[39mlambda\u001b[39;00m x:x[\u001b[39m1\u001b[39m], reverse\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=50'>51</a>\u001b[0m x \u001b[39m=\u001b[39m random()\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=52'>53</a>\u001b[0m result, pivot \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39;49m[\u001b[39m0\u001b[39;49m]\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=53'>54</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m1\u001b[39m, \u001b[39mlen\u001b[39m(\u001b[39mlist\u001b[39m)):\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000001?line=54'>55</a>\u001b[0m     key, value \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m[i]\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "text_generator.predict('vida', structure_graph_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' la verda q el pavimento y cuba q me alegro mucho teiempo porvida que dividirse en estar enrregresar al propio pais productor de pluriemplear a q nuestro beisboll que estén bien se dijohace años ahogados en cubasel por encimad q tanto estomatólogo por leerme, há sequelas relacionadas à Sochaux. Y agregale que como lesugiero la tentasión.mesa de paqar asociación ilícita política y reciclajes que leerle la víaestatal en cupa q compártieran con nuestros fueran animales y jodidamente filosofica la misma cituacion con pinzas ya todo esmero a nuestros cientificos para vendertela al respecto al magisterio en detrimento q pueda pienso en cup como'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_generator.predict('vida', structure_graph_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "list.append() takes exactly one argument (2 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 14'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000015?line=0'>1</a>\u001b[0m text_generator\u001b[39m.\u001b[39;49mpredict(\u001b[39m'\u001b[39;49m\u001b[39mvida\u001b[39;49m\u001b[39m'\u001b[39;49m, probabilistic_structure)\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 5'\u001b[0m in \u001b[0;36mMarkovGenerativeTextModel.predict\u001b[0;34m(self, word, structure_select_model)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=116'>117</a>\u001b[0m node: Node \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlanguage_graph[word]\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=117'>118</a>\u001b[0m struct \u001b[39m=\u001b[39m sampler(\u001b[39mlist\u001b[39m(node\u001b[39m.\u001b[39mtypes\u001b[39m.\u001b[39mitems()))\n\u001b[0;32m--> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=119'>120</a>\u001b[0m sent_struct \u001b[39m=\u001b[39m structure_select_model\u001b[39m.\u001b[39;49mpredict(struct)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=120'>121</a>\u001b[0m index \u001b[39m=\u001b[39m sent_struct\u001b[39m.\u001b[39mindex(struct)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=122'>123</a>\u001b[0m previous \u001b[39m=\u001b[39m [word]\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 4'\u001b[0m in \u001b[0;36mStructureSelectByProbabilisticDict.predict\u001b[0;34m(self, struct)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=38'>39</a>\u001b[0m         \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=39'>40</a>\u001b[0m     pivot \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m value\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=40'>41</a>\u001b[0m     dist\u001b[39m.\u001b[39;49mappend(key, value)\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=42'>43</a>\u001b[0m \u001b[39mreturn\u001b[39;00m sampler(dist)\n",
      "\u001b[0;31mTypeError\u001b[0m: list.append() takes exactly one argument (2 given)"
     ]
    }
   ],
   "source": [
    "text_generator.predict('vida', probabilistic_structure)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "list.append() takes exactly one argument (2 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 15'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000016?line=0'>1</a>\u001b[0m text_generator\u001b[39m.\u001b[39;49mpredict(\u001b[39m'\u001b[39;49m\u001b[39mvida\u001b[39;49m\u001b[39m'\u001b[39;49m, probabilistic_structure)\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 5'\u001b[0m in \u001b[0;36mMarkovGenerativeTextModel.predict\u001b[0;34m(self, word, structure_select_model)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=116'>117</a>\u001b[0m node: Node \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlanguage_graph[word]\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=117'>118</a>\u001b[0m struct \u001b[39m=\u001b[39m sampler(\u001b[39mlist\u001b[39m(node\u001b[39m.\u001b[39mtypes\u001b[39m.\u001b[39mitems()))\n\u001b[0;32m--> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=119'>120</a>\u001b[0m sent_struct \u001b[39m=\u001b[39m structure_select_model\u001b[39m.\u001b[39;49mpredict(struct)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=120'>121</a>\u001b[0m index \u001b[39m=\u001b[39m sent_struct\u001b[39m.\u001b[39mindex(struct)\n\u001b[1;32m    <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000004?line=122'>123</a>\u001b[0m previous \u001b[39m=\u001b[39m [word]\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 4'\u001b[0m in \u001b[0;36mStructureSelectByProbabilisticDict.predict\u001b[0;34m(self, struct)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=38'>39</a>\u001b[0m         \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=39'>40</a>\u001b[0m     pivot \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m value\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=40'>41</a>\u001b[0m     dist\u001b[39m.\u001b[39;49mappend(key, value)\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000003?line=42'>43</a>\u001b[0m \u001b[39mreturn\u001b[39;00m sampler(dist)\n",
      "\u001b[0;31mTypeError\u001b[0m: list.append() takes exactly one argument (2 given)"
     ]
    }
   ],
   "source": [
    "text_generator.predict('vida', probabilistic_structure)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵🥵                                ] training kd tree 252507  35%\r"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 7'\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000007?line=1'>2</a>\u001b[0m bar \u001b[39m=\u001b[39m pb(_len_, \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m training kd tree \u001b[39m\u001b[39m{\u001b[39;00m_len_\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000007?line=3'>4</a>\u001b[0m dataset \u001b[39m=\u001b[39m {}\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000007?line=4'>5</a>\u001b[0m \u001b[39mfor\u001b[39;00m i, data \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(load(\u001b[39m'\u001b[39m\u001b[39mwords\u001b[39m\u001b[39m'\u001b[39m)):\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000007?line=5'>6</a>\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39many\u001b[39m(data[\u001b[39m0\u001b[39m]): \u001b[39mcontinue\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000007?line=6'>7</a>\u001b[0m     dataset[data[\u001b[39m0\u001b[39m]] \u001b[39m=\u001b[39m data[\u001b[39m1\u001b[39m]\n",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 3'\u001b[0m in \u001b[0;36mload\u001b[0;34m(folder)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000002?line=10'>11</a>\u001b[0m text \u001b[39m=\u001b[39m f\u001b[39m.\u001b[39mread()\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000002?line=11'>12</a>\u001b[0m f\u001b[39m.\u001b[39mclose()\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000002?line=12'>13</a>\u001b[0m \u001b[39myield\u001b[39;00m ast\u001b[39m.\u001b[39;49mliteral_eval(text)\n",
      "File \u001b[0;32m/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ast.py:105\u001b[0m, in \u001b[0;36mliteral_eval\u001b[0;34m(node_or_string)\u001b[0m\n\u001b[1;32m    103\u001b[0m                 \u001b[39mreturn\u001b[39;00m left \u001b[39m-\u001b[39m right\n\u001b[1;32m    104\u001b[0m     \u001b[39mreturn\u001b[39;00m _convert_signed_num(node)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m _convert(node_or_string)\n",
      "File \u001b[0;32m/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ast.py:83\u001b[0m, in \u001b[0;36mliteral_eval.<locals>._convert\u001b[0;34m(node)\u001b[0m\n\u001b[1;32m     81\u001b[0m     \u001b[39mreturn\u001b[39;00m node\u001b[39m.\u001b[39mvalue\n\u001b[1;32m     82\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(node, Tuple):\n\u001b[0;32m---> 83\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mtuple\u001b[39;49m(\u001b[39mmap\u001b[39;49m(_convert, node\u001b[39m.\u001b[39;49melts))\n\u001b[1;32m     84\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(node, List):\n\u001b[1;32m     85\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mlist\u001b[39m(\u001b[39mmap\u001b[39m(_convert, node\u001b[39m.\u001b[39melts))\n",
      "File \u001b[0;32m/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ast.py:83\u001b[0m, in \u001b[0;36mliteral_eval.<locals>._convert\u001b[0;34m(node)\u001b[0m\n\u001b[1;32m     81\u001b[0m     \u001b[39mreturn\u001b[39;00m node\u001b[39m.\u001b[39mvalue\n\u001b[1;32m     82\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(node, Tuple):\n\u001b[0;32m---> 83\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mtuple\u001b[39;49m(\u001b[39mmap\u001b[39;49m(_convert, node\u001b[39m.\u001b[39;49melts))\n\u001b[1;32m     84\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(node, List):\n\u001b[1;32m     85\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mlist\u001b[39m(\u001b[39mmap\u001b[39m(_convert, node\u001b[39m.\u001b[39melts))\n",
      "File \u001b[0;32m/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ast.py:104\u001b[0m, in \u001b[0;36mliteral_eval.<locals>._convert\u001b[0;34m(node)\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    103\u001b[0m             \u001b[39mreturn\u001b[39;00m left \u001b[39m-\u001b[39m right\n\u001b[0;32m--> 104\u001b[0m \u001b[39mreturn\u001b[39;00m _convert_signed_num(node)\n",
      "File \u001b[0;32m/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ast.py:71\u001b[0m, in \u001b[0;36mliteral_eval.<locals>._convert_signed_num\u001b[0;34m(node)\u001b[0m\n\u001b[1;32m     69\u001b[0m         _raise_malformed_node(node)\n\u001b[1;32m     70\u001b[0m     \u001b[39mreturn\u001b[39;00m node\u001b[39m.\u001b[39mvalue\n\u001b[0;32m---> 71\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_convert_signed_num\u001b[39m(node):\n\u001b[1;32m     72\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(node, UnaryOp) \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(node\u001b[39m.\u001b[39mop, (UAdd, USub)):\n\u001b[1;32m     73\u001b[0m         operand \u001b[39m=\u001b[39m _convert_num(node\u001b[39m.\u001b[39moperand)\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/cuba-digital-lang-9Kd57kqM/lib/python3.9/site-packages/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch_regular.py:352\u001b[0m, in \u001b[0;36mThreadTracer.__call__\u001b[0;34m(self, frame, event, arg)\u001b[0m\n\u001b[1;32m    327\u001b[0m \u001b[39m''' This is the callback used when we enter some context in the debugger.\u001b[39;00m\n\u001b[1;32m    328\u001b[0m \n\u001b[1;32m    329\u001b[0m \u001b[39mWe also decorate the thread we are in with info about the debugging.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    337\u001b[0m \u001b[39m    This is the global debugger (this method should actually be added as a method to it).\u001b[39;00m\n\u001b[1;32m    338\u001b[0m \u001b[39m'''\u001b[39;00m\n\u001b[1;32m    339\u001b[0m \u001b[39m# IFDEF CYTHON\u001b[39;00m\n\u001b[1;32m    340\u001b[0m \u001b[39m# cdef str filename;\u001b[39;00m\n\u001b[1;32m    341\u001b[0m \u001b[39m# cdef str base;\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    350\u001b[0m \u001b[39m# DEBUG = 'code_to_debug' in frame.f_code.co_filename\u001b[39;00m\n\u001b[1;32m    351\u001b[0m \u001b[39m# if DEBUG: print('ENTER: trace_dispatch: %s %s %s %s' % (frame.f_code.co_filename, frame.f_lineno, event, frame.f_code.co_name))\u001b[39;00m\n\u001b[0;32m--> 352\u001b[0m py_db, t, additional_info, cache_skips, frame_skips_cache \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_args\n\u001b[1;32m    353\u001b[0m \u001b[39mif\u001b[39;00m additional_info\u001b[39m.\u001b[39mis_tracing:\n\u001b[1;32m    354\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mif\u001b[39;00m event \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mcall\u001b[39m\u001b[39m'\u001b[39m \u001b[39melse\u001b[39;00m NO_FTRACE  \u001b[39m# we don't wan't to trace code invoked from pydevd_frame.trace_dispatch\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# _len_ = length('words')\n",
    "# bar = pb(_len_, f\" training kd tree {_len_} \")\n",
    "\n",
    "# dataset = {}\n",
    "# for i, data in enumerate(load('words')):\n",
    "#     if not any(data[0]): continue\n",
    "#     dataset[data[0]] = data[1]\n",
    "#     bar.update(i+1)\n",
    "# bar.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "float() argument must be a string or a number, not 'dict_items'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb Cell 8'\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000008?line=0'>1</a>\u001b[0m X \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39marray(dataset\u001b[39m.\u001b[39mitems())\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000008?line=1'>2</a>\u001b[0m tree \u001b[39m=\u001b[39m KDTree(X)              \n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000008?line=2'>3</a>\u001b[0m dist, ind \u001b[39m=\u001b[39m tree\u001b[39m.\u001b[39mquery(X[:\u001b[39m1\u001b[39m], k\u001b[39m=\u001b[39m\u001b[39m3\u001b[39m)                \n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dany/src/school/cuba-digital-lang/nlp-probabilistic-text-generator/model.ipynb#ch0000008?line=3'>4</a>\u001b[0m \u001b[39mprint\u001b[39m(ind)  \u001b[39m# indices of 3 closest neighbors\u001b[39;00m\n",
      "File \u001b[0;32msklearn/neighbors/_binary_tree.pxi:831\u001b[0m, in \u001b[0;36msklearn.neighbors._kd_tree.BinaryTree.__init__\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/cuba-digital-lang-9Kd57kqM/lib/python3.9/site-packages/sklearn/utils/validation.py:856\u001b[0m, in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator, input_name)\u001b[0m\n\u001b[1;32m    854\u001b[0m         array \u001b[39m=\u001b[39m array\u001b[39m.\u001b[39mastype(dtype, casting\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39munsafe\u001b[39m\u001b[39m\"\u001b[39m, copy\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n\u001b[1;32m    855\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 856\u001b[0m         array \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39;49masarray(array, order\u001b[39m=\u001b[39;49morder, dtype\u001b[39m=\u001b[39;49mdtype)\n\u001b[1;32m    857\u001b[0m \u001b[39mexcept\u001b[39;00m ComplexWarning \u001b[39mas\u001b[39;00m complex_warning:\n\u001b[1;32m    858\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m    859\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mComplex data not supported\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m{}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(array)\n\u001b[1;32m    860\u001b[0m     ) \u001b[39mfrom\u001b[39;00m \u001b[39mcomplex_warning\u001b[39;00m\n",
      "\u001b[0;31mTypeError\u001b[0m: float() argument must be a string or a number, not 'dict_items'"
     ]
    }
   ],
   "source": [
    "# X = np.array(dataset.items())\n",
    "# tree = KDTree(X)              \n",
    "# dist, ind = tree.query(X[:1], k=3)                \n",
    "# print(ind)  # indices of 3 closest neighbors\n",
    "# print(dist)  # distances to 3 closest neighbors"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.10 ('nlp-probabilistic-text-generator-qxtpirU4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b81cbb5fb5d338a2a02423698167f6e84e035868697b94d25f64764139b1fb15"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}