dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
101f4a4080e74e99f2e34800bd9a197e32d9426e
geography-anki/playground/downloaded_data_inspection_lab/capital_xpath.ipynb

2150 lines
95 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "markdown",
"id": "a1e7252d-16d9-4b68-a855-d94a89132291",
"metadata": {},
"source": [
"# Capital extraction"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "e4bb164b-f8a7-4d21-86a0-618ca78e9386",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:44:25.083012Z",
"iopub.status.busy": "2022-06-25T20:44:25.082584Z",
"iopub.status.idle": "2022-06-25T20:44:25.400728Z",
"shell.execute_reply": "2022-06-25T20:44:25.399967Z",
"shell.execute_reply.started": "2022-06-25T20:44:25.082926Z"
},
"tags": []
},
"outputs": [],
"source": [
"import json\n",
"import pathlib\n",
"import re\n",
"import xml.etree.ElementTree as ET\n",
"\n",
"import pandas as pd\n",
"from lxml import etree"
]
},
{
"cell_type": "markdown",
"id": "9fd69c5d-2c4d-49c8-a042-82eade1d6ab7",
"metadata": {},
"source": [
"## load data\n",
"\n",
"### load the raw countries data"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "bfe405bb-7879-4f5f-85df-7215c5e8a4b8",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:44:25.402295Z",
"iopub.status.busy": "2022-06-25T20:44:25.401710Z",
"iopub.status.idle": "2022-06-25T20:44:25.406212Z",
"shell.execute_reply": "2022-06-25T20:44:25.405622Z",
"shell.execute_reply.started": "2022-06-25T20:44:25.402274Z"
},
"tags": []
},
"outputs": [],
"source": [
"data_directory = (\n",
" pathlib.Path(\".\").resolve().parents[1] / \"data\" / \"scrapy\" / \"raw_country_data\"\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "49fdb7c8-b9db-4d0a-8d2b-d4a0a3ddcdd9",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:45:11.573098Z",
"iopub.status.busy": "2022-06-25T20:45:11.572720Z",
"iopub.status.idle": "2022-06-25T20:45:11.650803Z",
"shell.execute_reply": "2022-06-25T20:45:11.650139Z",
"shell.execute_reply.started": "2022-06-25T20:45:11.573067Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>country_url</th>\n",
" <th>flag_description_url</th>\n",
" <th>short_country_name</th>\n",
" <th>country_html</th>\n",
" <th>flag_html</th>\n",
" <th>file_urls</th>\n",
" <th>files</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>https://en.wikipedia.org/wiki/Afghanistan</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Afghanistan</td>\n",
" <td>Afghanistan</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>https://en.wikipedia.org/wiki/Croatia</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Croatia</td>\n",
" <td>Croatia</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>https://en.wikipedia.org/wiki/Costa_Rica</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Costa_Rica</td>\n",
" <td>Costa_Rica</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;&lt;a href=\"/wiki/National_flag\" title=...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>https://en.wikipedia.org/wiki/Democratic_Repub...</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_the_Demo...</td>\n",
" <td>Democratic_Republic_of_the_Congo</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>https://en.wikipedia.org/wiki/Comoros</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Comoros</td>\n",
" <td>Comoros</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;&lt;a href=\"/wiki/National_flag\" title=...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>201</th>\n",
" <td>https://en.wikipedia.org/wiki/Antigua_and_Barbuda</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Antigua_...</td>\n",
" <td>Antigua_and_Barbuda</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>202</th>\n",
" <td>https://en.wikipedia.org/wiki/Angola</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Angola</td>\n",
" <td>Angola</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>203</th>\n",
" <td>https://en.wikipedia.org/wiki/Andorra</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Andorra</td>\n",
" <td>Andorra</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>204</th>\n",
" <td>https://en.wikipedia.org/wiki/Algeria</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Algeria</td>\n",
" <td>Algeria</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>205</th>\n",
" <td>https://en.wikipedia.org/wiki/Albania</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Albania</td>\n",
" <td>Albania</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of Albania&lt;/b&gt; (&lt;a href=\"/wiki/...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>206 rows × 7 columns</p>\n",
"</div>"
],
"text/plain": [
" country_url \\\n",
"0 https://en.wikipedia.org/wiki/Afghanistan \n",
"1 https://en.wikipedia.org/wiki/Croatia \n",
"2 https://en.wikipedia.org/wiki/Costa_Rica \n",
"3 https://en.wikipedia.org/wiki/Democratic_Repub... \n",
"4 https://en.wikipedia.org/wiki/Comoros \n",
".. ... \n",
"201 https://en.wikipedia.org/wiki/Antigua_and_Barbuda \n",
"202 https://en.wikipedia.org/wiki/Angola \n",
"203 https://en.wikipedia.org/wiki/Andorra \n",
"204 https://en.wikipedia.org/wiki/Algeria \n",
"205 https://en.wikipedia.org/wiki/Albania \n",
"\n",
" flag_description_url \\\n",
"0 https://en.wikipedia.org/wiki/Flag_of_Afghanistan \n",
"1 https://en.wikipedia.org/wiki/Flag_of_Croatia \n",
"2 https://en.wikipedia.org/wiki/Flag_of_Costa_Rica \n",
"3 https://en.wikipedia.org/wiki/Flag_of_the_Demo... \n",
"4 https://en.wikipedia.org/wiki/Flag_of_Comoros \n",
".. ... \n",
"201 https://en.wikipedia.org/wiki/Flag_of_Antigua_... \n",
"202 https://en.wikipedia.org/wiki/Flag_of_Angola \n",
"203 https://en.wikipedia.org/wiki/Flag_of_Andorra \n",
"204 https://en.wikipedia.org/wiki/Flag_of_Algeria \n",
"205 https://en.wikipedia.org/wiki/Flag_of_Albania \n",
"\n",
" short_country_name \\\n",
"0 Afghanistan \n",
"1 Croatia \n",
"2 Costa_Rica \n",
"3 Democratic_Republic_of_the_Congo \n",
"4 Comoros \n",
".. ... \n",
"201 Antigua_and_Barbuda \n",
"202 Angola \n",
"203 Andorra \n",
"204 Algeria \n",
"205 Albania \n",
"\n",
" country_html \\\n",
"0 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"1 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"2 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"3 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"4 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
".. ... \n",
"201 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"202 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"203 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"204 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"205 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"\n",
" flag_html \\\n",
"0 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"1 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"2 <p>The <b><a href=\"/wiki/National_flag\" title=... \n",
"3 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"4 <p>The <b><a href=\"/wiki/National_flag\" title=... \n",
".. ... \n",
"201 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"202 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"203 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"204 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"205 <p>The <b>flag of Albania</b> (<a href=\"/wiki/... \n",
"\n",
" file_urls \\\n",
"0 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"1 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"2 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"3 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"4 [https:////upload.wikimedia.org/wikipedia/comm... \n",
".. ... \n",
"201 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"202 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"203 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"204 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"205 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"\n",
" files \n",
"0 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"1 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"2 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"3 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"4 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
".. ... \n",
"201 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"202 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"203 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"204 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"205 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"\n",
"[206 rows x 7 columns]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries = pd.read_json(data_directory / \"countries.json\")\n",
"countries"
]
},
{
"cell_type": "markdown",
"id": "7e7398db-96ea-44e7-a13c-b075e8aecc30",
"metadata": {},
"source": [
"## cleaning\n",
"\n",
"### clean `short_country_name`"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "c2eb9f21-5614-4d87-bf54-b9583160da93",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:45:13.187066Z",
"iopub.status.busy": "2022-06-25T20:45:13.186714Z",
"iopub.status.idle": "2022-06-25T20:45:13.191480Z",
"shell.execute_reply": "2022-06-25T20:45:13.190663Z",
"shell.execute_reply.started": "2022-06-25T20:45:13.187037Z"
},
"tags": []
},
"outputs": [],
"source": [
"countries[\"short_country_name\"] = countries[\"short_country_name\"].map(\n",
" lambda country: country.replace(\"_\", \" \")\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "29e35334-b5fe-4851-81c1-4150f0ed8b00",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:45:13.645989Z",
"iopub.status.busy": "2022-06-25T20:45:13.645391Z",
"iopub.status.idle": "2022-06-25T20:45:13.689080Z",
"shell.execute_reply": "2022-06-25T20:45:13.688077Z",
"shell.execute_reply.started": "2022-06-25T20:45:13.645946Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>country_url</th>\n",
" <th>flag_description_url</th>\n",
" <th>short_country_name</th>\n",
" <th>country_html</th>\n",
" <th>flag_html</th>\n",
" <th>file_urls</th>\n",
" <th>files</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>https://en.wikipedia.org/wiki/Afghanistan</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Afghanistan</td>\n",
" <td>Afghanistan</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>https://en.wikipedia.org/wiki/Croatia</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Croatia</td>\n",
" <td>Croatia</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>https://en.wikipedia.org/wiki/Costa_Rica</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Costa_Rica</td>\n",
" <td>Costa Rica</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;&lt;a href=\"/wiki/National_flag\" title=...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>https://en.wikipedia.org/wiki/Democratic_Repub...</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_the_Demo...</td>\n",
" <td>Democratic Republic of the Congo</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>https://en.wikipedia.org/wiki/Comoros</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Comoros</td>\n",
" <td>Comoros</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;&lt;a href=\"/wiki/National_flag\" title=...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>201</th>\n",
" <td>https://en.wikipedia.org/wiki/Antigua_and_Barbuda</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Antigua_...</td>\n",
" <td>Antigua and Barbuda</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>202</th>\n",
" <td>https://en.wikipedia.org/wiki/Angola</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Angola</td>\n",
" <td>Angola</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>203</th>\n",
" <td>https://en.wikipedia.org/wiki/Andorra</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Andorra</td>\n",
" <td>Andorra</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>204</th>\n",
" <td>https://en.wikipedia.org/wiki/Algeria</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Algeria</td>\n",
" <td>Algeria</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>205</th>\n",
" <td>https://en.wikipedia.org/wiki/Albania</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Albania</td>\n",
" <td>Albania</td>\n",
" <td>[&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of Albania&lt;/b&gt; (&lt;a href=\"/wiki/...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>206 rows × 7 columns</p>\n",
"</div>"
],
"text/plain": [
" country_url \\\n",
"0 https://en.wikipedia.org/wiki/Afghanistan \n",
"1 https://en.wikipedia.org/wiki/Croatia \n",
"2 https://en.wikipedia.org/wiki/Costa_Rica \n",
"3 https://en.wikipedia.org/wiki/Democratic_Repub... \n",
"4 https://en.wikipedia.org/wiki/Comoros \n",
".. ... \n",
"201 https://en.wikipedia.org/wiki/Antigua_and_Barbuda \n",
"202 https://en.wikipedia.org/wiki/Angola \n",
"203 https://en.wikipedia.org/wiki/Andorra \n",
"204 https://en.wikipedia.org/wiki/Algeria \n",
"205 https://en.wikipedia.org/wiki/Albania \n",
"\n",
" flag_description_url \\\n",
"0 https://en.wikipedia.org/wiki/Flag_of_Afghanistan \n",
"1 https://en.wikipedia.org/wiki/Flag_of_Croatia \n",
"2 https://en.wikipedia.org/wiki/Flag_of_Costa_Rica \n",
"3 https://en.wikipedia.org/wiki/Flag_of_the_Demo... \n",
"4 https://en.wikipedia.org/wiki/Flag_of_Comoros \n",
".. ... \n",
"201 https://en.wikipedia.org/wiki/Flag_of_Antigua_... \n",
"202 https://en.wikipedia.org/wiki/Flag_of_Angola \n",
"203 https://en.wikipedia.org/wiki/Flag_of_Andorra \n",
"204 https://en.wikipedia.org/wiki/Flag_of_Algeria \n",
"205 https://en.wikipedia.org/wiki/Flag_of_Albania \n",
"\n",
" short_country_name \\\n",
"0 Afghanistan \n",
"1 Croatia \n",
"2 Costa Rica \n",
"3 Democratic Republic of the Congo \n",
"4 Comoros \n",
".. ... \n",
"201 Antigua and Barbuda \n",
"202 Angola \n",
"203 Andorra \n",
"204 Algeria \n",
"205 Albania \n",
"\n",
" country_html \\\n",
"0 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"1 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"2 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"3 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"4 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
".. ... \n",
"201 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"202 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"203 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"204 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"205 [<tr><th colspan=\"2\" class=\"infobox-above adr\"... \n",
"\n",
" flag_html \\\n",
"0 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"1 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"2 <p>The <b><a href=\"/wiki/National_flag\" title=... \n",
"3 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"4 <p>The <b><a href=\"/wiki/National_flag\" title=... \n",
".. ... \n",
"201 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"202 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"203 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"204 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"205 <p>The <b>flag of Albania</b> (<a href=\"/wiki/... \n",
"\n",
" file_urls \\\n",
"0 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"1 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"2 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"3 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"4 [https:////upload.wikimedia.org/wikipedia/comm... \n",
".. ... \n",
"201 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"202 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"203 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"204 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"205 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"\n",
" files \n",
"0 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"1 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"2 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"3 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"4 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
".. ... \n",
"201 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"202 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"203 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"204 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"205 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"\n",
"[206 rows x 7 columns]"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries"
]
},
{
"cell_type": "markdown",
"id": "e6bac76b-778f-4d4e-9a8b-8dc2838eb920",
"metadata": {},
"source": [
"### convert `country_html` to single string"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "169123c9-aabb-4972-af5c-ba076cc21f5a",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:45:14.942724Z",
"iopub.status.busy": "2022-06-25T20:45:14.942372Z",
"iopub.status.idle": "2022-06-25T20:45:14.978155Z",
"shell.execute_reply": "2022-06-25T20:45:14.977401Z",
"shell.execute_reply.started": "2022-06-25T20:45:14.942694Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>country_url</th>\n",
" <th>flag_description_url</th>\n",
" <th>short_country_name</th>\n",
" <th>country_html</th>\n",
" <th>flag_html</th>\n",
" <th>file_urls</th>\n",
" <th>files</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>https://en.wikipedia.org/wiki/Afghanistan</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Afghanistan</td>\n",
" <td>Afghanistan</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>https://en.wikipedia.org/wiki/Croatia</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Croatia</td>\n",
" <td>Croatia</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>https://en.wikipedia.org/wiki/Costa_Rica</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Costa_Rica</td>\n",
" <td>Costa Rica</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;&lt;a href=\"/wiki/National_flag\" title=...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>https://en.wikipedia.org/wiki/Democratic_Repub...</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_the_Demo...</td>\n",
" <td>Democratic Republic of the Congo</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>https://en.wikipedia.org/wiki/Comoros</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Comoros</td>\n",
" <td>Comoros</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;&lt;a href=\"/wiki/National_flag\" title=...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>201</th>\n",
" <td>https://en.wikipedia.org/wiki/Antigua_and_Barbuda</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Antigua_...</td>\n",
" <td>Antigua and Barbuda</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>202</th>\n",
" <td>https://en.wikipedia.org/wiki/Angola</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Angola</td>\n",
" <td>Angola</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>203</th>\n",
" <td>https://en.wikipedia.org/wiki/Andorra</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Andorra</td>\n",
" <td>Andorra</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>204</th>\n",
" <td>https://en.wikipedia.org/wiki/Algeria</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Algeria</td>\n",
" <td>Algeria</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>205</th>\n",
" <td>https://en.wikipedia.org/wiki/Albania</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Albania</td>\n",
" <td>Albania</td>\n",
" <td>&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above adr\"&gt;...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of Albania&lt;/b&gt; (&lt;a href=\"/wiki/...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>206 rows × 7 columns</p>\n",
"</div>"
],
"text/plain": [
" country_url \\\n",
"0 https://en.wikipedia.org/wiki/Afghanistan \n",
"1 https://en.wikipedia.org/wiki/Croatia \n",
"2 https://en.wikipedia.org/wiki/Costa_Rica \n",
"3 https://en.wikipedia.org/wiki/Democratic_Repub... \n",
"4 https://en.wikipedia.org/wiki/Comoros \n",
".. ... \n",
"201 https://en.wikipedia.org/wiki/Antigua_and_Barbuda \n",
"202 https://en.wikipedia.org/wiki/Angola \n",
"203 https://en.wikipedia.org/wiki/Andorra \n",
"204 https://en.wikipedia.org/wiki/Algeria \n",
"205 https://en.wikipedia.org/wiki/Albania \n",
"\n",
" flag_description_url \\\n",
"0 https://en.wikipedia.org/wiki/Flag_of_Afghanistan \n",
"1 https://en.wikipedia.org/wiki/Flag_of_Croatia \n",
"2 https://en.wikipedia.org/wiki/Flag_of_Costa_Rica \n",
"3 https://en.wikipedia.org/wiki/Flag_of_the_Demo... \n",
"4 https://en.wikipedia.org/wiki/Flag_of_Comoros \n",
".. ... \n",
"201 https://en.wikipedia.org/wiki/Flag_of_Antigua_... \n",
"202 https://en.wikipedia.org/wiki/Flag_of_Angola \n",
"203 https://en.wikipedia.org/wiki/Flag_of_Andorra \n",
"204 https://en.wikipedia.org/wiki/Flag_of_Algeria \n",
"205 https://en.wikipedia.org/wiki/Flag_of_Albania \n",
"\n",
" short_country_name \\\n",
"0 Afghanistan \n",
"1 Croatia \n",
"2 Costa Rica \n",
"3 Democratic Republic of the Congo \n",
"4 Comoros \n",
".. ... \n",
"201 Antigua and Barbuda \n",
"202 Angola \n",
"203 Andorra \n",
"204 Algeria \n",
"205 Albania \n",
"\n",
" country_html \\\n",
"0 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"1 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"2 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"3 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"4 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
".. ... \n",
"201 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"202 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"203 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"204 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"205 <tr><th colspan=\"2\" class=\"infobox-above adr\">... \n",
"\n",
" flag_html \\\n",
"0 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"1 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"2 <p>The <b><a href=\"/wiki/National_flag\" title=... \n",
"3 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"4 <p>The <b><a href=\"/wiki/National_flag\" title=... \n",
".. ... \n",
"201 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"202 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"203 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"204 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"205 <p>The <b>flag of Albania</b> (<a href=\"/wiki/... \n",
"\n",
" file_urls \\\n",
"0 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"1 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"2 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"3 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"4 [https:////upload.wikimedia.org/wikipedia/comm... \n",
".. ... \n",
"201 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"202 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"203 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"204 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"205 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"\n",
" files \n",
"0 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"1 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"2 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"3 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"4 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
".. ... \n",
"201 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"202 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"203 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"204 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"205 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"\n",
"[206 rows x 7 columns]"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries[\"country_html\"] = countries[\"country_html\"].map(lambda html: \"\".join(html))\n",
"countries"
]
},
{
"cell_type": "markdown",
"id": "3cf0621e-4ed8-44e4-883f-4fdece8ccbef",
"metadata": {},
"source": [
"### remove `<br>` tags"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "ab6166f9-daa2-4591-9989-8d33f3f98533",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:45:51.019933Z",
"iopub.status.busy": "2022-06-25T20:45:51.019577Z",
"iopub.status.idle": "2022-06-25T20:45:51.041851Z",
"shell.execute_reply": "2022-06-25T20:45:51.041008Z",
"shell.execute_reply.started": "2022-06-25T20:45:51.019903Z"
},
"tags": []
},
"outputs": [],
"source": [
"countries[\"country_html\"] = countries[\"country_html\"].map(\n",
" lambda html: html.replace(\"<br>\", \"\")\n",
")"
]
},
{
"cell_type": "markdown",
"id": "ce0e413d-1763-4520-bc36-29715963718c",
"metadata": {},
"source": [
"### add root node"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "df924f80-c239-4a35-bc30-888589b34f0b",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:47:29.097287Z",
"iopub.status.busy": "2022-06-25T20:47:29.096936Z",
"iopub.status.idle": "2022-06-25T20:47:29.109510Z",
"shell.execute_reply": "2022-06-25T20:47:29.108689Z",
"shell.execute_reply.started": "2022-06-25T20:47:29.097258Z"
},
"tags": []
},
"outputs": [],
"source": [
"countries[\"country_html\"] = countries[\"country_html\"].map(\n",
" lambda html: f\"<div>{html}</div>\"\n",
")"
]
},
{
"cell_type": "markdown",
"id": "f5e7249e-988c-420e-bd02-f47d96fd0685",
"metadata": {},
"source": [
"## parse\n",
"\n",
"### parse string as xml"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "691bd250-44b9-4adb-9430-b0ef624c986b",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:47:50.180864Z",
"iopub.status.busy": "2022-06-25T20:47:50.180516Z",
"iopub.status.idle": "2022-06-25T20:47:50.184798Z",
"shell.execute_reply": "2022-06-25T20:47:50.183956Z",
"shell.execute_reply.started": "2022-06-25T20:47:50.180835Z"
},
"tags": []
},
"outputs": [],
"source": [
"parser = etree.XMLParser(recover=True)"
]
},
{
"cell_type": "markdown",
"id": "9967cd92-390a-4b1e-a946-32d46a898eb9",
"metadata": {},
"source": [
"#### Afganistan"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "915c338e-9225-46a3-b833-89fdeb971c88",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:47:51.179025Z",
"iopub.status.busy": "2022-06-25T20:47:51.178417Z",
"iopub.status.idle": "2022-06-25T20:47:51.187219Z",
"shell.execute_reply": "2022-06-25T20:47:51.186016Z",
"shell.execute_reply.started": "2022-06-25T20:47:51.178977Z"
},
"tags": []
},
"outputs": [],
"source": [
"# root = etree.fromstringlist(countries[\"country_html\"].iloc[0], parser)\n",
"root = etree.fromstring(countries[\"country_html\"].iloc[0], parser)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "771453ab-2b00-42e5-a026-3bcea7fc6476",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:47:52.012467Z",
"iopub.status.busy": "2022-06-25T20:47:52.011430Z",
"iopub.status.idle": "2022-06-25T20:47:52.020811Z",
"shell.execute_reply": "2022-06-25T20:47:52.019658Z",
"shell.execute_reply.started": "2022-06-25T20:47:52.012399Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"<Element div at 0x123031f80>"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"root"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "516153ae-95b8-44b0-8bd3-51914ba95f4c",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:47:52.947860Z",
"iopub.status.busy": "2022-06-25T20:47:52.947172Z",
"iopub.status.idle": "2022-06-25T20:47:52.955283Z",
"shell.execute_reply": "2022-06-25T20:47:52.954038Z",
"shell.execute_reply.started": "2022-06-25T20:47:52.947809Z"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"b'<a href=\"/wiki/Kabul\" title=\"Kabul\">Kabul</a>'\n",
"Kabul\n",
"<Element a at 0x122f81100>\n"
]
}
],
"source": [
"for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td/a\"):\n",
" print(etree.tostring(element))\n",
" print(element.text)\n",
" print(element)"
]
},
{
"cell_type": "markdown",
"id": "a9d0a316-fa1b-4b04-9090-ab90f72dee19",
"metadata": {},
"source": [
"#### South Africa"
]
},
{
"cell_type": "code",
"execution_count": 36,
"id": "769b4492-28a0-4454-8e92-806c4e646660",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:47:58.319337Z",
"iopub.status.busy": "2022-06-25T20:47:58.317420Z",
"iopub.status.idle": "2022-06-25T20:47:58.339980Z",
"shell.execute_reply": "2022-06-25T20:47:58.339214Z",
"shell.execute_reply.started": "2022-06-25T20:47:58.319276Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>country_url</th>\n",
" <th>flag_description_url</th>\n",
" <th>short_country_name</th>\n",
" <th>country_html</th>\n",
" <th>flag_html</th>\n",
" <th>file_urls</th>\n",
" <th>files</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>https://en.wikipedia.org/wiki/Central_African_...</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_the_Cent...</td>\n",
" <td>Central African Republic</td>\n",
" <td>&lt;div&gt;&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above ...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of the &lt;a href=\"/wiki/Central_A...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>58</th>\n",
" <td>https://en.wikipedia.org/wiki/South_Africa</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_South_Af...</td>\n",
" <td>South Africa</td>\n",
" <td>&lt;div&gt;&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above ...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of South Africa&lt;/b&gt; was designe...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" country_url \\\n",
"9 https://en.wikipedia.org/wiki/Central_African_... \n",
"58 https://en.wikipedia.org/wiki/South_Africa \n",
"\n",
" flag_description_url \\\n",
"9 https://en.wikipedia.org/wiki/Flag_of_the_Cent... \n",
"58 https://en.wikipedia.org/wiki/Flag_of_South_Af... \n",
"\n",
" short_country_name \\\n",
"9 Central African Republic \n",
"58 South Africa \n",
"\n",
" country_html \\\n",
"9 <div><tr><th colspan=\"2\" class=\"infobox-above ... \n",
"58 <div><tr><th colspan=\"2\" class=\"infobox-above ... \n",
"\n",
" flag_html \\\n",
"9 <p>The <b>flag of the <a href=\"/wiki/Central_A... \n",
"58 <p>The <b>flag of South Africa</b> was designe... \n",
"\n",
" file_urls \\\n",
"9 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"58 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"\n",
" files \n",
"9 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"58 [{'url': 'https://upload.wikimedia.org/wikiped... "
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries[countries[\"short_country_name\"].map(lambda country: \"Africa\" in country)]"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "f06833fe-dee5-42dc-b0ad-71f9759e0fb1",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:21:03.803935Z",
"iopub.status.busy": "2022-06-25T21:21:03.803558Z",
"iopub.status.idle": "2022-06-25T21:21:03.809310Z",
"shell.execute_reply": "2022-06-25T21:21:03.808433Z",
"shell.execute_reply.started": "2022-06-25T21:21:03.803906Z"
},
"tags": []
},
"outputs": [],
"source": [
"# root = etree.fromstringlist(countries[\"country_html\"].iloc[58], parser)\n",
"root = etree.fromstring(countries[\"country_html\"].iloc[58], parser)"
]
},
{
"cell_type": "code",
"execution_count": 69,
"id": "70f25f0c-92bb-43ee-9223-15d1ad9fb691",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:22:53.884677Z",
"iopub.status.busy": "2022-06-25T21:22:53.884343Z",
"iopub.status.idle": "2022-06-25T21:22:53.893752Z",
"shell.execute_reply": "2022-06-25T21:22:53.893051Z",
"shell.execute_reply.started": "2022-06-25T21:22:53.884650Z"
},
"tags": []
},
"outputs": [],
"source": [
"for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td\"):\n",
" for capital in element:\n",
" capital_filter = capital.xpath(\"ul//a\")\n",
" branch_filter = capital.xpath(\"ul//li\")\n",
"\n",
" _capitals = []\n",
" for item in capital_filter:\n",
" if not isinstance(re.search(r\"\\d\", item.text), re.Match):\n",
" _capitals.append(item.text)\n",
" for match in branch_filter:\n",
" branch = match.xpath(\"text()\")\n",
" _capitals.append(re.search(r\"(?:\\()([^\\)]*)\", branch[0])[1])\n",
"\n",
" result = {\n",
" \"index\": 58,\n",
" \"country_name\": countries[\"short_country_name\"].iloc[58],\n",
" \"capital\": _capitals,\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 70,
"id": "4d6fffaa-c08f-4ae9-b1d4-d1d27550a903",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:22:57.761492Z",
"iopub.status.busy": "2022-06-25T21:22:57.761115Z",
"iopub.status.idle": "2022-06-25T21:22:57.767381Z",
"shell.execute_reply": "2022-06-25T21:22:57.766523Z",
"shell.execute_reply.started": "2022-06-25T21:22:57.761462Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'index': 58,\n",
" 'country_name': 'South Africa',\n",
" 'capital': ['Pretoria',\n",
" 'Cape Town',\n",
" 'Bloemfontein',\n",
" 'executive',\n",
" 'legislative',\n",
" 'judicial']}"
]
},
"execution_count": 70,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"result"
]
},
{
"cell_type": "markdown",
"id": "8af8a656-3e07-4de0-933e-012d218c18f9",
"metadata": {},
"source": [
"#### Albania"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "33205ffb-1b50-468b-8d5b-3c7302a8b285",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:48:07.893538Z",
"iopub.status.busy": "2022-06-25T20:48:07.892909Z",
"iopub.status.idle": "2022-06-25T20:48:07.905915Z",
"shell.execute_reply": "2022-06-25T20:48:07.905138Z",
"shell.execute_reply.started": "2022-06-25T20:48:07.893507Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>country_url</th>\n",
" <th>flag_description_url</th>\n",
" <th>short_country_name</th>\n",
" <th>country_html</th>\n",
" <th>flag_html</th>\n",
" <th>file_urls</th>\n",
" <th>files</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>205</th>\n",
" <td>https://en.wikipedia.org/wiki/Albania</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Albania</td>\n",
" <td>Albania</td>\n",
" <td>&lt;div&gt;&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above ...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of Albania&lt;/b&gt; (&lt;a href=\"/wiki/...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" country_url \\\n",
"205 https://en.wikipedia.org/wiki/Albania \n",
"\n",
" flag_description_url short_country_name \\\n",
"205 https://en.wikipedia.org/wiki/Flag_of_Albania Albania \n",
"\n",
" country_html \\\n",
"205 <div><tr><th colspan=\"2\" class=\"infobox-above ... \n",
"\n",
" flag_html \\\n",
"205 <p>The <b>flag of Albania</b> (<a href=\"/wiki/... \n",
"\n",
" file_urls \\\n",
"205 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"\n",
" files \n",
"205 [{'url': 'https://upload.wikimedia.org/wikiped... "
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries[countries[\"short_country_name\"].map(lambda country: \"Albania\" in country)]"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "482c1bae-c55c-43dd-b603-a297140fe428",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:48:15.103731Z",
"iopub.status.busy": "2022-06-25T20:48:15.103380Z",
"iopub.status.idle": "2022-06-25T20:48:15.108883Z",
"shell.execute_reply": "2022-06-25T20:48:15.108029Z",
"shell.execute_reply.started": "2022-06-25T20:48:15.103703Z"
},
"tags": []
},
"outputs": [],
"source": [
"root = etree.fromstring(countries[\"country_html\"].iloc[205], parser)"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "79966a1a-3017-40aa-a639-889770b68419",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:48:17.351871Z",
"iopub.status.busy": "2022-06-25T20:48:17.350564Z",
"iopub.status.idle": "2022-06-25T20:48:17.360415Z",
"shell.execute_reply": "2022-06-25T20:48:17.358988Z",
"shell.execute_reply.started": "2022-06-25T20:48:17.351827Z"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<Element td at 0x123fc9840>\n"
]
}
],
"source": [
"for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td\"):\n",
" print(element)"
]
},
{
"cell_type": "markdown",
"id": "5c316764-016f-49be-a3ad-1485dceb5b0c",
"metadata": {},
"source": [
"#### Sahrawi Arab Democratic Republic"
]
},
{
"cell_type": "code",
"execution_count": 80,
"id": "3dfd7e29-6166-483a-b454-09daadf2ea20",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:26:21.907262Z",
"iopub.status.busy": "2022-06-25T21:26:21.906430Z",
"iopub.status.idle": "2022-06-25T21:26:21.924749Z",
"shell.execute_reply": "2022-06-25T21:26:21.923775Z",
"shell.execute_reply.started": "2022-06-25T21:26:21.907228Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>country_url</th>\n",
" <th>flag_description_url</th>\n",
" <th>short_country_name</th>\n",
" <th>country_html</th>\n",
" <th>flag_html</th>\n",
" <th>file_urls</th>\n",
" <th>files</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>https://en.wikipedia.org/wiki/Sahrawi_Arab_Dem...</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Sahrawi_...</td>\n",
" <td>Sahrawi Arab Democratic Republic</td>\n",
" <td>&lt;div&gt;&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above ...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of Western Sahara&lt;/b&gt; (Arabic: ...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>39</th>\n",
" <td>https://en.wikipedia.org/wiki/United_Arab_Emir...</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_the_Unit...</td>\n",
" <td>United Arab Emirates</td>\n",
" <td>&lt;div&gt;&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above ...</td>\n",
" <td>&lt;p&gt;The &lt;a href=\"/wiki/National_flag\" title=\"Na...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>69</th>\n",
" <td>https://en.wikipedia.org/wiki/Saudi_Arabia</td>\n",
" <td>https://en.wikipedia.org/wiki/Flag_of_Saudi_Ar...</td>\n",
" <td>Saudi Arabia</td>\n",
" <td>&lt;div&gt;&lt;tr&gt;&lt;th colspan=\"2\" class=\"infobox-above ...</td>\n",
" <td>&lt;p&gt;The &lt;b&gt;flag of the Kingdom of Saudi Arabia&lt;...</td>\n",
" <td>[https:////upload.wikimedia.org/wikipedia/comm...</td>\n",
" <td>[{'url': 'https://upload.wikimedia.org/wikiped...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" country_url \\\n",
"20 https://en.wikipedia.org/wiki/Sahrawi_Arab_Dem... \n",
"39 https://en.wikipedia.org/wiki/United_Arab_Emir... \n",
"69 https://en.wikipedia.org/wiki/Saudi_Arabia \n",
"\n",
" flag_description_url \\\n",
"20 https://en.wikipedia.org/wiki/Flag_of_Sahrawi_... \n",
"39 https://en.wikipedia.org/wiki/Flag_of_the_Unit... \n",
"69 https://en.wikipedia.org/wiki/Flag_of_Saudi_Ar... \n",
"\n",
" short_country_name \\\n",
"20 Sahrawi Arab Democratic Republic \n",
"39 United Arab Emirates \n",
"69 Saudi Arabia \n",
"\n",
" country_html \\\n",
"20 <div><tr><th colspan=\"2\" class=\"infobox-above ... \n",
"39 <div><tr><th colspan=\"2\" class=\"infobox-above ... \n",
"69 <div><tr><th colspan=\"2\" class=\"infobox-above ... \n",
"\n",
" flag_html \\\n",
"20 <p>The <b>flag of Western Sahara</b> (Arabic: ... \n",
"39 <p>The <a href=\"/wiki/National_flag\" title=\"Na... \n",
"69 <p>The <b>flag of the Kingdom of Saudi Arabia<... \n",
"\n",
" file_urls \\\n",
"20 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"39 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"69 [https:////upload.wikimedia.org/wikipedia/comm... \n",
"\n",
" files \n",
"20 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"39 [{'url': 'https://upload.wikimedia.org/wikiped... \n",
"69 [{'url': 'https://upload.wikimedia.org/wikiped... "
]
},
"execution_count": 80,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries[countries[\"short_country_name\"].map(lambda country: \"arab\" in country.lower())]"
]
},
{
"cell_type": "code",
"execution_count": 81,
"id": "02e1f53a-9081-4edc-9a61-41da56166274",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:26:26.463033Z",
"iopub.status.busy": "2022-06-25T21:26:26.462631Z",
"iopub.status.idle": "2022-06-25T21:26:26.469880Z",
"shell.execute_reply": "2022-06-25T21:26:26.469125Z",
"shell.execute_reply.started": "2022-06-25T21:26:26.463004Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"country_url https://en.wikipedia.org/wiki/Sahrawi_Arab_Dem...\n",
"flag_description_url https://en.wikipedia.org/wiki/Flag_of_Sahrawi_...\n",
"short_country_name Sahrawi Arab Democratic Republic\n",
"country_html <div><tr><th colspan=\"2\" class=\"infobox-above ...\n",
"flag_html <p>The <b>flag of Western Sahara</b> (Arabic: ...\n",
"file_urls [https:////upload.wikimedia.org/wikipedia/comm...\n",
"files [{'url': 'https://upload.wikimedia.org/wikiped...\n",
"Name: 20, dtype: object"
]
},
"execution_count": 81,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries.iloc[20]"
]
},
{
"cell_type": "code",
"execution_count": 82,
"id": "3b74a848-af64-4ea8-b2b2-3b78c9fb843e",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:26:39.089757Z",
"iopub.status.busy": "2022-06-25T21:26:39.089324Z",
"iopub.status.idle": "2022-06-25T21:26:39.095857Z",
"shell.execute_reply": "2022-06-25T21:26:39.094902Z",
"shell.execute_reply.started": "2022-06-25T21:26:39.089723Z"
},
"tags": []
},
"outputs": [],
"source": [
"root = etree.fromstring(countries[\"country_html\"].iloc[20], parser)"
]
},
{
"cell_type": "code",
"execution_count": 114,
"id": "f8829405-0533-4349-9c99-05e9e1e41607",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:49:34.812182Z",
"iopub.status.busy": "2022-06-25T21:49:34.811795Z",
"iopub.status.idle": "2022-06-25T21:49:34.820550Z",
"shell.execute_reply": "2022-06-25T21:49:34.819761Z",
"shell.execute_reply.started": "2022-06-25T21:49:34.812153Z"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"b'<div class=\"plainlist\"><ul><li><a href=\"/wiki/Laayoune\" title=\"Laayoune\">El Aai&#250;n</a><sup>a</sup> (<i>de jure</i>)</li><li><span class=\"nowrap\"><a href=\"/wiki/Tifariti\" title=\"Tifariti\">Tifariti</a> (<i>de facto</i>)</span></li></ul></div>'\n",
"b'<i>de jure</i>)'\n",
"match de jure\n",
"b'<i>de facto</i>)'\n",
"match de facto\n"
]
}
],
"source": [
"for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td\"):\n",
" for capital in element:\n",
" print(etree.tostring(capital))\n",
" capital_filter = capital.xpath(\"ul//a\")\n",
" branch_filter = capital.xpath(\"ul//li//i\")\n",
"\n",
" _capitals = []\n",
" for item in capital_filter:\n",
" if not isinstance(re.search(r\"\\d\", item.text), re.Match):\n",
" _capitals.append(item.text)\n",
" for match in branch_filter:\n",
" print(etree.tostring(match))\n",
" print(\"match\", match.text)\n",
" if isinstance(re.search(r\"(?:i\\>)([^\\<)]*)\", etree.tostring(match).decode(\"utf-8\")), re.Match):\n",
" _capitals.append(match.text)\n",
" result = {\n",
" \"index\": 20,\n",
" \"country_name\": countries[\"short_country_name\"].iloc[20],\n",
" \"capital\": _capitals,\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 115,
"id": "92d7c4ac-44f9-46e6-b8ae-94dffa276f84",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:49:35.379470Z",
"iopub.status.busy": "2022-06-25T21:49:35.378687Z",
"iopub.status.idle": "2022-06-25T21:49:35.388295Z",
"shell.execute_reply": "2022-06-25T21:49:35.386602Z",
"shell.execute_reply.started": "2022-06-25T21:49:35.379399Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'index': 20,\n",
" 'country_name': 'Sahrawi Arab Democratic Republic',\n",
" 'capital': ['El Aaiún', 'Tifariti', 'de jure', 'de facto']}"
]
},
"execution_count": 115,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"result"
]
},
{
"cell_type": "markdown",
"id": "9d909d88-e086-41fe-a46f-1cd0a21ec625",
"metadata": {},
"source": [
"## define functions"
]
},
{
"cell_type": "code",
"execution_count": 120,
"id": "9263180c-92ed-4f5f-a479-70af90599b75",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T21:51:58.514700Z",
"iopub.status.busy": "2022-06-25T21:51:58.514295Z",
"iopub.status.idle": "2022-06-25T21:51:58.524855Z",
"shell.execute_reply": "2022-06-25T21:51:58.524175Z",
"shell.execute_reply.started": "2022-06-25T21:51:58.514671Z"
},
"tags": []
},
"outputs": [],
"source": [
"def extract_capital_0(index: int, country_name: str, country_html: str):\n",
" result = None\n",
" root = etree.fromstring(country_html, parser)\n",
" \n",
" # matches single capital\n",
" for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td/a\"):\n",
" result = {\"index\": index, \"country_name\": country_name, \"capital\": element.text}\n",
"\n",
" # matches multiple capitals with numbered footnote\n",
" try:\n",
" if result is None:\n",
" for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td\"):\n",
" for capital in element:\n",
" capital_filter = capital.xpath(\"ul//a\")\n",
" branch_filter = capital.xpath(\"ul//li\")\n",
"\n",
" _capitals = []\n",
" for item in capital_filter:\n",
" if not isinstance(re.search(r\"\\d\", item.text), re.Match):\n",
" _capitals.append(item.text)\n",
" for match in branch_filter:\n",
" branch = match.xpath(\"text()\")\n",
" _capitals.append(re.search(r\"(?:\\()([^\\)]*)\", branch[0])[1])\n",
" result = {\n",
" \"index\": index,\n",
" \"country_name\": country_name,\n",
" \"capital\": _capitals,\n",
" }\n",
" except IndexError:\n",
" result = None\n",
" \n",
" # matches mutiple capitals with italic footnote\n",
" if result is None:\n",
" for element in root.xpath(\"//th[text() = 'Capital']/following-sibling::td\"):\n",
" for capital in element:\n",
" capital_filter = capital.xpath(\"ul//a\")\n",
" branch_filter = capital.xpath(\"ul//li//i\")\n",
"\n",
" _capitals = []\n",
" for item in capital_filter:\n",
" if not isinstance(re.search(r\"\\d\", item.text), re.Match):\n",
" _capitals.append(item.text)\n",
" for match in branch_filter:\n",
" if isinstance(re.search(r\"(?:i\\>)([^\\<)]*)\", etree.tostring(match).decode(\"utf-8\")), re.Match):\n",
" _capitals.append(match.text)\n",
" result = {\n",
" \"index\": 20,\n",
" \"country_name\": countries[\"short_country_name\"].iloc[20],\n",
" \"capital\": _capitals,\n",
" }\n",
"\n",
" return result or None"
]
},
{
"cell_type": "code",
"execution_count": 128,
"id": "0b39d1c7-a1ee-4031-a0eb-93ab5ef525ff",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T22:10:22.757906Z",
"iopub.status.busy": "2022-06-25T22:10:22.757413Z",
"iopub.status.idle": "2022-06-25T22:10:22.881155Z",
"shell.execute_reply": "2022-06-25T22:10:22.880194Z",
"shell.execute_reply.started": "2022-06-25T22:10:22.757877Z"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'index': 0, 'country_name': 'Afghanistan', 'capital': None}\n",
"{'index': 1, 'country_name': 'Croatia', 'capital': None}\n",
"{'index': 2, 'country_name': 'Costa Rica', 'capital': None}\n",
"{'index': 3, 'country_name': 'Democratic Republic of the Congo', 'capital': None}\n",
"{'index': 4, 'country_name': 'Comoros', 'capital': None}\n",
"{'index': 5, 'country_name': 'Republic of the Congo', 'capital': None}\n",
"{'index': 6, 'country_name': 'China', 'capital': None}\n",
"{'index': 7, 'country_name': 'Chile', 'capital': None}\n",
"{'index': 8, 'country_name': 'Chad', 'capital': None}\n",
"{'index': 9, 'country_name': 'Central African Republic', 'capital': None}\n",
"{'index': 10, 'country_name': 'Cape Verde', 'capital': None}\n",
"{'index': 11, 'country_name': 'Colombia', 'capital': None}\n",
"{'index': 12, 'country_name': 'Cameroon', 'capital': None}\n",
"{'index': 13, 'country_name': 'Cambodia', 'capital': None}\n",
"{'index': 14, 'country_name': 'Burundi', 'capital': None}\n",
"{'index': 15, 'country_name': 'Transnistria', 'capital': None}\n",
"{'index': 16, 'country_name': 'Canada', 'capital': None}\n",
"{'index': 17, 'country_name': 'Taiwan', 'capital': None}\n",
"{'index': 18, 'country_name': 'South Ossetia', 'capital': None}\n",
"{'index': 19, 'country_name': 'Somaliland', 'capital': None}\n",
"{'index': 20, 'country_name': 'Sahrawi Arab Democratic Republic', 'capital': None}\n",
"{'index': 21, 'country_name': 'Northern Cyprus', 'capital': None}\n",
"{'index': 22, 'country_name': 'Niue', 'capital': None}\n",
"{'index': 23, 'country_name': 'Kosovo', 'capital': None}\n",
"{'index': 24, 'country_name': 'Cook Islands', 'capital': None}\n",
"{'index': 25, 'country_name': 'Republic of Artsakh', 'capital': None}\n",
"{'index': 26, 'country_name': 'Abkhazia', 'capital': None}\n",
"{'index': 27, 'country_name': 'Zimbabwe', 'capital': None}\n",
"{'index': 28, 'country_name': 'Zambia', 'capital': None}\n",
"{'index': 29, 'country_name': 'Yemen', 'capital': None}\n",
"{'index': 30, 'country_name': 'Vietnam', 'capital': None}\n",
"{'index': 31, 'country_name': 'Venezuela', 'capital': None}\n",
"{'index': 32, 'country_name': 'Luhansk People%27s Republic', 'capital': None}\n",
"None\n",
"{'index': 34, 'country_name': 'Vanuatu', 'capital': None}\n",
"{'index': 35, 'country_name': 'Uzbekistan', 'capital': None}\n",
"{'index': 36, 'country_name': 'Uruguay', 'capital': None}\n",
"{'index': 37, 'country_name': 'United Kingdom', 'capital': None}\n",
"{'index': 38, 'country_name': 'United States', 'capital': None}\n",
"{'index': 39, 'country_name': 'United Arab Emirates', 'capital': None}\n",
"{'index': 40, 'country_name': 'Uganda', 'capital': None}\n",
"{'index': 41, 'country_name': 'Tuvalu', 'capital': None}\n",
"{'index': 42, 'country_name': 'Tunisia', 'capital': None}\n",
"{'index': 43, 'country_name': 'Turkmenistan', 'capital': None}\n",
"{'index': 44, 'country_name': 'Tonga', 'capital': None}\n",
"{'index': 45, 'country_name': 'Trinidad and Tobago', 'capital': None}\n",
"{'index': 46, 'country_name': 'Togo', 'capital': None}\n",
"{'index': 47, 'country_name': 'Ukraine', 'capital': None}\n",
"{'index': 48, 'country_name': 'Thailand', 'capital': None}\n",
"{'index': 49, 'country_name': 'Tanzania', 'capital': None}\n",
"{'index': 50, 'country_name': 'Tajikistan', 'capital': None}\n",
"{'index': 51, 'country_name': 'Syria', 'capital': None}\n",
"{'index': 52, 'country_name': 'Switzerland', 'capital': None}\n",
"{'index': 53, 'country_name': 'Sweden', 'capital': None}\n",
"{'index': 54, 'country_name': 'Sudan', 'capital': None}\n",
"{'index': 55, 'country_name': 'Suriname', 'capital': None}\n",
"{'index': 56, 'country_name': 'Sri Lanka', 'capital': None}\n",
"{'index': 57, 'country_name': 'Spain', 'capital': None}\n",
"{'index': 58, 'country_name': 'South Africa', 'capital': None}\n",
"{'index': 59, 'country_name': 'Somalia', 'capital': None}\n",
"{'index': 60, 'country_name': 'Solomon Islands', 'capital': None}\n",
"{'index': 61, 'country_name': 'Slovenia', 'capital': None}\n",
"{'index': 62, 'country_name': 'South Sudan', 'capital': None}\n",
"{'index': 63, 'country_name': 'Slovakia', 'capital': None}\n",
"None\n",
"{'index': 65, 'country_name': 'Sierra Leone', 'capital': None}\n",
"{'index': 66, 'country_name': 'Seychelles', 'capital': None}\n",
"{'index': 67, 'country_name': 'Serbia', 'capital': None}\n",
"{'index': 68, 'country_name': 'Senegal', 'capital': None}\n",
"{'index': 69, 'country_name': 'Saudi Arabia', 'capital': None}\n",
"{'index': 70, 'country_name': 'S%C3%A3o Tom%C3%A9 and Pr%C3%ADncipe', 'capital': None}\n",
"{'index': 71, 'country_name': 'San Marino', 'capital': None}\n",
"{'index': 72, 'country_name': 'Samoa', 'capital': None}\n",
"{'index': 73, 'country_name': 'Saint Vincent and the Grenadines', 'capital': None}\n",
"{'index': 74, 'country_name': 'Saint Lucia', 'capital': None}\n",
"{'index': 75, 'country_name': 'Rwanda', 'capital': None}\n",
"{'index': 76, 'country_name': 'Romania', 'capital': None}\n",
"{'index': 77, 'country_name': 'Portugal', 'capital': None}\n",
"{'index': 78, 'country_name': 'Poland', 'capital': None}\n",
"{'index': 79, 'country_name': 'Saint Kitts and Nevis', 'capital': None}\n",
"{'index': 80, 'country_name': 'Philippines', 'capital': None}\n",
"{'index': 81, 'country_name': 'Qatar', 'capital': None}\n",
"{'index': 82, 'country_name': 'Russia', 'capital': None}\n",
"{'index': 83, 'country_name': 'Peru', 'capital': None}\n",
"{'index': 84, 'country_name': 'Papua New Guinea', 'capital': None}\n",
"{'index': 85, 'country_name': 'Paraguay', 'capital': None}\n",
"{'index': 86, 'country_name': 'Panama', 'capital': None}\n",
"None\n",
"{'index': 88, 'country_name': 'Palau', 'capital': None}\n",
"{'index': 89, 'country_name': 'Oman', 'capital': None}\n",
"{'index': 90, 'country_name': 'North Macedonia', 'capital': None}\n",
"{'index': 91, 'country_name': 'Norway', 'capital': None}\n",
"{'index': 92, 'country_name': 'Pakistan', 'capital': None}\n",
"{'index': 93, 'country_name': 'Nigeria', 'capital': None}\n",
"{'index': 94, 'country_name': 'Niger', 'capital': None}\n",
"{'index': 95, 'country_name': 'Nicaragua', 'capital': None}\n",
"{'index': 96, 'country_name': 'Kingdom of the Netherlands', 'capital': None}\n",
"{'index': 97, 'country_name': 'Nepal', 'capital': None}\n",
"{'index': 98, 'country_name': 'Nauru', 'capital': None}\n",
"{'index': 99, 'country_name': 'New Zealand', 'capital': None}\n",
"{'index': 100, 'country_name': 'Namibia', 'capital': None}\n",
"{'index': 101, 'country_name': 'Myanmar', 'capital': None}\n",
"{'index': 102, 'country_name': 'Mozambique', 'capital': None}\n",
"{'index': 103, 'country_name': 'Morocco', 'capital': None}\n",
"{'index': 104, 'country_name': 'Montenegro', 'capital': None}\n",
"{'index': 105, 'country_name': 'Monaco', 'capital': None}\n",
"{'index': 106, 'country_name': 'Moldova', 'capital': None}\n",
"{'index': 107, 'country_name': 'Federated States of Micronesia', 'capital': None}\n",
"{'index': 108, 'country_name': 'Mexico', 'capital': None}\n",
"{'index': 109, 'country_name': 'Mongolia', 'capital': None}\n",
"{'index': 110, 'country_name': 'Mauritius', 'capital': None}\n",
"{'index': 111, 'country_name': 'Mauritania', 'capital': None}\n",
"{'index': 112, 'country_name': 'Marshall Islands', 'capital': None}\n",
"{'index': 113, 'country_name': 'Malta', 'capital': None}\n",
"{'index': 114, 'country_name': 'Mali', 'capital': None}\n",
"{'index': 115, 'country_name': 'Maldives', 'capital': None}\n",
"{'index': 116, 'country_name': 'Malaysia', 'capital': None}\n",
"{'index': 117, 'country_name': 'Malawi', 'capital': None}\n",
"{'index': 118, 'country_name': 'Madagascar', 'capital': None}\n",
"{'index': 119, 'country_name': 'Luxembourg', 'capital': None}\n",
"{'index': 120, 'country_name': 'Lithuania', 'capital': None}\n",
"{'index': 121, 'country_name': 'Liechtenstein', 'capital': None}\n",
"{'index': 122, 'country_name': 'Libya', 'capital': None}\n",
"{'index': 123, 'country_name': 'Liberia', 'capital': None}\n",
"{'index': 124, 'country_name': 'Lesotho', 'capital': None}\n",
"{'index': 125, 'country_name': 'Lebanon', 'capital': None}\n",
"{'index': 126, 'country_name': 'Latvia', 'capital': None}\n",
"{'index': 127, 'country_name': 'Laos', 'capital': None}\n",
"{'index': 128, 'country_name': 'Kyrgyzstan', 'capital': None}\n",
"{'index': 129, 'country_name': 'Kuwait', 'capital': None}\n",
"{'index': 130, 'country_name': 'South Korea', 'capital': None}\n",
"{'index': 131, 'country_name': 'North Korea', 'capital': None}\n",
"{'index': 132, 'country_name': 'Kiribati', 'capital': None}\n",
"{'index': 133, 'country_name': 'Kenya', 'capital': None}\n",
"{'index': 134, 'country_name': 'Kazakhstan', 'capital': None}\n",
"{'index': 135, 'country_name': 'Jordan', 'capital': None}\n",
"{'index': 136, 'country_name': 'Japan', 'capital': None}\n",
"{'index': 137, 'country_name': 'Jamaica', 'capital': None}\n",
"{'index': 138, 'country_name': 'Ivory Coast', 'capital': None}\n",
"{'index': 139, 'country_name': 'Italy', 'capital': None}\n",
"{'index': 140, 'country_name': 'Israel', 'capital': None}\n",
"{'index': 141, 'country_name': 'Republic of Ireland', 'capital': None}\n",
"{'index': 142, 'country_name': 'Iraq', 'capital': None}\n",
"{'index': 143, 'country_name': 'Iran', 'capital': None}\n",
"{'index': 144, 'country_name': 'Indonesia', 'capital': None}\n",
"{'index': 145, 'country_name': 'India', 'capital': None}\n",
"{'index': 146, 'country_name': 'Iceland', 'capital': None}\n",
"{'index': 147, 'country_name': 'Hungary', 'capital': None}\n",
"{'index': 148, 'country_name': 'Honduras', 'capital': None}\n",
"{'index': 149, 'country_name': 'Guyana', 'capital': None}\n",
"{'index': 150, 'country_name': 'Haiti', 'capital': None}\n",
"{'index': 151, 'country_name': 'Guinea-Bissau', 'capital': None}\n",
"{'index': 152, 'country_name': 'Guinea', 'capital': None}\n",
"{'index': 153, 'country_name': 'Guatemala', 'capital': None}\n",
"{'index': 154, 'country_name': 'Grenada', 'capital': None}\n",
"{'index': 155, 'country_name': 'Greece', 'capital': None}\n",
"{'index': 156, 'country_name': 'Ghana', 'capital': None}\n",
"{'index': 157, 'country_name': 'Germany', 'capital': None}\n",
"{'index': 158, 'country_name': 'Georgia (country)', 'capital': None}\n",
"{'index': 159, 'country_name': 'The Gambia', 'capital': None}\n",
"{'index': 160, 'country_name': 'Gabon', 'capital': None}\n",
"{'index': 161, 'country_name': 'Finland', 'capital': None}\n",
"{'index': 162, 'country_name': 'Fiji', 'capital': None}\n",
"{'index': 163, 'country_name': 'Ethiopia', 'capital': None}\n",
"{'index': 164, 'country_name': 'Eswatini', 'capital': None}\n",
"{'index': 165, 'country_name': 'Estonia', 'capital': None}\n",
"{'index': 166, 'country_name': 'France', 'capital': None}\n",
"{'index': 167, 'country_name': 'Eritrea', 'capital': None}\n",
"{'index': 168, 'country_name': 'Equatorial Guinea', 'capital': None}\n",
"{'index': 169, 'country_name': 'El Salvador', 'capital': None}\n",
"{'index': 170, 'country_name': 'Egypt', 'capital': None}\n",
"{'index': 171, 'country_name': 'Ecuador', 'capital': None}\n",
"{'index': 172, 'country_name': 'East Timor', 'capital': None}\n",
"{'index': 173, 'country_name': 'Dominica', 'capital': None}\n",
"{'index': 174, 'country_name': 'Dominican Republic', 'capital': None}\n",
"{'index': 175, 'country_name': 'Djibouti', 'capital': None}\n",
"{'index': 176, 'country_name': 'Danish Realm', 'capital': None}\n",
"{'index': 177, 'country_name': 'Czech Republic', 'capital': None}\n",
"{'index': 178, 'country_name': 'Cyprus', 'capital': None}\n",
"{'index': 179, 'country_name': 'Cuba', 'capital': None}\n",
"{'index': 180, 'country_name': 'Burkina Faso', 'capital': None}\n",
"{'index': 181, 'country_name': 'Bulgaria', 'capital': None}\n",
"{'index': 182, 'country_name': 'Brunei', 'capital': None}\n",
"{'index': 183, 'country_name': 'Brazil', 'capital': None}\n",
"{'index': 184, 'country_name': 'Botswana', 'capital': None}\n",
"{'index': 185, 'country_name': 'Bosnia and Herzegovina', 'capital': None}\n",
"{'index': 186, 'country_name': 'Bolivia', 'capital': None}\n",
"{'index': 187, 'country_name': 'Bhutan', 'capital': None}\n",
"{'index': 188, 'country_name': 'Benin', 'capital': None}\n",
"{'index': 189, 'country_name': 'Belize', 'capital': None}\n",
"{'index': 190, 'country_name': 'Belgium', 'capital': None}\n",
"{'index': 191, 'country_name': 'Belarus', 'capital': None}\n",
"{'index': 192, 'country_name': 'Barbados', 'capital': None}\n",
"{'index': 193, 'country_name': 'Bangladesh', 'capital': None}\n",
"{'index': 194, 'country_name': 'Bahrain', 'capital': None}\n",
"{'index': 195, 'country_name': 'The Bahamas', 'capital': None}\n",
"{'index': 196, 'country_name': 'Azerbaijan', 'capital': None}\n",
"{'index': 197, 'country_name': 'Austria', 'capital': None}\n",
"{'index': 198, 'country_name': 'Australia', 'capital': None}\n",
"{'index': 199, 'country_name': 'Armenia', 'capital': None}\n",
"{'index': 200, 'country_name': 'Argentina', 'capital': None}\n",
"{'index': 201, 'country_name': 'Antigua and Barbuda', 'capital': '\\n'}\n",
"{'index': 202, 'country_name': 'Angola', 'capital': None}\n",
"{'index': 203, 'country_name': 'Andorra', 'capital': None}\n",
"{'index': 204, 'country_name': 'Algeria', 'capital': None}\n",
"{'index': 205, 'country_name': 'Albania', 'capital': None}\n"
]
}
],
"source": [
"for index, country_name, country_html in zip(\n",
" countries.index, countries[\"short_country_name\"], countries[\"country_html\"]\n",
"):\n",
" # print(json.dumps(extract_capital_0(index, country_name, country_html)))\n",
" print(extract_capital_0(index, country_name, country_html))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c147ae2-cc6f-4ac9-9c6a-81fb8e71d1d1",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "4cc8e0f7-36ab-41b8-8ce9-d0ebe02a9896",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "914e0498-c75d-4c40-8320-ec62f5db8764",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T20:16:19.646094Z",
"iopub.status.busy": "2022-06-25T20:16:19.645699Z",
"iopub.status.idle": "2022-06-25T20:16:19.649925Z",
"shell.execute_reply": "2022-06-25T20:16:19.648982Z",
"shell.execute_reply.started": "2022-06-25T20:16:19.646064Z"
}
},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 126,
"id": "57d8b5a9-dd20-4214-866a-b57ecc7ad5da",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T22:09:14.820234Z",
"iopub.status.busy": "2022-06-25T22:09:14.819573Z",
"iopub.status.idle": "2022-06-25T22:09:14.830451Z",
"shell.execute_reply": "2022-06-25T22:09:14.829525Z",
"shell.execute_reply.started": "2022-06-25T22:09:14.820174Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"country_url https://en.wikipedia.org/wiki/Malaysia\n",
"flag_description_url https://en.wikipedia.org/wiki/Flag_of_Malaysia\n",
"short_country_name Malaysia\n",
"country_html <div><tr><th colspan=\"2\" class=\"infobox-above ...\n",
"flag_html <p>The <a href=\"/wiki/National_flag\" title=\"Na...\n",
"file_urls [https:////upload.wikimedia.org/wikipedia/comm...\n",
"files [{'url': 'https://upload.wikimedia.org/wikiped...\n",
"Name: 116, dtype: object"
]
},
"execution_count": 126,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries.iloc[116]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2f9b1bf-e60a-44d0-9a77-bdc44daeb838",
"metadata": {
"execution": {
"iopub.execute_input": "2022-06-25T22:17:26.496436Z",
"iopub.status.busy": "2022-06-25T22:17:26.494903Z"
},
"tags": []
},
"outputs": [
{
"ename": "TypeError",
"evalue": "expected string or bytes-like object",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [131]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m index \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m116\u001b[39m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mextract_capital_0\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcountries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miloc\u001b[49m\u001b[43m[\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshort_country_name\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcountries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miloc\u001b[49m\u001b[43m[\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcountry_html\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
"Input \u001b[0;32mIn [120]\u001b[0m, in \u001b[0;36mextract_capital_0\u001b[0;34m(index, country_name, country_html)\u001b[0m\n\u001b[1;32m 17\u001b[0m _capitals \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m capital_filter:\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[43mre\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\\\u001b[39;49m\u001b[38;5;124;43md\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mitem\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m, re\u001b[38;5;241m.\u001b[39mMatch):\n\u001b[1;32m 20\u001b[0m _capitals\u001b[38;5;241m.\u001b[39mappend(item\u001b[38;5;241m.\u001b[39mtext)\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m match \u001b[38;5;129;01min\u001b[39;00m branch_filter:\n",
"File \u001b[0;32m~/.pyenv/versions/3.8.12/lib/python3.8/re.py:201\u001b[0m, in \u001b[0;36msearch\u001b[0;34m(pattern, string, flags)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msearch\u001b[39m(pattern, string, flags\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m):\n\u001b[1;32m 199\u001b[0m \u001b[38;5;124;03m\"\"\"Scan through string looking for a match to the pattern, returning\u001b[39;00m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;124;03m a Match object, or None if no match was found.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 201\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_compile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpattern\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstring\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mTypeError\u001b[0m: expected string or bytes-like object"
]
}
],
"source": [
"index = 116\n",
"extract_capital_0(index, countries.iloc[index][\"short_country_name\"], countries.iloc[index][\"country_html\"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e066fde4-b290-49d7-80d2-64e6003efcec",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink