dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: change indent

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-24 21:00:00 +01:00
parent 34d6980cac
commit 3108ab1c1f
3 changed files with 18 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/settings.py
View File

@@ -97,5 +97,5 @@ FEEDS = {
/ "data" / "data"
/ "scrapy" / "scrapy"
/ "raw_country_data" / "raw_country_data"
/ "countries.json": {"format": "json", "encoding": "utf8", "store_empty": False} / "countries.json": {"format": "json", "encoding": "utf8", "store_empty": False, "indent": 2}
} }

2
docs/scraping.md
View File

@@ -68,7 +68,7 @@ response
- Extract data using xpath: - Extract data using xpath:
``` ```
countries = response.xpath("//table[contains(@class, 'sortable') and contains(@class, 'wikitable')]/tbody/tr[not(contains(@style, 'background'))]/td[1 and contains(@style, 'vertical-align:top;')]/b/a/@href") countries = response.xpath("//table[contains(@class, 'sortable') and contains(@class, 'wikitable')]/tbody/tr[not(contains(@style, 'background'))]/td[1 and contains(@style, 'vertical-align:top;')]/b/a/@href").getall()
countries[0] countries[0]
``` ```

214
playground/downloaded_data_inspection_lab/Untitled.ipynb
View File

@@ -2,16 +2,9 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "d8185790-0793-4881-99e8-6730f95a8006", "id": "d8185790-0793-4881-99e8-6730f95a8006",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-23T23:01:31.065905Z",
"iopub.status.busy": "2022-06-23T23:01:31.065496Z",
"iopub.status.idle": "2022-06-23T23:01:31.544067Z",
"shell.execute_reply": "2022-06-23T23:01:31.542907Z",
"shell.execute_reply.started": "2022-06-23T23:01:31.065831Z"
},
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [],
@@ -24,30 +17,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": null,
"id": "ea2b3e33-d58e-4e30-a0cc-8218a1f252c9", "id": "ea2b3e33-d58e-4e30-a0cc-8218a1f252c9",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-23T23:01:31.792796Z",
"iopub.status.busy": "2022-06-23T23:01:31.792121Z",
"iopub.status.idle": "2022-06-23T23:01:31.808265Z",
"shell.execute_reply": "2022-06-23T23:01:31.807317Z",
"shell.execute_reply.started": "2022-06-23T23:01:31.792751Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"[None]"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"pd_options = {\n", "pd_options = {\n",
" \"display.max_rows\": None,\n", " \"display.max_rows\": None,\n",
@@ -58,16 +33,9 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 24, "execution_count": null,
"id": "36149580-91d9-431d-99c3-51feee829e79", "id": "36149580-91d9-431d-99c3-51feee829e79",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T00:47:50.283172Z",
"iopub.status.busy": "2022-06-24T00:47:50.282750Z",
"iopub.status.idle": "2022-06-24T00:47:50.301549Z",
"shell.execute_reply": "2022-06-24T00:47:50.300747Z",
"shell.execute_reply.started": "2022-06-24T00:47:50.283143Z"
},
"tags": [] "tags": []
}, },
"outputs": [], "outputs": [],
@@ -79,33 +47,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 36, "execution_count": null,
"id": "d03be94e-8642-4916-8a43-1711e0c21b36", "id": "d03be94e-8642-4916-8a43-1711e0c21b36",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T01:28:21.599365Z",
"iopub.status.busy": "2022-06-24T01:28:21.598512Z",
"iopub.status.idle": "2022-06-24T01:28:21.717735Z",
"shell.execute_reply": "2022-06-24T01:28:21.715894Z",
"shell.execute_reply.started": "2022-06-24T01:28:21.599295Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"ename": "JSONDecodeError",
"evalue": "Extra data: line 83 column 2 (char 2294639)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [36]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m countries_file \u001b[38;5;241m=\u001b[39m data_directory \u001b[38;5;241m/\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcountries.json\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 2\u001b[0m countries \u001b[38;5;241m=\u001b[39m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcountries_file\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_text\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.pyenv/versions/3.8.12/lib/python3.8/json/__init__.py:357\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m kw[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mencoding\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 355\u001b[0m parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 356\u001b[0m parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 357\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 358\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 359\u001b[0m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
"File \u001b[0;32m~/.pyenv/versions/3.8.12/lib/python3.8/json/decoder.py:340\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 338\u001b[0m end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n\u001b[0;32m--> 340\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExtra data\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, end)\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n",
"\u001b[0;31mJSONDecodeError\u001b[0m: Extra data: line 83 column 2 (char 2294639)"
]
}
],
"source": [ "source": [
"countries_file = data_directory / \"countries.json\"\n", "countries_file = data_directory / \"countries.json\"\n",
"countries = json.loads(countries_file.read_text())\n", "countries = json.loads(countries_file.read_text())\n",
@@ -114,67 +61,24 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 37, "execution_count": null,
"id": "29cca9ea-16d3-4534-8c9e-49fde37f8cdd", "id": "29cca9ea-16d3-4534-8c9e-49fde37f8cdd",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T01:28:23.587271Z",
"iopub.status.busy": "2022-06-24T01:28:23.586601Z",
"iopub.status.idle": "2022-06-24T01:28:23.803584Z",
"shell.execute_reply": "2022-06-24T01:28:23.801073Z",
"shell.execute_reply.started": "2022-06-24T01:28:23.587242Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"ename": "ValueError",
"evalue": "Trailing data",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_json\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcountries_file\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/util/_decorators.py:207\u001b[0m, in \u001b[0;36mdeprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 206\u001b[0m kwargs[new_arg_name] \u001b[38;5;241m=\u001b[39m new_arg_value\n\u001b[0;32m--> 207\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/util/_decorators.py:311\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m>\u001b[39m num_allow_args:\n\u001b[1;32m 306\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m 307\u001b[0m msg\u001b[38;5;241m.\u001b[39mformat(arguments\u001b[38;5;241m=\u001b[39marguments),\n\u001b[1;32m 308\u001b[0m \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[1;32m 309\u001b[0m stacklevel\u001b[38;5;241m=\u001b[39mstacklevel,\n\u001b[1;32m 310\u001b[0m )\n\u001b[0;32m--> 311\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/io/json/_json.py:612\u001b[0m, in \u001b[0;36mread_json\u001b[0;34m(path_or_buf, orient, typ, dtype, convert_axes, convert_dates, keep_default_dates, numpy, precise_float, date_unit, encoding, encoding_errors, lines, chunksize, compression, nrows, storage_options)\u001b[0m\n\u001b[1;32m 609\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m json_reader\n\u001b[1;32m 611\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m json_reader:\n\u001b[0;32m--> 612\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mjson_reader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/io/json/_json.py:746\u001b[0m, in \u001b[0;36mJsonReader.read\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 744\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_object_parser(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_combine_lines(data_lines))\n\u001b[1;32m 745\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 746\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_object_parser\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 747\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n\u001b[1;32m 748\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/io/json/_json.py:768\u001b[0m, in \u001b[0;36mJsonReader._get_object_parser\u001b[0;34m(self, json)\u001b[0m\n\u001b[1;32m 766\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 767\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m typ \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mframe\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 768\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[43mFrameParser\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 770\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m typ \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mseries\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m obj \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 771\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(dtype, \u001b[38;5;28mbool\u001b[39m):\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/io/json/_json.py:880\u001b[0m, in \u001b[0;36mParser.parse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 878\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parse_numpy()\n\u001b[1;32m 879\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 880\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_parse_no_numpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 883\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
"File \u001b[0;32m~/git-repos/geography-anki/playground/downloaded_data_inspection/.venv/lib/python3.8/site-packages/pandas/io/json/_json.py:1133\u001b[0m, in \u001b[0;36mFrameParser._parse_no_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1129\u001b[0m orient \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morient\n\u001b[1;32m 1131\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m orient \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcolumns\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 1132\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj \u001b[38;5;241m=\u001b[39m DataFrame(\n\u001b[0;32m-> 1133\u001b[0m \u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprecise_float\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprecise_float\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 1134\u001b[0m )\n\u001b[1;32m 1135\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m orient \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msplit\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 1136\u001b[0m decoded \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1137\u001b[0m \u001b[38;5;28mstr\u001b[39m(k): v\n\u001b[1;32m 1138\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m loads(json, precise_float\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprecise_float)\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1139\u001b[0m }\n",
"\u001b[0;31mValueError\u001b[0m: Trailing data"
]
}
],
"source": [ "source": [
"df = pd.read_json(countries_file)" "df = pd.read_json(countries_file)"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 33, "execution_count": null,
"id": "ef8bc3ce-08dd-4260-807c-2616b2e1c1ba", "id": "ef8bc3ce-08dd-4260-807c-2616b2e1c1ba",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T00:48:51.018167Z",
"iopub.status.busy": "2022-06-24T00:48:51.017745Z",
"iopub.status.idle": "2022-06-24T00:48:51.023756Z",
"shell.execute_reply": "2022-06-24T00:48:51.022902Z",
"shell.execute_reply.started": "2022-06-24T00:48:51.018137Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"(81,)\n",
"[False]\n",
"[False]\n"
]
}
],
"source": [ "source": [
"country_url = df[\"country_url\"]\n", "country_url = df[\"country_url\"]\n",
"print(country_url.shape)\n", "print(country_url.shape)\n",
@@ -184,29 +88,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 34, "execution_count": null,
"id": "48db8f93-659b-45a4-8477-a7cec139bebc", "id": "48db8f93-659b-45a4-8477-a7cec139bebc",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T00:48:52.316175Z",
"iopub.status.busy": "2022-06-24T00:48:52.315575Z",
"iopub.status.idle": "2022-06-24T00:48:52.323965Z",
"shell.execute_reply": "2022-06-24T00:48:52.323184Z",
"shell.execute_reply.started": "2022-06-24T00:48:52.316146Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"(81,)\n",
"[False]\n",
"[False]\n"
]
}
],
"source": [ "source": [
"short_country_name = df[\"short_country_name\"]\n", "short_country_name = df[\"short_country_name\"]\n",
"print(short_country_name.shape)\n", "print(short_country_name.shape)\n",
@@ -216,29 +103,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": null,
"id": "a52f6aa2-5bbd-46e4-9b2f-cdbd7269cb6e", "id": "a52f6aa2-5bbd-46e4-9b2f-cdbd7269cb6e",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T00:47:57.991196Z",
"iopub.status.busy": "2022-06-24T00:47:57.990582Z",
"iopub.status.idle": "2022-06-24T00:47:58.001189Z",
"shell.execute_reply": "2022-06-24T00:47:57.999654Z",
"shell.execute_reply.started": "2022-06-24T00:47:57.991142Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"(81,)\n",
"[False]\n",
"[False]\n"
]
}
],
"source": [ "source": [
"flag_description = df[\"flag_description\"]\n", "flag_description = df[\"flag_description\"]\n",
"print(flag_description.shape)\n", "print(flag_description.shape)\n",
@@ -248,64 +118,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 30, "execution_count": null,
"id": "643e6512-1e5b-4eb2-9f0a-6b680ada787b", "id": "643e6512-1e5b-4eb2-9f0a-6b680ada787b",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2022-06-24T00:47:59.412325Z",
"iopub.status.busy": "2022-06-24T00:47:59.411973Z",
"iopub.status.idle": "2022-06-24T00:47:59.420681Z",
"shell.execute_reply": "2022-06-24T00:47:59.419781Z",
"shell.execute_reply.started": "2022-06-24T00:47:59.412296Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"7\n"
]
},
{
"data": {
"text/plain": [
"[[{'url': 'https://upload.wikimedia.org/wikipedia/commons/5/5c/Flag_of_the_Taliban.svg',\n",
" 'path': 'files/flags/Flag_of_the_Taliban.svg',\n",
" 'checksum': '153b7b9dc8133d542e744f5ff6102710',\n",
" 'status': 'downloaded'}],\n",
" [{'url': 'https://upload.wikimedia.org/wikipedia/commons/0/01/Flag_of_Niue.svg',\n",
" 'path': 'files/flags/Flag_of_Niue.svg',\n",
" 'checksum': 'ce971e9afe79c9a63fd706a617b34ce2',\n",
" 'status': 'downloaded'}],\n",
" [{'url': 'https://upload.wikimedia.org/wikipedia/commons/4/4e/Flag_of_Uganda.svg',\n",
" 'path': 'files/flags/Flag_of_Uganda.svg',\n",
" 'checksum': 'f8bb736e5832232610b5b65dd3c0a121',\n",
" 'status': 'downloaded'}],\n",
" [{'url': 'https://upload.wikimedia.org/wikipedia/commons/e/e4/Flag_of_the_Federated_States_of_Micronesia.svg',\n",
" 'path': 'files/flags/Flag_of_the_Federated_States_of_Micronesia.svg',\n",
" 'checksum': 'cfc3756759f4002983b49217456fc8e4',\n",
" 'status': 'downloaded'}],\n",
" [{'url': 'https://upload.wikimedia.org/wikipedia/commons/4/49/Flag_of_Kenya.svg',\n",
" 'path': 'files/flags/Flag_of_Kenya.svg',\n",
" 'checksum': 'aa572e0e7ad47c23e37633f1b370da8d',\n",
" 'status': 'downloaded'}],\n",
" [{'url': 'https://upload.wikimedia.org/wikipedia/commons/9/91/Flag_of_Bhutan.svg',\n",
" 'path': 'files/flags/Flag_of_Bhutan.svg',\n",
" 'checksum': 'ce4684f240e15637d2c67eb222d63fe5',\n",
" 'status': 'downloaded'}],\n",
" [{'url': 'https://upload.wikimedia.org/wikipedia/commons/8/85/Flag_of_Belarus.svg',\n",
" 'path': 'files/flags/Flag_of_Belarus.svg',\n",
" 'checksum': '22ec6af94d36453ca6e7c0830000a6c1',\n",
" 'status': 'downloaded'}]]"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"no_anthem = [item for item in [df[\"files\"]][0] if len(item) == 1]\n", "no_anthem = [item for item in [df[\"files\"]][0] if len(item) == 1]\n",
"print(len(no_anthem))\n", "print(len(no_anthem))\n",