dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: update flags spider

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-26 17:27:37 +01:00
parent 42df945c25
commit b5eec4550d
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/pipelines.py
View File

@@ -47,5 +47,5 @@ class FlagDownloadFilesPipeline(FilesPipeline):
flag_filename = re.search(r"([^\/]*)$", request.url) flag_filename = re.search(r"([^\/]*)$", request.url)
if isinstance(flag_filename, re.Match): if isinstance(flag_filename, re.Match):
if filename := flag_filename[1].endswith(".svg"): if (filename := flag_filename[1]).endswith(".svg"):
return f"files/flags/{filename}" return f"files/flags/{filename}"

4
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/spiders/flags.py
View File

@@ -36,7 +36,7 @@ class FlagsSpider(scrapy.Spider):
def get_country_page(self, response: TextResponse, country_name: str): def get_country_page(self, response: TextResponse, country_name: str):
flag_data = response.xpath( flag_data = response.xpath(
"//table[contains(@class, 'infobox')]/tbody/tr/td/div/div[1]/div/a[not(contains(@href, 'cite_note'))]/@href" "//table[contains(@class, 'infobox')]/tbody/tr/td/div/div/div/a[not(contains(@href, 'cite_note'))]/@href"
).getall() ).getall()
flag_image_url = flag_data[0] flag_image_url = flag_data[0]
@@ -70,6 +70,6 @@ class FlagsSpider(scrapy.Spider):
flags_item = FlagsItem() flags_item = FlagsItem()
flags_item["country_name"] = country_name flags_item["country_name"] = country_name
flags_item["file_urls"] = [f"https://{flag_image_result}"] flags_item["file_urls"] = [f"https:{flag_image_result}"]
yield flags_item yield flags_item