dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: move filter into pipeline

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-27 00:05:43 +01:00
parent 5cc9297a01
commit 747c0e8d79
2 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/middlewares.py
View File

@@ -109,7 +109,7 @@ class WikipediaCountryScraperDecodeURLMiddleware:
request._url = request.url.replace("%27", "'") request._url = request.url.replace("%27", "'")
request._url = request.url.replace("%28", "(") request._url = request.url.replace("%28", "(")
request._url = request.url.replace("%29", ")") request._url = request.url.replace("%29", ")")
request._url = request.url.replace("%C3%A7", "ç") # request._url = request.url.replace("%C3%A7", "ç")
request._url = request.url.replace("%C3%B1", "ñ") # request._url = request.url.replace("%C3%B1", "ñ")
request._url = request.url.replace("%C3%B4", "ô") # request._url = request.url.replace("%C3%B4", "ô")
request._url = request.url.replace("%C3%85", "Å") # request._url = request.url.replace("%C3%85", "Å")

7
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/pipelines.py
View File

@@ -42,4 +42,11 @@ class FlagDownloadFilesPipeline(FilesPipeline):
if isinstance(flag_filename, re.Match): if isinstance(flag_filename, re.Match):
if (filename := flag_filename[1]).endswith(("svg", "png")): if (filename := flag_filename[1]).endswith(("svg", "png")):
# filename = filename.replace("%27", "'")
# filename = filename.replace("%28", "(")
# filename = filename.replace("%29", ")")
filename = filename.replace("%C3%A7", "ç")
filename = filename.replace("%C3%B1", "ñ")
filename = filename.replace("%C3%B4", "ô")
filename = filename.replace("%C3%85", "Å")
return f"files/flags/{filename}" return f"files/flags/{filename}"