dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: update filter middleware

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-26 23:57:07 +01:00
parent a07bb6f5bd
commit 5cc9297a01
2 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/middlewares.py
View File

@@ -105,5 +105,11 @@ class WikipediaCountryScraperDownloaderMiddleware:
class WikipediaCountryScraperDecodeURLMiddleware: class WikipediaCountryScraperDecodeURLMiddleware:
def process_request(self, request, spider): def process_request(self, request, spider):
# https://www.w3schools.com/tags/ref_urlencode.ASP
request._url = request.url.replace("%27", "'")
request._url = request.url.replace("%28", "(") request._url = request.url.replace("%28", "(")
request._url = request.url.replace("%29", ")") request._url = request.url.replace("%29", ")")
request._url = request.url.replace("%C3%A7", "ç")
request._url = request.url.replace("%C3%B1", "ñ")
request._url = request.url.replace("%C3%B4", "ô")
request._url = request.url.replace("%C3%85", "Å")

1
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/pipelines.py
View File

@@ -3,7 +3,6 @@
# Don't forget to add your pipeline to the ITEM_PIPELINES setting # Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import urllib
import re import re
# useful for handling different item types with a single interface # useful for handling different item types with a single interface