dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: disable dev filter for downloads

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-23 02:30:46 +01:00
parent 96c99fdb1f
commit a59d34d180
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/spiders/countrydownloader.py
View File

@@ -23,8 +23,8 @@ class CountrydownloaderSpider(scrapy.Spider):
"//table[contains(@class, 'sortable') and contains(@class, 'wikitable')]/tbody/tr[not(contains(@style, 'background'))]/td[1 and contains(@style, 'vertical-align:top;')]/b/a/@href" "//table[contains(@class, 'sortable') and contains(@class, 'wikitable')]/tbody/tr[not(contains(@style, 'background'))]/td[1 and contains(@style, 'vertical-align:top;')]/b/a/@href"
).getall() ).getall()
# for url in country_urls_xpath: for url in country_urls_xpath:
for url in country_urls_xpath[:3]: # for url in country_urls_xpath[:3]:
regex_match = re.search(r"\/wiki\/(?P<short_country_name>[^$]*)", url) regex_match = re.search(r"\/wiki\/(?P<short_country_name>[^$]*)", url)
yield scrapy.Request( yield scrapy.Request(
url=f"https://en.wikipedia.org{url}", url=f"https://en.wikipedia.org{url}",