dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: add json feeds output

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-22 21:46:33 +01:00
parent 522f766b49
commit 97be860627
2 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
01_scrapy/wikipedia_country_scraper/download_countries.sh
View File

@@ -1,2 +1,2 @@
source .venv/bin/activate source .venv/bin/activate
scrapy crawl CountrydownloaderSpider -O ../../data/scrapy/raw_country_data/countries.json scrapy crawl CountrydownloaderSpider

8
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/settings.py
View File

@@ -91,3 +91,11 @@ FILES_STORE = str(pathlib.Path(__file__).resolve().parents[3] / "data" / "scrapy
# HTTPCACHE_DIR = 'httpcache' # HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = [] # HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
FEEDS = {
pathlib.Path(__file__).resolve().parents[3]
/ "data"
/ "scrapy"
/ "raw_country_data"
/ "countries.json": {"format": "json", "encoding": "utf8", "store_empty": True, "indent": 2}
}