dtomlinson/geography-anki
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: add remove citations to item loader

Browse Source
This commit is contained in:
Daniel Tomlinson
2022-06-26 22:26:49 +01:00
parent 6fcae89c5d
commit 55e4ff7722
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
01_scrapy/wikipedia_country_scraper/wikipedia_country_scraper/itemloaders.py
View File

@@ -1,12 +1,15 @@
import re
from itemloaders.processors import TakeFirst, MapCompose from itemloaders.processors import TakeFirst, MapCompose
from scrapy.loader import ItemLoader from scrapy.loader import ItemLoader
from w3lib.html import remove_tags from w3lib.html import remove_tags
class FlagsItemLoader(ItemLoader): class FlagsItemLoader(ItemLoader):
default_output_processor = TakeFirst() default_output_processor = TakeFirst()
# country_name - convert "_" to " " # country_name - convert "_" to " "
country_name_in = MapCompose(lambda country: country.replace("_", " ")) country_name_in = MapCompose(lambda country: country.replace("_", " "))
# flag_description - remove html tags # flag_description - remove html tags, remove citations [1]
flag_description_in = MapCompose(remove_tags) flag_description_in = MapCompose(remove_tags, lambda line: re.sub(r"(\[\d+\])", "", line))