52 lines
1.8 KiB
Python
52 lines
1.8 KiB
Python
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
|
|
|
import re
|
|
|
|
# useful for handling different item types with a single interface
|
|
from itemadapter import ItemAdapter
|
|
from scrapy.pipelines.files import FilesPipeline
|
|
|
|
|
|
class WikipediaCountryScraperPipeline:
|
|
def process_item(self, item, spider):
|
|
return item
|
|
|
|
|
|
class WikipediaCountryScraperFilesPipeline(FilesPipeline):
|
|
def file_path(self, request, response=None, info=None, *, item=None):
|
|
print(f"request URLs: {request.url}")
|
|
flag_filename = re.search(r"([^\/]*)$", request.url)
|
|
|
|
if isinstance(flag_filename, re.Match):
|
|
if (filename := flag_filename[1]).endswith(".svg"):
|
|
return f"files/flags/{filename}"
|
|
elif filename.endswith(".ogg") or filename.endswith("oga"):
|
|
return f"files/anthems/{filename}"
|
|
|
|
|
|
class AnthemDownloadFilesPipeline(FilesPipeline):
|
|
def file_path(self, request, response=None, info=None, *, item=None):
|
|
flag_filename = re.search(r"([^\/]*)$", request.url)
|
|
|
|
if isinstance(flag_filename, re.Match):
|
|
if (
|
|
(filename := flag_filename[1]).endswith("ogg")
|
|
or filename.endswith("oga")
|
|
or filename.endswith("mp3")
|
|
or filename.endswith("wav")
|
|
):
|
|
return f"files/anthems/{filename}"
|
|
|
|
|
|
class FlagDownloadFilesPipeline(FilesPipeline):
|
|
def file_path(self, request, response=None, info=None, *, item=None):
|
|
flag_filename = re.search(r"([^\/]*)$", request.url)
|
|
|
|
if isinstance(flag_filename, re.Match):
|
|
if filename := flag_filename[1].endswith(".svg"):
|
|
return f"files/flags/{filename}"
|