diff --git a/slack-bot/__pycache__/pullTrafficInfo.cpython-37.pyc b/slack-bot/__pycache__/pullTrafficInfo.cpython-37.pyc index a8a9908..49036d2 100644 Binary files a/slack-bot/__pycache__/pullTrafficInfo.cpython-37.pyc and b/slack-bot/__pycache__/pullTrafficInfo.cpython-37.pyc differ diff --git a/slack-bot/pullTrafficInfo.py b/slack-bot/pullTrafficInfo.py index c3a5107..7d3af4b 100644 --- a/slack-bot/pullTrafficInfo.py +++ b/slack-bot/pullTrafficInfo.py @@ -147,7 +147,7 @@ class getTrafficInfo(object): f' reported on the {self.motorway}' f' :scream:', use_aliases=True)) - self.output.append(self.incidentInformation[i][0][0]) + self.output.append(self.incidentInformation[0][i][0]) self.output.append(random.choice(self.sarcasticMessage)) return self diff --git a/slack-bot/requirements.txt b/slack-bot/requirements.txt deleted file mode 100644 index 95dd7d2..0000000 --- a/slack-bot/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -beautifulsoup4==4.8.0 -bs4==0.0.1 -certifi==2019.9.11 -chardet==3.0.4 -emoji==0.5.4 -idna==2.8 -lxml==4.4.1 -numpy==1.17.2 -pandas==0.25.1 -python-dateutil==2.8.0 -pytz==2019.2 -requests==2.22.0 -selenium==3.141.0 -six==1.12.0 -soupsieve==1.9.4 -urllib3==1.25.6 diff --git a/slack-bot/cropped.png b/slack-bot/screenshot/cropped.png similarity index 100% rename from slack-bot/cropped.png rename to slack-bot/screenshot/cropped.png diff --git a/slack-bot/screenshot.png b/slack-bot/screenshot/screenshot.png similarity index 100% rename from slack-bot/screenshot.png rename to slack-bot/screenshot/screenshot.png diff --git a/slack-bot/screenshot.py b/slack-bot/screenshot/screenshot.py similarity index 100% rename from slack-bot/screenshot.py rename to slack-bot/screenshot/screenshot.py diff --git a/slack-bot/screenshotFull.py b/slack-bot/screenshot/screenshotFull.py similarity index 100% rename from slack-bot/screenshotFull.py rename to slack-bot/screenshot/screenshotFull.py diff --git a/slack-bot/web_screenshot.png b/slack-bot/screenshot/web_screenshot.png similarity index 100% rename from slack-bot/web_screenshot.png rename to slack-bot/screenshot/web_screenshot.png diff --git a/slack-bot/scraper.py b/slack-bot/traffic-scraper/dev/scraper.py similarity index 100% rename from slack-bot/scraper.py rename to slack-bot/traffic-scraper/dev/scraper.py diff --git a/slack-bot/scraper.py.old b/slack-bot/traffic-scraper/dev/scraper.py.old similarity index 100% rename from slack-bot/scraper.py.old rename to slack-bot/traffic-scraper/dev/scraper.py.old diff --git a/slack-bot/sendToSQS.py b/slack-bot/traffic-scraper/dev/sendToSQS_old.py similarity index 60% rename from slack-bot/sendToSQS.py rename to slack-bot/traffic-scraper/dev/sendToSQS_old.py index dce2d90..0e8d505 100644 --- a/slack-bot/sendToSQS.py +++ b/slack-bot/traffic-scraper/dev/sendToSQS_old.py @@ -2,11 +2,15 @@ import boto3 import os import sys sys.path.append(os.getcwd()) +sys.path.append('/home/dtomlinson/projects/slack-bot/traffic-scraper/prd') from pullTrafficInfo import getTrafficInfo import base64 -import emoji +# import emoji +import hashlib +from datetime import datetime -motorway = 'M1' +motorway = 'A3' +currentTime = datetime.now().strftime('%H:%M:%S') session = boto3.Session(profile_name='plex-aws') sqs = session.client('sqs') @@ -33,18 +37,23 @@ queueURL = 'https://sqs.eu-west-1.amazonaws.com/745437999005/slack-bot.fifo' # else: # print('Queue is empty') -for item in inst.output: - encoded = (base64.b64encode(item.encode())).decode() - sqs.send_message(QueueUrl=queueURL, - MessageBody=encoded, - MessageGroupId=f'slack-bot-{motorway}') - # print(encoded) - # print(encoded.encode()) - # print(base64.b64decode(encoded.encode()).decode()) +# for item in inst.output: +# print(item) +# encoded = (base64.b64encode(item.encode())).decode() +# dedupId = hashlib.md5((item + currentTime).encode()).hexdigest() +# msg = sqs.send_message(QueueUrl=queueURL, +# MessageBody=encoded, +# MessageGroupId=f'slack-bot-{motorway}', +# MessageDeduplicationId=dedupId) +# if msg is not None: +# print(msg["MessageId"]) +# # print(encoded) +# # print(encoded.encode()) +# # print(base64.b64decode(encoded.encode()).decode()) response = sqs.receive_message(QueueUrl=queueURL, - MaxNumberOfMessages=4) + MaxNumberOfMessages=10) if 'Messages' in response: for message in response['Messages']: # print(message['Body']) diff --git a/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc b/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc new file mode 100644 index 0000000..c32afae Binary files /dev/null and b/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc differ diff --git a/slack-bot/traffic-scraper/prd/pullTrafficInfo.py b/slack-bot/traffic-scraper/prd/pullTrafficInfo.py new file mode 100644 index 0000000..cd55365 --- /dev/null +++ b/slack-bot/traffic-scraper/prd/pullTrafficInfo.py @@ -0,0 +1,164 @@ +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException +from datetime import datetime +import re +import emoji +import random + + +class getTrafficInfo(object): + """ + Pulls the latest traffic information for a major road or + motorway from trafficdelays.co.uk. + + Parameters: + motorway: str -> a string containing the A-road or motorway + driver: str -> the browser/driver to run under (chrome default) + + Usage: + + ** required ** + Create a driver for your motorway: + getTrafficInfo.getTrafficURL(motorway, driver) + + **options** + Get information for incident types: + getTrafficInfo.findIncidents() + + Get HTML from the page: + getTrafficInfo.getIncidentHTML() + + Get a count of how many incidents for each type: + getTrafficInfo.getIncidentCount() + + Get the text detailing the incident: + getTrafficInfo.getIncidentInformation() + + Generate a sequential list for an output: + getTrafficInfo.getOutput() + """ + def __init__(self, browser, motorway): + super(getTrafficInfo, self).__init__() + self.browser = browser + self.motorway = motorway + self.incidentTypes = ['congestion', 'accident'] + + # def __del__(self): + # print('Quitting') + # self.browser.quit() + + @classmethod + def getTrafficURL(cls, motorway, driver='chrome'): + url = ('https://www.trafficdelays.co.uk/' + motorway.lower() + + '-traffic-delays') + browser = getTrafficInfo.getWebDriver(driver, url) + getTrafficInfo.verfiyMotorway(browser, motorway) + return cls(browser, motorway) + + @staticmethod + def getWebDriver(driver, url): + driver = driver.lower() + if driver == 'chrome': + # Prepare the option for the Chromedriver + options = webdriver.ChromeOptions() + options.add_argument('headless') + # Start the Chromedriver + browser = webdriver.Chrome(options=options) + browser.get(url) + return browser + else: + raise Exception(f'Driver {driver} not supported') + + @staticmethod + def verfiyMotorway(browser, motorway): + try: + verify = browser.find_element_by_xpath('/html/body/div[1]/div/div/' + 'div/section/div/div/' + 'div[1]') + if verify.text in ('It looks like the link pointing here ' + 'was faulty. Maybe try searching?'): + raise Exception(f'No traffic information available for' + f' {motorway}') + except NoSuchElementException: + pass + + def findIncidents(self): + self.incidentBrowser = [] + for item in self.incidentTypes: + xpath = f'//*[@id="{item}"]' + self.incidentBrowser.append(self.browser.find_element_by_xpath + (xpath)) + self.getIncidentHTML() + return self + + def getIncidentHTML(self): + self.incidentHTML = [] + for item in self.incidentBrowser: + self.incidentHTML.append(item.get_attribute('innerHTML')) + return self + + def getIncidentCount(self): + self.incidentCount = [] + for item, i in zip(self.incidentBrowser, + range(0, len(self.incidentHTML))): + self.incidentCount.append(self.incidentHTML[i].count('
  • ')) + return self + + def getIncidentInformation(self): + self.incidentInformation = [] + pattern = r".*\.*title=\".*\".*\>((.|\n)*?)\" + for item in self.incidentHTML: + self.incidentInformation.append(re.findall(pattern, item)) + return self + + def generateOutput(self): + self.getIncidentCount() + self.output = [] + self.sarcasticMessage = [(f'Hey Andy, have you thought about getting' + ' the train?' + + emoji.emojize(f' :bullettrain_front:', + use_aliases=True)), + (f'Hey Andy, maybe flying would be quicker?' + + emoji.emojize(f' :helicopter:', + use_aliases=True)), + (f'Don\'t fret, Andy can always work from' + ' home!' + + emoji.emojize(f' :house_with_garden:', + use_aliases=True))] + currentTime = datetime.now().strftime('%H:%M:%S') + self.output.append('START') + self.output.append(emoji.emojize(f'Did someone say {self.motorway}!?' + ' :anguished:' + ' Let\'s check the latest updates' + ' from Highways England as of' + f' {currentTime}!' + ' :police_car::rotating_light:', + use_aliases=True)) + for item, i in zip(self.incidentCount, + range(0, len(self.incidentTypes))): + if item == 0: + self.output.append(emoji.emojize + (f'There are currently no' + ' reported' + f' {self.incidentTypes[i]} incidents on' + f' the {self.motorway} :thinking_face:', + use_aliases=True)) + pass + else: + self.output.append(emoji.emojize + (f'There are currently' + f' {self.incidentCount[i]} reported' + f' {self.incidentTypes[i]} incidents' + f' reported on the {self.motorway}' + f' :scream:', + use_aliases=True)) + self.output.append(self.incidentInformation[0][i][0]) + self.output.append(random.choice(self.sarcasticMessage)) + self.output.append('END') + return self + + +# inst = getTrafficInfo.getTrafficURL('A50').findIncidents() \ +# .getIncidentInformation().generateOutput() +# for i in inst.output: +# print(i) diff --git a/slack-bot/traffic-scraper/prd/receiveFromSQS.py b/slack-bot/traffic-scraper/prd/receiveFromSQS.py new file mode 100644 index 0000000..fec997d --- /dev/null +++ b/slack-bot/traffic-scraper/prd/receiveFromSQS.py @@ -0,0 +1,44 @@ +import boto3 +import base64 + + +class receiveFromSQS(object): + """docstring for receiveFromSQS""" + def __init__(self, session, queueURL): + super(receiveFromSQS, self).__init__() + self.session = session + self.sqs = session.client('sqs') + self.queueURL = queueURL + + @classmethod + def createSession(cls, profileName, queueURL): + session = boto3.Session(profile_name=profileName) + return cls(session, queueURL) + + def receiveMessage(self, b64=True, MaxNumberOfMessages=10, delete=False, + endString='END'): + while True: + response = self.sqs\ + .receive_message(QueueUrl=self.queueURL, + MaxNumberOfMessages=MaxNumberOfMessages) + if 'Messages' in response: + for message in response['Messages']: + if b64: + decoded = base64.b64decode(message['Body']) + msg = decoded.decode() + print(msg) + if msg == endString: + return self + else: + msg = print(message['Body']) + if msg == endString: + return self + else: + print('Queue is empty') + break + + +inst = receiveFromSQS.createSession(profileName='plex-aws', + queueURL='https://sqs.eu-west-1.amazonaws' + '.com/745437999005/slack-bot.fifo') +inst.receiveMessage() diff --git a/slack-bot/traffic-scraper/prd/sendToSQS.py b/slack-bot/traffic-scraper/prd/sendToSQS.py new file mode 100644 index 0000000..9d5b479 --- /dev/null +++ b/slack-bot/traffic-scraper/prd/sendToSQS.py @@ -0,0 +1,50 @@ +import boto3 +import base64 +import hashlib +from datetime import datetime +import os +import sys +sys.path.append(os.getcwd()) +from pullTrafficInfo import getTrafficInfo + + +class sendToSQS(object): + """docstring for sendToSQS""" + def __init__(self, session, queueURL): + super(sendToSQS, self).__init__() + self.session = session + self.sqs = session.client('sqs') + self.queueURL = queueURL + + @classmethod + def createSession(cls, profileName, queueURL): + session = boto3.Session(profile_name=profileName) + return cls(session, queueURL) + + def sendMessage(self, message, messageGroupId, b64=True, dedup=True): + currentTime = datetime.now().strftime('%H:%M:%S') + if b64: + message = (base64.b64encode(message.encode())).decode() + if dedup: + dedupId = hashlib.md5((message + currentTime).encode()).hexdigest() + msg = self.sqs.send_message(QueueUrl=self.queueURL, + MessageBody=message, + MessageGroupId=messageGroupId, + MessageDeduplicationId=dedupId) + else: + msg = self.sqs.send_message(QueueUrl=self.queueURL, + MessageBody=message, + MessageGroupId=messageGroupId) + if msg is not None: + print(msg["MessageId"]) + + +inst = sendToSQS.createSession(profileName='plex-aws', + queueURL='https://sqs.eu-west-1.amazonaws.com' + '/745437999005/slack-bot.fifo') + +instM = getTrafficInfo.getTrafficURL('M62').findIncidents() \ + .getIncidentInformation().generateOutput() + +for item in instM.output: + inst.sendMessage(message=item, messageGroupId='slack-bot-M62', dedup=True)