diff --git a/slack-bot/traffic-scraper/dev/sendToSQS_old.py b/slack-bot/traffic-scraper/dev/sendToSQS_old.py index c0e1e6f..2b723c6 100644 --- a/slack-bot/traffic-scraper/dev/sendToSQS_old.py +++ b/slack-bot/traffic-scraper/dev/sendToSQS_old.py @@ -1,12 +1,14 @@ import boto3 import os import sys + sys.path.append(os.getcwd()) sys.path.append('/home/dtomlinson/projects/slack-bot/traffic-scraper/prd') from pullTrafficInfo import getTrafficInfo import base64 + # import emoji -import hashlib +# import hashlib from datetime import datetime motorway = 'A3' @@ -17,8 +19,12 @@ sqs = session.client('sqs') # queue = sqs.create_queue(QueueName='test', Attributes={'DelaySeconds': '5'}) -inst = getTrafficInfo.getTrafficURL(motorway).findIncidents() \ - .getIncidentInformation().generateOutput() +inst = ( + getTrafficInfo.getTrafficURL(motorway) + .findIncidents() + .getIncidentInformation() + .generateOutput() +) # encoded = base64.b64encode(inst.output[0].encode()) # decoded = base64.b64decode(encoded).decode() @@ -51,13 +57,12 @@ queueURL = 'https://sqs.eu-west-1.amazonaws.com/745437999005/slack-bot.fifo' # # print(encoded.encode()) # # print(base64.b64decode(encoded.encode()).decode()) -queueAttributes = sqs\ - .get_queue_attributes(QueueUrl=queueURL, - AttributeNames=['ApproximateNumberOfMessages']) +queueAttributes = sqs.get_queue_attributes( + QueueUrl=queueURL, AttributeNames=['ApproximateNumberOfMessages'] +) -response = sqs.receive_message(QueueUrl=queueURL, - MaxNumberOfMessages=10) +response = sqs.receive_message(QueueUrl=queueURL, MaxNumberOfMessages=10) if 'Messages' in response: for message in response['Messages']: # print(message['Body']) diff --git a/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc b/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc index 57f734c..534fadd 100644 Binary files a/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc and b/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc differ diff --git a/slack-bot/traffic-scraper/prd/pullTrafficInfo.py b/slack-bot/traffic-scraper/prd/pullTrafficInfo.py index cd55365..6212801 100644 --- a/slack-bot/traffic-scraper/prd/pullTrafficInfo.py +++ b/slack-bot/traffic-scraper/prd/pullTrafficInfo.py @@ -37,6 +37,7 @@ class getTrafficInfo(object): Generate a sequential list for an output: getTrafficInfo.getOutput() """ + def __init__(self, browser, motorway): super(getTrafficInfo, self).__init__() self.browser = browser @@ -49,8 +50,11 @@ class getTrafficInfo(object): @classmethod def getTrafficURL(cls, motorway, driver='chrome'): - url = ('https://www.trafficdelays.co.uk/' + motorway.lower() - + '-traffic-delays') + url = ( + 'https://www.trafficdelays.co.uk/' + + motorway.lower() + + '-traffic-delays' + ) browser = getTrafficInfo.getWebDriver(driver, url) getTrafficInfo.verfiyMotorway(browser, motorway) return cls(browser, motorway) @@ -72,13 +76,16 @@ class getTrafficInfo(object): @staticmethod def verfiyMotorway(browser, motorway): try: - verify = browser.find_element_by_xpath('/html/body/div[1]/div/div/' - 'div/section/div/div/' - 'div[1]') - if verify.text in ('It looks like the link pointing here ' - 'was faulty. Maybe try searching?'): - raise Exception(f'No traffic information available for' - f' {motorway}') + verify = browser.find_element_by_xpath( + '/html/body/div[1]/div/div/' 'div/section/div/div/' 'div[1]' + ) + if verify.text in ( + 'It looks like the link pointing here ' + 'was faulty. Maybe try searching?' + ): + raise Exception( + f'No traffic information available for' f' {motorway}' + ) except NoSuchElementException: pass @@ -86,8 +93,9 @@ class getTrafficInfo(object): self.incidentBrowser = [] for item in self.incidentTypes: xpath = f'//*[@id="{item}"]' - self.incidentBrowser.append(self.browser.find_element_by_xpath - (xpath)) + self.incidentBrowser.append( + self.browser.find_element_by_xpath(xpath) + ) self.getIncidentHTML() return self @@ -99,8 +107,9 @@ class getTrafficInfo(object): def getIncidentCount(self): self.incidentCount = [] - for item, i in zip(self.incidentBrowser, - range(0, len(self.incidentHTML))): + for item, i in zip( + self.incidentBrowser, range(0, len(self.incidentHTML)) + ): self.incidentCount.append(self.incidentHTML[i].count('
  • ')) return self @@ -114,44 +123,60 @@ class getTrafficInfo(object): def generateOutput(self): self.getIncidentCount() self.output = [] - self.sarcasticMessage = [(f'Hey Andy, have you thought about getting' - ' the train?' - + emoji.emojize(f' :bullettrain_front:', - use_aliases=True)), - (f'Hey Andy, maybe flying would be quicker?' - + emoji.emojize(f' :helicopter:', - use_aliases=True)), - (f'Don\'t fret, Andy can always work from' - ' home!' - + emoji.emojize(f' :house_with_garden:', - use_aliases=True))] + self.sarcasticMessage = [ + ( + f'Hey Andy, have you thought about getting' + ' the train?' + + emoji.emojize(f' :bullettrain_front:', use_aliases=True) + ), + ( + f'Hey Andy, maybe flying would be quicker?' + + emoji.emojize(f' :helicopter:', use_aliases=True) + ), + ( + f'Don\'t fret, Andy can always work from' + ' home!' + + emoji.emojize(f' :house_with_garden:', use_aliases=True) + ), + ] currentTime = datetime.now().strftime('%H:%M:%S') self.output.append('START') - self.output.append(emoji.emojize(f'Did someone say {self.motorway}!?' - ' :anguished:' - ' Let\'s check the latest updates' - ' from Highways England as of' - f' {currentTime}!' - ' :police_car::rotating_light:', - use_aliases=True)) - for item, i in zip(self.incidentCount, - range(0, len(self.incidentTypes))): + self.output.append( + emoji.emojize( + f'Did someone say {self.motorway}!?' + ' :anguished:' + ' Let\'s check the latest updates' + ' from Highways England as of' + f' {currentTime}!' + ' :police_car::rotating_light:', + use_aliases=True, + ) + ) + for item, i in zip( + self.incidentCount, range(0, len(self.incidentTypes)) + ): if item == 0: - self.output.append(emoji.emojize - (f'There are currently no' - ' reported' - f' {self.incidentTypes[i]} incidents on' - f' the {self.motorway} :thinking_face:', - use_aliases=True)) + self.output.append( + emoji.emojize( + f'There are currently no' + ' reported' + f' {self.incidentTypes[i]} incidents on' + f' the {self.motorway} :thinking_face:', + use_aliases=True, + ) + ) pass else: - self.output.append(emoji.emojize - (f'There are currently' - f' {self.incidentCount[i]} reported' - f' {self.incidentTypes[i]} incidents' - f' reported on the {self.motorway}' - f' :scream:', - use_aliases=True)) + self.output.append( + emoji.emojize( + f'There are currently' + f' {self.incidentCount[i]} reported' + f' {self.incidentTypes[i]} incidents' + f' reported on the {self.motorway}' + f' :scream:', + use_aliases=True, + ) + ) self.output.append(self.incidentInformation[0][i][0]) self.output.append(random.choice(self.sarcasticMessage)) self.output.append('END') diff --git a/slack-bot/traffic-scraper/prd/receiveFromSQS.py b/slack-bot/traffic-scraper/prd/receiveFromSQS.py index 9d9f582..c7b415d 100644 --- a/slack-bot/traffic-scraper/prd/receiveFromSQS.py +++ b/slack-bot/traffic-scraper/prd/receiveFromSQS.py @@ -1,5 +1,6 @@ import boto3 import base64 +import math class receiveFromSQS(object): @@ -19,59 +20,80 @@ class receiveFromSQS(object): def getQueueLength(self): attributeNames = ['ApproximateNumberOfMessages'] self.queueAttributes = self.sqs.get_queue_attributes( - QueueUrl=self.queueURL, - AttributeNames=attributeNames, + QueueUrl=self.queueURL, AttributeNames=attributeNames ) self.queueLength = int( - self.queueAttributes['Attributes'][ - 'ApproximateNumberOfMessages' - ] + self.queueAttributes['Attributes']['ApproximateNumberOfMessages'] ) return self.queueLength - def _receiveSQSMessage(self, maxNumberOfMessages=10): - response = self.sqs.receive_message( - QueueUrl=self.queueURL, - MaxNumberOfMessages=maxNumberOfMessages, - ) - return response - - def _extractMessageFromSQS(self): - pass - - def receiveMessage( + def _receiveSQSMessage( self, - b64=True, - MaxNumberOfMessages=10, - delete=False, - endString='END', + totalNumberOfMessages, + maxNumberOfMessages=10, + deleteOnReceipt=False, ): - while True: - response = self.sqs.receive_message( - QueueUrl=self.queueURL, - MaxNumberOfMessages=MaxNumberOfMessages, + self.response = [] + loops = int(math.ceil(totalNumberOfMessages / maxNumberOfMessages)) + for i in range(0, loops): + self.response.append( + self.sqs.receive_message( + QueueUrl=self.queueURL, + MaxNumberOfMessages=maxNumberOfMessages, + ) ) - if 'Messages' in response: - for message in response['Messages']: - if b64: - decoded = base64.b64decode( - message['Body'] - ) - msg = decoded.decode() - print(msg) - if msg == endString: - return self - else: - msg = print(message['Body']) - if msg == endString: - return self - else: - print('Queue is empty') - break + return self - def receiveAllMessages(self, b64=True, delete=False): - while True: - self.response = self._receiveSQSMessage() + def _extractMessageFromSQS(self, totalNumberOfMessages): + self.messages = [] + try: + loops = len(self.response) + for i in range(0, loops): + if 'Messages' in self.response[i]: + _message = self.response[i]['Messages'] + for rawMessage in _message: + self.messages.append(rawMessage['Body']) + else: + print("No messages in the queue") + except KeyError: + print("No messages in the queue") + return self + + # def receiveMessage( + # self, b64=True, MaxNumberOfMessages=10, delete=False, endString='END' + # ): + # while True: + # response = self.sqs.receive_message( + # QueueUrl=self.queueURL, + # MaxNumberOfMessages=MaxNumberOfMessages + # ) + # if 'Messages' in response: + # for message in response['Messages']: + # if b64: + # decoded = base64.b64decode(message['Body']) + # msg = decoded.decode() + # if msg == endString: + # return self + # else: + # msg = print(message['Body']) + # if msg == endString: + # return self + # else: + # print('Queue is empty') + # break + + def receiveAllMessages(self, b64=True, deleteOnReceipt=False): + totalNumberOfMessages = self.getQueueLength() + self._receiveSQSMessage( + totalNumberOfMessages, + deleteOnReceipt=deleteOnReceipt + ) + self._extractMessageFromSQS( + totalNumberOfMessages=totalNumberOfMessages + ) + if b64: + pass # WRITE THIS IN! + return self inst = receiveFromSQS.createSession( @@ -79,4 +101,16 @@ inst = receiveFromSQS.createSession( queueURL='https://sqs.eu-west-1.amazonaws' '.com/745437999005/slack-bot.fifo', ) -inst.getQueueLength() + +inst.receiveAllMessages() +for item in inst.messages: + print(item) + +# queueLength = inst.getQueueLength() + +# response = inst._receiveSQSMessage(queueLength)._extractMessageFromSQS( +# queueLength +# ) +# # messages = inst._receiveSQSMessage(12)._extractMessageFromSQS(12) +# for item in response.message: +# print(item) diff --git a/slack-bot/traffic-scraper/prd/sendToSQS.py b/slack-bot/traffic-scraper/prd/sendToSQS.py index 332c79e..6b7b81f 100644 --- a/slack-bot/traffic-scraper/prd/sendToSQS.py +++ b/slack-bot/traffic-scraper/prd/sendToSQS.py @@ -4,16 +4,18 @@ import hashlib from datetime import datetime import os import sys + sys.path.append(os.getcwd()) from pullTrafficInfo import getTrafficInfo class sendToSQS(object): """docstring for sendToSQS""" + def __init__(self, session, queueURL): super(sendToSQS, self).__init__() self.session = session - self.sqs = session.client('sqs') + self.sqs = session.client("sqs") self.queueURL = queueURL @classmethod @@ -22,29 +24,39 @@ class sendToSQS(object): return cls(session, queueURL) def sendMessage(self, message, messageGroupId, b64=True, dedup=False): - currentTime = datetime.now().strftime('%H:%M:%S.%f') + currentTime = datetime.now().strftime("%H:%M:%S.%f") if b64: message = (base64.b64encode(message.encode())).decode() if not dedup: dedupId = hashlib.md5((message + currentTime).encode()).hexdigest() - msg = self.sqs.send_message(QueueUrl=self.queueURL, - MessageBody=message, - MessageGroupId=messageGroupId, - MessageDeduplicationId=dedupId) + msg = self.sqs.send_message( + QueueUrl=self.queueURL, + MessageBody=message, + MessageGroupId=messageGroupId, + MessageDeduplicationId=dedupId, + ) else: - msg = self.sqs.send_message(QueueUrl=self.queueURL, - MessageBody=message, - MessageGroupId=messageGroupId) + msg = self.sqs.send_message( + QueueUrl=self.queueURL, + MessageBody=message, + MessageGroupId=messageGroupId, + ) if msg is not None: print(msg["MessageId"]) -inst = sendToSQS.createSession(profileName='plex-aws', - queueURL='https://sqs.eu-west-1.amazonaws.com' - '/745437999005/slack-bot.fifo') +inst = sendToSQS.createSession( + profileName="plex-aws", + queueURL="https://sqs.eu-west-1.amazonaws.com" + "/745437999005/slack-bot.fifo", +) -instM = getTrafficInfo.getTrafficURL('M62').findIncidents() \ - .getIncidentInformation().generateOutput() +instM = ( + getTrafficInfo.getTrafficURL("M62") + .findIncidents() + .getIncidentInformation() + .generateOutput() +) for item in instM.output: - inst.sendMessage(message=item, messageGroupId='slack-bot-M62') + inst.sendMessage(message=item, messageGroupId="slack-bot-M62")