dtomlinson/python-VM
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updating receiveFromSQS.py

Browse Source
This commit is contained in:
Daniel Tomlinson
2019-10-08 02:42:29 +01:00
parent 713084c336
commit 13b9a6d039
5 changed files with 190 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc
View File

Binary file not shown.

117
slack-bot/traffic-scraper/prd/pullTrafficInfo.py
View File

@@ -37,6 +37,7 @@ class getTrafficInfo(object):
Generate a sequential list for an output:
getTrafficInfo.getOutput()
"""
def __init__(self, browser, motorway):
super(getTrafficInfo, self).__init__()
self.browser = browser
@@ -49,8 +50,11 @@ class getTrafficInfo(object):
@classmethod
def getTrafficURL(cls, motorway, driver='chrome'):
url = ('https://www.trafficdelays.co.uk/' + motorway.lower()
+ '-traffic-delays')
url = (
'https://www.trafficdelays.co.uk/'
+ motorway.lower()
+ '-traffic-delays'
)
browser = getTrafficInfo.getWebDriver(driver, url)
getTrafficInfo.verfiyMotorway(browser, motorway)
return cls(browser, motorway)
@@ -72,13 +76,16 @@ class getTrafficInfo(object):
@staticmethod
def verfiyMotorway(browser, motorway):
try:
verify = browser.find_element_by_xpath('/html/body/div[1]/div/div/'
'div/section/div/div/'
'div[1]')
if verify.text in ('It looks like the link pointing here '
'was faulty. Maybe try searching?'):
raise Exception(f'No traffic information available for'
f' {motorway}')
verify = browser.find_element_by_xpath(
'/html/body/div[1]/div/div/' 'div/section/div/div/' 'div[1]'
)
if verify.text in (
'It looks like the link pointing here '
'was faulty. Maybe try searching?'
):
raise Exception(
f'No traffic information available for' f' {motorway}'
)
except NoSuchElementException:
pass
@@ -86,8 +93,9 @@ class getTrafficInfo(object):
self.incidentBrowser = []
for item in self.incidentTypes:
xpath = f'//*[@id="{item}"]'
self.incidentBrowser.append(self.browser.find_element_by_xpath
(xpath))
self.incidentBrowser.append(
self.browser.find_element_by_xpath(xpath)
)
self.getIncidentHTML()
return self
@@ -99,8 +107,9 @@ class getTrafficInfo(object):
def getIncidentCount(self):
self.incidentCount = []
for item, i in zip(self.incidentBrowser,
range(0, len(self.incidentHTML))):
for item, i in zip(
self.incidentBrowser, range(0, len(self.incidentHTML))
):
self.incidentCount.append(self.incidentHTML[i].count('<li>'))
return self
@@ -114,44 +123,60 @@ class getTrafficInfo(object):
def generateOutput(self):
self.getIncidentCount()
self.output = []
self.sarcasticMessage = [(f'Hey Andy, have you thought about getting'
' the train?'
+ emoji.emojize(f' :bullettrain_front:',
use_aliases=True)),
(f'Hey Andy, maybe flying would be quicker?'
+ emoji.emojize(f' :helicopter:',
use_aliases=True)),
(f'Don\'t fret, Andy can always work from'
' home!'
+ emoji.emojize(f' :house_with_garden:',
use_aliases=True))]
self.sarcasticMessage = [
(
f'Hey Andy, have you thought about getting'
' the train?'
+ emoji.emojize(f' :bullettrain_front:', use_aliases=True)
),
(
f'Hey Andy, maybe flying would be quicker?'
+ emoji.emojize(f' :helicopter:', use_aliases=True)
),
(
f'Don\'t fret, Andy can always work from'
' home!'
+ emoji.emojize(f' :house_with_garden:', use_aliases=True)
),
]
currentTime = datetime.now().strftime('%H:%M:%S')
self.output.append('START')
self.output.append(emoji.emojize(f'Did someone say {self.motorway}!?'
' :anguished:'
' Let\'s check the latest updates'
' from Highways England as of'
f' {currentTime}!'
' :police_car::rotating_light:',
use_aliases=True))
for item, i in zip(self.incidentCount,
range(0, len(self.incidentTypes))):
self.output.append(
emoji.emojize(
f'Did someone say {self.motorway}!?'
' :anguished:'
' Let\'s check the latest updates'
' from Highways England as of'
f' {currentTime}!'
' :police_car::rotating_light:',
use_aliases=True,
)
)
for item, i in zip(
self.incidentCount, range(0, len(self.incidentTypes))
):
if item == 0:
self.output.append(emoji.emojize
(f'There are currently no'
' reported'
f' {self.incidentTypes[i]} incidents on'
f' the {self.motorway} :thinking_face:',
use_aliases=True))
self.output.append(
emoji.emojize(
f'There are currently no'
' reported'
f' {self.incidentTypes[i]} incidents on'
f' the {self.motorway} :thinking_face:',
use_aliases=True,
)
)
pass
else:
self.output.append(emoji.emojize
(f'There are currently'
f' {self.incidentCount[i]} reported'
f' {self.incidentTypes[i]} incidents'
f' reported on the {self.motorway}'
f' :scream:',
use_aliases=True))
self.output.append(
emoji.emojize(
f'There are currently'
f' {self.incidentCount[i]} reported'
f' {self.incidentTypes[i]} incidents'
f' reported on the {self.motorway}'
f' :scream:',
use_aliases=True,
)
)
self.output.append(self.incidentInformation[0][i][0])
self.output.append(random.choice(self.sarcasticMessage))
self.output.append('END')

124
slack-bot/traffic-scraper/prd/receiveFromSQS.py
View File

@@ -1,5 +1,6 @@
import boto3
import base64
import math
class receiveFromSQS(object):
@@ -19,59 +20,80 @@ class receiveFromSQS(object):
def getQueueLength(self):
attributeNames = ['ApproximateNumberOfMessages']
self.queueAttributes = self.sqs.get_queue_attributes(
QueueUrl=self.queueURL,
AttributeNames=attributeNames,
QueueUrl=self.queueURL, AttributeNames=attributeNames
)
self.queueLength = int(
self.queueAttributes['Attributes'][
'ApproximateNumberOfMessages'
]
self.queueAttributes['Attributes']['ApproximateNumberOfMessages']
)
return self.queueLength
def _receiveSQSMessage(self, maxNumberOfMessages=10):
response = self.sqs.receive_message(
QueueUrl=self.queueURL,
MaxNumberOfMessages=maxNumberOfMessages,
)
return response
def _extractMessageFromSQS(self):
pass
def receiveMessage(
def _receiveSQSMessage(
self,
b64=True,
MaxNumberOfMessages=10,
delete=False,
endString='END',
totalNumberOfMessages,
maxNumberOfMessages=10,
deleteOnReceipt=False,
):
while True:
response = self.sqs.receive_message(
QueueUrl=self.queueURL,
MaxNumberOfMessages=MaxNumberOfMessages,
self.response = []
loops = int(math.ceil(totalNumberOfMessages / maxNumberOfMessages))
for i in range(0, loops):
self.response.append(
self.sqs.receive_message(
QueueUrl=self.queueURL,
MaxNumberOfMessages=maxNumberOfMessages,
)
)
if 'Messages' in response:
for message in response['Messages']:
if b64:
decoded = base64.b64decode(
message['Body']
)
msg = decoded.decode()
print(msg)
if msg == endString:
return self
else:
msg = print(message['Body'])
if msg == endString:
return self
else:
print('Queue is empty')
break
return self
def receiveAllMessages(self, b64=True, delete=False):
while True:
self.response = self._receiveSQSMessage()
def _extractMessageFromSQS(self, totalNumberOfMessages):
self.messages = []
try:
loops = len(self.response)
for i in range(0, loops):
if 'Messages' in self.response[i]:
_message = self.response[i]['Messages']
for rawMessage in _message:
self.messages.append(rawMessage['Body'])
else:
print("No messages in the queue")
except KeyError:
print("No messages in the queue")
return self
# def receiveMessage(
# self, b64=True, MaxNumberOfMessages=10, delete=False, endString='END'
# ):
# while True:
# response = self.sqs.receive_message(
# QueueUrl=self.queueURL,
# MaxNumberOfMessages=MaxNumberOfMessages
# )
# if 'Messages' in response:
# for message in response['Messages']:
# if b64:
# decoded = base64.b64decode(message['Body'])
# msg = decoded.decode()
# if msg == endString:
# return self
# else:
# msg = print(message['Body'])
# if msg == endString:
# return self
# else:
# print('Queue is empty')
# break
def receiveAllMessages(self, b64=True, deleteOnReceipt=False):
totalNumberOfMessages = self.getQueueLength()
self._receiveSQSMessage(
totalNumberOfMessages,
deleteOnReceipt=deleteOnReceipt
)
self._extractMessageFromSQS(
totalNumberOfMessages=totalNumberOfMessages
)
if b64:
pass # WRITE THIS IN!
return self
inst = receiveFromSQS.createSession(
@@ -79,4 +101,16 @@ inst = receiveFromSQS.createSession(
queueURL='https://sqs.eu-west-1.amazonaws'
'.com/745437999005/slack-bot.fifo',
)
inst.getQueueLength()
inst.receiveAllMessages()
for item in inst.messages:
print(item)
# queueLength = inst.getQueueLength()
# response = inst._receiveSQSMessage(queueLength)._extractMessageFromSQS(
# queueLength
# )
# # messages = inst._receiveSQSMessage(12)._extractMessageFromSQS(12)
# for item in response.message:
# print(item)

42
slack-bot/traffic-scraper/prd/sendToSQS.py
View File

@@ -4,16 +4,18 @@ import hashlib
from datetime import datetime
import os
import sys
sys.path.append(os.getcwd())
from pullTrafficInfo import getTrafficInfo
class sendToSQS(object):
"""docstring for sendToSQS"""
def __init__(self, session, queueURL):
super(sendToSQS, self).__init__()
self.session = session
self.sqs = session.client('sqs')
self.sqs = session.client("sqs")
self.queueURL = queueURL
@classmethod
@@ -22,29 +24,39 @@ class sendToSQS(object):
return cls(session, queueURL)
def sendMessage(self, message, messageGroupId, b64=True, dedup=False):
currentTime = datetime.now().strftime('%H:%M:%S.%f')
currentTime = datetime.now().strftime("%H:%M:%S.%f")
if b64:
message = (base64.b64encode(message.encode())).decode()
if not dedup:
dedupId = hashlib.md5((message + currentTime).encode()).hexdigest()
msg = self.sqs.send_message(QueueUrl=self.queueURL,
MessageBody=message,
MessageGroupId=messageGroupId,
MessageDeduplicationId=dedupId)
msg = self.sqs.send_message(
QueueUrl=self.queueURL,
MessageBody=message,
MessageGroupId=messageGroupId,
MessageDeduplicationId=dedupId,
)
else:
msg = self.sqs.send_message(QueueUrl=self.queueURL,
MessageBody=message,
MessageGroupId=messageGroupId)
msg = self.sqs.send_message(
QueueUrl=self.queueURL,
MessageBody=message,
MessageGroupId=messageGroupId,
)
if msg is not None:
print(msg["MessageId"])
inst = sendToSQS.createSession(profileName='plex-aws',
queueURL='https://sqs.eu-west-1.amazonaws.com'
'/745437999005/slack-bot.fifo')
inst = sendToSQS.createSession(
profileName="plex-aws",
queueURL="https://sqs.eu-west-1.amazonaws.com"
"/745437999005/slack-bot.fifo",
)
instM = getTrafficInfo.getTrafficURL('M62').findIncidents() \
.getIncidentInformation().generateOutput()
instM = (
getTrafficInfo.getTrafficURL("M62")
.findIncidents()
.getIncidentInformation()
.generateOutput()
)
for item in instM.output:
inst.sendMessage(message=item, messageGroupId='slack-bot-M62')
inst.sendMessage(message=item, messageGroupId="slack-bot-M62")