updating receiveFromSQS.py

slack-bot/traffic-scraper/dev/sendToSQS_old.py

@@ -1,12 +1,14 @@
 import boto3
 import os
 import sys
+
 sys.path.append(os.getcwd())
 sys.path.append('/home/dtomlinson/projects/slack-bot/traffic-scraper/prd')
 from pullTrafficInfo import getTrafficInfo
 import base64
+
 # import emoji
-import hashlib
+# import hashlib
 from datetime import datetime
 
 motorway = 'A3'
@@ -17,8 +19,12 @@ sqs = session.client('sqs')
 
 # queue = sqs.create_queue(QueueName='test', Attributes={'DelaySeconds': '5'})
 
-inst = getTrafficInfo.getTrafficURL(motorway).findIncidents() \
+inst = (
-    .getIncidentInformation().generateOutput()
+    getTrafficInfo.getTrafficURL(motorway)
+    .findIncidents()
+    .getIncidentInformation()
+    .generateOutput()
+)
 
 # encoded = base64.b64encode(inst.output[0].encode())
 # decoded = base64.b64decode(encoded).decode()
@@ -51,13 +57,12 @@ queueURL = 'https://sqs.eu-west-1.amazonaws.com/745437999005/slack-bot.fifo'
 #     # print(encoded.encode())
 #     # print(base64.b64decode(encoded.encode()).decode())
 
-queueAttributes = sqs\
+queueAttributes = sqs.get_queue_attributes(
-    .get_queue_attributes(QueueUrl=queueURL,
+    QueueUrl=queueURL, AttributeNames=['ApproximateNumberOfMessages']
-                          AttributeNames=['ApproximateNumberOfMessages'])
+)
 
 
-response = sqs.receive_message(QueueUrl=queueURL,
+response = sqs.receive_message(QueueUrl=queueURL, MaxNumberOfMessages=10)
-                               MaxNumberOfMessages=10)
 if 'Messages' in response:
     for message in response['Messages']:
         # print(message['Body'])

slack-bot/traffic-scraper/prd/pullTrafficInfo.py

@@ -37,6 +37,7 @@ class getTrafficInfo(object):
     Generate a sequential list for an output:
     getTrafficInfo.getOutput()
     """
+
     def __init__(self, browser, motorway):
         super(getTrafficInfo, self).__init__()
         self.browser = browser
@@ -49,8 +50,11 @@ class getTrafficInfo(object):
 
     @classmethod
     def getTrafficURL(cls, motorway, driver='chrome'):
-        url = ('https://www.trafficdelays.co.uk/' + motorway.lower()
+        url = (
-               + '-traffic-delays')
+            'https://www.trafficdelays.co.uk/'
+            + motorway.lower()
+            + '-traffic-delays'
+        )
         browser = getTrafficInfo.getWebDriver(driver, url)
         getTrafficInfo.verfiyMotorway(browser, motorway)
         return cls(browser, motorway)
@@ -72,13 +76,16 @@ class getTrafficInfo(object):
     @staticmethod
     def verfiyMotorway(browser, motorway):
         try:
-            verify = browser.find_element_by_xpath('/html/body/div[1]/div/div/'
+            verify = browser.find_element_by_xpath(
-                                                   'div/section/div/div/'
+                '/html/body/div[1]/div/div/' 'div/section/div/div/' 'div[1]'
-                                                   'div[1]')
+            )
-            if verify.text in ('It looks like the link pointing here '
+            if verify.text in (
-                               'was faulty. Maybe try searching?'):
+                'It looks like the link pointing here '
-                raise Exception(f'No traffic information available for'
+                'was faulty. Maybe try searching?'
-                                f' {motorway}')
+            ):
+                raise Exception(
+                    f'No traffic information available for' f' {motorway}'
+                )
         except NoSuchElementException:
             pass
 
@@ -86,8 +93,9 @@ class getTrafficInfo(object):
         self.incidentBrowser = []
         for item in self.incidentTypes:
             xpath = f'//*[@id="{item}"]'
-            self.incidentBrowser.append(self.browser.find_element_by_xpath
+            self.incidentBrowser.append(
-                                        (xpath))
+                self.browser.find_element_by_xpath(xpath)
+            )
         self.getIncidentHTML()
         return self
 
@@ -99,8 +107,9 @@ class getTrafficInfo(object):
 
     def getIncidentCount(self):
         self.incidentCount = []
-        for item, i in zip(self.incidentBrowser,
+        for item, i in zip(
-                           range(0, len(self.incidentHTML))):
+            self.incidentBrowser, range(0, len(self.incidentHTML))
+        ):
             self.incidentCount.append(self.incidentHTML[i].count('<li>'))
         return self
 
@@ -114,44 +123,60 @@ class getTrafficInfo(object):
     def generateOutput(self):
         self.getIncidentCount()
         self.output = []
-        self.sarcasticMessage = [(f'Hey Andy, have you thought about getting'
+        self.sarcasticMessage = [
+            (
+                f'Hey Andy, have you thought about getting'
                 ' the train?'
-                                  + emoji.emojize(f' :bullettrain_front:',
+                + emoji.emojize(f' :bullettrain_front:', use_aliases=True)
-                                                  use_aliases=True)),
+            ),
-                                 (f'Hey Andy, maybe flying would be quicker?'
+            (
-                                  + emoji.emojize(f' :helicopter:',
+                f'Hey Andy, maybe flying would be quicker?'
-                                                  use_aliases=True)),
+                + emoji.emojize(f' :helicopter:', use_aliases=True)
-                                 (f'Don\'t fret, Andy can always work from'
+            ),
+            (
+                f'Don\'t fret, Andy can always work from'
                 ' home!'
-                                  + emoji.emojize(f' :house_with_garden:',
+                + emoji.emojize(f' :house_with_garden:', use_aliases=True)
-                                                  use_aliases=True))]
+            ),
+        ]
         currentTime = datetime.now().strftime('%H:%M:%S')
         self.output.append('START')
-        self.output.append(emoji.emojize(f'Did someone say {self.motorway}!?'
+        self.output.append(
+            emoji.emojize(
+                f'Did someone say {self.motorway}!?'
                 ' :anguished:'
                 ' Let\'s check the latest updates'
                 ' from Highways England as of'
                 f' {currentTime}!'
                 ' :police_car::rotating_light:',
-                                         use_aliases=True))
+                use_aliases=True,
-        for item, i in zip(self.incidentCount,
+            )
-                           range(0, len(self.incidentTypes))):
+        )
+        for item, i in zip(
+            self.incidentCount, range(0, len(self.incidentTypes))
+        ):
             if item == 0:
-                self.output.append(emoji.emojize
+                self.output.append(
-                                   (f'There are currently no'
+                    emoji.emojize(
+                        f'There are currently no'
                         ' reported'
                         f' {self.incidentTypes[i]} incidents on'
                         f' the {self.motorway} :thinking_face:',
-                                    use_aliases=True))
+                        use_aliases=True,
+                    )
+                )
                 pass
             else:
-                self.output.append(emoji.emojize
+                self.output.append(
-                                   (f'There are currently'
+                    emoji.emojize(
+                        f'There are currently'
                         f' {self.incidentCount[i]} reported'
                         f' {self.incidentTypes[i]} incidents'
                         f' reported on the {self.motorway}'
                         f' :scream:',
-                                    use_aliases=True))
+                        use_aliases=True,
+                    )
+                )
                 self.output.append(self.incidentInformation[0][i][0])
         self.output.append(random.choice(self.sarcasticMessage))
         self.output.append('END')

slack-bot/traffic-scraper/prd/receiveFromSQS.py

@@ -1,5 +1,6 @@
 import boto3
 import base64
+import math
 
 
 class receiveFromSQS(object):
@@ -19,59 +20,80 @@ class receiveFromSQS(object):
     def getQueueLength(self):
         attributeNames = ['ApproximateNumberOfMessages']
         self.queueAttributes = self.sqs.get_queue_attributes(
-            QueueUrl=self.queueURL,
+            QueueUrl=self.queueURL, AttributeNames=attributeNames
-            AttributeNames=attributeNames,
         )
         self.queueLength = int(
-            self.queueAttributes['Attributes'][
+            self.queueAttributes['Attributes']['ApproximateNumberOfMessages']
-                'ApproximateNumberOfMessages'
-            ]
         )
         return self.queueLength
 
-    def _receiveSQSMessage(self, maxNumberOfMessages=10):
+    def _receiveSQSMessage(
-        response = self.sqs.receive_message(
+        self,
+        totalNumberOfMessages,
+        maxNumberOfMessages=10,
+        deleteOnReceipt=False,
+    ):
+        self.response = []
+        loops = int(math.ceil(totalNumberOfMessages / maxNumberOfMessages))
+        for i in range(0, loops):
+            self.response.append(
+                self.sqs.receive_message(
                     QueueUrl=self.queueURL,
                     MaxNumberOfMessages=maxNumberOfMessages,
                 )
-        return response
-
-    def _extractMessageFromSQS(self):
-        pass
-
-    def receiveMessage(
-        self,
-        b64=True,
-        MaxNumberOfMessages=10,
-        delete=False,
-        endString='END',
-    ):
-        while True:
-            response = self.sqs.receive_message(
-                QueueUrl=self.queueURL,
-                MaxNumberOfMessages=MaxNumberOfMessages,
             )
-            if 'Messages' in response:
+        return self
-                for message in response['Messages']:
+
+    def _extractMessageFromSQS(self, totalNumberOfMessages):
+        self.messages = []
+        try:
+            loops = len(self.response)
+            for i in range(0, loops):
+                if 'Messages' in self.response[i]:
+                    _message = self.response[i]['Messages']
+                    for rawMessage in _message:
+                        self.messages.append(rawMessage['Body'])
+                else:
+                    print("No messages in the queue")
+        except KeyError:
+            print("No messages in the queue")
+        return self
+
+    # def receiveMessage(
+    #     self, b64=True, MaxNumberOfMessages=10, delete=False, endString='END'
+    # ):
+    #     while True:
+    #         response = self.sqs.receive_message(
+    #             QueueUrl=self.queueURL,
+    # MaxNumberOfMessages=MaxNumberOfMessages
+    #         )
+    #         if 'Messages' in response:
+    #             for message in response['Messages']:
+    #                 if b64:
+    #                     decoded = base64.b64decode(message['Body'])
+    #                     msg = decoded.decode()
+    #                     if msg == endString:
+    #                         return self
+    #                 else:
+    #                     msg = print(message['Body'])
+    #                     if msg == endString:
+    #                         return self
+    #         else:
+    #             print('Queue is empty')
+    #             break
+
+    def receiveAllMessages(self, b64=True, deleteOnReceipt=False):
+        totalNumberOfMessages = self.getQueueLength()
+        self._receiveSQSMessage(
+            totalNumberOfMessages,
+            deleteOnReceipt=deleteOnReceipt
+        )
+        self._extractMessageFromSQS(
+            totalNumberOfMessages=totalNumberOfMessages
+        )
         if b64:
-                        decoded = base64.b64decode(
+            pass  # WRITE THIS IN!
-                            message['Body']
-                        )
-                        msg = decoded.decode()
-                        print(msg)
-                        if msg == endString:
         return self
-                    else:
-                        msg = print(message['Body'])
-                        if msg == endString:
-                            return self
-            else:
-                print('Queue is empty')
-                break
-
-    def receiveAllMessages(self, b64=True, delete=False):
-        while True:
-            self.response = self._receiveSQSMessage()
 
 
 inst = receiveFromSQS.createSession(
@@ -79,4 +101,16 @@ inst = receiveFromSQS.createSession(
     queueURL='https://sqs.eu-west-1.amazonaws'
     '.com/745437999005/slack-bot.fifo',
 )
-inst.getQueueLength()
+
+inst.receiveAllMessages()
+for item in inst.messages:
+    print(item)
+
+# queueLength = inst.getQueueLength()
+
+# response = inst._receiveSQSMessage(queueLength)._extractMessageFromSQS(
+#     queueLength
+# )
+# # messages = inst._receiveSQSMessage(12)._extractMessageFromSQS(12)
+# for item in response.message:
+#     print(item)

slack-bot/traffic-scraper/prd/sendToSQS.py

@@ -4,16 +4,18 @@ import hashlib
 from datetime import datetime
 import os
 import sys
+
 sys.path.append(os.getcwd())
 from pullTrafficInfo import getTrafficInfo
 
 
 class sendToSQS(object):
     """docstring for sendToSQS"""
+
     def __init__(self, session, queueURL):
         super(sendToSQS, self).__init__()
         self.session = session
-        self.sqs = session.client('sqs')
+        self.sqs = session.client("sqs")
         self.queueURL = queueURL
 
     @classmethod
@@ -22,29 +24,39 @@ class sendToSQS(object):
         return cls(session, queueURL)
 
     def sendMessage(self, message, messageGroupId, b64=True, dedup=False):
-        currentTime = datetime.now().strftime('%H:%M:%S.%f')
+        currentTime = datetime.now().strftime("%H:%M:%S.%f")
         if b64:
             message = (base64.b64encode(message.encode())).decode()
         if not dedup:
             dedupId = hashlib.md5((message + currentTime).encode()).hexdigest()
-            msg = self.sqs.send_message(QueueUrl=self.queueURL,
+            msg = self.sqs.send_message(
+                QueueUrl=self.queueURL,
                 MessageBody=message,
                 MessageGroupId=messageGroupId,
-                                        MessageDeduplicationId=dedupId)
+                MessageDeduplicationId=dedupId,
+            )
         else:
-            msg = self.sqs.send_message(QueueUrl=self.queueURL,
+            msg = self.sqs.send_message(
+                QueueUrl=self.queueURL,
                 MessageBody=message,
-                                        MessageGroupId=messageGroupId)
+                MessageGroupId=messageGroupId,
+            )
         if msg is not None:
             print(msg["MessageId"])
 
 
-inst = sendToSQS.createSession(profileName='plex-aws',
+inst = sendToSQS.createSession(
-                               queueURL='https://sqs.eu-west-1.amazonaws.com'
+    profileName="plex-aws",
-                                        '/745437999005/slack-bot.fifo')
+    queueURL="https://sqs.eu-west-1.amazonaws.com"
+    "/745437999005/slack-bot.fifo",
+)
 
-instM = getTrafficInfo.getTrafficURL('M62').findIncidents() \
+instM = (
-    .getIncidentInformation().generateOutput()
+    getTrafficInfo.getTrafficURL("M62")
+    .findIncidents()
+    .getIncidentInformation()
+    .generateOutput()
+)
 
 for item in instM.output:
-    inst.sendMessage(message=item, messageGroupId='slack-bot-M62')
+    inst.sendMessage(message=item, messageGroupId="slack-bot-M62")