From 36bf49f0ca189fe78566e6e698c1d8ba9fd86956 Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Fri, 11 Oct 2019 01:07:42 +0100 Subject: [PATCH] completed receiveFromSQS.py --- .../prd/11-10-2019_00:30:27.json | 1 + .../pullTrafficInfo.cpython-37.pyc | Bin 5044 -> 5192 bytes .../traffic-scraper/prd/pullTrafficInfo.py | 13 ++-- .../traffic-scraper/prd/receiveFromSQS.py | 57 +++++++++++++----- 4 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 slack-bot/traffic-scraper/prd/11-10-2019_00:30:27.json diff --git a/slack-bot/traffic-scraper/prd/11-10-2019_00:30:27.json b/slack-bot/traffic-scraper/prd/11-10-2019_00:30:27.json new file mode 100644 index 0000000..7c384a9 --- /dev/null +++ b/slack-bot/traffic-scraper/prd/11-10-2019_00:30:27.json @@ -0,0 +1 @@ +["START", "Did someone say M62!? \ud83d\ude27 Let's check the latest updates from Highways England as of 00:30:21! \ud83d\ude93\ud83d\udea8", "There are currently no reported congestion incidents on the M62 \ud83e\udd14", "There are currently no reported accident incidents on the M62 \ud83e\udd14", "Hey Andy, have you thought about getting the train? \ud83d\ude85", "END"] \ No newline at end of file diff --git a/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc b/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc index a78f1a7769435790789439f93d3f0a845b43ca17..0678d9c380cece1a110a93f226090175e1baf202 100644 GIT binary patch delta 1164 zcmZWoO;6N781C%0``PX%SwIs&Ism+9015;=I0 zlj2+rE=Qw@A$Ty+#4EpH;@OLT!g*T^s5_ZwXWn_hqG_;TnyAnnq`!?D9Ju3zpkc5fX7T?J;j7aePWYIrb&? zNy!6^+40-B04ougY7f?lTgM9xkoUN%n`T8PRlklM%46Xok5)0|$>}Wv2eh(GN+8e* za0%ew?I2b>L5XEF&oSzed8pLIb;yPEh8aiFSD9B!=UL(%B(w9fB2_ESNvL-|-I;F%~^(ih# z0twO!>lcm-rai2Y{yGMkTp_x+0UounL$1SGAf*_t@6aDM|r_9Z?qmOJqL#MVocBL9>CIn`8qwSKJ3HybUF^632C!E%8oJS<*@<=4YcicJ)I_l6;F3}8}#INH@3HfW#}I1L@0 z`;aZiqvSp$z|nA}K`e)!`w2(jh9}1AuIbx&9Eq6+<*5i-0$Mb;CykUM5f%Lg!Qd6v delta 967 zcmZWnU279T6y0gEyJ?b5nr;0^3;nifw6tO=Dg;4&P@%ShMPWh1c6ZuLyB~3OR#f@} zd~6+jDYmq4BBDrzK8OfD`2&1$^w|e}@*g-iTW!!?IJ0x_nX~8KJDj3U(ys=dYBkV+yKwy`<)-IKn=jdz3tmdl4V^xB_0-6W$j;!Pmr}2bPb^alb!_AS{Sot%!G>laD`ZU zYk7n%{;D))Ls=B&Lr(yV0*nDv0AMWilmwy4ez>78G^Y^~2UpX{AaYJjsT|G%@o`d6 z6^^v^m0vWEqZ+r&^QfP1niW*!AI)i^gJ{n2>?ravYlJR63Xy!oY(|`ePzkh5fMI|^ z04Rx00i2T{jC#E#;$Y9K*AIVOMKlkj0>9fmVOFzNCc<)-B=(d0&Ud@V<|cq~24DuX kv~Y0+JB+FjIV|DtT@{&{AG)!JFCfW?KtGS*&A^-b4bN!fj{pDw diff --git a/slack-bot/traffic-scraper/prd/pullTrafficInfo.py b/slack-bot/traffic-scraper/prd/pullTrafficInfo.py index 6212801..f44ec19 100644 --- a/slack-bot/traffic-scraper/prd/pullTrafficInfo.py +++ b/slack-bot/traffic-scraper/prd/pullTrafficInfo.py @@ -177,13 +177,16 @@ class getTrafficInfo(object): use_aliases=True, ) ) - self.output.append(self.incidentInformation[0][i][0]) + try: + self.output.append(self.incidentInformation[i][0][0]) + except IndexError: + pass self.output.append(random.choice(self.sarcasticMessage)) self.output.append('END') return self -# inst = getTrafficInfo.getTrafficURL('A50').findIncidents() \ -# .getIncidentInformation().generateOutput() -# for i in inst.output: -# print(i) +inst = getTrafficInfo.getTrafficURL('M62').findIncidents() \ + .getIncidentInformation().generateOutput() +for i in inst.output: + print(i) diff --git a/slack-bot/traffic-scraper/prd/receiveFromSQS.py b/slack-bot/traffic-scraper/prd/receiveFromSQS.py index dc68e01..d2f92b7 100644 --- a/slack-bot/traffic-scraper/prd/receiveFromSQS.py +++ b/slack-bot/traffic-scraper/prd/receiveFromSQS.py @@ -1,6 +1,8 @@ import boto3 import base64 import math +import json +from datetime import datetime class receiveFromSQS(object): @@ -35,20 +37,18 @@ class receiveFromSQS(object): self.loops = int( math.ceil(totalNumberOfMessages / maxNumberOfMessages) ) + loopTrack = 0 if totalNumberOfMessages <= 10: maxNumberOfMessages = totalNumberOfMessages else: - # Find how many times total-loop*max, on last loop - # replace max with this - maxNumberOfMessagesFinal = self.loops * maxNumberOfMessages - abs(10 - ( - totalNumberOfMessages - (self.loops * maxNumberOfMessages) - )) + maxNumberOfMessagesFinal = 10 - ( + (self.loops * maxNumberOfMessages) - totalNumberOfMessages + ) print(maxNumberOfMessagesFinal) if self.loops == 0: raise RuntimeError('No messages in the queue') for i in range(0, self.loops): - loopTrack = 0 - if loopTrack == self.loops: + if loopTrack == self.loops - 1 and totalNumberOfMessages > 10: maxNumberOfMessages = maxNumberOfMessagesFinal self.resp.append( self.sqs.receive_message( @@ -56,12 +56,18 @@ class receiveFromSQS(object): MaxNumberOfMessages=maxNumberOfMessages, ) ) - entries = [ # Needs Keyerror try - {'Id': msg['MessageId'], 'ReceiptHandle': msg['ReceiptHandle']} - for msg in self.resp[i]['Messages'] - ] - self._deleteSQSMessages(entries) - loopTrack += 1 + try: + entries = [ + { + 'Id': msg['MessageId'], + 'ReceiptHandle': msg['ReceiptHandle'], + } + for msg in self.resp[i]['Messages'] + ] + self._deleteSQSMessages(entries) + loopTrack += 1 + except KeyError: + print("No messages in the queue") return self def _extractMessageFromSQS(self, totalNumberOfMessages): @@ -119,7 +125,22 @@ class receiveFromSQS(object): self.receiveAllMessages( b64=b64, _totalNumberOfMessages=numberOfMessages ) - pass + return self + + def generateOutput(self, type='json'): + if type == 'json': + self.output = json.dumps(self.messages) + return self.output + + def savetoDisk(self, path): + self.timeNow = datetime.now().strftime('%d-%m-%Y_%H:%M:%S') + if self.output is None: + self.generateOutput() + if len(self.messages) > 0: + with open(f'{path}/{self.timeNow}.json', 'w+') as outputFile: + outputFile.write(self.output) + else: + print('No messages to save') inst = receiveFromSQS.createSession( @@ -128,6 +149,12 @@ inst = receiveFromSQS.createSession( '.com/745437999005/slack-bot.fifo', ) -inst.receiveNMessages(numberOfMessages=12) +output = inst.receiveNMessages(numberOfMessages=6).generateOutput(type='json') +inst.savetoDisk( + '/Users/dtomlinson/OneDrive - William Hill' + ' Organisation Limited/Mac/git_repos/python-VM/slack-bot/traffic-scraper' + '/prd' +) +# inst.receiveAllMessages() for item in inst.messages: print(item)