From a843d35c7b98662c119d12822b1f034dc7359fef Mon Sep 17 00:00:00 2001 From: Daniel Tomlinson Date: Sun, 6 Oct 2019 23:04:08 +0100 Subject: [PATCH] v0.1 of send and receive scripts --- .../pullTrafficInfo.cpython-37.pyc | Bin 4973 -> 4912 bytes slack-bot/pullTrafficInfo.py | 2 +- slack-bot/requirements.txt | 16 -- slack-bot/{ => screenshot}/cropped.png | Bin slack-bot/{ => screenshot}/screenshot.png | Bin slack-bot/{ => screenshot}/screenshot.py | 0 slack-bot/{ => screenshot}/screenshotFull.py | 0 slack-bot/{ => screenshot}/web_screenshot.png | Bin .../{ => traffic-scraper/dev}/scraper.py | 0 .../{ => traffic-scraper/dev}/scraper.py.old | 0 .../dev/sendToSQS_old.py} | 31 ++-- .../pullTrafficInfo.cpython-37.pyc | Bin 0 -> 4975 bytes .../traffic-scraper/prd/pullTrafficInfo.py | 164 ++++++++++++++++++ .../traffic-scraper/prd/receiveFromSQS.py | 44 +++++ slack-bot/traffic-scraper/prd/sendToSQS.py | 50 ++++++ 15 files changed, 279 insertions(+), 28 deletions(-) delete mode 100644 slack-bot/requirements.txt rename slack-bot/{ => screenshot}/cropped.png (100%) rename slack-bot/{ => screenshot}/screenshot.png (100%) rename slack-bot/{ => screenshot}/screenshot.py (100%) rename slack-bot/{ => screenshot}/screenshotFull.py (100%) rename slack-bot/{ => screenshot}/web_screenshot.png (100%) rename slack-bot/{ => traffic-scraper/dev}/scraper.py (100%) rename slack-bot/{ => traffic-scraper/dev}/scraper.py.old (100%) rename slack-bot/{sendToSQS.py => traffic-scraper/dev/sendToSQS_old.py} (60%) create mode 100644 slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc create mode 100644 slack-bot/traffic-scraper/prd/pullTrafficInfo.py create mode 100644 slack-bot/traffic-scraper/prd/receiveFromSQS.py create mode 100644 slack-bot/traffic-scraper/prd/sendToSQS.py diff --git a/slack-bot/__pycache__/pullTrafficInfo.cpython-37.pyc b/slack-bot/__pycache__/pullTrafficInfo.cpython-37.pyc index a8a99081727e4e9245da21bfbc0ca21881d052d0..49036d280b132cd7fa83d8b8cd675bbe027e3e39 100644 GIT binary patch delta 52 zcmaE>wn2^CiI~vm6}{qyt#t) GD=z?yCJ`6_ delta 113 zcmdm>_EwGCiIZ{zd7Dd6~tDC7Jno3O<>+nI)+y`o4+D`stY^@kOZx`NjGLl_eSZ OdAeb~o2^;D@&W)LpeFDD diff --git a/slack-bot/pullTrafficInfo.py b/slack-bot/pullTrafficInfo.py index c3a5107..7d3af4b 100644 --- a/slack-bot/pullTrafficInfo.py +++ b/slack-bot/pullTrafficInfo.py @@ -147,7 +147,7 @@ class getTrafficInfo(object): f' reported on the {self.motorway}' f' :scream:', use_aliases=True)) - self.output.append(self.incidentInformation[i][0][0]) + self.output.append(self.incidentInformation[0][i][0]) self.output.append(random.choice(self.sarcasticMessage)) return self diff --git a/slack-bot/requirements.txt b/slack-bot/requirements.txt deleted file mode 100644 index 95dd7d2..0000000 --- a/slack-bot/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -beautifulsoup4==4.8.0 -bs4==0.0.1 -certifi==2019.9.11 -chardet==3.0.4 -emoji==0.5.4 -idna==2.8 -lxml==4.4.1 -numpy==1.17.2 -pandas==0.25.1 -python-dateutil==2.8.0 -pytz==2019.2 -requests==2.22.0 -selenium==3.141.0 -six==1.12.0 -soupsieve==1.9.4 -urllib3==1.25.6 diff --git a/slack-bot/cropped.png b/slack-bot/screenshot/cropped.png similarity index 100% rename from slack-bot/cropped.png rename to slack-bot/screenshot/cropped.png diff --git a/slack-bot/screenshot.png b/slack-bot/screenshot/screenshot.png similarity index 100% rename from slack-bot/screenshot.png rename to slack-bot/screenshot/screenshot.png diff --git a/slack-bot/screenshot.py b/slack-bot/screenshot/screenshot.py similarity index 100% rename from slack-bot/screenshot.py rename to slack-bot/screenshot/screenshot.py diff --git a/slack-bot/screenshotFull.py b/slack-bot/screenshot/screenshotFull.py similarity index 100% rename from slack-bot/screenshotFull.py rename to slack-bot/screenshot/screenshotFull.py diff --git a/slack-bot/web_screenshot.png b/slack-bot/screenshot/web_screenshot.png similarity index 100% rename from slack-bot/web_screenshot.png rename to slack-bot/screenshot/web_screenshot.png diff --git a/slack-bot/scraper.py b/slack-bot/traffic-scraper/dev/scraper.py similarity index 100% rename from slack-bot/scraper.py rename to slack-bot/traffic-scraper/dev/scraper.py diff --git a/slack-bot/scraper.py.old b/slack-bot/traffic-scraper/dev/scraper.py.old similarity index 100% rename from slack-bot/scraper.py.old rename to slack-bot/traffic-scraper/dev/scraper.py.old diff --git a/slack-bot/sendToSQS.py b/slack-bot/traffic-scraper/dev/sendToSQS_old.py similarity index 60% rename from slack-bot/sendToSQS.py rename to slack-bot/traffic-scraper/dev/sendToSQS_old.py index dce2d90..0e8d505 100644 --- a/slack-bot/sendToSQS.py +++ b/slack-bot/traffic-scraper/dev/sendToSQS_old.py @@ -2,11 +2,15 @@ import boto3 import os import sys sys.path.append(os.getcwd()) +sys.path.append('/home/dtomlinson/projects/slack-bot/traffic-scraper/prd') from pullTrafficInfo import getTrafficInfo import base64 -import emoji +# import emoji +import hashlib +from datetime import datetime -motorway = 'M1' +motorway = 'A3' +currentTime = datetime.now().strftime('%H:%M:%S') session = boto3.Session(profile_name='plex-aws') sqs = session.client('sqs') @@ -33,18 +37,23 @@ queueURL = 'https://sqs.eu-west-1.amazonaws.com/745437999005/slack-bot.fifo' # else: # print('Queue is empty') -for item in inst.output: - encoded = (base64.b64encode(item.encode())).decode() - sqs.send_message(QueueUrl=queueURL, - MessageBody=encoded, - MessageGroupId=f'slack-bot-{motorway}') - # print(encoded) - # print(encoded.encode()) - # print(base64.b64decode(encoded.encode()).decode()) +# for item in inst.output: +# print(item) +# encoded = (base64.b64encode(item.encode())).decode() +# dedupId = hashlib.md5((item + currentTime).encode()).hexdigest() +# msg = sqs.send_message(QueueUrl=queueURL, +# MessageBody=encoded, +# MessageGroupId=f'slack-bot-{motorway}', +# MessageDeduplicationId=dedupId) +# if msg is not None: +# print(msg["MessageId"]) +# # print(encoded) +# # print(encoded.encode()) +# # print(base64.b64decode(encoded.encode()).decode()) response = sqs.receive_message(QueueUrl=queueURL, - MaxNumberOfMessages=4) + MaxNumberOfMessages=10) if 'Messages' in response: for message in response['Messages']: # print(message['Body']) diff --git a/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc b/slack-bot/traffic-scraper/prd/__pycache__/pullTrafficInfo.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c32afae98b206e7876e5f7963b7fcf32918bff34 GIT binary patch literal 4975 zcmai2TW=f372X>!qDV?sWI0Y!XYAUEsMsQU0}8^ilh{U_AXb7{X&Oe(V!1OES6c4U zvqQ^VP<^NjvC$~~2ULxQnJpZM4wu0*Au(9niXRXC-h)u|MWh}) z_r*X3afF#N_mogUUu^lbae5JxbR`iQVYE$dv@LG7?eC4Q$t`ZbD7PI^;1@Lu=8@Sh ziV`pJ@(Tm~GWrX=qWcSckyl?B?Fz5)I-ZMsi7(?>8JGm{1j%pX=O*o2Z@l)e2S@<$zjBX5f_rJ`#ta{7#Ce{7^+6j z)K`hOD_VJfV}nWYbQnm%*~W(c#yu$j2soQiBn(FJP|i5XMLCP>=IQXOFF#wKGJSJG zWb);WjaVy7GWpcULj4TxAo2qa&0=aa5EIx=iPITwc7uq2GFg_aPiPaZhue=nn;Z#{ z8E7%-tk3PoaJrp#%v11i5JP-C?y_EdfRm3#;4tSX(dr;Pzo&V76%13-_lVFuZ8|L_ zo+}&&jyjwk^_2Yoqer4o^zxKyBxHVA2~H8H1zyO)0NSiIH)8Qn4TkDfq(9fQphh71 zunc|Cb9yK17$`%TW8=s=woVMw7^CmnV+*arVZgi8gGYh6*<0-j6nPis1nX&+JwHdT zQOpX-a3G{CfLvB~U1+*;-K;dx-mE;;n5;I%W1G&uVP{Sv!mcc1qg+5k{>XLx&`T26 z{fF`2Pn$icc$2HR4@i?ZY7S()Cw!GOlhE_`Z*^kToY?lQ#Frk}z!KgZzyZv8hRwl9 zmO!5bPUvffSvQOLrI*gB&y4$GZeBCsNI3F#_i!h_MZ?Xpd2E~*m*9>s8Hctqk1YUa zpICgt5L{oKTI^1SuwZq%pS@xhqwl|rEcozWwle;Vb>&6j*Y!+ zR`E;UI-S?1c{;>$-@;2hiBIx$UX+@3Q=aQMMI zw8!>w0YKsH*nVuNB6r5dkBC?=?IZKA2q%E8xFwfBw_WZDkB1^jpeCh>vq?)i5-hE= zC{_%bIEbYZyit&yPn(~Qj}{K^U2JHeA&Wj&~xM}5x7Xr zC2F*qp2bjJz(fmoLLW1%vRTGmv8rapbgcBvQ^=+W&dzQIAekeP)|&MJI-_?8LJm*} zv^|*NeoU6H37ZfE2^A9$*l zIpj&@8-PL*I1T&MLu5s0e9#@`YCO|rxq@{MaVJ$YhGm%*yJD?abu<0dxidIxF}_Lb zanXm0xb-P~gw8G$*fpMSj*UI@zVYPxp-C1G)wEvPI2U8jfV~@g_AylXL@V&q`q!lV z1?@<4mzgd!n;YML5b*b}rPj44@Rdtb7x+;g+a+aySrML?GT|t zV*p=Ax=JRTBLEGww%Q9pgaD>lce3hKY8uNcASd5O(dJ1V$XQ)KO(lW}cG=I*} z7~Xw{d*Z%HqBS$Ua_(@?fi{!o%;}PzOr(Kxf)tj51PYF4uW$rUePrRh*^YHc_D3=n zX=ZF)HjYc=z_g@{&RY9)Nh*{&x5MC0qkJ0Tbc&GKX)w_8W(DaXcxHAeBKd3V&;Vow zosvurR|}q*!8zHpxs@4#LuFQw5mIbmm~Nq{vF7h+Y^@HN6C1fb2T@C3VQw<{+6Pxjzq#108wYOmDmz zn7N%Zcqo>UrD=HngtN`#nVG``C!nl7#4&Dj=LZCSUeM#>Y+Tag@^rjwQ!b?n$Hfy; zp9MH|QaSUu%qu$T>Av2xi1!O{FjehlYA*{p4O;d5pQ?h{MOEc>RYR{nQ`EmfQ7yz# zi`Nlt|CQ6$^pv_)vRU(Kv@o=}N znAbsiOUb>CJ376VsG;trtE|;Qc83WXyC~8k)!J^D?ZsgtTrUi~L?r3@De1lrQQdGv z0qh_ihMXZ}p)T_Gg}j^ASgR+(z(+zMWGj8|ejL4v6k7^)Q}f1rFJfMZf-u2uxu4e{ zl$Ty5UcfFsdl0CeyX#35H?6dM<6-N@qt=bb?ZV^j55L?_m+uFhCD;*1f+eUd(@(G7 zWi9xFVUY9$Z?VsWdN*NyPx$-uW$SQ2WjCTNRiY1r-Ja&+L9`os5oZX)C}q+sS6OQi zLjvLYo@}*btWe-ayKaaLs+GGZwyi_Ahw$wWr9|Enj*u{+K+Mx6mafdzLa2@j@pPHB z6hitwQR{lXXr-&aL?^Ax(Rp)@&W!q$-~wwQO&4CjmDvxr?l)HC223q0=A}$l5dC;B zXqWUO7478&IfaLU!+(Ud4<(-@>&)yZKFG=_uDkS)gm$Dl<&`%uE3Z@Y4mIymLvB*O zhbAk+1M|3_A+?WzR>R37>f!{DZK9<^PlL9cjX(5XLtc;Nb(O@Dgc3gk{U?@vE9|6)W01BO^3lcPdy_F|QXY<`r`lvp>sB&VHd`>a4f1OeIx>e;QCyRb00p z^I=H+s_Q--df{ZI@*4FRHFOZN zNzEV8WVJ~h=8G@{*%=yOi&88}iG{BCsBTN>8|5o-xE0H+ma6q?t-Me!mP_T5Uel a string containing the A-road or motorway + driver: str -> the browser/driver to run under (chrome default) + + Usage: + + ** required ** + Create a driver for your motorway: + getTrafficInfo.getTrafficURL(motorway, driver) + + **options** + Get information for incident types: + getTrafficInfo.findIncidents() + + Get HTML from the page: + getTrafficInfo.getIncidentHTML() + + Get a count of how many incidents for each type: + getTrafficInfo.getIncidentCount() + + Get the text detailing the incident: + getTrafficInfo.getIncidentInformation() + + Generate a sequential list for an output: + getTrafficInfo.getOutput() + """ + def __init__(self, browser, motorway): + super(getTrafficInfo, self).__init__() + self.browser = browser + self.motorway = motorway + self.incidentTypes = ['congestion', 'accident'] + + # def __del__(self): + # print('Quitting') + # self.browser.quit() + + @classmethod + def getTrafficURL(cls, motorway, driver='chrome'): + url = ('https://www.trafficdelays.co.uk/' + motorway.lower() + + '-traffic-delays') + browser = getTrafficInfo.getWebDriver(driver, url) + getTrafficInfo.verfiyMotorway(browser, motorway) + return cls(browser, motorway) + + @staticmethod + def getWebDriver(driver, url): + driver = driver.lower() + if driver == 'chrome': + # Prepare the option for the Chromedriver + options = webdriver.ChromeOptions() + options.add_argument('headless') + # Start the Chromedriver + browser = webdriver.Chrome(options=options) + browser.get(url) + return browser + else: + raise Exception(f'Driver {driver} not supported') + + @staticmethod + def verfiyMotorway(browser, motorway): + try: + verify = browser.find_element_by_xpath('/html/body/div[1]/div/div/' + 'div/section/div/div/' + 'div[1]') + if verify.text in ('It looks like the link pointing here ' + 'was faulty. Maybe try searching?'): + raise Exception(f'No traffic information available for' + f' {motorway}') + except NoSuchElementException: + pass + + def findIncidents(self): + self.incidentBrowser = [] + for item in self.incidentTypes: + xpath = f'//*[@id="{item}"]' + self.incidentBrowser.append(self.browser.find_element_by_xpath + (xpath)) + self.getIncidentHTML() + return self + + def getIncidentHTML(self): + self.incidentHTML = [] + for item in self.incidentBrowser: + self.incidentHTML.append(item.get_attribute('innerHTML')) + return self + + def getIncidentCount(self): + self.incidentCount = [] + for item, i in zip(self.incidentBrowser, + range(0, len(self.incidentHTML))): + self.incidentCount.append(self.incidentHTML[i].count('
  • ')) + return self + + def getIncidentInformation(self): + self.incidentInformation = [] + pattern = r".*\.*title=\".*\".*\>((.|\n)*?)\" + for item in self.incidentHTML: + self.incidentInformation.append(re.findall(pattern, item)) + return self + + def generateOutput(self): + self.getIncidentCount() + self.output = [] + self.sarcasticMessage = [(f'Hey Andy, have you thought about getting' + ' the train?' + + emoji.emojize(f' :bullettrain_front:', + use_aliases=True)), + (f'Hey Andy, maybe flying would be quicker?' + + emoji.emojize(f' :helicopter:', + use_aliases=True)), + (f'Don\'t fret, Andy can always work from' + ' home!' + + emoji.emojize(f' :house_with_garden:', + use_aliases=True))] + currentTime = datetime.now().strftime('%H:%M:%S') + self.output.append('START') + self.output.append(emoji.emojize(f'Did someone say {self.motorway}!?' + ' :anguished:' + ' Let\'s check the latest updates' + ' from Highways England as of' + f' {currentTime}!' + ' :police_car::rotating_light:', + use_aliases=True)) + for item, i in zip(self.incidentCount, + range(0, len(self.incidentTypes))): + if item == 0: + self.output.append(emoji.emojize + (f'There are currently no' + ' reported' + f' {self.incidentTypes[i]} incidents on' + f' the {self.motorway} :thinking_face:', + use_aliases=True)) + pass + else: + self.output.append(emoji.emojize + (f'There are currently' + f' {self.incidentCount[i]} reported' + f' {self.incidentTypes[i]} incidents' + f' reported on the {self.motorway}' + f' :scream:', + use_aliases=True)) + self.output.append(self.incidentInformation[0][i][0]) + self.output.append(random.choice(self.sarcasticMessage)) + self.output.append('END') + return self + + +# inst = getTrafficInfo.getTrafficURL('A50').findIncidents() \ +# .getIncidentInformation().generateOutput() +# for i in inst.output: +# print(i) diff --git a/slack-bot/traffic-scraper/prd/receiveFromSQS.py b/slack-bot/traffic-scraper/prd/receiveFromSQS.py new file mode 100644 index 0000000..fec997d --- /dev/null +++ b/slack-bot/traffic-scraper/prd/receiveFromSQS.py @@ -0,0 +1,44 @@ +import boto3 +import base64 + + +class receiveFromSQS(object): + """docstring for receiveFromSQS""" + def __init__(self, session, queueURL): + super(receiveFromSQS, self).__init__() + self.session = session + self.sqs = session.client('sqs') + self.queueURL = queueURL + + @classmethod + def createSession(cls, profileName, queueURL): + session = boto3.Session(profile_name=profileName) + return cls(session, queueURL) + + def receiveMessage(self, b64=True, MaxNumberOfMessages=10, delete=False, + endString='END'): + while True: + response = self.sqs\ + .receive_message(QueueUrl=self.queueURL, + MaxNumberOfMessages=MaxNumberOfMessages) + if 'Messages' in response: + for message in response['Messages']: + if b64: + decoded = base64.b64decode(message['Body']) + msg = decoded.decode() + print(msg) + if msg == endString: + return self + else: + msg = print(message['Body']) + if msg == endString: + return self + else: + print('Queue is empty') + break + + +inst = receiveFromSQS.createSession(profileName='plex-aws', + queueURL='https://sqs.eu-west-1.amazonaws' + '.com/745437999005/slack-bot.fifo') +inst.receiveMessage() diff --git a/slack-bot/traffic-scraper/prd/sendToSQS.py b/slack-bot/traffic-scraper/prd/sendToSQS.py new file mode 100644 index 0000000..9d5b479 --- /dev/null +++ b/slack-bot/traffic-scraper/prd/sendToSQS.py @@ -0,0 +1,50 @@ +import boto3 +import base64 +import hashlib +from datetime import datetime +import os +import sys +sys.path.append(os.getcwd()) +from pullTrafficInfo import getTrafficInfo + + +class sendToSQS(object): + """docstring for sendToSQS""" + def __init__(self, session, queueURL): + super(sendToSQS, self).__init__() + self.session = session + self.sqs = session.client('sqs') + self.queueURL = queueURL + + @classmethod + def createSession(cls, profileName, queueURL): + session = boto3.Session(profile_name=profileName) + return cls(session, queueURL) + + def sendMessage(self, message, messageGroupId, b64=True, dedup=True): + currentTime = datetime.now().strftime('%H:%M:%S') + if b64: + message = (base64.b64encode(message.encode())).decode() + if dedup: + dedupId = hashlib.md5((message + currentTime).encode()).hexdigest() + msg = self.sqs.send_message(QueueUrl=self.queueURL, + MessageBody=message, + MessageGroupId=messageGroupId, + MessageDeduplicationId=dedupId) + else: + msg = self.sqs.send_message(QueueUrl=self.queueURL, + MessageBody=message, + MessageGroupId=messageGroupId) + if msg is not None: + print(msg["MessageId"]) + + +inst = sendToSQS.createSession(profileName='plex-aws', + queueURL='https://sqs.eu-west-1.amazonaws.com' + '/745437999005/slack-bot.fifo') + +instM = getTrafficInfo.getTrafficURL('M62').findIncidents() \ + .getIncidentInformation().generateOutput() + +for item in instM.output: + inst.sendMessage(message=item, messageGroupId='slack-bot-M62', dedup=True)