adding all files done so far
This commit is contained in:
11
boto3/blocking_suggestions/.history.txt
Normal file
11
boto3/blocking_suggestions/.history.txt
Normal file
@@ -0,0 +1,11 @@
|
||||
|
||||
blocking_suggestions_2019-07-01T00.csv
|
||||
blocking_suggestions_2019-07-01T02.csv
|
||||
blocking_suggestions_2019-07-01T04.csv
|
||||
blocking_suggestions_2019-07-01T06.csv
|
||||
blocking_suggestions_2019-07-01T08.csv
|
||||
blocking_suggestions_2019-07-01T10.csv
|
||||
blocking_suggestions_2019-07-01T12.csv
|
||||
blocking_suggestions_2019-07-01T14.csv
|
||||
blocking_suggestions_2019-07-01T16.csv
|
||||
blocking_suggestions_2019-07-01T18.csv
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
167
boto3/pull.log
Normal file
167
boto3/pull.log
Normal file
@@ -0,0 +1,167 @@
|
||||
2019-07-01 21:27:23,774 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:27:23,775 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:27:24,113 - INFO - opening history file
|
||||
2019-07-01 21:27:24,113 - CRITICAL - history file cannot be found or created - check permissions of the folder.
|
||||
2019-07-01 21:28:25,693 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:28:25,693 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:28:26,137 - INFO - opening history file
|
||||
2019-07-01 21:28:26,137 - CRITICAL - history file cannot be found or created - check permissions of the folder.
|
||||
2019-07-01 21:28:55,767 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:28:55,767 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:28:56,074 - INFO - opening history file
|
||||
2019-07-01 21:28:56,075 - INFO - attempting to clear current files
|
||||
2019-07-01 21:28:56,075 - INFO - no files to remove
|
||||
2019-07-01 21:28:56,805 - INFO - downloaded blocking_suggestions_2019-07-01T00.csv
|
||||
2019-07-01 21:28:56,947 - INFO - downloaded blocking_suggestions_2019-07-01T02.csv
|
||||
2019-07-01 21:28:57,080 - INFO - downloaded blocking_suggestions_2019-07-01T04.csv
|
||||
2019-07-01 21:28:57,238 - INFO - downloaded blocking_suggestions_2019-07-01T06.csv
|
||||
2019-07-01 21:28:57,435 - INFO - downloaded blocking_suggestions_2019-07-01T08.csv
|
||||
2019-07-01 21:28:57,570 - INFO - downloaded blocking_suggestions_2019-07-01T10.csv
|
||||
2019-07-01 21:28:57,686 - INFO - downloaded blocking_suggestions_2019-07-01T12.csv
|
||||
2019-07-01 21:28:58,131 - INFO - downloaded blocking_suggestions_2019-07-01T14.csv
|
||||
2019-07-01 21:28:58,395 - INFO - downloaded blocking_suggestions_2019-07-01T16.csv
|
||||
2019-07-01 21:28:58,395 - INFO - ========= SCRIPT FINISHED =========
|
||||
2019-07-01 21:31:10,636 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:31:10,637 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:31:10,921 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:31:30,515 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:31:30,515 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:31:31,077 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:31:52,287 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:31:52,288 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:31:52,566 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:32:33,361 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:32:33,361 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:32:33,642 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:32:52,126 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:32:52,127 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:32:52,395 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:33:03,360 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:33:03,360 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:33:03,638 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:33:37,285 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:33:37,285 - INFO - path entered is blocking_suggestions/
|
||||
2019-07-01 21:33:37,571 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:33:59,988 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:33:59,988 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:34:00,308 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:34:34,827 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:34:34,827 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:34:35,107 - INFO - no files available to download -- exiting
|
||||
2019-07-01 21:34:48,900 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:34:48,900 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:34:49,188 - INFO - opening history file
|
||||
2019-07-01 21:34:49,189 - INFO - attempting to clear current files
|
||||
2019-07-01 21:34:49,189 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T14.csv
|
||||
2019-07-01 21:34:49,189 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T04.csv
|
||||
2019-07-01 21:34:49,190 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T16.csv
|
||||
2019-07-01 21:34:49,190 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T06.csv
|
||||
2019-07-01 21:34:49,190 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T00.csv
|
||||
2019-07-01 21:34:49,191 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T08.csv
|
||||
2019-07-01 21:34:49,191 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T12.csv
|
||||
2019-07-01 21:34:49,191 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T02.csv
|
||||
2019-07-01 21:34:49,191 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T10.csv
|
||||
2019-07-01 21:34:50,976 - INFO - downloaded test.json
|
||||
2019-07-01 21:34:50,976 - INFO - ========= SCRIPT FINISHED =========
|
||||
2019-07-01 21:35:10,779 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 21:35:10,779 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 21:35:11,169 - INFO - opening history file
|
||||
2019-07-01 21:35:11,169 - INFO - attempting to clear current files
|
||||
2019-07-01 21:35:11,170 - INFO - no files to remove
|
||||
2019-07-01 21:35:13,482 - INFO - downloaded results.2019-06-03_00:00.json
|
||||
2019-07-01 21:35:15,199 - INFO - downloaded results.2019-06-03_00:01.json
|
||||
2019-07-01 21:35:18,301 - INFO - downloaded results.2019-06-03_00:02.json
|
||||
2019-07-01 21:35:19,611 - INFO - downloaded results.2019-06-03_00:03.json
|
||||
2019-07-01 21:35:19,681 - INFO - downloaded results.2019-06-03_00:04.json
|
||||
2019-07-01 21:35:19,755 - INFO - downloaded results.2019-06-03_00:05.json
|
||||
2019-07-01 21:35:19,823 - INFO - downloaded results.2019-06-03_00:06.json
|
||||
2019-07-01 21:35:19,909 - INFO - downloaded results.2019-06-03_00:07.json
|
||||
2019-07-01 21:35:19,978 - INFO - downloaded results.2019-06-03_00:08.json
|
||||
2019-07-01 21:35:20,049 - INFO - downloaded results.2019-06-03_00:09.json
|
||||
2019-07-01 21:35:20,116 - INFO - downloaded results.2019-06-03_00:10.json
|
||||
2019-07-01 21:35:20,184 - INFO - downloaded results.2019-06-03_00:11.json
|
||||
2019-07-01 21:35:20,255 - INFO - downloaded results.2019-06-03_00:12.json
|
||||
2019-07-01 21:35:20,319 - INFO - downloaded results.2019-06-03_00:13.json
|
||||
2019-07-01 21:35:20,389 - INFO - downloaded results.2019-06-03_00:14.json
|
||||
2019-07-01 21:35:20,480 - INFO - downloaded results.2019-06-03_00:15.json
|
||||
2019-07-01 21:35:20,554 - INFO - downloaded results.2019-06-03_00:16.json
|
||||
2019-07-01 21:35:20,625 - INFO - downloaded results.2019-06-03_00:17.json
|
||||
2019-07-01 21:35:20,699 - INFO - downloaded results.2019-06-03_00:18.json
|
||||
2019-07-01 21:35:20,768 - INFO - downloaded results.2019-06-03_00:19.json
|
||||
2019-07-01 21:35:20,837 - INFO - downloaded results.2019-06-03_00:20.json
|
||||
2019-07-01 21:35:20,917 - INFO - downloaded results.2019-06-03_00:21.json
|
||||
2019-07-01 21:35:20,985 - INFO - downloaded results.2019-06-03_00:22.json
|
||||
2019-07-01 21:35:21,052 - INFO - downloaded results.2019-06-03_00:23.json
|
||||
2019-07-01 21:35:21,121 - INFO - downloaded results.2019-06-03_00:24.json
|
||||
2019-07-01 21:35:21,195 - INFO - downloaded results.2019-06-03_00:25.json
|
||||
2019-07-01 21:35:23,040 - INFO - downloaded results.2019-06-03_00:26.json
|
||||
2019-07-01 21:35:24,395 - INFO - downloaded results.2019-06-03_00:27.json
|
||||
2019-07-01 21:35:25,530 - INFO - downloaded results.2019-06-03_00:28.json
|
||||
2019-07-01 21:35:26,554 - INFO - downloaded results.2019-06-03_00:29.json
|
||||
2019-07-01 21:35:27,933 - INFO - downloaded results.2019-06-03_00:30.json
|
||||
2019-07-01 21:35:29,166 - INFO - downloaded results.2019-06-03_00:31.json
|
||||
2019-07-01 21:35:30,229 - INFO - downloaded results.2019-06-03_00:32.json
|
||||
2019-07-01 21:35:31,218 - INFO - downloaded results.2019-06-03_00:33.json
|
||||
2019-07-01 21:35:32,235 - INFO - downloaded results.2019-06-03_00:34.json
|
||||
2019-07-01 22:10:03,817 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 22:10:03,818 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 22:10:04,137 - INFO - no files available to download -- exiting
|
||||
2019-07-01 22:10:12,537 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 22:10:12,537 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 22:10:12,832 - INFO - opening history file
|
||||
2019-07-01 22:10:12,832 - INFO - attempting to clear current files
|
||||
2019-07-01 22:10:12,833 - INFO - no files to remove
|
||||
2019-07-01 22:10:13,506 - INFO - downloaded blocking_suggestions_2019-07-01T00.csv
|
||||
2019-07-01 22:10:13,672 - INFO - downloaded blocking_suggestions_2019-07-01T02.csv
|
||||
2019-07-01 22:10:13,778 - INFO - downloaded blocking_suggestions_2019-07-01T04.csv
|
||||
2019-07-01 22:10:13,901 - INFO - downloaded blocking_suggestions_2019-07-01T06.csv
|
||||
2019-07-01 22:10:14,100 - INFO - downloaded blocking_suggestions_2019-07-01T08.csv
|
||||
2019-07-01 22:10:14,210 - INFO - downloaded blocking_suggestions_2019-07-01T10.csv
|
||||
2019-07-01 22:10:14,316 - INFO - downloaded blocking_suggestions_2019-07-01T12.csv
|
||||
2019-07-01 22:10:14,474 - INFO - downloaded blocking_suggestions_2019-07-01T14.csv
|
||||
2019-07-01 22:10:14,709 - INFO - downloaded blocking_suggestions_2019-07-01T16.csv
|
||||
2019-07-01 22:10:14,841 - INFO - downloaded blocking_suggestions_2019-07-01T18.csv
|
||||
2019-07-01 22:10:14,842 - INFO - ========= SCRIPT FINISHED =========
|
||||
2019-07-01 22:10:21,141 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 22:10:21,141 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 22:10:21,465 - INFO - opening history file
|
||||
2019-07-01 22:10:21,466 - INFO - attempting to clear current files
|
||||
2019-07-01 22:10:21,467 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T14.csv
|
||||
2019-07-01 22:10:21,467 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T04.csv
|
||||
2019-07-01 22:10:21,467 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T16.csv
|
||||
2019-07-01 22:10:21,467 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T06.csv
|
||||
2019-07-01 22:10:21,468 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T00.csv
|
||||
2019-07-01 22:10:21,468 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T18.csv
|
||||
2019-07-01 22:10:21,468 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T08.csv
|
||||
2019-07-01 22:10:21,468 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T12.csv
|
||||
2019-07-01 22:10:21,468 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T02.csv
|
||||
2019-07-01 22:10:21,468 - INFO - removed /home/dtomlinson/projects/boto3/blocking_suggestions/blocking_suggestions_2019-07-01T10.csv
|
||||
2019-07-01 22:10:21,469 - INFO - ========= SCRIPT FINISHED =========
|
||||
2019-07-01 22:10:28,475 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 22:10:28,475 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 22:10:28,757 - INFO - opening history file
|
||||
2019-07-01 22:10:28,757 - INFO - attempting to clear current files
|
||||
2019-07-01 22:10:28,759 - INFO - no files to remove
|
||||
2019-07-01 22:10:28,759 - INFO - ========= SCRIPT FINISHED =========
|
||||
2019-07-01 22:12:55,531 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 22:12:55,531 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 22:12:55,833 - INFO - opening history file
|
||||
2019-07-01 22:12:55,833 - INFO - attempting to clear current files
|
||||
2019-07-01 22:12:55,834 - INFO - no files to remove
|
||||
2019-07-01 22:12:55,834 - INFO - ========= SCRIPT FINISHED =========
|
||||
2019-07-01 22:13:07,271 - INFO - ========= SCRIPT STARTED =========
|
||||
2019-07-01 22:13:07,272 - INFO - no path entered, using current directory /home/dtomlinson/projects/boto3/blocking_suggestions/
|
||||
2019-07-01 22:13:07,571 - INFO - opening history file
|
||||
2019-07-01 22:13:07,572 - INFO - attempting to clear current files
|
||||
2019-07-01 22:13:07,572 - INFO - no files to remove
|
||||
2019-07-01 22:13:08,239 - INFO - downloaded blocking_suggestions_2019-07-01T00.csv
|
||||
2019-07-01 22:13:08,373 - INFO - downloaded blocking_suggestions_2019-07-01T02.csv
|
||||
2019-07-01 22:13:08,522 - INFO - downloaded blocking_suggestions_2019-07-01T04.csv
|
||||
2019-07-01 22:13:08,636 - INFO - downloaded blocking_suggestions_2019-07-01T06.csv
|
||||
2019-07-01 22:13:08,816 - INFO - downloaded blocking_suggestions_2019-07-01T08.csv
|
||||
2019-07-01 22:13:08,949 - INFO - downloaded blocking_suggestions_2019-07-01T10.csv
|
||||
2019-07-01 22:13:09,075 - INFO - downloaded blocking_suggestions_2019-07-01T12.csv
|
||||
2019-07-01 22:13:09,558 - INFO - downloaded blocking_suggestions_2019-07-01T14.csv
|
||||
2019-07-01 22:13:09,814 - INFO - downloaded blocking_suggestions_2019-07-01T16.csv
|
||||
2019-07-01 22:13:10,020 - INFO - downloaded blocking_suggestions_2019-07-01T18.csv
|
||||
2019-07-01 22:13:10,020 - INFO - ========= SCRIPT FINISHED =========
|
||||
249
boto3/pull.py
Normal file
249
boto3/pull.py
Normal file
@@ -0,0 +1,249 @@
|
||||
import boto3
|
||||
import botocore
|
||||
import os
|
||||
from datetime import date
|
||||
import sys
|
||||
import logging
|
||||
import argparse
|
||||
import glob
|
||||
|
||||
# Set Global Variables
|
||||
log_location = 'pull.log'
|
||||
|
||||
remote_folder = ['bot_predictions']
|
||||
remote_file_prefix = ['blocking_suggestions_']
|
||||
|
||||
append_date = [True]
|
||||
date_format = ['%Y-%m-%d']
|
||||
|
||||
bucket = ['td-ingest-storage-williamhill']
|
||||
access_key = ['AKIAYJXVWMRHQ2OGNHLA']
|
||||
secret_key = ['0/4wxdBmpiU3gK1QHLk4me0zj2RHuNAcSOfgJm1B']
|
||||
|
||||
|
||||
class downloadFiles(object):
|
||||
"""docstring for downloadFiles"""
|
||||
today = date.today()
|
||||
|
||||
def __init__(self,
|
||||
client,
|
||||
resource,
|
||||
bucket,
|
||||
remote_folder,
|
||||
remote_file_prefix,
|
||||
local_path,
|
||||
append_date=False,
|
||||
date_format=''):
|
||||
super(downloadFiles, self).__init__()
|
||||
self.client = client
|
||||
self.resource = resource
|
||||
self.bucket = bucket
|
||||
self.append_date = append_date
|
||||
self.date_format = date_format
|
||||
self.remote_folder = self._folder_fixer(remote_folder)
|
||||
self.dest = f'{self.remote_folder!s}{remote_file_prefix!s}'
|
||||
self.local_path = local_path
|
||||
self.remote_list, self.local_list, self.local_file_list = \
|
||||
(list() for _ in range(3))
|
||||
|
||||
@staticmethod
|
||||
def generate_date(date_format):
|
||||
date = downloadFiles.today.strftime(date_format)
|
||||
return date
|
||||
|
||||
@staticmethod
|
||||
def _folder_fixer(folder):
|
||||
try:
|
||||
if folder[-1] != '/':
|
||||
folder = f'{folder}/'
|
||||
except IndexError:
|
||||
folder = ''
|
||||
return folder
|
||||
|
||||
def get_path(self):
|
||||
if self.local_path:
|
||||
self.local_path = self._folder_fixer(self.local_path)
|
||||
logger.info(f'path entered is {self.local_path}')
|
||||
return self
|
||||
else:
|
||||
self.local_path = os.getcwd()
|
||||
self.local_path = self._folder_fixer(self.local_path)
|
||||
self.local_path = f'{self.local_path}blocking_suggestions/'
|
||||
logger.info(f'no path entered, using current directory '
|
||||
f'{self.local_path}')
|
||||
return self
|
||||
|
||||
def get_files(self):
|
||||
if self.append_date:
|
||||
date = f'{self.generate_date(self.date_format)!s}'
|
||||
else:
|
||||
date = ''
|
||||
self.dest = f'{self.dest!s}{date!s}'
|
||||
|
||||
paginator = self.client.get_paginator('list_objects')
|
||||
iterator = paginator.paginate(Bucket=self.bucket, Prefix=self.dest)
|
||||
self.filtered = iterator.search('Contents[*].Key')
|
||||
for i in self.filtered:
|
||||
try:
|
||||
self.remote_list.append(i)
|
||||
self.local_list.append(
|
||||
f'{self.local_path}{i[len(self.remote_folder):]}'
|
||||
)
|
||||
self.local_file_list.append(
|
||||
f'{i[len(self.remote_folder):]}'
|
||||
)
|
||||
except TypeError:
|
||||
logger.info('no files available to download -- exiting')
|
||||
raise SystemExit
|
||||
logger.debug(f'remote files are {self.remote_list}')
|
||||
logger.debug(f'saving files locally to {self.local_list}')
|
||||
return self
|
||||
|
||||
def get_history(self):
|
||||
self.history_file = f'{self.local_path}.history.txt'
|
||||
try:
|
||||
logger.info('opening history file')
|
||||
open(self.history_file, 'a').close()
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
logger.critical('history file cannot be found or created'
|
||||
' - check permissions of the folder.')
|
||||
raise
|
||||
self.history_list = \
|
||||
[line.rstrip('\n') for line in open(self.history_file)]
|
||||
return self
|
||||
|
||||
def remove_files(self):
|
||||
logger.info('attempting to clear current files')
|
||||
current_files = glob.glob(f'{self.local_path}[!history.txt]*')
|
||||
if current_files:
|
||||
for i in current_files:
|
||||
try:
|
||||
os.remove(i)
|
||||
logger.info(f'removed {i}')
|
||||
except OSError:
|
||||
logger.exception('Error:')
|
||||
else:
|
||||
logger.info('no files to remove')
|
||||
return self
|
||||
|
||||
def download_files(self):
|
||||
for remote_file, local_file_with_path, local_file in zip(
|
||||
self.remote_list, self.local_list, self.local_file_list):
|
||||
if local_file not in self.history_list:
|
||||
with open(local_file_with_path, 'wb'), \
|
||||
open(self.history_file, 'a') as hist:
|
||||
try:
|
||||
self.resource.Bucket(self.bucket).download_file(
|
||||
remote_file, local_file_with_path)
|
||||
hist.write(f'\n{local_file}')
|
||||
logger.info(f'downloaded {local_file}')
|
||||
except botocore.exceptions.ClientError as e:
|
||||
if e.response['Error']['Code'] == '404':
|
||||
print(f'The object {remote_file} does not exist.')
|
||||
else:
|
||||
raise
|
||||
if local_file in self.history_list:
|
||||
logger.debug(f'{local_file} already downloaded - skipping')
|
||||
return self
|
||||
|
||||
|
||||
def _call():
|
||||
global args, debug
|
||||
parser = argparse.ArgumentParser(description="""
|
||||
downloads any new files for the current day from an S3 bucket. \
|
||||
uses a local history file to track what has been \
|
||||
previously downloaded in the download path.
|
||||
""")
|
||||
parser.add_argument('--path', type=str,
|
||||
help='enter pull path to download to. if left \
|
||||
blank will use the same location as the script.',
|
||||
default='')
|
||||
parser.add_argument('--debug', action='store_true', default=False,
|
||||
help='Use this to log DEBUG information.')
|
||||
|
||||
args = parser.parse_args()
|
||||
debug = vars(args)['debug']
|
||||
|
||||
if debug:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
else:
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
main(_clients=_clients,
|
||||
_resources=_resources,
|
||||
_buckets=_buckets,
|
||||
remote_folder=remote_folder,
|
||||
remote_file_prefix=remote_file_prefix,
|
||||
append_date=append_date,
|
||||
date_format=date_format,
|
||||
**vars(args))
|
||||
|
||||
|
||||
def main(*args,
|
||||
_clients={'client0': ''},
|
||||
_resources={'resource0': ''},
|
||||
_buckets={'bucket0': ''},
|
||||
remote_folder=[''],
|
||||
remote_file_prefix=[''],
|
||||
append_date=['True'],
|
||||
date_format=[''],
|
||||
path='',
|
||||
**kwargs):
|
||||
logger.info('========= SCRIPT STARTED =========')
|
||||
instance = downloadFiles(client=_clients['client0'],
|
||||
resource=_resources['resource0'],
|
||||
bucket=_buckets['bucket0'],
|
||||
remote_folder=remote_folder[0],
|
||||
remote_file_prefix=remote_file_prefix[0],
|
||||
local_path=path,
|
||||
append_date=append_date[0],
|
||||
date_format=date_format[0])
|
||||
instance.get_path().get_files().get_history().remove_files()\
|
||||
.download_files()
|
||||
logger.info('========= SCRIPT FINISHED =========')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args, debug = '', ''
|
||||
|
||||
# define logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
c_handler = logging.StreamHandler(sys.stdout)
|
||||
f_handler = logging.FileHandler(log_location)
|
||||
|
||||
c_format = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
f_format = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
c_handler.setFormatter(c_format)
|
||||
f_handler.setFormatter(f_format)
|
||||
|
||||
logger.addHandler(c_handler)
|
||||
logger.addHandler(f_handler)
|
||||
|
||||
_clients = {}
|
||||
_resources = {}
|
||||
_buckets = {}
|
||||
|
||||
for i in range(0, len(bucket)):
|
||||
_clients[f'client{i}'] =\
|
||||
boto3.client('s3',
|
||||
aws_access_key_id=f'{access_key[i]}',
|
||||
aws_secret_access_key=f'{secret_key[i]}')
|
||||
_resources[f'resource{i}'] =\
|
||||
boto3.resource('s3',
|
||||
aws_access_key_id=f'{access_key[i]}',
|
||||
aws_secret_access_key=f'{secret_key[i]}')
|
||||
_buckets[f'bucket{i}'] = f'{bucket[i]}'
|
||||
|
||||
try:
|
||||
_length = len(remote_folder)
|
||||
if _length == 0:
|
||||
remote_folder = ['']
|
||||
elif remote_folder[0] == 'root':
|
||||
remote_folder = ['']
|
||||
else:
|
||||
pass
|
||||
except NameError:
|
||||
remote_folder = ['']
|
||||
_call()
|
||||
Reference in New Issue
Block a user