cv/street_group_tech_test
1
0
Fork 0
mirror of https://github.com/dtomlinson91/street_group_tech_test synced 2025-12-22 11:55:45 +00:00
Code Issues Projects Releases Wiki Activity

adding latest beam pipeline code

Browse Source
This commit is contained in:
Daniel Tomlinson
2021-09-25 01:44:37 +01:00
parent aa61ea9c57
commit 47a4ac4bc3
1 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
analyse_properties/__init__.py
View File

@@ -17,7 +17,7 @@ def csv_reader(csv_file):
return csv.reader(io.TextIOWrapper(csv_file.open()))
def slicer(element, *ranges):
def slice_by_range(element, *ranges):
return chain(*(islice(element, *r) for r in ranges))
@@ -27,12 +27,18 @@ class SplitPAON(beam.DoFn):
if len(paon_split) == 0:
return element
elif len(paon_split) == 1:
pass
else:
elif len(paon_split) > 0:
pass
class DropEmptyColumn(bean.doFn):
def __init__(self, index):
self.index = index
def process(self, element):
column = element[self.index]
def main():
csv_data = resources.path(
"analyse_properties.data", "pp-monthly-update-new-version.csv"
@@ -53,7 +59,7 @@ def main():
clean = (
load
| "Drop unneeded columns"
>> beam.Map(lambda element: list(slicer(element, (1, 4), (7, 14))))
>> beam.Map(lambda element: list(slice_by_range(element, (1, 4), (7, 14))))
| "Convert to Upper Case"
>> beam.Map(lambda element: [e.upper() for e in element])
| beam.Map(print)