cv/street_group_tech_test
1
0
Fork 0
mirror of https://github.com/dtomlinson91/street_group_tech_test synced 2025-12-22 03:55:43 +00:00
Code Issues Projects Releases Wiki Activity

adding latest beam pipeline code

Browse Source
This commit is contained in:
Daniel Tomlinson
2021-09-25 01:44:37 +01:00
parent aa61ea9c57
commit 47a4ac4bc3
1 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
analyse_properties/__init__.py
View File

@@ -17,7 +17,7 @@ def csv_reader(csv_file):
return csv.reader(io.TextIOWrapper(csv_file.open())) return csv.reader(io.TextIOWrapper(csv_file.open()))
def slicer(element, *ranges): def slice_by_range(element, *ranges):
return chain(*(islice(element, *r) for r in ranges)) return chain(*(islice(element, *r) for r in ranges))
@@ -27,12 +27,18 @@ class SplitPAON(beam.DoFn):
if len(paon_split) == 0: if len(paon_split) == 0:
return element return element
elif len(paon_split) == 1: elif len(paon_split) > 0:
pass
else:
pass pass
class DropEmptyColumn(bean.doFn):
def __init__(self, index):
self.index = index
def process(self, element):
column = element[self.index]
def main(): def main():
csv_data = resources.path( csv_data = resources.path(
"analyse_properties.data", "pp-monthly-update-new-version.csv" "analyse_properties.data", "pp-monthly-update-new-version.csv"
@@ -53,7 +59,7 @@ def main():
clean = ( clean = (
load load
| "Drop unneeded columns" | "Drop unneeded columns"
>> beam.Map(lambda element: list(slicer(element, (1, 4), (7, 14)))) >> beam.Map(lambda element: list(slice_by_range(element, (1, 4), (7, 14))))
| "Convert to Upper Case" | "Convert to Upper Case"
>> beam.Map(lambda element: [e.upper() for e in element]) >> beam.Map(lambda element: [e.upper() for e in element])
| beam.Map(print) | beam.Map(print)