mirror of
https://github.com/dtomlinson91/street_group_tech_test
synced 2025-12-22 11:55:45 +00:00
861 lines
35 KiB
HTML
861 lines
35 KiB
HTML
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
|
|
|
|
|
|
<link rel="icon" href="../assets/images/favicon.png">
|
|
<meta name="generator" content="mkdocs-1.2.2, mkdocs-material-7.3.0">
|
|
|
|
|
|
|
|
<title>Approach - The Street Group Technical Test</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../assets/stylesheets/main.8b42a75e.min.css">
|
|
|
|
|
|
<link rel="stylesheet" href="../assets/stylesheets/palette.3f5d1f46.min.css">
|
|
|
|
|
|
|
|
<meta name="theme-color" content="#4051b5">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
|
|
<style>:root{--md-text-font-family:"Roboto";--md-code-font-family:"Roboto Mono"}</style>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="indigo" data-md-color-accent="blue">
|
|
|
|
|
|
<script>function __prefix(e){return new URL("..",location).pathname+"."+e}function __get(e,t=localStorage){return JSON.parse(t.getItem(__prefix(e)))}</script>
|
|
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" for="__drawer"></label>
|
|
<div data-md-component="skip">
|
|
|
|
|
|
<a href="#approach" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
</div>
|
|
<div data-md-component="announce">
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<header class="md-header" data-md-component="header">
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
|
<a href=".." title="The Street Group Technical Test" class="md-header__button md-logo" aria-label="The Street Group Technical Test" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54z"/></svg>
|
|
|
|
</a>
|
|
<label class="md-header__button md-icon" for="__drawer">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
|
|
</label>
|
|
<div class="md-header__title" data-md-component="header-title">
|
|
<div class="md-header__ellipsis">
|
|
<div class="md-header__topic">
|
|
<span class="md-ellipsis">
|
|
The Street Group Technical Test
|
|
</span>
|
|
</div>
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
|
<span class="md-ellipsis">
|
|
|
|
Approach
|
|
|
|
</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<label class="md-header__button md-icon" for="__search">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
|
|
</label>
|
|
|
|
<div class="md-search" data-md-component="search" role="dialog">
|
|
<label class="md-search__overlay" for="__search"></label>
|
|
<div class="md-search__inner" role="search">
|
|
<form class="md-search__form" name="search">
|
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
|
<label class="md-search__icon md-icon" for="__search">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
|
|
</label>
|
|
<nav class="md-search__options" aria-label="Search">
|
|
|
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
|
|
</button>
|
|
</nav>
|
|
|
|
</form>
|
|
<div class="md-search__output">
|
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
|
<div class="md-search-result" data-md-component="search-result">
|
|
<div class="md-search-result__meta">
|
|
Initializing search
|
|
</div>
|
|
<ol class="md-search-result__list"></ol>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-header__source">
|
|
|
|
<a href="https://github.com/dtomlinson91/street_group_tech_test/" title="Go to repository" class="md-source" data-md-component="source">
|
|
<div class="md-source__icon md-icon">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
|
</div>
|
|
<div class="md-source__repository">
|
|
GitHub
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
</nav>
|
|
|
|
</header>
|
|
|
|
<div class="md-container" data-md-component="container">
|
|
|
|
|
|
|
|
|
|
|
|
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
|
|
<div class="md-tabs__inner md-grid">
|
|
<ul class="md-tabs__list">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-tabs__item">
|
|
<a href="../index.html" class="md-tabs__link">
|
|
Documentation
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-tabs__item">
|
|
<a href="introduction.html" class="md-tabs__link md-tabs__link--active">
|
|
Discussion
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-tabs__item">
|
|
<a href="../dataflow/index.html" class="md-tabs__link">
|
|
DataFlow
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-tabs__item">
|
|
<a href="../pandas-profiling/report.html" class="md-tabs__link">
|
|
Data Exploration Report
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</div>
|
|
</nav>
|
|
|
|
|
|
|
|
<main class="md-main" data-md-component="main">
|
|
<div class="md-main__inner md-grid">
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
|
|
<label class="md-nav__title" for="__drawer">
|
|
<a href=".." title="The Street Group Technical Test" class="md-nav__button md-logo" aria-label="The Street Group Technical Test" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54z"/></svg>
|
|
|
|
</a>
|
|
The Street Group Technical Test
|
|
</label>
|
|
|
|
<div class="md-nav__source">
|
|
|
|
<a href="https://github.com/dtomlinson91/street_group_tech_test/" title="Go to repository" class="md-source" data-md-component="source">
|
|
<div class="md-source__icon md-icon">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
|
</div>
|
|
<div class="md-source__repository">
|
|
GitHub
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_1" type="checkbox" id="__nav_1" >
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_1">
|
|
Documentation
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="Documentation" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_1">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Documentation
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../index.html" class="md-nav__link">
|
|
Welcome
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../documentation/installation.html" class="md-nav__link">
|
|
Installation
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../documentation/usage.html" class="md-nav__link">
|
|
Usage
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_2" type="checkbox" id="__nav_2" checked>
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_2">
|
|
Discussion
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="Discussion" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_2">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Discussion
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="introduction.html" class="md-nav__link">
|
|
Introduction
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="exploration.html" class="md-nav__link">
|
|
Data Exploration Report
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="cleaning.html" class="md-nav__link">
|
|
Cleaning
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
Approach
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<a href="approach.html" class="md-nav__link md-nav__link--active">
|
|
Approach
|
|
</a>
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#loading-stage" class="md-nav__link">
|
|
Loading stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#cleaning-stage" class="md-nav__link">
|
|
Cleaning stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#create-a-mapping-table" class="md-nav__link">
|
|
Create a mapping table
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#prepare-stage" class="md-nav__link">
|
|
Prepare stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#format-stage" class="md-nav__link">
|
|
Format stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#save-stage" class="md-nav__link">
|
|
Save stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="results.html" class="md-nav__link">
|
|
Results
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_3">
|
|
DataFlow
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="DataFlow" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_3">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
DataFlow
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../dataflow/index.html" class="md-nav__link">
|
|
Running on DataFlow
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../dataflow/scaling.html" class="md-nav__link">
|
|
Scaling to the Full DataSet
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../pandas-profiling/report.html" class="md-nav__link">
|
|
Data Exploration Report
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#loading-stage" class="md-nav__link">
|
|
Loading stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#cleaning-stage" class="md-nav__link">
|
|
Cleaning stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#create-a-mapping-table" class="md-nav__link">
|
|
Create a mapping table
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#prepare-stage" class="md-nav__link">
|
|
Prepare stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#format-stage" class="md-nav__link">
|
|
Format stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#save-stage" class="md-nav__link">
|
|
Save stage
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-content" data-md-component="content">
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
<a href="https://github.com/dtomlinson91/street_group_tech_test/edit/master/docs/discussion/approach.md" title="Edit this page" class="md-content__button md-icon">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25z"/></svg>
|
|
</a>
|
|
|
|
|
|
<h1 id="approach">Approach<a class="headerlink" href="#approach" title="Permanent link">¶</a></h1>
|
|
<p>The general approach to the pipeline is:</p>
|
|
<h2 id="loading-stage">Loading stage<a class="headerlink" href="#loading-stage" title="Permanent link">¶</a></h2>
|
|
<ul>
|
|
<li>Load using <code class="highlight"><span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">()</span></code></li>
|
|
<li>Split the string loaded by <code>,</code> as it's a comma delimited <code>.csv</code>.</li>
|
|
<li>Strip the leading/trailing <code>"</code> marks.</li>
|
|
</ul>
|
|
<p>The result is an array with each element representing a single column in that row.</p>
|
|
<h2 id="cleaning-stage">Cleaning stage<a class="headerlink" href="#cleaning-stage" title="Permanent link">¶</a></h2>
|
|
<p>Already discussed.</p>
|
|
<h2 id="create-a-mapping-table">Create a mapping table<a class="headerlink" href="#create-a-mapping-table" title="Permanent link">¶</a></h2>
|
|
<p>The mapping table takes each row and creates a <code>(key,value)</code> pair with:</p>
|
|
<ul>
|
|
<li>The key being the id across all columns (<code>id_all_columns</code>).</li>
|
|
<li>The value being the raw data as an array.</li>
|
|
</ul>
|
|
<p>The mapping table is then condensed to a single dictionary with these key, value pairs (automatically deduplicating repeated rows) and is used as a side input further down the pipeline.</p>
|
|
<p>This mapping table is created to ensure the <code>GroupByKey</code> operation is as quick as possible. The more data you have to process in a <code>GroupByKey</code>, the longer the operation takes. By doing the <code>GroupByKey</code> using just the ids, the pipeline can process the files much quicker than if we included the raw data in this operation.</p>
|
|
<h2 id="prepare-stage">Prepare stage<a class="headerlink" href="#prepare-stage" title="Permanent link">¶</a></h2>
|
|
<ul>
|
|
<li>Take the mapping table data (before it is condensed) and create a unique id ignoring the price and date (<code>id_without_price_date</code>).</li>
|
|
</ul>
|
|
<p>This id will not be unique: for properties with more than one transaction they will share this id.</p>
|
|
<ul>
|
|
<li>Create a <code>(key, value)</code> pair with:<ul>
|
|
<li>The key being <code>id_without_price_date</code>.</li>
|
|
<li>The value being <code>id_all_columns</code>.</li>
|
|
</ul>
|
|
</li>
|
|
<li>Group by <code>id_without_price_date</code>.</li>
|
|
</ul>
|
|
<p>This results in a PCollection that looks like: <code>(id_without_price_date, [id_all_columns,...])</code></p>
|
|
<ul>
|
|
<li>Deduplicate the <code>id_all_columns</code> inside this array to eliminate repeated rows that are exactly the same.</li>
|
|
<li>Use the mapping table as a side input to reinsert the raw data using the <code>id_all_columns</code>.</li>
|
|
</ul>
|
|
<details>
|
|
<summary>Example for No.1 B90 3LA</summary>
|
|
|
|
Mapping table (pre condensed):
|
|
|
|
<div class="highlight"><pre><span></span><code><span class="err">('</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">'</span><span class="p">,</span> <span class="p">[</span><span class="err">'</span><span class="mi">317500</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="mi">2020-11-13</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="err">'</span><span class="p">,</span> <span class="err">'B</span><span class="mi">90</span> <span class="mi">3</span><span class="err">LA'</span><span class="p">,</span> <span class="err">'</span><span class="mi">1</span><span class="err">'</span><span class="p">,</span> <span class="err">''</span><span class="p">,</span> <span class="err">'VERSTONE</span> <span class="err">ROAD'</span><span class="p">,</span> <span class="err">'SHIRLEY'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'WEST</span> <span class="err">MIDLANDS'</span><span class="p">,</span> <span class="err">''</span><span class="p">]</span><span class="err">)</span>
|
|
<span class="err">('</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">'</span><span class="p">,</span> <span class="p">[</span><span class="err">'</span><span class="mi">317500</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="mi">2020-11-13</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="err">'</span><span class="p">,</span> <span class="err">'B</span><span class="mi">90</span> <span class="mi">3</span><span class="err">LA'</span><span class="p">,</span> <span class="err">'</span><span class="mi">1</span><span class="err">'</span><span class="p">,</span> <span class="err">''</span><span class="p">,</span> <span class="err">'VERSTONE</span> <span class="err">ROAD'</span><span class="p">,</span> <span class="err">'SHIRLEY'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'WEST</span> <span class="err">MIDLANDS'</span><span class="p">,</span> <span class="err">''</span><span class="p">]</span><span class="err">)</span>
|
|
</code></pre></div>
|
|
|
|
Mapping table (condensed):
|
|
|
|
<div class="highlight"><pre><span></span><code><span class="p">{</span><span class="err">'</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">'</span><span class="p">:</span> <span class="p">[</span><span class="err">'</span><span class="mi">317500</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="mi">2020-11-13</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="err">'</span><span class="p">,</span> <span class="err">'B</span><span class="mi">90</span> <span class="mi">3</span><span class="err">LA'</span><span class="p">,</span> <span class="err">'</span><span class="mi">1</span><span class="err">'</span><span class="p">,</span> <span class="err">''</span><span class="p">,</span> <span class="err">'VERSTONE</span> <span class="err">ROAD'</span><span class="p">,</span> <span class="err">'SHIRLEY'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'WEST</span> <span class="err">MIDLANDS'</span><span class="p">,</span> <span class="err">''</span><span class="p">]}</span>
|
|
</code></pre></div>
|
|
|
|
Prepared (key, value):
|
|
|
|
<div class="highlight"><pre><span></span><code><span class="err">('</span><span class="kc">fe</span><span class="mi">205</span><span class="err">b</span><span class="kc">fe</span><span class="mi">66</span><span class="err">bc</span><span class="mi">7</span><span class="kc">f</span><span class="mi">18</span><span class="err">c</span><span class="mi">50</span><span class="err">c</span><span class="mi">8</span><span class="kc">f</span><span class="mi">3</span><span class="err">d</span><span class="mf">77e</span><span class="err">c</span><span class="mf">3e30</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">')</span>
|
|
<span class="err">('</span><span class="kc">fe</span><span class="mi">205</span><span class="err">b</span><span class="kc">fe</span><span class="mi">66</span><span class="err">bc</span><span class="mi">7</span><span class="kc">f</span><span class="mi">18</span><span class="err">c</span><span class="mi">50</span><span class="err">c</span><span class="mi">8</span><span class="kc">f</span><span class="mi">3</span><span class="err">d</span><span class="mf">77e</span><span class="err">c</span><span class="mf">3e30</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">')</span>
|
|
</code></pre></div>
|
|
|
|
Prepared (GroupByKey):
|
|
|
|
<div class="highlight"><pre><span></span><code><span class="err">('</span><span class="kc">fe</span><span class="mi">205</span><span class="err">b</span><span class="kc">fe</span><span class="mi">66</span><span class="err">bc</span><span class="mi">7</span><span class="kc">f</span><span class="mi">18</span><span class="err">c</span><span class="mi">50</span><span class="err">c</span><span class="mi">8</span><span class="kc">f</span><span class="mi">3</span><span class="err">d</span><span class="mf">77e</span><span class="err">c</span><span class="mf">3e30</span><span class="err">'</span><span class="p">,</span> <span class="p">[</span><span class="err">'</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">'</span><span class="p">]</span><span class="err">)</span>
|
|
</code></pre></div>
|
|
|
|
Prepared (Deduplicated):
|
|
|
|
<div class="highlight"><pre><span></span><code><span class="err">('</span><span class="kc">fe</span><span class="mi">205</span><span class="err">b</span><span class="kc">fe</span><span class="mi">66</span><span class="err">bc</span><span class="mi">7</span><span class="kc">f</span><span class="mi">18</span><span class="err">c</span><span class="mi">50</span><span class="err">c</span><span class="mi">8</span><span class="kc">f</span><span class="mi">3</span><span class="err">d</span><span class="mf">77e</span><span class="err">c</span><span class="mf">3e30</span><span class="err">'</span><span class="p">,</span> <span class="p">[</span><span class="err">'</span><span class="kc">f</span><span class="err">d</span><span class="mi">4634</span><span class="kc">fae</span><span class="err">c</span><span class="mi">47</span><span class="err">c</span><span class="mi">29</span><span class="err">de</span><span class="mi">40</span><span class="err">bb</span><span class="kc">f</span><span class="mi">7840723</span><span class="err">b</span><span class="mi">41</span><span class="err">'</span><span class="p">]</span><span class="err">)</span>
|
|
</code></pre></div>
|
|
|
|
Use mapping table as side input:
|
|
|
|
<div class="highlight"><pre><span></span><code><span class="err">('</span><span class="kc">fe</span><span class="mi">205</span><span class="err">b</span><span class="kc">fe</span><span class="mi">66</span><span class="err">bc</span><span class="mi">7</span><span class="kc">f</span><span class="mi">18</span><span class="err">c</span><span class="mi">50</span><span class="err">c</span><span class="mi">8</span><span class="kc">f</span><span class="mi">3</span><span class="err">d</span><span class="mf">77e</span><span class="err">c</span><span class="mf">3e30</span><span class="err">'</span><span class="p">,</span> <span class="p">[</span><span class="err">'</span><span class="mi">317500</span><span class="err">'</span><span class="p">,</span> <span class="err">'</span><span class="mi">2020-11-13</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="err">'</span><span class="p">,</span> <span class="err">'B</span><span class="mi">90</span> <span class="mi">3</span><span class="err">LA'</span><span class="p">,</span> <span class="err">'</span><span class="mi">1</span><span class="err">'</span><span class="p">,</span> <span class="err">''</span><span class="p">,</span> <span class="err">'VERSTONE</span> <span class="err">ROAD'</span><span class="p">,</span> <span class="err">'SHIRLEY'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'SOLIHULL'</span><span class="p">,</span> <span class="err">'WEST</span> <span class="err">MIDLANDS'</span><span class="p">,</span> <span class="err">''</span><span class="p">]</span><span class="err">)</span>
|
|
</code></pre></div>
|
|
|
|
</details>
|
|
|
|
<h2 id="format-stage">Format stage<a class="headerlink" href="#format-stage" title="Permanent link">¶</a></h2>
|
|
<p>This stage takes the result and constructs a <code>json</code> object out of the grouped data. The schema for this output is discussed in the following page.</p>
|
|
<h2 id="save-stage">Save stage<a class="headerlink" href="#save-stage" title="Permanent link">¶</a></h2>
|
|
<ul>
|
|
<li>The PCollection is combined with <code class="highlight"><span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">ToList</span><span class="p">()</span></code></li>
|
|
<li>Apply <code>json.dumps()</code> for proper quotation marks for strings.</li>
|
|
<li>Write to text with <code class="highlight"><span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">WriteToText</span></code>.</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
</div>
|
|
|
|
</main>
|
|
|
|
|
|
<footer class="md-footer">
|
|
|
|
<nav class="md-footer__inner md-grid" aria-label="Footer">
|
|
|
|
|
|
<a href="cleaning.html" class="md-footer__link md-footer__link--prev" aria-label="Previous: Cleaning" rel="prev">
|
|
<div class="md-footer__button md-icon">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
|
|
</div>
|
|
<div class="md-footer__title">
|
|
<div class="md-ellipsis">
|
|
<span class="md-footer__direction">
|
|
Previous
|
|
</span>
|
|
Cleaning
|
|
</div>
|
|
</div>
|
|
</a>
|
|
|
|
|
|
|
|
<a href="results.html" class="md-footer__link md-footer__link--next" aria-label="Next: Results" rel="next">
|
|
<div class="md-footer__title">
|
|
<div class="md-ellipsis">
|
|
<span class="md-footer__direction">
|
|
Next
|
|
</span>
|
|
Results
|
|
</div>
|
|
</div>
|
|
<div class="md-footer__button md-icon">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
|
|
</div>
|
|
</a>
|
|
|
|
</nav>
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-footer-copyright">
|
|
|
|
|
|
Made with
|
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
|
Material for MkDocs
|
|
</a>
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
<div class="md-dialog" data-md-component="dialog">
|
|
<div class="md-dialog__inner md-typeset"></div>
|
|
</div>
|
|
<script id="__config" type="application/json">{"base": "..", "features": {"navigation.tabs": true}, "translations": {"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "search": "../assets/javascripts/workers/search.f8263e09.min.js", "version": null}</script>
|
|
|
|
|
|
<script src="../assets/javascripts/bundle.4fc53ad4.min.js"></script>
|
|
|
|
<script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
|
|
|
|
<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |