Merge final release (#1)

* adding initial skeleton * updating .gitignore * updating dev dependencies * adding report.py * updating notes * adding prospector.yaml * updating beam to install gcp extras * adding documentation * adding data exploration report + code * adding latest beam pipeline code * adding latest beam pipeline code * adding debug.py * adding latesty beam pipeline code * adding latest beam pipeline code * adding latest beam pipeline code * updating .gitignore * updating folder structure for data input/output * updating prospector.yaml * adding latest beam pipeline code * updating prospector.yaml * migrate beam pipeline to main.py * updating .gitignore * updating .gitignore * adding download script for data set * adding initial docs * moving inputs/outputs to use pathlib * removing shard_name_template from output file * adding pyenv 3.7.9 * removing requirements.txt for documentation * updating README.md * updating download data script for new location in GCS * adding latest beam pipeline code for dataflow * adding latest beam pipeline code for dataflow * adding latest beam pipeline code for dataflow * moving dataflow notes * updating prospector.yaml * adding latest beam pipeline code for dataflow * updating beam pipeline to use GroupByKey * updating download_data script with new bucket * update prospector.yaml * update dataflow documentation with new commands for vpc * adding latest beam pipeline code for dataflow with group optimisation * updating dataflow documentation * adding latest beam pipeline code for dataflow with group optimisation * updating download_data script with pp-2020 dataset * adding temporary notes * updating dataflow notes * adding latest beam pipeline code * updating dataflow notes * adding latest beam pipeline code for dataflow * adding debug print * moving panda-profiling report into docs * updating report.py * adding entrypoint command * adding initial docs * adding commands.md to notes * commenting out debug imports * updating documentation * updating latest beam pipeline with default inputs * updating poetry * adding requirements.txt * updating documentation
2025-12-22 11:55:45 +00:00 · 2021-09-28 00:31:09 +01:00
parent 8a22bfebe1
commit 80376a662e
34 changed files with 5667 additions and 1 deletions
--- a/notes/tmp/exampledata
+++ b/notes/tmp/exampledata
@@ -0,0 +1,44 @@
+[{
+  "property_id": "3cf3c06632c46754696f2017933702f3",
+  "flat_appartment": "",
+  "builing": "",
+  "number": "63",
+  "street": "ROTTON PARK STREET",
+  "locality": "",
+  "town": "BIRMINGHAM",
+  "district": "BIRMINGHAM",
+  "county": "WEST MIDLANDS",
+  "postcode": "B16 0AE",
+  "property_transactions": [
+    { "price": "385000", "transaction_date": "2021-01-08", "year": "2021" },
+    { "price": "701985", "transaction_date": "2019-03-28", "year": "2019" },
+    { "price": "1748761", "transaction_date": "2020-05-27", "year": "2020" }
+  ],
+  "latest_transaction_year": "2021"
+},
+{
+    "property_id": "c650d5d7bb0daf0a19bb2cacabbee74e",
+    "readable_address": "16 STATION ROAD\nPARKGATE\nNESTON\nCHESHIRE WEST AND CHESTER\nCH64 6QJ",
+    "flat_appartment": "",
+    "builing": "",
+    "number": "16",
+    "street": "STATION ROAD",
+    "locality": "PARKGATE",
+    "town": "NESTON",
+    "district": "CHESHIRE WEST AND CHESTER",
+    "county": "CHESHIRE WEST AND CHESTER",
+    "postcode": "CH64 6QJ",
+    "property_transactions": [
+        {
+        "price": "280000",
+        "transaction_date": "2020-11-30",
+        "year": "2020"
+        },
+        {
+        "price": "265000",
+        "transaction_date": "2020-05-29",
+        "year": "2020"
+        }
+    ],
+    "latest_transaction_year": "2020"
+}]