From 205580d7fe56c68ba3c96a54d4e2721c39746028 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Thu, 22 Sep 2022 01:28:04 +0200 Subject: [PATCH] better display as unusual data in planning view applications with unandled statuses are shown not available application id is not shown as unknown anymore --- app/map_styles/polygon.xml | 4 +- .../frontend/config/category-maps-config.ts | 2 +- etl/planning_data/load_into_database.py | 36 ++++++++-------- .../load_into_database_dropped_data.py | 41 ++++++++++--------- .../033.planning_livestream_data.up.sql | 2 +- 5 files changed, 45 insertions(+), 40 deletions(-) diff --git a/app/map_styles/polygon.xml b/app/map_styles/polygon.xml index 994fe086..67617cfe 100644 --- a/app/map_styles/polygon.xml +++ b/app/map_styles/polygon.xml @@ -355,11 +355,11 @@ - [status] = "Unknown" + [status] != "Submitted" and [status] != "Approved" and [status] != "Appeal In Progress" and [status] != "Rejected" and [status] != "Withdrawn" - [status] = "Unknown" + [status] != "Submitted" and [status] != "Approved" and [status] != "Appeal In Progress" and [status] != "Rejected" and [status] != "Withdrawn" diff --git a/app/src/frontend/config/category-maps-config.ts b/app/src/frontend/config/category-maps-config.ts index b7a36df3..5072302e 100644 --- a/app/src/frontend/config/category-maps-config.ts +++ b/app/src/frontend/config/category-maps-config.ts @@ -178,7 +178,7 @@ export const categoryMapsConfig: {[key in Category]: CategoryMapDefinition[]} = { color: '#fff200', text: 'Appeal In Progress' }, { color: '#e31d23', text: 'Rejected' }, { color: '#999999', text: 'Withdrawn' }, - { color: '#eacad0', text: 'Unknown' }, + { color: '#eacad0', text: 'Other' }, ] } }, diff --git a/etl/planning_data/load_into_database.py b/etl/planning_data/load_into_database.py index 9b4c581d..d9005577 100644 --- a/etl/planning_data/load_into_database.py +++ b/etl/planning_data/load_into_database.py @@ -35,6 +35,25 @@ def parse_date_string_into_datestring(incoming): date = datetime.datetime.strptime(incoming, "%Y-%m-%dT%H:%M:%S.%fZ") # '2022-08-08T20:07:22.238Z' return datetime.datetime.strftime(date, "%Y-%m-%d") +def process_status(status): + """return None if status is invalid""" + if status == "Refused": + status = "Rejected" + if status == "Appeal Received": + status = "Appeal In Progress" + if status == None: + status = "Unknown" + if (status in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]): + return status + print("Unexpected status " + status) + if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]: + print("New unexpected status " + status) + status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql + if len(status) > 50: + print("Status was too long and was skipped:", status) + return None + return status + def main(): connection = get_connection() with connection.cursor() as cur: @@ -53,22 +72,7 @@ def main(): decision_date = parse_date_string_into_datestring(entry['_source']['decision_date']) last_synced_date = parse_date_string_into_datestring(entry['_source']['last_synced']) uprn = entry['_source']['uprn'] - status = entry['_source']['status'] - if status in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", None, "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]: - continue - if status in []: - opts = jsbeautifier.default_options() - opts.indent_size = 2 - print(jsbeautifier.beautify(json.dumps(entry), opts)) - continue - if status == "Refused": - status = "Rejected" - if status == "Appeal Received": - status = "Appeal In Progress" - if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]): - print("Unexpected status " + status) - continue - #raise Exception("Unexpected status " + status) + status = process_status(entry['_source']['status']) if uprn == None: continue entry = { diff --git a/etl/planning_data/load_into_database_dropped_data.py b/etl/planning_data/load_into_database_dropped_data.py index 6fe7a354..07a590bd 100644 --- a/etl/planning_data/load_into_database_dropped_data.py +++ b/etl/planning_data/load_into_database_dropped_data.py @@ -53,6 +53,25 @@ def shorten_description(original_description): description += "... (show more)" return description +def process_status(status): + """return None if status is invalid""" + if status == "Refused": + status = "Rejected" + if status == "Appeal Received": + status = "Appeal In Progress" + if status == None: + status = "Unknown" + if (status in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]): + return status + print("Unexpected status " + status) + if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]: + print("New unexpected status " + status) + status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql + if len(status) > 50: + print("Status was too long and was skipped:", status) + return None + return status + def main(): connection = get_connection() with connection.cursor() as cur: @@ -61,29 +80,11 @@ def main(): data = json.load(content_file) for entry in data['features']: description = entry['properties']['description'] - application_id = "unknown" + application_id = "not available" decision_date = parse_date_string_into_datestring(entry['properties']['decision_date']) last_synced_date = parse_date_string_into_datestring(entry['properties']['decision_date']) uprn = entry['properties']['uprn'] - status = entry['properties']['status'] - if status in ["No Objection to Proposal (OBS only)", "Not Required", None, "Lapsed", "SECS", "Comment Issued", - - # new ones - "ALL DECISIONS ISSUED", "Closed", "?", "" - ]: - continue - if status in []: - opts = jsbeautifier.default_options() - opts.indent_size = 2 - print(jsbeautifier.beautify(json.dumps(entry), opts)) - continue - if status == "Refused": - status = "Rejected" - if status == "Appeal Received": - status = "Appeal In Progress" - if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]): - print("Unexpected status " + status) - continue + status = process_status(entry['properties']['status']) if uprn == None: continue entry = { diff --git a/migrations/033.planning_livestream_data.up.sql b/migrations/033.planning_livestream_data.up.sql index 1e593dbf..940b5e11 100644 --- a/migrations/033.planning_livestream_data.up.sql +++ b/migrations/033.planning_livestream_data.up.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS planning_data ( registered_with_local_authority_date date, decision_date date, last_synced_date date, - status VARCHAR(20), + status VARCHAR(50), data_source VARCHAR(70), data_source_link VARCHAR(150), uprn bigint