diff --git a/app/map_styles/polygon.xml b/app/map_styles/polygon.xml
index 994fe086..67617cfe 100644
--- a/app/map_styles/polygon.xml
+++ b/app/map_styles/polygon.xml
@@ -355,11 +355,11 @@
- [status] = "Unknown"
+ [status] != "Submitted" and [status] != "Approved" and [status] != "Appeal In Progress" and [status] != "Rejected" and [status] != "Withdrawn"
- [status] = "Unknown"
+ [status] != "Submitted" and [status] != "Approved" and [status] != "Appeal In Progress" and [status] != "Rejected" and [status] != "Withdrawn"
diff --git a/app/src/frontend/config/category-maps-config.ts b/app/src/frontend/config/category-maps-config.ts
index b7a36df3..5072302e 100644
--- a/app/src/frontend/config/category-maps-config.ts
+++ b/app/src/frontend/config/category-maps-config.ts
@@ -178,7 +178,7 @@ export const categoryMapsConfig: {[key in Category]: CategoryMapDefinition[]} =
{ color: '#fff200', text: 'Appeal In Progress' },
{ color: '#e31d23', text: 'Rejected' },
{ color: '#999999', text: 'Withdrawn' },
- { color: '#eacad0', text: 'Unknown' },
+ { color: '#eacad0', text: 'Other' },
]
}
},
diff --git a/etl/planning_data/load_into_database.py b/etl/planning_data/load_into_database.py
index 9b4c581d..d9005577 100644
--- a/etl/planning_data/load_into_database.py
+++ b/etl/planning_data/load_into_database.py
@@ -35,6 +35,25 @@ def parse_date_string_into_datestring(incoming):
date = datetime.datetime.strptime(incoming, "%Y-%m-%dT%H:%M:%S.%fZ") # '2022-08-08T20:07:22.238Z'
return datetime.datetime.strftime(date, "%Y-%m-%d")
+def process_status(status):
+ """return None if status is invalid"""
+ if status == "Refused":
+ status = "Rejected"
+ if status == "Appeal Received":
+ status = "Appeal In Progress"
+ if status == None:
+ status = "Unknown"
+ if (status in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
+ return status
+ print("Unexpected status " + status)
+ if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
+ print("New unexpected status " + status)
+ status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql
+ if len(status) > 50:
+ print("Status was too long and was skipped:", status)
+ return None
+ return status
+
def main():
connection = get_connection()
with connection.cursor() as cur:
@@ -53,22 +72,7 @@ def main():
decision_date = parse_date_string_into_datestring(entry['_source']['decision_date'])
last_synced_date = parse_date_string_into_datestring(entry['_source']['last_synced'])
uprn = entry['_source']['uprn']
- status = entry['_source']['status']
- if status in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", None, "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
- continue
- if status in []:
- opts = jsbeautifier.default_options()
- opts.indent_size = 2
- print(jsbeautifier.beautify(json.dumps(entry), opts))
- continue
- if status == "Refused":
- status = "Rejected"
- if status == "Appeal Received":
- status = "Appeal In Progress"
- if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
- print("Unexpected status " + status)
- continue
- #raise Exception("Unexpected status " + status)
+ status = process_status(entry['_source']['status'])
if uprn == None:
continue
entry = {
diff --git a/etl/planning_data/load_into_database_dropped_data.py b/etl/planning_data/load_into_database_dropped_data.py
index 6fe7a354..07a590bd 100644
--- a/etl/planning_data/load_into_database_dropped_data.py
+++ b/etl/planning_data/load_into_database_dropped_data.py
@@ -53,6 +53,25 @@ def shorten_description(original_description):
description += "... (show more)"
return description
+def process_status(status):
+ """return None if status is invalid"""
+ if status == "Refused":
+ status = "Rejected"
+ if status == "Appeal Received":
+ status = "Appeal In Progress"
+ if status == None:
+ status = "Unknown"
+ if (status in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
+ return status
+ print("Unexpected status " + status)
+ if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
+ print("New unexpected status " + status)
+ status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql
+ if len(status) > 50:
+ print("Status was too long and was skipped:", status)
+ return None
+ return status
+
def main():
connection = get_connection()
with connection.cursor() as cur:
@@ -61,29 +80,11 @@ def main():
data = json.load(content_file)
for entry in data['features']:
description = entry['properties']['description']
- application_id = "unknown"
+ application_id = "not available"
decision_date = parse_date_string_into_datestring(entry['properties']['decision_date'])
last_synced_date = parse_date_string_into_datestring(entry['properties']['decision_date'])
uprn = entry['properties']['uprn']
- status = entry['properties']['status']
- if status in ["No Objection to Proposal (OBS only)", "Not Required", None, "Lapsed", "SECS", "Comment Issued",
-
- # new ones
- "ALL DECISIONS ISSUED", "Closed", "?", ""
- ]:
- continue
- if status in []:
- opts = jsbeautifier.default_options()
- opts.indent_size = 2
- print(jsbeautifier.beautify(json.dumps(entry), opts))
- continue
- if status == "Refused":
- status = "Rejected"
- if status == "Appeal Received":
- status = "Appeal In Progress"
- if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
- print("Unexpected status " + status)
- continue
+ status = process_status(entry['properties']['status'])
if uprn == None:
continue
entry = {
diff --git a/migrations/033.planning_livestream_data.up.sql b/migrations/033.planning_livestream_data.up.sql
index 1e593dbf..940b5e11 100644
--- a/migrations/033.planning_livestream_data.up.sql
+++ b/migrations/033.planning_livestream_data.up.sql
@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS planning_data (
registered_with_local_authority_date date,
decision_date date,
last_synced_date date,
- status VARCHAR(20),
+ status VARCHAR(50),
data_source VARCHAR(70),
data_source_link VARCHAR(150),
uprn bigint