better display as unusual data in planning view

applications with unandled statuses are shown
not available application id is not shown as unknown anymore

app/map_styles/polygon.xml

@@ -355,11 +355,11 @@
             <LineSymbolizer stroke="#999999" stroke-width="1.5" />
         </Rule>
         <Rule>
-            <Filter>[status] = "Unknown"</Filter>
+            <Filter>[status] != "Submitted" and [status] != "Approved" and [status] != "Appeal In Progress" and [status] != "Rejected" and [status] != "Withdrawn"</Filter>
             <PolygonSymbolizer fill="#eacad0"/>
         </Rule>
         <Rule>
-            <Filter>[status] = "Unknown"</Filter>
+            <Filter>[status] != "Submitted" and [status] != "Approved" and [status] != "Appeal In Progress" and [status] != "Rejected" and [status] != "Withdrawn"</Filter>
             <LineSymbolizer stroke="#eacad0" stroke-width="15" />
         </Rule>
     </Style>

app/src/frontend/config/category-maps-config.ts

@@ -178,7 +178,7 @@ export const categoryMapsConfig: {[key in Category]: CategoryMapDefinition[]} =
                     { color: '#fff200', text: 'Appeal In Progress' },
                     { color: '#e31d23', text: 'Rejected' },
                     { color: '#999999', text: 'Withdrawn' },
-                    { color: '#eacad0', text: 'Unknown' },
+                    { color: '#eacad0', text: 'Other' },
                 ]
             }
         },

etl/planning_data/load_into_database.py

@@ -35,6 +35,25 @@ def parse_date_string_into_datestring(incoming):
         date = datetime.datetime.strptime(incoming, "%Y-%m-%dT%H:%M:%S.%fZ") # '2022-08-08T20:07:22.238Z'
     return datetime.datetime.strftime(date, "%Y-%m-%d")
 
+def process_status(status):
+    """return None if status is invalid"""
+    if status == "Refused":
+        status = "Rejected"
+    if status == "Appeal Received":
+        status = "Appeal In Progress"
+    if status == None:
+        status = "Unknown"
+    if (status in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
+        return status
+    print("Unexpected status " + status)
+    if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
+        print("New unexpected status " + status)
+    status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql
+    if len(status) > 50:
+        print("Status was too long and was skipped:", status)
+        return None
+    return status
+
 def main():
     connection = get_connection()
     with connection.cursor() as cur:
@@ -53,22 +72,7 @@ def main():
             decision_date = parse_date_string_into_datestring(entry['_source']['decision_date'])
             last_synced_date = parse_date_string_into_datestring(entry['_source']['last_synced'])
             uprn = entry['_source']['uprn']
-            status = entry['_source']['status']
+            status = process_status(entry['_source']['status'])
-            if status in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", None, "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
-                continue
-            if status in []:
-                opts = jsbeautifier.default_options()
-                opts.indent_size = 2
-                print(jsbeautifier.beautify(json.dumps(entry), opts))            
-                continue
-            if status == "Refused":
-                status = "Rejected"
-            if status == "Appeal Received":
-                status = "Appeal In Progress"
-            if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
-                print("Unexpected status " + status)
-                continue
-                #raise Exception("Unexpected status " + status)
             if uprn == None:
                 continue
             entry = {

etl/planning_data/load_into_database_dropped_data.py

@@ -53,6 +53,25 @@ def shorten_description(original_description):
         description += "... <i>(show more)</i>"
     return description
 
+def process_status(status):
+    """return None if status is invalid"""
+    if status == "Refused":
+        status = "Rejected"
+    if status == "Appeal Received":
+        status = "Appeal In Progress"
+    if status == None:
+        status = "Unknown"
+    if (status in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
+        return status
+    print("Unexpected status " + status)
+    if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
+        print("New unexpected status " + status)
+    status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql
+    if len(status) > 50:
+        print("Status was too long and was skipped:", status)
+        return None
+    return status
+
 def main():
     connection = get_connection()
     with connection.cursor() as cur:
@@ -61,29 +80,11 @@ def main():
         data = json.load(content_file)
         for entry in data['features']:
             description = entry['properties']['description']
-            application_id = "unknown"
+            application_id = "not available"
             decision_date = parse_date_string_into_datestring(entry['properties']['decision_date'])
             last_synced_date = parse_date_string_into_datestring(entry['properties']['decision_date'])
             uprn = entry['properties']['uprn']
-            status = entry['properties']['status']
+            status = process_status(entry['properties']['status'])
-            if status in ["No Objection to Proposal (OBS only)", "Not Required", None, "Lapsed", "SECS", "Comment Issued",
-
-            # new ones
-            "ALL DECISIONS ISSUED", "Closed", "?", ""
-            ]: 
-                continue
-            if status in []:
-                opts = jsbeautifier.default_options()
-                opts.indent_size = 2
-                print(jsbeautifier.beautify(json.dumps(entry), opts))            
-                continue
-            if status == "Refused":
-                status = "Rejected"
-            if status == "Appeal Received":
-                status = "Appeal In Progress"
-            if (status not in ["Approved", "Rejected", "Appeal In Progress", "Withdrawn", "Unknown"]):
-                print("Unexpected status " + status)
-                continue
             if uprn == None:
                 continue
             entry = {

migrations/033.planning_livestream_data.up.sql

@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS planning_data (
         registered_with_local_authority_date date,
         decision_date date,
         last_synced_date date,
-        status VARCHAR(20),
+        status VARCHAR(50),
         data_source VARCHAR(70),
         data_source_link VARCHAR(150),
         uprn bigint