diff --git a/etl/planning_data/obtain_livestream_data_and_load_into_database.py b/etl/planning_data/obtain_livestream_data_and_load_into_database.py index 9fef6086..bd663dea 100644 --- a/etl/planning_data/obtain_livestream_data_and_load_into_database.py +++ b/etl/planning_data/obtain_livestream_data_and_load_into_database.py @@ -66,6 +66,19 @@ def load_data_into_database(cursor, data): "data_source": "Greater London Authority's Planning London DataHub", "data_source_link": None } + if date_in_future(entry["registered_with_local_authority_date"]): + print("registered_with_local_authority_date is treated as invalid:", entry["registered_with_local_authority_date"]) + # Brent-87_0946 has "valid_date": "23/04/9187" + entry["registered_with_local_authority_date"] = None + + if date_in_future(entry["decision_date"]): + print("decision_date is treated as invalid:", entry["decision_date"]) + entry["decision_date"] = None + + if date_in_future(entry["last_synced_date"]): + print("last_synced_date is treated as invalid:", entry["last_synced_date"]) + entry["last_synced_date"] = None + if "Hackney" in entry["application_id"]: if entry["application_url"] != None: if "https://" not in entry["application_url"]: @@ -80,6 +93,11 @@ def load_data_into_database(cursor, data): print(json.dumps(entry, indent = 4)) raise e +def date_in_future(date): + if date == None: + return False + return date > datetime.datetime.now() + def query(search_after): headers = { 'X-API-AllowRequest': os.environ['PLANNNING_DATA_API_ALLOW_REQUEST_CODE'],