a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

apply PEP8 standards

Browse Source
This commit is contained in:
Mateusz Konieczny 2022-11-10 08:28:44 +01:00
parent a679054a0d
commit d544b5228d
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
etl/planning_data/obtain_livestream_data_and_load_into_database.py
View File

@ -26,10 +26,9 @@ def main():
connection.commit() connection.commit()
def load_data_into_database(cursor, data): def load_data_into_database(cursor, data):
if "timed_out" not in data: if "timed_out" not in data:
print(json.dumps(data, indent = 4)) print(json.dumps(data, indent=4))
print("timed_out field missing in provided data") print("timed_out field missing in provided data")
else: else:
if data['timed_out']: if data['timed_out']:
@ -93,11 +92,13 @@ def load_data_into_database(cursor, data):
show_dictionary(entry) show_dictionary(entry)
raise e raise e
def date_in_future(date): def date_in_future(date):
if date == None: if date == None:
return False return False
return date > datetime.datetime.now() return date > datetime.datetime.now()
def query(search_after): def query(search_after):
headers = { headers = {
'X-API-AllowRequest': os.environ['PLANNNING_DATA_API_ALLOW_REQUEST_CODE'], 'X-API-AllowRequest': os.environ['PLANNNING_DATA_API_ALLOW_REQUEST_CODE'],
@ -141,6 +142,7 @@ def query(search_after):
print(json_data) print(json_data)
return requests.post('https://planningdata.london.gov.uk/api-guest/applications/_search', headers=headers, json=json_data) return requests.post('https://planningdata.london.gov.uk/api-guest/applications/_search', headers=headers, json=json_data)
def get_connection(): def get_connection():
return psycopg2.connect( return psycopg2.connect(
host=os.environ['PGHOST'], host=os.environ['PGHOST'],
@ -149,9 +151,11 @@ def get_connection():
password=os.environ['PGPASSWORD'] password=os.environ['PGPASSWORD']
) )
def filepath(): def filepath():
return os.path.dirname(os.path.realpath(__file__)) + os.sep + "data.json" return os.path.dirname(os.path.realpath(__file__)) + os.sep + "data.json"
def insert_entry(cursor, e): def insert_entry(cursor, e):
try: try:
now = datetime.datetime.now() now = datetime.datetime.now()
@ -255,6 +259,7 @@ def obtain_entry_link(provided_link, application_id):
return None return None
# Richmond is simply broken # Richmond is simply broken
def process_status(status, decision_date): def process_status(status, decision_date):
"""return None if status is invalid""" """return None if status is invalid"""
if status in ["Application Under Consideration", "Application Received"]: if status in ["Application Under Consideration", "Application Received"]:
@ -275,11 +280,12 @@ def process_status(status, decision_date):
print("Unexpected status " + status) print("Unexpected status " + status)
if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]: if status not in ["No Objection to Proposal (OBS only)", "Objection Raised to Proposal (OBS only)", "Not Required", "Unknown", "Lapsed", "SECS", "Comment Issued", "ALL DECISIONS ISSUED", "Closed", "Declined to Determine"]:
print("New unexpected status " + status) print("New unexpected status " + status)
status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql status_length_limit = 50 # see migrations/033.planning_livestream_data.up.sql
if len(status) > 50: if len(status) > 50:
print("Status was too long and was skipped:", status) print("Status was too long and was skipped:", status)
return None return None
return status return status
if __name__ == '__main__': if __name__ == '__main__':
main() main()