a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/history-paging

Browse Source
This commit is contained in:
Maciej Ziarkowski 2020-02-02 16:40:17 +00:00
commit aa43ab7711
10 changed files with 36 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -1,6 +1,6 @@
language: node_js
node_js:
- 8
- 12
cache: npm
before_script:
- cd $TRAVIS_BUILD_DIR/app && npm ci

26
app/package-lock.json generated
View File

@ -425,15 +425,6 @@
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-syntax-typescript": {
"version": "7.7.4",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.7.4.tgz",
"integrity": "sha512-77blgY18Hud4NM1ggTA8xVT/dBENQf17OpiToSa2jSmEY3fWXD2jwrdVlO4kq5yzUTeF15WSQ6b4fByNvJcjpQ==",
"dev": true,
"requires": {
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-transform-arrow-functions": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.2.0.tgz",
@ -1189,9 +1180,9 @@
"dev": true
},
"@types/node": {
"version": "8.10.59",
"resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.59.tgz",
"integrity": "sha512-8RkBivJrDCyPpBXhVZcjh7cQxVBSmRk9QM7hOketZzp6Tg79c0N8kkpAIito9bnJ3HCVCHVYz+KHTEbfQNfeVQ==",
"version": "12.12.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.25.tgz",
"integrity": "sha512-nf1LMGZvgFX186geVZR1xMZKKblJiRfiASTHw85zED2kI1yDKHDwTKMdkaCbTlXoRKlGKaDfYywt+V0As30q3w==",
"dev": true
},
"@types/nodemailer": {
@ -2420,17 +2411,6 @@
"integrity": "sha512-f49NsaohQ1ByY20nUrpc30QFdbeT4ntV4PAL2vSZe6uCB5nqAcqXS/qzU+aI6ZfYhWASx5eIsTFvFrs1B2ffGg==",
"dev": true
},
"babel-plugin-typescript-to-proptypes": {
"version": "0.17.1",
"resolved": "https://registry.npmjs.org/babel-plugin-typescript-to-proptypes/-/babel-plugin-typescript-to-proptypes-0.17.1.tgz",
"integrity": "sha512-yREUfvDlmn6QjM0QbywXUkXBQMD/iFfLVTl+jig4X7ZLUg9lq8ZLuex8HIM2SQ4X3vcjGnWPFowodlMcXhwxdQ==",
"dev": true,
"requires": {
"@babel/helper-module-imports": "^7.0.0",
"@babel/helper-plugin-utils": "^7.0.0",
"@babel/plugin-syntax-typescript": "^7.2.0"
}
},
"babel-preset-jest": {
"version": "23.2.0",
"resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-23.2.0.tgz",

7
app/package.json
View File

@ -26,7 +26,6 @@
"node-fs": "^0.1.7",
"nodemailer": "^6.3.0",
"pg-promise": "^8.7.5",
"prop-types": "^15.7.2",
"query-string": "^6.8.2",
"react": "^16.9.0",
"react-dom": "^16.9.0",
@ -37,16 +36,13 @@
"sharp": "^0.22.1"
},
"devDependencies": {
"@babel/plugin-syntax-jsx": "^7.7.4",
"@babel/plugin-syntax-typescript": "^7.7.4",
"@types/express": "^4.17.2",
"@types/express-session": "^1.15.16",
"@types/jest": "^24.0.23",
"@types/lodash": "^4.14.149",
"@types/mapbox__sphericalmercator": "^1.1.3",
"@types/node": "^8.10.59",
"@types/node": "^12.12.25",
"@types/nodemailer": "^6.2.2",
"@types/prop-types": "^15.7.3",
"@types/react": "^16.9.16",
"@types/react-dom": "^16.9.4",
"@types/react-leaflet": "^2.5.0",
@ -54,7 +50,6 @@
"@types/sharp": "^0.22.3",
"@types/webpack-env": "^1.14.1",
"babel-eslint": "^10.0.3",
"babel-plugin-typescript-to-proptypes": "^0.17.1",
"eslint": "^5.16.0",
"eslint-plugin-jest": "^22.21.0",
"eslint-plugin-react": "^7.17.0",

3
app/src/api/services/dataExtract.ts
View File

@ -19,7 +19,8 @@ async function listDataExtracts(): Promise<DataExtract[]> {
const extractRecords = await db.manyOrNone<DataExtractRow>(
`SELECT
extract_id, extracted_on, extract_path
FROM bulk_extracts`
FROM bulk_extracts
ORDER BY extracted_on DESC`
);
return extractRecords.map(getDataExtractFromRow);

5
app/src/frontend/pages/data-extracts.tsx
View File

@ -30,10 +30,7 @@ export default class DataExtracts extends React.Component<{}, DataExtractsState>
async componentDidMount() {
let data = await apiGet('/api/extracts', { jsonReviver: dateReviver});
const extracts = (data.extracts as ExtractViewModel[])
.sort((a, b) => b.extracted_on.valueOf() - a.extracted_on.valueOf());
const extracts = (data.extracts as ExtractViewModel[]);
this.setState({ extracts: extracts, latestExtract: extracts[0], previousExtracts: extracts.slice(1) });
}

4
docs/setup-dev-environment.md
View File

@ -37,7 +37,7 @@ Now clone the colouring london codebase.
Now install Node. It is helpful to define some local variables.
```
NODE_VERSION=v8.11.3
NODE_VERSION=v12.14.1
DISTRO=linux-x64
wget -nc https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$DISTRO.tar.xz
sudo mkdir /usr/local/lib/node
@ -137,7 +137,7 @@ root user profile. Don't forget to exit from root at the end.
```
sudo su root
export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
export PATH=$NODEJS_HOME:$PATH`
npm install -g npm@next`
exit

25
docs/setup-production-environment.md
View File

@ -32,6 +32,10 @@ Install Postgres and associated tools
`sudo apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev`
Install Python 3 and pip
`sudo apt-get install python3 python3-pip`
Install Nginx
@ -68,7 +72,7 @@ Now set appropriate permissions on the `colouring-london` directory
First define a couple of convenience variables:
`NODE_VERSION=v8.11.3`
`NODE_VERSION=v12.14.1`
`DISTRO=linux-x64`
@ -107,7 +111,7 @@ Now upgrade the `npm` package manager to the most recent release with global pri
`sudo su root`
`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
`export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
`export PATH=$NODEJS_HOME:$PATH`
@ -240,7 +244,7 @@ Perform a global install of PM2
`sudo su root`
`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
`export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
`export PATH=$NODEJS_HOME:$PATH`
@ -303,6 +307,21 @@ To stop the colouring-london app type:
***
#### Set up data extracts
Install requirements for the maintenance Python scripts
`cd /var/www/colouring-london/maintenance`
`sudo pip3 install -r requirements.txt`
The maintenance scripts might need environment variables present at the time of execution, notably the database connection details.
If running the scripts manually, the variables can be provided just before execution, for example
`PGHOST=localhost PGPORT=5432 PGDATABASE=dbname PGUSER=username PGPASSWORD=secretpassword EXTRACTS_DIRECTORY=/var/www/colouring-london/downloads python3 maintenance/extract_data/extract_data.py`
If the maintenance script is to be run on a schedule, the variables should be loaded before running the script, for example from a `.env` file.
#### Set up SSL - TO DO

4
maintenance/extract_data/extract_data.py
View File

@ -64,9 +64,11 @@ def make_data_extract(current_time, connection, zip_file_path):
zip_file_path.parent.mkdir(parents=True, exist_ok=True)
source_dir_path = Path(__file__).parent
try:
with zipfile.ZipFile(zip_file_path, mode='w') as newzip:
newzip.write('README.txt')
newzip.write(source_dir_path / 'README.txt', arcname='README.txt')
newzip.write('/tmp/building_attributes.csv', arcname='building_attributes.csv')
newzip.write('/tmp/building_uprns.csv', arcname='building_uprns.csv')
newzip.write('/tmp/edit_history.csv', arcname='edit_history.csv')

1
maintenance/tasks.cron.txt
View File

@ -1 +0,0 @@
0 5 * * * /var/www/colouringlondon/maintenance/extract_data/extract_data.py

2
provision/vm_provision.sh
View File

@ -40,7 +40,7 @@ apt-get install -y \
#
# node version and platform
NODE_VERSION=v8.11.3
NODE_VERSION=v12.14.1
DISTRO=linux-x64
# download