From 9bdd3dc90b71d23aa46f703819dcf41fbc5c1f8f Mon Sep 17 00:00:00 2001 From: Martin-dJ Date: Tue, 26 Mar 2019 17:12:00 +0000 Subject: [PATCH 1/3] Added documentation for setting up a development environment --- docs/setup-dev-environment.md | 206 ++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 docs/setup-dev-environment.md diff --git a/docs/setup-dev-environment.md b/docs/setup-dev-environment.md new file mode 100644 index 00000000..bba5425d --- /dev/null +++ b/docs/setup-dev-environment.md @@ -0,0 +1,206 @@ +## Setting up a local development environment + +This document is intended to guide you through setting up a local development environment for Colouring London. This guide assumes you already have Ubuntu 18.04 server installed, typically installed in a virtual environment such a Virtual Box and are able to SSH into your Ubuntu installation for convenience. +
+
+ + + + +First upgrade the installed packages to the latest versions, to remove any security warnings. + +`sudo apt-get update` + + +`sudo apt-get upgrade` + +
+ + + +#### Installing the tools and components + +Now we install some essential tools. + +`sudo apt-get install -y build-essential git vim-nox wget curl` + +
+Now install python and related tools. + +`sudo apt-get install -y python3 python3-pip python3-dev python3-venv` + +
+Next install postgres and postgis to enable support for geographical objects. + +`sudo apt-get install -y postgresql postgresql-contrib libpq-dev postgis postgresql-10-postgis-2.4` + +and additional geo-spatial tools + +`sudo apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev` + +
+Now clone the colouring london codebase. + +`git clone https://github.com/tomalrussell/colouring-london.git` + +
+Now install Node. It is helpful to define some local variables. + +`NODE_VERSION=v8.11.3` + +`DISTRO=linux-x64` + +`wget -nc https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$DISTRO.tar.xz` + +`sudo mkdir /usr/local/lib/node` + +`sudo tar xf node-$NODE_VERSION-$DISTRO.tar.xz -C /usr/local/lib/node` + +`sudo mv /usr/local/lib/node/node-$NODE_VERSION-$DISTRO /usr/local/lib/node/node-$NODE_VERSION` + +`rm node-$NODE_VERSION-​$DISTRO.tar.xz` + +
+Now add the Node installation to the path and export this to your bash profile. + +`cat >> ~/.profile < + +#### Configuring Postgres + +Now we configure postgres. First ensure postgres is running. + +`service postgresql start` + +
+Ensure the `en_US` locale exists. + +`sudo locale-gen en_US.UTF-8` + +
+Configure the database to listen on network connection. + +`sudo sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/10/main/postgresql.conf` + +
+Allow authenticated connections from any IP (so includes host). + +`echo "host all all all md5" | sudo tee --append /etc/postgresql/10/main/pg_hba.conf > /dev/null` + +
+Restart postgres to pick up config changes. + +`service postgresql restart` + +
+Create a superuser role for this user (``) if it does not already exist. The password `` is arbitrary and probably should not be your Ubuntu login password. + +`sudo -u postgres psql -c "SELECT 1 FROM pg_user WHERE usename = '';" | grep -q 1 || sudo -u postgres psql -c "CREATE ROLE SUPERUSER LOGIN PASSWORD '';"` + +
+Create a colouring london database if none exists. The name (``) is arbitrary. + +`sudo -u postgres psql -c "SELECT 1 FROM pg_database WHERE datname = '';" | grep -q 1 || sudo -u postgres createdb -E UTF8 -T template0 --locale=en_US.utf8 -O ` + +
+Create the necessary postgres extensions. + +`psql -d -c "create extension postgis;"` + +`psql -d -c "create extension pgcrypto;"` + +`psql -d -c "create extension pg_trgm;"` + +
+Now run all 'up' migrations to create tables, data types, indexes etc. The `.sql` scripts to do this are located in the `migrations` folder of your local repository. + +`ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d < $migration; done;` + +
+ + + +#### Setting up Python + +Now set up a virtual environment for python. In the following example we have named the virtual environment *colouringlondon* but it can have any name. + +`pyvenv colouringlondon` + +Activate the virtual environment so we can install python packages into it. + +`source colouringlondon/bin/activate` + +Install python pip package manager and related tools. + +`pip install --upgrade pip` +`pip install --upgrade setuptools wheel` + +Now install the required python packages. This relies on the `requirements.txt` file located in the `etl` folder of your local repository. + +`pip install -r ./colouring-london/etl/requirements.txt` + + +
+#### Setting up Node + +Now upgrade the npm package manager to the most recent release with global privileges. This needs to be performed as root user, so it is necessary to export the node variables to the root user profile. Don't forget to exit from root at the end. + +`sudo su root` + +`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/` + +`export PATH=$NODEJS_HOME:​$PATH` + +`npm install -g npm@next` + +`exit` + + +
+Now install the required Node packages. This needs to done from the `app` directory of your local repository, so that it can read from the `package.json` file. + +`cd ./colouring-london/app && npm install` + + +
+ +#### Running the application + +Now we are ready to run the application. The `APP_COOKIE_SECRET` is arbitrary. + +`PGPASSWORD= PGDATABASE= PGUSER= PGHOST=localhost PGPORT=5432 APP_COOKIE_SECRET=123456 npm start` + + +
+If you a running Ubuntu in a virtual environment you will need to configure networking to forward ports from the guest to the host. For Virtual Box the following was configured under NAT port forwarding. + + + +Name | Protocol | Host Port | Guest Port +----------| ---------- | --------- | --------- +app | TCP | 8080 | 3000 +app_dev | TCP | 3001 | 3001 +ssh | TCP | 4022 | 22 + + + +The site can then be viewed on http://localhost:8080. The `app_dev` mapping is used in development by Razzle which rebuilds and serves client side assets on the fly. + +
+Finally to quit the application type `Ctrl-C`. + + + + + + From bad21c9eb0c5ee11f1f63b7642dd8e12416ee255 Mon Sep 17 00:00:00 2001 From: Martin-dJ Date: Tue, 26 Mar 2019 17:43:19 +0000 Subject: [PATCH 2/3] Revisions to setup-dev-environment.md to correct formatting errors --- docs/setup-dev-environment.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/setup-dev-environment.md b/docs/setup-dev-environment.md index bba5425d..eaf16fc7 100644 --- a/docs/setup-dev-environment.md +++ b/docs/setup-dev-environment.md @@ -25,11 +25,13 @@ Now we install some essential tools. `sudo apt-get install -y build-essential git vim-nox wget curl`
+ Now install python and related tools. `sudo apt-get install -y python3 python3-pip python3-dev python3-venv`
+ Next install postgres and postgis to enable support for geographical objects. `sudo apt-get install -y postgresql postgresql-contrib libpq-dev postgis postgresql-10-postgis-2.4` @@ -39,11 +41,13 @@ and additional geo-spatial tools `sudo apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev`
+ Now clone the colouring london codebase. `git clone https://github.com/tomalrussell/colouring-london.git`
+ Now install Node. It is helpful to define some local variables. `NODE_VERSION=v8.11.3` @@ -61,6 +65,7 @@ Now install Node. It is helpful to define some local variables. `rm node-$NODE_VERSION-​$DISTRO.tar.xz`
+ Now add the Node installation to the path and export this to your bash profile. `cat >> ~/.profile < + Ensure the `en_US` locale exists. `sudo locale-gen en_US.UTF-8`
+ Configure the database to listen on network connection. `sudo sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/10/main/postgresql.conf`
-Allow authenticated connections from any IP (so includes host). + +Allow authenticated connections from any IP (so includes the host). `echo "host all all all md5" | sudo tee --append /etc/postgresql/10/main/pg_hba.conf > /dev/null`
+ Restart postgres to pick up config changes. `service postgresql restart`
+ Create a superuser role for this user (``) if it does not already exist. The password `` is arbitrary and probably should not be your Ubuntu login password. `sudo -u postgres psql -c "SELECT 1 FROM pg_user WHERE usename = '';" | grep -q 1 || sudo -u postgres psql -c "CREATE ROLE SUPERUSER LOGIN PASSWORD '';"`
+ Create a colouring london database if none exists. The name (``) is arbitrary. `sudo -u postgres psql -c "SELECT 1 FROM pg_database WHERE datname = '';" | grep -q 1 || sudo -u postgres createdb -E UTF8 -T template0 --locale=en_US.utf8 -O `
+ Create the necessary postgres extensions. `psql -d -c "create extension postgis;"` @@ -122,6 +134,7 @@ Create the necessary postgres extensions. `psql -d -c "create extension pg_trgm;"`
+ Now run all 'up' migrations to create tables, data types, indexes etc. The `.sql` scripts to do this are located in the `migrations` folder of your local repository. `ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d < $migration; done;` @@ -151,6 +164,7 @@ Now install the required python packages. This relies on the `requirements.txt`
+ #### Setting up Node Now upgrade the npm package manager to the most recent release with global privileges. This needs to be performed as root user, so it is necessary to export the node variables to the root user profile. Don't forget to exit from root at the end. @@ -167,6 +181,7 @@ Now upgrade the npm package manager to the most recent release with global privi
+ Now install the required Node packages. This needs to done from the `app` directory of your local repository, so that it can read from the `package.json` file. `cd ./colouring-london/app && npm install` @@ -182,6 +197,7 @@ Now we are ready to run the application. The `APP_COOKIE_SECRET` is arbitrary.
+ If you a running Ubuntu in a virtual environment you will need to configure networking to forward ports from the guest to the host. For Virtual Box the following was configured under NAT port forwarding. @@ -197,6 +213,7 @@ ssh | TCP | 4022 | 22 The site can then be viewed on http://localhost:8080. The `app_dev` mapping is used in development by Razzle which rebuilds and serves client side assets on the fly.
+ Finally to quit the application type `Ctrl-C`. From c04010555709962c742f48f6953ca94a169ae86c Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Tue, 9 Apr 2019 09:23:50 +0100 Subject: [PATCH 3/3] Remove breaks, use multiline code blocks, reflow paragraphs --- docs/setup-dev-environment.md | 188 ++++++++++++---------------------- 1 file changed, 68 insertions(+), 120 deletions(-) diff --git a/docs/setup-dev-environment.md b/docs/setup-dev-environment.md index eaf16fc7..8317b969 100644 --- a/docs/setup-dev-environment.md +++ b/docs/setup-dev-environment.md @@ -1,22 +1,16 @@ ## Setting up a local development environment -This document is intended to guide you through setting up a local development environment for Colouring London. This guide assumes you already have Ubuntu 18.04 server installed, typically installed in a virtual environment such a Virtual Box and are able to SSH into your Ubuntu installation for convenience. -
-
- - - +This document is intended to guide you through setting up a local development environment for +Colouring London. This guide assumes you already have Ubuntu 18.04 server installed, typically +installed in a virtual environment such a Virtual Box and are able to SSH into your Ubuntu +installation for convenience. First upgrade the installed packages to the latest versions, to remove any security warnings. -`sudo apt-get update` - - -`sudo apt-get upgrade` - -
- - +``` +sudo apt-get update +sudo apt-get upgrade +``` #### Installing the tools and components @@ -24,13 +18,9 @@ Now we install some essential tools. `sudo apt-get install -y build-essential git vim-nox wget curl` -
- Now install python and related tools. -`sudo apt-get install -y python3 python3-pip python3-dev python3-venv` - -
+`sudo apt-get install -y python3 python3-pip python3-dev python3-venv` Next install postgres and postgis to enable support for geographical objects. @@ -40,46 +30,37 @@ and additional geo-spatial tools `sudo apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev` -
- Now clone the colouring london codebase. `git clone https://github.com/tomalrussell/colouring-london.git` -
- Now install Node. It is helpful to define some local variables. -`NODE_VERSION=v8.11.3` - -`DISTRO=linux-x64` - -`wget -nc https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$DISTRO.tar.xz` - -`sudo mkdir /usr/local/lib/node` - -`sudo tar xf node-$NODE_VERSION-$DISTRO.tar.xz -C /usr/local/lib/node` - -`sudo mv /usr/local/lib/node/node-$NODE_VERSION-$DISTRO /usr/local/lib/node/node-$NODE_VERSION` - -`rm node-$NODE_VERSION-​$DISTRO.tar.xz` - -
+``` +NODE_VERSION=v8.11.3 +DISTRO=linux-x64 +wget -nc https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$DISTRO.tar.xz +sudo mkdir /usr/local/lib/node +sudo tar xf node-$NODE_VERSION-$DISTRO.tar.xz -C /usr/local/lib/node +sudo mv /usr/local/lib/node/node-$NODE_VERSION-$DISTRO /usr/local/lib/node/node-$NODE_VERSION +rm node-$NODE_VERSION-$DISTRO.tar.xz +``` Now add the Node installation to the path and export this to your bash profile. -`cat >> ~/.profile <> ~/.profile < +``` +echo $PATH +echo $NODEJS_HOME +``` #### Configuring Postgres @@ -87,65 +68,48 @@ Now we configure postgres. First ensure postgres is running. `service postgresql start` -
- Ensure the `en_US` locale exists. `sudo locale-gen en_US.UTF-8` -
- Configure the database to listen on network connection. `sudo sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/10/main/postgresql.conf` -
- Allow authenticated connections from any IP (so includes the host). `echo "host all all all md5" | sudo tee --append /etc/postgresql/10/main/pg_hba.conf > /dev/null` -
- Restart postgres to pick up config changes. `service postgresql restart` -
- -Create a superuser role for this user (``) if it does not already exist. The password `` is arbitrary and probably should not be your Ubuntu login password. +Create a superuser role for this user (``) if it does not already exist. The +password `` is arbitrary and probably should not be your Ubuntu login password. `sudo -u postgres psql -c "SELECT 1 FROM pg_user WHERE usename = '';" | grep -q 1 || sudo -u postgres psql -c "CREATE ROLE SUPERUSER LOGIN PASSWORD '';"` -
+Create a colouring london database if none exists. The name (``) is arbitrary. -Create a colouring london database if none exists. The name (``) is arbitrary. - -`sudo -u postgres psql -c "SELECT 1 FROM pg_database WHERE datname = '';" | grep -q 1 || sudo -u postgres createdb -E UTF8 -T template0 --locale=en_US.utf8 -O ` - -
+`sudo -u postgres psql -c "SELECT 1 FROM pg_database WHERE datname = '';" | grep -q 1 || sudo -u postgres createdb -E UTF8 -T template0 --locale=en_US.utf8 -O ` Create the necessary postgres extensions. -`psql -d -c "create extension postgis;"` +`psql -d -c "create extension postgis;"` -`psql -d -c "create extension pgcrypto;"` +`psql -d -c "create extension pgcrypto;"` -`psql -d -c "create extension pg_trgm;"` - -
- -Now run all 'up' migrations to create tables, data types, indexes etc. The `.sql` scripts to do this are located in the `migrations` folder of your local repository. - -`ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d < $migration; done;` - -
+`psql -d -c "create extension pg_trgm;"` +Now run all 'up' migrations to create tables, data types, indexes etc. The `.sql` scripts to +do this are located in the `migrations` folder of your local repository. +`ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d < $migration; done;` #### Setting up Python -Now set up a virtual environment for python. In the following example we have named the virtual environment *colouringlondon* but it can have any name. +Now set up a virtual environment for python. In the following example we have named the +virtual environment *colouringlondon* but it can have any name. `pyvenv colouringlondon` @@ -155,69 +119,53 @@ Activate the virtual environment so we can install python packages into it. Install python pip package manager and related tools. -`pip install --upgrade pip` -`pip install --upgrade setuptools wheel` +``` +pip install --upgrade pip +pip install --upgrade setuptools wheel +``` -Now install the required python packages. This relies on the `requirements.txt` file located in the `etl` folder of your local repository. +Now install the required python packages. This relies on the `requirements.txt` file located +in the `etl` folder of your local repository. `pip install -r ./colouring-london/etl/requirements.txt` - -
- #### Setting up Node -Now upgrade the npm package manager to the most recent release with global privileges. This needs to be performed as root user, so it is necessary to export the node variables to the root user profile. Don't forget to exit from root at the end. +Now upgrade the npm package manager to the most recent release with global privileges. This +needs to be performed as root user, so it is necessary to export the node variables to the +root user profile. Don't forget to exit from root at the end. -`sudo su root` +``` +sudo su root +export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/` +export PATH=$NODEJS_HOME:$PATH` +npm install -g npm@next` +exit +``` -`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/` - -`export PATH=$NODEJS_HOME:​$PATH` - -`npm install -g npm@next` - -`exit` - - -
- -Now install the required Node packages. This needs to done from the `app` directory of your local repository, so that it can read from the `package.json` file. +Now install the required Node packages. This needs to done from the `app` directory of your +local repository, so that it can read from the `package.json` file. `cd ./colouring-london/app && npm install` -
- #### Running the application -Now we are ready to run the application. The `APP_COOKIE_SECRET` is arbitrary. +Now we are ready to run the application. The `APP_COOKIE_SECRET` is arbitrary. -`PGPASSWORD= PGDATABASE= PGUSER= PGHOST=localhost PGPORT=5432 APP_COOKIE_SECRET=123456 npm start` +`PGPASSWORD= PGDATABASE= PGUSER= PGHOST=localhost PGPORT=5432 APP_COOKIE_SECRET=123456 npm start` +If you a running Ubuntu in a virtual environment you will need to configure networking to +forward ports from the guest to the host. For Virtual Box the following was configured under +NAT port forwarding. -
+Name | Protocol | Host Port | Guest Port +-------- | --------- | ---------- | ----------- +app | TCP | 8080 | 3000 +app_dev | TCP | 3001 | 3001 +ssh | TCP | 4022 | 22 -If you a running Ubuntu in a virtual environment you will need to configure networking to forward ports from the guest to the host. For Virtual Box the following was configured under NAT port forwarding. - - - -Name | Protocol | Host Port | Guest Port -----------| ---------- | --------- | --------- -app | TCP | 8080 | 3000 -app_dev | TCP | 3001 | 3001 -ssh | TCP | 4022 | 22 - - - -The site can then be viewed on http://localhost:8080. The `app_dev` mapping is used in development by Razzle which rebuilds and serves client side assets on the fly. - -
+The site can then be viewed on http://localhost:8080. The `app_dev` mapping is used in +development by Razzle which rebuilds and serves client side assets on the fly. Finally to quit the application type `Ctrl-C`. - - - - - -