diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..d4dfb50 --- /dev/null +++ b/infra/README.md @@ -0,0 +1,2 @@ +# infra +Here is everything required to deploy the block explorer diff --git a/infra/deployment/deploy-rpc.md b/infra/deployment/deploy-rpc.md new file mode 100644 index 0000000..d365323 --- /dev/null +++ b/infra/deployment/deploy-rpc.md @@ -0,0 +1,34 @@ +# Deploy the rpc server (xsnd) +The following steps are for setting up the rpc server that runs on system startup. + +## Requisites +- set up aws, see [aws-setup.md](/infra/misc/setup-aws.md) + +## Set up a new user +- create the user: `sudo adduser --system rpc` +- switch to the new user: `sudo su -s /bin/bash - rpc` +- set aws credentials: `aws configure` +- create the folder for xsn data: `mkdir /home/rpc/.xsncore` +- set the xsn config (see [xsn.conf](/infra/misc/xsn.conf)): `vim /home/rpc/.xsncore/xsn.conf` + +## Install the rpc server (use the new user) +- switch to the new user: `sudo su -s /bin/bash - rpc` +- download the client (ensure it is the latest version): `wget https://github.com/X9Developers/XSN/releases/download/v1.0.9/xsncore-1.0.9-linux64.tar.gz -O xsn.tar.gz` +- unpack the file: `tar -zxvf xsn.tar.gz` +- create a folder for the executables: `mkdir /home/rpc/xsn` +- move the executables: `mv xsncore-1.0.9/bin/xsn* xsn/` +- add the script for new blocks: `vim xsn/script.sh` +- set the script as executable (see [script.sh](/infra/misc/script.sh)): `chmod +x xsn/script.sh` +- test the script: `./xsn/script.sh working` + +## Add the systemd service (as super user) +- create the service file (see [xsn-rpc.service](/infra/systemd-services/xsn-rpc.service)): `sudo cp xsn-rpc.service /etc/systemd/system/xsn-rpc.service` +- reload services: `sudo systemctl daemon-reload` +- check that the service is recognized: `sudo service xsn-rpc status` +- start the service: `sudo service xsn-rpc start` +- verify it is working: `sudo service xsn-rpc status` +- run the service on system startup: `sudo systemctl enable xsn-rpc` + +## Test the rpc service +- switch to the new user: `sudo su -s /bin/bash - rpc` +- test the service: `./xsn/xsn-cli getinfo` diff --git a/infra/deployment/deploy-server.md b/infra/deployment/deploy-server.md new file mode 100644 index 0000000..6d711d2 --- /dev/null +++ b/infra/deployment/deploy-server.md @@ -0,0 +1,47 @@ +# Deploy the server project +The following instructions are for setting up the server project (backend). + +## Build the project + +These steps should be run in a place where you have cloned the repository, like your local machine: +- Move to the [server](/server) folder: `cd server` +- Build the project: `sbt dist` +- Copy the built project to the server: `scp target/universal/xsn-block-explorer-0.1.0-SNAPSHOT.zip xsnexplorer.io:~/` + + +## Setup the server (first steps) + +### Install java 8 + +- `sudo add-apt-repository ppa:webupd8team/java` +- `sudo apt-get update && sudo apt-get install oracle-java8-installer` +- verify the version: `java -version` + +[source](https://www.digitalocean.com/community/tutorials/how-to-install-java-with-apt-get-on-ubuntu-16-04) + +### Setup a new user +- add the new user: `sudo adduser --system play` +- switch to the new user: `sudo su -s /bin/bash - play` +- create the project folder: `mkdir /home/play/server` + +### Unpack the project +- Unzip the project: `sudo unzip ~/xsn-block-explorer-0.1.0-SNAPSHOT.zip -d /home/play/server` +- Set the config: `sudo vim /home/play/server/xsn-block-explorer-0.1.0-SNAPSHOT/conf/application.conf` +- Restore permissions: `sudo chown -R play:nogroup /home/play/server` +- Restart the service: `sudo service xsn-backend restart` + +### Run the service on system startup +- add the systemd service (see [xsn-backend.service](/infra/systemd-services/xsn-backend.service)): `cp xsn-backend.service /etc/systemd/system/xsn-backend.service` +- reload the services: `sudo systemctl daemon-reload` +- verify the service registration: `sudo service xsn-backend status` +- start the service: `sudo service xsn-backend start` +- verify the status: `sudo service xsn-backend status` +- verify the service: `curl localhost:9000/health` +- enable the service to run on system startup: `sudo systemctl enable xsn-backend` + + +## Troubleshooting +- systemd logs: `sudo journalctl -u xsn-backend` +- syslog: `sudo tail -f /var/log/syslog` +- check `.env` file permissions +- ensure the `PLAY_APPLICATION_SECRET` is set diff --git a/infra/deployment/deploy-web-ui.md b/infra/deployment/deploy-web-ui.md new file mode 100644 index 0000000..0cdf1c5 --- /dev/null +++ b/infra/deployment/deploy-web-ui.md @@ -0,0 +1,19 @@ +# Deploy the web-ui project + +## Requisites +- nginx working (see [setup-nginx.md](/infra/misc/setup-nginx.md)) + + +## Build the project + +These steps should be run in a place where you have cloned the repository, like your local machine: +- move to the [web-ui](/web-ui) folder: `cd web-ui` +- build the project: `ng build --prod` +- zip the result: `zip -r web-ui.zip dist/*` +- copy to the server: `scp web-ui.zip xsnexplorer.io:~/` + + +## Server +- login. `ssh xsnexplorer.io` +- unzip the project: `unzip ~/web-ui.zip -d ~/`` +- move the files: `sudo rsync -a ~/dist/ /var/www/html/ --remove-source-files` diff --git a/infra/misc/queries.sql b/infra/misc/queries.sql new file mode 100644 index 0000000..fb936ca --- /dev/null +++ b/infra/misc/queries.sql @@ -0,0 +1,90 @@ +-- clear data +delete from blocks; delete from transactions; delete from transaction_inputs; delete from transaction_outputs; delete from balances; + +-- avg time to get a new block +SELECT AVG(b.time - a.time) AS new_block_avg_time +FROM (SELECT height, time FROM blocks) a JOIN + (SELECT height, time FROM blocks) b ON (a.height + 1 = b.height); + + +-- find blocks with corrupted next_blockhash +SELECT height, blockhash, next_blockhash +FROM blocks b +WHERE 0 = (SELECT COUNT(*) FROM blocks WHERE blockhash = b.next_blockhash) AND + height < (SELECT MAX(height) FROM blocks); + +-- find blocks with corrupted previous_blockhash +SELECT height, blockhash, previous_blockhash +FROM blocks b +WHERE 0 = (SELECT COUNT(*) FROM blocks WHERE blockhash = b.previous_blockhash) AND + height > (SELECT MIN(height) FROM blocks); + +-- find missing blocks in the chain +SELECT height - 1 AS missing +FROM blocks b +WHERE height > 1 AND + height - 1 NOT IN ( + SELECT height + FROM blocks + WHERE height = b.height - 1 +); + +-- find corrupted balances +SELECT address, one.available AS one, (two.received - two.spent) AS two +FROM ( + SELECT address, received - spent AS available + FROM ( + SELECT address, SUM(value) AS spent + FROM transaction_inputs + GROUP BY address + ) s JOIN + ( + SELECT address, SUM(value) AS received + FROM transaction_outputs + GROUP BY address + ) r USING (address) + ) one JOIN balances two USING (address) +WHERE one.available <> (two.received - two.spent); +-- + +-- rebuild balances table +-- 1. count number of balances +SELECT COUNT(*) +FROM balances; + +-- 2. verify you would write the same amount +SELECT COUNT(*) +FROM + ( + SELECT address, SUM(value) AS received + FROM transaction_outputs + GROUP BY address + ) r LEFT JOIN ( + SELECT address, SUM(value) AS spent + FROM transaction_inputs + GROUP BY address + ) s USING (address); + +-- 3. delete balances, be sure that the explorer is turned off +DELETE FROM balances; + +-- 4. insert the balances +INSERT INTO balances + ( + SELECT address, received, COALESCE(spent, 0) AS spent + FROM + ( + SELECT address, SUM(value) AS received + FROM transaction_outputs + GROUP BY address + ) r LEFT JOIN ( + SELECT address, SUM(value) AS spent + FROM transaction_inputs + GROUP BY address + ) s USING (address) + ); + +-- 5. verify you have the same amount +SELECT COUNT(*) FROM balances; + +-- 6. start explorer diff --git a/infra/misc/script.sh b/infra/misc/script.sh new file mode 100644 index 0000000..e241c32 --- /dev/null +++ b/infra/misc/script.sh @@ -0,0 +1,5 @@ +#!/bin/bash +BLOCK=$1 +QUEUE="https://sqs.us-east-2.amazonaws.com/984148963792/blocks.fifo" +DID=$(head /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) +aws sqs send-message --message-body $BLOCK --queue-url $QUEUE --message-group-id none --message-deduplication-id $DID diff --git a/infra/misc/setup-aws.md b/infra/misc/setup-aws.md new file mode 100644 index 0000000..5c42f14 --- /dev/null +++ b/infra/misc/setup-aws.md @@ -0,0 +1,11 @@ +# Setup AWS + +## set up the locale +- `export LC_ALL="en_US.UTF-8"` +- `export LC_CTYPE="en_US.UTF-8"` +- `sudo dpkg-reconfigure locales` + +## install aws cli +- `sudo apt update && sudo apt install -y python-pip python-setuptools` +- `sudo pip install --upgrade pip` +- `sudo pip install awscli` diff --git a/infra/misc/setup-nginx.md b/infra/misc/setup-nginx.md new file mode 100644 index 0000000..2d2993c --- /dev/null +++ b/infra/misc/setup-nginx.md @@ -0,0 +1,2 @@ +# Setup nginx +TODO diff --git a/infra/misc/sycn-heroku-db.md b/infra/misc/sycn-heroku-db.md new file mode 100644 index 0000000..f3e4ea7 --- /dev/null +++ b/infra/misc/sycn-heroku-db.md @@ -0,0 +1,16 @@ +# Sync Heroku psql +At the moment, we are using heroku postgres, if we let the server sync the whole chain, it will take more than a day to complete the sync which is unreasonable. + +The whole process takes a couple of hours if we use postgres locally, these steps allow us to avoid the bottleneck caused by heroku, we'll seed a local database and then import that into heroku. + +Heroku is a bottleneck on the initial seeding, the idea is to seed the database locally and export it to heroku. + +## Commands +- export from local: `pg_dump -Fc --no-acl --no-owner -U postgres xsn_blockchain > backup.dump` +- upload the file to a public location: `scp backup.dump xsnexplorer.io:~/`, in the server `mv backup.dump /var/www/html/ +` +- restore dump: `heroku pg:backups:restore 'http://xsnexplorer.io/backup.dump' postgresql-graceful-31330 -a xsnexplorer` +- delete the file from the public location: `rm /var/www/html/backup.dump` + +### Source +- https://devcenter.heroku.com/articles/heroku-postgres-import-export diff --git a/infra/misc/xsn.conf b/infra/misc/xsn.conf new file mode 100644 index 0000000..ab23797 --- /dev/null +++ b/infra/misc/xsn.conf @@ -0,0 +1,7 @@ +rpcuser=[REPLACE_ME] +rpcpassword=[REPLACE_ME] +rpcport=51473 +txindex=1 +addressindex=1 +spentindex=1 +blocknotify=/home/rpc/xsn/script.sh %s diff --git a/infra/systemd-services/xsn-backend.service b/infra/systemd-services/xsn-backend.service new file mode 100644 index 0000000..d55b535 --- /dev/null +++ b/infra/systemd-services/xsn-backend.service @@ -0,0 +1,15 @@ +[Unit] +Description=XSN Backend Server + +[Service] +Type=simple +WorkingDirectory=/home/play/server/xsn-block-explorer-0.1.0-SNAPSHOT +StandardOutput=tty +StandardError=tty +EnvironmentFile=/home/play/server/.env +User=play +ExecStart=/home/play/server/xsn-block-explorer-0.1.0-SNAPSHOT/bin/xsn-block-explorer -Dhttp.port=9000 -Dpidfile.path=/dev/null +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/infra/systemd-services/xsn-rpc.service b/infra/systemd-services/xsn-rpc.service new file mode 100644 index 0000000..5dc31cd --- /dev/null +++ b/infra/systemd-services/xsn-rpc.service @@ -0,0 +1,15 @@ +[Unit] +Description=XSN RPC Server + +[Service] +Type=simple +WorkingDirectory=/home/rpc/ +StandardOutput=tty +StandardError=tty +User=rpc +ExecStart=/home/rpc/xsn/xsnd + +Restart=on-failure + +[Install] +WantedBy=multi-user.target