Add Cirrus CI (#7431)

* Cirrus: Add Tox task

* Cirrus: Add Locale task

* Cirrus: Add Flake8 Task

* Cirrus: Add Regtest task

* Regtest: Flush stdout

Allows viewing output sooner.

* Regtest: Read process.stdout in text mode

Improves ability to quickly see output.

* Cirrus: Add Windows task

* Cirrus: Add Android task

* Cirrus: Add macOS task

* Cirrus: Add AppImage task

* Cirrus: Add tarball task

* Cirrus: Add Submodules task

* Android: remove superfluous cp/rm

* Add .dockerignore

Symlink to .gitignore.

.cirrus.yml

@@ -0,0 +1,221 @@
+task:
+  container:
+    image: $ELECTRUM_IMAGE
+    cpu: 1
+    memory: 1G
+  matrix:
+    - name: Tox Python $ELECTRUM_PYTHON_VERSION
+      env:
+        ELECTRUM_IMAGE: python:$ELECTRUM_PYTHON_VERSION
+        TOXENV: py3
+        ELECTRUM_PYTHON_NAME: python3
+      matrix:
+       - env:
+           ELECTRUM_PYTHON_VERSION: 3.6
+       - env:
+           ELECTRUM_PYTHON_VERSION: 3.7
+       - env:
+           ELECTRUM_PYTHON_VERSION: 3.8
+       - env:
+           ELECTRUM_PYTHON_VERSION: 3.9
+       - env:
+           ELECTRUM_PYTHON_VERSION: 3
+       - env:
+           ELECTRUM_PYTHON_VERSION: rc
+    - name: Tox PyPy
+      allow_failures: true
+      env:
+        ELECTRUM_IMAGE: pypy:3
+        TOXENV: pypy3
+        ELECTRUM_PYTHON_NAME: pypy3
+  pip_cache:
+    folder: ~/.cache/pip
+    fingerprint_script: echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS
+    populate_script: mkdir -p ~/.cache/pip
+  electrum_cache:
+    folder: /tmp/electrum-build
+    populate_script: mkdir -p /tmp/electrum-build
+  version_script:
+    - $ELECTRUM_PYTHON_NAME --version
+  tag_script:
+    - git tag
+  install_script:
+    - apt-get update
+    - apt-get -y install libsecp256k1-0
+    - pip install -r $ELECTRUM_REQUIREMENTS
+  tox_script:
+    - tox
+  coveralls_script:
+    - coveralls
+  env:
+    ELECTRUM_REQUIREMENTS: contrib/requirements/requirements-travis.txt
+
+task:
+  name: Locale
+  container:
+    image: $ELECTRUM_IMAGE
+    cpu: 1
+    memory: 1G
+  pip_cache:
+    folder: ~/.cache/pip
+    fingerprint_script: echo Locale && echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS
+    populate_script: mkdir -p ~/.cache/pip
+  electrum_cache:
+    folder: /tmp/electrum-build
+    populate_script: mkdir -p /tmp/electrum-build
+  install_script:
+    - apt-get update
+    - apt-get -y install libsecp256k1-0
+    - pip install -r $ELECTRUM_REQUIREMENTS
+    - pip install requests
+  locale_script:
+    - contrib/push_locale
+  env:
+    ELECTRUM_IMAGE: python:3.7
+    ELECTRUM_REQUIREMENTS: contrib/requirements/requirements-travis.txt
+  depends_on:
+    - Tox Python 3.9
+  only_if: $CIRRUS_BRANCH == 'master'
+
+task:
+  name: Regtest functional tests
+  container:
+    image: $ELECTRUM_IMAGE
+    cpu: 1
+    memory: 1G
+  pip_cache:
+    folder: ~/.cache/pip
+    fingerprint_script: echo Regtest && echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS
+    populate_script: mkdir -p ~/.cache/pip
+  electrum_cache:
+    folder: /tmp/electrum-build
+    populate_script: mkdir -p /tmp/electrum-build
+  bitcoind_cache:
+    folder: /tmp/bitcoind
+    populate_script: mkdir -p /tmp/bitcoind
+  install_script:
+    - apt-get update
+    - apt-get -y install libsecp256k1-0 curl jq bc
+    - pip3 install .[tests]
+    - pip3 install electrumx
+    - "BITCOIND_VERSION=$(curl https://bitcoincore.org/en/download/ | grep -E -i --only-matching 'Latest version: [0-9\\.]+' | grep -E --only-matching '[0-9\\.]+')"
+    - BITCOIND_FILENAME=bitcoin-$BITCOIND_VERSION-x86_64-linux-gnu.tar.gz
+    - BITCOIND_PATH=/tmp/bitcoind/$BITCOIND_FILENAME
+    - BITCOIND_URL=https://bitcoincore.org/bin/bitcoin-core-$BITCOIND_VERSION/$BITCOIND_FILENAME
+    - tar -xaf $BITCOIND_PATH || (rm -f /tmp/bitcoind/* && curl --output $BITCOIND_PATH $BITCOIND_URL && tar -xaf $BITCOIND_PATH)
+    - cp -a bitcoin-$BITCOIND_VERSION/* /usr/
+  bitcoind_service_background_script:
+    - electrum/tests/regtest/run_bitcoind.sh
+  electrumx_service_background_script:
+    - electrum/tests/regtest/run_electrumx.sh
+  regtest_script:
+    - sleep 10s
+    - python3 -m unittest electrum/tests/regtest.py
+  env:
+    ELECTRUM_IMAGE: python:3.7
+    ELECTRUM_REQUIREMENTS: contrib/requirements/requirements-travis.txt
+    # ElectrumX exits with an error without this:
+    ALLOW_ROOT: 1
+
+task:
+  container:
+    image: $ELECTRUM_IMAGE
+    cpu: 1
+    memory: 1G
+  pip_cache:
+    folder: ~/.cache/pip
+    fingerprint_script: echo Flake8 && echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS
+    populate_script: mkdir -p ~/.cache/pip
+  electrum_cache:
+    folder: /tmp/electrum-build
+    populate_script: mkdir -p /tmp/electrum-build
+  install_script:
+    - pip install flake8
+  flake8_script:
+    - flake8 . --count --select=$ELECTRUM_LINTERS --show-source --statistics
+  env:
+    ELECTRUM_IMAGE: python:3.7
+    ELECTRUM_REQUIREMENTS: contrib/requirements/requirements-travis.txt
+  matrix:
+    - name: Flake8 Mandatory
+      env:
+        ELECTRUM_LINTERS: E9,F63,F7,F82
+    - name: Flake8 Non-Mandatory
+      env:
+        ELECTRUM_LINTERS: E,F,W,C90
+      allow_failures: true
+
+task:
+  name: Windows build
+  container:
+    dockerfile: contrib/build-wine/Dockerfile
+    cpu: 1
+    memory: 2G
+  build_script:
+    - cd contrib/build-wine
+    - ./make_win.sh
+  binaries_artifacts:
+    path: "contrib/build-wine/dist/*"
+  env:
+    CIRRUS_WORKING_DIR: /opt/wine64/drive_c/electrum
+
+task:
+  name: Android build
+  container:
+    dockerfile: contrib/android/Dockerfile
+    cpu: 2
+    memory: 2G
+  build_script:
+    - ./contrib/android/make_apk
+  binaries_artifacts:
+    path: "dist/*"
+
+task:
+  name: MacOS build
+  macos_instance:
+    image: catalina-xcode-11.3.1
+  env:
+    TARGET_OS: macOS
+  install_script:
+    - git fetch --all --tags
+  build_script:
+    - ./contrib/osx/make_osx
+  sum_script:
+    - ls -lah dist
+    - shasum -a 256 dist/*.dmg
+  binaries_artifacts:
+    path: "dist/*"
+
+task:
+  name: AppImage build
+  container:
+    dockerfile: contrib/build-linux/appimage/Dockerfile
+    cpu: 2
+    memory: 1G
+  build_script:
+    - ./contrib/build-linux/appimage/make_appimage.sh
+  binaries_artifacts:
+    path: "dist/*"
+
+task:
+  name: tarball build
+  container:
+    dockerfile: contrib/build-linux/sdist/Dockerfile
+    cpu: 1
+    memory: 1G
+  build_script:
+    - ./contrib/build-linux/sdist/make_sdist.sh
+  binaries_artifacts:
+    path: "dist/*"
+
+task:
+  name: Submodules
+  container:
+    image: python:3.7
+    cpu: 1
+    memory: 1G
+  fetch_script:
+    - git fetch --all --tags
+  check_script:
+    - ./contrib/deterministic-build/check_submodules.sh
+  only_if: $CIRRUS_TAG != ''

.dockerignore

@@ -0,0 +1 @@
+.gitignore

contrib/android/Dockerfile

@@ -151,7 +151,7 @@ RUN chown ${USER} /opt
 USER ${USER}
 
 
-COPY requirements-build-android.txt /opt/deterministic-build/
+COPY contrib/deterministic-build/requirements-build-android.txt /opt/deterministic-build/
 RUN python3 -m pip install --no-dependencies --user \
     -r /opt/deterministic-build/requirements-build-android.txt
 

contrib/android/build.sh

@@ -22,12 +22,11 @@ if [ ! -z "$ELECBUILD_NOCACHE" ] ; then
 fi
 
 info "building docker image."
-cp "$CONTRIB/deterministic-build/requirements-build-android.txt" "$CONTRIB_ANDROID/requirements-build-android.txt"
 sudo docker build \
     $DOCKER_BUILD_FLAGS \
     -t electrum-android-builder-img \
-    "$CONTRIB_ANDROID"
+    --file "$CONTRIB_ANDROID/Dockerfile" \
-rm "$CONTRIB_ANDROID/requirements-build-android.txt"
+    "$PROJECT_ROOT"
 
 
 # maybe do fresh clone

electrum/tests/regtest.py

@@ -7,9 +7,10 @@ class TestLightning(unittest.TestCase):
 
     @staticmethod
     def run_shell(args, timeout=30):
-        process = subprocess.Popen(['electrum/tests/regtest/regtest.sh'] + args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+        process = subprocess.Popen(['electrum/tests/regtest/regtest.sh'] + args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True)
-        for line in iter(process.stdout.readline, b''):
+        for line in iter(process.stdout.readline, ''):
-            sys.stdout.write(line.decode(sys.stdout.encoding))
+            sys.stdout.write(line)
+            sys.stdout.flush()
         process.wait(timeout=timeout)
         process.stdout.close()
         assert process.returncode == 0

electrum/tests/regtest/start_bitcoind.sh → electrum/tests/regtest/run_bitcoind.sh

@@ -17,9 +17,9 @@ rpcbind=0.0.0.0
 rpcport=18554
 EOF
 rm -rf ~/.bitcoin/regtest
-screen -S bitcoind -X quit || true
+bitcoind -regtest &
-screen -S bitcoind -m -d bitcoind -regtest
 sleep 6
 bitcoin-cli createwallet test_wallet
 addr=$(bitcoin-cli getnewaddress)
-bitcoin-cli generatetoaddress 150 $addr > /dev/null
+bitcoin-cli generatetoaddress 150 $addr
+tail -f ~/.bitcoin/regtest/debug.log

electrum/tests/regtest/start_electrumx.sh → electrum/tests/regtest/run_electrumx.sh

@@ -4,4 +4,4 @@ set -eux pipefail
 cd
 rm -rf $HOME/electrumx_db
 mkdir $HOME/electrumx_db
-COST_SOFT_LIMIT=0 COST_HARD_LIMIT=0 COIN=BitcoinSegwit SERVICES=tcp://:51001,rpc:// NET=regtest DAEMON_URL=http://doggman:donkey@127.0.0.1:18554 DB_DIRECTORY=$HOME/electrumx_db electrumx_server > $HOME/electrumx.log &
+COST_SOFT_LIMIT=0 COST_HARD_LIMIT=0 COIN=BitcoinSegwit SERVICES=tcp://:51001,rpc:// NET=regtest DAEMON_URL=http://doggman:donkey@127.0.0.1:18554 DB_DIRECTORY=$HOME/electrumx_db electrumx_server